From 3c337a441c883a8255ec766dd92ecd419a17f791 Mon Sep 17 00:00:00 2001 From: Matt Lord Date: Sun, 8 Dec 2024 09:27:30 -0500 Subject: [PATCH 01/39] WiP Signed-off-by: Matt Lord --- go/mysql/binlog/binlog_json.go | 115 ++- go/mysql/binlog/rbr.go | 44 +- go/mysql/binlog/rbr_test.go | 2 +- go/mysql/binlog_event.go | 27 +- go/mysql/binlog_event_common.go | 5 + go/mysql/binlog_event_filepos.go | 4 + go/mysql/binlog_event_mysql56_test.go | 109 +++ go/mysql/binlog_event_rbr.go | 45 +- go/mysql/replication_constants.go | 3 + go/vt/binlog/binlog_streamer.go | 4 +- go/vt/proto/binlogdata/binlogdata.pb.go | 740 +++++++++--------- .../proto/binlogdata/binlogdata_vtproto.pb.go | 51 ++ .../vreplication/replicator_plan.go | 13 + .../tabletserver/vstreamer/vstreamer.go | 30 +- proto/binlogdata.proto | 11 +- web/vtadmin/src/proto/vtadmin.d.ts | 6 + web/vtadmin/src/proto/vtadmin.js | 28 + 17 files changed, 833 insertions(+), 404 deletions(-) diff --git a/go/mysql/binlog/binlog_json.go b/go/mysql/binlog/binlog_json.go index 03bf604fb2d..73146abd06b 100644 --- a/go/mysql/binlog/binlog_json.go +++ b/go/mysql/binlog/binlog_json.go @@ -17,7 +17,9 @@ limitations under the License. package binlog import ( + "bytes" "encoding/binary" + "encoding/hex" "fmt" "math" "strconv" @@ -25,9 +27,12 @@ import ( "vitess.io/vitess/go/hack" "vitess.io/vitess/go/mysql/format" "vitess.io/vitess/go/mysql/json" + "vitess.io/vitess/go/sqltypes" + "vitess.io/vitess/go/vt/log" + "vitess.io/vitess/go/vt/vterrors" + querypb "vitess.io/vitess/go/vt/proto/query" vtrpcpb "vitess.io/vitess/go/vt/proto/vtrpc" - "vitess.io/vitess/go/vt/vterrors" ) /* @@ -44,6 +49,14 @@ https://github.com/shyiko/mysql-binlog-connector-java/pull/119/files https://github.com/noplay/python-mysql-replication/blob/175df28cc8b536a68522ff9b09dc5440adad6094/pymysqlreplication/packet.py */ +type jsonDiffOp uint8 + +const ( + jsonDiffOpReplace = jsonDiffOp(0) + jsonDiffOpInsert = jsonDiffOp(1) + jsonDiffOpRemove = jsonDiffOp(2) +) + // ParseBinaryJSON provides the parsing function from the mysql binary json // representation to a JSON value instance. func ParseBinaryJSON(data []byte) (*json.Value, error) { @@ -60,6 +73,62 @@ func ParseBinaryJSON(data []byte) (*json.Value, error) { return node, nil } +// ParseBinaryJSONDiff provides the parsing function from the MySQL JSON +// diff representation to an SQL expression. +func ParseBinaryJSONDiff(data []byte) (sqltypes.Value, error) { + log.Errorf("DEBUG: json diff data hex: %s, string: %s", hex.EncodeToString(data), string(data)) + pos := 0 + opType := jsonDiffOp(data[pos]) + pos++ + diff := bytes.Buffer{} + + switch opType { + case jsonDiffOpReplace: + diff.WriteString("JSON_REPLACE(") + case jsonDiffOpInsert: + diff.WriteString("JSON_INSERT(") + case jsonDiffOpRemove: + diff.WriteString("JSON_REMOVE(") + } + diff.WriteString("%s, ") // This will later be replaced by the field name + + log.Errorf("DEBUG: json diff opType: %d", opType) + + pathLen, readTo, ok := readLenEncInt(data, pos) + if !ok { + return sqltypes.Value{}, fmt.Errorf("cannot read JSON diff path length") + } + pos = readTo + + log.Errorf("DEBUG: json diff path length: %d", pathLen) + + path := data[pos : uint64(pos)+pathLen] + pos += int(pathLen) + log.Errorf("DEBUG: json diff path: %s", string(path)) + diff.WriteString(fmt.Sprintf("'%s', ", path)) + + if opType == jsonDiffOpRemove { // No value for remove + diff.WriteString(")") + return sqltypes.MakeTrusted(sqltypes.Expression, diff.Bytes()), nil + } + + valueLen, readTo, ok := readLenEncInt(data, pos) + if !ok { + return sqltypes.Value{}, fmt.Errorf("cannot read JSON diff path length") + } + pos = readTo + log.Errorf("DEBUG: json diff value length: %d", valueLen) + + value, err := ParseBinaryJSON(data[pos : uint64(pos)+valueLen]) + if err != nil { + return sqltypes.Value{}, fmt.Errorf("cannot read JSON diff value for path %s: %w", path, err) + } + log.Errorf("DEBUG: json diff value: %v", value) + diff.WriteString(fmt.Sprintf("%s)", value)) + + return sqltypes.MakeTrusted(sqltypes.Expression, diff.Bytes()), nil +} + // jsonDataType has the values used in the mysql json binary representation to denote types. // We have string, literal(true/false/null), number, object or array types. // large object => doc size > 64K: you get pointers instead of inline values. @@ -315,7 +384,7 @@ func binparserOpaque(_ jsonDataType, data []byte, pos int) (node *json.Value, er precision := decimalData[0] scale := decimalData[1] metadata := (uint16(precision) << 8) + uint16(scale) - val, _, err := CellValue(decimalData, 2, TypeNewDecimal, metadata, &querypb.Field{Type: querypb.Type_DECIMAL}) + val, _, err := CellValue(decimalData, 2, TypeNewDecimal, metadata, &querypb.Field{Type: querypb.Type_DECIMAL}, false) if err != nil { return nil, err } @@ -394,3 +463,45 @@ func binparserObject(typ jsonDataType, data []byte, pos int) (node *json.Value, return json.NewObject(object), nil } + +func readLenEncInt(data []byte, pos int) (uint64, int, bool) { + if pos >= len(data) { + return 0, 0, false + } + + // reslice to avoid arithmetic below + data = data[pos:] + + switch data[0] { + case 0xfc: + // Encoded in the next 2 bytes. + if 2 >= len(data) { + return 0, 0, false + } + return uint64(data[1]) | + uint64(data[2])<<8, pos + 3, true + case 0xfd: + // Encoded in the next 3 bytes. + if 3 >= len(data) { + return 0, 0, false + } + return uint64(data[1]) | + uint64(data[2])<<8 | + uint64(data[3])<<16, pos + 4, true + case 0xfe: + // Encoded in the next 8 bytes. + if 8 >= len(data) { + return 0, 0, false + } + return uint64(data[1]) | + uint64(data[2])<<8 | + uint64(data[3])<<16 | + uint64(data[4])<<24 | + uint64(data[5])<<32 | + uint64(data[6])<<40 | + uint64(data[7])<<48 | + uint64(data[8])<<56, pos + 9, true + default: + return uint64(data[0]), pos + 1, true + } +} diff --git a/go/mysql/binlog/rbr.go b/go/mysql/binlog/rbr.go index 8b95b0daee9..9f92c18c530 100644 --- a/go/mysql/binlog/rbr.go +++ b/go/mysql/binlog/rbr.go @@ -26,9 +26,11 @@ import ( "time" "vitess.io/vitess/go/sqltypes" - querypb "vitess.io/vitess/go/vt/proto/query" - "vitess.io/vitess/go/vt/proto/vtrpc" + "vitess.io/vitess/go/vt/log" "vitess.io/vitess/go/vt/vterrors" + + querypb "vitess.io/vitess/go/vt/proto/query" + vtrpcpb "vitess.io/vitess/go/vt/proto/vtrpc" ) // ZeroTimestamp is the special value 0 for a timestamp. @@ -130,7 +132,7 @@ func CellLength(data []byte, pos int, typ byte, metadata uint16) (int, error) { uint32(data[pos+2])<<16| uint32(data[pos+3])<<24), nil default: - return 0, vterrors.Errorf(vtrpc.Code_INTERNAL, "unsupported blob/geometry metadata value %v (data: %v pos: %v)", metadata, data, pos) + return 0, vterrors.Errorf(vtrpcpb.Code_INTERNAL, "unsupported blob/geometry metadata value %v (data: %v pos: %v)", metadata, data, pos) } case TypeString: // This may do String, Enum, and Set. The type is in @@ -151,7 +153,7 @@ func CellLength(data []byte, pos int, typ byte, metadata uint16) (int, error) { return l + 1, nil default: - return 0, vterrors.Errorf(vtrpc.Code_INTERNAL, "unsupported type %v (data: %v pos: %v)", typ, data, pos) + return 0, vterrors.Errorf(vtrpcpb.Code_INTERNAL, "unsupported type %v (data: %v pos: %v)", typ, data, pos) } } @@ -176,7 +178,7 @@ func printTimestamp(v uint32) *bytes.Buffer { // byte to determine general shared aspects of types and the querypb.Field to // determine other info specifically about its underlying column (SQL column // type, column length, charset, etc) -func CellValue(data []byte, pos int, typ byte, metadata uint16, field *querypb.Field) (sqltypes.Value, int, error) { +func CellValue(data []byte, pos int, typ byte, metadata uint16, field *querypb.Field, partialJSON bool) (sqltypes.Value, int, error) { switch typ { case TypeTiny: if sqltypes.IsSigned(field.Type) { @@ -644,7 +646,7 @@ func CellValue(data []byte, pos int, typ byte, metadata uint16, field *querypb.F return sqltypes.MakeTrusted(querypb.Type_ENUM, strconv.AppendUint(nil, uint64(val), 10)), 2, nil default: - return sqltypes.NULL, 0, vterrors.Errorf(vtrpc.Code_INTERNAL, "unexpected enum size: %v", metadata&0xff) + return sqltypes.NULL, 0, vterrors.Errorf(vtrpcpb.Code_INTERNAL, "unexpected enum size: %v", metadata&0xff) } case TypeSet: @@ -672,21 +674,29 @@ func CellValue(data []byte, pos int, typ byte, metadata uint16, field *querypb.F uint32(data[pos+2])<<16 | uint32(data[pos+3])<<24) default: - return sqltypes.NULL, 0, vterrors.Errorf(vtrpc.Code_INTERNAL, "unsupported blob metadata value %v (data: %v pos: %v)", metadata, data, pos) + return sqltypes.NULL, 0, vterrors.Errorf(vtrpcpb.Code_INTERNAL, "unsupported blob metadata value %v (data: %v pos: %v)", metadata, data, pos) } pos += int(metadata) // For JSON, we parse the data, and emit SQL. if typ == TypeJSON { - var err error jsonData := data[pos : pos+l] - jsonVal, err := ParseBinaryJSON(jsonData) - if err != nil { - panic(err) + if partialJSON { + log.Errorf("DEBUG: partialJSON cell value: %s", string(jsonData)) + val, err := ParseBinaryJSONDiff(jsonData) + if err != nil { + panic(err) + } + log.Errorf("DEBUG: decoded partialJSON cell value: %v", val) + return val, l + int(metadata), nil + } else { + jsonVal, err := ParseBinaryJSON(jsonData) + if err != nil { + panic(err) + } + jd := jsonVal.MarshalTo(nil) + return sqltypes.MakeTrusted(sqltypes.Expression, jd), l + int(metadata), nil } - d := jsonVal.MarshalTo(nil) - return sqltypes.MakeTrusted(sqltypes.Expression, - d), l + int(metadata), nil } return sqltypes.MakeTrusted(querypb.Type_VARBINARY, @@ -710,7 +720,7 @@ func CellValue(data []byte, pos int, typ byte, metadata uint16, field *querypb.F return sqltypes.MakeTrusted(querypb.Type_UINT16, strconv.AppendUint(nil, uint64(val), 10)), 2, nil default: - return sqltypes.NULL, 0, vterrors.Errorf(vtrpc.Code_INTERNAL, "unexpected enum size: %v", metadata&0xff) + return sqltypes.NULL, 0, vterrors.Errorf(vtrpcpb.Code_INTERNAL, "unexpected enum size: %v", metadata&0xff) } } if t == TypeSet { @@ -776,13 +786,13 @@ func CellValue(data []byte, pos int, typ byte, metadata uint16, field *querypb.F uint32(data[pos+2])<<16 | uint32(data[pos+3])<<24) default: - return sqltypes.NULL, 0, vterrors.Errorf(vtrpc.Code_INTERNAL, "unsupported geometry metadata value %v (data: %v pos: %v)", metadata, data, pos) + return sqltypes.NULL, 0, vterrors.Errorf(vtrpcpb.Code_INTERNAL, "unsupported geometry metadata value %v (data: %v pos: %v)", metadata, data, pos) } pos += int(metadata) return sqltypes.MakeTrusted(querypb.Type_GEOMETRY, data[pos:pos+l]), l + int(metadata), nil default: - return sqltypes.NULL, 0, vterrors.Errorf(vtrpc.Code_INTERNAL, "unsupported type %v", typ) + return sqltypes.NULL, 0, vterrors.Errorf(vtrpcpb.Code_INTERNAL, "unsupported type %v", typ) } } diff --git a/go/mysql/binlog/rbr_test.go b/go/mysql/binlog/rbr_test.go index 1dfaf90a33e..ce49e587e6c 100644 --- a/go/mysql/binlog/rbr_test.go +++ b/go/mysql/binlog/rbr_test.go @@ -550,7 +550,7 @@ func TestCellLengthAndData(t *testing.T) { } // Test CellValue. - out, l, err := CellValue(padded, 1, tcase.typ, tcase.metadata, &querypb.Field{Type: tcase.styp}) + out, l, err := CellValue(padded, 1, tcase.typ, tcase.metadata, &querypb.Field{Type: tcase.styp}, false) if err != nil || l != len(tcase.data) || out.Type() != tcase.out.Type() || !bytes.Equal(out.Raw(), tcase.out.Raw()) { t.Errorf("testcase cellData(%v,%v) returned unexpected result: %v %v %v, was expecting %v %v \nwant: %s\ngot: %s", tcase.typ, tcase.data, out, l, err, tcase.out, len(tcase.data), tcase.out.Raw(), out.Raw()) diff --git a/go/mysql/binlog_event.go b/go/mysql/binlog_event.go index 5d472230d0e..76660f4ee90 100644 --- a/go/mysql/binlog_event.go +++ b/go/mysql/binlog_event.go @@ -86,9 +86,20 @@ type BinlogEvent interface { IsWriteRows() bool // IsUpdateRows returns true if this is a UPDATE_ROWS_EVENT. IsUpdateRows() bool + // IsPartialUpdateRows returs true if a partial JSON update event + // is found. These events are only seen in MySQL 8.0 if the mysqld + // instance has binlog_row_value_options=PARTIAL_JSON set. + IsPartialUpdateRows() bool // IsDeleteRows returns true if this is a DELETE_ROWS_EVENT. IsDeleteRows() bool + // IsPseudo is for custom implementations of GTID. + IsPseudo() bool + + // IsTransactionPayload returns true if a compressed transaction + // payload event is found (binlog_transaction_compression=ON). + IsTransactionPayload() bool + // Timestamp returns the timestamp from the event header. Timestamp() uint32 // ServerID returns the server ID from the event header. @@ -123,8 +134,8 @@ type BinlogEvent interface { TableMap(BinlogFormat) (*TableMap, error) // Rows returns a Rows struct representing data from a // {WRITE,UPDATE,DELETE}_ROWS_EVENT. This is only valid if - // IsWriteRows(), IsUpdateRows(), or IsDeleteRows() returns - // true. + // IsWriteRows(), IsUpdateRows(), IsPartialUpdateRows(), or + // IsDeleteRows() returns true. Rows(BinlogFormat, *TableMap) (Rows, error) // TransactionPayload returns a TransactionPayload type which provides // a GetNextEvent() method to iterate over the events contained within @@ -141,13 +152,6 @@ type BinlogEvent interface { // the same event and a nil checksum. StripChecksum(BinlogFormat) (ev BinlogEvent, checksum []byte, err error) - // IsPseudo is for custom implementations of GTID. - IsPseudo() bool - - // IsTransactionPayload returns true if a compressed transaction - // payload event is found (binlog_transaction_compression=ON). - IsTransactionPayload() bool - // Bytes returns the binary representation of the event Bytes() []byte } @@ -266,6 +270,11 @@ type Row struct { // It is only set for UPDATE and DELETE events. Identify []byte + // If this row represents a PartialUpdateRow event there will be a + // shared-image that sits between the before and after images that + // defines how the JSON column values are represented. + JSONPartialValues Bitmap + // Data is the raw data. // It is only set for WRITE and UPDATE events. Data []byte diff --git a/go/mysql/binlog_event_common.go b/go/mysql/binlog_event_common.go index c95873614f0..548875c44f7 100644 --- a/go/mysql/binlog_event_common.go +++ b/go/mysql/binlog_event_common.go @@ -187,6 +187,11 @@ func (ev binlogEvent) IsUpdateRows() bool { ev.Type() == eUpdateRowsEventV2 } +// IsPartialUpdateRows implements BinlogEvent.IsPartialUpdateRows(). +func (ev binlogEvent) IsPartialUpdateRows() bool { + return ev.Type() == ePartialUpdateRowsEvent +} + // IsDeleteRows implements BinlogEvent.IsDeleteRows(). // We do not support v0. func (ev binlogEvent) IsDeleteRows() bool { diff --git a/go/mysql/binlog_event_filepos.go b/go/mysql/binlog_event_filepos.go index c71c8346964..b7e6ed9e0f2 100644 --- a/go/mysql/binlog_event_filepos.go +++ b/go/mysql/binlog_event_filepos.go @@ -203,6 +203,10 @@ func (ev filePosFakeEvent) IsUpdateRows() bool { return false } +func (ev filePosFakeEvent) IsPartialUpdateRows() bool { + return false +} + func (ev filePosFakeEvent) IsDeleteRows() bool { return false } diff --git a/go/mysql/binlog_event_mysql56_test.go b/go/mysql/binlog_event_mysql56_test.go index 5844779de63..be9ae071ded 100644 --- a/go/mysql/binlog_event_mysql56_test.go +++ b/go/mysql/binlog_event_mysql56_test.go @@ -27,6 +27,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "vitess.io/vitess/go/mysql/collations" "vitess.io/vitess/go/mysql/replication" ) @@ -248,3 +249,111 @@ func TestMysql56SemiSyncAck(t *testing.T) { assert.True(t, e.IsQuery()) } } + +func TestMySQL56PartialUpdateRowsEvent(t *testing.T) { + // The mysqlbinlog -vvv --base64-output=decode-rows output for the following event: + // ### UPDATE `vt_commerce`.`customer` + // ### WHERE + // ### @1=1 /* LONGINT meta=0 nullable=0 is_null=0 */ + // ### @2='alice@domain.com' /* VARSTRING(128) meta=128 nullable=1 is_null=0 */ + // ### @3='{"salary": 100}' /* JSON meta=4 nullable=1 is_null=0 */ + // ### SET + // ### @1=1 /* LONGINT meta=0 nullable=0 is_null=0 */ + // ### @2='alice@domain.com' /* VARSTRING(128) meta=128 nullable=1 is_null=0 */ + // ### @3=JSON_INSERT(@3, '$.role', 'manager') /* JSON meta=4 nullable=1 is_null=0 */ + // ### UPDATE `vt_commerce`.`customer` + // ### WHERE + // ### @1=2 /* LONGINT meta=0 nullable=0 is_null=0 */ + // ### @2='bob@domain.com' /* VARSTRING(128) meta=128 nullable=1 is_null=0 */ + // ### @3='{"salary": 99}' /* JSON meta=4 nullable=1 is_null=0 */ + // ### SET + // ### @1=2 /* LONGINT meta=0 nullable=0 is_null=0 */ + // ### @2='bob@domain.com' /* VARSTRING(128) meta=128 nullable=1 is_null=0 */ + // ### @3=JSON_INSERT(@3, '$.role', 'manager') /* JSON meta=4 nullable=1 is_null=0 */ + // ### UPDATE `vt_commerce`.`customer` + // ### WHERE + // ### @1=3 /* LONGINT meta=0 nullable=0 is_null=0 */ + // ### @2='charlie@domain.com' /* VARSTRING(128) meta=128 nullable=1 is_null=0 */ + // ### @3='{"salary": 99}' /* JSON meta=4 nullable=1 is_null=0 */ + // ### SET + // ### @1=3 /* LONGINT meta=0 nullable=0 is_null=0 */ + // ### @2='charlie@domain.com' /* VARSTRING(128) meta=128 nullable=1 is_null=0 */ + // ### @3=JSON_INSERT(@3, '$.role', 'manager') /* JSON meta=4 nullable=1 is_null=0 */ + // ### UPDATE `vt_commerce`.`customer` + // ### WHERE + // ### @1=4 /* LONGINT meta=0 nullable=0 is_null=0 */ + // ### @2='dan@domain.com' /* VARSTRING(128) meta=128 nullable=1 is_null=0 */ + // ### @3='{"salary": 99}' /* JSON meta=4 nullable=1 is_null=0 */ + // ### SET + // ### @1=4 /* LONGINT meta=0 nullable=0 is_null=0 */ + // ### @2='dan@domain.com' /* VARSTRING(128) meta=128 nullable=1 is_null=0 */ + // ### @3=JSON_INSERT(@3, '$.role', 'manager') /* JSON meta=4 nullable=1 is_null=0 */ + // ### UPDATE `vt_commerce`.`customer` + // ### WHERE + // ### @1=5 /* LONGINT meta=0 nullable=0 is_null=0 */ + // ### @2='eve@domain.com' /* VARSTRING(128) meta=128 nullable=1 is_null=0 */ + // ### @3='{"salary": 100}' /* JSON meta=4 nullable=1 is_null=0 */ + // ### SET + // ### @1=5 /* LONGINT meta=0 nullable=0 is_null=0 */ + // ### @2='eve@domain.com' /* VARSTRING(128) meta=128 nullable=1 is_null=0 */ + // ### @3=JSON_INSERT(@3, '$.role', 'manager') /* JSON meta=4 nullable=1 is_null=0 */ + mysql56PartialUpdateRowEvent := NewMysql56BinlogEvent([]byte{0x67, 0xc4, 0x54, 0x67, 0x27, 0x1f, 0x91, 0x10, 0x76, 0x14, 0x02, 0x00, 0x00, 0x19, 0x69, + 0x00, 0x00, 0x00, 0x00, 0xb0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x02, 0x00, 0x03, 0xff, 0xff, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x10, 0x61, 0x6c, 0x69, 0x63, 0x65, 0x40, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x2e, 0x63, 0x6f, 0x6d, 0x16, 0x00, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x15, 0x00, 0x0b, 0x00, 0x03, 0x00, 0x0c, 0x0e, 0x00, 0x73, 0x65, 0x78, 0x06, 0x66, 0x65, 0x6d, 0x61, 0x6c, 0x65, 0x01, 0x01, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x61, 0x6c, 0x69, 0x63, 0x65, 0x40, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x2e, 0x63, 0x6f, + 0x6d, 0x12, 0x00, 0x00, 0x00, 0x01, 0x06, 0x24, 0x2e, 0x72, 0x6f, 0x6c, 0x65, 0x09, 0x0c, 0x07, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x00, + 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0e, 0x62, 0x6f, 0x62, 0x40, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x2e, 0x63, 0x6f, 0x6d, 0x14, + 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x13, 0x00, 0x0b, 0x00, 0x03, 0x00, 0x0c, 0x0e, 0x00, 0x73, 0x65, 0x78, 0x04, 0x6d, 0x61, 0x6c, 0x65, 0x01, + 0x01, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0e, 0x62, 0x6f, 0x62, 0x40, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x2e, 0x63, 0x6f, + 0x6d, 0x12, 0x00, 0x00, 0x00, 0x01, 0x06, 0x24, 0x2e, 0x72, 0x6f, 0x6c, 0x65, 0x09, 0x0c, 0x07, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x00, + 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x12, 0x63, 0x68, 0x61, 0x72, 0x6c, 0x69, 0x65, 0x40, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x2e, + 0x63, 0x6f, 0x6d, 0x14, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x13, 0x00, 0x0b, 0x00, 0x03, 0x00, 0x0c, 0x0e, 0x00, 0x73, 0x65, 0x78, 0x04, 0x6d, + 0x61, 0x6c, 0x65, 0x01, 0x01, 0x00, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x12, 0x63, 0x68, 0x61, 0x72, 0x6c, 0x69, 0x65, 0x40, 0x64, + 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x2e, 0x63, 0x6f, 0x6d, 0x12, 0x00, 0x00, 0x00, 0x01, 0x06, 0x24, 0x2e, 0x72, 0x6f, 0x6c, 0x65, 0x09, 0x0c, 0x07, + 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0e, 0x64, 0x61, 0x6e, 0x40, 0x64, 0x6f, 0x6d, + 0x61, 0x69, 0x6e, 0x2e, 0x63, 0x6f, 0x6d, 0x14, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x13, 0x00, 0x0b, 0x00, 0x03, 0x00, 0x0c, 0x0e, 0x00, 0x73, + 0x65, 0x78, 0x04, 0x6d, 0x61, 0x6c, 0x65, 0x01, 0x01, 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0e, 0x64, 0x61, 0x6e, 0x40, 0x64, + 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x2e, 0x63, 0x6f, 0x6d, 0x12, 0x00, 0x00, 0x00, 0x01, 0x06, 0x24, 0x2e, 0x72, 0x6f, 0x6c, 0x65, 0x09, 0x0c, 0x07, + 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x00, 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0e, 0x65, 0x76, 0x65, 0x40, 0x64, 0x6f, 0x6d, + 0x61, 0x69, 0x6e, 0x2e, 0x63, 0x6f, 0x6d, 0x16, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x15, 0x00, 0x0b, 0x00, 0x03, 0x00, 0x0c, 0x0e, 0x00, 0x73, + 0x65, 0x78, 0x06, 0x66, 0x65, 0x6d, 0x61, 0x6c, 0x65, 0x01, 0x01, 0x00, 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0e, 0x65, 0x76, 0x65, + 0x40, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x2e, 0x63, 0x6f, 0x6d, 0x12, 0x00, 0x00, 0x00, 0x01, 0x06, 0x24, 0x2e, 0x72, 0x6f, 0x6c, 0x65, 0x09, + 0x0c, 0x07, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72}) + + require.True(t, mysql56PartialUpdateRowEvent.IsPartialUpdateRows()) + + format := BinlogFormat{ + HeaderSizes: []byte{ + 0, 13, 0, 8, 0, 0, 0, 0, 4, 0, 4, 0, 0, 0, 98, 0, 4, 26, 8, 0, 0, 0, 8, 8, 8, 2, 0, 0, 0, 10, 10, 10, 42, 42, 0, 18, 52, 0, 10, 40, 0, + }, + ServerVersion: "8.0.40", + FormatVersion: 4, + HeaderLength: 19, + ChecksumAlgorithm: 1, + } + tm := &TableMap{ + Flags: 1, + Database: "vt_commerce", + Name: "customer", + Types: []byte{8, 15, 245}, + CanBeNull: Bitmap{ + data: []byte{6}, + count: 3, + }, + Metadata: []uint16{0, 128, 4}, + ColumnCollationIDs: []collations.ID{63}, + } + ev, err := mysql56PartialUpdateRowEvent.Rows(format, tm) + require.NoError(t, err) + + assert.Equal(t, 5, len(ev.Rows)) + require.NoError(t, err) + for i := range ev.Rows { + vals, err := ev.StringValuesForTests(tm, i) + require.NoError(t, err) + // The third column is the JSON column. + require.Equal(t, vals[2], `JSON_INSERT(%s, '$.role', "manager")`) + t.Logf("Rows: %v", vals) + } +} diff --git a/go/mysql/binlog_event_rbr.go b/go/mysql/binlog_event_rbr.go index d77b7bcb9a0..ea59a2a2247 100644 --- a/go/mysql/binlog_event_rbr.go +++ b/go/mysql/binlog_event_rbr.go @@ -21,6 +21,7 @@ import ( "vitess.io/vitess/go/mysql/binlog" "vitess.io/vitess/go/mysql/collations" + "vitess.io/vitess/go/vt/log" "vitess.io/vitess/go/vt/vterrors" querypb "vitess.io/vitess/go/vt/proto/query" @@ -283,10 +284,10 @@ func readColumnCollationIDs(data []byte, pos, count int) ([]collations.ID, error func (ev binlogEvent) Rows(f BinlogFormat, tm *TableMap) (Rows, error) { typ := ev.Type() data := ev.Bytes()[f.HeaderLength:] - hasIdentify := typ == eUpdateRowsEventV1 || typ == eUpdateRowsEventV2 || + hasIdentify := typ == eUpdateRowsEventV1 || typ == eUpdateRowsEventV2 || typ == ePartialUpdateRowsEvent || typ == eDeleteRowsEventV1 || typ == eDeleteRowsEventV2 hasData := typ == eWriteRowsEventV1 || typ == eWriteRowsEventV2 || - typ == eUpdateRowsEventV1 || typ == eUpdateRowsEventV2 + typ == eUpdateRowsEventV1 || typ == ePartialUpdateRowsEvent || typ == eUpdateRowsEventV2 result := Rows{} pos := 6 @@ -297,7 +298,7 @@ func (ev binlogEvent) Rows(f BinlogFormat, tm *TableMap) (Rows, error) { pos += 2 // version=2 have extra data here. - if typ == eWriteRowsEventV2 || typ == eUpdateRowsEventV2 || typ == eDeleteRowsEventV2 { + if typ == eWriteRowsEventV2 || typ == eUpdateRowsEventV2 || typ == ePartialUpdateRowsEvent || typ == eDeleteRowsEventV2 { // This extraDataLength contains the 2 bytes length. extraDataLength := binary.LittleEndian.Uint16(data[pos : pos+2]) pos += int(extraDataLength) @@ -311,6 +312,7 @@ func (ev binlogEvent) Rows(f BinlogFormat, tm *TableMap) (Rows, error) { numIdentifyColumns := 0 numDataColumns := 0 + numJSONColumns := 0 if hasIdentify { // Bitmap of the columns used for identify. @@ -324,6 +326,15 @@ func (ev binlogEvent) Rows(f BinlogFormat, tm *TableMap) (Rows, error) { numDataColumns = result.DataColumns.BitCount() } + // For PartialUpdateRowsEvents, we need to know how many JSON columns there are. + if ev.Type() == ePartialUpdateRowsEvent { + for c := 0; c < int(columnCount); c++ { + if tm.Types[c] == binlog.TypeJSON { + numJSONColumns++ + } + } + } + // One row at a time. for pos < len(data) { row := Row{} @@ -358,6 +369,19 @@ func (ev binlogEvent) Rows(f BinlogFormat, tm *TableMap) (Rows, error) { row.Identify = data[startPos:pos] } + if ev.Type() == ePartialUpdateRowsEvent { + log.Errorf("DEBUG: PartialUpdateRowsEvent found with %d JSON columns", numJSONColumns) + // The first byte indicates whether or not any JSON values are partial. + // If it's 0 then there's nothing else to do. + partialJSON := uint8(data[pos]) + log.Errorf("DEBUG: PartialJSON: %d", partialJSON) + pos++ + if partialJSON == 1 { + row.JSONPartialValues, pos = newBitmap(data, pos, numJSONColumns) + log.Errorf("DEBUG: PartialUpdateRowsEvent: JSONPartialValues: %08b", row.JSONPartialValues) + } + } + if hasData { // Bitmap of columns that are null (amongst the ones that are present). row.NullColumns, pos = newBitmap(data, pos, numDataColumns) @@ -402,6 +426,7 @@ func (rs *Rows) StringValuesForTests(tm *TableMap, rowIndex int) ([]string, erro var result []string valueIndex := 0 + jsonIndex := 0 data := rs.Rows[rowIndex].Data pos := 0 for c := 0; c < rs.DataColumns.Count(); c++ { @@ -416,12 +441,18 @@ func (rs *Rows) StringValuesForTests(tm *TableMap, rowIndex int) ([]string, erro continue } - // We have real data - value, l, err := binlog.CellValue(data, pos, tm.Types[c], tm.Metadata[c], &querypb.Field{Type: querypb.Type_UINT64}) + partialJSON := false + if rs.Rows[rowIndex].JSONPartialValues.Count() > 0 && tm.Types[c] == binlog.TypeJSON { + partialJSON = rs.Rows[rowIndex].JSONPartialValues.Bit(jsonIndex) + jsonIndex++ + } + + // We have real data. + value, l, err := binlog.CellValue(data, pos, tm.Types[c], tm.Metadata[c], &querypb.Field{Type: querypb.Type_UINT64}, partialJSON) if err != nil { return nil, err } - result = append(result, value.ToString()) + result = append(result, value.RawStr()) pos += l valueIndex++ } @@ -452,7 +483,7 @@ func (rs *Rows) StringIdentifiesForTests(tm *TableMap, rowIndex int) ([]string, } // We have real data - value, l, err := binlog.CellValue(data, pos, tm.Types[c], tm.Metadata[c], &querypb.Field{Type: querypb.Type_UINT64}) + value, l, err := binlog.CellValue(data, pos, tm.Types[c], tm.Metadata[c], &querypb.Field{Type: querypb.Type_UINT64}, false) if err != nil { return nil, err } diff --git a/go/mysql/replication_constants.go b/go/mysql/replication_constants.go index 6b6e34b2333..27d0bd331ce 100644 --- a/go/mysql/replication_constants.go +++ b/go/mysql/replication_constants.go @@ -110,6 +110,9 @@ const ( //eViewChangeEvent = 37 //eXAPrepareLogEvent = 38 + // PartialUpdateRowsEvent when binlog_row_value_options=PARTIAL_JSON. + ePartialUpdateRowsEvent = 39 + // Transaction_payload_event when binlog_transaction_compression=ON. eTransactionPayloadEvent = 40 diff --git a/go/vt/binlog/binlog_streamer.go b/go/vt/binlog/binlog_streamer.go index d62fcc3a915..08e06ec803c 100644 --- a/go/vt/binlog/binlog_streamer.go +++ b/go/vt/binlog/binlog_streamer.go @@ -760,7 +760,7 @@ func writeValuesAsSQL(sql *sqlparser.TrackedBuffer, tce *tableCacheEntry, rs *my } // We have real data. - value, l, err := binlog.CellValue(data, pos, tce.tm.Types[c], tce.tm.Metadata[c], &querypb.Field{Type: tce.ti.Fields[c].Type}) + value, l, err := binlog.CellValue(data, pos, tce.tm.Types[c], tce.tm.Metadata[c], &querypb.Field{Type: tce.ti.Fields[c].Type}, false) if err != nil { return keyspaceIDCell, nil, err } @@ -825,7 +825,7 @@ func writeIdentifiersAsSQL(sql *sqlparser.TrackedBuffer, tce *tableCacheEntry, r sql.WriteByte('=') // We have real data. - value, l, err := binlog.CellValue(data, pos, tce.tm.Types[c], tce.tm.Metadata[c], &querypb.Field{Type: tce.ti.Fields[c].Type}) + value, l, err := binlog.CellValue(data, pos, tce.tm.Types[c], tce.tm.Metadata[c], &querypb.Field{Type: tce.ti.Fields[c].Type}, false) if err != nil { return keyspaceIDCell, nil, err } diff --git a/go/vt/proto/binlogdata/binlogdata.pb.go b/go/vt/proto/binlogdata/binlogdata.pb.go index 35e738f772e..51736c606db 100644 --- a/go/vt/proto/binlogdata/binlogdata.pb.go +++ b/go/vt/proto/binlogdata/binlogdata.pb.go @@ -1333,8 +1333,17 @@ type RowChange struct { Before *query.Row `protobuf:"bytes,1,opt,name=before,proto3" json:"before,omitempty"` After *query.Row `protobuf:"bytes,2,opt,name=after,proto3" json:"after,omitempty"` - // DataColumns is a bitmap of all columns: bit is set if column is present in the after image + // DataColumns is a bitmap of all columns: bit is set if column is + // present in the after image. DataColumns *RowChange_Bitmap `protobuf:"bytes,3,opt,name=data_columns,json=dataColumns,proto3" json:"data_columns,omitempty"` + // JsonPartialValues is a bitmap of any JSON columns where the bit is + // set if the value in the after image is a partial JSON value that + // is represented as an expression of + // JSON_[INSERT|REPLACE|REMOVE](%s, '$.path', value) which then is + // used to add/update/replace a path in the JSON document. When the + // value is used the fmt directive must be replaced by the actual + // column name of the JSON field. + JsonPartialValues *RowChange_Bitmap `protobuf:"bytes,4,opt,name=json_partial_values,json=jsonPartialValues,proto3" json:"json_partial_values,omitempty"` } func (x *RowChange) Reset() { @@ -1388,6 +1397,13 @@ func (x *RowChange) GetDataColumns() *RowChange_Bitmap { return nil } +func (x *RowChange) GetJsonPartialValues() *RowChange_Bitmap { + if x != nil { + return x.JsonPartialValues + } + return nil +} + // RowEvent represent row events for one table. type RowEvent struct { state protoimpl.MessageState @@ -3223,7 +3239,7 @@ var file_binlogdata_proto_rawDesc = []byte{ 0x65, 0x5a, 0x6f, 0x6e, 0x65, 0x12, 0x28, 0x0a, 0x10, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x5f, 0x7a, 0x6f, 0x6e, 0x65, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x5a, 0x6f, 0x6e, 0x65, 0x22, - 0xc6, 0x01, 0x0a, 0x09, 0x52, 0x6f, 0x77, 0x43, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x12, 0x22, 0x0a, + 0x94, 0x02, 0x0a, 0x09, 0x52, 0x6f, 0x77, 0x43, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x12, 0x22, 0x0a, 0x06, 0x62, 0x65, 0x66, 0x6f, 0x72, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0a, 0x2e, 0x71, 0x75, 0x65, 0x72, 0x79, 0x2e, 0x52, 0x6f, 0x77, 0x52, 0x06, 0x62, 0x65, 0x66, 0x6f, 0x72, 0x65, 0x12, 0x20, 0x0a, 0x05, 0x61, 0x66, 0x74, 0x65, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, @@ -3232,313 +3248,318 @@ var file_binlogdata_proto_rawDesc = []byte{ 0x6d, 0x6e, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x62, 0x69, 0x6e, 0x6c, 0x6f, 0x67, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x52, 0x6f, 0x77, 0x43, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x2e, 0x42, 0x69, 0x74, 0x6d, 0x61, 0x70, 0x52, 0x0b, 0x64, 0x61, 0x74, 0x61, 0x43, 0x6f, 0x6c, - 0x75, 0x6d, 0x6e, 0x73, 0x1a, 0x32, 0x0a, 0x06, 0x42, 0x69, 0x74, 0x6d, 0x61, 0x70, 0x12, 0x14, - 0x0a, 0x05, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x05, 0x63, - 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x63, 0x6f, 0x6c, 0x73, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x0c, 0x52, 0x04, 0x63, 0x6f, 0x6c, 0x73, 0x22, 0xd5, 0x01, 0x0a, 0x08, 0x52, 0x6f, 0x77, - 0x45, 0x76, 0x65, 0x6e, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x6e, - 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x74, 0x61, 0x62, 0x6c, 0x65, - 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x36, 0x0a, 0x0b, 0x72, 0x6f, 0x77, 0x5f, 0x63, 0x68, 0x61, 0x6e, - 0x67, 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x62, 0x69, 0x6e, 0x6c, - 0x6f, 0x67, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x52, 0x6f, 0x77, 0x43, 0x68, 0x61, 0x6e, 0x67, 0x65, - 0x52, 0x0a, 0x72, 0x6f, 0x77, 0x43, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x73, 0x12, 0x1a, 0x0a, 0x08, - 0x6b, 0x65, 0x79, 0x73, 0x70, 0x61, 0x63, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, - 0x6b, 0x65, 0x79, 0x73, 0x70, 0x61, 0x63, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x73, 0x68, 0x61, 0x72, - 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x73, 0x68, 0x61, 0x72, 0x64, 0x12, 0x14, - 0x0a, 0x05, 0x66, 0x6c, 0x61, 0x67, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x05, 0x66, - 0x6c, 0x61, 0x67, 0x73, 0x12, 0x2a, 0x0a, 0x11, 0x69, 0x73, 0x5f, 0x69, 0x6e, 0x74, 0x65, 0x72, - 0x6e, 0x61, 0x6c, 0x5f, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x08, 0x52, - 0x0f, 0x69, 0x73, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x54, 0x61, 0x62, 0x6c, 0x65, - 0x22, 0xe4, 0x01, 0x0a, 0x0a, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x12, - 0x1d, 0x0a, 0x0a, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x09, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x24, - 0x0a, 0x06, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0c, - 0x2e, 0x71, 0x75, 0x65, 0x72, 0x79, 0x2e, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x52, 0x06, 0x66, 0x69, - 0x65, 0x6c, 0x64, 0x73, 0x12, 0x1a, 0x0a, 0x08, 0x6b, 0x65, 0x79, 0x73, 0x70, 0x61, 0x63, 0x65, - 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x6b, 0x65, 0x79, 0x73, 0x70, 0x61, 0x63, 0x65, - 0x12, 0x14, 0x0a, 0x05, 0x73, 0x68, 0x61, 0x72, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x05, 0x73, 0x68, 0x61, 0x72, 0x64, 0x12, 0x33, 0x0a, 0x16, 0x65, 0x6e, 0x75, 0x6d, 0x5f, 0x73, - 0x65, 0x74, 0x5f, 0x73, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x73, - 0x18, 0x19, 0x20, 0x01, 0x28, 0x08, 0x52, 0x13, 0x65, 0x6e, 0x75, 0x6d, 0x53, 0x65, 0x74, 0x53, - 0x74, 0x72, 0x69, 0x6e, 0x67, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x12, 0x2a, 0x0a, 0x11, 0x69, - 0x73, 0x5f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x5f, 0x74, 0x61, 0x62, 0x6c, 0x65, - 0x18, 0x1a, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0f, 0x69, 0x73, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x6e, - 0x61, 0x6c, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x22, 0x88, 0x01, 0x0a, 0x09, 0x53, 0x68, 0x61, 0x72, - 0x64, 0x47, 0x74, 0x69, 0x64, 0x12, 0x1a, 0x0a, 0x08, 0x6b, 0x65, 0x79, 0x73, 0x70, 0x61, 0x63, - 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x6b, 0x65, 0x79, 0x73, 0x70, 0x61, 0x63, - 0x65, 0x12, 0x14, 0x0a, 0x05, 0x73, 0x68, 0x61, 0x72, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x05, 0x73, 0x68, 0x61, 0x72, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x67, 0x74, 0x69, 0x64, 0x18, - 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x67, 0x74, 0x69, 0x64, 0x12, 0x35, 0x0a, 0x0a, 0x74, - 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x70, 0x5f, 0x6b, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, - 0x17, 0x2e, 0x62, 0x69, 0x6e, 0x6c, 0x6f, 0x67, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x54, 0x61, 0x62, - 0x6c, 0x65, 0x4c, 0x61, 0x73, 0x74, 0x50, 0x4b, 0x52, 0x08, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x50, - 0x4b, 0x73, 0x22, 0x3f, 0x0a, 0x05, 0x56, 0x47, 0x74, 0x69, 0x64, 0x12, 0x36, 0x0a, 0x0b, 0x73, - 0x68, 0x61, 0x72, 0x64, 0x5f, 0x67, 0x74, 0x69, 0x64, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, - 0x32, 0x15, 0x2e, 0x62, 0x69, 0x6e, 0x6c, 0x6f, 0x67, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x53, 0x68, - 0x61, 0x72, 0x64, 0x47, 0x74, 0x69, 0x64, 0x52, 0x0a, 0x73, 0x68, 0x61, 0x72, 0x64, 0x47, 0x74, - 0x69, 0x64, 0x73, 0x22, 0x41, 0x0a, 0x0d, 0x4b, 0x65, 0x79, 0x73, 0x70, 0x61, 0x63, 0x65, 0x53, - 0x68, 0x61, 0x72, 0x64, 0x12, 0x1a, 0x0a, 0x08, 0x6b, 0x65, 0x79, 0x73, 0x70, 0x61, 0x63, 0x65, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x6b, 0x65, 0x79, 0x73, 0x70, 0x61, 0x63, 0x65, - 0x12, 0x14, 0x0a, 0x05, 0x73, 0x68, 0x61, 0x72, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x05, 0x73, 0x68, 0x61, 0x72, 0x64, 0x22, 0xbc, 0x02, 0x0a, 0x07, 0x4a, 0x6f, 0x75, 0x72, 0x6e, - 0x61, 0x6c, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x02, - 0x69, 0x64, 0x12, 0x40, 0x0a, 0x0e, 0x6d, 0x69, 0x67, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, - 0x74, 0x79, 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x19, 0x2e, 0x62, 0x69, 0x6e, - 0x6c, 0x6f, 0x67, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x4d, 0x69, 0x67, 0x72, 0x61, 0x74, 0x69, 0x6f, - 0x6e, 0x54, 0x79, 0x70, 0x65, 0x52, 0x0d, 0x6d, 0x69, 0x67, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, - 0x54, 0x79, 0x70, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x73, 0x18, 0x03, - 0x20, 0x03, 0x28, 0x09, 0x52, 0x06, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x73, 0x12, 0x25, 0x0a, 0x0e, - 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x5f, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x04, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x50, 0x6f, 0x73, 0x69, 0x74, - 0x69, 0x6f, 0x6e, 0x12, 0x36, 0x0a, 0x0b, 0x73, 0x68, 0x61, 0x72, 0x64, 0x5f, 0x67, 0x74, 0x69, - 0x64, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x62, 0x69, 0x6e, 0x6c, 0x6f, - 0x67, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x53, 0x68, 0x61, 0x72, 0x64, 0x47, 0x74, 0x69, 0x64, 0x52, - 0x0a, 0x73, 0x68, 0x61, 0x72, 0x64, 0x47, 0x74, 0x69, 0x64, 0x73, 0x12, 0x3d, 0x0a, 0x0c, 0x70, - 0x61, 0x72, 0x74, 0x69, 0x63, 0x69, 0x70, 0x61, 0x6e, 0x74, 0x73, 0x18, 0x06, 0x20, 0x03, 0x28, - 0x0b, 0x32, 0x19, 0x2e, 0x62, 0x69, 0x6e, 0x6c, 0x6f, 0x67, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x4b, - 0x65, 0x79, 0x73, 0x70, 0x61, 0x63, 0x65, 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x0c, 0x70, 0x61, - 0x72, 0x74, 0x69, 0x63, 0x69, 0x70, 0x61, 0x6e, 0x74, 0x73, 0x12, 0x29, 0x0a, 0x10, 0x73, 0x6f, - 0x75, 0x72, 0x63, 0x65, 0x5f, 0x77, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x73, 0x18, 0x07, - 0x20, 0x03, 0x28, 0x09, 0x52, 0x0f, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x57, 0x6f, 0x72, 0x6b, - 0x66, 0x6c, 0x6f, 0x77, 0x73, 0x22, 0xb6, 0x04, 0x0a, 0x06, 0x56, 0x45, 0x76, 0x65, 0x6e, 0x74, - 0x12, 0x2a, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x16, + 0x75, 0x6d, 0x6e, 0x73, 0x12, 0x4c, 0x0a, 0x13, 0x6a, 0x73, 0x6f, 0x6e, 0x5f, 0x70, 0x61, 0x72, + 0x74, 0x69, 0x61, 0x6c, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x1c, 0x2e, 0x62, 0x69, 0x6e, 0x6c, 0x6f, 0x67, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x52, + 0x6f, 0x77, 0x43, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x2e, 0x42, 0x69, 0x74, 0x6d, 0x61, 0x70, 0x52, + 0x11, 0x6a, 0x73, 0x6f, 0x6e, 0x50, 0x61, 0x72, 0x74, 0x69, 0x61, 0x6c, 0x56, 0x61, 0x6c, 0x75, + 0x65, 0x73, 0x1a, 0x32, 0x0a, 0x06, 0x42, 0x69, 0x74, 0x6d, 0x61, 0x70, 0x12, 0x14, 0x0a, 0x05, + 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x05, 0x63, 0x6f, 0x75, + 0x6e, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x63, 0x6f, 0x6c, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, + 0x52, 0x04, 0x63, 0x6f, 0x6c, 0x73, 0x22, 0xd5, 0x01, 0x0a, 0x08, 0x52, 0x6f, 0x77, 0x45, 0x76, + 0x65, 0x6e, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x6e, 0x61, 0x6d, + 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x4e, 0x61, + 0x6d, 0x65, 0x12, 0x36, 0x0a, 0x0b, 0x72, 0x6f, 0x77, 0x5f, 0x63, 0x68, 0x61, 0x6e, 0x67, 0x65, + 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x62, 0x69, 0x6e, 0x6c, 0x6f, 0x67, + 0x64, 0x61, 0x74, 0x61, 0x2e, 0x52, 0x6f, 0x77, 0x43, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x52, 0x0a, + 0x72, 0x6f, 0x77, 0x43, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x73, 0x12, 0x1a, 0x0a, 0x08, 0x6b, 0x65, + 0x79, 0x73, 0x70, 0x61, 0x63, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x6b, 0x65, + 0x79, 0x73, 0x70, 0x61, 0x63, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x73, 0x68, 0x61, 0x72, 0x64, 0x18, + 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x73, 0x68, 0x61, 0x72, 0x64, 0x12, 0x14, 0x0a, 0x05, + 0x66, 0x6c, 0x61, 0x67, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x05, 0x66, 0x6c, 0x61, + 0x67, 0x73, 0x12, 0x2a, 0x0a, 0x11, 0x69, 0x73, 0x5f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, + 0x6c, 0x5f, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0f, 0x69, + 0x73, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x22, 0xe4, + 0x01, 0x0a, 0x0a, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x12, 0x1d, 0x0a, + 0x0a, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x09, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x24, 0x0a, 0x06, + 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0c, 0x2e, 0x71, + 0x75, 0x65, 0x72, 0x79, 0x2e, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x52, 0x06, 0x66, 0x69, 0x65, 0x6c, + 0x64, 0x73, 0x12, 0x1a, 0x0a, 0x08, 0x6b, 0x65, 0x79, 0x73, 0x70, 0x61, 0x63, 0x65, 0x18, 0x03, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x6b, 0x65, 0x79, 0x73, 0x70, 0x61, 0x63, 0x65, 0x12, 0x14, + 0x0a, 0x05, 0x73, 0x68, 0x61, 0x72, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x73, + 0x68, 0x61, 0x72, 0x64, 0x12, 0x33, 0x0a, 0x16, 0x65, 0x6e, 0x75, 0x6d, 0x5f, 0x73, 0x65, 0x74, + 0x5f, 0x73, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x18, 0x19, + 0x20, 0x01, 0x28, 0x08, 0x52, 0x13, 0x65, 0x6e, 0x75, 0x6d, 0x53, 0x65, 0x74, 0x53, 0x74, 0x72, + 0x69, 0x6e, 0x67, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x73, 0x12, 0x2a, 0x0a, 0x11, 0x69, 0x73, 0x5f, + 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x5f, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x18, 0x1a, + 0x20, 0x01, 0x28, 0x08, 0x52, 0x0f, 0x69, 0x73, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, + 0x54, 0x61, 0x62, 0x6c, 0x65, 0x22, 0x88, 0x01, 0x0a, 0x09, 0x53, 0x68, 0x61, 0x72, 0x64, 0x47, + 0x74, 0x69, 0x64, 0x12, 0x1a, 0x0a, 0x08, 0x6b, 0x65, 0x79, 0x73, 0x70, 0x61, 0x63, 0x65, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x6b, 0x65, 0x79, 0x73, 0x70, 0x61, 0x63, 0x65, 0x12, + 0x14, 0x0a, 0x05, 0x73, 0x68, 0x61, 0x72, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, + 0x73, 0x68, 0x61, 0x72, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x67, 0x74, 0x69, 0x64, 0x18, 0x03, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x04, 0x67, 0x74, 0x69, 0x64, 0x12, 0x35, 0x0a, 0x0a, 0x74, 0x61, 0x62, + 0x6c, 0x65, 0x5f, 0x70, 0x5f, 0x6b, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x17, 0x2e, + 0x62, 0x69, 0x6e, 0x6c, 0x6f, 0x67, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x54, 0x61, 0x62, 0x6c, 0x65, + 0x4c, 0x61, 0x73, 0x74, 0x50, 0x4b, 0x52, 0x08, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x50, 0x4b, 0x73, + 0x22, 0x3f, 0x0a, 0x05, 0x56, 0x47, 0x74, 0x69, 0x64, 0x12, 0x36, 0x0a, 0x0b, 0x73, 0x68, 0x61, + 0x72, 0x64, 0x5f, 0x67, 0x74, 0x69, 0x64, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x15, + 0x2e, 0x62, 0x69, 0x6e, 0x6c, 0x6f, 0x67, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x53, 0x68, 0x61, 0x72, + 0x64, 0x47, 0x74, 0x69, 0x64, 0x52, 0x0a, 0x73, 0x68, 0x61, 0x72, 0x64, 0x47, 0x74, 0x69, 0x64, + 0x73, 0x22, 0x41, 0x0a, 0x0d, 0x4b, 0x65, 0x79, 0x73, 0x70, 0x61, 0x63, 0x65, 0x53, 0x68, 0x61, + 0x72, 0x64, 0x12, 0x1a, 0x0a, 0x08, 0x6b, 0x65, 0x79, 0x73, 0x70, 0x61, 0x63, 0x65, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x6b, 0x65, 0x79, 0x73, 0x70, 0x61, 0x63, 0x65, 0x12, 0x14, + 0x0a, 0x05, 0x73, 0x68, 0x61, 0x72, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x73, + 0x68, 0x61, 0x72, 0x64, 0x22, 0xbc, 0x02, 0x0a, 0x07, 0x4a, 0x6f, 0x75, 0x72, 0x6e, 0x61, 0x6c, + 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x02, 0x69, 0x64, + 0x12, 0x40, 0x0a, 0x0e, 0x6d, 0x69, 0x67, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x74, 0x79, + 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x19, 0x2e, 0x62, 0x69, 0x6e, 0x6c, 0x6f, + 0x67, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x4d, 0x69, 0x67, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x54, + 0x79, 0x70, 0x65, 0x52, 0x0d, 0x6d, 0x69, 0x67, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x79, + 0x70, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, + 0x28, 0x09, 0x52, 0x06, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x73, 0x12, 0x25, 0x0a, 0x0e, 0x6c, 0x6f, + 0x63, 0x61, 0x6c, 0x5f, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x04, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x0d, 0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x50, 0x6f, 0x73, 0x69, 0x74, 0x69, 0x6f, + 0x6e, 0x12, 0x36, 0x0a, 0x0b, 0x73, 0x68, 0x61, 0x72, 0x64, 0x5f, 0x67, 0x74, 0x69, 0x64, 0x73, + 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x62, 0x69, 0x6e, 0x6c, 0x6f, 0x67, 0x64, + 0x61, 0x74, 0x61, 0x2e, 0x53, 0x68, 0x61, 0x72, 0x64, 0x47, 0x74, 0x69, 0x64, 0x52, 0x0a, 0x73, + 0x68, 0x61, 0x72, 0x64, 0x47, 0x74, 0x69, 0x64, 0x73, 0x12, 0x3d, 0x0a, 0x0c, 0x70, 0x61, 0x72, + 0x74, 0x69, 0x63, 0x69, 0x70, 0x61, 0x6e, 0x74, 0x73, 0x18, 0x06, 0x20, 0x03, 0x28, 0x0b, 0x32, + 0x19, 0x2e, 0x62, 0x69, 0x6e, 0x6c, 0x6f, 0x67, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x4b, 0x65, 0x79, + 0x73, 0x70, 0x61, 0x63, 0x65, 0x53, 0x68, 0x61, 0x72, 0x64, 0x52, 0x0c, 0x70, 0x61, 0x72, 0x74, + 0x69, 0x63, 0x69, 0x70, 0x61, 0x6e, 0x74, 0x73, 0x12, 0x29, 0x0a, 0x10, 0x73, 0x6f, 0x75, 0x72, + 0x63, 0x65, 0x5f, 0x77, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x73, 0x18, 0x07, 0x20, 0x03, + 0x28, 0x09, 0x52, 0x0f, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, + 0x6f, 0x77, 0x73, 0x22, 0xb6, 0x04, 0x0a, 0x06, 0x56, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x12, 0x2a, + 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x16, 0x2e, 0x62, + 0x69, 0x6e, 0x6c, 0x6f, 0x67, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x56, 0x45, 0x76, 0x65, 0x6e, 0x74, + 0x54, 0x79, 0x70, 0x65, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x74, 0x69, + 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x09, 0x74, + 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x12, 0x0a, 0x04, 0x67, 0x74, 0x69, 0x64, + 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x67, 0x74, 0x69, 0x64, 0x12, 0x1c, 0x0a, 0x09, + 0x73, 0x74, 0x61, 0x74, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x09, 0x73, 0x74, 0x61, 0x74, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x31, 0x0a, 0x09, 0x72, 0x6f, + 0x77, 0x5f, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, + 0x62, 0x69, 0x6e, 0x6c, 0x6f, 0x67, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x52, 0x6f, 0x77, 0x45, 0x76, + 0x65, 0x6e, 0x74, 0x52, 0x08, 0x72, 0x6f, 0x77, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x12, 0x37, 0x0a, + 0x0b, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x5f, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x18, 0x06, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x62, 0x69, 0x6e, 0x6c, 0x6f, 0x67, 0x64, 0x61, 0x74, 0x61, 0x2e, + 0x46, 0x69, 0x65, 0x6c, 0x64, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x52, 0x0a, 0x66, 0x69, 0x65, 0x6c, + 0x64, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x12, 0x27, 0x0a, 0x05, 0x76, 0x67, 0x74, 0x69, 0x64, 0x18, + 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x11, 0x2e, 0x62, 0x69, 0x6e, 0x6c, 0x6f, 0x67, 0x64, 0x61, + 0x74, 0x61, 0x2e, 0x56, 0x47, 0x74, 0x69, 0x64, 0x52, 0x05, 0x76, 0x67, 0x74, 0x69, 0x64, 0x12, + 0x2d, 0x0a, 0x07, 0x6a, 0x6f, 0x75, 0x72, 0x6e, 0x61, 0x6c, 0x18, 0x08, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x13, 0x2e, 0x62, 0x69, 0x6e, 0x6c, 0x6f, 0x67, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x4a, 0x6f, + 0x75, 0x72, 0x6e, 0x61, 0x6c, 0x52, 0x07, 0x6a, 0x6f, 0x75, 0x72, 0x6e, 0x61, 0x6c, 0x12, 0x10, + 0x0a, 0x03, 0x64, 0x6d, 0x6c, 0x18, 0x09, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x64, 0x6d, 0x6c, + 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, 0x5f, 0x74, 0x69, 0x6d, 0x65, + 0x18, 0x14, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0b, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, 0x54, + 0x69, 0x6d, 0x65, 0x12, 0x3c, 0x0a, 0x0e, 0x6c, 0x61, 0x73, 0x74, 0x5f, 0x70, 0x5f, 0x6b, 0x5f, + 0x65, 0x76, 0x65, 0x6e, 0x74, 0x18, 0x15, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x62, 0x69, + 0x6e, 0x6c, 0x6f, 0x67, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x4c, 0x61, 0x73, 0x74, 0x50, 0x4b, 0x45, + 0x76, 0x65, 0x6e, 0x74, 0x52, 0x0b, 0x6c, 0x61, 0x73, 0x74, 0x50, 0x4b, 0x45, 0x76, 0x65, 0x6e, + 0x74, 0x12, 0x1a, 0x0a, 0x08, 0x6b, 0x65, 0x79, 0x73, 0x70, 0x61, 0x63, 0x65, 0x18, 0x16, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x08, 0x6b, 0x65, 0x79, 0x73, 0x70, 0x61, 0x63, 0x65, 0x12, 0x14, 0x0a, + 0x05, 0x73, 0x68, 0x61, 0x72, 0x64, 0x18, 0x17, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x73, 0x68, + 0x61, 0x72, 0x64, 0x12, 0x1c, 0x0a, 0x09, 0x74, 0x68, 0x72, 0x6f, 0x74, 0x74, 0x6c, 0x65, 0x64, + 0x18, 0x18, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x74, 0x68, 0x72, 0x6f, 0x74, 0x74, 0x6c, 0x65, + 0x64, 0x12, 0x29, 0x0a, 0x10, 0x74, 0x68, 0x72, 0x6f, 0x74, 0x74, 0x6c, 0x65, 0x64, 0x5f, 0x72, + 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x18, 0x19, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x74, 0x68, 0x72, + 0x6f, 0x74, 0x74, 0x6c, 0x65, 0x64, 0x52, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x22, 0x8d, 0x01, 0x0a, + 0x0c, 0x4d, 0x69, 0x6e, 0x69, 0x6d, 0x61, 0x6c, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x12, 0x12, 0x0a, + 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, + 0x65, 0x12, 0x24, 0x0a, 0x06, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, + 0x0b, 0x32, 0x0c, 0x2e, 0x71, 0x75, 0x65, 0x72, 0x79, 0x2e, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x52, + 0x06, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x12, 0x1e, 0x0a, 0x0b, 0x70, 0x5f, 0x6b, 0x5f, 0x63, + 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x03, 0x52, 0x09, 0x70, 0x4b, + 0x43, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x73, 0x12, 0x23, 0x0a, 0x0e, 0x70, 0x5f, 0x6b, 0x5f, 0x69, + 0x6e, 0x64, 0x65, 0x78, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x0b, 0x70, 0x4b, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x4e, 0x61, 0x6d, 0x65, 0x22, 0x41, 0x0a, 0x0d, + 0x4d, 0x69, 0x6e, 0x69, 0x6d, 0x61, 0x6c, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x12, 0x30, 0x0a, + 0x06, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x18, 0x2e, + 0x62, 0x69, 0x6e, 0x6c, 0x6f, 0x67, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x4d, 0x69, 0x6e, 0x69, 0x6d, + 0x61, 0x6c, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x52, 0x06, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x73, 0x22, + 0xd9, 0x01, 0x0a, 0x0e, 0x56, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x4f, 0x70, 0x74, 0x69, 0x6f, + 0x6e, 0x73, 0x12, 0x27, 0x0a, 0x0f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x5f, 0x74, + 0x61, 0x62, 0x6c, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0e, 0x69, 0x6e, 0x74, + 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x73, 0x12, 0x5a, 0x0a, 0x10, 0x63, + 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x5f, 0x6f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x18, + 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2f, 0x2e, 0x62, 0x69, 0x6e, 0x6c, 0x6f, 0x67, 0x64, 0x61, + 0x74, 0x61, 0x2e, 0x56, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, + 0x73, 0x2e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, + 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x4f, 0x76, + 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x1a, 0x42, 0x0a, 0x14, 0x43, 0x6f, 0x6e, 0x66, 0x69, + 0x67, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, + 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, + 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0xfd, 0x02, 0x0a, 0x0e, + 0x56, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3f, + 0x0a, 0x13, 0x65, 0x66, 0x66, 0x65, 0x63, 0x74, 0x69, 0x76, 0x65, 0x5f, 0x63, 0x61, 0x6c, 0x6c, + 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x76, 0x74, + 0x72, 0x70, 0x63, 0x2e, 0x43, 0x61, 0x6c, 0x6c, 0x65, 0x72, 0x49, 0x44, 0x52, 0x11, 0x65, 0x66, + 0x66, 0x65, 0x63, 0x74, 0x69, 0x76, 0x65, 0x43, 0x61, 0x6c, 0x6c, 0x65, 0x72, 0x49, 0x64, 0x12, + 0x45, 0x0a, 0x13, 0x69, 0x6d, 0x6d, 0x65, 0x64, 0x69, 0x61, 0x74, 0x65, 0x5f, 0x63, 0x61, 0x6c, + 0x6c, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x71, + 0x75, 0x65, 0x72, 0x79, 0x2e, 0x56, 0x54, 0x47, 0x61, 0x74, 0x65, 0x43, 0x61, 0x6c, 0x6c, 0x65, + 0x72, 0x49, 0x44, 0x52, 0x11, 0x69, 0x6d, 0x6d, 0x65, 0x64, 0x69, 0x61, 0x74, 0x65, 0x43, 0x61, + 0x6c, 0x6c, 0x65, 0x72, 0x49, 0x64, 0x12, 0x25, 0x0a, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, + 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0d, 0x2e, 0x71, 0x75, 0x65, 0x72, 0x79, 0x2e, 0x54, + 0x61, 0x72, 0x67, 0x65, 0x74, 0x52, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x12, 0x1a, 0x0a, + 0x08, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x08, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x2a, 0x0a, 0x06, 0x66, 0x69, 0x6c, + 0x74, 0x65, 0x72, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x62, 0x69, 0x6e, 0x6c, + 0x6f, 0x67, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x52, 0x06, 0x66, + 0x69, 0x6c, 0x74, 0x65, 0x72, 0x12, 0x3e, 0x0a, 0x0f, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x6c, + 0x61, 0x73, 0x74, 0x5f, 0x70, 0x5f, 0x6b, 0x73, 0x18, 0x06, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x17, + 0x2e, 0x62, 0x69, 0x6e, 0x6c, 0x6f, 0x67, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x54, 0x61, 0x62, 0x6c, + 0x65, 0x4c, 0x61, 0x73, 0x74, 0x50, 0x4b, 0x52, 0x0c, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x4c, 0x61, + 0x73, 0x74, 0x50, 0x4b, 0x73, 0x12, 0x34, 0x0a, 0x07, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, + 0x18, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x62, 0x69, 0x6e, 0x6c, 0x6f, 0x67, 0x64, + 0x61, 0x74, 0x61, 0x2e, 0x56, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x4f, 0x70, 0x74, 0x69, 0x6f, + 0x6e, 0x73, 0x52, 0x07, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x22, 0x3d, 0x0a, 0x0f, 0x56, + 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2a, + 0x0a, 0x06, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x62, 0x69, 0x6e, 0x6c, 0x6f, 0x67, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x56, 0x45, 0x76, 0x65, - 0x6e, 0x74, 0x54, 0x79, 0x70, 0x65, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x1c, 0x0a, 0x09, - 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, - 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x12, 0x0a, 0x04, 0x67, 0x74, - 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x67, 0x74, 0x69, 0x64, 0x12, 0x1c, - 0x0a, 0x09, 0x73, 0x74, 0x61, 0x74, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x09, 0x73, 0x74, 0x61, 0x74, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x31, 0x0a, 0x09, - 0x72, 0x6f, 0x77, 0x5f, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x14, 0x2e, 0x62, 0x69, 0x6e, 0x6c, 0x6f, 0x67, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x52, 0x6f, 0x77, - 0x45, 0x76, 0x65, 0x6e, 0x74, 0x52, 0x08, 0x72, 0x6f, 0x77, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x12, - 0x37, 0x0a, 0x0b, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x5f, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x18, 0x06, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x62, 0x69, 0x6e, 0x6c, 0x6f, 0x67, 0x64, 0x61, 0x74, - 0x61, 0x2e, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x52, 0x0a, 0x66, 0x69, - 0x65, 0x6c, 0x64, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x12, 0x27, 0x0a, 0x05, 0x76, 0x67, 0x74, 0x69, - 0x64, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x11, 0x2e, 0x62, 0x69, 0x6e, 0x6c, 0x6f, 0x67, - 0x64, 0x61, 0x74, 0x61, 0x2e, 0x56, 0x47, 0x74, 0x69, 0x64, 0x52, 0x05, 0x76, 0x67, 0x74, 0x69, - 0x64, 0x12, 0x2d, 0x0a, 0x07, 0x6a, 0x6f, 0x75, 0x72, 0x6e, 0x61, 0x6c, 0x18, 0x08, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x62, 0x69, 0x6e, 0x6c, 0x6f, 0x67, 0x64, 0x61, 0x74, 0x61, 0x2e, - 0x4a, 0x6f, 0x75, 0x72, 0x6e, 0x61, 0x6c, 0x52, 0x07, 0x6a, 0x6f, 0x75, 0x72, 0x6e, 0x61, 0x6c, - 0x12, 0x10, 0x0a, 0x03, 0x64, 0x6d, 0x6c, 0x18, 0x09, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x64, - 0x6d, 0x6c, 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, 0x5f, 0x74, 0x69, - 0x6d, 0x65, 0x18, 0x14, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0b, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, - 0x74, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x3c, 0x0a, 0x0e, 0x6c, 0x61, 0x73, 0x74, 0x5f, 0x70, 0x5f, - 0x6b, 0x5f, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x18, 0x15, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, - 0x62, 0x69, 0x6e, 0x6c, 0x6f, 0x67, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x4c, 0x61, 0x73, 0x74, 0x50, - 0x4b, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x52, 0x0b, 0x6c, 0x61, 0x73, 0x74, 0x50, 0x4b, 0x45, 0x76, - 0x65, 0x6e, 0x74, 0x12, 0x1a, 0x0a, 0x08, 0x6b, 0x65, 0x79, 0x73, 0x70, 0x61, 0x63, 0x65, 0x18, - 0x16, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x6b, 0x65, 0x79, 0x73, 0x70, 0x61, 0x63, 0x65, 0x12, - 0x14, 0x0a, 0x05, 0x73, 0x68, 0x61, 0x72, 0x64, 0x18, 0x17, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, - 0x73, 0x68, 0x61, 0x72, 0x64, 0x12, 0x1c, 0x0a, 0x09, 0x74, 0x68, 0x72, 0x6f, 0x74, 0x74, 0x6c, - 0x65, 0x64, 0x18, 0x18, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x74, 0x68, 0x72, 0x6f, 0x74, 0x74, - 0x6c, 0x65, 0x64, 0x12, 0x29, 0x0a, 0x10, 0x74, 0x68, 0x72, 0x6f, 0x74, 0x74, 0x6c, 0x65, 0x64, - 0x5f, 0x72, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x18, 0x19, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x74, - 0x68, 0x72, 0x6f, 0x74, 0x74, 0x6c, 0x65, 0x64, 0x52, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x22, 0x8d, - 0x01, 0x0a, 0x0c, 0x4d, 0x69, 0x6e, 0x69, 0x6d, 0x61, 0x6c, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x12, - 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, - 0x61, 0x6d, 0x65, 0x12, 0x24, 0x0a, 0x06, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x18, 0x02, 0x20, - 0x03, 0x28, 0x0b, 0x32, 0x0c, 0x2e, 0x71, 0x75, 0x65, 0x72, 0x79, 0x2e, 0x46, 0x69, 0x65, 0x6c, - 0x64, 0x52, 0x06, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x12, 0x1e, 0x0a, 0x0b, 0x70, 0x5f, 0x6b, - 0x5f, 0x63, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x03, 0x52, 0x09, - 0x70, 0x4b, 0x43, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x73, 0x12, 0x23, 0x0a, 0x0e, 0x70, 0x5f, 0x6b, - 0x5f, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x0b, 0x70, 0x4b, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x4e, 0x61, 0x6d, 0x65, 0x22, 0x41, - 0x0a, 0x0d, 0x4d, 0x69, 0x6e, 0x69, 0x6d, 0x61, 0x6c, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x12, - 0x30, 0x0a, 0x06, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, - 0x18, 0x2e, 0x62, 0x69, 0x6e, 0x6c, 0x6f, 0x67, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x4d, 0x69, 0x6e, - 0x69, 0x6d, 0x61, 0x6c, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x52, 0x06, 0x74, 0x61, 0x62, 0x6c, 0x65, - 0x73, 0x22, 0xd9, 0x01, 0x0a, 0x0e, 0x56, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x4f, 0x70, 0x74, - 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x27, 0x0a, 0x0f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, - 0x5f, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0e, 0x69, - 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x73, 0x12, 0x5a, 0x0a, - 0x10, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x5f, 0x6f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, - 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2f, 0x2e, 0x62, 0x69, 0x6e, 0x6c, 0x6f, 0x67, - 0x64, 0x61, 0x74, 0x61, 0x2e, 0x56, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x4f, 0x70, 0x74, 0x69, - 0x6f, 0x6e, 0x73, 0x2e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, - 0x64, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, - 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x1a, 0x42, 0x0a, 0x14, 0x43, 0x6f, 0x6e, - 0x66, 0x69, 0x67, 0x4f, 0x76, 0x65, 0x72, 0x72, 0x69, 0x64, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, - 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, - 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0xfd, 0x02, - 0x0a, 0x0e, 0x56, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x12, 0x3f, 0x0a, 0x13, 0x65, 0x66, 0x66, 0x65, 0x63, 0x74, 0x69, 0x76, 0x65, 0x5f, 0x63, 0x61, - 0x6c, 0x6c, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0f, 0x2e, - 0x76, 0x74, 0x72, 0x70, 0x63, 0x2e, 0x43, 0x61, 0x6c, 0x6c, 0x65, 0x72, 0x49, 0x44, 0x52, 0x11, - 0x65, 0x66, 0x66, 0x65, 0x63, 0x74, 0x69, 0x76, 0x65, 0x43, 0x61, 0x6c, 0x6c, 0x65, 0x72, 0x49, - 0x64, 0x12, 0x45, 0x0a, 0x13, 0x69, 0x6d, 0x6d, 0x65, 0x64, 0x69, 0x61, 0x74, 0x65, 0x5f, 0x63, - 0x61, 0x6c, 0x6c, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, - 0x2e, 0x71, 0x75, 0x65, 0x72, 0x79, 0x2e, 0x56, 0x54, 0x47, 0x61, 0x74, 0x65, 0x43, 0x61, 0x6c, - 0x6c, 0x65, 0x72, 0x49, 0x44, 0x52, 0x11, 0x69, 0x6d, 0x6d, 0x65, 0x64, 0x69, 0x61, 0x74, 0x65, - 0x43, 0x61, 0x6c, 0x6c, 0x65, 0x72, 0x49, 0x64, 0x12, 0x25, 0x0a, 0x06, 0x74, 0x61, 0x72, 0x67, - 0x65, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0d, 0x2e, 0x71, 0x75, 0x65, 0x72, 0x79, - 0x2e, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x52, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x12, - 0x1a, 0x0a, 0x08, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x08, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x2a, 0x0a, 0x06, 0x66, - 0x69, 0x6c, 0x74, 0x65, 0x72, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x62, 0x69, - 0x6e, 0x6c, 0x6f, 0x67, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x52, - 0x06, 0x66, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x12, 0x3e, 0x0a, 0x0f, 0x74, 0x61, 0x62, 0x6c, 0x65, - 0x5f, 0x6c, 0x61, 0x73, 0x74, 0x5f, 0x70, 0x5f, 0x6b, 0x73, 0x18, 0x06, 0x20, 0x03, 0x28, 0x0b, - 0x32, 0x17, 0x2e, 0x62, 0x69, 0x6e, 0x6c, 0x6f, 0x67, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x54, 0x61, - 0x62, 0x6c, 0x65, 0x4c, 0x61, 0x73, 0x74, 0x50, 0x4b, 0x52, 0x0c, 0x74, 0x61, 0x62, 0x6c, 0x65, - 0x4c, 0x61, 0x73, 0x74, 0x50, 0x4b, 0x73, 0x12, 0x34, 0x0a, 0x07, 0x6f, 0x70, 0x74, 0x69, 0x6f, - 0x6e, 0x73, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x62, 0x69, 0x6e, 0x6c, 0x6f, + 0x6e, 0x74, 0x52, 0x06, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x22, 0xbb, 0x02, 0x0a, 0x12, 0x56, + 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x52, 0x6f, 0x77, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x12, 0x3f, 0x0a, 0x13, 0x65, 0x66, 0x66, 0x65, 0x63, 0x74, 0x69, 0x76, 0x65, 0x5f, 0x63, + 0x61, 0x6c, 0x6c, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0f, + 0x2e, 0x76, 0x74, 0x72, 0x70, 0x63, 0x2e, 0x43, 0x61, 0x6c, 0x6c, 0x65, 0x72, 0x49, 0x44, 0x52, + 0x11, 0x65, 0x66, 0x66, 0x65, 0x63, 0x74, 0x69, 0x76, 0x65, 0x43, 0x61, 0x6c, 0x6c, 0x65, 0x72, + 0x49, 0x64, 0x12, 0x45, 0x0a, 0x13, 0x69, 0x6d, 0x6d, 0x65, 0x64, 0x69, 0x61, 0x74, 0x65, 0x5f, + 0x63, 0x61, 0x6c, 0x6c, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x15, 0x2e, 0x71, 0x75, 0x65, 0x72, 0x79, 0x2e, 0x56, 0x54, 0x47, 0x61, 0x74, 0x65, 0x43, 0x61, + 0x6c, 0x6c, 0x65, 0x72, 0x49, 0x44, 0x52, 0x11, 0x69, 0x6d, 0x6d, 0x65, 0x64, 0x69, 0x61, 0x74, + 0x65, 0x43, 0x61, 0x6c, 0x6c, 0x65, 0x72, 0x49, 0x64, 0x12, 0x25, 0x0a, 0x06, 0x74, 0x61, 0x72, + 0x67, 0x65, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0d, 0x2e, 0x71, 0x75, 0x65, 0x72, + 0x79, 0x2e, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x52, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, + 0x12, 0x14, 0x0a, 0x05, 0x71, 0x75, 0x65, 0x72, 0x79, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x05, 0x71, 0x75, 0x65, 0x72, 0x79, 0x12, 0x2a, 0x0a, 0x06, 0x6c, 0x61, 0x73, 0x74, 0x70, 0x6b, + 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x71, 0x75, 0x65, 0x72, 0x79, 0x2e, 0x51, + 0x75, 0x65, 0x72, 0x79, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x52, 0x06, 0x6c, 0x61, 0x73, 0x74, + 0x70, 0x6b, 0x12, 0x34, 0x0a, 0x07, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x06, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x62, 0x69, 0x6e, 0x6c, 0x6f, 0x67, 0x64, 0x61, 0x74, 0x61, + 0x2e, 0x56, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, + 0x07, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x22, 0xa4, 0x02, 0x0a, 0x13, 0x56, 0x53, 0x74, + 0x72, 0x65, 0x61, 0x6d, 0x52, 0x6f, 0x77, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x12, 0x24, 0x0a, 0x06, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, + 0x32, 0x0c, 0x2e, 0x71, 0x75, 0x65, 0x72, 0x79, 0x2e, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x52, 0x06, + 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x12, 0x28, 0x0a, 0x08, 0x70, 0x6b, 0x66, 0x69, 0x65, 0x6c, + 0x64, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0c, 0x2e, 0x71, 0x75, 0x65, 0x72, 0x79, + 0x2e, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x52, 0x08, 0x70, 0x6b, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, + 0x12, 0x12, 0x0a, 0x04, 0x67, 0x74, 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, + 0x67, 0x74, 0x69, 0x64, 0x12, 0x1e, 0x0a, 0x04, 0x72, 0x6f, 0x77, 0x73, 0x18, 0x04, 0x20, 0x03, + 0x28, 0x0b, 0x32, 0x0a, 0x2e, 0x71, 0x75, 0x65, 0x72, 0x79, 0x2e, 0x52, 0x6f, 0x77, 0x52, 0x04, + 0x72, 0x6f, 0x77, 0x73, 0x12, 0x22, 0x0a, 0x06, 0x6c, 0x61, 0x73, 0x74, 0x70, 0x6b, 0x18, 0x05, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0a, 0x2e, 0x71, 0x75, 0x65, 0x72, 0x79, 0x2e, 0x52, 0x6f, 0x77, + 0x52, 0x06, 0x6c, 0x61, 0x73, 0x74, 0x70, 0x6b, 0x12, 0x1c, 0x0a, 0x09, 0x74, 0x68, 0x72, 0x6f, + 0x74, 0x74, 0x6c, 0x65, 0x64, 0x18, 0x06, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x74, 0x68, 0x72, + 0x6f, 0x74, 0x74, 0x6c, 0x65, 0x64, 0x12, 0x1c, 0x0a, 0x09, 0x68, 0x65, 0x61, 0x72, 0x74, 0x62, + 0x65, 0x61, 0x74, 0x18, 0x07, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x68, 0x65, 0x61, 0x72, 0x74, + 0x62, 0x65, 0x61, 0x74, 0x12, 0x29, 0x0a, 0x10, 0x74, 0x68, 0x72, 0x6f, 0x74, 0x74, 0x6c, 0x65, + 0x64, 0x5f, 0x72, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, + 0x74, 0x68, 0x72, 0x6f, 0x74, 0x74, 0x6c, 0x65, 0x64, 0x52, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x22, + 0xfb, 0x01, 0x0a, 0x14, 0x56, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x54, 0x61, 0x62, 0x6c, 0x65, + 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3f, 0x0a, 0x13, 0x65, 0x66, 0x66, 0x65, + 0x63, 0x74, 0x69, 0x76, 0x65, 0x5f, 0x63, 0x61, 0x6c, 0x6c, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x76, 0x74, 0x72, 0x70, 0x63, 0x2e, 0x43, 0x61, + 0x6c, 0x6c, 0x65, 0x72, 0x49, 0x44, 0x52, 0x11, 0x65, 0x66, 0x66, 0x65, 0x63, 0x74, 0x69, 0x76, + 0x65, 0x43, 0x61, 0x6c, 0x6c, 0x65, 0x72, 0x49, 0x64, 0x12, 0x45, 0x0a, 0x13, 0x69, 0x6d, 0x6d, + 0x65, 0x64, 0x69, 0x61, 0x74, 0x65, 0x5f, 0x63, 0x61, 0x6c, 0x6c, 0x65, 0x72, 0x5f, 0x69, 0x64, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x71, 0x75, 0x65, 0x72, 0x79, 0x2e, 0x56, + 0x54, 0x47, 0x61, 0x74, 0x65, 0x43, 0x61, 0x6c, 0x6c, 0x65, 0x72, 0x49, 0x44, 0x52, 0x11, 0x69, + 0x6d, 0x6d, 0x65, 0x64, 0x69, 0x61, 0x74, 0x65, 0x43, 0x61, 0x6c, 0x6c, 0x65, 0x72, 0x49, 0x64, + 0x12, 0x25, 0x0a, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x0d, 0x2e, 0x71, 0x75, 0x65, 0x72, 0x79, 0x2e, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x52, + 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x12, 0x34, 0x0a, 0x07, 0x6f, 0x70, 0x74, 0x69, 0x6f, + 0x6e, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x62, 0x69, 0x6e, 0x6c, 0x6f, 0x67, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x56, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x4f, 0x70, 0x74, - 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x07, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x22, 0x3d, 0x0a, - 0x0f, 0x56, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x12, 0x2a, 0x0a, 0x06, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, - 0x32, 0x12, 0x2e, 0x62, 0x69, 0x6e, 0x6c, 0x6f, 0x67, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x56, 0x45, - 0x76, 0x65, 0x6e, 0x74, 0x52, 0x06, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x22, 0xbb, 0x02, 0x0a, - 0x12, 0x56, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x52, 0x6f, 0x77, 0x73, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x12, 0x3f, 0x0a, 0x13, 0x65, 0x66, 0x66, 0x65, 0x63, 0x74, 0x69, 0x76, 0x65, - 0x5f, 0x63, 0x61, 0x6c, 0x6c, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x0f, 0x2e, 0x76, 0x74, 0x72, 0x70, 0x63, 0x2e, 0x43, 0x61, 0x6c, 0x6c, 0x65, 0x72, 0x49, - 0x44, 0x52, 0x11, 0x65, 0x66, 0x66, 0x65, 0x63, 0x74, 0x69, 0x76, 0x65, 0x43, 0x61, 0x6c, 0x6c, - 0x65, 0x72, 0x49, 0x64, 0x12, 0x45, 0x0a, 0x13, 0x69, 0x6d, 0x6d, 0x65, 0x64, 0x69, 0x61, 0x74, - 0x65, 0x5f, 0x63, 0x61, 0x6c, 0x6c, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x15, 0x2e, 0x71, 0x75, 0x65, 0x72, 0x79, 0x2e, 0x56, 0x54, 0x47, 0x61, 0x74, 0x65, - 0x43, 0x61, 0x6c, 0x6c, 0x65, 0x72, 0x49, 0x44, 0x52, 0x11, 0x69, 0x6d, 0x6d, 0x65, 0x64, 0x69, - 0x61, 0x74, 0x65, 0x43, 0x61, 0x6c, 0x6c, 0x65, 0x72, 0x49, 0x64, 0x12, 0x25, 0x0a, 0x06, 0x74, - 0x61, 0x72, 0x67, 0x65, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0d, 0x2e, 0x71, 0x75, - 0x65, 0x72, 0x79, 0x2e, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x52, 0x06, 0x74, 0x61, 0x72, 0x67, - 0x65, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x71, 0x75, 0x65, 0x72, 0x79, 0x18, 0x04, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x05, 0x71, 0x75, 0x65, 0x72, 0x79, 0x12, 0x2a, 0x0a, 0x06, 0x6c, 0x61, 0x73, 0x74, - 0x70, 0x6b, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x71, 0x75, 0x65, 0x72, 0x79, - 0x2e, 0x51, 0x75, 0x65, 0x72, 0x79, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x52, 0x06, 0x6c, 0x61, - 0x73, 0x74, 0x70, 0x6b, 0x12, 0x34, 0x0a, 0x07, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, - 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x62, 0x69, 0x6e, 0x6c, 0x6f, 0x67, 0x64, 0x61, - 0x74, 0x61, 0x2e, 0x56, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, - 0x73, 0x52, 0x07, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x22, 0xa4, 0x02, 0x0a, 0x13, 0x56, - 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x52, 0x6f, 0x77, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x12, 0x24, 0x0a, 0x06, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x18, 0x01, 0x20, 0x03, - 0x28, 0x0b, 0x32, 0x0c, 0x2e, 0x71, 0x75, 0x65, 0x72, 0x79, 0x2e, 0x46, 0x69, 0x65, 0x6c, 0x64, - 0x52, 0x06, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x12, 0x28, 0x0a, 0x08, 0x70, 0x6b, 0x66, 0x69, - 0x65, 0x6c, 0x64, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0c, 0x2e, 0x71, 0x75, 0x65, - 0x72, 0x79, 0x2e, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x52, 0x08, 0x70, 0x6b, 0x66, 0x69, 0x65, 0x6c, + 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x07, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x22, 0xde, 0x01, + 0x0a, 0x15, 0x56, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x73, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x74, 0x61, 0x62, 0x6c, 0x65, + 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x74, 0x61, 0x62, + 0x6c, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x24, 0x0a, 0x06, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, + 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0c, 0x2e, 0x71, 0x75, 0x65, 0x72, 0x79, 0x2e, 0x46, + 0x69, 0x65, 0x6c, 0x64, 0x52, 0x06, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x12, 0x28, 0x0a, 0x08, + 0x70, 0x6b, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0c, + 0x2e, 0x71, 0x75, 0x65, 0x72, 0x79, 0x2e, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x52, 0x08, 0x70, 0x6b, + 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x67, 0x74, 0x69, 0x64, 0x18, 0x04, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x67, 0x74, 0x69, 0x64, 0x12, 0x1e, 0x0a, 0x04, 0x72, 0x6f, + 0x77, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0a, 0x2e, 0x71, 0x75, 0x65, 0x72, 0x79, + 0x2e, 0x52, 0x6f, 0x77, 0x52, 0x04, 0x72, 0x6f, 0x77, 0x73, 0x12, 0x22, 0x0a, 0x06, 0x6c, 0x61, + 0x73, 0x74, 0x70, 0x6b, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0a, 0x2e, 0x71, 0x75, 0x65, + 0x72, 0x79, 0x2e, 0x52, 0x6f, 0x77, 0x52, 0x06, 0x6c, 0x61, 0x73, 0x74, 0x70, 0x6b, 0x22, 0x69, + 0x0a, 0x0b, 0x4c, 0x61, 0x73, 0x74, 0x50, 0x4b, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x12, 0x3c, 0x0a, + 0x0e, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x6c, 0x61, 0x73, 0x74, 0x5f, 0x70, 0x5f, 0x6b, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x62, 0x69, 0x6e, 0x6c, 0x6f, 0x67, 0x64, 0x61, + 0x74, 0x61, 0x2e, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x4c, 0x61, 0x73, 0x74, 0x50, 0x4b, 0x52, 0x0b, + 0x74, 0x61, 0x62, 0x6c, 0x65, 0x4c, 0x61, 0x73, 0x74, 0x50, 0x4b, 0x12, 0x1c, 0x0a, 0x09, 0x63, + 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, + 0x63, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x22, 0x58, 0x0a, 0x0b, 0x54, 0x61, 0x62, + 0x6c, 0x65, 0x4c, 0x61, 0x73, 0x74, 0x50, 0x4b, 0x12, 0x1d, 0x0a, 0x0a, 0x74, 0x61, 0x62, 0x6c, + 0x65, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x74, 0x61, + 0x62, 0x6c, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x2a, 0x0a, 0x06, 0x6c, 0x61, 0x73, 0x74, 0x70, + 0x6b, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x71, 0x75, 0x65, 0x72, 0x79, 0x2e, + 0x51, 0x75, 0x65, 0x72, 0x79, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x52, 0x06, 0x6c, 0x61, 0x73, + 0x74, 0x70, 0x6b, 0x22, 0xdc, 0x01, 0x0a, 0x15, 0x56, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x52, + 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3f, 0x0a, + 0x13, 0x65, 0x66, 0x66, 0x65, 0x63, 0x74, 0x69, 0x76, 0x65, 0x5f, 0x63, 0x61, 0x6c, 0x6c, 0x65, + 0x72, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x76, 0x74, 0x72, + 0x70, 0x63, 0x2e, 0x43, 0x61, 0x6c, 0x6c, 0x65, 0x72, 0x49, 0x44, 0x52, 0x11, 0x65, 0x66, 0x66, + 0x65, 0x63, 0x74, 0x69, 0x76, 0x65, 0x43, 0x61, 0x6c, 0x6c, 0x65, 0x72, 0x49, 0x64, 0x12, 0x45, + 0x0a, 0x13, 0x69, 0x6d, 0x6d, 0x65, 0x64, 0x69, 0x61, 0x74, 0x65, 0x5f, 0x63, 0x61, 0x6c, 0x6c, + 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x71, 0x75, + 0x65, 0x72, 0x79, 0x2e, 0x56, 0x54, 0x47, 0x61, 0x74, 0x65, 0x43, 0x61, 0x6c, 0x6c, 0x65, 0x72, + 0x49, 0x44, 0x52, 0x11, 0x69, 0x6d, 0x6d, 0x65, 0x64, 0x69, 0x61, 0x74, 0x65, 0x43, 0x61, 0x6c, + 0x6c, 0x65, 0x72, 0x49, 0x64, 0x12, 0x25, 0x0a, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x18, + 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0d, 0x2e, 0x71, 0x75, 0x65, 0x72, 0x79, 0x2e, 0x54, 0x61, + 0x72, 0x67, 0x65, 0x74, 0x52, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x12, 0x14, 0x0a, 0x05, + 0x71, 0x75, 0x65, 0x72, 0x79, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x71, 0x75, 0x65, + 0x72, 0x79, 0x22, 0x72, 0x0a, 0x16, 0x56, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x52, 0x65, 0x73, + 0x75, 0x6c, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x24, 0x0a, 0x06, + 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0c, 0x2e, 0x71, + 0x75, 0x65, 0x72, 0x79, 0x2e, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x52, 0x06, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x67, 0x74, 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x67, 0x74, 0x69, 0x64, 0x12, 0x1e, 0x0a, 0x04, 0x72, 0x6f, 0x77, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0a, 0x2e, 0x71, 0x75, 0x65, 0x72, 0x79, 0x2e, 0x52, 0x6f, 0x77, - 0x52, 0x04, 0x72, 0x6f, 0x77, 0x73, 0x12, 0x22, 0x0a, 0x06, 0x6c, 0x61, 0x73, 0x74, 0x70, 0x6b, - 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0a, 0x2e, 0x71, 0x75, 0x65, 0x72, 0x79, 0x2e, 0x52, - 0x6f, 0x77, 0x52, 0x06, 0x6c, 0x61, 0x73, 0x74, 0x70, 0x6b, 0x12, 0x1c, 0x0a, 0x09, 0x74, 0x68, - 0x72, 0x6f, 0x74, 0x74, 0x6c, 0x65, 0x64, 0x18, 0x06, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x74, - 0x68, 0x72, 0x6f, 0x74, 0x74, 0x6c, 0x65, 0x64, 0x12, 0x1c, 0x0a, 0x09, 0x68, 0x65, 0x61, 0x72, - 0x74, 0x62, 0x65, 0x61, 0x74, 0x18, 0x07, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x68, 0x65, 0x61, - 0x72, 0x74, 0x62, 0x65, 0x61, 0x74, 0x12, 0x29, 0x0a, 0x10, 0x74, 0x68, 0x72, 0x6f, 0x74, 0x74, - 0x6c, 0x65, 0x64, 0x5f, 0x72, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x0f, 0x74, 0x68, 0x72, 0x6f, 0x74, 0x74, 0x6c, 0x65, 0x64, 0x52, 0x65, 0x61, 0x73, 0x6f, - 0x6e, 0x22, 0xfb, 0x01, 0x0a, 0x14, 0x56, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x54, 0x61, 0x62, - 0x6c, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3f, 0x0a, 0x13, 0x65, 0x66, - 0x66, 0x65, 0x63, 0x74, 0x69, 0x76, 0x65, 0x5f, 0x63, 0x61, 0x6c, 0x6c, 0x65, 0x72, 0x5f, 0x69, - 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x76, 0x74, 0x72, 0x70, 0x63, 0x2e, - 0x43, 0x61, 0x6c, 0x6c, 0x65, 0x72, 0x49, 0x44, 0x52, 0x11, 0x65, 0x66, 0x66, 0x65, 0x63, 0x74, - 0x69, 0x76, 0x65, 0x43, 0x61, 0x6c, 0x6c, 0x65, 0x72, 0x49, 0x64, 0x12, 0x45, 0x0a, 0x13, 0x69, - 0x6d, 0x6d, 0x65, 0x64, 0x69, 0x61, 0x74, 0x65, 0x5f, 0x63, 0x61, 0x6c, 0x6c, 0x65, 0x72, 0x5f, - 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x71, 0x75, 0x65, 0x72, 0x79, - 0x2e, 0x56, 0x54, 0x47, 0x61, 0x74, 0x65, 0x43, 0x61, 0x6c, 0x6c, 0x65, 0x72, 0x49, 0x44, 0x52, - 0x11, 0x69, 0x6d, 0x6d, 0x65, 0x64, 0x69, 0x61, 0x74, 0x65, 0x43, 0x61, 0x6c, 0x6c, 0x65, 0x72, - 0x49, 0x64, 0x12, 0x25, 0x0a, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x18, 0x03, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x0d, 0x2e, 0x71, 0x75, 0x65, 0x72, 0x79, 0x2e, 0x54, 0x61, 0x72, 0x67, 0x65, - 0x74, 0x52, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x12, 0x34, 0x0a, 0x07, 0x6f, 0x70, 0x74, - 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x62, 0x69, 0x6e, - 0x6c, 0x6f, 0x67, 0x64, 0x61, 0x74, 0x61, 0x2e, 0x56, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x4f, - 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x07, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x22, - 0xde, 0x01, 0x0a, 0x15, 0x56, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x54, 0x61, 0x62, 0x6c, 0x65, - 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x74, 0x61, 0x62, - 0x6c, 0x65, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x74, - 0x61, 0x62, 0x6c, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x24, 0x0a, 0x06, 0x66, 0x69, 0x65, 0x6c, - 0x64, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0c, 0x2e, 0x71, 0x75, 0x65, 0x72, 0x79, - 0x2e, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x52, 0x06, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x12, 0x28, - 0x0a, 0x08, 0x70, 0x6b, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, - 0x32, 0x0c, 0x2e, 0x71, 0x75, 0x65, 0x72, 0x79, 0x2e, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x52, 0x08, - 0x70, 0x6b, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x67, 0x74, 0x69, 0x64, - 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x67, 0x74, 0x69, 0x64, 0x12, 0x1e, 0x0a, 0x04, - 0x72, 0x6f, 0x77, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0a, 0x2e, 0x71, 0x75, 0x65, - 0x72, 0x79, 0x2e, 0x52, 0x6f, 0x77, 0x52, 0x04, 0x72, 0x6f, 0x77, 0x73, 0x12, 0x22, 0x0a, 0x06, - 0x6c, 0x61, 0x73, 0x74, 0x70, 0x6b, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0a, 0x2e, 0x71, - 0x75, 0x65, 0x72, 0x79, 0x2e, 0x52, 0x6f, 0x77, 0x52, 0x06, 0x6c, 0x61, 0x73, 0x74, 0x70, 0x6b, - 0x22, 0x69, 0x0a, 0x0b, 0x4c, 0x61, 0x73, 0x74, 0x50, 0x4b, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x12, - 0x3c, 0x0a, 0x0e, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x6c, 0x61, 0x73, 0x74, 0x5f, 0x70, 0x5f, - 0x6b, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x62, 0x69, 0x6e, 0x6c, 0x6f, 0x67, - 0x64, 0x61, 0x74, 0x61, 0x2e, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x4c, 0x61, 0x73, 0x74, 0x50, 0x4b, - 0x52, 0x0b, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x4c, 0x61, 0x73, 0x74, 0x50, 0x4b, 0x12, 0x1c, 0x0a, - 0x09, 0x63, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, - 0x52, 0x09, 0x63, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x22, 0x58, 0x0a, 0x0b, 0x54, - 0x61, 0x62, 0x6c, 0x65, 0x4c, 0x61, 0x73, 0x74, 0x50, 0x4b, 0x12, 0x1d, 0x0a, 0x0a, 0x74, 0x61, - 0x62, 0x6c, 0x65, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, - 0x74, 0x61, 0x62, 0x6c, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x2a, 0x0a, 0x06, 0x6c, 0x61, 0x73, - 0x74, 0x70, 0x6b, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x71, 0x75, 0x65, 0x72, - 0x79, 0x2e, 0x51, 0x75, 0x65, 0x72, 0x79, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x52, 0x06, 0x6c, - 0x61, 0x73, 0x74, 0x70, 0x6b, 0x22, 0xdc, 0x01, 0x0a, 0x15, 0x56, 0x53, 0x74, 0x72, 0x65, 0x61, - 0x6d, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, - 0x3f, 0x0a, 0x13, 0x65, 0x66, 0x66, 0x65, 0x63, 0x74, 0x69, 0x76, 0x65, 0x5f, 0x63, 0x61, 0x6c, - 0x6c, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x76, - 0x74, 0x72, 0x70, 0x63, 0x2e, 0x43, 0x61, 0x6c, 0x6c, 0x65, 0x72, 0x49, 0x44, 0x52, 0x11, 0x65, - 0x66, 0x66, 0x65, 0x63, 0x74, 0x69, 0x76, 0x65, 0x43, 0x61, 0x6c, 0x6c, 0x65, 0x72, 0x49, 0x64, - 0x12, 0x45, 0x0a, 0x13, 0x69, 0x6d, 0x6d, 0x65, 0x64, 0x69, 0x61, 0x74, 0x65, 0x5f, 0x63, 0x61, - 0x6c, 0x6c, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x15, 0x2e, - 0x71, 0x75, 0x65, 0x72, 0x79, 0x2e, 0x56, 0x54, 0x47, 0x61, 0x74, 0x65, 0x43, 0x61, 0x6c, 0x6c, - 0x65, 0x72, 0x49, 0x44, 0x52, 0x11, 0x69, 0x6d, 0x6d, 0x65, 0x64, 0x69, 0x61, 0x74, 0x65, 0x43, - 0x61, 0x6c, 0x6c, 0x65, 0x72, 0x49, 0x64, 0x12, 0x25, 0x0a, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, - 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0d, 0x2e, 0x71, 0x75, 0x65, 0x72, 0x79, 0x2e, - 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x52, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x12, 0x14, - 0x0a, 0x05, 0x71, 0x75, 0x65, 0x72, 0x79, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x71, - 0x75, 0x65, 0x72, 0x79, 0x22, 0x72, 0x0a, 0x16, 0x56, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x52, - 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x24, - 0x0a, 0x06, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0c, - 0x2e, 0x71, 0x75, 0x65, 0x72, 0x79, 0x2e, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x52, 0x06, 0x66, 0x69, - 0x65, 0x6c, 0x64, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x67, 0x74, 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x04, 0x67, 0x74, 0x69, 0x64, 0x12, 0x1e, 0x0a, 0x04, 0x72, 0x6f, 0x77, 0x73, - 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0a, 0x2e, 0x71, 0x75, 0x65, 0x72, 0x79, 0x2e, 0x52, - 0x6f, 0x77, 0x52, 0x04, 0x72, 0x6f, 0x77, 0x73, 0x2a, 0x3e, 0x0a, 0x0b, 0x4f, 0x6e, 0x44, 0x44, - 0x4c, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x0a, 0x0a, 0x06, 0x49, 0x47, 0x4e, 0x4f, 0x52, - 0x45, 0x10, 0x00, 0x12, 0x08, 0x0a, 0x04, 0x53, 0x54, 0x4f, 0x50, 0x10, 0x01, 0x12, 0x08, 0x0a, - 0x04, 0x45, 0x58, 0x45, 0x43, 0x10, 0x02, 0x12, 0x0f, 0x0a, 0x0b, 0x45, 0x58, 0x45, 0x43, 0x5f, - 0x49, 0x47, 0x4e, 0x4f, 0x52, 0x45, 0x10, 0x03, 0x2a, 0x7b, 0x0a, 0x18, 0x56, 0x52, 0x65, 0x70, - 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, - 0x54, 0x79, 0x70, 0x65, 0x12, 0x0f, 0x0a, 0x0b, 0x4d, 0x61, 0x74, 0x65, 0x72, 0x69, 0x61, 0x6c, - 0x69, 0x7a, 0x65, 0x10, 0x00, 0x12, 0x0e, 0x0a, 0x0a, 0x4d, 0x6f, 0x76, 0x65, 0x54, 0x61, 0x62, - 0x6c, 0x65, 0x73, 0x10, 0x01, 0x12, 0x15, 0x0a, 0x11, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x4c, - 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x10, 0x02, 0x12, 0x0b, 0x0a, 0x07, - 0x4d, 0x69, 0x67, 0x72, 0x61, 0x74, 0x65, 0x10, 0x03, 0x12, 0x0b, 0x0a, 0x07, 0x52, 0x65, 0x73, - 0x68, 0x61, 0x72, 0x64, 0x10, 0x04, 0x12, 0x0d, 0x0a, 0x09, 0x4f, 0x6e, 0x6c, 0x69, 0x6e, 0x65, - 0x44, 0x44, 0x4c, 0x10, 0x05, 0x2a, 0x44, 0x0a, 0x1b, 0x56, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, - 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x53, 0x75, 0x62, - 0x54, 0x79, 0x70, 0x65, 0x12, 0x08, 0x0a, 0x04, 0x4e, 0x6f, 0x6e, 0x65, 0x10, 0x00, 0x12, 0x0b, - 0x0a, 0x07, 0x50, 0x61, 0x72, 0x74, 0x69, 0x61, 0x6c, 0x10, 0x01, 0x12, 0x0e, 0x0a, 0x0a, 0x41, - 0x74, 0x6f, 0x6d, 0x69, 0x63, 0x43, 0x6f, 0x70, 0x79, 0x10, 0x02, 0x2a, 0x71, 0x0a, 0x19, 0x56, - 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x57, 0x6f, 0x72, 0x6b, 0x66, - 0x6c, 0x6f, 0x77, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x55, 0x6e, 0x6b, 0x6e, - 0x6f, 0x77, 0x6e, 0x10, 0x00, 0x12, 0x08, 0x0a, 0x04, 0x49, 0x6e, 0x69, 0x74, 0x10, 0x01, 0x12, - 0x0b, 0x0a, 0x07, 0x53, 0x74, 0x6f, 0x70, 0x70, 0x65, 0x64, 0x10, 0x02, 0x12, 0x0b, 0x0a, 0x07, - 0x43, 0x6f, 0x70, 0x79, 0x69, 0x6e, 0x67, 0x10, 0x03, 0x12, 0x0b, 0x0a, 0x07, 0x52, 0x75, 0x6e, - 0x6e, 0x69, 0x6e, 0x67, 0x10, 0x04, 0x12, 0x09, 0x0a, 0x05, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x10, - 0x05, 0x12, 0x0b, 0x0a, 0x07, 0x4c, 0x61, 0x67, 0x67, 0x69, 0x6e, 0x67, 0x10, 0x06, 0x2a, 0x8d, - 0x02, 0x0a, 0x0a, 0x56, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a, - 0x07, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x08, 0x0a, 0x04, 0x47, 0x54, - 0x49, 0x44, 0x10, 0x01, 0x12, 0x09, 0x0a, 0x05, 0x42, 0x45, 0x47, 0x49, 0x4e, 0x10, 0x02, 0x12, - 0x0a, 0x0a, 0x06, 0x43, 0x4f, 0x4d, 0x4d, 0x49, 0x54, 0x10, 0x03, 0x12, 0x0c, 0x0a, 0x08, 0x52, - 0x4f, 0x4c, 0x4c, 0x42, 0x41, 0x43, 0x4b, 0x10, 0x04, 0x12, 0x07, 0x0a, 0x03, 0x44, 0x44, 0x4c, - 0x10, 0x05, 0x12, 0x0a, 0x0a, 0x06, 0x49, 0x4e, 0x53, 0x45, 0x52, 0x54, 0x10, 0x06, 0x12, 0x0b, - 0x0a, 0x07, 0x52, 0x45, 0x50, 0x4c, 0x41, 0x43, 0x45, 0x10, 0x07, 0x12, 0x0a, 0x0a, 0x06, 0x55, - 0x50, 0x44, 0x41, 0x54, 0x45, 0x10, 0x08, 0x12, 0x0a, 0x0a, 0x06, 0x44, 0x45, 0x4c, 0x45, 0x54, - 0x45, 0x10, 0x09, 0x12, 0x07, 0x0a, 0x03, 0x53, 0x45, 0x54, 0x10, 0x0a, 0x12, 0x09, 0x0a, 0x05, - 0x4f, 0x54, 0x48, 0x45, 0x52, 0x10, 0x0b, 0x12, 0x07, 0x0a, 0x03, 0x52, 0x4f, 0x57, 0x10, 0x0c, - 0x12, 0x09, 0x0a, 0x05, 0x46, 0x49, 0x45, 0x4c, 0x44, 0x10, 0x0d, 0x12, 0x0d, 0x0a, 0x09, 0x48, - 0x45, 0x41, 0x52, 0x54, 0x42, 0x45, 0x41, 0x54, 0x10, 0x0e, 0x12, 0x09, 0x0a, 0x05, 0x56, 0x47, - 0x54, 0x49, 0x44, 0x10, 0x0f, 0x12, 0x0b, 0x0a, 0x07, 0x4a, 0x4f, 0x55, 0x52, 0x4e, 0x41, 0x4c, - 0x10, 0x10, 0x12, 0x0b, 0x0a, 0x07, 0x56, 0x45, 0x52, 0x53, 0x49, 0x4f, 0x4e, 0x10, 0x11, 0x12, - 0x0a, 0x0a, 0x06, 0x4c, 0x41, 0x53, 0x54, 0x50, 0x4b, 0x10, 0x12, 0x12, 0x0d, 0x0a, 0x09, 0x53, - 0x41, 0x56, 0x45, 0x50, 0x4f, 0x49, 0x4e, 0x54, 0x10, 0x13, 0x12, 0x12, 0x0a, 0x0e, 0x43, 0x4f, - 0x50, 0x59, 0x5f, 0x43, 0x4f, 0x4d, 0x50, 0x4c, 0x45, 0x54, 0x45, 0x44, 0x10, 0x14, 0x2a, 0x27, - 0x0a, 0x0d, 0x4d, 0x69, 0x67, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x12, - 0x0a, 0x0a, 0x06, 0x54, 0x41, 0x42, 0x4c, 0x45, 0x53, 0x10, 0x00, 0x12, 0x0a, 0x0a, 0x06, 0x53, - 0x48, 0x41, 0x52, 0x44, 0x53, 0x10, 0x01, 0x42, 0x29, 0x5a, 0x27, 0x76, 0x69, 0x74, 0x65, 0x73, - 0x73, 0x2e, 0x69, 0x6f, 0x2f, 0x76, 0x69, 0x74, 0x65, 0x73, 0x73, 0x2f, 0x67, 0x6f, 0x2f, 0x76, - 0x74, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x62, 0x69, 0x6e, 0x6c, 0x6f, 0x67, 0x64, 0x61, - 0x74, 0x61, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x52, 0x04, 0x72, 0x6f, 0x77, 0x73, 0x2a, 0x3e, 0x0a, 0x0b, 0x4f, 0x6e, 0x44, 0x44, 0x4c, 0x41, + 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x0a, 0x0a, 0x06, 0x49, 0x47, 0x4e, 0x4f, 0x52, 0x45, 0x10, + 0x00, 0x12, 0x08, 0x0a, 0x04, 0x53, 0x54, 0x4f, 0x50, 0x10, 0x01, 0x12, 0x08, 0x0a, 0x04, 0x45, + 0x58, 0x45, 0x43, 0x10, 0x02, 0x12, 0x0f, 0x0a, 0x0b, 0x45, 0x58, 0x45, 0x43, 0x5f, 0x49, 0x47, + 0x4e, 0x4f, 0x52, 0x45, 0x10, 0x03, 0x2a, 0x7b, 0x0a, 0x18, 0x56, 0x52, 0x65, 0x70, 0x6c, 0x69, + 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x54, 0x79, + 0x70, 0x65, 0x12, 0x0f, 0x0a, 0x0b, 0x4d, 0x61, 0x74, 0x65, 0x72, 0x69, 0x61, 0x6c, 0x69, 0x7a, + 0x65, 0x10, 0x00, 0x12, 0x0e, 0x0a, 0x0a, 0x4d, 0x6f, 0x76, 0x65, 0x54, 0x61, 0x62, 0x6c, 0x65, + 0x73, 0x10, 0x01, 0x12, 0x15, 0x0a, 0x11, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x4c, 0x6f, 0x6f, + 0x6b, 0x75, 0x70, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x10, 0x02, 0x12, 0x0b, 0x0a, 0x07, 0x4d, 0x69, + 0x67, 0x72, 0x61, 0x74, 0x65, 0x10, 0x03, 0x12, 0x0b, 0x0a, 0x07, 0x52, 0x65, 0x73, 0x68, 0x61, + 0x72, 0x64, 0x10, 0x04, 0x12, 0x0d, 0x0a, 0x09, 0x4f, 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x44, 0x44, + 0x4c, 0x10, 0x05, 0x2a, 0x44, 0x0a, 0x1b, 0x56, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, + 0x69, 0x6f, 0x6e, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, 0x77, 0x53, 0x75, 0x62, 0x54, 0x79, + 0x70, 0x65, 0x12, 0x08, 0x0a, 0x04, 0x4e, 0x6f, 0x6e, 0x65, 0x10, 0x00, 0x12, 0x0b, 0x0a, 0x07, + 0x50, 0x61, 0x72, 0x74, 0x69, 0x61, 0x6c, 0x10, 0x01, 0x12, 0x0e, 0x0a, 0x0a, 0x41, 0x74, 0x6f, + 0x6d, 0x69, 0x63, 0x43, 0x6f, 0x70, 0x79, 0x10, 0x02, 0x2a, 0x71, 0x0a, 0x19, 0x56, 0x52, 0x65, + 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x57, 0x6f, 0x72, 0x6b, 0x66, 0x6c, 0x6f, + 0x77, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x55, 0x6e, 0x6b, 0x6e, 0x6f, 0x77, + 0x6e, 0x10, 0x00, 0x12, 0x08, 0x0a, 0x04, 0x49, 0x6e, 0x69, 0x74, 0x10, 0x01, 0x12, 0x0b, 0x0a, + 0x07, 0x53, 0x74, 0x6f, 0x70, 0x70, 0x65, 0x64, 0x10, 0x02, 0x12, 0x0b, 0x0a, 0x07, 0x43, 0x6f, + 0x70, 0x79, 0x69, 0x6e, 0x67, 0x10, 0x03, 0x12, 0x0b, 0x0a, 0x07, 0x52, 0x75, 0x6e, 0x6e, 0x69, + 0x6e, 0x67, 0x10, 0x04, 0x12, 0x09, 0x0a, 0x05, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x10, 0x05, 0x12, + 0x0b, 0x0a, 0x07, 0x4c, 0x61, 0x67, 0x67, 0x69, 0x6e, 0x67, 0x10, 0x06, 0x2a, 0x8d, 0x02, 0x0a, + 0x0a, 0x56, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x55, + 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x08, 0x0a, 0x04, 0x47, 0x54, 0x49, 0x44, + 0x10, 0x01, 0x12, 0x09, 0x0a, 0x05, 0x42, 0x45, 0x47, 0x49, 0x4e, 0x10, 0x02, 0x12, 0x0a, 0x0a, + 0x06, 0x43, 0x4f, 0x4d, 0x4d, 0x49, 0x54, 0x10, 0x03, 0x12, 0x0c, 0x0a, 0x08, 0x52, 0x4f, 0x4c, + 0x4c, 0x42, 0x41, 0x43, 0x4b, 0x10, 0x04, 0x12, 0x07, 0x0a, 0x03, 0x44, 0x44, 0x4c, 0x10, 0x05, + 0x12, 0x0a, 0x0a, 0x06, 0x49, 0x4e, 0x53, 0x45, 0x52, 0x54, 0x10, 0x06, 0x12, 0x0b, 0x0a, 0x07, + 0x52, 0x45, 0x50, 0x4c, 0x41, 0x43, 0x45, 0x10, 0x07, 0x12, 0x0a, 0x0a, 0x06, 0x55, 0x50, 0x44, + 0x41, 0x54, 0x45, 0x10, 0x08, 0x12, 0x0a, 0x0a, 0x06, 0x44, 0x45, 0x4c, 0x45, 0x54, 0x45, 0x10, + 0x09, 0x12, 0x07, 0x0a, 0x03, 0x53, 0x45, 0x54, 0x10, 0x0a, 0x12, 0x09, 0x0a, 0x05, 0x4f, 0x54, + 0x48, 0x45, 0x52, 0x10, 0x0b, 0x12, 0x07, 0x0a, 0x03, 0x52, 0x4f, 0x57, 0x10, 0x0c, 0x12, 0x09, + 0x0a, 0x05, 0x46, 0x49, 0x45, 0x4c, 0x44, 0x10, 0x0d, 0x12, 0x0d, 0x0a, 0x09, 0x48, 0x45, 0x41, + 0x52, 0x54, 0x42, 0x45, 0x41, 0x54, 0x10, 0x0e, 0x12, 0x09, 0x0a, 0x05, 0x56, 0x47, 0x54, 0x49, + 0x44, 0x10, 0x0f, 0x12, 0x0b, 0x0a, 0x07, 0x4a, 0x4f, 0x55, 0x52, 0x4e, 0x41, 0x4c, 0x10, 0x10, + 0x12, 0x0b, 0x0a, 0x07, 0x56, 0x45, 0x52, 0x53, 0x49, 0x4f, 0x4e, 0x10, 0x11, 0x12, 0x0a, 0x0a, + 0x06, 0x4c, 0x41, 0x53, 0x54, 0x50, 0x4b, 0x10, 0x12, 0x12, 0x0d, 0x0a, 0x09, 0x53, 0x41, 0x56, + 0x45, 0x50, 0x4f, 0x49, 0x4e, 0x54, 0x10, 0x13, 0x12, 0x12, 0x0a, 0x0e, 0x43, 0x4f, 0x50, 0x59, + 0x5f, 0x43, 0x4f, 0x4d, 0x50, 0x4c, 0x45, 0x54, 0x45, 0x44, 0x10, 0x14, 0x2a, 0x27, 0x0a, 0x0d, + 0x4d, 0x69, 0x67, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0a, 0x0a, + 0x06, 0x54, 0x41, 0x42, 0x4c, 0x45, 0x53, 0x10, 0x00, 0x12, 0x0a, 0x0a, 0x06, 0x53, 0x48, 0x41, + 0x52, 0x44, 0x53, 0x10, 0x01, 0x42, 0x29, 0x5a, 0x27, 0x76, 0x69, 0x74, 0x65, 0x73, 0x73, 0x2e, + 0x69, 0x6f, 0x2f, 0x76, 0x69, 0x74, 0x65, 0x73, 0x73, 0x2f, 0x67, 0x6f, 0x2f, 0x76, 0x74, 0x2f, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x62, 0x69, 0x6e, 0x6c, 0x6f, 0x67, 0x64, 0x61, 0x74, 0x61, + 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -3631,61 +3652,62 @@ var file_binlogdata_proto_depIdxs = []int32{ 48, // 16: binlogdata.RowChange.before:type_name -> query.Row 48, // 17: binlogdata.RowChange.after:type_name -> query.Row 43, // 18: binlogdata.RowChange.data_columns:type_name -> binlogdata.RowChange.Bitmap - 18, // 19: binlogdata.RowEvent.row_changes:type_name -> binlogdata.RowChange - 49, // 20: binlogdata.FieldEvent.fields:type_name -> query.Field - 36, // 21: binlogdata.ShardGtid.table_p_ks:type_name -> binlogdata.TableLastPK - 21, // 22: binlogdata.VGtid.shard_gtids:type_name -> binlogdata.ShardGtid - 5, // 23: binlogdata.Journal.migration_type:type_name -> binlogdata.MigrationType - 21, // 24: binlogdata.Journal.shard_gtids:type_name -> binlogdata.ShardGtid - 23, // 25: binlogdata.Journal.participants:type_name -> binlogdata.KeyspaceShard - 4, // 26: binlogdata.VEvent.type:type_name -> binlogdata.VEventType - 19, // 27: binlogdata.VEvent.row_event:type_name -> binlogdata.RowEvent - 20, // 28: binlogdata.VEvent.field_event:type_name -> binlogdata.FieldEvent - 22, // 29: binlogdata.VEvent.vgtid:type_name -> binlogdata.VGtid - 24, // 30: binlogdata.VEvent.journal:type_name -> binlogdata.Journal - 35, // 31: binlogdata.VEvent.last_p_k_event:type_name -> binlogdata.LastPKEvent - 49, // 32: binlogdata.MinimalTable.fields:type_name -> query.Field - 26, // 33: binlogdata.MinimalSchema.tables:type_name -> binlogdata.MinimalTable - 44, // 34: binlogdata.VStreamOptions.config_overrides:type_name -> binlogdata.VStreamOptions.ConfigOverridesEntry - 50, // 35: binlogdata.VStreamRequest.effective_caller_id:type_name -> vtrpc.CallerID - 51, // 36: binlogdata.VStreamRequest.immediate_caller_id:type_name -> query.VTGateCallerID - 52, // 37: binlogdata.VStreamRequest.target:type_name -> query.Target - 16, // 38: binlogdata.VStreamRequest.filter:type_name -> binlogdata.Filter - 36, // 39: binlogdata.VStreamRequest.table_last_p_ks:type_name -> binlogdata.TableLastPK - 28, // 40: binlogdata.VStreamRequest.options:type_name -> binlogdata.VStreamOptions - 25, // 41: binlogdata.VStreamResponse.events:type_name -> binlogdata.VEvent - 50, // 42: binlogdata.VStreamRowsRequest.effective_caller_id:type_name -> vtrpc.CallerID - 51, // 43: binlogdata.VStreamRowsRequest.immediate_caller_id:type_name -> query.VTGateCallerID - 52, // 44: binlogdata.VStreamRowsRequest.target:type_name -> query.Target - 53, // 45: binlogdata.VStreamRowsRequest.lastpk:type_name -> query.QueryResult - 28, // 46: binlogdata.VStreamRowsRequest.options:type_name -> binlogdata.VStreamOptions - 49, // 47: binlogdata.VStreamRowsResponse.fields:type_name -> query.Field - 49, // 48: binlogdata.VStreamRowsResponse.pkfields:type_name -> query.Field - 48, // 49: binlogdata.VStreamRowsResponse.rows:type_name -> query.Row - 48, // 50: binlogdata.VStreamRowsResponse.lastpk:type_name -> query.Row - 50, // 51: binlogdata.VStreamTablesRequest.effective_caller_id:type_name -> vtrpc.CallerID - 51, // 52: binlogdata.VStreamTablesRequest.immediate_caller_id:type_name -> query.VTGateCallerID - 52, // 53: binlogdata.VStreamTablesRequest.target:type_name -> query.Target - 28, // 54: binlogdata.VStreamTablesRequest.options:type_name -> binlogdata.VStreamOptions - 49, // 55: binlogdata.VStreamTablesResponse.fields:type_name -> query.Field - 49, // 56: binlogdata.VStreamTablesResponse.pkfields:type_name -> query.Field - 48, // 57: binlogdata.VStreamTablesResponse.rows:type_name -> query.Row - 48, // 58: binlogdata.VStreamTablesResponse.lastpk:type_name -> query.Row - 36, // 59: binlogdata.LastPKEvent.table_last_p_k:type_name -> binlogdata.TableLastPK - 53, // 60: binlogdata.TableLastPK.lastpk:type_name -> query.QueryResult - 50, // 61: binlogdata.VStreamResultsRequest.effective_caller_id:type_name -> vtrpc.CallerID - 51, // 62: binlogdata.VStreamResultsRequest.immediate_caller_id:type_name -> query.VTGateCallerID - 52, // 63: binlogdata.VStreamResultsRequest.target:type_name -> query.Target - 49, // 64: binlogdata.VStreamResultsResponse.fields:type_name -> query.Field - 48, // 65: binlogdata.VStreamResultsResponse.rows:type_name -> query.Row - 6, // 66: binlogdata.BinlogTransaction.Statement.category:type_name -> binlogdata.BinlogTransaction.Statement.Category - 8, // 67: binlogdata.BinlogTransaction.Statement.charset:type_name -> binlogdata.Charset - 14, // 68: binlogdata.Rule.ConvertCharsetEntry.value:type_name -> binlogdata.CharsetConversion - 69, // [69:69] is the sub-list for method output_type - 69, // [69:69] is the sub-list for method input_type - 69, // [69:69] is the sub-list for extension type_name - 69, // [69:69] is the sub-list for extension extendee - 0, // [0:69] is the sub-list for field type_name + 43, // 19: binlogdata.RowChange.json_partial_values:type_name -> binlogdata.RowChange.Bitmap + 18, // 20: binlogdata.RowEvent.row_changes:type_name -> binlogdata.RowChange + 49, // 21: binlogdata.FieldEvent.fields:type_name -> query.Field + 36, // 22: binlogdata.ShardGtid.table_p_ks:type_name -> binlogdata.TableLastPK + 21, // 23: binlogdata.VGtid.shard_gtids:type_name -> binlogdata.ShardGtid + 5, // 24: binlogdata.Journal.migration_type:type_name -> binlogdata.MigrationType + 21, // 25: binlogdata.Journal.shard_gtids:type_name -> binlogdata.ShardGtid + 23, // 26: binlogdata.Journal.participants:type_name -> binlogdata.KeyspaceShard + 4, // 27: binlogdata.VEvent.type:type_name -> binlogdata.VEventType + 19, // 28: binlogdata.VEvent.row_event:type_name -> binlogdata.RowEvent + 20, // 29: binlogdata.VEvent.field_event:type_name -> binlogdata.FieldEvent + 22, // 30: binlogdata.VEvent.vgtid:type_name -> binlogdata.VGtid + 24, // 31: binlogdata.VEvent.journal:type_name -> binlogdata.Journal + 35, // 32: binlogdata.VEvent.last_p_k_event:type_name -> binlogdata.LastPKEvent + 49, // 33: binlogdata.MinimalTable.fields:type_name -> query.Field + 26, // 34: binlogdata.MinimalSchema.tables:type_name -> binlogdata.MinimalTable + 44, // 35: binlogdata.VStreamOptions.config_overrides:type_name -> binlogdata.VStreamOptions.ConfigOverridesEntry + 50, // 36: binlogdata.VStreamRequest.effective_caller_id:type_name -> vtrpc.CallerID + 51, // 37: binlogdata.VStreamRequest.immediate_caller_id:type_name -> query.VTGateCallerID + 52, // 38: binlogdata.VStreamRequest.target:type_name -> query.Target + 16, // 39: binlogdata.VStreamRequest.filter:type_name -> binlogdata.Filter + 36, // 40: binlogdata.VStreamRequest.table_last_p_ks:type_name -> binlogdata.TableLastPK + 28, // 41: binlogdata.VStreamRequest.options:type_name -> binlogdata.VStreamOptions + 25, // 42: binlogdata.VStreamResponse.events:type_name -> binlogdata.VEvent + 50, // 43: binlogdata.VStreamRowsRequest.effective_caller_id:type_name -> vtrpc.CallerID + 51, // 44: binlogdata.VStreamRowsRequest.immediate_caller_id:type_name -> query.VTGateCallerID + 52, // 45: binlogdata.VStreamRowsRequest.target:type_name -> query.Target + 53, // 46: binlogdata.VStreamRowsRequest.lastpk:type_name -> query.QueryResult + 28, // 47: binlogdata.VStreamRowsRequest.options:type_name -> binlogdata.VStreamOptions + 49, // 48: binlogdata.VStreamRowsResponse.fields:type_name -> query.Field + 49, // 49: binlogdata.VStreamRowsResponse.pkfields:type_name -> query.Field + 48, // 50: binlogdata.VStreamRowsResponse.rows:type_name -> query.Row + 48, // 51: binlogdata.VStreamRowsResponse.lastpk:type_name -> query.Row + 50, // 52: binlogdata.VStreamTablesRequest.effective_caller_id:type_name -> vtrpc.CallerID + 51, // 53: binlogdata.VStreamTablesRequest.immediate_caller_id:type_name -> query.VTGateCallerID + 52, // 54: binlogdata.VStreamTablesRequest.target:type_name -> query.Target + 28, // 55: binlogdata.VStreamTablesRequest.options:type_name -> binlogdata.VStreamOptions + 49, // 56: binlogdata.VStreamTablesResponse.fields:type_name -> query.Field + 49, // 57: binlogdata.VStreamTablesResponse.pkfields:type_name -> query.Field + 48, // 58: binlogdata.VStreamTablesResponse.rows:type_name -> query.Row + 48, // 59: binlogdata.VStreamTablesResponse.lastpk:type_name -> query.Row + 36, // 60: binlogdata.LastPKEvent.table_last_p_k:type_name -> binlogdata.TableLastPK + 53, // 61: binlogdata.TableLastPK.lastpk:type_name -> query.QueryResult + 50, // 62: binlogdata.VStreamResultsRequest.effective_caller_id:type_name -> vtrpc.CallerID + 51, // 63: binlogdata.VStreamResultsRequest.immediate_caller_id:type_name -> query.VTGateCallerID + 52, // 64: binlogdata.VStreamResultsRequest.target:type_name -> query.Target + 49, // 65: binlogdata.VStreamResultsResponse.fields:type_name -> query.Field + 48, // 66: binlogdata.VStreamResultsResponse.rows:type_name -> query.Row + 6, // 67: binlogdata.BinlogTransaction.Statement.category:type_name -> binlogdata.BinlogTransaction.Statement.Category + 8, // 68: binlogdata.BinlogTransaction.Statement.charset:type_name -> binlogdata.Charset + 14, // 69: binlogdata.Rule.ConvertCharsetEntry.value:type_name -> binlogdata.CharsetConversion + 70, // [70:70] is the sub-list for method output_type + 70, // [70:70] is the sub-list for method input_type + 70, // [70:70] is the sub-list for extension type_name + 70, // [70:70] is the sub-list for extension extendee + 0, // [0:70] is the sub-list for field type_name } func init() { file_binlogdata_proto_init() } diff --git a/go/vt/proto/binlogdata/binlogdata_vtproto.pb.go b/go/vt/proto/binlogdata/binlogdata_vtproto.pb.go index 98fba617973..93b378738dd 100644 --- a/go/vt/proto/binlogdata/binlogdata_vtproto.pb.go +++ b/go/vt/proto/binlogdata/binlogdata_vtproto.pb.go @@ -314,6 +314,7 @@ func (m *RowChange) CloneVT() *RowChange { r.Before = m.Before.CloneVT() r.After = m.After.CloneVT() r.DataColumns = m.DataColumns.CloneVT() + r.JsonPartialValues = m.JsonPartialValues.CloneVT() if len(m.unknownFields) > 0 { r.unknownFields = make([]byte, len(m.unknownFields)) copy(r.unknownFields, m.unknownFields) @@ -1675,6 +1676,16 @@ func (m *RowChange) MarshalToSizedBufferVT(dAtA []byte) (int, error) { i -= len(m.unknownFields) copy(dAtA[i:], m.unknownFields) } + if m.JsonPartialValues != nil { + size, err := m.JsonPartialValues.MarshalToSizedBufferVT(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = protohelpers.EncodeVarint(dAtA, i, uint64(size)) + i-- + dAtA[i] = 0x22 + } if m.DataColumns != nil { size, err := m.DataColumns.MarshalToSizedBufferVT(dAtA[:i]) if err != nil { @@ -3636,6 +3647,10 @@ func (m *RowChange) SizeVT() (n int) { l = m.DataColumns.SizeVT() n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) } + if m.JsonPartialValues != nil { + l = m.JsonPartialValues.SizeVT() + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } n += len(m.unknownFields) return n } @@ -6619,6 +6634,42 @@ func (m *RowChange) UnmarshalVT(dAtA []byte) error { return err } iNdEx = postIndex + case 4: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field JsonPartialValues", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.JsonPartialValues == nil { + m.JsonPartialValues = &RowChange_Bitmap{} + } + if err := m.JsonPartialValues.UnmarshalVT(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := protohelpers.Skip(dAtA[iNdEx:]) diff --git a/go/vt/vttablet/tabletmanager/vreplication/replicator_plan.go b/go/vt/vttablet/tabletmanager/vreplication/replicator_plan.go index 62d6166b5ca..1e4524987f2 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/replicator_plan.go +++ b/go/vt/vttablet/tabletmanager/vreplication/replicator_plan.go @@ -28,8 +28,10 @@ import ( "vitess.io/vitess/go/mysql/collations/colldata" vjson "vitess.io/vitess/go/mysql/json" "vitess.io/vitess/go/mysql/sqlerror" + "vitess.io/vitess/go/ptr" "vitess.io/vitess/go/sqltypes" "vitess.io/vitess/go/vt/binlog/binlogplayer" + "vitess.io/vitess/go/vt/log" "vitess.io/vitess/go/vt/sqlparser" "vitess.io/vitess/go/vt/vterrors" "vitess.io/vitess/go/vt/vtgate/evalengine" @@ -377,6 +379,7 @@ func (tp *TablePlan) applyChange(rowChange *binlogdatapb.RowChange, executor fun } } if rowChange.After != nil { + jsonIndex := 0 after = true vals := sqltypes.MakeRowTrusted(tp.Fields, rowChange.After) for i, field := range tp.Fields { @@ -384,15 +387,25 @@ func (tp *TablePlan) applyChange(rowChange *binlogdatapb.RowChange, executor fun var newVal *sqltypes.Value var err error if field.Type == querypb.Type_JSON { + log.Errorf("DEBUG: vplayer applyChange: field.Type == querypb.Type_JSON, val type: %v, vals[i]: %+v", vals[i].Type(), vals[i].RawStr()) if vals[i].IsNull() { // An SQL NULL and not an actual JSON value newVal = &sqltypes.NULL + } else if rowChange.JsonPartialValues != nil && isBitSet(rowChange.JsonPartialValues.Cols, jsonIndex) { + // An SQL expression that can be converted to a JSON value such + // as JSON_INSERT(). + // This occurs e.g. when using partial JSON values as a result of + // mysqld using binlog-row-value-options=PARTIAL_JSON. + s := fmt.Sprintf(vals[i].RawStr(), field.Name) + newVal = ptr.Of(sqltypes.MakeTrusted(querypb.Type_EXPRESSION, []byte(s))) } else { // A JSON value (which may be a JSON null literal value) newVal, err = vjson.MarshalSQLValue(vals[i].Raw()) if err != nil { return nil, err } } + log.Errorf("DEBUG: vplayer applyChange: field.Type == querypb.Type_JSON, newVal: %+v", newVal) bindVar, err = tp.bindFieldVal(field, newVal) + jsonIndex++ } else { bindVar, err = tp.bindFieldVal(field, &vals[i]) } diff --git a/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go b/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go index 59db723ff2b..cbed5e4fcf8 100644 --- a/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go +++ b/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go @@ -623,7 +623,7 @@ func (vs *vstreamer) parseEvent(ev mysql.BinlogEvent, bufferAndTransmit func(vev if vevent != nil { vevents = append(vevents, vevent) } - case ev.IsWriteRows() || ev.IsDeleteRows() || ev.IsUpdateRows(): + case ev.IsWriteRows() || ev.IsDeleteRows() || ev.IsUpdateRows() || ev.IsPartialUpdateRows(): // The existence of before and after images can be used to // identify statement types. It's also possible that the // before and after images end up going to different shards. @@ -973,7 +973,7 @@ func (vs *vstreamer) processJournalEvent(vevents []*binlogdatapb.VEvent, plan *s } nextrow: for _, row := range rows.Rows { - afterOK, afterValues, _, err := vs.extractRowAndFilter(plan, row.Data, rows.DataColumns, row.NullColumns) + afterOK, afterValues, _, err := vs.extractRowAndFilter(plan, row.Data, rows.DataColumns, row.NullColumns, row.JSONPartialValues) if err != nil { return nil, err } @@ -1011,11 +1011,14 @@ nextrow: func (vs *vstreamer) processRowEvent(vevents []*binlogdatapb.VEvent, plan *streamerPlan, rows mysql.Rows) ([]*binlogdatapb.VEvent, error) { rowChanges := make([]*binlogdatapb.RowChange, 0, len(rows.Rows)) for _, row := range rows.Rows { - beforeOK, beforeValues, _, err := vs.extractRowAndFilter(plan, row.Identify, rows.IdentifyColumns, row.NullIdentifyColumns) + // The BEFORE image does not have partial JSON values so we pass an empty bitmap. + beforeOK, beforeValues, _, err := vs.extractRowAndFilter(plan, row.Identify, rows.IdentifyColumns, row.NullIdentifyColumns, mysql.Bitmap{}) if err != nil { return nil, err } - afterOK, afterValues, partial, err := vs.extractRowAndFilter(plan, row.Data, rows.DataColumns, row.NullColumns) + // The AFTER image is where we may have partial JSON values, as reflected in the + // row's JSONPartialValues bitmap. + afterOK, afterValues, partial, err := vs.extractRowAndFilter(plan, row.Data, rows.DataColumns, row.NullColumns, row.JSONPartialValues) if err != nil { return nil, err } @@ -1036,6 +1039,12 @@ func (vs *vstreamer) processRowEvent(vevents []*binlogdatapb.VEvent, plan *strea Cols: rows.DataColumns.Bits(), } } + if row.JSONPartialValues.Count() > 0 { + rowChange.JsonPartialValues = &binlogdatapb.RowChange_Bitmap{ + Count: int64(row.JSONPartialValues.Count()), + Cols: row.JSONPartialValues.Bits(), + } + } } rowChanges = append(rowChanges, rowChange) } @@ -1081,13 +1090,14 @@ func (vs *vstreamer) rebuildPlans() error { // - true, if row needs to be skipped because of workflow filter rules // - data values, array of one value per column // - true, if the row image was partial (i.e. binlog_row_image=noblob and dml doesn't update one or more blob/text columns) -func (vs *vstreamer) extractRowAndFilter(plan *streamerPlan, data []byte, dataColumns, nullColumns mysql.Bitmap) (bool, []sqltypes.Value, bool, error) { +func (vs *vstreamer) extractRowAndFilter(plan *streamerPlan, data []byte, dataColumns, nullColumns mysql.Bitmap, jsonPartialValues mysql.Bitmap) (bool, []sqltypes.Value, bool, error) { if len(data) == 0 { return false, nil, false, nil } values := make([]sqltypes.Value, dataColumns.Count()) charsets := make([]collations.ID, len(values)) valueIndex := 0 + jsonIndex := 0 pos := 0 partial := false for colNum := 0; colNum < dataColumns.Count(); colNum++ { @@ -1103,12 +1113,20 @@ func (vs *vstreamer) extractRowAndFilter(plan *streamerPlan, data []byte, dataCo valueIndex++ continue } - value, l, err := mysqlbinlog.CellValue(data, pos, plan.TableMap.Types[colNum], plan.TableMap.Metadata[colNum], plan.Table.Fields[colNum]) + partialJSON := false + if jsonPartialValues.Count() > 0 && plan.Table.Fields[colNum].Type == querypb.Type_JSON { + partialJSON = jsonPartialValues.Bit(jsonIndex) + log.Errorf("DEBUG: extractRowAndFilter: table: %s, column: %v, partialJSON: %v", plan.Table.Name, plan.Table.Fields[colNum], partialJSON) + jsonIndex++ + } + value, l, err := mysqlbinlog.CellValue(data, pos, plan.TableMap.Types[colNum], plan.TableMap.Metadata[colNum], plan.Table.Fields[colNum], partialJSON) if err != nil { log.Errorf("extractRowAndFilter: %s, table: %s, colNum: %d, fields: %+v, current values: %+v", err, plan.Table.Name, colNum, plan.Table.Fields, values) return false, nil, false, err } + log.Errorf("DEBUG: extractRowAndFilter: table: %s, column: %v, type: %v, value: %v", + plan.Table.Name, plan.Table.Fields[colNum], plan.TableMap.Types[colNum], value) pos += l if !value.IsNull() { // ENUMs and SETs require no special handling if they are NULL diff --git a/proto/binlogdata.proto b/proto/binlogdata.proto index e1df792776b..0fdf69b0773 100644 --- a/proto/binlogdata.proto +++ b/proto/binlogdata.proto @@ -333,8 +333,17 @@ message RowChange { } query.Row before = 1; query.Row after = 2; - // DataColumns is a bitmap of all columns: bit is set if column is present in the after image + // DataColumns is a bitmap of all columns: bit is set if column is + // present in the after image. Bitmap data_columns = 3; + // JsonPartialValues is a bitmap of any JSON columns where the bit is + // set if the value in the after image is a partial JSON value that + // is represented as an expression of + // JSON_[INSERT|REPLACE|REMOVE](%s, '$.path', value) which then is + // used to add/update/replace a path in the JSON document. When the + // value is used the fmt directive must be replaced by the actual + // column name of the JSON field. + Bitmap json_partial_values = 4; } // RowEvent represent row events for one table. diff --git a/web/vtadmin/src/proto/vtadmin.d.ts b/web/vtadmin/src/proto/vtadmin.d.ts index 410aaa644ff..707c6627ec4 100644 --- a/web/vtadmin/src/proto/vtadmin.d.ts +++ b/web/vtadmin/src/proto/vtadmin.d.ts @@ -37406,6 +37406,9 @@ export namespace binlogdata { /** RowChange data_columns */ data_columns?: (binlogdata.RowChange.IBitmap|null); + + /** RowChange json_partial_values */ + json_partial_values?: (binlogdata.RowChange.IBitmap|null); } /** Represents a RowChange. */ @@ -37426,6 +37429,9 @@ export namespace binlogdata { /** RowChange data_columns. */ public data_columns?: (binlogdata.RowChange.IBitmap|null); + /** RowChange json_partial_values. */ + public json_partial_values?: (binlogdata.RowChange.IBitmap|null); + /** * Creates a new RowChange instance using the specified properties. * @param [properties] Properties to set diff --git a/web/vtadmin/src/proto/vtadmin.js b/web/vtadmin/src/proto/vtadmin.js index b8ab0c1186a..84cc786a0e1 100644 --- a/web/vtadmin/src/proto/vtadmin.js +++ b/web/vtadmin/src/proto/vtadmin.js @@ -87921,6 +87921,7 @@ export const binlogdata = $root.binlogdata = (() => { * @property {query.IRow|null} [before] RowChange before * @property {query.IRow|null} [after] RowChange after * @property {binlogdata.RowChange.IBitmap|null} [data_columns] RowChange data_columns + * @property {binlogdata.RowChange.IBitmap|null} [json_partial_values] RowChange json_partial_values */ /** @@ -87962,6 +87963,14 @@ export const binlogdata = $root.binlogdata = (() => { */ RowChange.prototype.data_columns = null; + /** + * RowChange json_partial_values. + * @member {binlogdata.RowChange.IBitmap|null|undefined} json_partial_values + * @memberof binlogdata.RowChange + * @instance + */ + RowChange.prototype.json_partial_values = null; + /** * Creates a new RowChange instance using the specified properties. * @function create @@ -87992,6 +88001,8 @@ export const binlogdata = $root.binlogdata = (() => { $root.query.Row.encode(message.after, writer.uint32(/* id 2, wireType 2 =*/18).fork()).ldelim(); if (message.data_columns != null && Object.hasOwnProperty.call(message, "data_columns")) $root.binlogdata.RowChange.Bitmap.encode(message.data_columns, writer.uint32(/* id 3, wireType 2 =*/26).fork()).ldelim(); + if (message.json_partial_values != null && Object.hasOwnProperty.call(message, "json_partial_values")) + $root.binlogdata.RowChange.Bitmap.encode(message.json_partial_values, writer.uint32(/* id 4, wireType 2 =*/34).fork()).ldelim(); return writer; }; @@ -88038,6 +88049,10 @@ export const binlogdata = $root.binlogdata = (() => { message.data_columns = $root.binlogdata.RowChange.Bitmap.decode(reader, reader.uint32()); break; } + case 4: { + message.json_partial_values = $root.binlogdata.RowChange.Bitmap.decode(reader, reader.uint32()); + break; + } default: reader.skipType(tag & 7); break; @@ -88088,6 +88103,11 @@ export const binlogdata = $root.binlogdata = (() => { if (error) return "data_columns." + error; } + if (message.json_partial_values != null && message.hasOwnProperty("json_partial_values")) { + let error = $root.binlogdata.RowChange.Bitmap.verify(message.json_partial_values); + if (error) + return "json_partial_values." + error; + } return null; }; @@ -88118,6 +88138,11 @@ export const binlogdata = $root.binlogdata = (() => { throw TypeError(".binlogdata.RowChange.data_columns: object expected"); message.data_columns = $root.binlogdata.RowChange.Bitmap.fromObject(object.data_columns); } + if (object.json_partial_values != null) { + if (typeof object.json_partial_values !== "object") + throw TypeError(".binlogdata.RowChange.json_partial_values: object expected"); + message.json_partial_values = $root.binlogdata.RowChange.Bitmap.fromObject(object.json_partial_values); + } return message; }; @@ -88138,6 +88163,7 @@ export const binlogdata = $root.binlogdata = (() => { object.before = null; object.after = null; object.data_columns = null; + object.json_partial_values = null; } if (message.before != null && message.hasOwnProperty("before")) object.before = $root.query.Row.toObject(message.before, options); @@ -88145,6 +88171,8 @@ export const binlogdata = $root.binlogdata = (() => { object.after = $root.query.Row.toObject(message.after, options); if (message.data_columns != null && message.hasOwnProperty("data_columns")) object.data_columns = $root.binlogdata.RowChange.Bitmap.toObject(message.data_columns, options); + if (message.json_partial_values != null && message.hasOwnProperty("json_partial_values")) + object.json_partial_values = $root.binlogdata.RowChange.Bitmap.toObject(message.json_partial_values, options); return object; }; From c9cf71ccc04c5fa2c92a77397e5b564ab37e3093 Mon Sep 17 00:00:00 2001 From: Matt Lord Date: Sun, 8 Dec 2024 21:08:54 -0500 Subject: [PATCH 02/39] Remove `set names binary` in controller Signed-off-by: Matt Lord --- go/vt/vttablet/tabletmanager/rpc_vreplication_test.go | 2 +- go/vt/vttablet/tabletmanager/vreplication/controller.go | 5 ----- go/vt/vttablet/tabletmanager/vreplication/replicator_plan.go | 1 + 3 files changed, 2 insertions(+), 6 deletions(-) diff --git a/go/vt/vttablet/tabletmanager/rpc_vreplication_test.go b/go/vt/vttablet/tabletmanager/rpc_vreplication_test.go index 0a5bd9f26fd..80870de23ce 100644 --- a/go/vt/vttablet/tabletmanager/rpc_vreplication_test.go +++ b/go/vt/vttablet/tabletmanager/rpc_vreplication_test.go @@ -1753,7 +1753,7 @@ func addInvariants(dbClient *binlogplayer.MockDBClient, vreplID, sourceTabletUID fmt.Sprintf("%s||0|0|Stopped|1|%s|0|0", position, workflow), )) dbClient.AddInvariant(setSessionTZ, &sqltypes.Result{}) - dbClient.AddInvariant(setNames, &sqltypes.Result{}) + //dbClient.AddInvariant(setNames, &sqltypes.Result{}) dbClient.AddInvariant(setNetReadTimeout, &sqltypes.Result{}) dbClient.AddInvariant(setNetWriteTimeout, &sqltypes.Result{}) diff --git a/go/vt/vttablet/tabletmanager/vreplication/controller.go b/go/vt/vttablet/tabletmanager/vreplication/controller.go index 7067211ff10..51a37b09284 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/controller.go +++ b/go/vt/vttablet/tabletmanager/vreplication/controller.go @@ -215,11 +215,6 @@ func setDBClientSettings(dbClient binlogplayer.DBClient, workflowConfig *vttable if _, err := dbClient.ExecuteFetch("set @@session.time_zone = '+00:00'", maxRows); err != nil { return err } - // Tables may have varying character sets. To ship the bits without interpreting them - // we set the character set to be binary. - if _, err := dbClient.ExecuteFetch("set names 'binary'", maxRows); err != nil { - return err - } if _, err := dbClient.ExecuteFetch(fmt.Sprintf("set @@session.net_read_timeout = %v", workflowConfig.NetReadTimeout), maxRows); err != nil { return err diff --git a/go/vt/vttablet/tabletmanager/vreplication/replicator_plan.go b/go/vt/vttablet/tabletmanager/vreplication/replicator_plan.go index 1e4524987f2..bb4d4d8548d 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/replicator_plan.go +++ b/go/vt/vttablet/tabletmanager/vreplication/replicator_plan.go @@ -595,6 +595,7 @@ func execParsedQuery(pq *sqlparser.ParsedQuery, bindvars map[string]*querypb.Bin if err != nil { return nil, err } + log.Errorf("DEBUG: vplayer execParsedQuery: query: %s", query) return executor(query) } From 2aed1aa2671569dbc96993d9ea21e4f03bdd48e9 Mon Sep 17 00:00:00 2001 From: Matt Lord Date: Sun, 8 Dec 2024 21:43:52 -0500 Subject: [PATCH 03/39] Fix test Signed-off-by: Matt Lord --- go/mysql/endtoend/replication_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/mysql/endtoend/replication_test.go b/go/mysql/endtoend/replication_test.go index d3b9a6722ea..a04f75c6b43 100644 --- a/go/mysql/endtoend/replication_test.go +++ b/go/mysql/endtoend/replication_test.go @@ -1064,7 +1064,7 @@ func valuesForTests(t *testing.T, rs *mysql.Rows, tm *mysql.TableMap, rowIndex i } // We have real data - value, l, err := binlog.CellValue(data, pos, tm.Types[c], tm.Metadata[c], &querypb.Field{Type: querypb.Type_UINT64}) + value, l, err := binlog.CellValue(data, pos, tm.Types[c], tm.Metadata[c], &querypb.Field{Type: querypb.Type_UINT64}, false) if err != nil { return nil, err } From 7b75ed61d76302efeaf5f0ec29a6c9b451ee6f26 Mon Sep 17 00:00:00 2001 From: Matt Lord Date: Sun, 8 Dec 2024 22:09:40 -0500 Subject: [PATCH 04/39] Use charset literals in partial JSON expression Signed-off-by: Matt Lord --- go/mysql/binlog/binlog_json.go | 8 +++++++- go/mysql/binlog_event_mysql56_test.go | 2 +- go/vt/proto/binlogdata/binlogdata.pb.go | 2 +- go/vt/vttablet/tabletmanager/rpc_vreplication_test.go | 2 +- go/vt/vttablet/tabletmanager/vreplication/controller.go | 5 +++++ proto/binlogdata.proto | 2 +- 6 files changed, 16 insertions(+), 5 deletions(-) diff --git a/go/mysql/binlog/binlog_json.go b/go/mysql/binlog/binlog_json.go index 73146abd06b..2a096728724 100644 --- a/go/mysql/binlog/binlog_json.go +++ b/go/mysql/binlog/binlog_json.go @@ -105,7 +105,10 @@ func ParseBinaryJSONDiff(data []byte) (sqltypes.Value, error) { path := data[pos : uint64(pos)+pathLen] pos += int(pathLen) log.Errorf("DEBUG: json diff path: %s", string(path)) - diff.WriteString(fmt.Sprintf("'%s', ", path)) + // We have to specify the unicode character set for the strings we + // use in the expression as the connection can be using a different + // character set (e.g. vreplication always uses set names binary). + diff.WriteString(fmt.Sprintf("_utf8mb4'%s', ", path)) if opType == jsonDiffOpRemove { // No value for remove diff.WriteString(")") @@ -124,6 +127,9 @@ func ParseBinaryJSONDiff(data []byte) (sqltypes.Value, error) { return sqltypes.Value{}, fmt.Errorf("cannot read JSON diff value for path %s: %w", path, err) } log.Errorf("DEBUG: json diff value: %v", value) + if value.Type() == json.TypeString { + diff.WriteString("_utf8mb4") + } diff.WriteString(fmt.Sprintf("%s)", value)) return sqltypes.MakeTrusted(sqltypes.Expression, diff.Bytes()), nil diff --git a/go/mysql/binlog_event_mysql56_test.go b/go/mysql/binlog_event_mysql56_test.go index be9ae071ded..4a61da04a20 100644 --- a/go/mysql/binlog_event_mysql56_test.go +++ b/go/mysql/binlog_event_mysql56_test.go @@ -353,7 +353,7 @@ func TestMySQL56PartialUpdateRowsEvent(t *testing.T) { vals, err := ev.StringValuesForTests(tm, i) require.NoError(t, err) // The third column is the JSON column. - require.Equal(t, vals[2], `JSON_INSERT(%s, '$.role', "manager")`) + require.Equal(t, `JSON_INSERT(%s, _utf8mb4'$.role', _utf8mb4"manager")`, vals[2]) t.Logf("Rows: %v", vals) } } diff --git a/go/vt/proto/binlogdata/binlogdata.pb.go b/go/vt/proto/binlogdata/binlogdata.pb.go index 51736c606db..0cc9d5b17cf 100644 --- a/go/vt/proto/binlogdata/binlogdata.pb.go +++ b/go/vt/proto/binlogdata/binlogdata.pb.go @@ -1340,7 +1340,7 @@ type RowChange struct { // set if the value in the after image is a partial JSON value that // is represented as an expression of // JSON_[INSERT|REPLACE|REMOVE](%s, '$.path', value) which then is - // used to add/update/replace a path in the JSON document. When the + // used to add/update/remove a path in the JSON document. When the // value is used the fmt directive must be replaced by the actual // column name of the JSON field. JsonPartialValues *RowChange_Bitmap `protobuf:"bytes,4,opt,name=json_partial_values,json=jsonPartialValues,proto3" json:"json_partial_values,omitempty"` diff --git a/go/vt/vttablet/tabletmanager/rpc_vreplication_test.go b/go/vt/vttablet/tabletmanager/rpc_vreplication_test.go index 80870de23ce..0a5bd9f26fd 100644 --- a/go/vt/vttablet/tabletmanager/rpc_vreplication_test.go +++ b/go/vt/vttablet/tabletmanager/rpc_vreplication_test.go @@ -1753,7 +1753,7 @@ func addInvariants(dbClient *binlogplayer.MockDBClient, vreplID, sourceTabletUID fmt.Sprintf("%s||0|0|Stopped|1|%s|0|0", position, workflow), )) dbClient.AddInvariant(setSessionTZ, &sqltypes.Result{}) - //dbClient.AddInvariant(setNames, &sqltypes.Result{}) + dbClient.AddInvariant(setNames, &sqltypes.Result{}) dbClient.AddInvariant(setNetReadTimeout, &sqltypes.Result{}) dbClient.AddInvariant(setNetWriteTimeout, &sqltypes.Result{}) diff --git a/go/vt/vttablet/tabletmanager/vreplication/controller.go b/go/vt/vttablet/tabletmanager/vreplication/controller.go index 51a37b09284..7067211ff10 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/controller.go +++ b/go/vt/vttablet/tabletmanager/vreplication/controller.go @@ -215,6 +215,11 @@ func setDBClientSettings(dbClient binlogplayer.DBClient, workflowConfig *vttable if _, err := dbClient.ExecuteFetch("set @@session.time_zone = '+00:00'", maxRows); err != nil { return err } + // Tables may have varying character sets. To ship the bits without interpreting them + // we set the character set to be binary. + if _, err := dbClient.ExecuteFetch("set names 'binary'", maxRows); err != nil { + return err + } if _, err := dbClient.ExecuteFetch(fmt.Sprintf("set @@session.net_read_timeout = %v", workflowConfig.NetReadTimeout), maxRows); err != nil { return err diff --git a/proto/binlogdata.proto b/proto/binlogdata.proto index 0fdf69b0773..4351e48d12a 100644 --- a/proto/binlogdata.proto +++ b/proto/binlogdata.proto @@ -340,7 +340,7 @@ message RowChange { // set if the value in the after image is a partial JSON value that // is represented as an expression of // JSON_[INSERT|REPLACE|REMOVE](%s, '$.path', value) which then is - // used to add/update/replace a path in the JSON document. When the + // used to add/update/remove a path in the JSON document. When the // value is used the fmt directive must be replaced by the actual // column name of the JSON field. Bitmap json_partial_values = 4; From 748589d5d105b7c096edf284c5d1984763d92b1f Mon Sep 17 00:00:00 2001 From: Matt Lord Date: Sun, 8 Dec 2024 22:47:06 -0500 Subject: [PATCH 05/39] Minor changes Signed-off-by: Matt Lord --- go/mysql/binlog/rbr.go | 13 ++++++------- .../tabletmanager/vreplication/replicator_plan.go | 7 ++++--- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/go/mysql/binlog/rbr.go b/go/mysql/binlog/rbr.go index 9f92c18c530..ef1e5fdd439 100644 --- a/go/mysql/binlog/rbr.go +++ b/go/mysql/binlog/rbr.go @@ -689,14 +689,13 @@ func CellValue(data []byte, pos int, typ byte, metadata uint16, field *querypb.F } log.Errorf("DEBUG: decoded partialJSON cell value: %v", val) return val, l + int(metadata), nil - } else { - jsonVal, err := ParseBinaryJSON(jsonData) - if err != nil { - panic(err) - } - jd := jsonVal.MarshalTo(nil) - return sqltypes.MakeTrusted(sqltypes.Expression, jd), l + int(metadata), nil } + jsonVal, err := ParseBinaryJSON(jsonData) + if err != nil { + panic(err) + } + jd := jsonVal.MarshalTo(nil) + return sqltypes.MakeTrusted(sqltypes.Expression, jd), l + int(metadata), nil } return sqltypes.MakeTrusted(querypb.Type_VARBINARY, diff --git a/go/vt/vttablet/tabletmanager/vreplication/replicator_plan.go b/go/vt/vttablet/tabletmanager/vreplication/replicator_plan.go index bb4d4d8548d..037f5777077 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/replicator_plan.go +++ b/go/vt/vttablet/tabletmanager/vreplication/replicator_plan.go @@ -388,16 +388,17 @@ func (tp *TablePlan) applyChange(rowChange *binlogdatapb.RowChange, executor fun var err error if field.Type == querypb.Type_JSON { log.Errorf("DEBUG: vplayer applyChange: field.Type == querypb.Type_JSON, val type: %v, vals[i]: %+v", vals[i].Type(), vals[i].RawStr()) - if vals[i].IsNull() { // An SQL NULL and not an actual JSON value + switch { + case vals[i].IsNull(): // An SQL NULL and not an actual JSON value newVal = &sqltypes.NULL - } else if rowChange.JsonPartialValues != nil && isBitSet(rowChange.JsonPartialValues.Cols, jsonIndex) { + case rowChange.JsonPartialValues != nil && isBitSet(rowChange.JsonPartialValues.Cols, jsonIndex): // An SQL expression that can be converted to a JSON value such // as JSON_INSERT(). // This occurs e.g. when using partial JSON values as a result of // mysqld using binlog-row-value-options=PARTIAL_JSON. s := fmt.Sprintf(vals[i].RawStr(), field.Name) newVal = ptr.Of(sqltypes.MakeTrusted(querypb.Type_EXPRESSION, []byte(s))) - } else { // A JSON value (which may be a JSON null literal value) + default: // A JSON value (which may be a JSON null literal value) newVal, err = vjson.MarshalSQLValue(vals[i].Raw()) if err != nil { return nil, err From 0b2b9cfaf592bfd68f86b90e11dde18ee5414b86 Mon Sep 17 00:00:00 2001 From: Matt Lord Date: Sun, 8 Dec 2024 22:49:09 -0500 Subject: [PATCH 06/39] Remove DEBUG printfs Signed-off-by: Matt Lord --- go/mysql/binlog/binlog_json.go | 10 ---------- go/mysql/binlog/rbr.go | 3 --- go/mysql/binlog_event_rbr.go | 4 ---- .../tabletmanager/vreplication/replicator_plan.go | 4 ---- go/vt/vttablet/tabletserver/vstreamer/vstreamer.go | 3 --- 5 files changed, 24 deletions(-) diff --git a/go/mysql/binlog/binlog_json.go b/go/mysql/binlog/binlog_json.go index 2a096728724..5a32365fcc9 100644 --- a/go/mysql/binlog/binlog_json.go +++ b/go/mysql/binlog/binlog_json.go @@ -19,7 +19,6 @@ package binlog import ( "bytes" "encoding/binary" - "encoding/hex" "fmt" "math" "strconv" @@ -28,7 +27,6 @@ import ( "vitess.io/vitess/go/mysql/format" "vitess.io/vitess/go/mysql/json" "vitess.io/vitess/go/sqltypes" - "vitess.io/vitess/go/vt/log" "vitess.io/vitess/go/vt/vterrors" querypb "vitess.io/vitess/go/vt/proto/query" @@ -76,7 +74,6 @@ func ParseBinaryJSON(data []byte) (*json.Value, error) { // ParseBinaryJSONDiff provides the parsing function from the MySQL JSON // diff representation to an SQL expression. func ParseBinaryJSONDiff(data []byte) (sqltypes.Value, error) { - log.Errorf("DEBUG: json diff data hex: %s, string: %s", hex.EncodeToString(data), string(data)) pos := 0 opType := jsonDiffOp(data[pos]) pos++ @@ -92,19 +89,14 @@ func ParseBinaryJSONDiff(data []byte) (sqltypes.Value, error) { } diff.WriteString("%s, ") // This will later be replaced by the field name - log.Errorf("DEBUG: json diff opType: %d", opType) - pathLen, readTo, ok := readLenEncInt(data, pos) if !ok { return sqltypes.Value{}, fmt.Errorf("cannot read JSON diff path length") } pos = readTo - log.Errorf("DEBUG: json diff path length: %d", pathLen) - path := data[pos : uint64(pos)+pathLen] pos += int(pathLen) - log.Errorf("DEBUG: json diff path: %s", string(path)) // We have to specify the unicode character set for the strings we // use in the expression as the connection can be using a different // character set (e.g. vreplication always uses set names binary). @@ -120,13 +112,11 @@ func ParseBinaryJSONDiff(data []byte) (sqltypes.Value, error) { return sqltypes.Value{}, fmt.Errorf("cannot read JSON diff path length") } pos = readTo - log.Errorf("DEBUG: json diff value length: %d", valueLen) value, err := ParseBinaryJSON(data[pos : uint64(pos)+valueLen]) if err != nil { return sqltypes.Value{}, fmt.Errorf("cannot read JSON diff value for path %s: %w", path, err) } - log.Errorf("DEBUG: json diff value: %v", value) if value.Type() == json.TypeString { diff.WriteString("_utf8mb4") } diff --git a/go/mysql/binlog/rbr.go b/go/mysql/binlog/rbr.go index ef1e5fdd439..42494b0a346 100644 --- a/go/mysql/binlog/rbr.go +++ b/go/mysql/binlog/rbr.go @@ -26,7 +26,6 @@ import ( "time" "vitess.io/vitess/go/sqltypes" - "vitess.io/vitess/go/vt/log" "vitess.io/vitess/go/vt/vterrors" querypb "vitess.io/vitess/go/vt/proto/query" @@ -682,12 +681,10 @@ func CellValue(data []byte, pos int, typ byte, metadata uint16, field *querypb.F if typ == TypeJSON { jsonData := data[pos : pos+l] if partialJSON { - log.Errorf("DEBUG: partialJSON cell value: %s", string(jsonData)) val, err := ParseBinaryJSONDiff(jsonData) if err != nil { panic(err) } - log.Errorf("DEBUG: decoded partialJSON cell value: %v", val) return val, l + int(metadata), nil } jsonVal, err := ParseBinaryJSON(jsonData) diff --git a/go/mysql/binlog_event_rbr.go b/go/mysql/binlog_event_rbr.go index ea59a2a2247..0e0ad0c458f 100644 --- a/go/mysql/binlog_event_rbr.go +++ b/go/mysql/binlog_event_rbr.go @@ -21,7 +21,6 @@ import ( "vitess.io/vitess/go/mysql/binlog" "vitess.io/vitess/go/mysql/collations" - "vitess.io/vitess/go/vt/log" "vitess.io/vitess/go/vt/vterrors" querypb "vitess.io/vitess/go/vt/proto/query" @@ -370,15 +369,12 @@ func (ev binlogEvent) Rows(f BinlogFormat, tm *TableMap) (Rows, error) { } if ev.Type() == ePartialUpdateRowsEvent { - log.Errorf("DEBUG: PartialUpdateRowsEvent found with %d JSON columns", numJSONColumns) // The first byte indicates whether or not any JSON values are partial. // If it's 0 then there's nothing else to do. partialJSON := uint8(data[pos]) - log.Errorf("DEBUG: PartialJSON: %d", partialJSON) pos++ if partialJSON == 1 { row.JSONPartialValues, pos = newBitmap(data, pos, numJSONColumns) - log.Errorf("DEBUG: PartialUpdateRowsEvent: JSONPartialValues: %08b", row.JSONPartialValues) } } diff --git a/go/vt/vttablet/tabletmanager/vreplication/replicator_plan.go b/go/vt/vttablet/tabletmanager/vreplication/replicator_plan.go index 037f5777077..bf282e663c2 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/replicator_plan.go +++ b/go/vt/vttablet/tabletmanager/vreplication/replicator_plan.go @@ -31,7 +31,6 @@ import ( "vitess.io/vitess/go/ptr" "vitess.io/vitess/go/sqltypes" "vitess.io/vitess/go/vt/binlog/binlogplayer" - "vitess.io/vitess/go/vt/log" "vitess.io/vitess/go/vt/sqlparser" "vitess.io/vitess/go/vt/vterrors" "vitess.io/vitess/go/vt/vtgate/evalengine" @@ -387,7 +386,6 @@ func (tp *TablePlan) applyChange(rowChange *binlogdatapb.RowChange, executor fun var newVal *sqltypes.Value var err error if field.Type == querypb.Type_JSON { - log.Errorf("DEBUG: vplayer applyChange: field.Type == querypb.Type_JSON, val type: %v, vals[i]: %+v", vals[i].Type(), vals[i].RawStr()) switch { case vals[i].IsNull(): // An SQL NULL and not an actual JSON value newVal = &sqltypes.NULL @@ -404,7 +402,6 @@ func (tp *TablePlan) applyChange(rowChange *binlogdatapb.RowChange, executor fun return nil, err } } - log.Errorf("DEBUG: vplayer applyChange: field.Type == querypb.Type_JSON, newVal: %+v", newVal) bindVar, err = tp.bindFieldVal(field, newVal) jsonIndex++ } else { @@ -596,7 +593,6 @@ func execParsedQuery(pq *sqlparser.ParsedQuery, bindvars map[string]*querypb.Bin if err != nil { return nil, err } - log.Errorf("DEBUG: vplayer execParsedQuery: query: %s", query) return executor(query) } diff --git a/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go b/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go index cbed5e4fcf8..568461aed3d 100644 --- a/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go +++ b/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go @@ -1116,7 +1116,6 @@ func (vs *vstreamer) extractRowAndFilter(plan *streamerPlan, data []byte, dataCo partialJSON := false if jsonPartialValues.Count() > 0 && plan.Table.Fields[colNum].Type == querypb.Type_JSON { partialJSON = jsonPartialValues.Bit(jsonIndex) - log.Errorf("DEBUG: extractRowAndFilter: table: %s, column: %v, partialJSON: %v", plan.Table.Name, plan.Table.Fields[colNum], partialJSON) jsonIndex++ } value, l, err := mysqlbinlog.CellValue(data, pos, plan.TableMap.Types[colNum], plan.TableMap.Metadata[colNum], plan.Table.Fields[colNum], partialJSON) @@ -1125,8 +1124,6 @@ func (vs *vstreamer) extractRowAndFilter(plan *streamerPlan, data []byte, dataCo err, plan.Table.Name, colNum, plan.Table.Fields, values) return false, nil, false, err } - log.Errorf("DEBUG: extractRowAndFilter: table: %s, column: %v, type: %v, value: %v", - plan.Table.Name, plan.Table.Fields[colNum], plan.TableMap.Types[colNum], value) pos += l if !value.IsNull() { // ENUMs and SETs require no special handling if they are NULL From 7be99a3ba3a8416ccbcf9847c1361003b850db0a Mon Sep 17 00:00:00 2001 From: Matt Lord Date: Mon, 9 Dec 2024 10:44:03 -0500 Subject: [PATCH 07/39] Get N diffs working Signed-off-by: Matt Lord --- go/mysql/binlog/binlog_json.go | 92 +++++++------- go/mysql/binlog_event_mysql56_test.go | 172 ++++++++++++++------------ go/mysql/binlog_event_rbr.go | 3 + 3 files changed, 143 insertions(+), 124 deletions(-) diff --git a/go/mysql/binlog/binlog_json.go b/go/mysql/binlog/binlog_json.go index 5a32365fcc9..6f02456e495 100644 --- a/go/mysql/binlog/binlog_json.go +++ b/go/mysql/binlog/binlog_json.go @@ -74,53 +74,61 @@ func ParseBinaryJSON(data []byte) (*json.Value, error) { // ParseBinaryJSONDiff provides the parsing function from the MySQL JSON // diff representation to an SQL expression. func ParseBinaryJSONDiff(data []byte) (sqltypes.Value, error) { - pos := 0 - opType := jsonDiffOp(data[pos]) - pos++ diff := bytes.Buffer{} + diff.Grow(int(float32(len(data)) * 1.5)) + pos := 0 + outer := false + innerStr := "" - switch opType { - case jsonDiffOpReplace: - diff.WriteString("JSON_REPLACE(") - case jsonDiffOpInsert: - diff.WriteString("JSON_INSERT(") - case jsonDiffOpRemove: - diff.WriteString("JSON_REMOVE(") - } - diff.WriteString("%s, ") // This will later be replaced by the field name - - pathLen, readTo, ok := readLenEncInt(data, pos) - if !ok { - return sqltypes.Value{}, fmt.Errorf("cannot read JSON diff path length") - } - pos = readTo - - path := data[pos : uint64(pos)+pathLen] - pos += int(pathLen) - // We have to specify the unicode character set for the strings we - // use in the expression as the connection can be using a different - // character set (e.g. vreplication always uses set names binary). - diff.WriteString(fmt.Sprintf("_utf8mb4'%s', ", path)) - - if opType == jsonDiffOpRemove { // No value for remove - diff.WriteString(")") - return sqltypes.MakeTrusted(sqltypes.Expression, diff.Bytes()), nil - } + for pos < len(data) { + if outer { + innerStr = diff.String() + diff.Reset() + } + opType := jsonDiffOp(data[pos]) + pos++ + switch opType { + case jsonDiffOpReplace: + diff.WriteString("JSON_REPLACE(") + case jsonDiffOpInsert: + diff.WriteString("JSON_INSERT(") + case jsonDiffOpRemove: + diff.WriteString("JSON_REMOVE(") + } + if outer { + diff.WriteString(innerStr) + diff.WriteString(", ") + } else { // Only the inner most function has the field name + diff.WriteString("%s, ") // This will later be replaced by the field name + } - valueLen, readTo, ok := readLenEncInt(data, pos) - if !ok { - return sqltypes.Value{}, fmt.Errorf("cannot read JSON diff path length") - } - pos = readTo + pathLen, readTo := readVariableLength(data, pos) + pos = readTo + path := data[pos : pos+pathLen] + pos += pathLen + // We have to specify the unicode character set for the strings we + // use in the expression as the connection can be using a different + // character set (e.g. vreplication always uses set names binary). + diff.WriteString(fmt.Sprintf("_utf8mb4'%s'", path)) + + if opType == jsonDiffOpRemove { // No value for remove + diff.WriteString(")") + } else { + valueLen, readTo := readVariableLength(data, pos) + pos = readTo + value, err := ParseBinaryJSON(data[pos : pos+valueLen]) + if err != nil { + return sqltypes.Value{}, fmt.Errorf("cannot read JSON diff value for path %s: %w", path, err) + } + pos += valueLen + if value.Type() == json.TypeString { + diff.WriteString(", _utf8mb4") + } + diff.WriteString(fmt.Sprintf("%s)", value)) + } - value, err := ParseBinaryJSON(data[pos : uint64(pos)+valueLen]) - if err != nil { - return sqltypes.Value{}, fmt.Errorf("cannot read JSON diff value for path %s: %w", path, err) - } - if value.Type() == json.TypeString { - diff.WriteString("_utf8mb4") + outer = true } - diff.WriteString(fmt.Sprintf("%s)", value)) return sqltypes.MakeTrusted(sqltypes.Expression, diff.Bytes()), nil } diff --git a/go/mysql/binlog_event_mysql56_test.go b/go/mysql/binlog_event_mysql56_test.go index 4a61da04a20..b7725e69691 100644 --- a/go/mysql/binlog_event_mysql56_test.go +++ b/go/mysql/binlog_event_mysql56_test.go @@ -251,78 +251,6 @@ func TestMysql56SemiSyncAck(t *testing.T) { } func TestMySQL56PartialUpdateRowsEvent(t *testing.T) { - // The mysqlbinlog -vvv --base64-output=decode-rows output for the following event: - // ### UPDATE `vt_commerce`.`customer` - // ### WHERE - // ### @1=1 /* LONGINT meta=0 nullable=0 is_null=0 */ - // ### @2='alice@domain.com' /* VARSTRING(128) meta=128 nullable=1 is_null=0 */ - // ### @3='{"salary": 100}' /* JSON meta=4 nullable=1 is_null=0 */ - // ### SET - // ### @1=1 /* LONGINT meta=0 nullable=0 is_null=0 */ - // ### @2='alice@domain.com' /* VARSTRING(128) meta=128 nullable=1 is_null=0 */ - // ### @3=JSON_INSERT(@3, '$.role', 'manager') /* JSON meta=4 nullable=1 is_null=0 */ - // ### UPDATE `vt_commerce`.`customer` - // ### WHERE - // ### @1=2 /* LONGINT meta=0 nullable=0 is_null=0 */ - // ### @2='bob@domain.com' /* VARSTRING(128) meta=128 nullable=1 is_null=0 */ - // ### @3='{"salary": 99}' /* JSON meta=4 nullable=1 is_null=0 */ - // ### SET - // ### @1=2 /* LONGINT meta=0 nullable=0 is_null=0 */ - // ### @2='bob@domain.com' /* VARSTRING(128) meta=128 nullable=1 is_null=0 */ - // ### @3=JSON_INSERT(@3, '$.role', 'manager') /* JSON meta=4 nullable=1 is_null=0 */ - // ### UPDATE `vt_commerce`.`customer` - // ### WHERE - // ### @1=3 /* LONGINT meta=0 nullable=0 is_null=0 */ - // ### @2='charlie@domain.com' /* VARSTRING(128) meta=128 nullable=1 is_null=0 */ - // ### @3='{"salary": 99}' /* JSON meta=4 nullable=1 is_null=0 */ - // ### SET - // ### @1=3 /* LONGINT meta=0 nullable=0 is_null=0 */ - // ### @2='charlie@domain.com' /* VARSTRING(128) meta=128 nullable=1 is_null=0 */ - // ### @3=JSON_INSERT(@3, '$.role', 'manager') /* JSON meta=4 nullable=1 is_null=0 */ - // ### UPDATE `vt_commerce`.`customer` - // ### WHERE - // ### @1=4 /* LONGINT meta=0 nullable=0 is_null=0 */ - // ### @2='dan@domain.com' /* VARSTRING(128) meta=128 nullable=1 is_null=0 */ - // ### @3='{"salary": 99}' /* JSON meta=4 nullable=1 is_null=0 */ - // ### SET - // ### @1=4 /* LONGINT meta=0 nullable=0 is_null=0 */ - // ### @2='dan@domain.com' /* VARSTRING(128) meta=128 nullable=1 is_null=0 */ - // ### @3=JSON_INSERT(@3, '$.role', 'manager') /* JSON meta=4 nullable=1 is_null=0 */ - // ### UPDATE `vt_commerce`.`customer` - // ### WHERE - // ### @1=5 /* LONGINT meta=0 nullable=0 is_null=0 */ - // ### @2='eve@domain.com' /* VARSTRING(128) meta=128 nullable=1 is_null=0 */ - // ### @3='{"salary": 100}' /* JSON meta=4 nullable=1 is_null=0 */ - // ### SET - // ### @1=5 /* LONGINT meta=0 nullable=0 is_null=0 */ - // ### @2='eve@domain.com' /* VARSTRING(128) meta=128 nullable=1 is_null=0 */ - // ### @3=JSON_INSERT(@3, '$.role', 'manager') /* JSON meta=4 nullable=1 is_null=0 */ - mysql56PartialUpdateRowEvent := NewMysql56BinlogEvent([]byte{0x67, 0xc4, 0x54, 0x67, 0x27, 0x1f, 0x91, 0x10, 0x76, 0x14, 0x02, 0x00, 0x00, 0x19, 0x69, - 0x00, 0x00, 0x00, 0x00, 0xb0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x02, 0x00, 0x03, 0xff, 0xff, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x10, 0x61, 0x6c, 0x69, 0x63, 0x65, 0x40, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x2e, 0x63, 0x6f, 0x6d, 0x16, 0x00, 0x00, 0x00, 0x00, - 0x01, 0x00, 0x15, 0x00, 0x0b, 0x00, 0x03, 0x00, 0x0c, 0x0e, 0x00, 0x73, 0x65, 0x78, 0x06, 0x66, 0x65, 0x6d, 0x61, 0x6c, 0x65, 0x01, 0x01, 0x00, - 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x61, 0x6c, 0x69, 0x63, 0x65, 0x40, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x2e, 0x63, 0x6f, - 0x6d, 0x12, 0x00, 0x00, 0x00, 0x01, 0x06, 0x24, 0x2e, 0x72, 0x6f, 0x6c, 0x65, 0x09, 0x0c, 0x07, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x00, - 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0e, 0x62, 0x6f, 0x62, 0x40, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x2e, 0x63, 0x6f, 0x6d, 0x14, - 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x13, 0x00, 0x0b, 0x00, 0x03, 0x00, 0x0c, 0x0e, 0x00, 0x73, 0x65, 0x78, 0x04, 0x6d, 0x61, 0x6c, 0x65, 0x01, - 0x01, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0e, 0x62, 0x6f, 0x62, 0x40, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x2e, 0x63, 0x6f, - 0x6d, 0x12, 0x00, 0x00, 0x00, 0x01, 0x06, 0x24, 0x2e, 0x72, 0x6f, 0x6c, 0x65, 0x09, 0x0c, 0x07, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x00, - 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x12, 0x63, 0x68, 0x61, 0x72, 0x6c, 0x69, 0x65, 0x40, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x2e, - 0x63, 0x6f, 0x6d, 0x14, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x13, 0x00, 0x0b, 0x00, 0x03, 0x00, 0x0c, 0x0e, 0x00, 0x73, 0x65, 0x78, 0x04, 0x6d, - 0x61, 0x6c, 0x65, 0x01, 0x01, 0x00, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x12, 0x63, 0x68, 0x61, 0x72, 0x6c, 0x69, 0x65, 0x40, 0x64, - 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x2e, 0x63, 0x6f, 0x6d, 0x12, 0x00, 0x00, 0x00, 0x01, 0x06, 0x24, 0x2e, 0x72, 0x6f, 0x6c, 0x65, 0x09, 0x0c, 0x07, - 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0e, 0x64, 0x61, 0x6e, 0x40, 0x64, 0x6f, 0x6d, - 0x61, 0x69, 0x6e, 0x2e, 0x63, 0x6f, 0x6d, 0x14, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x13, 0x00, 0x0b, 0x00, 0x03, 0x00, 0x0c, 0x0e, 0x00, 0x73, - 0x65, 0x78, 0x04, 0x6d, 0x61, 0x6c, 0x65, 0x01, 0x01, 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0e, 0x64, 0x61, 0x6e, 0x40, 0x64, - 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x2e, 0x63, 0x6f, 0x6d, 0x12, 0x00, 0x00, 0x00, 0x01, 0x06, 0x24, 0x2e, 0x72, 0x6f, 0x6c, 0x65, 0x09, 0x0c, 0x07, - 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x00, 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0e, 0x65, 0x76, 0x65, 0x40, 0x64, 0x6f, 0x6d, - 0x61, 0x69, 0x6e, 0x2e, 0x63, 0x6f, 0x6d, 0x16, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x15, 0x00, 0x0b, 0x00, 0x03, 0x00, 0x0c, 0x0e, 0x00, 0x73, - 0x65, 0x78, 0x06, 0x66, 0x65, 0x6d, 0x61, 0x6c, 0x65, 0x01, 0x01, 0x00, 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0e, 0x65, 0x76, 0x65, - 0x40, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x2e, 0x63, 0x6f, 0x6d, 0x12, 0x00, 0x00, 0x00, 0x01, 0x06, 0x24, 0x2e, 0x72, 0x6f, 0x6c, 0x65, 0x09, - 0x0c, 0x07, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72}) - - require.True(t, mysql56PartialUpdateRowEvent.IsPartialUpdateRows()) - format := BinlogFormat{ HeaderSizes: []byte{ 0, 13, 0, 8, 0, 0, 0, 0, 4, 0, 4, 0, 0, 0, 98, 0, 4, 26, 8, 0, 0, 0, 8, 8, 8, 2, 0, 0, 0, 10, 10, 10, 42, 42, 0, 18, 52, 0, 10, 40, 0, @@ -344,16 +272,96 @@ func TestMySQL56PartialUpdateRowsEvent(t *testing.T) { Metadata: []uint16{0, 128, 4}, ColumnCollationIDs: []collations.ID{63}, } - ev, err := mysql56PartialUpdateRowEvent.Rows(format, tm) - require.NoError(t, err) - assert.Equal(t, 5, len(ev.Rows)) - require.NoError(t, err) - for i := range ev.Rows { - vals, err := ev.StringValuesForTests(tm, i) - require.NoError(t, err) - // The third column is the JSON column. - require.Equal(t, `JSON_INSERT(%s, _utf8mb4'$.role', _utf8mb4"manager")`, vals[2]) - t.Logf("Rows: %v", vals) + testCases := []struct { + name string + rawEvent []byte + want string + }{ + { + name: "REMOVE and REPLACE", + // The mysqlbinlog -vvv --base64-output=decode-rows output for the following event: + // ### UPDATE `vt_commerce`.`customer` + // ### WHERE + // ### @1=1 /* LONGINT meta=0 nullable=0 is_null=0 */ + // ### @2='alice@domain.com' /* VARSTRING(128) meta=128 nullable=1 is_null=0 */ + // ### @3='{"day": "friday", "role": "manager", "color": "red", "salary": 100, "favorite_color": "black"}' /* JSON meta=4 nullable=1 is_null=0 */ + // ### SET + // ### @1=1 /* LONGINT meta=0 nullable=0 is_null=0 */ + // ### @2='alice@domain.com' /* VARSTRING(128) meta=128 nullable=1 is_null=0 */ + // ### @3=JSON_REMOVE( + // ### JSON_REPLACE(@3, '$.day', 'monday'), + // ### '$.favorite_color') /* JSON meta=4 nullable=1 is_null=0 */ + // ### UPDATE `vt_commerce`.`customer` + // ### WHERE + // ### @1=2 /* LONGINT meta=0 nullable=0 is_null=0 */ + // ### @2='bob@domain.com' /* VARSTRING(128) meta=128 nullable=1 is_null=0 */ + // ### @3='{"day": "friday", "role": "manager", "color": "red", "salary": 99, "favorite_color": "black"}' /* JSON meta=4 nullable=1 is_null=0 */ + // ### SET + // ### @1=2 /* LONGINT meta=0 nullable=0 is_null=0 */ + // ### @2='bob@domain.com' /* VARSTRING(128) meta=128 nullable=1 is_null=0 */ + // ### @3=JSON_REMOVE( + // ### JSON_REPLACE(@3, '$.day', 'monday'), + // ### '$.favorite_color') /* JSON meta=4 nullable=1 is_null=0 */ + // ### UPDATE `vt_commerce`.`customer` + // ### WHERE + // ### @1=3 /* LONGINT meta=0 nullable=0 is_null=0 */ + // ### @2='charlie@domain.com' /* VARSTRING(128) meta=128 nullable=1 is_null=0 */ + // ### @3='{"day": "friday", "role": "manager", "color": "red", "salary": 99, "favorite_color": "black"}' /* JSON meta=4 nullable=1 is_null=0 */ + // ### SET + // ### @1=3 /* LONGINT meta=0 nullable=0 is_null=0 */ + // ### @2='charlie@domain.com' /* VARSTRING(128) meta=128 nullable=1 is_null=0 */ + // ### @3=JSON_REMOVE( + // ### JSON_REPLACE(@3, '$.day', 'monday'), + // ### '$.favorite_color') /* JSON meta=4 nullable=1 is_null=0 */ + // ### UPDATE `vt_commerce`.`customer` + // ### WHERE + // ### @1=4 /* LONGINT meta=0 nullable=0 is_null=0 */ + // ### @2='dan@domain.com' /* VARSTRING(128) meta=128 nullable=1 is_null=0 */ + // ### @3='{"day": "friday", "role": "manager", "color": "red", "salary": 99, "favorite_color": "black"}' /* JSON meta=4 nullable=1 is_null=0 */ + // ### SET + // ### @1=4 /* LONGINT meta=0 nullable=0 is_null=0 */ + // ### @2='dan@domain.com' /* VARSTRING(128) meta=128 nullable=1 is_null=0 */ + // ### @3=JSON_REMOVE( + // ### JSON_REPLACE(@3, '$.day', 'monday'), + // ### '$.favorite_color') /* JSON meta=4 nullable=1 is_null=0 */ + // ### UPDATE `vt_commerce`.`customer` + // ### WHERE + // ### @1=5 /* LONGINT meta=0 nullable=0 is_null=0 */ + // ### @2='eve@domain.com' /* VARSTRING(128) meta=128 nullable=1 is_null=0 */ + // ### @3='{"day": "friday", "role": "manager", "color": "red", "salary": 100, "favorite_color": "black"}' /* JSON meta=4 nullable=1 is_null=0 */ + // ### SET + // ### @1=5 /* LONGINT meta=0 nullable=0 is_null=0 */ + // ### @2='eve@domain.com' /* VARSTRING(128) meta=128 nullable=1 is_null=0 */ + // ### @3=JSON_REMOVE( + // ### JSON_REPLACE(@3, '$.day', 'monday'), + // ### '$.favorite_color') /* JSON meta=4 nullable=1 is_null=0 */ + rawEvent: []byte{ + 227, 240, 86, 103, 39, 74, 58, 208, 33, 225, 3, 0, 0, 173, 122, 0, 0, 0, 0, 176, 0, 0, 0, 0, 0, 1, 0, 2, 0, 3, 255, 255, 0, 1, 0, 0, 0, 0, + 0, 0, 0, 16, 97, 108, 105, 99, 101, 64, 100, 111, 109, 97, 105, 110, 46, 99, 111, 109, 97, 0, 0, 0, 0, 5, 0, 96, 0, 39, 0, 3, 0, 42, 0, 4, + 0, 46, 0, 5, 0, 51, 0, 6, 0, 57, 0, 14, 0, 12, 71, 0, 12, 78, 0, 12, 86, 0, 5, 100, 0, 12, 90, 0, 100, 97, 121, 114, 111, 108, 101, 99, 111, 108, 111, 114, 115, 97, 108, 97, 114, 121, 102, 97, 118, 111, 114, 105, 116, 101, 95, 99, 111, 108, 111, 114, 6, 102, 114, 105, 100, 97, 121, 7, 109, 97, 110, 97, 103, 101, 114, 3, 114, 101, 100, 5, 98, 108, 97, 99, 107, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 16, 97, 108, 105, 99, 101, 64, 100, 111, 109, 97, 105, 110, 46, 99, 111, 109, 34, 0, 0, 0, 0, 5, 36, 46, 100, 97, 121, 8, 12, 6, 109, 111, 110, 100, 97, 121, 2, 16, 36, 46, 102, 97, 118, 111, 114, 105, 116, 101, 95, 99, 111, 108, 111, 114, 0, 2, 0, 0, 0, 0, 0, 0, 0, 14, 98, 111, 98, 64, 100, 111, 109, 97, 105, 110, 46, 99, 111, 109, 97, 0, 0, 0, 0, 5, 0, 96, 0, 39, 0, 3, 0, 42, 0, 4, 0, 46, 0, 5, 0, 51, 0, 6, 0, 57, 0, 14, 0, 12, 71, 0, 12, 78, 0, 12, 86, 0, 5, 99, 0, 12, 90, 0, 100, 97, 121, 114, 111, 108, 101, 99, 111, 108, 111, 114, 115, 97, 108, 97, 114, 121, 102, 97, 118, 111, 114, 105, 116, 101, 95, 99, 111, 108, 111, 114, 6, 102, 114, 105, 100, 97, 121, 7, 109, 97, 110, 97, 103, 101, 114, 3, 114, 101, 100, 5, 98, 108, 97, 99, 107, 1, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 14, 98, 111, 98, 64, 100, 111, 109, 97, 105, 110, 46, 99, 111, 109, 34, 0, 0, 0, 0, 5, 36, 46, 100, 97, 121, 8, 12, 6, 109, 111, 110, 100, 97, 121, 2, 16, 36, 46, 102, 97, 118, 111, 114, 105, 116, 101, 95, 99, 111, 108, 111, 114, 0, 3, 0, 0, 0, 0, 0, 0, 0, 18, 99, 104, 97, 114, 108, 105, 101, 64, 100, 111, 109, 97, 105, 110, 46, 99, 111, 109, 97, 0, 0, 0, 0, 5, 0, 96, 0, 39, 0, 3, 0, 42, 0, 4, 0, 46, 0, 5, 0, 51, 0, 6, 0, 57, 0, 14, 0, 12, 71, 0, 12, 78, 0, 12, 86, 0, 5, 99, 0, 12, 90, 0, 100, 97, 121, 114, 111, 108, 101, 99, 111, 108, 111, 114, 115, 97, 108, 97, 114, 121, 102, 97, 118, 111, 114, 105, 116, 101, 95, 99, 111, 108, 111, 114, 6, 102, 114, 105, 100, 97, 121, 7, 109, 97, 110, 97, 103, 101, 114, 3, 114, 101, 100, 5, 98, 108, 97, 99, 107, 1, 1, 0, 3, 0, 0, 0, 0, 0, 0, 0, 18, 99, 104, 97, 114, 108, 105, 101, 64, 100, 111, 109, 97, 105, 110, 46, 99, 111, 109, 34, 0, 0, 0, 0, 5, 36, 46, 100, 97, 121, 8, 12, 6, 109, 111, 110, 100, 97, 121, 2, 16, 36, 46, 102, 97, 118, 111, 114, 105, 116, 101, 95, 99, 111, 108, 111, 114, 0, 4, 0, 0, 0, 0, 0, 0, 0, 14, 100, 97, 110, 64, 100, 111, 109, 97, 105, 110, 46, 99, 111, 109, 97, 0, 0, 0, 0, 5, 0, 96, 0, 39, 0, 3, 0, 42, 0, 4, 0, 46, 0, 5, 0, 51, 0, 6, 0, 57, 0, 14, 0, 12, 71, 0, 12, 78, 0, 12, 86, 0, 5, 99, 0, 12, 90, 0, 100, 97, 121, 114, 111, 108, 101, 99, 111, 108, 111, 114, 115, 97, 108, 97, 114, 121, 102, 97, 118, 111, 114, 105, 116, 101, 95, 99, 111, 108, 111, 114, 6, 102, 114, 105, 100, 97, 121, 7, 109, 97, 110, 97, 103, 101, 114, 3, 114, 101, 100, 5, 98, 108, 97, 99, 107, 1, 1, 0, 4, 0, 0, 0, 0, 0, 0, 0, 14, 100, 97, 110, 64, 100, 111, 109, 97, 105, 110, 46, 99, 111, 109, 34, 0, 0, 0, 0, 5, 36, 46, 100, 97, 121, 8, 12, 6, 109, 111, 110, 100, 97, 121, 2, 16, 36, 46, 102, 97, 118, 111, 114, 105, 116, 101, 95, 99, 111, 108, 111, 114, 0, 5, 0, 0, 0, 0, 0, 0, 0, 14, 101, 118, 101, 64, 100, 111, 109, 97, 105, 110, 46, 99, 111, 109, 97, 0, 0, 0, 0, 5, 0, 96, 0, 39, 0, 3, 0, 42, 0, 4, 0, 46, 0, 5, 0, 51, 0, 6, 0, 57, 0, 14, 0, 12, 71, 0, 12, 78, 0, 12, 86, 0, 5, 100, 0, 12, 90, 0, 100, 97, 121, 114, 111, 108, 101, 99, 111, 108, 111, 114, 115, 97, 108, 97, 114, 121, 102, 97, 118, 111, 114, 105, 116, 101, 95, 99, 111, 108, 111, 114, 6, 102, 114, 105, 100, 97, 121, 7, 109, 97, 110, 97, 103, 101, 114, 3, 114, 101, 100, 5, 98, 108, 97, 99, 107, 1, 1, 0, 5, 0, 0, 0, 0, 0, 0, 0, 14, 101, 118, 101, 64, 100, 111, 109, 97, 105, 110, 46, 99, 111, 109, 34, 0, 0, 0, 0, 5, 36, 46, 100, 97, 121, 8, 12, 6, 109, 111, 110, 100, 97, 121, 2, 16, 36, 46, 102, 97, 118, 111, 114, 105, 116, 101, 95, 99, 111, 108, 111, 114, + }, + want: "JSON_REMOVE(JSON_REPLACE(%s, _utf8mb4'$.day', _utf8mb4\"monday\"), _utf8mb4'$.favorite_color')", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + mysql56PartialUpdateRowEvent := NewMysql56BinlogEvent(tc.rawEvent) + require.True(t, mysql56PartialUpdateRowEvent.IsPartialUpdateRows()) + + ev, err := mysql56PartialUpdateRowEvent.Rows(format, tm) + require.NoError(t, err) + + assert.Equal(t, 5, len(ev.Rows)) + require.NoError(t, err) + for i := range ev.Rows { + vals, err := ev.StringValuesForTests(tm, i) + require.NoError(t, err) + // The third column is the JSON column. + require.Equal(t, `JSON_REMOVE(JSON_REPLACE(%s, _utf8mb4'$.day', _utf8mb4"monday"), _utf8mb4'$.favorite_color')`, vals[2]) + t.Logf("Rows: %v", vals) + } + }) } } diff --git a/go/mysql/binlog_event_rbr.go b/go/mysql/binlog_event_rbr.go index 0e0ad0c458f..259d0441751 100644 --- a/go/mysql/binlog_event_rbr.go +++ b/go/mysql/binlog_event_rbr.go @@ -287,6 +287,9 @@ func (ev binlogEvent) Rows(f BinlogFormat, tm *TableMap) (Rows, error) { typ == eDeleteRowsEventV1 || typ == eDeleteRowsEventV2 hasData := typ == eWriteRowsEventV1 || typ == eWriteRowsEventV2 || typ == eUpdateRowsEventV1 || typ == ePartialUpdateRowsEvent || typ == eUpdateRowsEventV2 + //if typ == ePartialUpdateRowsEvent { + // log.Errorf("DEBUG: PartialUpdateRowsEvent bytes: %v", ev.Bytes()) + //} result := Rows{} pos := 6 From bd3aa673ec8e1a43916665c63094da366e8d0a0e Mon Sep 17 00:00:00 2001 From: Matt Lord Date: Mon, 9 Dec 2024 11:14:43 -0500 Subject: [PATCH 08/39] Build out unit test Signed-off-by: Matt Lord --- go/mysql/binlog/binlog_json.go | 10 +- go/mysql/binlog_event_mysql56_test.go | 224 +++++++++++++++++++++++++- go/mysql/binlog_event_rbr.go | 3 - 3 files changed, 227 insertions(+), 10 deletions(-) diff --git a/go/mysql/binlog/binlog_json.go b/go/mysql/binlog/binlog_json.go index 6f02456e495..204e5186be4 100644 --- a/go/mysql/binlog/binlog_json.go +++ b/go/mysql/binlog/binlog_json.go @@ -75,18 +75,21 @@ func ParseBinaryJSON(data []byte) (*json.Value, error) { // diff representation to an SQL expression. func ParseBinaryJSONDiff(data []byte) (sqltypes.Value, error) { diff := bytes.Buffer{} + // Reasonable estimate of the space we'll need to build the SQL + // expression in order to try and avoid reallocations w/o + // overallocating too much. diff.Grow(int(float32(len(data)) * 1.5)) pos := 0 outer := false innerStr := "" for pos < len(data) { + opType := jsonDiffOp(data[pos]) + pos++ if outer { innerStr = diff.String() diff.Reset() } - opType := jsonDiffOp(data[pos]) - pos++ switch opType { case jsonDiffOpReplace: diff.WriteString("JSON_REPLACE(") @@ -114,6 +117,7 @@ func ParseBinaryJSONDiff(data []byte) (sqltypes.Value, error) { if opType == jsonDiffOpRemove { // No value for remove diff.WriteString(")") } else { + diff.WriteString(", ") valueLen, readTo := readVariableLength(data, pos) pos = readTo value, err := ParseBinaryJSON(data[pos : pos+valueLen]) @@ -122,7 +126,7 @@ func ParseBinaryJSONDiff(data []byte) (sqltypes.Value, error) { } pos += valueLen if value.Type() == json.TypeString { - diff.WriteString(", _utf8mb4") + diff.WriteString("_utf8mb4") } diff.WriteString(fmt.Sprintf("%s)", value)) } diff --git a/go/mysql/binlog_event_mysql56_test.go b/go/mysql/binlog_event_mysql56_test.go index b7725e69691..28c359d1430 100644 --- a/go/mysql/binlog_event_mysql56_test.go +++ b/go/mysql/binlog_event_mysql56_test.go @@ -260,6 +260,13 @@ func TestMySQL56PartialUpdateRowsEvent(t *testing.T) { HeaderLength: 19, ChecksumAlgorithm: 1, } + // This is from the following table structure: + // CREATE TABLE `customer` ( + // `customer_id` bigint NOT NULL AUTO_INCREMENT, + // `email` varbinary(128) DEFAULT NULL, + // `jd` json DEFAULT NULL, + // PRIMARY KEY (`customer_id`) + // ) tm := &TableMap{ Flags: 1, Database: "vt_commerce", @@ -276,8 +283,119 @@ func TestMySQL56PartialUpdateRowsEvent(t *testing.T) { testCases := []struct { name string rawEvent []byte + numRows int want string }{ + { + name: "INSERT", + // The mysqlbinlog -vvv --base64-output=decode-rows output for the following event: + // ### UPDATE `vt_commerce`.`customer` + // ### WHERE + // ### @1=1 /* LONGINT meta=0 nullable=0 is_null=0 */ + // ### @2='alice@domain.com' /* VARSTRING(128) meta=128 nullable=1 is_null=0 */ + // ### @3='{"salary": 100}' /* JSON meta=4 nullable=1 is_null=0 */ + // ### SET + // ### @1=1 /* LONGINT meta=0 nullable=0 is_null=0 */ + // ### @2='alice@domain.com' /* VARSTRING(128) meta=128 nullable=1 is_null=0 */ + // ### @3=JSON_INSERT(@3, '$.role', 'manager') /* JSON meta=4 nullable=1 is_null=0 */ + // ### UPDATE `vt_commerce`.`customer` + // ### WHERE + // ### @1=2 /* LONGINT meta=0 nullable=0 is_null=0 */ + // ### @2='bob@domain.com' /* VARSTRING(128) meta=128 nullable=1 is_null=0 */ + // ### @3='{"salary": 99}' /* JSON meta=4 nullable=1 is_null=0 */ + // ### SET + // ### @1=2 /* LONGINT meta=0 nullable=0 is_null=0 */ + // ### @2='bob@domain.com' /* VARSTRING(128) meta=128 nullable=1 is_null=0 */ + // ### @3=JSON_INSERT(@3, '$.role', 'manager') /* JSON meta=4 nullable=1 is_null=0 */ + // ### UPDATE `vt_commerce`.`customer` + // ### WHERE + // ### @1=3 /* LONGINT meta=0 nullable=0 is_null=0 */ + // ### @2='charlie@domain.com' /* VARSTRING(128) meta=128 nullable=1 is_null=0 */ + // ### @3='{"salary": 99}' /* JSON meta=4 nullable=1 is_null=0 */ + // ### SET + // ### @1=3 /* LONGINT meta=0 nullable=0 is_null=0 */ + // ### @2='charlie@domain.com' /* VARSTRING(128) meta=128 nullable=1 is_null=0 */ + // ### @3=JSON_INSERT(@3, '$.role', 'manager') /* JSON meta=4 nullable=1 is_null=0 */ + // ### UPDATE `vt_commerce`.`customer` + // ### WHERE + // ### @1=4 /* LONGINT meta=0 nullable=0 is_null=0 */ + // ### @2='dan@domain.com' /* VARSTRING(128) meta=128 nullable=1 is_null=0 */ + // ### @3='{"salary": 99}' /* JSON meta=4 nullable=1 is_null=0 */ + // ### SET + // ### @1=4 /* LONGINT meta=0 nullable=0 is_null=0 */ + // ### @2='dan@domain.com' /* VARSTRING(128) meta=128 nullable=1 is_null=0 */ + // ### @3=JSON_INSERT(@3, '$.role', 'manager') /* JSON meta=4 nullable=1 is_null=0 */ + // ### UPDATE `vt_commerce`.`customer` + // ### WHERE + // ### @1=5 /* LONGINT meta=0 nullable=0 is_null=0 */ + // ### @2='eve@domain.com' /* VARSTRING(128) meta=128 nullable=1 is_null=0 */ + // ### @3='{"salary": 100}' /* JSON meta=4 nullable=1 is_null=0 */ + // ### SET + // ### @1=5 /* LONGINT meta=0 nullable=0 is_null=0 */ + // ### @2='eve@domain.com' /* VARSTRING(128) meta=128 nullable=1 is_null=0 */ + // ### @3=JSON_INSERT(@3, '$.role', 'manager') /* JSON meta=4 nullable=1 is_null=0 */ + rawEvent: []byte{ + 196, 19, 87, 103, 39, 47, 142, 143, 12, 6, 2, 0, 0, 229, 104, 0, 0, 0, 0, 176, 0, 0, 0, 0, 0, 1, 0, 2, 0, 3, 255, 255, 0, 1, 0, 0, 0, 0, 0, + 0, 0, 16, 97, 108, 105, 99, 101, 64, 100, 111, 109, 97, 105, 110, 46, 99, 111, 109, 18, 0, 0, 0, 0, 1, 0, 17, 0, 11, 0, 6, 0, 5, 100, 0, 115, + 97, 108, 97, 114, 121, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 16, 97, 108, 105, 99, 101, 64, 100, 111, 109, 97, 105, 110, 46, 99, 111, 109, 18, 0, + 0, 0, 1, 6, 36, 46, 114, 111, 108, 101, 9, 12, 7, 109, 97, 110, 97, 103, 101, 114, 0, 2, 0, 0, 0, 0, 0, 0, 0, 14, 98, 111, 98, 64, 100, 111, + 109, 97, 105, 110, 46, 99, 111, 109, 18, 0, 0, 0, 0, 1, 0, 17, 0, 11, 0, 6, 0, 5, 99, 0, 115, 97, 108, 97, 114, 121, 1, 1, 0, 2, 0, 0, 0, 0, + 0, 0, 0, 14, 98, 111, 98, 64, 100, 111, 109, 97, 105, 110, 46, 99, 111, 109, 18, 0, 0, 0, 1, 6, 36, 46, 114, 111, 108, 101, 9, 12, 7, 109, 97, + 110, 97, 103, 101, 114, 0, 3, 0, 0, 0, 0, 0, 0, 0, 18, 99, 104, 97, 114, 108, 105, 101, 64, 100, 111, 109, 97, 105, 110, 46, 99, 111, 109, 18, + 0, 0, 0, 0, 1, 0, 17, 0, 11, 0, 6, 0, 5, 99, 0, 115, 97, 108, 97, 114, 121, 1, 1, 0, 3, 0, 0, 0, 0, 0, 0, 0, 18, 99, 104, 97, 114, 108, 105, + 101, 64, 100, 111, 109, 97, 105, 110, 46, 99, 111, 109, 18, 0, 0, 0, 1, 6, 36, 46, 114, 111, 108, 101, 9, 12, 7, 109, 97, 110, 97, 103, 101, + 114, 0, 4, 0, 0, 0, 0, 0, 0, 0, 14, 100, 97, 110, 64, 100, 111, 109, 97, 105, 110, 46, 99, 111, 109, 18, 0, 0, 0, 0, 1, 0, 17, 0, 11, 0, 6, 0, + 5, 99, 0, 115, 97, 108, 97, 114, 121, 1, 1, 0, 4, 0, 0, 0, 0, 0, 0, 0, 14, 100, 97, 110, 64, 100, 111, 109, 97, 105, 110, 46, 99, 111, 109, 18, + 0, 0, 0, 1, 6, 36, 46, 114, 111, 108, 101, 9, 12, 7, 109, 97, 110, 97, 103, 101, 114, 0, 5, 0, 0, 0, 0, 0, 0, 0, 14, 101, 118, 101, 64, 100, + 111, 109, 97, 105, 110, 46, 99, 111, 109, 18, 0, 0, 0, 0, 1, 0, 17, 0, 11, 0, 6, 0, 5, 100, 0, 115, 97, 108, 97, 114, 121, 1, 1, 0, 5, 0, 0, 0, + 0, 0, 0, 0, 14, 101, 118, 101, 64, 100, 111, 109, 97, 105, 110, 46, 99, 111, 109, 18, 0, 0, 0, 1, 6, 36, 46, 114, 111, 108, 101, 9, 12, 7, 109, + 97, 110, 97, 103, 101, 114, + }, + numRows: 5, + want: "JSON_INSERT(%s, _utf8mb4'$.role', _utf8mb4\"manager\")", + }, + { + // The mysqlbinlog -vvv --base64-output=decode-rows output for the following event: + // ### UPDATE `vt_commerce`.`customer` + // ### WHERE + // ### @1=1 /* LONGINT meta=0 nullable=0 is_null=0 */ + // ### @2='alice@domain.com' /* VARSTRING(128) meta=128 nullable=1 is_null=0 */ + // ### @3='{"role": "manager", "salary": 100}' /* JSON meta=4 nullable=1 is_null=0 */ + // ### SET + // ### @1=1 /* LONGINT meta=0 nullable=0 is_null=0 */ + // ### @2='alice@domain.com' /* VARSTRING(128) meta=128 nullable=1 is_null=0 */ + // ### @3=JSON_REPLACE(@3, '$.role', 'IC') /* JSON meta=4 nullable=1 is_null=0 */ + rawEvent: []byte{ + 155, 21, 87, 103, 39, 47, 142, 143, 12, 148, 0, 0, 0, 135, 106, 0, 0, 0, 0, 176, 0, 0, 0, 0, 0, 1, 0, 2, 0, 3, 255, 255, 0, 1, 0, 0, 0, 0, 0, 0, + 0, 16, 97, 108, 105, 99, 101, 64, 100, 111, 109, 97, 105, 110, 46, 99, 111, 109, 37, 0, 0, 0, 0, 2, 0, 36, 0, 18, 0, 4, 0, 22, 0, 6, 0, 12, 28, + 0, 5, 100, 0, 114, 111, 108, 101, 115, 97, 108, 97, 114, 121, 7, 109, 97, 110, 97, 103, 101, 114, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 16, 97, 108, + 105, 99, 101, 64, 100, 111, 109, 97, 105, 110, 46, 99, 111, 109, 13, 0, 0, 0, 0, 6, 36, 46, 114, 111, 108, 101, 4, 12, 2, 73, 67, + }, + name: "REPLACE", + numRows: 1, + want: "JSON_REPLACE(%s, _utf8mb4'$.role', _utf8mb4\"IC\")", + }, + { + name: "REMOVE", + // The mysqlbinlog -vvv --base64-output=decode-rows output for the following event: + // ### UPDATE `vt_commerce`.`customer` + // ### WHERE + // ### @1=2 /* LONGINT meta=0 nullable=0 is_null=0 */ + // ### @2='bob@domain.com' /* VARSTRING(128) meta=128 nullable=1 is_null=0 */ + // ### @3='{"role": "manager", "salary": 99}' /* JSON meta=4 nullable=1 is_null=0 */ + // ### SET + // ### @1=2 /* LONGINT meta=0 nullable=0 is_null=0 */ + // ### @2='bob@domain.com' /* VARSTRING(128) meta=128 nullable=1 is_null=0 */ + // ### @3=JSON_REMOVE(@3, '$.salary') /* JSON meta=4 nullable=1 is_null=0 */ + numRows: 1, + rawEvent: []byte{ + 176, 22, 87, 103, 39, 47, 142, 143, 12, 141, 0, 0, 0, 34, 108, 0, 0, 0, 0, 176, 0, 0, 0, 0, 0, 1, 0, 2, 0, 3, 255, 255, 0, 2, 0, 0, 0, 0, 0, 0, 0, + 14, 98, 111, 98, 64, 100, 111, 109, 97, 105, 110, 46, 99, 111, 109, 37, 0, 0, 0, 0, 2, 0, 36, 0, 18, 0, 4, 0, 22, 0, 6, 0, 12, 28, 0, 5, 99, 0, 114, + 111, 108, 101, 115, 97, 108, 97, 114, 121, 7, 109, 97, 110, 97, 103, 101, 114, 1, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 14, 98, 111, 98, 64, 100, 111, 109, + 97, 105, 110, 46, 99, 111, 109, 10, 0, 0, 0, 2, 8, 36, 46, 115, 97, 108, 97, 114, 121, + }, + want: "JSON_REMOVE(%s, _utf8mb4'$.salary')", + }, { name: "REMOVE and REPLACE", // The mysqlbinlog -vvv --base64-output=decode-rows output for the following event: @@ -339,9 +457,107 @@ func TestMySQL56PartialUpdateRowsEvent(t *testing.T) { rawEvent: []byte{ 227, 240, 86, 103, 39, 74, 58, 208, 33, 225, 3, 0, 0, 173, 122, 0, 0, 0, 0, 176, 0, 0, 0, 0, 0, 1, 0, 2, 0, 3, 255, 255, 0, 1, 0, 0, 0, 0, 0, 0, 0, 16, 97, 108, 105, 99, 101, 64, 100, 111, 109, 97, 105, 110, 46, 99, 111, 109, 97, 0, 0, 0, 0, 5, 0, 96, 0, 39, 0, 3, 0, 42, 0, 4, - 0, 46, 0, 5, 0, 51, 0, 6, 0, 57, 0, 14, 0, 12, 71, 0, 12, 78, 0, 12, 86, 0, 5, 100, 0, 12, 90, 0, 100, 97, 121, 114, 111, 108, 101, 99, 111, 108, 111, 114, 115, 97, 108, 97, 114, 121, 102, 97, 118, 111, 114, 105, 116, 101, 95, 99, 111, 108, 111, 114, 6, 102, 114, 105, 100, 97, 121, 7, 109, 97, 110, 97, 103, 101, 114, 3, 114, 101, 100, 5, 98, 108, 97, 99, 107, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 16, 97, 108, 105, 99, 101, 64, 100, 111, 109, 97, 105, 110, 46, 99, 111, 109, 34, 0, 0, 0, 0, 5, 36, 46, 100, 97, 121, 8, 12, 6, 109, 111, 110, 100, 97, 121, 2, 16, 36, 46, 102, 97, 118, 111, 114, 105, 116, 101, 95, 99, 111, 108, 111, 114, 0, 2, 0, 0, 0, 0, 0, 0, 0, 14, 98, 111, 98, 64, 100, 111, 109, 97, 105, 110, 46, 99, 111, 109, 97, 0, 0, 0, 0, 5, 0, 96, 0, 39, 0, 3, 0, 42, 0, 4, 0, 46, 0, 5, 0, 51, 0, 6, 0, 57, 0, 14, 0, 12, 71, 0, 12, 78, 0, 12, 86, 0, 5, 99, 0, 12, 90, 0, 100, 97, 121, 114, 111, 108, 101, 99, 111, 108, 111, 114, 115, 97, 108, 97, 114, 121, 102, 97, 118, 111, 114, 105, 116, 101, 95, 99, 111, 108, 111, 114, 6, 102, 114, 105, 100, 97, 121, 7, 109, 97, 110, 97, 103, 101, 114, 3, 114, 101, 100, 5, 98, 108, 97, 99, 107, 1, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 14, 98, 111, 98, 64, 100, 111, 109, 97, 105, 110, 46, 99, 111, 109, 34, 0, 0, 0, 0, 5, 36, 46, 100, 97, 121, 8, 12, 6, 109, 111, 110, 100, 97, 121, 2, 16, 36, 46, 102, 97, 118, 111, 114, 105, 116, 101, 95, 99, 111, 108, 111, 114, 0, 3, 0, 0, 0, 0, 0, 0, 0, 18, 99, 104, 97, 114, 108, 105, 101, 64, 100, 111, 109, 97, 105, 110, 46, 99, 111, 109, 97, 0, 0, 0, 0, 5, 0, 96, 0, 39, 0, 3, 0, 42, 0, 4, 0, 46, 0, 5, 0, 51, 0, 6, 0, 57, 0, 14, 0, 12, 71, 0, 12, 78, 0, 12, 86, 0, 5, 99, 0, 12, 90, 0, 100, 97, 121, 114, 111, 108, 101, 99, 111, 108, 111, 114, 115, 97, 108, 97, 114, 121, 102, 97, 118, 111, 114, 105, 116, 101, 95, 99, 111, 108, 111, 114, 6, 102, 114, 105, 100, 97, 121, 7, 109, 97, 110, 97, 103, 101, 114, 3, 114, 101, 100, 5, 98, 108, 97, 99, 107, 1, 1, 0, 3, 0, 0, 0, 0, 0, 0, 0, 18, 99, 104, 97, 114, 108, 105, 101, 64, 100, 111, 109, 97, 105, 110, 46, 99, 111, 109, 34, 0, 0, 0, 0, 5, 36, 46, 100, 97, 121, 8, 12, 6, 109, 111, 110, 100, 97, 121, 2, 16, 36, 46, 102, 97, 118, 111, 114, 105, 116, 101, 95, 99, 111, 108, 111, 114, 0, 4, 0, 0, 0, 0, 0, 0, 0, 14, 100, 97, 110, 64, 100, 111, 109, 97, 105, 110, 46, 99, 111, 109, 97, 0, 0, 0, 0, 5, 0, 96, 0, 39, 0, 3, 0, 42, 0, 4, 0, 46, 0, 5, 0, 51, 0, 6, 0, 57, 0, 14, 0, 12, 71, 0, 12, 78, 0, 12, 86, 0, 5, 99, 0, 12, 90, 0, 100, 97, 121, 114, 111, 108, 101, 99, 111, 108, 111, 114, 115, 97, 108, 97, 114, 121, 102, 97, 118, 111, 114, 105, 116, 101, 95, 99, 111, 108, 111, 114, 6, 102, 114, 105, 100, 97, 121, 7, 109, 97, 110, 97, 103, 101, 114, 3, 114, 101, 100, 5, 98, 108, 97, 99, 107, 1, 1, 0, 4, 0, 0, 0, 0, 0, 0, 0, 14, 100, 97, 110, 64, 100, 111, 109, 97, 105, 110, 46, 99, 111, 109, 34, 0, 0, 0, 0, 5, 36, 46, 100, 97, 121, 8, 12, 6, 109, 111, 110, 100, 97, 121, 2, 16, 36, 46, 102, 97, 118, 111, 114, 105, 116, 101, 95, 99, 111, 108, 111, 114, 0, 5, 0, 0, 0, 0, 0, 0, 0, 14, 101, 118, 101, 64, 100, 111, 109, 97, 105, 110, 46, 99, 111, 109, 97, 0, 0, 0, 0, 5, 0, 96, 0, 39, 0, 3, 0, 42, 0, 4, 0, 46, 0, 5, 0, 51, 0, 6, 0, 57, 0, 14, 0, 12, 71, 0, 12, 78, 0, 12, 86, 0, 5, 100, 0, 12, 90, 0, 100, 97, 121, 114, 111, 108, 101, 99, 111, 108, 111, 114, 115, 97, 108, 97, 114, 121, 102, 97, 118, 111, 114, 105, 116, 101, 95, 99, 111, 108, 111, 114, 6, 102, 114, 105, 100, 97, 121, 7, 109, 97, 110, 97, 103, 101, 114, 3, 114, 101, 100, 5, 98, 108, 97, 99, 107, 1, 1, 0, 5, 0, 0, 0, 0, 0, 0, 0, 14, 101, 118, 101, 64, 100, 111, 109, 97, 105, 110, 46, 99, 111, 109, 34, 0, 0, 0, 0, 5, 36, 46, 100, 97, 121, 8, 12, 6, 109, 111, 110, 100, 97, 121, 2, 16, 36, 46, 102, 97, 118, 111, 114, 105, 116, 101, 95, 99, 111, 108, 111, 114, + 0, 46, 0, 5, 0, 51, 0, 6, 0, 57, 0, 14, 0, 12, 71, 0, 12, 78, 0, 12, 86, 0, 5, 100, 0, 12, 90, 0, 100, 97, 121, 114, 111, 108, 101, 99, 111, + 108, 111, 114, 115, 97, 108, 97, 114, 121, 102, 97, 118, 111, 114, 105, 116, 101, 95, 99, 111, 108, 111, 114, 6, 102, 114, 105, 100, 97, 121, + 7, 109, 97, 110, 97, 103, 101, 114, 3, 114, 101, 100, 5, 98, 108, 97, 99, 107, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 16, 97, 108, 105, 99, 101, + 64, 100, 111, 109, 97, 105, 110, 46, 99, 111, 109, 34, 0, 0, 0, 0, 5, 36, 46, 100, 97, 121, 8, 12, 6, 109, 111, 110, 100, 97, 121, 2, 16, 36, + 46, 102, 97, 118, 111, 114, 105, 116, 101, 95, 99, 111, 108, 111, 114, 0, 2, 0, 0, 0, 0, 0, 0, 0, 14, 98, 111, 98, 64, 100, 111, 109, 97, + 105, 110, 46, 99, 111, 109, 97, 0, 0, 0, 0, 5, 0, 96, 0, 39, 0, 3, 0, 42, 0, 4, 0, 46, 0, 5, 0, 51, 0, 6, 0, 57, 0, 14, 0, 12, 71, 0, 12, 78, + 0, 12, 86, 0, 5, 99, 0, 12, 90, 0, 100, 97, 121, 114, 111, 108, 101, 99, 111, 108, 111, 114, 115, 97, 108, 97, 114, 121, 102, 97, 118, 111, + 114, 105, 116, 101, 95, 99, 111, 108, 111, 114, 6, 102, 114, 105, 100, 97, 121, 7, 109, 97, 110, 97, 103, 101, 114, 3, 114, 101, 100, 5, 98, + 108, 97, 99, 107, 1, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 14, 98, 111, 98, 64, 100, 111, 109, 97, 105, 110, 46, 99, 111, 109, 34, 0, 0, 0, 0, 5, 36, + 46, 100, 97, 121, 8, 12, 6, 109, 111, 110, 100, 97, 121, 2, 16, 36, 46, 102, 97, 118, 111, 114, 105, 116, 101, 95, 99, 111, 108, 111, 114, 0, + 3, 0, 0, 0, 0, 0, 0, 0, 18, 99, 104, 97, 114, 108, 105, 101, 64, 100, 111, 109, 97, 105, 110, 46, 99, 111, 109, 97, 0, 0, 0, 0, 5, 0, 96, 0, + 39, 0, 3, 0, 42, 0, 4, 0, 46, 0, 5, 0, 51, 0, 6, 0, 57, 0, 14, 0, 12, 71, 0, 12, 78, 0, 12, 86, 0, 5, 99, 0, 12, 90, 0, 100, 97, 121, 114, + 111, 108, 101, 99, 111, 108, 111, 114, 115, 97, 108, 97, 114, 121, 102, 97, 118, 111, 114, 105, 116, 101, 95, 99, 111, 108, 111, 114, 6, 102, + 114, 105, 100, 97, 121, 7, 109, 97, 110, 97, 103, 101, 114, 3, 114, 101, 100, 5, 98, 108, 97, 99, 107, 1, 1, 0, 3, 0, 0, 0, 0, 0, 0, 0, 18, + 99, 104, 97, 114, 108, 105, 101, 64, 100, 111, 109, 97, 105, 110, 46, 99, 111, 109, 34, 0, 0, 0, 0, 5, 36, 46, 100, 97, 121, 8, 12, 6, 109, + 111, 110, 100, 97, 121, 2, 16, 36, 46, 102, 97, 118, 111, 114, 105, 116, 101, 95, 99, 111, 108, 111, 114, 0, 4, 0, 0, 0, 0, 0, 0, 0, 14, 100, + 97, 110, 64, 100, 111, 109, 97, 105, 110, 46, 99, 111, 109, 97, 0, 0, 0, 0, 5, 0, 96, 0, 39, 0, 3, 0, 42, 0, 4, 0, 46, 0, 5, 0, 51, 0, 6, 0, + 57, 0, 14, 0, 12, 71, 0, 12, 78, 0, 12, 86, 0, 5, 99, 0, 12, 90, 0, 100, 97, 121, 114, 111, 108, 101, 99, 111, 108, 111, 114, 115, 97, 108, + 97, 114, 121, 102, 97, 118, 111, 114, 105, 116, 101, 95, 99, 111, 108, 111, 114, 6, 102, 114, 105, 100, 97, 121, 7, 109, 97, 110, 97, 103, + 101, 114, 3, 114, 101, 100, 5, 98, 108, 97, 99, 107, 1, 1, 0, 4, 0, 0, 0, 0, 0, 0, 0, 14, 100, 97, 110, 64, 100, 111, 109, 97, 105, 110, 46, + 99, 111, 109, 34, 0, 0, 0, 0, 5, 36, 46, 100, 97, 121, 8, 12, 6, 109, 111, 110, 100, 97, 121, 2, 16, 36, 46, 102, 97, 118, 111, 114, 105, 116, + 101, 95, 99, 111, 108, 111, 114, 0, 5, 0, 0, 0, 0, 0, 0, 0, 14, 101, 118, 101, 64, 100, 111, 109, 97, 105, 110, 46, 99, 111, 109, 97, 0, 0, 0, + 0, 5, 0, 96, 0, 39, 0, 3, 0, 42, 0, 4, 0, 46, 0, 5, 0, 51, 0, 6, 0, 57, 0, 14, 0, 12, 71, 0, 12, 78, 0, 12, 86, 0, 5, 100, 0, 12, 90, 0, 100, + 97, 121, 114, 111, 108, 101, 99, 111, 108, 111, 114, 115, 97, 108, 97, 114, 121, 102, 97, 118, 111, 114, 105, 116, 101, 95, 99, 111, 108, 111, + 114, 6, 102, 114, 105, 100, 97, 121, 7, 109, 97, 110, 97, 103, 101, 114, 3, 114, 101, 100, 5, 98, 108, 97, 99, 107, 1, 1, 0, 5, 0, 0, 0, 0, 0, + 0, 0, 14, 101, 118, 101, 64, 100, 111, 109, 97, 105, 110, 46, 99, 111, 109, 34, 0, 0, 0, 0, 5, 36, 46, 100, 97, 121, 8, 12, 6, 109, 111, 110, + 100, 97, 121, 2, 16, 36, 46, 102, 97, 118, 111, 114, 105, 116, 101, 95, 99, 111, 108, 111, 114, + }, + numRows: 5, + want: "JSON_REMOVE(JSON_REPLACE(%s, _utf8mb4'$.day', _utf8mb4\"monday\"), _utf8mb4'$.favorite_color')", + }, + { + name: "INSERT and REMOVE and REPLACE", + // The mysqlbinlog -vvv --base64-output=decode-rows output for the following event: + // ### UPDATE `vt_commerce`.`customer` + // ### WHERE + // ### @1=3 /* LONGINT meta=0 nullable=0 is_null=0 */ + // ### @2='charlie@domain.com' /* VARSTRING(128) meta=128 nullable=1 is_null=0 */ + // ### @3='{"day": "monday", "role": "manager", "salary": 99, "favorite_color": "red"}' /* JSON meta=4 nullable=1 is_null=0 */ + // ### SET + // ### @1=3 /* LONGINT meta=0 nullable=0 is_null=0 */ + // ### @2='charlie@domain.com' /* VARSTRING(128) meta=128 nullable=1 is_null=0 */ + // ### @3=JSON_INSERT( + // ### JSON_REMOVE( + // ### JSON_REPLACE(@3, '$.day', 'tuesday'), + // ### '$.favorite_color'), + // ### '$.hobby', 'skiing') /* JSON meta=4 nullable=1 is_null=0 */ + rawEvent: []byte{ + 48, 25, 87, 103, 39, 47, 142, 143, 12, 234, 0, 0, 0, 0, 117, 0, 0, 0, 0, 176, 0, 0, 0, 0, 0, 1, 0, 2, 0, 3, 255, 255, 0, 3, 0, 0, 0, 0, 0, 0, 0, + 18, 99, 104, 97, 114, 108, 105, 101, 64, 100, 111, 109, 97, 105, 110, 46, 99, 111, 109, 79, 0, 0, 0, 0, 4, 0, 78, 0, 32, 0, 3, 0, 35, 0, 4, 0, + 39, 0, 6, 0, 45, 0, 14, 0, 12, 59, 0, 12, 66, 0, 5, 99, 0, 12, 74, 0, 100, 97, 121, 114, 111, 108, 101, 115, 97, 108, 97, 114, 121, 102, 97, + 118, 111, 114, 105, 116, 101, 95, 99, 111, 108, 111, 114, 6, 109, 111, 110, 100, 97, 121, 7, 109, 97, 110, 97, 103, 101, 114, 3, 114, 101, 100, + 1, 1, 0, 3, 0, 0, 0, 0, 0, 0, 0, 18, 99, 104, 97, 114, 108, 105, 101, 64, 100, 111, 109, 97, 105, 110, 46, 99, 111, 109, 53, 0, 0, 0, 0, 5, 36, + 46, 100, 97, 121, 9, 12, 7, 116, 117, 101, 115, 100, 97, 121, 2, 16, 36, 46, 102, 97, 118, 111, 114, 105, 116, 101, 95, 99, 111, 108, 111, 114, + 1, 7, 36, 46, 104, 111, 98, 98, 121, 8, 12, 6, 115, 107, 105, 105, 110, 103, + }, + numRows: 1, + want: "JSON_INSERT(JSON_REMOVE(JSON_REPLACE(%s, _utf8mb4'$.day', _utf8mb4\"tuesday\"), _utf8mb4'$.favorite_color'), _utf8mb4'$.hobby', _utf8mb4\"skiing\")", + }, + { + name: "REPLACE with null", + // The mysqlbinlog -vvv --base64-output=decode-rows output for the following event: + // ### UPDATE `vt_commerce`.`customer` + // ### WHERE + // ### @1=4 /* LONGINT meta=0 nullable=0 is_null=0 */ + // ### @2='dan@domain.com' /* VARSTRING(128) meta=128 nullable=1 is_null=0 */ + // ### @3='{"role": "manager", "salary": 99}' /* JSON meta=4 nullable=1 is_null=0 */ + // ### SET + // ### @1=4 /* LONGINT meta=0 nullable=0 is_null=0 */ + // ### @2='dan@domain.com' /* VARSTRING(128) meta=128 nullable=1 is_null=0 */ + // ### @3=JSON_REPLACE(@3, '$.salary', null) /* JSON meta=4 nullable=1 is_null=0 * + rawEvent: []byte{ + 148, 26, 87, 103, 39, 47, 142, 143, 12, 144, 0, 0, 0, 158, 118, 0, 0, 0, 0, 176, 0, 0, 0, 0, 0, 1, 0, 2, 0, 3, 255, 255, 0, 4, 0, 0, 0, 0, 0, 0, 0, + 14, 100, 97, 110, 64, 100, 111, 109, 97, 105, 110, 46, 99, 111, 109, 37, 0, 0, 0, 0, 2, 0, 36, 0, 18, 0, 4, 0, 22, 0, 6, 0, 12, 28, 0, 5, 99, 0, + 114, 111, 108, 101, 115, 97, 108, 97, 114, 121, 7, 109, 97, 110, 97, 103, 101, 114, 1, 1, 0, 4, 0, 0, 0, 0, 0, 0, 0, 14, 100, 97, 110, 64, 100, 111, + 109, 97, 105, 110, 46, 99, 111, 109, 13, 0, 0, 0, 0, 8, 36, 46, 115, 97, 108, 97, 114, 121, 2, 4, 0, + }, + numRows: 1, + want: "JSON_REPLACE(%s, _utf8mb4'$.salary', null)", + }, + { + name: "REPLACE 2 paths", + // The mysqlbinlog -vvv --base64-output=decode-rows output for the following event: + // ### UPDATE `vt_commerce`.`customer` + // ### WHERE + // ### @1=4 /* LONGINT meta=0 nullable=0 is_null=0 */ + // ### @2='dan@domain.com' /* VARSTRING(128) meta=128 nullable=1 is_null=0 */ + // ### @3='{"role": "manager", "salary": null}' /* JSON meta=4 nullable=1 is_null=0 */ + // ### SET + // ### @1=4 /* LONGINT meta=0 nullable=0 is_null=0 */ + // ### @2='dan@domain.com' /* VARSTRING(128) meta=128 nullable=1 is_null=0 */ + // ### @3=JSON_REPLACE(@3, '$.salary', 110, + // ### '$.role', 'IC') /* JSON meta=4 nullable=1 is_null=0 */ + rawEvent: []byte{ + 32, 32, 87, 103, 39, 26, 45, 78, 117, 158, 0, 0, 0, 145, 106, 0, 0, 0, 0, 176, 0, 0, 0, 0, 0, 1, 0, 2, 0, 3, 255, 255, 0, 4, 0, 0, 0, 0, 0, 0, 0, 14, + 100, 97, 110, 64, 100, 111, 109, 97, 105, 110, 46, 99, 111, 109, 37, 0, 0, 0, 0, 2, 0, 36, 0, 18, 0, 4, 0, 22, 0, 6, 0, 12, 28, 0, 5, 99, 0, 114, 111, + 108, 101, 115, 97, 108, 97, 114, 121, 7, 109, 97, 110, 97, 103, 101, 114, 1, 1, 0, 4, 0, 0, 0, 0, 0, 0, 0, 14, 100, 97, 110, 64, 100, 111, 109, 97, + 105, 110, 46, 99, 111, 109, 27, 0, 0, 0, 0, 8, 36, 46, 115, 97, 108, 97, 114, 121, 3, 5, 110, 0, 0, 6, 36, 46, 114, 111, 108, 101, 4, 12, 2, 73, 67, }, - want: "JSON_REMOVE(JSON_REPLACE(%s, _utf8mb4'$.day', _utf8mb4\"monday\"), _utf8mb4'$.favorite_color')", + numRows: 1, + want: "JSON_REPLACE(JSON_REPLACE(%s, _utf8mb4'$.salary', 110), _utf8mb4'$.role', _utf8mb4\"IC\")", }, } @@ -353,13 +569,13 @@ func TestMySQL56PartialUpdateRowsEvent(t *testing.T) { ev, err := mysql56PartialUpdateRowEvent.Rows(format, tm) require.NoError(t, err) - assert.Equal(t, 5, len(ev.Rows)) + assert.Equal(t, tc.numRows, len(ev.Rows)) require.NoError(t, err) for i := range ev.Rows { vals, err := ev.StringValuesForTests(tm, i) require.NoError(t, err) // The third column is the JSON column. - require.Equal(t, `JSON_REMOVE(JSON_REPLACE(%s, _utf8mb4'$.day', _utf8mb4"monday"), _utf8mb4'$.favorite_color')`, vals[2]) + require.Equal(t, tc.want, vals[2]) t.Logf("Rows: %v", vals) } }) diff --git a/go/mysql/binlog_event_rbr.go b/go/mysql/binlog_event_rbr.go index 259d0441751..0e0ad0c458f 100644 --- a/go/mysql/binlog_event_rbr.go +++ b/go/mysql/binlog_event_rbr.go @@ -287,9 +287,6 @@ func (ev binlogEvent) Rows(f BinlogFormat, tm *TableMap) (Rows, error) { typ == eDeleteRowsEventV1 || typ == eDeleteRowsEventV2 hasData := typ == eWriteRowsEventV1 || typ == eWriteRowsEventV2 || typ == eUpdateRowsEventV1 || typ == ePartialUpdateRowsEvent || typ == eUpdateRowsEventV2 - //if typ == ePartialUpdateRowsEvent { - // log.Errorf("DEBUG: PartialUpdateRowsEvent bytes: %v", ev.Bytes()) - //} result := Rows{} pos := 6 From 96ffca453316ac9427c9fd58bb45f83da81095b0 Mon Sep 17 00:00:00 2001 From: Matt Lord Date: Mon, 9 Dec 2024 12:37:05 -0500 Subject: [PATCH 09/39] Enable PARTIAL_JSON for vrepl e2e tests Signed-off-by: Matt Lord --- .github/workflows/cluster_endtoend_onlineddl_vrepl.yml | 4 ++++ .../workflows/cluster_endtoend_onlineddl_vrepl_stress.yml | 4 ++++ .../cluster_endtoend_onlineddl_vrepl_stress_suite.yml | 4 ++++ .../workflows/cluster_endtoend_onlineddl_vrepl_suite.yml | 4 ++++ .github/workflows/cluster_endtoend_schemadiff_vrepl.yml | 4 ++++ .../cluster_endtoend_vreplication_across_db_versions.yml | 4 ++++ .github/workflows/cluster_endtoend_vreplication_basic.yml | 4 ++++ .../workflows/cluster_endtoend_vreplication_cellalias.yml | 4 ++++ .../cluster_endtoend_vreplication_copy_parallel.yml | 4 ++++ .../cluster_endtoend_vreplication_foreign_key_stress.yml | 4 ++++ .../cluster_endtoend_vreplication_mariadb_to_mysql.yml | 4 ++++ .github/workflows/cluster_endtoend_vreplication_migrate.yml | 4 ++++ .../cluster_endtoend_vreplication_multi_tenant.yml | 4 ++++ ...oend_vreplication_partial_movetables_and_materialize.yml | 4 ++++ .github/workflows/cluster_endtoend_vreplication_v2.yml | 4 ++++ ...toend_vreplication_vtctldclient_vdiff2_movetables_tz.yml | 4 ++++ test/ci_workflow_gen.go | 2 ++ test/templates/cluster_endtoend_test.tpl | 6 ++++++ 18 files changed, 72 insertions(+) diff --git a/.github/workflows/cluster_endtoend_onlineddl_vrepl.yml b/.github/workflows/cluster_endtoend_onlineddl_vrepl.yml index 5056e46d895..ce0ba65f74f 100644 --- a/.github/workflows/cluster_endtoend_onlineddl_vrepl.yml +++ b/.github/workflows/cluster_endtoend_onlineddl_vrepl.yml @@ -149,6 +149,10 @@ jobs: binlog-transaction-compression=ON EOF + cat <<-EOF>>./config/mycnf/mysql8026.cnf + binlog-row-value-options=PARTIAL_JSON + EOF + # run the tests however you normally do, then produce a JUnit XML file eatmydata -- go run test.go -docker=false -follow -shard onlineddl_vrepl | tee -a output.txt | go-junit-report -set-exit-code > report.xml diff --git a/.github/workflows/cluster_endtoend_onlineddl_vrepl_stress.yml b/.github/workflows/cluster_endtoend_onlineddl_vrepl_stress.yml index 3597b37ede7..57413f649e1 100644 --- a/.github/workflows/cluster_endtoend_onlineddl_vrepl_stress.yml +++ b/.github/workflows/cluster_endtoend_onlineddl_vrepl_stress.yml @@ -149,6 +149,10 @@ jobs: binlog-transaction-compression=ON EOF + cat <<-EOF>>./config/mycnf/mysql8026.cnf + binlog-row-value-options=PARTIAL_JSON + EOF + # run the tests however you normally do, then produce a JUnit XML file eatmydata -- go run test.go -docker=false -follow -shard onlineddl_vrepl_stress | tee -a output.txt | go-junit-report -set-exit-code > report.xml diff --git a/.github/workflows/cluster_endtoend_onlineddl_vrepl_stress_suite.yml b/.github/workflows/cluster_endtoend_onlineddl_vrepl_stress_suite.yml index a5322578981..e1f949df6f8 100644 --- a/.github/workflows/cluster_endtoend_onlineddl_vrepl_stress_suite.yml +++ b/.github/workflows/cluster_endtoend_onlineddl_vrepl_stress_suite.yml @@ -149,6 +149,10 @@ jobs: binlog-transaction-compression=ON EOF + cat <<-EOF>>./config/mycnf/mysql8026.cnf + binlog-row-value-options=PARTIAL_JSON + EOF + # run the tests however you normally do, then produce a JUnit XML file eatmydata -- go run test.go -docker=false -follow -shard onlineddl_vrepl_stress_suite | tee -a output.txt | go-junit-report -set-exit-code > report.xml diff --git a/.github/workflows/cluster_endtoend_onlineddl_vrepl_suite.yml b/.github/workflows/cluster_endtoend_onlineddl_vrepl_suite.yml index 12bde509048..18090dd2430 100644 --- a/.github/workflows/cluster_endtoend_onlineddl_vrepl_suite.yml +++ b/.github/workflows/cluster_endtoend_onlineddl_vrepl_suite.yml @@ -149,6 +149,10 @@ jobs: binlog-transaction-compression=ON EOF + cat <<-EOF>>./config/mycnf/mysql8026.cnf + binlog-row-value-options=PARTIAL_JSON + EOF + # run the tests however you normally do, then produce a JUnit XML file eatmydata -- go run test.go -docker=false -follow -shard onlineddl_vrepl_suite | tee -a output.txt | go-junit-report -set-exit-code > report.xml diff --git a/.github/workflows/cluster_endtoend_schemadiff_vrepl.yml b/.github/workflows/cluster_endtoend_schemadiff_vrepl.yml index e452d28606a..2053a75db4c 100644 --- a/.github/workflows/cluster_endtoend_schemadiff_vrepl.yml +++ b/.github/workflows/cluster_endtoend_schemadiff_vrepl.yml @@ -149,6 +149,10 @@ jobs: binlog-transaction-compression=ON EOF + cat <<-EOF>>./config/mycnf/mysql8026.cnf + binlog-row-value-options=PARTIAL_JSON + EOF + # run the tests however you normally do, then produce a JUnit XML file eatmydata -- go run test.go -docker=false -follow -shard schemadiff_vrepl | tee -a output.txt | go-junit-report -set-exit-code > report.xml diff --git a/.github/workflows/cluster_endtoend_vreplication_across_db_versions.yml b/.github/workflows/cluster_endtoend_vreplication_across_db_versions.yml index a38506ddd7d..6f502c94d32 100644 --- a/.github/workflows/cluster_endtoend_vreplication_across_db_versions.yml +++ b/.github/workflows/cluster_endtoend_vreplication_across_db_versions.yml @@ -166,6 +166,10 @@ jobs: binlog-transaction-compression=ON EOF + cat <<-EOF>>./config/mycnf/mysql8026.cnf + binlog-row-value-options=PARTIAL_JSON + EOF + # run the tests however you normally do, then produce a JUnit XML file eatmydata -- go run test.go -docker=false -follow -shard vreplication_across_db_versions | tee -a output.txt | go-junit-report -set-exit-code > report.xml diff --git a/.github/workflows/cluster_endtoend_vreplication_basic.yml b/.github/workflows/cluster_endtoend_vreplication_basic.yml index 283d0451a34..120cf541305 100644 --- a/.github/workflows/cluster_endtoend_vreplication_basic.yml +++ b/.github/workflows/cluster_endtoend_vreplication_basic.yml @@ -166,6 +166,10 @@ jobs: binlog-transaction-compression=ON EOF + cat <<-EOF>>./config/mycnf/mysql8026.cnf + binlog-row-value-options=PARTIAL_JSON + EOF + # run the tests however you normally do, then produce a JUnit XML file eatmydata -- go run test.go -docker=false -follow -shard vreplication_basic | tee -a output.txt | go-junit-report -set-exit-code > report.xml diff --git a/.github/workflows/cluster_endtoend_vreplication_cellalias.yml b/.github/workflows/cluster_endtoend_vreplication_cellalias.yml index b316c075614..760912b4818 100644 --- a/.github/workflows/cluster_endtoend_vreplication_cellalias.yml +++ b/.github/workflows/cluster_endtoend_vreplication_cellalias.yml @@ -166,6 +166,10 @@ jobs: binlog-transaction-compression=ON EOF + cat <<-EOF>>./config/mycnf/mysql8026.cnf + binlog-row-value-options=PARTIAL_JSON + EOF + # run the tests however you normally do, then produce a JUnit XML file eatmydata -- go run test.go -docker=false -follow -shard vreplication_cellalias | tee -a output.txt | go-junit-report -set-exit-code > report.xml diff --git a/.github/workflows/cluster_endtoend_vreplication_copy_parallel.yml b/.github/workflows/cluster_endtoend_vreplication_copy_parallel.yml index 48f506709e7..d944a253ce3 100644 --- a/.github/workflows/cluster_endtoend_vreplication_copy_parallel.yml +++ b/.github/workflows/cluster_endtoend_vreplication_copy_parallel.yml @@ -166,6 +166,10 @@ jobs: binlog-transaction-compression=ON EOF + cat <<-EOF>>./config/mycnf/mysql8026.cnf + binlog-row-value-options=PARTIAL_JSON + EOF + # run the tests however you normally do, then produce a JUnit XML file eatmydata -- go run test.go -docker=false -follow -shard vreplication_copy_parallel | tee -a output.txt | go-junit-report -set-exit-code > report.xml diff --git a/.github/workflows/cluster_endtoend_vreplication_foreign_key_stress.yml b/.github/workflows/cluster_endtoend_vreplication_foreign_key_stress.yml index a1293e3688e..5bba7760d7e 100644 --- a/.github/workflows/cluster_endtoend_vreplication_foreign_key_stress.yml +++ b/.github/workflows/cluster_endtoend_vreplication_foreign_key_stress.yml @@ -166,6 +166,10 @@ jobs: binlog-transaction-compression=ON EOF + cat <<-EOF>>./config/mycnf/mysql8026.cnf + binlog-row-value-options=PARTIAL_JSON + EOF + # run the tests however you normally do, then produce a JUnit XML file eatmydata -- go run test.go -docker=false -follow -shard vreplication_foreign_key_stress | tee -a output.txt | go-junit-report -set-exit-code > report.xml diff --git a/.github/workflows/cluster_endtoend_vreplication_mariadb_to_mysql.yml b/.github/workflows/cluster_endtoend_vreplication_mariadb_to_mysql.yml index 2013d89a83d..1184fe493ef 100644 --- a/.github/workflows/cluster_endtoend_vreplication_mariadb_to_mysql.yml +++ b/.github/workflows/cluster_endtoend_vreplication_mariadb_to_mysql.yml @@ -166,6 +166,10 @@ jobs: binlog-transaction-compression=ON EOF + cat <<-EOF>>./config/mycnf/mysql8026.cnf + binlog-row-value-options=PARTIAL_JSON + EOF + # run the tests however you normally do, then produce a JUnit XML file eatmydata -- go run test.go -docker=false -follow -shard vreplication_mariadb_to_mysql | tee -a output.txt | go-junit-report -set-exit-code > report.xml diff --git a/.github/workflows/cluster_endtoend_vreplication_migrate.yml b/.github/workflows/cluster_endtoend_vreplication_migrate.yml index e7a8a8cb5ce..009840800d2 100644 --- a/.github/workflows/cluster_endtoend_vreplication_migrate.yml +++ b/.github/workflows/cluster_endtoend_vreplication_migrate.yml @@ -166,6 +166,10 @@ jobs: binlog-transaction-compression=ON EOF + cat <<-EOF>>./config/mycnf/mysql8026.cnf + binlog-row-value-options=PARTIAL_JSON + EOF + # run the tests however you normally do, then produce a JUnit XML file eatmydata -- go run test.go -docker=false -follow -shard vreplication_migrate | tee -a output.txt | go-junit-report -set-exit-code > report.xml diff --git a/.github/workflows/cluster_endtoend_vreplication_multi_tenant.yml b/.github/workflows/cluster_endtoend_vreplication_multi_tenant.yml index c0c80a8dc61..9a5935a8907 100644 --- a/.github/workflows/cluster_endtoend_vreplication_multi_tenant.yml +++ b/.github/workflows/cluster_endtoend_vreplication_multi_tenant.yml @@ -166,6 +166,10 @@ jobs: binlog-transaction-compression=ON EOF + cat <<-EOF>>./config/mycnf/mysql8026.cnf + binlog-row-value-options=PARTIAL_JSON + EOF + # run the tests however you normally do, then produce a JUnit XML file eatmydata -- go run test.go -docker=false -follow -shard vreplication_multi_tenant | tee -a output.txt | go-junit-report -set-exit-code > report.xml diff --git a/.github/workflows/cluster_endtoend_vreplication_partial_movetables_and_materialize.yml b/.github/workflows/cluster_endtoend_vreplication_partial_movetables_and_materialize.yml index 0b80b82fd7f..0a10e37e6c4 100644 --- a/.github/workflows/cluster_endtoend_vreplication_partial_movetables_and_materialize.yml +++ b/.github/workflows/cluster_endtoend_vreplication_partial_movetables_and_materialize.yml @@ -166,6 +166,10 @@ jobs: binlog-transaction-compression=ON EOF + cat <<-EOF>>./config/mycnf/mysql8026.cnf + binlog-row-value-options=PARTIAL_JSON + EOF + # run the tests however you normally do, then produce a JUnit XML file eatmydata -- go run test.go -docker=false -follow -shard vreplication_partial_movetables_and_materialize | tee -a output.txt | go-junit-report -set-exit-code > report.xml diff --git a/.github/workflows/cluster_endtoend_vreplication_v2.yml b/.github/workflows/cluster_endtoend_vreplication_v2.yml index fcbe7057b4c..f023bf9718b 100644 --- a/.github/workflows/cluster_endtoend_vreplication_v2.yml +++ b/.github/workflows/cluster_endtoend_vreplication_v2.yml @@ -166,6 +166,10 @@ jobs: binlog-transaction-compression=ON EOF + cat <<-EOF>>./config/mycnf/mysql8026.cnf + binlog-row-value-options=PARTIAL_JSON + EOF + # run the tests however you normally do, then produce a JUnit XML file eatmydata -- go run test.go -docker=false -follow -shard vreplication_v2 | tee -a output.txt | go-junit-report -set-exit-code > report.xml diff --git a/.github/workflows/cluster_endtoend_vreplication_vtctldclient_vdiff2_movetables_tz.yml b/.github/workflows/cluster_endtoend_vreplication_vtctldclient_vdiff2_movetables_tz.yml index 40f2002f9a3..7d96ac60306 100644 --- a/.github/workflows/cluster_endtoend_vreplication_vtctldclient_vdiff2_movetables_tz.yml +++ b/.github/workflows/cluster_endtoend_vreplication_vtctldclient_vdiff2_movetables_tz.yml @@ -166,6 +166,10 @@ jobs: binlog-transaction-compression=ON EOF + cat <<-EOF>>./config/mycnf/mysql8026.cnf + binlog-row-value-options=PARTIAL_JSON + EOF + # run the tests however you normally do, then produce a JUnit XML file eatmydata -- go run test.go -docker=false -follow -shard vreplication_vtctldclient_vdiff2_movetables_tz | tee -a output.txt | go-junit-report -set-exit-code > report.xml diff --git a/test/ci_workflow_gen.go b/test/ci_workflow_gen.go index 7956c491408..52da65d8041 100644 --- a/test/ci_workflow_gen.go +++ b/test/ci_workflow_gen.go @@ -172,6 +172,7 @@ type clusterTest struct { Docker bool LimitResourceUsage bool EnableBinlogTransactionCompression bool + EnablePartialJSON bool PartialKeyspace bool Cores16 bool } @@ -294,6 +295,7 @@ func generateClusterWorkflows(list []string, tpl string) { } if strings.Contains(cluster, "vrepl") { test.EnableBinlogTransactionCompression = true + test.EnablePartialJSON = true } mysqlVersionIndicator := "" if mysqlVersion != defaultMySQLVersion && len(clusterMySQLVersions()) > 1 { diff --git a/test/templates/cluster_endtoend_test.tpl b/test/templates/cluster_endtoend_test.tpl index 6fe58fae361..387d2e8509a 100644 --- a/test/templates/cluster_endtoend_test.tpl +++ b/test/templates/cluster_endtoend_test.tpl @@ -215,6 +215,12 @@ jobs: EOF {{end}} + {{if .EnablePartialJSON}} + cat <<-EOF>>./config/mycnf/mysql8026.cnf + binlog-row-value-options=PARTIAL_JSON + EOF + {{end}} + # run the tests however you normally do, then produce a JUnit XML file eatmydata -- go run test.go -docker={{if .Docker}}true -flavor={{.Platform}}{{else}}false{{end}} -follow -shard {{.Shard}}{{if .PartialKeyspace}} -partial-keyspace=true {{end}}{{if .BuildTag}} -build-tag={{.BuildTag}} {{end}} | tee -a output.txt | go-junit-report -set-exit-code > report.xml From 42d5db4648e29ffb8e71846f6c471a46c9fce198 Mon Sep 17 00:00:00 2001 From: Matt Lord Date: Mon, 9 Dec 2024 19:18:57 -0500 Subject: [PATCH 10/39] Get rid of unused func Signed-off-by: Matt Lord --- go/mysql/binlog/binlog_json.go | 42 ---------------------------------- 1 file changed, 42 deletions(-) diff --git a/go/mysql/binlog/binlog_json.go b/go/mysql/binlog/binlog_json.go index 204e5186be4..c0bbf8aec21 100644 --- a/go/mysql/binlog/binlog_json.go +++ b/go/mysql/binlog/binlog_json.go @@ -471,45 +471,3 @@ func binparserObject(typ jsonDataType, data []byte, pos int) (node *json.Value, return json.NewObject(object), nil } - -func readLenEncInt(data []byte, pos int) (uint64, int, bool) { - if pos >= len(data) { - return 0, 0, false - } - - // reslice to avoid arithmetic below - data = data[pos:] - - switch data[0] { - case 0xfc: - // Encoded in the next 2 bytes. - if 2 >= len(data) { - return 0, 0, false - } - return uint64(data[1]) | - uint64(data[2])<<8, pos + 3, true - case 0xfd: - // Encoded in the next 3 bytes. - if 3 >= len(data) { - return 0, 0, false - } - return uint64(data[1]) | - uint64(data[2])<<8 | - uint64(data[3])<<16, pos + 4, true - case 0xfe: - // Encoded in the next 8 bytes. - if 8 >= len(data) { - return 0, 0, false - } - return uint64(data[1]) | - uint64(data[2])<<8 | - uint64(data[3])<<16 | - uint64(data[4])<<24 | - uint64(data[5])<<32 | - uint64(data[6])<<40 | - uint64(data[7])<<48 | - uint64(data[8])<<56, pos + 9, true - default: - return uint64(data[0]), pos + 1, true - } -} From 6e288df6895fec500591c71e258b531e79593d9e Mon Sep 17 00:00:00 2001 From: Matt Lord Date: Tue, 10 Dec 2024 09:01:28 -0500 Subject: [PATCH 11/39] Get binlog_row_image = noblob and binlog-row-value-options = PARTIAL_JSON working together Signed-off-by: Matt Lord --- go/mysql/binlog_event_rbr.go | 3 ++- .../tabletmanager/vreplication/replicator_plan.go | 15 +++++++++++++++ .../vreplication/table_plan_partial.go | 10 ++++++++++ .../vttablet/tabletserver/vstreamer/vstreamer.go | 6 +++++- 4 files changed, 32 insertions(+), 2 deletions(-) diff --git a/go/mysql/binlog_event_rbr.go b/go/mysql/binlog_event_rbr.go index 0e0ad0c458f..9612780b3e8 100644 --- a/go/mysql/binlog_event_rbr.go +++ b/go/mysql/binlog_event_rbr.go @@ -370,7 +370,8 @@ func (ev binlogEvent) Rows(f BinlogFormat, tm *TableMap) (Rows, error) { if ev.Type() == ePartialUpdateRowsEvent { // The first byte indicates whether or not any JSON values are partial. - // If it's 0 then there's nothing else to do. + // If it's not 1 then there's nothing else to do for the row as any + // columns use the full value. partialJSON := uint8(data[pos]) pos++ if partialJSON == 1 { diff --git a/go/vt/vttablet/tabletmanager/vreplication/replicator_plan.go b/go/vt/vttablet/tabletmanager/vreplication/replicator_plan.go index bf282e663c2..90cdafd1316 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/replicator_plan.go +++ b/go/vt/vttablet/tabletmanager/vreplication/replicator_plan.go @@ -386,6 +386,7 @@ func (tp *TablePlan) applyChange(rowChange *binlogdatapb.RowChange, executor fun var newVal *sqltypes.Value var err error if field.Type == querypb.Type_JSON { + //log.Errorf("DEBUG: JSON field %v, value: %v", field.Name, vals[i].RawStr()) switch { case vals[i].IsNull(): // An SQL NULL and not an actual JSON value newVal = &sqltypes.NULL @@ -394,6 +395,12 @@ func (tp *TablePlan) applyChange(rowChange *binlogdatapb.RowChange, executor fun // as JSON_INSERT(). // This occurs e.g. when using partial JSON values as a result of // mysqld using binlog-row-value-options=PARTIAL_JSON. + if len(vals[i].Raw()) == 0 { + // When using BOTH binlog_row_image=NOBLOB AND + // binlog_row_value_options=PARTIAL_JSON then the JSON column + // has the data bit set and the diff is empty. + setBit(rowChange.DataColumns.Cols, i, false) + } s := fmt.Sprintf(vals[i].RawStr(), field.Name) newVal = ptr.Of(sqltypes.MakeTrusted(querypb.Type_EXPRESSION, []byte(s))) default: // A JSON value (which may be a JSON null literal value) @@ -437,6 +444,7 @@ func (tp *TablePlan) applyChange(rowChange *binlogdatapb.RowChange, executor fun case before && after: if !tp.pkChanged(bindvars) && !tp.HasExtraSourcePkColumns { if tp.isPartial(rowChange) { + //log.Errorf("DEBUG: building partial update query using DataColumns: %08b", rowChange.DataColumns.Cols) upd, err := tp.getPartialUpdateQuery(rowChange.DataColumns) if err != nil { return nil, err @@ -452,6 +460,12 @@ func (tp *TablePlan) applyChange(rowChange *binlogdatapb.RowChange, executor fun return nil, err } } + // TODO: the INSERTs done here after deleting the row with the original PK + // need to use the values from the BEFORE image for the columns NOT present + // in the AFTER image due to being a partial image due to the source's usage + // of binlog-row-image=NOBLOB. + // For JSON columns when binlog-row-value-options=PARTIAL_JSON is used, we + // need to wrap the JSON diff function(s) around the BEFORE value. if tp.isOutsidePKRange(bindvars, before, after, "insert") { return nil, nil } @@ -593,6 +607,7 @@ func execParsedQuery(pq *sqlparser.ParsedQuery, bindvars map[string]*querypb.Bin if err != nil { return nil, err } + //log.Errorf("DEBUG: execParsedQuery: %s", query) return executor(query) } diff --git a/go/vt/vttablet/tabletmanager/vreplication/table_plan_partial.go b/go/vt/vttablet/tabletmanager/vreplication/table_plan_partial.go index 85e0fd8e50f..a4e177a9f14 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/table_plan_partial.go +++ b/go/vt/vttablet/tabletmanager/vreplication/table_plan_partial.go @@ -36,6 +36,16 @@ func isBitSet(data []byte, index int) bool { return data[byteIndex]&bitMask > 0 } +func setBit(data []byte, index int, value bool) { + byteIndex := index / 8 + bitMask := byte(1 << (uint(index) & 0x7)) + if value { + data[byteIndex] |= bitMask + } else { + data[byteIndex] &= 0xff - bitMask + } +} + func (tp *TablePlan) isPartial(rowChange *binlogdatapb.RowChange) bool { if (tp.WorkflowConfig.ExperimentalFlags /**/ & /**/ vttablet.VReplicationExperimentalFlagAllowNoBlobBinlogRowImage) == 0 || rowChange.DataColumns == nil || diff --git a/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go b/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go index 568461aed3d..ac648077c19 100644 --- a/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go +++ b/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go @@ -1031,19 +1031,23 @@ func (vs *vstreamer) processRowEvent(vevents []*binlogdatapb.VEvent, plan *strea } if afterOK { rowChange.After = sqltypes.RowToProto3(afterValues) + //log.Errorf("DEBUG: partial = %v", partial) + //log.Errorf("DEBUG: rowChange: After: %+v", rowChange.After) if (vs.config.ExperimentalFlags /**/ & /**/ vttablet.VReplicationExperimentalFlagAllowNoBlobBinlogRowImage != 0) && - partial { + (partial || row.JSONPartialValues.Count() > 0) { rowChange.DataColumns = &binlogdatapb.RowChange_Bitmap{ Count: int64(rows.DataColumns.Count()), Cols: rows.DataColumns.Bits(), } + //log.Errorf("DEBUG: rowChange: DataColumns: %08b", rowChange.DataColumns.Cols) } if row.JSONPartialValues.Count() > 0 { rowChange.JsonPartialValues = &binlogdatapb.RowChange_Bitmap{ Count: int64(row.JSONPartialValues.Count()), Cols: row.JSONPartialValues.Bits(), } + //log.Errorf("DEBUG: rowChange: JSONPartialColumns: %08b", rowChange.JsonPartialValues.Cols) } } rowChanges = append(rowChanges, rowChange) From f2a5445e602614fde338cd6c6ec742ce0ddecb4f Mon Sep 17 00:00:00 2001 From: Matt Lord Date: Tue, 10 Dec 2024 18:26:52 -0500 Subject: [PATCH 12/39] Get edge cases working Signed-off-by: Matt Lord --- go/mysql/binlog/binlog_json.go | 9 ++++- go/mysql/binlog/rbr.go | 5 ++- go/mysql/binlog_event_mysql56_test.go | 37 +++++++++++++++---- .../vreplication/vreplication_test.go | 9 ++++- .../vreplication/replicator_plan.go | 14 ++++--- .../tabletserver/vstreamer/vstreamer.go | 4 -- 6 files changed, 56 insertions(+), 22 deletions(-) diff --git a/go/mysql/binlog/binlog_json.go b/go/mysql/binlog/binlog_json.go index c0bbf8aec21..29c37d1eb78 100644 --- a/go/mysql/binlog/binlog_json.go +++ b/go/mysql/binlog/binlog_json.go @@ -97,12 +97,19 @@ func ParseBinaryJSONDiff(data []byte) (sqltypes.Value, error) { diff.WriteString("JSON_INSERT(") case jsonDiffOpRemove: diff.WriteString("JSON_REMOVE(") + default: + // Can be a literal JSON null. + js, err := ParseBinaryJSON(data) + if err == nil && js.Type() == json.TypeNull { + return sqltypes.MakeTrusted(sqltypes.Expression, js.MarshalTo(nil)), nil + } + return sqltypes.Value{}, fmt.Errorf("invalid JSON diff operation: %d", opType) } if outer { diff.WriteString(innerStr) diff.WriteString(", ") } else { // Only the inner most function has the field name - diff.WriteString("%s, ") // This will later be replaced by the field name + diff.WriteString("`%s`, ") // This will later be replaced by the field name } pathLen, readTo := readVariableLength(data, pos) diff --git a/go/mysql/binlog/rbr.go b/go/mysql/binlog/rbr.go index 42494b0a346..7512413f606 100644 --- a/go/mysql/binlog/rbr.go +++ b/go/mysql/binlog/rbr.go @@ -691,8 +691,9 @@ func CellValue(data []byte, pos int, typ byte, metadata uint16, field *querypb.F if err != nil { panic(err) } - jd := jsonVal.MarshalTo(nil) - return sqltypes.MakeTrusted(sqltypes.Expression, jd), l + int(metadata), nil + d := jsonVal.MarshalTo(nil) + return sqltypes.MakeTrusted(sqltypes.Expression, + d), l + int(metadata), nil } return sqltypes.MakeTrusted(querypb.Type_VARBINARY, diff --git a/go/mysql/binlog_event_mysql56_test.go b/go/mysql/binlog_event_mysql56_test.go index 28c359d1430..a138f28e16e 100644 --- a/go/mysql/binlog_event_mysql56_test.go +++ b/go/mysql/binlog_event_mysql56_test.go @@ -352,7 +352,7 @@ func TestMySQL56PartialUpdateRowsEvent(t *testing.T) { 97, 110, 97, 103, 101, 114, }, numRows: 5, - want: "JSON_INSERT(%s, _utf8mb4'$.role', _utf8mb4\"manager\")", + want: "JSON_INSERT(`%s`, _utf8mb4'$.role', _utf8mb4\"manager\")", }, { // The mysqlbinlog -vvv --base64-output=decode-rows output for the following event: @@ -373,7 +373,7 @@ func TestMySQL56PartialUpdateRowsEvent(t *testing.T) { }, name: "REPLACE", numRows: 1, - want: "JSON_REPLACE(%s, _utf8mb4'$.role', _utf8mb4\"IC\")", + want: "JSON_REPLACE(`%s`, _utf8mb4'$.role', _utf8mb4\"IC\")", }, { name: "REMOVE", @@ -394,7 +394,7 @@ func TestMySQL56PartialUpdateRowsEvent(t *testing.T) { 111, 108, 101, 115, 97, 108, 97, 114, 121, 7, 109, 97, 110, 97, 103, 101, 114, 1, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 14, 98, 111, 98, 64, 100, 111, 109, 97, 105, 110, 46, 99, 111, 109, 10, 0, 0, 0, 2, 8, 36, 46, 115, 97, 108, 97, 114, 121, }, - want: "JSON_REMOVE(%s, _utf8mb4'$.salary')", + want: "JSON_REMOVE(`%s`, _utf8mb4'$.salary')", }, { name: "REMOVE and REPLACE", @@ -486,7 +486,7 @@ func TestMySQL56PartialUpdateRowsEvent(t *testing.T) { 100, 97, 121, 2, 16, 36, 46, 102, 97, 118, 111, 114, 105, 116, 101, 95, 99, 111, 108, 111, 114, }, numRows: 5, - want: "JSON_REMOVE(JSON_REPLACE(%s, _utf8mb4'$.day', _utf8mb4\"monday\"), _utf8mb4'$.favorite_color')", + want: "JSON_REMOVE(JSON_REPLACE(`%s`, _utf8mb4'$.day', _utf8mb4\"monday\"), _utf8mb4'$.favorite_color')", }, { name: "INSERT and REMOVE and REPLACE", @@ -514,7 +514,7 @@ func TestMySQL56PartialUpdateRowsEvent(t *testing.T) { 1, 7, 36, 46, 104, 111, 98, 98, 121, 8, 12, 6, 115, 107, 105, 105, 110, 103, }, numRows: 1, - want: "JSON_INSERT(JSON_REMOVE(JSON_REPLACE(%s, _utf8mb4'$.day', _utf8mb4\"tuesday\"), _utf8mb4'$.favorite_color'), _utf8mb4'$.hobby', _utf8mb4\"skiing\")", + want: "JSON_INSERT(JSON_REMOVE(JSON_REPLACE(`%s`, _utf8mb4'$.day', _utf8mb4\"tuesday\"), _utf8mb4'$.favorite_color'), _utf8mb4'$.hobby', _utf8mb4\"skiing\")", }, { name: "REPLACE with null", @@ -535,7 +535,7 @@ func TestMySQL56PartialUpdateRowsEvent(t *testing.T) { 109, 97, 105, 110, 46, 99, 111, 109, 13, 0, 0, 0, 0, 8, 36, 46, 115, 97, 108, 97, 114, 121, 2, 4, 0, }, numRows: 1, - want: "JSON_REPLACE(%s, _utf8mb4'$.salary', null)", + want: "JSON_REPLACE(`%s`, _utf8mb4'$.salary', null)", }, { name: "REPLACE 2 paths", @@ -557,7 +557,30 @@ func TestMySQL56PartialUpdateRowsEvent(t *testing.T) { 105, 110, 46, 99, 111, 109, 27, 0, 0, 0, 0, 8, 36, 46, 115, 97, 108, 97, 114, 121, 3, 5, 110, 0, 0, 6, 36, 46, 114, 111, 108, 101, 4, 12, 2, 73, 67, }, numRows: 1, - want: "JSON_REPLACE(JSON_REPLACE(%s, _utf8mb4'$.salary', 110), _utf8mb4'$.role', _utf8mb4\"IC\")", + want: "JSON_REPLACE(JSON_REPLACE(`%s`, _utf8mb4'$.salary', 110), _utf8mb4'$.role', _utf8mb4\"IC\")", + }, + { + name: "JSON null", + // The mysqlbinlog -vvv --base64-output=decode-rows output for the following event: + // ### UPDATE `vt_commerce`.`customer` + // ### WHERE + // ### @1=5 /* LONGINT meta=0 nullable=0 is_null=0 */ + // ### @2='neweve@domain.com' /* VARSTRING(128) meta=128 nullable=1 is_null=0 */ + // ### @3='{"day": "friday", "role": "manager", "color": "red", "salary": 100, "favorite_color": "black"}' /* JSON meta=4 nullable=1 is_null=0 */ + // ### SET + // ### @1=5 /* LONGINT meta=0 nullable=0 is_null=0 */ + // ### @2='neweve@domain.com' /* VARSTRING(128) meta=128 nullable=1 is_null=0 */ + // ### @3='null' /* JSON meta=4 nullable=1 is_null=0 */ + rawEvent: []byte{ + 109, 200, 88, 103, 39, 57, 91, 186, 0, 194, 0, 0, 0, 0, 0, 0, 0, 0, 0, 178, 0, 0, 0, 0, 0, 1, 0, 2, 0, 3, 7, 7, 0, 5, 0, 0, 0, 0, 0, 0, 0, 17, 110, + 101, 119, 101, 118, 101, 64, 100, 111, 109, 97, 105, 110, 46, 99, 111, 109, 97, 0, 0, 0, 0, 5, 0, 96, 0, 39, 0, 3, 0, 42, 0, 4, 0, 46, 0, 5, 0, 51, + 0, 6, 0, 57, 0, 14, 0, 12, 71, 0, 12, 78, 0, 12, 86, 0, 5, 100, 0, 12, 90, 0, 100, 97, 121, 114, 111, 108, 101, 99, 111, 108, 111, 114, 115, 97, + 108, 97, 114, 121, 102, 97, 118, 111, 114, 105, 116, 101, 95, 99, 111, 108, 111, 114, 6, 102, 114, 105, 100, 97, 121, 7, 109, 97, 110, 97, 103, 101, + 114, 3, 114, 101, 100, 5, 98, 108, 97, 99, 107, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 17, 110, 101, 119, 101, 118, 101, 64, 100, 111, 109, 97, 105, 110, 46, + 99, 111, 109, 2, 0, 0, 0, 4, 0, + }, + numRows: 1, + want: "null", }, } diff --git a/go/test/endtoend/vreplication/vreplication_test.go b/go/test/endtoend/vreplication/vreplication_test.go index d3193298a0c..d1c7f5aa8ea 100644 --- a/go/test/endtoend/vreplication/vreplication_test.go +++ b/go/test/endtoend/vreplication/vreplication_test.go @@ -721,8 +721,13 @@ func shardCustomer(t *testing.T, testReverse bool, cells []*Cell, sourceCellOrAl // Confirm that the 0 scale decimal field, dec80, is replicated correctly execVtgateQuery(t, vtgateConn, sourceKs, "update customer set dec80 = 0") execVtgateQuery(t, vtgateConn, sourceKs, "update customer set blb = \"new blob data\" where cid=3") - execVtgateQuery(t, vtgateConn, sourceKs, "update json_tbl set j1 = null, j2 = 'null', j3 = '\"null\"'") - execVtgateQuery(t, vtgateConn, sourceKs, "insert into json_tbl(id, j1, j2, j3) values (7, null, 'null', '\"null\"')") + // TODO: file a MySQL bug for this. The following query results in the quoted string literal "null" + // stored in the j3 column. But with and without the literal quotes, the value in the PARTIAL_JSON + // diff is the unquoted literal null which is a JSON null type. This leads to a vdiff mismatch. + //execVtgateQuery(t, vtgateConn, sourceKs, "update json_tbl set j1 = null, j2 = 'null', j3 = '\"null\"'") + //execVtgateQuery(t, vtgateConn, sourceKs, "insert into json_tbl(id, j1, j2, j3) values (7, null, 'null', '\"null\"')") + execVtgateQuery(t, vtgateConn, sourceKs, "update json_tbl set j1 = null, j2 = 'null', j3 = 'null'") + execVtgateQuery(t, vtgateConn, sourceKs, "insert into json_tbl(id, j1, j2, j3) values (7, null, 'null', 'null')") waitForNoWorkflowLag(t, vc, targetKs, workflow) dec80Replicated := false for _, tablet := range []*cluster.VttabletProcess{customerTab1, customerTab2} { diff --git a/go/vt/vttablet/tabletmanager/vreplication/replicator_plan.go b/go/vt/vttablet/tabletmanager/vreplication/replicator_plan.go index 90cdafd1316..7a51d5f5aea 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/replicator_plan.go +++ b/go/vt/vttablet/tabletmanager/vreplication/replicator_plan.go @@ -19,6 +19,7 @@ package vreplication import ( "encoding/json" "fmt" + "slices" "sort" "strings" @@ -386,11 +387,11 @@ func (tp *TablePlan) applyChange(rowChange *binlogdatapb.RowChange, executor fun var newVal *sqltypes.Value var err error if field.Type == querypb.Type_JSON { - //log.Errorf("DEBUG: JSON field %v, value: %v", field.Name, vals[i].RawStr()) switch { case vals[i].IsNull(): // An SQL NULL and not an actual JSON value newVal = &sqltypes.NULL - case rowChange.JsonPartialValues != nil && isBitSet(rowChange.JsonPartialValues.Cols, jsonIndex): + case rowChange.JsonPartialValues != nil && isBitSet(rowChange.JsonPartialValues.Cols, jsonIndex) && + !slices.Equal(vals[i].Raw(), sqltypes.NullBytes): // An SQL expression that can be converted to a JSON value such // as JSON_INSERT(). // This occurs e.g. when using partial JSON values as a result of @@ -400,9 +401,12 @@ func (tp *TablePlan) applyChange(rowChange *binlogdatapb.RowChange, executor fun // binlog_row_value_options=PARTIAL_JSON then the JSON column // has the data bit set and the diff is empty. setBit(rowChange.DataColumns.Cols, i, false) + newVal = ptr.Of(sqltypes.MakeTrusted(querypb.Type_EXPRESSION, nil)) + } else { + newVal = ptr.Of(sqltypes.MakeTrusted(querypb.Type_EXPRESSION, []byte( + fmt.Sprintf(vals[i].RawStr(), field.Name), + ))) } - s := fmt.Sprintf(vals[i].RawStr(), field.Name) - newVal = ptr.Of(sqltypes.MakeTrusted(querypb.Type_EXPRESSION, []byte(s))) default: // A JSON value (which may be a JSON null literal value) newVal, err = vjson.MarshalSQLValue(vals[i].Raw()) if err != nil { @@ -444,7 +448,6 @@ func (tp *TablePlan) applyChange(rowChange *binlogdatapb.RowChange, executor fun case before && after: if !tp.pkChanged(bindvars) && !tp.HasExtraSourcePkColumns { if tp.isPartial(rowChange) { - //log.Errorf("DEBUG: building partial update query using DataColumns: %08b", rowChange.DataColumns.Cols) upd, err := tp.getPartialUpdateQuery(rowChange.DataColumns) if err != nil { return nil, err @@ -607,7 +610,6 @@ func execParsedQuery(pq *sqlparser.ParsedQuery, bindvars map[string]*querypb.Bin if err != nil { return nil, err } - //log.Errorf("DEBUG: execParsedQuery: %s", query) return executor(query) } diff --git a/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go b/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go index ac648077c19..b964f5d7e8e 100644 --- a/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go +++ b/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go @@ -1031,8 +1031,6 @@ func (vs *vstreamer) processRowEvent(vevents []*binlogdatapb.VEvent, plan *strea } if afterOK { rowChange.After = sqltypes.RowToProto3(afterValues) - //log.Errorf("DEBUG: partial = %v", partial) - //log.Errorf("DEBUG: rowChange: After: %+v", rowChange.After) if (vs.config.ExperimentalFlags /**/ & /**/ vttablet.VReplicationExperimentalFlagAllowNoBlobBinlogRowImage != 0) && (partial || row.JSONPartialValues.Count() > 0) { @@ -1040,14 +1038,12 @@ func (vs *vstreamer) processRowEvent(vevents []*binlogdatapb.VEvent, plan *strea Count: int64(rows.DataColumns.Count()), Cols: rows.DataColumns.Bits(), } - //log.Errorf("DEBUG: rowChange: DataColumns: %08b", rowChange.DataColumns.Cols) } if row.JSONPartialValues.Count() > 0 { rowChange.JsonPartialValues = &binlogdatapb.RowChange_Bitmap{ Count: int64(row.JSONPartialValues.Count()), Cols: row.JSONPartialValues.Bits(), } - //log.Errorf("DEBUG: rowChange: JSONPartialColumns: %08b", rowChange.JsonPartialValues.Cols) } } rowChanges = append(rowChanges, rowChange) From 1e492b9ee79fc726bc67ea85919cf656d0cc7654 Mon Sep 17 00:00:00 2001 From: Matt Lord Date: Tue, 10 Dec 2024 20:03:59 -0500 Subject: [PATCH 13/39] Minor changes from self review Signed-off-by: Matt Lord --- go/mysql/binlog/binlog_json.go | 14 ++++++++------ go/mysql/binlog_event.go | 7 ++++--- go/mysql/binlog_event_mysql56_test.go | 1 + go/mysql/binlog_event_rbr.go | 2 +- go/test/endtoend/vreplication/vreplication_test.go | 2 ++ go/vt/proto/binlogdata/binlogdata.pb.go | 6 +++--- .../tabletmanager/vreplication/replicator_plan.go | 8 +++++--- proto/binlogdata.proto | 6 +++--- 8 files changed, 27 insertions(+), 19 deletions(-) diff --git a/go/mysql/binlog/binlog_json.go b/go/mysql/binlog/binlog_json.go index 29c37d1eb78..e3ca52096f2 100644 --- a/go/mysql/binlog/binlog_json.go +++ b/go/mysql/binlog/binlog_json.go @@ -71,14 +71,14 @@ func ParseBinaryJSON(data []byte) (*json.Value, error) { return node, nil } -// ParseBinaryJSONDiff provides the parsing function from the MySQL JSON -// diff representation to an SQL expression. +// ParseBinaryJSONDiff provides the parsing function from the binary MySQL +// JSON diff representation to an SQL expression. func ParseBinaryJSONDiff(data []byte) (sqltypes.Value, error) { diff := bytes.Buffer{} // Reasonable estimate of the space we'll need to build the SQL // expression in order to try and avoid reallocations w/o // overallocating too much. - diff.Grow(int(float32(len(data)) * 1.5)) + diff.Grow(int(float32(len(data)) * 1.25)) pos := 0 outer := false innerStr := "" @@ -98,12 +98,13 @@ func ParseBinaryJSONDiff(data []byte) (sqltypes.Value, error) { case jsonDiffOpRemove: diff.WriteString("JSON_REMOVE(") default: - // Can be a literal JSON null. + // Can be a JSON null. js, err := ParseBinaryJSON(data) if err == nil && js.Type() == json.TypeNull { return sqltypes.MakeTrusted(sqltypes.Expression, js.MarshalTo(nil)), nil } - return sqltypes.Value{}, fmt.Errorf("invalid JSON diff operation: %d", opType) + return sqltypes.Value{}, vterrors.Errorf(vtrpcpb.Code_INVALID_ARGUMENT, + "invalid JSON diff operation: %d", opType) } if outer { diff.WriteString(innerStr) @@ -129,7 +130,8 @@ func ParseBinaryJSONDiff(data []byte) (sqltypes.Value, error) { pos = readTo value, err := ParseBinaryJSON(data[pos : pos+valueLen]) if err != nil { - return sqltypes.Value{}, fmt.Errorf("cannot read JSON diff value for path %s: %w", path, err) + return sqltypes.Value{}, vterrors.Errorf(vtrpcpb.Code_INVALID_ARGUMENT, + "cannot read JSON diff value for path %s: %v", path, err) } pos += valueLen if value.Type() == json.TypeString { diff --git a/go/mysql/binlog_event.go b/go/mysql/binlog_event.go index 76660f4ee90..72c228b594e 100644 --- a/go/mysql/binlog_event.go +++ b/go/mysql/binlog_event.go @@ -270,9 +270,10 @@ type Row struct { // It is only set for UPDATE and DELETE events. Identify []byte - // If this row represents a PartialUpdateRow event there will be a - // shared-image that sits between the before and after images that - // defines how the JSON column values are represented. + // If this row was from a PartialUpdateRows event and it contains + // 1 or more JSON columns with partial values, then this will be + // set as a bitmap of which JSON columns in the AFTER image have + // partial values. JSONPartialValues Bitmap // Data is the raw data. diff --git a/go/mysql/binlog_event_mysql56_test.go b/go/mysql/binlog_event_mysql56_test.go index a138f28e16e..77ce2c11d60 100644 --- a/go/mysql/binlog_event_mysql56_test.go +++ b/go/mysql/binlog_event_mysql56_test.go @@ -594,6 +594,7 @@ func TestMySQL56PartialUpdateRowsEvent(t *testing.T) { assert.Equal(t, tc.numRows, len(ev.Rows)) require.NoError(t, err) + for i := range ev.Rows { vals, err := ev.StringValuesForTests(tm, i) require.NoError(t, err) diff --git a/go/mysql/binlog_event_rbr.go b/go/mysql/binlog_event_rbr.go index 9612780b3e8..f4f005b5dcd 100644 --- a/go/mysql/binlog_event_rbr.go +++ b/go/mysql/binlog_event_rbr.go @@ -370,7 +370,7 @@ func (ev binlogEvent) Rows(f BinlogFormat, tm *TableMap) (Rows, error) { if ev.Type() == ePartialUpdateRowsEvent { // The first byte indicates whether or not any JSON values are partial. - // If it's not 1 then there's nothing else to do for the row as any + // If it's not 1 then there's nothing special to do for the row as any // columns use the full value. partialJSON := uint8(data[pos]) pos++ diff --git a/go/test/endtoend/vreplication/vreplication_test.go b/go/test/endtoend/vreplication/vreplication_test.go index d1c7f5aa8ea..870b5d0d348 100644 --- a/go/test/endtoend/vreplication/vreplication_test.go +++ b/go/test/endtoend/vreplication/vreplication_test.go @@ -724,6 +724,8 @@ func shardCustomer(t *testing.T, testReverse bool, cells []*Cell, sourceCellOrAl // TODO: file a MySQL bug for this. The following query results in the quoted string literal "null" // stored in the j3 column. But with and without the literal quotes, the value in the PARTIAL_JSON // diff is the unquoted literal null which is a JSON null type. This leads to a vdiff mismatch. + // I'm not sure if the bug is that it allows the quoted value "null" to be inserted or that it + // doesn't reflect this in the partial diff value, but the combination of the two is certainly a bug. //execVtgateQuery(t, vtgateConn, sourceKs, "update json_tbl set j1 = null, j2 = 'null', j3 = '\"null\"'") //execVtgateQuery(t, vtgateConn, sourceKs, "insert into json_tbl(id, j1, j2, j3) values (7, null, 'null', '\"null\"')") execVtgateQuery(t, vtgateConn, sourceKs, "update json_tbl set j1 = null, j2 = 'null', j3 = 'null'") diff --git a/go/vt/proto/binlogdata/binlogdata.pb.go b/go/vt/proto/binlogdata/binlogdata.pb.go index 0cc9d5b17cf..e3523b6b384 100644 --- a/go/vt/proto/binlogdata/binlogdata.pb.go +++ b/go/vt/proto/binlogdata/binlogdata.pb.go @@ -1336,9 +1336,9 @@ type RowChange struct { // DataColumns is a bitmap of all columns: bit is set if column is // present in the after image. DataColumns *RowChange_Bitmap `protobuf:"bytes,3,opt,name=data_columns,json=dataColumns,proto3" json:"data_columns,omitempty"` - // JsonPartialValues is a bitmap of any JSON columns where the bit is - // set if the value in the after image is a partial JSON value that - // is represented as an expression of + // JsonPartialValues is a bitmap of any JSON columns, where the bit + // is set if the value in the AFTER image is a partial JSON value + // that is represented as an expression of // JSON_[INSERT|REPLACE|REMOVE](%s, '$.path', value) which then is // used to add/update/remove a path in the JSON document. When the // value is used the fmt directive must be replaced by the actual diff --git a/go/vt/vttablet/tabletmanager/vreplication/replicator_plan.go b/go/vt/vttablet/tabletmanager/vreplication/replicator_plan.go index 7a51d5f5aea..206a1d9f037 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/replicator_plan.go +++ b/go/vt/vttablet/tabletmanager/vreplication/replicator_plan.go @@ -394,12 +394,14 @@ func (tp *TablePlan) applyChange(rowChange *binlogdatapb.RowChange, executor fun !slices.Equal(vals[i].Raw(), sqltypes.NullBytes): // An SQL expression that can be converted to a JSON value such // as JSON_INSERT(). - // This occurs e.g. when using partial JSON values as a result of + // This occurs when using partial JSON values as a result of // mysqld using binlog-row-value-options=PARTIAL_JSON. if len(vals[i].Raw()) == 0 { // When using BOTH binlog_row_image=NOBLOB AND - // binlog_row_value_options=PARTIAL_JSON then the JSON column - // has the data bit set and the diff is empty. + // binlog_row_value_options=PARTIAL_JSON then the JSON + // column has the data bit set and the diff is empty. So + // we have to account for this by unsetting the data bit + // so that the current JSON value is not overwritten. setBit(rowChange.DataColumns.Cols, i, false) newVal = ptr.Of(sqltypes.MakeTrusted(querypb.Type_EXPRESSION, nil)) } else { diff --git a/proto/binlogdata.proto b/proto/binlogdata.proto index 4351e48d12a..3d55de7ea14 100644 --- a/proto/binlogdata.proto +++ b/proto/binlogdata.proto @@ -336,9 +336,9 @@ message RowChange { // DataColumns is a bitmap of all columns: bit is set if column is // present in the after image. Bitmap data_columns = 3; - // JsonPartialValues is a bitmap of any JSON columns where the bit is - // set if the value in the after image is a partial JSON value that - // is represented as an expression of + // JsonPartialValues is a bitmap of any JSON columns, where the bit + // is set if the value in the AFTER image is a partial JSON value + // that is represented as an expression of // JSON_[INSERT|REPLACE|REMOVE](%s, '$.path', value) which then is // used to add/update/remove a path in the JSON document. When the // value is used the fmt directive must be replaced by the actual From 4fbbe4947e574c63c8474f1f66b2360c481e61af Mon Sep 17 00:00:00 2001 From: Matt Lord Date: Wed, 11 Dec 2024 09:30:39 -0500 Subject: [PATCH 14/39] Add new unit test cases Signed-off-by: Matt Lord --- go/mysql/binlog_event_mysql56_test.go | 44 +++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/go/mysql/binlog_event_mysql56_test.go b/go/mysql/binlog_event_mysql56_test.go index 77ce2c11d60..81eaa475b90 100644 --- a/go/mysql/binlog_event_mysql56_test.go +++ b/go/mysql/binlog_event_mysql56_test.go @@ -582,6 +582,50 @@ func TestMySQL56PartialUpdateRowsEvent(t *testing.T) { numRows: 1, want: "null", }, + { + name: "null literal string", + // The mysqlbinlog -vvv --base64-output=decode-rows output for the following event: + // ### UPDATE `vt_commerce`.`customer` + // ### WHERE + // ### @1=10 /* LONGINT meta=0 nullable=0 is_null=0 */ + // ### @2='mlord@planetscale.com' /* VARSTRING(128) meta=128 nullable=1 is_null=0 */ + // ### @3=NULL /* JSON meta=4 nullable=1 is_null=1 */ + // ### SET + // ### @1=10 /* LONGINT meta=0 nullable=0 is_null=0 */ + // ### @2='mlord@planetscale.com' /* VARSTRING(128) meta=128 nullable=1 is_null=0 */ + // ### @3='null' /* JSON meta=4 nullable=1 is_null=0 */ + rawEvent: []byte{ + 178, 168, 89, 103, 39, 37, 191, 137, 18, 105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 153, 0, 0, 0, 0, 0, 1, 0, 2, 0, 3, 7, 7, 4, 10, 0, 0, 0, 0, 0, 0, 0, 21, + 109, 108, 111, 114, 100, 64, 112, 108, 97, 110, 101, 116, 115, 99, 97, 108, 101, 46, 99, 111, 109, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 21, 109, 108, 111, + 114, 100, 64, 112, 108, 97, 110, 101, 116, 115, 99, 97, 108, 101, 46, 99, 111, 109, 6, 0, 0, 0, 12, 4, 110, 117, 108, 108, + }, + numRows: 1, + want: "\"null\"", + }, + { + name: "JSON object", + // The mysqlbinlog -vvv --base64-output=decode-rows output for the following event: + // ### UPDATE `vt_commerce`.`customer` + // ### WHERE + // ### @1=1 /* LONGINT meta=0 nullable=0 is_null=0 */ + // ### @2='newalice@domain.com' /* VARSTRING(128) meta=128 nullable=1 is_null=0 */ + // ### @3='{"day": "wednesday", "role": "manager", "color": "red", "salary": 100}' /* JSON meta=4 nullable=1 is_null=0 */ + // ### SET + // ### @1=1 /* LONGINT meta=0 nullable=0 is_null=0 */ + // ### @2='newalice@domain.com' /* VARSTRING(128) meta=128 nullable=1 is_null=0 */ + // ### @3=JSON_INSERT(@3, '$.misc', '{"address":"1012 S Park", "town":"Hastings", "state":"MI"}') /* JSON meta=4 nullable=1 is_null=0 */ + rawEvent: []byte{ + 208, 160, 89, 103, 39, 202, 59, 214, 68, 242, 0, 0, 0, 0, 0, 0, 0, 0, 0, 153, 0, 0, 0, 0, 0, 1, 0, 2, 0, 3, 7, 7, 0, 1, 0, 0, 0, 0, 0, 0, 0, 19, 110, + 101, 119, 97, 108, 105, 99, 101, 64, 100, 111, 109, 97, 105, 110, 46, 99, 111, 109, 73, 0, 0, 0, 0, 4, 0, 72, 0, 32, 0, 3, 0, 35, 0, 4, 0, 39, 0, 5, + 0, 44, 0, 6, 0, 12, 50, 0, 12, 60, 0, 12, 68, 0, 5, 100, 0, 100, 97, 121, 114, 111, 108, 101, 99, 111, 108, 111, 114, 115, 97, 108, 97, 114, 121, 9, + 119, 101, 100, 110, 101, 115, 100, 97, 121, 7, 109, 97, 110, 97, 103, 101, 114, 3, 114, 101, 100, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 19, 110, 101, 119, + 97, 108, 105, 99, 101, 64, 100, 111, 109, 97, 105, 110, 46, 99, 111, 109, 69, 0, 0, 0, 1, 6, 36, 46, 109, 105, 115, 99, 60, 12, 58, 123, 34, 97, 100, + 100, 114, 101, 115, 115, 34, 58, 34, 49, 48, 49, 50, 32, 83, 32, 80, 97, 114, 107, 34, 44, 32, 34, 116, 111, 119, 110, 34, 58, 34, 72, 97, 115, 116, + 105, 110, 103, 115, 34, 44, 32, 34, 115, 116, 97, 116, 101, 34, 58, 34, 77, 73, 34, 125, + }, + numRows: 1, + want: "JSON_INSERT(`%s`, _utf8mb4'$.misc', _utf8mb4\"{\\\"address\\\":\\\"1012 S Park\\\", \\\"town\\\":\\\"Hastings\\\", \\\"state\\\":\\\"MI\\\"}\")", + }, } for _, tc := range testCases { From 4d15ca2021f2a748ba756592a557fd610880bc8c Mon Sep 17 00:00:00 2001 From: Matt Lord Date: Wed, 11 Dec 2024 11:48:52 -0500 Subject: [PATCH 15/39] Fix bug with NULL JSON columns We still needed to increment the json col index for those. Signed-off-by: Matt Lord --- go/mysql/binlog_event_rbr.go | 3 +++ go/test/endtoend/vreplication/vreplication_test.go | 11 ++--------- go/vt/vttablet/tabletserver/vstreamer/vstreamer.go | 3 +++ 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/go/mysql/binlog_event_rbr.go b/go/mysql/binlog_event_rbr.go index f4f005b5dcd..29e0211bc36 100644 --- a/go/mysql/binlog_event_rbr.go +++ b/go/mysql/binlog_event_rbr.go @@ -435,6 +435,9 @@ func (rs *Rows) StringValuesForTests(tm *TableMap, rowIndex int) ([]string, erro // This column is represented, but its value is NULL. result = append(result, "NULL") valueIndex++ + if tm.Types[c] == binlog.TypeJSON { + jsonIndex++ + } continue } diff --git a/go/test/endtoend/vreplication/vreplication_test.go b/go/test/endtoend/vreplication/vreplication_test.go index 870b5d0d348..d3193298a0c 100644 --- a/go/test/endtoend/vreplication/vreplication_test.go +++ b/go/test/endtoend/vreplication/vreplication_test.go @@ -721,15 +721,8 @@ func shardCustomer(t *testing.T, testReverse bool, cells []*Cell, sourceCellOrAl // Confirm that the 0 scale decimal field, dec80, is replicated correctly execVtgateQuery(t, vtgateConn, sourceKs, "update customer set dec80 = 0") execVtgateQuery(t, vtgateConn, sourceKs, "update customer set blb = \"new blob data\" where cid=3") - // TODO: file a MySQL bug for this. The following query results in the quoted string literal "null" - // stored in the j3 column. But with and without the literal quotes, the value in the PARTIAL_JSON - // diff is the unquoted literal null which is a JSON null type. This leads to a vdiff mismatch. - // I'm not sure if the bug is that it allows the quoted value "null" to be inserted or that it - // doesn't reflect this in the partial diff value, but the combination of the two is certainly a bug. - //execVtgateQuery(t, vtgateConn, sourceKs, "update json_tbl set j1 = null, j2 = 'null', j3 = '\"null\"'") - //execVtgateQuery(t, vtgateConn, sourceKs, "insert into json_tbl(id, j1, j2, j3) values (7, null, 'null', '\"null\"')") - execVtgateQuery(t, vtgateConn, sourceKs, "update json_tbl set j1 = null, j2 = 'null', j3 = 'null'") - execVtgateQuery(t, vtgateConn, sourceKs, "insert into json_tbl(id, j1, j2, j3) values (7, null, 'null', 'null')") + execVtgateQuery(t, vtgateConn, sourceKs, "update json_tbl set j1 = null, j2 = 'null', j3 = '\"null\"'") + execVtgateQuery(t, vtgateConn, sourceKs, "insert into json_tbl(id, j1, j2, j3) values (7, null, 'null', '\"null\"')") waitForNoWorkflowLag(t, vc, targetKs, workflow) dec80Replicated := false for _, tablet := range []*cluster.VttabletProcess{customerTab1, customerTab2} { diff --git a/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go b/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go index b964f5d7e8e..1cedc01dbf1 100644 --- a/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go +++ b/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go @@ -1111,6 +1111,9 @@ func (vs *vstreamer) extractRowAndFilter(plan *streamerPlan, data []byte, dataCo } if nullColumns.Bit(valueIndex) { valueIndex++ + if plan.Table.Fields[colNum].Type == querypb.Type_JSON { + jsonIndex++ + } continue } partialJSON := false From 8171e14e76cb1d8c70ebc9f3ea21cb1ae47f950f Mon Sep 17 00:00:00 2001 From: Matt Lord Date: Wed, 11 Dec 2024 15:04:08 -0500 Subject: [PATCH 16/39] Add test queries to e2e test Signed-off-by: Matt Lord --- go/test/endtoend/vreplication/vreplication_test.go | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/go/test/endtoend/vreplication/vreplication_test.go b/go/test/endtoend/vreplication/vreplication_test.go index d3193298a0c..a6da24ef8fc 100644 --- a/go/test/endtoend/vreplication/vreplication_test.go +++ b/go/test/endtoend/vreplication/vreplication_test.go @@ -721,8 +721,17 @@ func shardCustomer(t *testing.T, testReverse bool, cells []*Cell, sourceCellOrAl // Confirm that the 0 scale decimal field, dec80, is replicated correctly execVtgateQuery(t, vtgateConn, sourceKs, "update customer set dec80 = 0") execVtgateQuery(t, vtgateConn, sourceKs, "update customer set blb = \"new blob data\" where cid=3") - execVtgateQuery(t, vtgateConn, sourceKs, "update json_tbl set j1 = null, j2 = 'null', j3 = '\"null\"'") + execVtgateQuery(t, vtgateConn, sourceKs, "update json_tbl set j1 = null, j2 = 'null', j3 = '\"null\"' where id = 5") execVtgateQuery(t, vtgateConn, sourceKs, "insert into json_tbl(id, j1, j2, j3) values (7, null, 'null', '\"null\"')") + // Test binlog-row-value-options=PARTIAL_JSON + execVtgateQuery(t, vtgateConn, sourceKs, "update json_tbl set j3 = JSON_SET(j3, '$.role', 'manager')") + execVtgateQuery(t, vtgateConn, sourceKs, "update json_tbl set j3 = JSON_SET(j3, '$.color', 'red')") + execVtgateQuery(t, vtgateConn, sourceKs, "update json_tbl set j3 = JSON_SET(j3, '$.day', 'wednesday')") + execVtgateQuery(t, vtgateConn, sourceKs, "update json_tbl set j3 = JSON_INSERT(JSON_REPLACE(j3, '$.day', 'friday'), '$.favorite_color', 'black')") + execVtgateQuery(t, vtgateConn, sourceKs, "update json_tbl set j3 = JSON_SET(JSON_REMOVE(JSON_REPLACE(j3, '$.day', 'monday'), '$.favorite_color'), '$.hobby', 'skiing') where id = 3") + execVtgateQuery(t, vtgateConn, sourceKs, "update json_tbl set j3 = JSON_SET(JSON_REMOVE(JSON_REPLACE(j3, '$.day', 'tuesday'), '$.favorite_color'), '$.hobby', 'skiing') where id = 4") + execVtgateQuery(t, vtgateConn, sourceKs, "update json_tbl set j3 = JSON_SET(JSON_SET(j3, '$.salary', 110), '$.role', 'IC') where id = 4") + execVtgateQuery(t, vtgateConn, sourceKs, "update json_tbl set j3 = JSON_SET(j3, '$.misc', '{\"address\":\"1012 S Park St\", \"town\":\"Hastings\", \"state\":\"MI\"}') where id = 1") waitForNoWorkflowLag(t, vc, targetKs, workflow) dec80Replicated := false for _, tablet := range []*cluster.VttabletProcess{customerTab1, customerTab2} { From dd324ef894f2e9b213e32777933a5af3cfea21ff Mon Sep 17 00:00:00 2001 From: Matt Lord Date: Thu, 12 Dec 2024 13:27:33 -0500 Subject: [PATCH 17/39] Properly handle PK changes Signed-off-by: Matt Lord --- go/mysql/binlog/binlog_json.go | 10 +- go/mysql/binlog_event_mysql56_test.go | 16 +-- .../vreplication/vreplication_test.go | 1 + .../vreplication/replicator_plan.go | 111 ++++++++++++++---- .../vreplication/vplayer_flaky_test.go | 61 ++++++++++ 5 files changed, 162 insertions(+), 37 deletions(-) diff --git a/go/mysql/binlog/binlog_json.go b/go/mysql/binlog/binlog_json.go index e3ca52096f2..8db78edfd23 100644 --- a/go/mysql/binlog/binlog_json.go +++ b/go/mysql/binlog/binlog_json.go @@ -27,6 +27,7 @@ import ( "vitess.io/vitess/go/mysql/format" "vitess.io/vitess/go/mysql/json" "vitess.io/vitess/go/sqltypes" + "vitess.io/vitess/go/vt/sqlparser" "vitess.io/vitess/go/vt/vterrors" querypb "vitess.io/vitess/go/vt/proto/query" @@ -110,7 +111,7 @@ func ParseBinaryJSONDiff(data []byte) (sqltypes.Value, error) { diff.WriteString(innerStr) diff.WriteString(", ") } else { // Only the inner most function has the field name - diff.WriteString("`%s`, ") // This will later be replaced by the field name + diff.WriteString("%s, ") // This will later be replaced by the field name } pathLen, readTo := readVariableLength(data, pos) @@ -120,7 +121,10 @@ func ParseBinaryJSONDiff(data []byte) (sqltypes.Value, error) { // We have to specify the unicode character set for the strings we // use in the expression as the connection can be using a different // character set (e.g. vreplication always uses set names binary). - diff.WriteString(fmt.Sprintf("_utf8mb4'%s'", path)) + diff.WriteString(sqlparser.Utf8mb4Str) + diff.WriteByte('\'') + diff.Write(path) + diff.WriteByte('\'') if opType == jsonDiffOpRemove { // No value for remove diff.WriteString(")") @@ -135,7 +139,7 @@ func ParseBinaryJSONDiff(data []byte) (sqltypes.Value, error) { } pos += valueLen if value.Type() == json.TypeString { - diff.WriteString("_utf8mb4") + diff.WriteString(sqlparser.Utf8mb4Str) } diff.WriteString(fmt.Sprintf("%s)", value)) } diff --git a/go/mysql/binlog_event_mysql56_test.go b/go/mysql/binlog_event_mysql56_test.go index 81eaa475b90..9ad38473205 100644 --- a/go/mysql/binlog_event_mysql56_test.go +++ b/go/mysql/binlog_event_mysql56_test.go @@ -352,7 +352,7 @@ func TestMySQL56PartialUpdateRowsEvent(t *testing.T) { 97, 110, 97, 103, 101, 114, }, numRows: 5, - want: "JSON_INSERT(`%s`, _utf8mb4'$.role', _utf8mb4\"manager\")", + want: "JSON_INSERT(%s, _utf8mb4'$.role', _utf8mb4\"manager\")", }, { // The mysqlbinlog -vvv --base64-output=decode-rows output for the following event: @@ -373,7 +373,7 @@ func TestMySQL56PartialUpdateRowsEvent(t *testing.T) { }, name: "REPLACE", numRows: 1, - want: "JSON_REPLACE(`%s`, _utf8mb4'$.role', _utf8mb4\"IC\")", + want: "JSON_REPLACE(%s, _utf8mb4'$.role', _utf8mb4\"IC\")", }, { name: "REMOVE", @@ -394,7 +394,7 @@ func TestMySQL56PartialUpdateRowsEvent(t *testing.T) { 111, 108, 101, 115, 97, 108, 97, 114, 121, 7, 109, 97, 110, 97, 103, 101, 114, 1, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 14, 98, 111, 98, 64, 100, 111, 109, 97, 105, 110, 46, 99, 111, 109, 10, 0, 0, 0, 2, 8, 36, 46, 115, 97, 108, 97, 114, 121, }, - want: "JSON_REMOVE(`%s`, _utf8mb4'$.salary')", + want: "JSON_REMOVE(%s, _utf8mb4'$.salary')", }, { name: "REMOVE and REPLACE", @@ -486,7 +486,7 @@ func TestMySQL56PartialUpdateRowsEvent(t *testing.T) { 100, 97, 121, 2, 16, 36, 46, 102, 97, 118, 111, 114, 105, 116, 101, 95, 99, 111, 108, 111, 114, }, numRows: 5, - want: "JSON_REMOVE(JSON_REPLACE(`%s`, _utf8mb4'$.day', _utf8mb4\"monday\"), _utf8mb4'$.favorite_color')", + want: "JSON_REMOVE(JSON_REPLACE(%s, _utf8mb4'$.day', _utf8mb4\"monday\"), _utf8mb4'$.favorite_color')", }, { name: "INSERT and REMOVE and REPLACE", @@ -514,7 +514,7 @@ func TestMySQL56PartialUpdateRowsEvent(t *testing.T) { 1, 7, 36, 46, 104, 111, 98, 98, 121, 8, 12, 6, 115, 107, 105, 105, 110, 103, }, numRows: 1, - want: "JSON_INSERT(JSON_REMOVE(JSON_REPLACE(`%s`, _utf8mb4'$.day', _utf8mb4\"tuesday\"), _utf8mb4'$.favorite_color'), _utf8mb4'$.hobby', _utf8mb4\"skiing\")", + want: "JSON_INSERT(JSON_REMOVE(JSON_REPLACE(%s, _utf8mb4'$.day', _utf8mb4\"tuesday\"), _utf8mb4'$.favorite_color'), _utf8mb4'$.hobby', _utf8mb4\"skiing\")", }, { name: "REPLACE with null", @@ -535,7 +535,7 @@ func TestMySQL56PartialUpdateRowsEvent(t *testing.T) { 109, 97, 105, 110, 46, 99, 111, 109, 13, 0, 0, 0, 0, 8, 36, 46, 115, 97, 108, 97, 114, 121, 2, 4, 0, }, numRows: 1, - want: "JSON_REPLACE(`%s`, _utf8mb4'$.salary', null)", + want: "JSON_REPLACE(%s, _utf8mb4'$.salary', null)", }, { name: "REPLACE 2 paths", @@ -557,7 +557,7 @@ func TestMySQL56PartialUpdateRowsEvent(t *testing.T) { 105, 110, 46, 99, 111, 109, 27, 0, 0, 0, 0, 8, 36, 46, 115, 97, 108, 97, 114, 121, 3, 5, 110, 0, 0, 6, 36, 46, 114, 111, 108, 101, 4, 12, 2, 73, 67, }, numRows: 1, - want: "JSON_REPLACE(JSON_REPLACE(`%s`, _utf8mb4'$.salary', 110), _utf8mb4'$.role', _utf8mb4\"IC\")", + want: "JSON_REPLACE(JSON_REPLACE(%s, _utf8mb4'$.salary', 110), _utf8mb4'$.role', _utf8mb4\"IC\")", }, { name: "JSON null", @@ -624,7 +624,7 @@ func TestMySQL56PartialUpdateRowsEvent(t *testing.T) { 105, 110, 103, 115, 34, 44, 32, 34, 115, 116, 97, 116, 101, 34, 58, 34, 77, 73, 34, 125, }, numRows: 1, - want: "JSON_INSERT(`%s`, _utf8mb4'$.misc', _utf8mb4\"{\\\"address\\\":\\\"1012 S Park\\\", \\\"town\\\":\\\"Hastings\\\", \\\"state\\\":\\\"MI\\\"}\")", + want: "JSON_INSERT(%s, _utf8mb4'$.misc', _utf8mb4\"{\\\"address\\\":\\\"1012 S Park\\\", \\\"town\\\":\\\"Hastings\\\", \\\"state\\\":\\\"MI\\\"}\")", }, } diff --git a/go/test/endtoend/vreplication/vreplication_test.go b/go/test/endtoend/vreplication/vreplication_test.go index a6da24ef8fc..4269bd68cab 100644 --- a/go/test/endtoend/vreplication/vreplication_test.go +++ b/go/test/endtoend/vreplication/vreplication_test.go @@ -732,6 +732,7 @@ func shardCustomer(t *testing.T, testReverse bool, cells []*Cell, sourceCellOrAl execVtgateQuery(t, vtgateConn, sourceKs, "update json_tbl set j3 = JSON_SET(JSON_REMOVE(JSON_REPLACE(j3, '$.day', 'tuesday'), '$.favorite_color'), '$.hobby', 'skiing') where id = 4") execVtgateQuery(t, vtgateConn, sourceKs, "update json_tbl set j3 = JSON_SET(JSON_SET(j3, '$.salary', 110), '$.role', 'IC') where id = 4") execVtgateQuery(t, vtgateConn, sourceKs, "update json_tbl set j3 = JSON_SET(j3, '$.misc', '{\"address\":\"1012 S Park St\", \"town\":\"Hastings\", \"state\":\"MI\"}') where id = 1") + execVtgateQuery(t, vtgateConn, sourceKs, "update json_tbl set id=id+1000, j3=JSON_SET(j3, '$.day', 'friday')") waitForNoWorkflowLag(t, vc, targetKs, workflow) dec80Replicated := false for _, tablet := range []*cluster.VttabletProcess{customerTab1, customerTab2} { diff --git a/go/vt/vttablet/tabletmanager/vreplication/replicator_plan.go b/go/vt/vttablet/tabletmanager/vreplication/replicator_plan.go index 206a1d9f037..43e1e2dc1e1 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/replicator_plan.go +++ b/go/vt/vttablet/tabletmanager/vreplication/replicator_plan.go @@ -17,6 +17,7 @@ limitations under the License. package vreplication import ( + "bytes" "encoding/json" "fmt" "slices" @@ -30,6 +31,7 @@ import ( vjson "vitess.io/vitess/go/mysql/json" "vitess.io/vitess/go/mysql/sqlerror" "vitess.io/vitess/go/ptr" + "vitess.io/vitess/go/sqlescape" "vitess.io/vitess/go/sqltypes" "vitess.io/vitess/go/vt/binlog/binlogplayer" "vitess.io/vitess/go/vt/sqlparser" @@ -365,7 +367,10 @@ func (tp *TablePlan) bindFieldVal(field *querypb.Field, val *sqltypes.Value) (*q func (tp *TablePlan) applyChange(rowChange *binlogdatapb.RowChange, executor func(string) (*sqltypes.Result, error)) (*sqltypes.Result, error) { // MakeRowTrusted is needed here because Proto3ToResult is not convenient. - var before, after bool + var ( + before, after bool + afterVals []sqltypes.Value + ) bindvars := make(map[string]*querypb.BindVariable, len(tp.Fields)) if rowChange.Before != nil { before = true @@ -381,36 +386,37 @@ func (tp *TablePlan) applyChange(rowChange *binlogdatapb.RowChange, executor fun if rowChange.After != nil { jsonIndex := 0 after = true - vals := sqltypes.MakeRowTrusted(tp.Fields, rowChange.After) + afterVals = sqltypes.MakeRowTrusted(tp.Fields, rowChange.After) for i, field := range tp.Fields { - var bindVar *querypb.BindVariable - var newVal *sqltypes.Value - var err error + var ( + bindVar *querypb.BindVariable + newVal *sqltypes.Value + err error + ) if field.Type == querypb.Type_JSON { switch { - case vals[i].IsNull(): // An SQL NULL and not an actual JSON value + case afterVals[i].IsNull(): // An SQL NULL and not an actual JSON value newVal = &sqltypes.NULL case rowChange.JsonPartialValues != nil && isBitSet(rowChange.JsonPartialValues.Cols, jsonIndex) && - !slices.Equal(vals[i].Raw(), sqltypes.NullBytes): - // An SQL expression that can be converted to a JSON value such - // as JSON_INSERT(). - // This occurs when using partial JSON values as a result of - // mysqld using binlog-row-value-options=PARTIAL_JSON. - if len(vals[i].Raw()) == 0 { + !slices.Equal(afterVals[i].Raw(), sqltypes.NullBytes): + // An SQL expression that can be converted to a JSON value such as JSON_INSERT(). + // This occurs when using partial JSON values as a result of mysqld using + // binlog-row-value-options=PARTIAL_JSON. + if len(afterVals[i].Raw()) == 0 { // When using BOTH binlog_row_image=NOBLOB AND - // binlog_row_value_options=PARTIAL_JSON then the JSON - // column has the data bit set and the diff is empty. So - // we have to account for this by unsetting the data bit - // so that the current JSON value is not overwritten. + // binlog_row_value_options=PARTIAL_JSON then the JSON column has the data bit + // set and the diff is empty when it's not present. So we have to account for + // this by unsetting the data bit so that the current JSON value is not lost. setBit(rowChange.DataColumns.Cols, i, false) newVal = ptr.Of(sqltypes.MakeTrusted(querypb.Type_EXPRESSION, nil)) } else { + escapedName := sqlescape.EscapeID(field.Name) newVal = ptr.Of(sqltypes.MakeTrusted(querypb.Type_EXPRESSION, []byte( - fmt.Sprintf(vals[i].RawStr(), field.Name), + fmt.Sprintf(afterVals[i].RawStr(), escapedName), ))) } default: // A JSON value (which may be a JSON null literal value) - newVal, err = vjson.MarshalSQLValue(vals[i].Raw()) + newVal, err = vjson.MarshalSQLValue(afterVals[i].Raw()) if err != nil { return nil, err } @@ -418,7 +424,7 @@ func (tp *TablePlan) applyChange(rowChange *binlogdatapb.RowChange, executor fun bindVar, err = tp.bindFieldVal(field, newVal) jsonIndex++ } else { - bindVar, err = tp.bindFieldVal(field, &vals[i]) + bindVar, err = tp.bindFieldVal(field, &afterVals[i]) } if err != nil { return nil, err @@ -428,7 +434,7 @@ func (tp *TablePlan) applyChange(rowChange *binlogdatapb.RowChange, executor fun } switch { case !before && after: - // only apply inserts for rows whose primary keys are within the range of rows already copied + // Only apply inserts for rows whose primary keys are within the range of rows already copied. if tp.isOutsidePKRange(bindvars, before, after, "insert") { return nil, nil } @@ -465,15 +471,68 @@ func (tp *TablePlan) applyChange(rowChange *binlogdatapb.RowChange, executor fun return nil, err } } - // TODO: the INSERTs done here after deleting the row with the original PK - // need to use the values from the BEFORE image for the columns NOT present - // in the AFTER image due to being a partial image due to the source's usage - // of binlog-row-image=NOBLOB. - // For JSON columns when binlog-row-value-options=PARTIAL_JSON is used, we - // need to wrap the JSON diff function(s) around the BEFORE value. if tp.isOutsidePKRange(bindvars, before, after, "insert") { return nil, nil } + if tp.isPartial(rowChange) { + // We need to use a combination of the values in the BEFORE and AFTER image to generate + // the new row. + jsonIndex := 0 + for i, field := range tp.Fields { + if field.Type == querypb.Type_JSON && rowChange.JsonPartialValues != nil { + if !isBitSet(rowChange.JsonPartialValues.Cols, jsonIndex) { + // We use the full AFTER value which we already have. + jsonIndex++ + continue + } + if len(afterVals[i].Raw()) == 0 { + // When using BOTH binlog_row_image=NOBLOB AND + // binlog_row_value_options=PARTIAL_JSON then the JSON column has the data bit + // set and the diff is empty when it's not present. So we want to use the + // BEFORE image value. + beforeVal, err := vjson.MarshalSQLValue(bindvars["b_"+field.Name].Value) + if err != nil { + return nil, vterrors.Wrapf(err, "failed to convert JSON to SQL field value for %s.%s when building insert query", + tp.TargetName, field.Name) + } + bindvars["a_"+field.Name], err = tp.bindFieldVal(field, beforeVal) + if err != nil { + return nil, vterrors.Wrapf(err, "failed to bind field value for %s.%s when building insert query", + tp.TargetName, field.Name) + } + } else { + // For JSON columns when binlog-row-value-options=PARTIAL_JSON is used, we + // need to wrap the JSON diff function(s) around the BEFORE value. + diff := bindvars["a_"+field.Name].Value + beforeVal := bindvars["b_"+field.Name].Value + afterVal := bytes.Buffer{} + afterVal.Grow(len(diff) + len(beforeVal) + len(sqlparser.Utf8mb4Str) + 2) // +2 is for the enclosing quotes + // If the JSON column is partial, we need to specify the BEFORE value as + // the input for the diff function(s). + afterVal.WriteString(sqlparser.Utf8mb4Str) + afterVal.WriteByte('\'') + afterVal.Write(beforeVal) + afterVal.WriteByte('\'') + newVal := sqltypes.MakeTrusted(querypb.Type_EXPRESSION, []byte( + fmt.Sprintf(afterVals[i].RawStr(), afterVal.String()), + )) + bindVar, err := tp.bindFieldVal(field, &newVal) + if err != nil { + return nil, vterrors.Wrapf(err, "failed to bind field value for %s.%s when building insert query", + tp.TargetName, field.Name) + } + bindvars["a_"+field.Name] = bindVar + } + jsonIndex++ + continue + } + if !isBitSet(rowChange.DataColumns.Cols, i) { + return nil, vterrors.Errorf(vtrpcpb.Code_INTERNAL, + "binary log event missing a needed value for %s.%s due to the usage of binlog-row-image=NOBLOB; you will need to re-run the workflow with binlog-row-image=FULL", + tp.TargetName, field.Name) + } + } + } return execParsedQuery(tp.Insert, bindvars, executor) } // Unreachable. diff --git a/go/vt/vttablet/tabletmanager/vreplication/vplayer_flaky_test.go b/go/vt/vttablet/tabletmanager/vreplication/vplayer_flaky_test.go index 50d93e60e5a..9c17780b66e 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/vplayer_flaky_test.go +++ b/go/vt/vttablet/tabletmanager/vreplication/vplayer_flaky_test.go @@ -1519,6 +1519,67 @@ func TestPlayerRowMove(t *testing.T) { validateQueryCountStat(t, "replicate", 3) } +/* TODO: build this out and get it working +func TestPlayerUpdatePK(t *testing.T) { + defer deleteTablet(addTablet(100)) + + execStatements(t, []string{ + "create table src(id int, bd blob, jd json, primary key(id))", + fmt.Sprintf("create table %s.dst(id int, bd blob, jd json, primary key(id))", vrepldb), + }) + defer execStatements(t, []string{ + "drop table src", + fmt.Sprintf("drop table %s.dst", vrepldb), + }) + + filter := &binlogdatapb.Filter{ + Rules: []*binlogdatapb.Rule{{ + Match: "dst", + Filter: "select * from src", + }}, + } + bls := &binlogdatapb.BinlogSource{ + Keyspace: env.KeyspaceName, + Shard: env.ShardName, + Filter: filter, + OnDdl: binlogdatapb.OnDDLAction_IGNORE, + } + cancel, _ := startVReplication(t, bls, "") + defer cancel() + + execStatements(t, []string{ + "insert into src values(1, 'blob data', _utf8mb4'{\"key1\":\"val1\"}'), (2, 'blob data2', _utf8mb4'{\"key2\":\"val2\"}'), (3, 'blob data3', _utf8mb4'{\"key3\":\"val3\"}')", + }) + expectDBClientQueries(t, qh.Expect( + "begin", + "insert into dst(id,bd,jd) values (1,_binary'blob data','{\"key1\": \"val1\"}'), (2,_binary'blob data2','{\"key2\": \"val2\"}'), (3,_binary'blob data3','{\"key3\": \"val3\"}')", + "/update _vt.vreplication set pos=", + "commit", + )) + expectData(t, "dst", [][]string{ + {"1", "1", "1"}, + {"2", "5", "2"}, + }) + validateQueryCountStat(t, "replicate", 1) + + execStatements(t, []string{ + "update src set val1=1, val2=4 where id=3", + }) + expectDBClientQueries(t, qh.Expect( + "begin", + "update dst set sval2=sval2-ifnull(3, 0), rcount=rcount-1 where val1=2", + "insert into dst(val1,sval2,rcount) values (1,ifnull(4, 0),1) on duplicate key update sval2=sval2+ifnull(values(sval2), 0), rcount=rcount+1", + "/update _vt.vreplication set pos=", + "commit", + )) + expectData(t, "dst", [][]string{ + {"1", "5", "2"}, + {"2", "2", "1"}, + }) + validateQueryCountStat(t, "replicate", 3) +} +*/ + func TestPlayerTypes(t *testing.T) { defer deleteTablet(addTablet(100)) execStatements(t, []string{ From eb3f1b0a7f8cb673fc4e7cafd097fae1cf97f136 Mon Sep 17 00:00:00 2001 From: Matt Lord Date: Fri, 13 Dec 2024 12:13:53 -0500 Subject: [PATCH 18/39] Add unit test Signed-off-by: Matt Lord --- go/mysql/binlog/binlog_json.go | 5 +- go/test/utils/binlog.go | 12 +- .../vreplication/replicator_plan.go | 21 ++- .../vreplication/vplayer_flaky_test.go | 145 ++++++++++++------ 4 files changed, 134 insertions(+), 49 deletions(-) diff --git a/go/mysql/binlog/binlog_json.go b/go/mysql/binlog/binlog_json.go index 8db78edfd23..33a9c474bd8 100644 --- a/go/mysql/binlog/binlog_json.go +++ b/go/mysql/binlog/binlog_json.go @@ -127,7 +127,7 @@ func ParseBinaryJSONDiff(data []byte) (sqltypes.Value, error) { diff.WriteByte('\'') if opType == jsonDiffOpRemove { // No value for remove - diff.WriteString(")") + diff.WriteByte(')') } else { diff.WriteString(", ") valueLen, readTo := readVariableLength(data, pos) @@ -141,7 +141,8 @@ func ParseBinaryJSONDiff(data []byte) (sqltypes.Value, error) { if value.Type() == json.TypeString { diff.WriteString(sqlparser.Utf8mb4Str) } - diff.WriteString(fmt.Sprintf("%s)", value)) + diff.Write(value.MarshalTo(nil)) + diff.WriteByte(')') } outer = true diff --git a/go/test/utils/binlog.go b/go/test/utils/binlog.go index d3f686f1a8a..a9eafc4941d 100644 --- a/go/test/utils/binlog.go +++ b/go/test/utils/binlog.go @@ -27,7 +27,8 @@ const ( BinlogRowImageCnf = "binlog-row-image.cnf" ) -// SetBinlogRowImageMode creates a temp cnf file to set binlog_row_image to noblob for vreplication unit tests. +// SetBinlogRowImageMode creates a temp cnf file to set binlog_row_image=NOBLOB and +// binlog_row_value_options=PARTIAL_JSON for vreplication unit tests. // It adds it to the EXTRA_MY_CNF environment variable which appends text from them into my.cnf. func SetBinlogRowImageMode(mode string, cnfDir string) error { var newCnfs []string @@ -55,6 +56,15 @@ func SetBinlogRowImageMode(mode string, cnfDir string) error { if err != nil { return err } + lm := strings.ToLower(mode) + if lm == "noblob" || lm == "minimal" { + // We're testing partial binlog row images so let's also test partial + // JSON values in the images. + _, err = f.WriteString("\nbinlog_row_value_options=PARTIAL_JSON\n") + if err != nil { + return err + } + } err = f.Close() if err != nil { return err diff --git a/go/vt/vttablet/tabletmanager/vreplication/replicator_plan.go b/go/vt/vttablet/tabletmanager/vreplication/replicator_plan.go index 43e1e2dc1e1..fa3a225b18b 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/replicator_plan.go +++ b/go/vt/vttablet/tabletmanager/vreplication/replicator_plan.go @@ -528,7 +528,7 @@ func (tp *TablePlan) applyChange(rowChange *binlogdatapb.RowChange, executor fun } if !isBitSet(rowChange.DataColumns.Cols, i) { return nil, vterrors.Errorf(vtrpcpb.Code_INTERNAL, - "binary log event missing a needed value for %s.%s due to the usage of binlog-row-image=NOBLOB; you will need to re-run the workflow with binlog-row-image=FULL", + "binary log event missing a needed value for %s.%s due to not using binlog-row-image=FULL; you will need to re-run the workflow with binlog-row-image=FULL", tp.TargetName, field.Name) } } @@ -629,11 +629,28 @@ func (tp *TablePlan) applyBulkInsertChanges(rowInserts []*binlogdatapb.RowChange newStmt := true for _, rowInsert := range rowInserts { + var ( + err error + bindVar *querypb.BindVariable + ) rowValues := &strings.Builder{} bindvars := make(map[string]*querypb.BindVariable, len(tp.Fields)) vals := sqltypes.MakeRowTrusted(tp.Fields, rowInsert.After) for n, field := range tp.Fields { - bindVar, err := tp.bindFieldVal(field, &vals[n]) + if field.Type == querypb.Type_JSON { + var jsVal *sqltypes.Value + if vals[n].IsNull() { // An SQL NULL and not an actual JSON value + jsVal = &sqltypes.NULL + } else { // A JSON value (which may be a JSON null literal value) + jsVal, err = vjson.MarshalSQLValue(vals[n].Raw()) + if err != nil { + return nil, err + } + } + bindVar, err = tp.bindFieldVal(field, jsVal) + } else { + bindVar, err = tp.bindFieldVal(field, &vals[n]) + } if err != nil { return nil, err } diff --git a/go/vt/vttablet/tabletmanager/vreplication/vplayer_flaky_test.go b/go/vt/vttablet/tabletmanager/vreplication/vplayer_flaky_test.go index 9c17780b66e..90b514b5b6d 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/vplayer_flaky_test.go +++ b/go/vt/vttablet/tabletmanager/vreplication/vplayer_flaky_test.go @@ -1519,13 +1519,20 @@ func TestPlayerRowMove(t *testing.T) { validateQueryCountStat(t, "replicate", 3) } -/* TODO: build this out and get it working -func TestPlayerUpdatePK(t *testing.T) { - defer deleteTablet(addTablet(100)) +// TestPlayerPartialImagesUpdatePK tests the behavior of the vplayer when we +// have partial binlog images, meaning that binlog-row-image=NOBLOB and +// binlog-row-value-options=PARTIAL_JSON. These are both set together when +// running the unit tests with runNoBlobTest=true. So we skip the test if +// it's not set. +func TestPlayerPartialImagesUpdatePK(t *testing.T) { + if !runNoBlobTest { + t.Skip("Skipping test as runNoBlobTest is not set") + } + defer deleteTablet(addTablet(100)) execStatements(t, []string{ - "create table src(id int, bd blob, jd json, primary key(id))", - fmt.Sprintf("create table %s.dst(id int, bd blob, jd json, primary key(id))", vrepldb), + "create table src (id int, jd json, bd blob, primary key(id))", + fmt.Sprintf("create table %s.dst (id int, jd json, bd blob, primary key(id))", vrepldb), }) defer execStatements(t, []string{ "drop table src", @@ -1547,38 +1554,73 @@ func TestPlayerUpdatePK(t *testing.T) { cancel, _ := startVReplication(t, bls, "") defer cancel() - execStatements(t, []string{ - "insert into src values(1, 'blob data', _utf8mb4'{\"key1\":\"val1\"}'), (2, 'blob data2', _utf8mb4'{\"key2\":\"val2\"}'), (3, 'blob data3', _utf8mb4'{\"key3\":\"val3\"}')", - }) - expectDBClientQueries(t, qh.Expect( - "begin", - "insert into dst(id,bd,jd) values (1,_binary'blob data','{\"key1\": \"val1\"}'), (2,_binary'blob data2','{\"key2\": \"val2\"}'), (3,_binary'blob data3','{\"key3\": \"val3\"}')", - "/update _vt.vreplication set pos=", - "commit", - )) - expectData(t, "dst", [][]string{ - {"1", "1", "1"}, - {"2", "5", "2"}, - }) - validateQueryCountStat(t, "replicate", 1) - - execStatements(t, []string{ - "update src set val1=1, val2=4 where id=3", - }) - expectDBClientQueries(t, qh.Expect( - "begin", - "update dst set sval2=sval2-ifnull(3, 0), rcount=rcount-1 where val1=2", - "insert into dst(val1,sval2,rcount) values (1,ifnull(4, 0),1) on duplicate key update sval2=sval2+ifnull(values(sval2), 0), rcount=rcount+1", - "/update _vt.vreplication set pos=", - "commit", - )) - expectData(t, "dst", [][]string{ - {"1", "5", "2"}, - {"2", "2", "1"}, - }) - validateQueryCountStat(t, "replicate", 3) + testCases := []struct { + input string + output []string + data [][]string + error string + }{ + { + input: "insert into src (id, jd, bd) values (1,'{\"key1\": \"val1\"}','blob data'), (2,'{\"key2\": \"val2\"}','blob data2'), (3,'{\"key3\": \"val3\"}','blob data3')", + output: []string{"insert into dst(id,jd,bd) values (1,JSON_OBJECT(_utf8mb4'key1', _utf8mb4'val1'),_binary'blob data'), (2,JSON_OBJECT(_utf8mb4'key2', _utf8mb4'val2'),_binary'blob data2'), (3,JSON_OBJECT(_utf8mb4'key3', _utf8mb4'val3'),_binary'blob data3')"}, + data: [][]string{ + {"1", "{\"key1\": \"val1\"}", "blob data"}, + {"2", "{\"key2\": \"val2\"}", "blob data2"}, + {"3", "{\"key3\": \"val3\"}", "blob data3"}, + }, + }, + { + input: `update src set jd=JSON_SET(jd, '$.color', 'red') where id = 1`, + output: []string{"update dst set jd=JSON_INSERT(`jd`, _utf8mb4'$.color', _utf8mb4\"red\") where id=1"}, + data: [][]string{ + {"1", "{\"key1\": \"val1\", \"color\": \"red\"}", "blob data"}, + {"2", "{\"key2\": \"val2\"}", "blob data2"}, + {"3", "{\"key3\": \"val3\"}", "blob data3"}, + }, + }, + { + input: `update src set id = id+10, bd = 'new blob data' where id = 2`, + output: []string{ + "delete from dst where id=2", + "insert into dst(id,jd,bd) values (12,JSON_OBJECT(_utf8mb4'key2', _utf8mb4'val2'),_binary'new blob data')", + }, + data: [][]string{ + {"1", "{\"key1\": \"val1\", \"color\": \"red\"}", "blob data"}, + {"3", "{\"key3\": \"val3\"}", "blob data3"}, + {"12", "{\"key2\": \"val2\"}", "new blob data"}, + }, + }, + { + input: `update src set id = id+10 where id = 3`, + error: "binary log event missing a needed value for dst.bd due to not using binlog-row-image=FULL", + }, + } + + for _, tc := range testCases { + t.Run(tc.input, func(t *testing.T) { + execStatements(t, []string{tc.input}) + var want qh.ExpectationSequencer + if tc.error != "" { + want = qh.Expect( + "rollback", + ).Then(qh.Immediately( + fmt.Sprintf("/update _vt.vreplication set message=.*%s.*", tc.error), + )) + expectDBClientQueries(t, want) + } else { + want = qh.Expect( + "begin", + tc.output..., + ).Then(qh.Immediately( + "/update _vt.vreplication set pos=", + "commit", + )) + expectDBClientQueries(t, want) + expectData(t, "dst", tc.data) + } + }) + } } -*/ func TestPlayerTypes(t *testing.T) { defer deleteTablet(addTablet(100)) @@ -1714,15 +1756,30 @@ func TestPlayerTypes(t *testing.T) { {"1", "", "{}", "123", `{"a": [42, 100]}`, `{"foo": "bar"}`}, {"2", "null", `{"name": null}`, "123", `{"a": [42, 100]}`, `{"foo": "bar"}`}, }, - }, { - input: "update vitess_json set val1 = '{\"bar\": \"foo\"}', val4 = '{\"a\": [98, 123]}', val5 = convert(x'7b7d' using utf8mb4) where id=1", - output: "update vitess_json set val1=JSON_OBJECT(_utf8mb4'bar', _utf8mb4'foo'), val2=JSON_OBJECT(), val3=CAST(123 as JSON), val4=JSON_OBJECT(_utf8mb4'a', JSON_ARRAY(98, 123)), val5=JSON_OBJECT() where id=1", - table: "vitess_json", - data: [][]string{ - {"1", `{"bar": "foo"}`, "{}", "123", `{"a": [98, 123]}`, `{}`}, - {"2", "null", `{"name": null}`, "123", `{"a": [42, 100]}`, `{"foo": "bar"}`}, - }, }} + if !runNoBlobTest { + testcases = append(testcases, testcase{ + input: "update vitess_json set val1 = '{\"bar\": \"foo\"}', val4 = '{\"a\": [98, 123]}', val5 = convert(x'7b7d' using utf8mb4) where id=1", + output: "update vitess_json set val1=JSON_OBJECT(_utf8mb4'bar', _utf8mb4'foo'), val2=JSON_OBJECT(), val3=CAST(123 as JSON), val4=JSON_OBJECT(_utf8mb4'a', JSON_ARRAY(98, 123)), val5=JSON_OBJECT() where id=1", + table: "vitess_json", + data: [][]string{ + {"1", `{"bar": "foo"}`, "{}", "123", `{"a": [98, 123]}`, `{}`}, + {"2", "null", `{"name": null}`, "123", `{"a": [42, 100]}`, `{"foo": "bar"}`}, + }, + }) + } else { + // With partial JSON values we don't replicate the JSON columns that aren't + // actually updated. + testcases = append(testcases, testcase{ + input: "update vitess_json set val1 = '{\"bar\": \"foo\"}', val4 = '{\"a\": [98, 123]}', val5 = convert(x'7b7d' using utf8mb4) where id=1", + output: "update vitess_json set val1=JSON_OBJECT(_utf8mb4'bar', _utf8mb4'foo'), val4=JSON_OBJECT(_utf8mb4'a', JSON_ARRAY(98, 123)), val5=JSON_OBJECT() where id=1", + table: "vitess_json", + data: [][]string{ + {"1", `{"bar": "foo"}`, "{}", "123", `{"a": [98, 123]}`, `{}`}, + {"2", "null", `{"name": null}`, "123", `{"a": [42, 100]}`, `{"foo": "bar"}`}, + }, + }) + } for _, tcases := range testcases { execStatements(t, []string{tcases.input}) From 91b1e33f344af109121fb350d67079e19149100e Mon Sep 17 00:00:00 2001 From: Matt Lord Date: Fri, 13 Dec 2024 18:52:57 -0500 Subject: [PATCH 19/39] Support optional unit test behavior for 5.7 Signed-off-by: Matt Lord --- .../unit_test_evalengine_mysql57.yml | 3 ++ .../unit_test_evalengine_mysql80.yml | 3 ++ .../unit_test_evalengine_mysql84.yml | 3 ++ .github/workflows/unit_test_mysql57.yml | 3 ++ .github/workflows/unit_test_mysql80.yml | 3 ++ .github/workflows/unit_test_mysql84.yml | 3 ++ .../vreplication/vreplication_test.go | 12 ++++-- go/test/utils/binlog.go | 43 ++++++++++++++++--- go/test/utils/binlog_test.go | 16 ++++++- .../vreplication/framework_test.go | 21 ++++++--- .../vreplication/vplayer_flaky_test.go | 29 ++++++++----- test/templates/unit_test.tpl | 3 ++ 12 files changed, 114 insertions(+), 28 deletions(-) diff --git a/.github/workflows/unit_test_evalengine_mysql57.yml b/.github/workflows/unit_test_evalengine_mysql57.yml index ecc366b38fe..d55b2732c86 100644 --- a/.github/workflows/unit_test_evalengine_mysql57.yml +++ b/.github/workflows/unit_test_evalengine_mysql57.yml @@ -163,6 +163,9 @@ jobs: export NOVTADMINBUILD=1 export VTEVALENGINETEST="1" + # We sometimes need to alter the behavior based on the platform we're + # testing, e.g. MySQL 5.7 vs 8.0. + export CI_DB_PLATFORM="mysql57" eatmydata -- make unit_test | tee -a output.txt | go-junit-report -set-exit-code > report.xml diff --git a/.github/workflows/unit_test_evalengine_mysql80.yml b/.github/workflows/unit_test_evalengine_mysql80.yml index e6e802b52d8..96af579742e 100644 --- a/.github/workflows/unit_test_evalengine_mysql80.yml +++ b/.github/workflows/unit_test_evalengine_mysql80.yml @@ -153,6 +153,9 @@ jobs: export NOVTADMINBUILD=1 export VTEVALENGINETEST="1" + # We sometimes need to alter the behavior based on the platform we're + # testing, e.g. MySQL 5.7 vs 8.0. + export CI_DB_PLATFORM="mysql80" eatmydata -- make unit_test | tee -a output.txt | go-junit-report -set-exit-code > report.xml diff --git a/.github/workflows/unit_test_evalengine_mysql84.yml b/.github/workflows/unit_test_evalengine_mysql84.yml index 46736dac349..efbe2b0eb9f 100644 --- a/.github/workflows/unit_test_evalengine_mysql84.yml +++ b/.github/workflows/unit_test_evalengine_mysql84.yml @@ -153,6 +153,9 @@ jobs: export NOVTADMINBUILD=1 export VTEVALENGINETEST="1" + # We sometimes need to alter the behavior based on the platform we're + # testing, e.g. MySQL 5.7 vs 8.0. + export CI_DB_PLATFORM="mysql84" eatmydata -- make unit_test | tee -a output.txt | go-junit-report -set-exit-code > report.xml diff --git a/.github/workflows/unit_test_mysql57.yml b/.github/workflows/unit_test_mysql57.yml index 3eaf02d1538..eed08e9ce4c 100644 --- a/.github/workflows/unit_test_mysql57.yml +++ b/.github/workflows/unit_test_mysql57.yml @@ -163,6 +163,9 @@ jobs: export NOVTADMINBUILD=1 export VTEVALENGINETEST="0" + # We sometimes need to alter the behavior based on the platform we're + # testing, e.g. MySQL 5.7 vs 8.0. + export CI_DB_PLATFORM="mysql57" eatmydata -- make unit_test | tee -a output.txt | go-junit-report -set-exit-code > report.xml diff --git a/.github/workflows/unit_test_mysql80.yml b/.github/workflows/unit_test_mysql80.yml index c036e6dd477..9e0ed7e6977 100644 --- a/.github/workflows/unit_test_mysql80.yml +++ b/.github/workflows/unit_test_mysql80.yml @@ -153,6 +153,9 @@ jobs: export NOVTADMINBUILD=1 export VTEVALENGINETEST="0" + # We sometimes need to alter the behavior based on the platform we're + # testing, e.g. MySQL 5.7 vs 8.0. + export CI_DB_PLATFORM="mysql80" eatmydata -- make unit_test | tee -a output.txt | go-junit-report -set-exit-code > report.xml diff --git a/.github/workflows/unit_test_mysql84.yml b/.github/workflows/unit_test_mysql84.yml index 84447ce390b..5948eb0836a 100644 --- a/.github/workflows/unit_test_mysql84.yml +++ b/.github/workflows/unit_test_mysql84.yml @@ -153,6 +153,9 @@ jobs: export NOVTADMINBUILD=1 export VTEVALENGINETEST="0" + # We sometimes need to alter the behavior based on the platform we're + # testing, e.g. MySQL 5.7 vs 8.0. + export CI_DB_PLATFORM="mysql84" eatmydata -- make unit_test | tee -a output.txt | go-junit-report -set-exit-code > report.xml diff --git a/go/test/endtoend/vreplication/vreplication_test.go b/go/test/endtoend/vreplication/vreplication_test.go index 4269bd68cab..6eab1d5495e 100644 --- a/go/test/endtoend/vreplication/vreplication_test.go +++ b/go/test/endtoend/vreplication/vreplication_test.go @@ -323,8 +323,10 @@ func testVreplicationWorkflows(t *testing.T, limited bool, binlogRowImage string defer func() { defaultReplicas = 1 }() if binlogRowImage != "" { - require.NoError(t, utils.SetBinlogRowImageMode("noblob", vc.ClusterConfig.tmpDir)) - defer utils.SetBinlogRowImageMode("", vc.ClusterConfig.tmpDir) + // Run the e2e test with binlog_row_image=NOBLOB and + // binlog_row_value_options=PARTIAL_JSON. + require.NoError(t, utils.SetBinlogRowImageMode("noblob", vc.ClusterConfig.tmpDir, true)) + defer utils.SetBinlogRowImageMode("", vc.ClusterConfig.tmpDir, false) } defaultCell := vc.Cells[defaultCellName] @@ -600,8 +602,10 @@ func TestCellAliasVreplicationWorkflow(t *testing.T) { keyspace := "product" shard := "0" - require.NoError(t, utils.SetBinlogRowImageMode("noblob", vc.ClusterConfig.tmpDir)) - defer utils.SetBinlogRowImageMode("", vc.ClusterConfig.tmpDir) + // Run the e2e test with binlog_row_image=NOBLOB and + // binlog_row_value_options=PARTIAL_JSON. + require.NoError(t, utils.SetBinlogRowImageMode("noblob", vc.ClusterConfig.tmpDir, true)) + defer utils.SetBinlogRowImageMode("", vc.ClusterConfig.tmpDir, false) cell1 := vc.Cells["zone1"] cell2 := vc.Cells["zone2"] diff --git a/go/test/utils/binlog.go b/go/test/utils/binlog.go index a9eafc4941d..af24e3b1b57 100644 --- a/go/test/utils/binlog.go +++ b/go/test/utils/binlog.go @@ -19,6 +19,7 @@ package utils import ( "fmt" "os" + "strconv" "strings" ) @@ -28,9 +29,11 @@ const ( ) // SetBinlogRowImageMode creates a temp cnf file to set binlog_row_image=NOBLOB and -// binlog_row_value_options=PARTIAL_JSON for vreplication unit tests. -// It adds it to the EXTRA_MY_CNF environment variable which appends text from them into my.cnf. -func SetBinlogRowImageMode(mode string, cnfDir string) error { +// optionally binlog_row_value_options=PARTIAL_JSON (since it does not exist in 5.7) +// for vreplication unit tests. +// It adds it to the EXTRA_MY_CNF environment variable which appends text from them +// into my.cnf. +func SetBinlogRowImageMode(mode string, cnfDir string, includePartialJSON bool) error { var newCnfs []string // remove any existing extra cnfs for binlog row image @@ -56,8 +59,10 @@ func SetBinlogRowImageMode(mode string, cnfDir string) error { if err != nil { return err } - lm := strings.ToLower(mode) - if lm == "noblob" || lm == "minimal" { + if includePartialJSON { + if !CIDBPlatformIsMySQL8orLater() { + return fmt.Errorf("partial JSON values are only supported in MySQL 8.0 or later") + } // We're testing partial binlog row images so let's also test partial // JSON values in the images. _, err = f.WriteString("\nbinlog_row_value_options=PARTIAL_JSON\n") @@ -78,3 +83,31 @@ func SetBinlogRowImageMode(mode string, cnfDir string) error { } return nil } + +// CIDBPlatformIsMySQL8orLater returns true if the CI_DB_PLATFORM environment +// variable is empty -- meaning we're not running in the CI and we assume +// MySQL8.0 or later is used, and you can understand the failures and make +// adjustments as necessary -- or it's set to reflect usage of MySQL 8.0 or +// later. This relies on the current standard values used such as mysql5.7, +// mysql8.0, mysql8.4, mariadb10.7, etc. This can be used when the CI test +// behavior needs to be altered based on the specific database platform +// we're testing against. +func CIDBPlatformIsMySQL8orLater() bool { + dbPlatform := strings.ToLower(os.Getenv("CI_DB_PLATFORM")) + if dbPlatform == "" { + // This is for local testing where we don't set the env var via + // the CI. + return true + } + if strings.HasPrefix(dbPlatform, "mysql") { + _, v, ok := strings.Cut(dbPlatform, "mysql") + if ok { + // We only want the major version. + version, err := strconv.Atoi(string(v[0])) + if err == nil && version >= 8 { + return true + } + } + } + return false +} diff --git a/go/test/utils/binlog_test.go b/go/test/utils/binlog_test.go index 593b964a171..45f787bc85a 100644 --- a/go/test/utils/binlog_test.go +++ b/go/test/utils/binlog_test.go @@ -28,15 +28,27 @@ import ( func TestUtils(t *testing.T) { tmpDir := "/tmp" cnfFile := fmt.Sprintf("%s/%s", tmpDir, BinlogRowImageCnf) + // Test that setting the mode will create the cnf file and add it to the EXTRA_MY_CNF env var. - require.NoError(t, SetBinlogRowImageMode("noblob", tmpDir)) + require.NoError(t, SetBinlogRowImageMode("noblob", tmpDir, false)) data, err := os.ReadFile(cnfFile) require.NoError(t, err) require.Contains(t, string(data), "binlog_row_image=noblob") require.Contains(t, os.Getenv(ExtraCnf), BinlogRowImageCnf) + // Test that setting the mode and passing true for includePartialJSON will set both options + // as expected. + if CIDBPlatformIsMySQL8orLater() { + require.NoError(t, SetBinlogRowImageMode("noblob", tmpDir, true)) + data, err = os.ReadFile(cnfFile) + require.NoError(t, err) + require.Contains(t, string(data), "binlog_row_image=noblob") + require.Contains(t, string(data), "binlog_row_value_options=PARTIAL_JSON") + require.Contains(t, os.Getenv(ExtraCnf), BinlogRowImageCnf) + } + // Test that clearing the mode will remove the cnf file and the cnf from the EXTRA_MY_CNF env var. - require.NoError(t, SetBinlogRowImageMode("", tmpDir)) + require.NoError(t, SetBinlogRowImageMode("", tmpDir, false)) require.NotContains(t, os.Getenv(ExtraCnf), BinlogRowImageCnf) _, err = os.Stat(cnfFile) require.True(t, os.IsNotExist(err)) diff --git a/go/vt/vttablet/tabletmanager/vreplication/framework_test.go b/go/vt/vttablet/tabletmanager/vreplication/framework_test.go index 12a05a69dbc..c7eee1e3092 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/framework_test.go +++ b/go/vt/vttablet/tabletmanager/vreplication/framework_test.go @@ -166,8 +166,13 @@ func setup(ctx context.Context) (func(), int) { return cleanup, 0 } -// We run Tests twice, first with full binlog_row_image, then with noblob. -var runNoBlobTest = false +var ( + // We run unit tests twice, first with binlog_row_image=FULL, then with NOBLOB. + runNoBlobTest = false + // When using MySQL 8.0 or later, we also set binlog_row_value_options=PARTIAL_JSON + // when using NOBLOB. + runPartialJSONTest = false +) // We use this tempDir for creating the external cnfs, since we create the test cluster afterwards. const tempDir = "/tmp" @@ -178,10 +183,10 @@ func TestMain(m *testing.M) { exitCode := func() int { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - if err := utils.SetBinlogRowImageMode("full", tempDir); err != nil { + if err := utils.SetBinlogRowImageMode("full", tempDir, false); err != nil { panic(err) } - defer utils.SetBinlogRowImageMode("", tempDir) + defer utils.SetBinlogRowImageMode("", tempDir, false) cancel, ret := setup(ctx) if ret > 0 { return ret @@ -193,10 +198,14 @@ func TestMain(m *testing.M) { cancel() runNoBlobTest = true - if err := utils.SetBinlogRowImageMode("noblob", tempDir); err != nil { + // binlog-row-value-options=PARTIAL_JSON is only supported in MySQL 8.0 and later. + // We still run unit tests with MySQL 5.7, so we cannot add it to the cnf file + // when using 5.7 or mysqld will fail to start. + runPartialJSONTest = utils.CIDBPlatformIsMySQL8orLater() + if err := utils.SetBinlogRowImageMode("noblob", tempDir, runPartialJSONTest); err != nil { panic(err) } - defer utils.SetBinlogRowImageMode("", tempDir) + defer utils.SetBinlogRowImageMode("", tempDir, false) cancel, ret = setup(ctx) if ret > 0 { return ret diff --git a/go/vt/vttablet/tabletmanager/vreplication/vplayer_flaky_test.go b/go/vt/vttablet/tabletmanager/vreplication/vplayer_flaky_test.go index 90b514b5b6d..8248dcf6476 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/vplayer_flaky_test.go +++ b/go/vt/vttablet/tabletmanager/vreplication/vplayer_flaky_test.go @@ -1521,12 +1521,18 @@ func TestPlayerRowMove(t *testing.T) { // TestPlayerPartialImagesUpdatePK tests the behavior of the vplayer when we // have partial binlog images, meaning that binlog-row-image=NOBLOB and -// binlog-row-value-options=PARTIAL_JSON. These are both set together when -// running the unit tests with runNoBlobTest=true. So we skip the test if -// it's not set. +// binlog-row-value-options=PARTIAL_JSON. These are both set when running the +// unit tests with runNoBlobTest=true and runPartialJSONTest=true. +// If the test is running in the CI and the database platform is not MySQL +// 8.0 or later (which you can control using the CI_DB_PLATFORM env variable), +// then runPartialJSONTest will be false and the test will be skipped. +// the test if it's not set. func TestPlayerPartialImagesUpdatePK(t *testing.T) { if !runNoBlobTest { - t.Skip("Skipping test as runNoBlobTest is not set") + t.Skip("Skipping test as binlog_row_image=NOBLOB is not set") + } + if !runPartialJSONTest { + t.Skip("Skipping test as binlog-row-value-options=PARTIAL_JSON is not set (most likely because the database type is not MySQL 8.0 or later)") } defer deleteTablet(addTablet(100)) @@ -1554,12 +1560,13 @@ func TestPlayerPartialImagesUpdatePK(t *testing.T) { cancel, _ := startVReplication(t, bls, "") defer cancel() - testCases := []struct { + type testCase struct { input string output []string data [][]string error string - }{ + } + testCases := []testCase{ { input: "insert into src (id, jd, bd) values (1,'{\"key1\": \"val1\"}','blob data'), (2,'{\"key2\": \"val2\"}','blob data2'), (3,'{\"key3\": \"val3\"}','blob data3')", output: []string{"insert into dst(id,jd,bd) values (1,JSON_OBJECT(_utf8mb4'key1', _utf8mb4'val1'),_binary'blob data'), (2,JSON_OBJECT(_utf8mb4'key2', _utf8mb4'val2'),_binary'blob data2'), (3,JSON_OBJECT(_utf8mb4'key3', _utf8mb4'val3'),_binary'blob data3')"}, @@ -1757,10 +1764,12 @@ func TestPlayerTypes(t *testing.T) { {"2", "null", `{"name": null}`, "123", `{"a": [42, 100]}`, `{"foo": "bar"}`}, }, }} - if !runNoBlobTest { + if runNoBlobTest && runPartialJSONTest { + // With partial JSON values we don't replicate the JSON columns that aren't + // actually updated. testcases = append(testcases, testcase{ input: "update vitess_json set val1 = '{\"bar\": \"foo\"}', val4 = '{\"a\": [98, 123]}', val5 = convert(x'7b7d' using utf8mb4) where id=1", - output: "update vitess_json set val1=JSON_OBJECT(_utf8mb4'bar', _utf8mb4'foo'), val2=JSON_OBJECT(), val3=CAST(123 as JSON), val4=JSON_OBJECT(_utf8mb4'a', JSON_ARRAY(98, 123)), val5=JSON_OBJECT() where id=1", + output: "update vitess_json set val1=JSON_OBJECT(_utf8mb4'bar', _utf8mb4'foo'), val4=JSON_OBJECT(_utf8mb4'a', JSON_ARRAY(98, 123)), val5=JSON_OBJECT() where id=1", table: "vitess_json", data: [][]string{ {"1", `{"bar": "foo"}`, "{}", "123", `{"a": [98, 123]}`, `{}`}, @@ -1768,11 +1777,9 @@ func TestPlayerTypes(t *testing.T) { }, }) } else { - // With partial JSON values we don't replicate the JSON columns that aren't - // actually updated. testcases = append(testcases, testcase{ input: "update vitess_json set val1 = '{\"bar\": \"foo\"}', val4 = '{\"a\": [98, 123]}', val5 = convert(x'7b7d' using utf8mb4) where id=1", - output: "update vitess_json set val1=JSON_OBJECT(_utf8mb4'bar', _utf8mb4'foo'), val4=JSON_OBJECT(_utf8mb4'a', JSON_ARRAY(98, 123)), val5=JSON_OBJECT() where id=1", + output: "update vitess_json set val1=JSON_OBJECT(_utf8mb4'bar', _utf8mb4'foo'), val2=JSON_OBJECT(), val3=CAST(123 as JSON), val4=JSON_OBJECT(_utf8mb4'a', JSON_ARRAY(98, 123)), val5=JSON_OBJECT() where id=1", table: "vitess_json", data: [][]string{ {"1", `{"bar": "foo"}`, "{}", "123", `{"a": [98, 123]}`, `{}`}, diff --git a/test/templates/unit_test.tpl b/test/templates/unit_test.tpl index 3704aebac4e..f802ee7ad4a 100644 --- a/test/templates/unit_test.tpl +++ b/test/templates/unit_test.tpl @@ -177,6 +177,9 @@ jobs: export NOVTADMINBUILD=1 export VTEVALENGINETEST="{{.Evalengine}}" + # We sometimes need to alter the behavior based on the platform we're + # testing, e.g. MySQL 5.7 vs 8.0. + export CI_DB_PLATFORM="{{.Platform}}" eatmydata -- make unit_test | tee -a output.txt | go-junit-report -set-exit-code > report.xml From 1980b897e6d6f68b239390114a1ce838e26cb092 Mon Sep 17 00:00:00 2001 From: Matt Lord Date: Fri, 13 Dec 2024 22:06:02 -0500 Subject: [PATCH 20/39] Try upgrading codecov to see if that helps Signed-off-by: Matt Lord --- .github/workflows/codecov.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/codecov.yml b/.github/workflows/codecov.yml index fc30974212b..c971466b998 100644 --- a/.github/workflows/codecov.yml +++ b/.github/workflows/codecov.yml @@ -111,7 +111,7 @@ jobs: - name: Upload coverage reports to codecov.io if: steps.changes.outputs.changed_files == 'true' - uses: codecov/codecov-action@v4 + uses: codecov/codecov-action@015f24e6818733317a2da2edd6290ab26238649a # https://github.com/codecov/codecov-action/releases/tag/v5.0.7 with: fail_ci_if_error: true verbose: true From 7b25395999f51bcd3dd5ddada39befe969b6430e Mon Sep 17 00:00:00 2001 From: Matt Lord Date: Fri, 13 Dec 2024 23:04:51 -0500 Subject: [PATCH 21/39] Minor changes from self review Signed-off-by: Matt Lord --- go/mysql/binlog/binlog_json.go | 33 +++++++++---------- go/test/utils/binlog.go | 7 ++-- go/test/utils/binlog_test.go | 2 ++ .../vreplication/replicator_plan.go | 10 +++--- .../vreplication/vplayer_flaky_test.go | 32 ++++++++++++------ 5 files changed, 47 insertions(+), 37 deletions(-) diff --git a/go/mysql/binlog/binlog_json.go b/go/mysql/binlog/binlog_json.go index 33a9c474bd8..2b83780df72 100644 --- a/go/mysql/binlog/binlog_json.go +++ b/go/mysql/binlog/binlog_json.go @@ -113,6 +113,7 @@ func ParseBinaryJSONDiff(data []byte) (sqltypes.Value, error) { } else { // Only the inner most function has the field name diff.WriteString("%s, ") // This will later be replaced by the field name } + outer = true pathLen, readTo := readVariableLength(data, pos) pos = readTo @@ -125,27 +126,25 @@ func ParseBinaryJSONDiff(data []byte) (sqltypes.Value, error) { diff.WriteByte('\'') diff.Write(path) diff.WriteByte('\'') - if opType == jsonDiffOpRemove { // No value for remove diff.WriteByte(')') - } else { - diff.WriteString(", ") - valueLen, readTo := readVariableLength(data, pos) - pos = readTo - value, err := ParseBinaryJSON(data[pos : pos+valueLen]) - if err != nil { - return sqltypes.Value{}, vterrors.Errorf(vtrpcpb.Code_INVALID_ARGUMENT, - "cannot read JSON diff value for path %s: %v", path, err) - } - pos += valueLen - if value.Type() == json.TypeString { - diff.WriteString(sqlparser.Utf8mb4Str) - } - diff.Write(value.MarshalTo(nil)) - diff.WriteByte(')') + continue } - outer = true + diff.WriteString(", ") + valueLen, readTo := readVariableLength(data, pos) + pos = readTo + value, err := ParseBinaryJSON(data[pos : pos+valueLen]) + if err != nil { + return sqltypes.Value{}, vterrors.Errorf(vtrpcpb.Code_INVALID_ARGUMENT, + "cannot read JSON diff value for path %s: %v", path, err) + } + pos += valueLen + if value.Type() == json.TypeString { + diff.WriteString(sqlparser.Utf8mb4Str) + } + diff.Write(value.MarshalTo(nil)) + diff.WriteByte(')') } return sqltypes.MakeTrusted(sqltypes.Expression, diff.Bytes()), nil diff --git a/go/test/utils/binlog.go b/go/test/utils/binlog.go index af24e3b1b57..7287bfee44c 100644 --- a/go/test/utils/binlog.go +++ b/go/test/utils/binlog.go @@ -88,10 +88,9 @@ func SetBinlogRowImageMode(mode string, cnfDir string, includePartialJSON bool) // variable is empty -- meaning we're not running in the CI and we assume // MySQL8.0 or later is used, and you can understand the failures and make // adjustments as necessary -- or it's set to reflect usage of MySQL 8.0 or -// later. This relies on the current standard values used such as mysql5.7, -// mysql8.0, mysql8.4, mariadb10.7, etc. This can be used when the CI test -// behavior needs to be altered based on the specific database platform -// we're testing against. +// later. This relies on the current standard values used such as mysql57, +// mysql80, mysql84, etc. This can be used when the CI test behavior needs +// to be altered based on the specific database platform we're testing against. func CIDBPlatformIsMySQL8orLater() bool { dbPlatform := strings.ToLower(os.Getenv("CI_DB_PLATFORM")) if dbPlatform == "" { diff --git a/go/test/utils/binlog_test.go b/go/test/utils/binlog_test.go index 45f787bc85a..5b307595107 100644 --- a/go/test/utils/binlog_test.go +++ b/go/test/utils/binlog_test.go @@ -45,6 +45,8 @@ func TestUtils(t *testing.T) { require.Contains(t, string(data), "binlog_row_image=noblob") require.Contains(t, string(data), "binlog_row_value_options=PARTIAL_JSON") require.Contains(t, os.Getenv(ExtraCnf), BinlogRowImageCnf) + } else { + require.Error(t, SetBinlogRowImageMode("noblob", tmpDir, true)) } // Test that clearing the mode will remove the cnf file and the cnf from the EXTRA_MY_CNF env var. diff --git a/go/vt/vttablet/tabletmanager/vreplication/replicator_plan.go b/go/vt/vttablet/tabletmanager/vreplication/replicator_plan.go index fa3a225b18b..6320ee38d55 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/replicator_plan.go +++ b/go/vt/vttablet/tabletmanager/vreplication/replicator_plan.go @@ -480,12 +480,10 @@ func (tp *TablePlan) applyChange(rowChange *binlogdatapb.RowChange, executor fun jsonIndex := 0 for i, field := range tp.Fields { if field.Type == querypb.Type_JSON && rowChange.JsonPartialValues != nil { - if !isBitSet(rowChange.JsonPartialValues.Cols, jsonIndex) { + switch { + case !isBitSet(rowChange.JsonPartialValues.Cols, jsonIndex): // We use the full AFTER value which we already have. - jsonIndex++ - continue - } - if len(afterVals[i].Raw()) == 0 { + case len(afterVals[i].Raw()) == 0: // When using BOTH binlog_row_image=NOBLOB AND // binlog_row_value_options=PARTIAL_JSON then the JSON column has the data bit // set and the diff is empty when it's not present. So we want to use the @@ -500,7 +498,7 @@ func (tp *TablePlan) applyChange(rowChange *binlogdatapb.RowChange, executor fun return nil, vterrors.Wrapf(err, "failed to bind field value for %s.%s when building insert query", tp.TargetName, field.Name) } - } else { + default: // For JSON columns when binlog-row-value-options=PARTIAL_JSON is used, we // need to wrap the JSON diff function(s) around the BEFORE value. diff := bindvars["a_"+field.Name].Value diff --git a/go/vt/vttablet/tabletmanager/vreplication/vplayer_flaky_test.go b/go/vt/vttablet/tabletmanager/vreplication/vplayer_flaky_test.go index 8248dcf6476..b4a50ab529f 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/vplayer_flaky_test.go +++ b/go/vt/vttablet/tabletmanager/vreplication/vplayer_flaky_test.go @@ -1519,14 +1519,15 @@ func TestPlayerRowMove(t *testing.T) { validateQueryCountStat(t, "replicate", 3) } -// TestPlayerPartialImagesUpdatePK tests the behavior of the vplayer when we -// have partial binlog images, meaning that binlog-row-image=NOBLOB and -// binlog-row-value-options=PARTIAL_JSON. These are both set when running the -// unit tests with runNoBlobTest=true and runPartialJSONTest=true. +// TestPlayerPartialImagesUpdatePK tests the behavior of the vplayer when +// updating the Primary Key for a row when we have partial binlog +// images, meaning that binlog-row-image=NOBLOB and +// binlog-row-value-options=PARTIAL_JSON. These are both set when running +// the unit tests with runNoBlobTest=true and runPartialJSONTest=true. // If the test is running in the CI and the database platform is not MySQL -// 8.0 or later (which you can control using the CI_DB_PLATFORM env variable), -// then runPartialJSONTest will be false and the test will be skipped. -// the test if it's not set. +// 8.0 or later (which you can control using the CI_DB_PLATFORM env +// variable), then runPartialJSONTest will be false and the test will be +// skipped. func TestPlayerPartialImagesUpdatePK(t *testing.T) { if !runNoBlobTest { t.Skip("Skipping test as binlog_row_image=NOBLOB is not set") @@ -1586,15 +1587,26 @@ func TestPlayerPartialImagesUpdatePK(t *testing.T) { }, }, { - input: `update src set id = id+10, bd = 'new blob data' where id = 2`, + input: `update src set bd = 'new blob data' where id = 2`, + output: []string{ + "update dst set bd=_binary'new blob data' where id=2", + }, + data: [][]string{ + {"1", "{\"key1\": \"val1\", \"color\": \"red\"}", "blob data"}, + {"2", "{\"key2\": \"val2\"}", "new blob data"}, + {"3", "{\"key3\": \"val3\"}", "blob data3"}, + }, + }, + { + input: `update src set id = id+10, bd = 'newest blob data' where id = 2`, output: []string{ "delete from dst where id=2", - "insert into dst(id,jd,bd) values (12,JSON_OBJECT(_utf8mb4'key2', _utf8mb4'val2'),_binary'new blob data')", + "insert into dst(id,jd,bd) values (12,JSON_OBJECT(_utf8mb4'key2', _utf8mb4'val2'),_binary'newest blob data')", }, data: [][]string{ {"1", "{\"key1\": \"val1\", \"color\": \"red\"}", "blob data"}, {"3", "{\"key3\": \"val3\"}", "blob data3"}, - {"12", "{\"key2\": \"val2\"}", "new blob data"}, + {"12", "{\"key2\": \"val2\"}", "newest blob data"}, }, }, { From 799850d993517454abea56941281cb0cb52cf5ae Mon Sep 17 00:00:00 2001 From: Matt Lord Date: Sun, 15 Dec 2024 09:23:36 -0500 Subject: [PATCH 22/39] Improve comments Signed-off-by: Matt Lord --- .../vreplication/replicator_plan.go | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/go/vt/vttablet/tabletmanager/vreplication/replicator_plan.go b/go/vt/vttablet/tabletmanager/vreplication/replicator_plan.go index 6320ee38d55..adfa9e85471 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/replicator_plan.go +++ b/go/vt/vttablet/tabletmanager/vreplication/replicator_plan.go @@ -404,9 +404,11 @@ func (tp *TablePlan) applyChange(rowChange *binlogdatapb.RowChange, executor fun // binlog-row-value-options=PARTIAL_JSON. if len(afterVals[i].Raw()) == 0 { // When using BOTH binlog_row_image=NOBLOB AND - // binlog_row_value_options=PARTIAL_JSON then the JSON column has the data bit - // set and the diff is empty when it's not present. So we have to account for - // this by unsetting the data bit so that the current JSON value is not lost. + // binlog_row_value_options=PARTIAL_JSON, if the JSON column was NOT updated + // then the JSON column is marked as partial and the diff is empty as a way + // to exclude it from the AFTER image. It still has the data bit set, however, + // even though it's not really present. So we have to account for this by + // unsetting the data bit so that the current JSON value is not lost. setBit(rowChange.DataColumns.Cols, i, false) newVal = ptr.Of(sqltypes.MakeTrusted(querypb.Type_EXPRESSION, nil)) } else { @@ -475,8 +477,8 @@ func (tp *TablePlan) applyChange(rowChange *binlogdatapb.RowChange, executor fun return nil, nil } if tp.isPartial(rowChange) { - // We need to use a combination of the values in the BEFORE and AFTER image to generate - // the new row. + // We need to use a combination of the values in the BEFORE and AFTER image to generate the + // new row. jsonIndex := 0 for i, field := range tp.Fields { if field.Type == querypb.Type_JSON && rowChange.JsonPartialValues != nil { @@ -485,9 +487,9 @@ func (tp *TablePlan) applyChange(rowChange *binlogdatapb.RowChange, executor fun // We use the full AFTER value which we already have. case len(afterVals[i].Raw()) == 0: // When using BOTH binlog_row_image=NOBLOB AND - // binlog_row_value_options=PARTIAL_JSON then the JSON column has the data bit - // set and the diff is empty when it's not present. So we want to use the - // BEFORE image value. + // binlog_row_value_options=PARTIAL_JSON, if the JSON column was NOT updated + // then the JSON column is marked as partial and the diff is empty as a way + // to exclude it from the AFTER image. So we want to use the BEFORE image value. beforeVal, err := vjson.MarshalSQLValue(bindvars["b_"+field.Name].Value) if err != nil { return nil, vterrors.Wrapf(err, "failed to convert JSON to SQL field value for %s.%s when building insert query", From e68c0352887abbcf5422445e0f61328e992ed83d Mon Sep 17 00:00:00 2001 From: Matt Lord Date: Sun, 15 Dec 2024 13:03:50 -0500 Subject: [PATCH 23/39] Minor readability improvement Signed-off-by: Matt Lord --- .../vreplication/replicator_plan.go | 27 +++++++++---------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/go/vt/vttablet/tabletmanager/vreplication/replicator_plan.go b/go/vt/vttablet/tabletmanager/vreplication/replicator_plan.go index adfa9e85471..952dde672d1 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/replicator_plan.go +++ b/go/vt/vttablet/tabletmanager/vreplication/replicator_plan.go @@ -501,27 +501,26 @@ func (tp *TablePlan) applyChange(rowChange *binlogdatapb.RowChange, executor fun tp.TargetName, field.Name) } default: - // For JSON columns when binlog-row-value-options=PARTIAL_JSON is used, we - // need to wrap the JSON diff function(s) around the BEFORE value. - diff := bindvars["a_"+field.Name].Value + // For JSON columns when binlog-row-value-options=PARTIAL_JSON is used and the + // column is marked as partial, we need to wrap the JSON diff function(s) + // around the BEFORE value. + diff := afterVals[i].RawStr() beforeVal := bindvars["b_"+field.Name].Value - afterVal := bytes.Buffer{} - afterVal.Grow(len(diff) + len(beforeVal) + len(sqlparser.Utf8mb4Str) + 2) // +2 is for the enclosing quotes - // If the JSON column is partial, we need to specify the BEFORE value as - // the input for the diff function(s). - afterVal.WriteString(sqlparser.Utf8mb4Str) - afterVal.WriteByte('\'') - afterVal.Write(beforeVal) - afterVal.WriteByte('\'') + buf := bytes.Buffer{} + buf.Grow(len(diff) + len(beforeVal) + len(sqlparser.Utf8mb4Str) + 2) // +2 is for the enclosing quotes + buf.WriteString(sqlparser.Utf8mb4Str) + buf.WriteByte('\'') + buf.Write(beforeVal) + buf.WriteByte('\'') newVal := sqltypes.MakeTrusted(querypb.Type_EXPRESSION, []byte( - fmt.Sprintf(afterVals[i].RawStr(), afterVal.String()), + fmt.Sprintf(diff, buf.String()), )) - bindVar, err := tp.bindFieldVal(field, &newVal) + bv, err := tp.bindFieldVal(field, &newVal) if err != nil { return nil, vterrors.Wrapf(err, "failed to bind field value for %s.%s when building insert query", tp.TargetName, field.Name) } - bindvars["a_"+field.Name] = bindVar + bindvars["a_"+field.Name] = bv } jsonIndex++ continue From 8ad1fc09d3aa8dcac917ad19aec312dc095cb3a2 Mon Sep 17 00:00:00 2001 From: Matt Lord Date: Thu, 19 Dec 2024 09:07:59 -0500 Subject: [PATCH 24/39] Nitting while waiting for reviews Signed-off-by: Matt Lord --- .../vreplication/vreplication_test.go | 8 +-- go/test/utils/binlog.go | 6 +-- go/test/utils/binlog_test.go | 10 ++-- .../vreplication/framework_test.go | 8 +-- .../vreplication/replicator_plan.go | 18 +++---- .../vreplication/vplayer_flaky_test.go | 53 +++++++++++++------ 6 files changed, 61 insertions(+), 42 deletions(-) diff --git a/go/test/endtoend/vreplication/vreplication_test.go b/go/test/endtoend/vreplication/vreplication_test.go index 6eab1d5495e..955afde2f18 100644 --- a/go/test/endtoend/vreplication/vreplication_test.go +++ b/go/test/endtoend/vreplication/vreplication_test.go @@ -325,8 +325,8 @@ func testVreplicationWorkflows(t *testing.T, limited bool, binlogRowImage string if binlogRowImage != "" { // Run the e2e test with binlog_row_image=NOBLOB and // binlog_row_value_options=PARTIAL_JSON. - require.NoError(t, utils.SetBinlogRowImageMode("noblob", vc.ClusterConfig.tmpDir, true)) - defer utils.SetBinlogRowImageMode("", vc.ClusterConfig.tmpDir, false) + require.NoError(t, utils.SetBinlogRowImageOptions("noblob", true, vc.ClusterConfig.tmpDir)) + defer utils.SetBinlogRowImageOptions("", false, vc.ClusterConfig.tmpDir) } defaultCell := vc.Cells[defaultCellName] @@ -604,8 +604,8 @@ func TestCellAliasVreplicationWorkflow(t *testing.T) { // Run the e2e test with binlog_row_image=NOBLOB and // binlog_row_value_options=PARTIAL_JSON. - require.NoError(t, utils.SetBinlogRowImageMode("noblob", vc.ClusterConfig.tmpDir, true)) - defer utils.SetBinlogRowImageMode("", vc.ClusterConfig.tmpDir, false) + require.NoError(t, utils.SetBinlogRowImageOptions("noblob", true, vc.ClusterConfig.tmpDir)) + defer utils.SetBinlogRowImageOptions("", false, vc.ClusterConfig.tmpDir) cell1 := vc.Cells["zone1"] cell2 := vc.Cells["zone2"] diff --git a/go/test/utils/binlog.go b/go/test/utils/binlog.go index 7287bfee44c..0db6929054c 100644 --- a/go/test/utils/binlog.go +++ b/go/test/utils/binlog.go @@ -28,12 +28,12 @@ const ( BinlogRowImageCnf = "binlog-row-image.cnf" ) -// SetBinlogRowImageMode creates a temp cnf file to set binlog_row_image=NOBLOB and +// SetBinlogRowImageOptions creates a temp cnf file to set binlog_row_image=NOBLOB and // optionally binlog_row_value_options=PARTIAL_JSON (since it does not exist in 5.7) // for vreplication unit tests. // It adds it to the EXTRA_MY_CNF environment variable which appends text from them // into my.cnf. -func SetBinlogRowImageMode(mode string, cnfDir string, includePartialJSON bool) error { +func SetBinlogRowImageOptions(mode string, partialJSON bool, cnfDir string) error { var newCnfs []string // remove any existing extra cnfs for binlog row image @@ -59,7 +59,7 @@ func SetBinlogRowImageMode(mode string, cnfDir string, includePartialJSON bool) if err != nil { return err } - if includePartialJSON { + if partialJSON { if !CIDBPlatformIsMySQL8orLater() { return fmt.Errorf("partial JSON values are only supported in MySQL 8.0 or later") } diff --git a/go/test/utils/binlog_test.go b/go/test/utils/binlog_test.go index 5b307595107..d8a0d6d4222 100644 --- a/go/test/utils/binlog_test.go +++ b/go/test/utils/binlog_test.go @@ -24,13 +24,13 @@ import ( "github.com/stretchr/testify/require" ) -// TestSetBinlogRowImageMode tests the SetBinlogRowImageMode function. +// TestSetBinlogRowImageOptions tests the SetBinlogRowImageOptions function. func TestUtils(t *testing.T) { tmpDir := "/tmp" cnfFile := fmt.Sprintf("%s/%s", tmpDir, BinlogRowImageCnf) // Test that setting the mode will create the cnf file and add it to the EXTRA_MY_CNF env var. - require.NoError(t, SetBinlogRowImageMode("noblob", tmpDir, false)) + require.NoError(t, SetBinlogRowImageOptions("noblob", false, tmpDir)) data, err := os.ReadFile(cnfFile) require.NoError(t, err) require.Contains(t, string(data), "binlog_row_image=noblob") @@ -39,18 +39,18 @@ func TestUtils(t *testing.T) { // Test that setting the mode and passing true for includePartialJSON will set both options // as expected. if CIDBPlatformIsMySQL8orLater() { - require.NoError(t, SetBinlogRowImageMode("noblob", tmpDir, true)) + require.NoError(t, SetBinlogRowImageOptions("noblob", true, tmpDir)) data, err = os.ReadFile(cnfFile) require.NoError(t, err) require.Contains(t, string(data), "binlog_row_image=noblob") require.Contains(t, string(data), "binlog_row_value_options=PARTIAL_JSON") require.Contains(t, os.Getenv(ExtraCnf), BinlogRowImageCnf) } else { - require.Error(t, SetBinlogRowImageMode("noblob", tmpDir, true)) + require.Error(t, SetBinlogRowImageOptions("noblob", true, tmpDir)) } // Test that clearing the mode will remove the cnf file and the cnf from the EXTRA_MY_CNF env var. - require.NoError(t, SetBinlogRowImageMode("", tmpDir, false)) + require.NoError(t, SetBinlogRowImageOptions("", false, tmpDir)) require.NotContains(t, os.Getenv(ExtraCnf), BinlogRowImageCnf) _, err = os.Stat(cnfFile) require.True(t, os.IsNotExist(err)) diff --git a/go/vt/vttablet/tabletmanager/vreplication/framework_test.go b/go/vt/vttablet/tabletmanager/vreplication/framework_test.go index c7eee1e3092..c9c6451bbeb 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/framework_test.go +++ b/go/vt/vttablet/tabletmanager/vreplication/framework_test.go @@ -183,10 +183,10 @@ func TestMain(m *testing.M) { exitCode := func() int { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - if err := utils.SetBinlogRowImageMode("full", tempDir, false); err != nil { + if err := utils.SetBinlogRowImageOptions("full", false, tempDir); err != nil { panic(err) } - defer utils.SetBinlogRowImageMode("", tempDir, false) + defer utils.SetBinlogRowImageOptions("", false, tempDir) cancel, ret := setup(ctx) if ret > 0 { return ret @@ -202,10 +202,10 @@ func TestMain(m *testing.M) { // We still run unit tests with MySQL 5.7, so we cannot add it to the cnf file // when using 5.7 or mysqld will fail to start. runPartialJSONTest = utils.CIDBPlatformIsMySQL8orLater() - if err := utils.SetBinlogRowImageMode("noblob", tempDir, runPartialJSONTest); err != nil { + if err := utils.SetBinlogRowImageOptions("noblob", runPartialJSONTest, tempDir); err != nil { panic(err) } - defer utils.SetBinlogRowImageMode("", tempDir, false) + defer utils.SetBinlogRowImageOptions("", false, tempDir) cancel, ret = setup(ctx) if ret > 0 { return ret diff --git a/go/vt/vttablet/tabletmanager/vreplication/replicator_plan.go b/go/vt/vttablet/tabletmanager/vreplication/replicator_plan.go index 952dde672d1..ec9de26d7e6 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/replicator_plan.go +++ b/go/vt/vttablet/tabletmanager/vreplication/replicator_plan.go @@ -403,12 +403,11 @@ func (tp *TablePlan) applyChange(rowChange *binlogdatapb.RowChange, executor fun // This occurs when using partial JSON values as a result of mysqld using // binlog-row-value-options=PARTIAL_JSON. if len(afterVals[i].Raw()) == 0 { - // When using BOTH binlog_row_image=NOBLOB AND - // binlog_row_value_options=PARTIAL_JSON, if the JSON column was NOT updated - // then the JSON column is marked as partial and the diff is empty as a way - // to exclude it from the AFTER image. It still has the data bit set, however, - // even though it's not really present. So we have to account for this by - // unsetting the data bit so that the current JSON value is not lost. + // If the JSON column was NOT updated then the JSON column is marked as + // partial and the diff is empty as a way to exclude it from the AFTER image. + // It still has the data bit set, however, even though it's not really + // present. So we have to account for this by unsetting the data bit so + // that the column's current JSON value is not lost. setBit(rowChange.DataColumns.Cols, i, false) newVal = ptr.Of(sqltypes.MakeTrusted(querypb.Type_EXPRESSION, nil)) } else { @@ -486,10 +485,9 @@ func (tp *TablePlan) applyChange(rowChange *binlogdatapb.RowChange, executor fun case !isBitSet(rowChange.JsonPartialValues.Cols, jsonIndex): // We use the full AFTER value which we already have. case len(afterVals[i].Raw()) == 0: - // When using BOTH binlog_row_image=NOBLOB AND - // binlog_row_value_options=PARTIAL_JSON, if the JSON column was NOT updated - // then the JSON column is marked as partial and the diff is empty as a way - // to exclude it from the AFTER image. So we want to use the BEFORE image value. + // If the JSON column was NOT updated then the JSON column is marked as partial + // and the diff is empty as a way to exclude it from the AFTER image. So we + // want to use the BEFORE image value. beforeVal, err := vjson.MarshalSQLValue(bindvars["b_"+field.Name].Value) if err != nil { return nil, vterrors.Wrapf(err, "failed to convert JSON to SQL field value for %s.%s when building insert query", diff --git a/go/vt/vttablet/tabletmanager/vreplication/vplayer_flaky_test.go b/go/vt/vttablet/tabletmanager/vreplication/vplayer_flaky_test.go index b4a50ab529f..d6efcb5d5d4 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/vplayer_flaky_test.go +++ b/go/vt/vttablet/tabletmanager/vreplication/vplayer_flaky_test.go @@ -1521,19 +1521,11 @@ func TestPlayerRowMove(t *testing.T) { // TestPlayerPartialImagesUpdatePK tests the behavior of the vplayer when // updating the Primary Key for a row when we have partial binlog -// images, meaning that binlog-row-image=NOBLOB and -// binlog-row-value-options=PARTIAL_JSON. These are both set when running -// the unit tests with runNoBlobTest=true and runPartialJSONTest=true. -// If the test is running in the CI and the database platform is not MySQL -// 8.0 or later (which you can control using the CI_DB_PLATFORM env -// variable), then runPartialJSONTest will be false and the test will be -// skipped. +// images, meaning that binlog-row-image=NOBLOB and/or +// binlog-row-value-options=PARTIAL_JSON. func TestPlayerPartialImagesUpdatePK(t *testing.T) { - if !runNoBlobTest { - t.Skip("Skipping test as binlog_row_image=NOBLOB is not set") - } if !runPartialJSONTest { - t.Skip("Skipping test as binlog-row-value-options=PARTIAL_JSON is not set (most likely because the database type is not MySQL 8.0 or later)") + t.Skip("Skipping test as binlog_row_value_options=PARTIAL_JSON is not enabled") } defer deleteTablet(addTablet(100)) @@ -1577,7 +1569,9 @@ func TestPlayerPartialImagesUpdatePK(t *testing.T) { {"3", "{\"key3\": \"val3\"}", "blob data3"}, }, }, - { + } + if runNoBlobTest { + testCases = append(testCases, testCase{ input: `update src set jd=JSON_SET(jd, '$.color', 'red') where id = 1`, output: []string{"update dst set jd=JSON_INSERT(`jd`, _utf8mb4'$.color', _utf8mb4\"red\") where id=1"}, data: [][]string{ @@ -1585,7 +1579,19 @@ func TestPlayerPartialImagesUpdatePK(t *testing.T) { {"2", "{\"key2\": \"val2\"}", "blob data2"}, {"3", "{\"key3\": \"val3\"}", "blob data3"}, }, - }, + }) + } else { + testCases = append(testCases, testCase{ + input: `update src set jd=JSON_SET(jd, '$.color', 'red') where id = 1`, + output: []string{"update dst set jd=JSON_INSERT(`jd`, _utf8mb4'$.color', _utf8mb4\"red\"), bd=_binary'blob data' where id=1"}, + data: [][]string{ + {"1", "{\"key1\": \"val1\", \"color\": \"red\"}", "blob data"}, + {"2", "{\"key2\": \"val2\"}", "blob data2"}, + {"3", "{\"key3\": \"val3\"}", "blob data3"}, + }, + }) + } + testCases = append(testCases, []testCase{ { input: `update src set bd = 'new blob data' where id = 2`, output: []string{ @@ -1609,10 +1615,25 @@ func TestPlayerPartialImagesUpdatePK(t *testing.T) { {"12", "{\"key2\": \"val2\"}", "newest blob data"}, }, }, - { + }...) + if runNoBlobTest { + testCases = append(testCases, testCase{ input: `update src set id = id+10 where id = 3`, error: "binary log event missing a needed value for dst.bd due to not using binlog-row-image=FULL", - }, + }) + } else { + testCases = append(testCases, testCase{ + input: `update src set id = id+10 where id = 3`, + output: []string{ + "delete from dst where id=3", + "insert into dst(id,jd,bd) values (13,JSON_OBJECT(_utf8mb4'key3', _utf8mb4'val3'),_binary'blob data3')", + }, + data: [][]string{ + {"1", "{\"key1\": \"val1\", \"color\": \"red\"}", "blob data"}, + {"12", "{\"key2\": \"val2\"}", "newest blob data"}, + {"13", "{\"key3\": \"val3\"}", "blob data3"}, + }, + }) } for _, tc := range testCases { @@ -1776,7 +1797,7 @@ func TestPlayerTypes(t *testing.T) { {"2", "null", `{"name": null}`, "123", `{"a": [42, 100]}`, `{"foo": "bar"}`}, }, }} - if runNoBlobTest && runPartialJSONTest { + if runPartialJSONTest { // With partial JSON values we don't replicate the JSON columns that aren't // actually updated. testcases = append(testcases, testcase{ From c168ff0fb71604f9401386054bb62a40539ebc78 Mon Sep 17 00:00:00 2001 From: Matt Lord Date: Thu, 19 Dec 2024 11:39:15 -0500 Subject: [PATCH 25/39] Add another unit test case Signed-off-by: Matt Lord --- go/mysql/binlog_event_mysql56_test.go | 77 +++++++++++++++++++++++++++ 1 file changed, 77 insertions(+) diff --git a/go/mysql/binlog_event_mysql56_test.go b/go/mysql/binlog_event_mysql56_test.go index 9ad38473205..161e48a50ae 100644 --- a/go/mysql/binlog_event_mysql56_test.go +++ b/go/mysql/binlog_event_mysql56_test.go @@ -626,6 +626,83 @@ func TestMySQL56PartialUpdateRowsEvent(t *testing.T) { numRows: 1, want: "JSON_INSERT(%s, _utf8mb4'$.misc', _utf8mb4\"{\\\"address\\\":\\\"1012 S Park\\\", \\\"town\\\":\\\"Hastings\\\", \\\"state\\\":\\\"MI\\\"}\")", }, + { + name: "JSON field not updated", + // The mysqlbinlog -vvv --base64-output=decode-rows output for the following event: + // ### UPDATE `vt_commerce`.`customer` + // ### WHERE + // ### @1=1 /* LONGINT meta=0 nullable=0 is_null=0 */ + // ### @2='newalice@domain.com' /* VARSTRING(128) meta=128 nullable=1 is_null=0 */ + // ### @3='{"day": "friday", "role": "manager", "color": "red", "salary": 100, "favorite_color": "black"}' /* JSON meta=4 nullable=1 is_null=0 */ + // ### SET + // ### @1=101 /* LONGINT meta=0 nullable=0 is_null=0 */ + // ### @2='newalice@domain.com' /* VARSTRING(128) meta=128 nullable=1 is_null=0 */ + // ### @3=@3 /* JSON meta=4 nullable=1 is_null=0 */ + // ### UPDATE `vt_commerce`.`customer` + // ### WHERE + // ### @1=2 /* LONGINT meta=0 nullable=0 is_null=0 */ + // ### @2='newbob@domain.com' /* VARSTRING(128) meta=128 nullable=1 is_null=0 */ + // ### @3='{"day": "friday", "role": "manager", "color": "red", "salary": 99, "favorite_color": "black"}' /* JSON meta=4 nullable=1 is_null=0 */ + // ### SET + // ### @1=102 /* LONGINT meta=0 nullable=0 is_null=0 */ + // ### @2='newbob@domain.com' /* VARSTRING(128) meta=128 nullable=1 is_null=0 */ + // ### @3=@3 /* JSON meta=4 nullable=1 is_null=0 */ + // ### UPDATE `vt_commerce`.`customer` + // ### WHERE + // ### @1=3 /* LONGINT meta=0 nullable=0 is_null=0 */ + // ### @2='newcharlie@domain.com' /* VARSTRING(128) meta=128 nullable=1 is_null=0 */ + // ### @3='{"day": "monday", "role": "manager", "color": "red", "hobby": "skiing", "salary": 99}' /* JSON meta=4 nullable=1 is_null=0 */ + // ### SET + // ### @1=103 /* LONGINT meta=0 nullable=0 is_null=0 */ + // ### @2='newcharlie@domain.com' /* VARSTRING(128) meta=128 nullable=1 is_null=0 */ + // ### @3=@3 /* JSON meta=4 nullable=1 is_null=0 */ + // ### UPDATE `vt_commerce`.`customer` + // ### WHERE + // ### @1=4 /* LONGINT meta=0 nullable=0 is_null=0 */ + // ### @2='newdan@domain.com' /* VARSTRING(128) meta=128 nullable=1 is_null=0 */ + // ### @3='{"day": "friday", "role": "manager", "color": "red", "salary": 99, "favorite_color": "black"}' /* JSON meta=4 nullable=1 is_null=0 */ + // ### SET + // ### @1=104 /* LONGINT meta=0 nullable=0 is_null=0 */ + // ### @2='newdan@domain.com' /* VARSTRING(128) meta=128 nullable=1 is_null=0 */ + // ### @3=@3 /* JSON meta=4 nullable=1 is_null=0 */ + // ### UPDATE `vt_commerce`.`customer` + // ### WHERE + // ### @1=5 /* LONGINT meta=0 nullable=0 is_null=0 */ + // ### @2='neweve@domain.com' /* VARSTRING(128) meta=128 nullable=1 is_null=0 */ + // ### @3='{"day": "friday", "role": "manager", "color": "red", "salary": 100, "favorite_color": "black"}' /* JSON meta=4 nullable=1 is_null=0 */ + // ### SET + // ### @1=105 /* LONGINT meta=0 nullable=0 is_null=0 */ + // ### @2='neweve@domain.com' /* VARSTRING(128) meta=128 nullable=1 is_null=0 */ + // ### @3=@3 /* JSON meta=4 nullable=1 is_null=0 */ + rawEvent: []byte{ + 194, 74, 100, 103, 39, 46, 144, 133, 54, 77, 3, 0, 0, 77, 128, 0, 0, 0, 0, 153, 0, 0, 0, 0, 0, 1, 0, 2, 0, 3, 255, 255, 0, 1, 0, 0, 0, 0, 0, 0, 0, + 19, 110, 101, 119, 97, 108, 105, 99, 101, 64, 100, 111, 109, 97, 105, 110, 46, 99, 111, 109, 97, 0, 0, 0, 0, 5, 0, 96, 0, 39, 0, 3, 0, 42, 0, 4, 0, + 46, 0, 5, 0, 51, 0, 6, 0, 57, 0, 14, 0, 12, 71, 0, 12, 78, 0, 12, 86, 0, 5, 100, 0, 12, 90, 0, 100, 97, 121, 114, 111, 108, 101, 99, 111, 108, 111, + 114, 115, 97, 108, 97, 114, 121, 102, 97, 118, 111, 114, 105, 116, 101, 95, 99, 111, 108, 111, 114, 6, 102, 114, 105, 100, 97, 121, 7, 109, 97, + 110, 97, 103, 101, 114, 3, 114, 101, 100, 5, 98, 108, 97, 99, 107, 1, 1, 0, 101, 0, 0, 0, 0, 0, 0, 0, 19, 110, 101, 119, 97, 108, 105, 99, 101, 64, + 100, 111, 109, 97, 105, 110, 46, 99, 111, 109, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 17, 110, 101, 119, 98, 111, 98, 64, 100, 111, 109, 97, 105, + 110, 46, 99, 111, 109, 97, 0, 0, 0, 0, 5, 0, 96, 0, 39, 0, 3, 0, 42, 0, 4, 0, 46, 0, 5, 0, 51, 0, 6, 0, 57, 0, 14, 0, 12, 71, 0, 12, 78, 0, 12, 86, + 0, 5, 99, 0, 12, 90, 0, 100, 97, 121, 114, 111, 108, 101, 99, 111, 108, 111, 114, 115, 97, 108, 97, 114, 121, 102, 97, 118, 111, 114, 105, 116, 101, + 95, 99, 111, 108, 111, 114, 6, 102, 114, 105, 100, 97, 121, 7, 109, 97, 110, 97, 103, 101, 114, 3, 114, 101, 100, 5, 98, 108, 97, 99, 107, 1, 1, 0, + 102, 0, 0, 0, 0, 0, 0, 0, 17, 110, 101, 119, 98, 111, 98, 64, 100, 111, 109, 97, 105, 110, 46, 99, 111, 109, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, + 21, 110, 101, 119, 99, 104, 97, 114, 108, 105, 101, 64, 100, 111, 109, 97, 105, 110, 46, 99, 111, 109, 89, 0, 0, 0, 0, 5, 0, 88, 0, 39, 0, 3, 0, 42, + 0, 4, 0, 46, 0, 5, 0, 51, 0, 5, 0, 56, 0, 6, 0, 12, 62, 0, 12, 69, 0, 12, 77, 0, 12, 81, 0, 5, 99, 0, 100, 97, 121, 114, 111, 108, 101, 99, 111, 108, + 111, 114, 104, 111, 98, 98, 121, 115, 97, 108, 97, 114, 121, 6, 109, 111, 110, 100, 97, 121, 7, 109, 97, 110, 97, 103, 101, 114, 3, 114, 101, 100, 6, + 115, 107, 105, 105, 110, 103, 1, 1, 0, 103, 0, 0, 0, 0, 0, 0, 0, 21, 110, 101, 119, 99, 104, 97, 114, 108, 105, 101, 64, 100, 111, 109, 97, 105, 110, + 46, 99, 111, 109, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 17, 110, 101, 119, 100, 97, 110, 64, 100, 111, 109, 97, 105, 110, 46, 99, 111, 109, 97, 0, + 0, 0, 0, 5, 0, 96, 0, 39, 0, 3, 0, 42, 0, 4, 0, 46, 0, 5, 0, 51, 0, 6, 0, 57, 0, 14, 0, 12, 71, 0, 12, 78, 0, 12, 86, 0, 5, 99, 0, 12, 90, 0, 100, + 97, 121, 114, 111, 108, 101, 99, 111, 108, 111, 114, 115, 97, 108, 97, 114, 121, 102, 97, 118, 111, 114, 105, 116, 101, 95, 99, 111, 108, 111, 114, + 6, 102, 114, 105, 100, 97, 121, 7, 109, 97, 110, 97, 103, 101, 114, 3, 114, 101, 100, 5, 98, 108, 97, 99, 107, 1, 1, 0, 104, 0, 0, 0, 0, 0, 0, 0, 17, + 110, 101, 119, 100, 97, 110, 64, 100, 111, 109, 97, 105, 110, 46, 99, 111, 109, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 17, 110, 101, 119, 101, 118, + 101, 64, 100, 111, 109, 97, 105, 110, 46, 99, 111, 109, 97, 0, 0, 0, 0, 5, 0, 96, 0, 39, 0, 3, 0, 42, 0, 4, 0, 46, 0, 5, 0, 51, 0, 6, 0, 57, 0, 14, + 0, 12, 71, 0, 12, 78, 0, 12, 86, 0, 5, 100, 0, 12, 90, 0, 100, 97, 121, 114, 111, 108, 101, 99, 111, 108, 111, 114, 115, 97, 108, 97, 114, 121, 102, + 97, 118, 111, 114, 105, 116, 101, 95, 99, 111, 108, 111, 114, 6, 102, 114, 105, 100, 97, 121, 7, 109, 97, 110, 97, 103, 101, 114, 3, 114, 101, 100, + 5, 98, 108, 97, 99, 107, 1, 1, 0, 105, 0, 0, 0, 0, 0, 0, 0, 17, 110, 101, 119, 101, 118, 101, 64, 100, 111, 109, 97, 105, 110, 46, 99, 111, 109, 0, + 0, 0, 0, + }, + numRows: 5, + want: "", + }, } for _, tc := range testCases { From dcec5f6e89082ebaf4b9222287d83ba48bdf1b70 Mon Sep 17 00:00:00 2001 From: Matt Lord Date: Thu, 19 Dec 2024 11:55:35 -0500 Subject: [PATCH 26/39] Update unit test name and comment to reflect content Signed-off-by: Matt Lord --- .../vreplication/vplayer_flaky_test.go | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/go/vt/vttablet/tabletmanager/vreplication/vplayer_flaky_test.go b/go/vt/vttablet/tabletmanager/vreplication/vplayer_flaky_test.go index d6efcb5d5d4..9db9e73ab8a 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/vplayer_flaky_test.go +++ b/go/vt/vttablet/tabletmanager/vreplication/vplayer_flaky_test.go @@ -1519,11 +1519,11 @@ func TestPlayerRowMove(t *testing.T) { validateQueryCountStat(t, "replicate", 3) } -// TestPlayerPartialImagesUpdatePK tests the behavior of the vplayer when -// updating the Primary Key for a row when we have partial binlog -// images, meaning that binlog-row-image=NOBLOB and/or -// binlog-row-value-options=PARTIAL_JSON. -func TestPlayerPartialImagesUpdatePK(t *testing.T) { +// TestPlayerPartialImages tests the behavior of the vplayer when modifying +// a table with BLOB and JSON columns, including when modifying the Primary +// Key for a row, when we have partial binlog images, meaning that +// binlog-row-image=NOBLOB and/or binlog-row-value-options=PARTIAL_JSON. +func TestPlayerPartialImages(t *testing.T) { if !runPartialJSONTest { t.Skip("Skipping test as binlog_row_value_options=PARTIAL_JSON is not enabled") } @@ -1559,6 +1559,7 @@ func TestPlayerPartialImagesUpdatePK(t *testing.T) { data [][]string error string } + testCases := []testCase{ { input: "insert into src (id, jd, bd) values (1,'{\"key1\": \"val1\"}','blob data'), (2,'{\"key2\": \"val2\"}','blob data2'), (3,'{\"key3\": \"val3\"}','blob data3')", @@ -1718,12 +1719,14 @@ func TestPlayerTypes(t *testing.T) { } cancel, _ := startVReplication(t, bls, "") defer cancel() + type testcase struct { input string output string table string data [][]string } + testcases := []testcase{{ input: "insert into vitess_ints values(-128, 255, -32768, 65535, -8388608, 16777215, -2147483648, 4294967295, -9223372036854775808, 18446744073709551615, 2012)", output: "insert into vitess_ints(tiny,tinyu,small,smallu,medium,mediumu,normal,normalu,big,bigu,y) values (-128,255,-32768,65535,-8388608,16777215,-2147483648,4294967295,-9223372036854775808,18446744073709551615,2012)", From 64e0422dcbeea0647d1f912ddb8210139c9df0ed Mon Sep 17 00:00:00 2001 From: Matt Lord Date: Sat, 21 Dec 2024 12:20:31 -0500 Subject: [PATCH 27/39] Another self review 1. Use iota for op type 2. Optimize binary diff parsing when 0 bytes (don't allocate) Signed-off-by: Matt Lord --- go/mysql/binlog/binlog_json.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/go/mysql/binlog/binlog_json.go b/go/mysql/binlog/binlog_json.go index 2b83780df72..5d132ff81d1 100644 --- a/go/mysql/binlog/binlog_json.go +++ b/go/mysql/binlog/binlog_json.go @@ -51,9 +51,9 @@ https://github.com/noplay/python-mysql-replication/blob/175df28cc8b536a68522ff9b type jsonDiffOp uint8 const ( - jsonDiffOpReplace = jsonDiffOp(0) - jsonDiffOpInsert = jsonDiffOp(1) - jsonDiffOpRemove = jsonDiffOp(2) + jsonDiffOpReplace = jsonDiffOp(iota) + jsonDiffOpInsert + jsonDiffOpRemove ) // ParseBinaryJSON provides the parsing function from the mysql binary json @@ -75,6 +75,10 @@ func ParseBinaryJSON(data []byte) (*json.Value, error) { // ParseBinaryJSONDiff provides the parsing function from the binary MySQL // JSON diff representation to an SQL expression. func ParseBinaryJSONDiff(data []byte) (sqltypes.Value, error) { + if len(data) == 0 { + return sqltypes.MakeTrusted(sqltypes.Expression, data), nil + } + diff := bytes.Buffer{} // Reasonable estimate of the space we'll need to build the SQL // expression in order to try and avoid reallocations w/o From a230cb15d8fd414daa993f2fbca55d86beb93c1e Mon Sep 17 00:00:00 2001 From: Matt Lord Date: Sat, 21 Dec 2024 12:37:05 -0500 Subject: [PATCH 28/39] Error nits (send halp) Signed-off-by: Matt Lord --- go/mysql/binlog/binlog_json.go | 4 ++-- go/test/utils/binlog.go | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/go/mysql/binlog/binlog_json.go b/go/mysql/binlog/binlog_json.go index 5d132ff81d1..ed3a4ac646b 100644 --- a/go/mysql/binlog/binlog_json.go +++ b/go/mysql/binlog/binlog_json.go @@ -140,8 +140,8 @@ func ParseBinaryJSONDiff(data []byte) (sqltypes.Value, error) { pos = readTo value, err := ParseBinaryJSON(data[pos : pos+valueLen]) if err != nil { - return sqltypes.Value{}, vterrors.Errorf(vtrpcpb.Code_INVALID_ARGUMENT, - "cannot read JSON diff value for path %s: %v", path, err) + return sqltypes.Value{}, vterrors.Wrapf(err, + "cannot read JSON diff value for path %q", path) } pos += valueLen if value.Type() == json.TypeString { diff --git a/go/test/utils/binlog.go b/go/test/utils/binlog.go index 0db6929054c..6e43dd511ad 100644 --- a/go/test/utils/binlog.go +++ b/go/test/utils/binlog.go @@ -17,6 +17,7 @@ limitations under the License. package utils import ( + "errors" "fmt" "os" "strconv" @@ -61,7 +62,7 @@ func SetBinlogRowImageOptions(mode string, partialJSON bool, cnfDir string) erro } if partialJSON { if !CIDBPlatformIsMySQL8orLater() { - return fmt.Errorf("partial JSON values are only supported in MySQL 8.0 or later") + return errors.New("partial JSON values are only supported in MySQL 8.0 or later") } // We're testing partial binlog row images so let's also test partial // JSON values in the images. From f23f73cdcc02265643eb98f7482664c98115985f Mon Sep 17 00:00:00 2001 From: Matt Lord Date: Sat, 21 Dec 2024 13:17:03 -0500 Subject: [PATCH 29/39] Grow byte buffer to exact size Signed-off-by: Matt Lord --- go/vt/vttablet/tabletmanager/vreplication/replicator_plan.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/vt/vttablet/tabletmanager/vreplication/replicator_plan.go b/go/vt/vttablet/tabletmanager/vreplication/replicator_plan.go index ec9de26d7e6..a201ce25847 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/replicator_plan.go +++ b/go/vt/vttablet/tabletmanager/vreplication/replicator_plan.go @@ -505,7 +505,7 @@ func (tp *TablePlan) applyChange(rowChange *binlogdatapb.RowChange, executor fun diff := afterVals[i].RawStr() beforeVal := bindvars["b_"+field.Name].Value buf := bytes.Buffer{} - buf.Grow(len(diff) + len(beforeVal) + len(sqlparser.Utf8mb4Str) + 2) // +2 is for the enclosing quotes + buf.Grow(len(beforeVal) + len(sqlparser.Utf8mb4Str) + 2) // +2 is for the enclosing quotes buf.WriteString(sqlparser.Utf8mb4Str) buf.WriteByte('\'') buf.Write(beforeVal) From 93aa37cf90b3585f1613a123247ac4a8e4a96a69 Mon Sep 17 00:00:00 2001 From: Matt Lord Date: Sat, 21 Dec 2024 13:49:32 -0500 Subject: [PATCH 30/39] Add relese note summary section Signed-off-by: Matt Lord --- changelog/22.0/22.0.0/summary.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/changelog/22.0/22.0.0/summary.md b/changelog/22.0/22.0.0/summary.md index cb8372cd60e..3f63b2d868f 100644 --- a/changelog/22.0/22.0.0/summary.md +++ b/changelog/22.0/22.0.0/summary.md @@ -8,6 +8,7 @@ - **[RPC Changes](#rpc-changes)** - **[Prefer not promoting a replica that is currently taking a backup](#reparents-prefer-not-backing-up)** - **[VTOrc Config File Changes](#vtorc-config-file-changes)** + - **[Support for More Efficient JSON Replication](#efficient-json-replication)** - **[Minor Changes](#minor-changes)** - **[VTTablet Flags](#flags-vttablet)** - **[Topology read concurrency behaviour changes](#topo-read-concurrency-changes)** @@ -59,6 +60,12 @@ The following fields can be dynamically changed - To upgrade to the newer version of the configuration file, first switch to using the flags in your current deployment before upgrading. Then you can switch to using the configuration file in the newer release. +### Support for More Efficient JSON Replication + +In [#7345](https://github.com/vitessio/vitess/pull/17345) we added support for [`--binlog-row-value-options=PARTIAL_JSON`](https://dev.mysql.com/doc/refman/en/replication-options-binary-log.html#sysvar_binlog_row_value_options). You can read more about [this feature added to MySQL 8.0 here](https://dev.mysql.com/blog-archive/efficient-json-replication-in-mysql-8-0/). + +If you are using MySQL 8.0 or later and using JSON columns, you can now enable this MySQL feature across your Vitess cluster(s) to lower the disk space needed for binary logs and improve the CPU and memory usage in both `mysqld` (standard intrashard MySQL replication) and `vttablet` ([VReplication](https://vitess.io/docs/reference/vreplication/vreplication/)) without losing any capabilities or features. + ## Minor Changes From 626439704cce91511fcba6f09f85a11af3b34487 Mon Sep 17 00:00:00 2001 From: Matt Lord Date: Sat, 21 Dec 2024 21:47:26 -0500 Subject: [PATCH 31/39] Add more unit test cases Now we're covering all JSON types: - string - number - object (JSON object) - array - boolean - null Signed-off-by: Matt Lord --- go/mysql/binlog/binlog_json.go | 6 +- .../vreplication/vplayer_flaky_test.go | 79 ++++++++++++++++--- 2 files changed, 72 insertions(+), 13 deletions(-) diff --git a/go/mysql/binlog/binlog_json.go b/go/mysql/binlog/binlog_json.go index ed3a4ac646b..99845515823 100644 --- a/go/mysql/binlog/binlog_json.go +++ b/go/mysql/binlog/binlog_json.go @@ -144,10 +144,8 @@ func ParseBinaryJSONDiff(data []byte) (sqltypes.Value, error) { "cannot read JSON diff value for path %q", path) } pos += valueLen - if value.Type() == json.TypeString { - diff.WriteString(sqlparser.Utf8mb4Str) - } - diff.Write(value.MarshalTo(nil)) + buf := value.MarshalSQLTo(nil) + diff.Write(buf) diff.WriteByte(')') } diff --git a/go/vt/vttablet/tabletmanager/vreplication/vplayer_flaky_test.go b/go/vt/vttablet/tabletmanager/vreplication/vplayer_flaky_test.go index 9db9e73ab8a..872ba7a59b1 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/vplayer_flaky_test.go +++ b/go/vt/vttablet/tabletmanager/vreplication/vplayer_flaky_test.go @@ -1562,8 +1562,10 @@ func TestPlayerPartialImages(t *testing.T) { testCases := []testCase{ { - input: "insert into src (id, jd, bd) values (1,'{\"key1\": \"val1\"}','blob data'), (2,'{\"key2\": \"val2\"}','blob data2'), (3,'{\"key3\": \"val3\"}','blob data3')", - output: []string{"insert into dst(id,jd,bd) values (1,JSON_OBJECT(_utf8mb4'key1', _utf8mb4'val1'),_binary'blob data'), (2,JSON_OBJECT(_utf8mb4'key2', _utf8mb4'val2'),_binary'blob data2'), (3,JSON_OBJECT(_utf8mb4'key3', _utf8mb4'val3'),_binary'blob data3')"}, + input: "insert into src (id, jd, bd) values (1,'{\"key1\": \"val1\"}','blob data'), (2,'{\"key2\": \"val2\"}','blob data2'), (3,'{\"key3\": \"val3\"}','blob data3')", + output: []string{ + "insert into dst(id,jd,bd) values (1,JSON_OBJECT(_utf8mb4'key1', _utf8mb4'val1'),_binary'blob data'), (2,JSON_OBJECT(_utf8mb4'key2', _utf8mb4'val2'),_binary'blob data2'), (3,JSON_OBJECT(_utf8mb4'key3', _utf8mb4'val3'),_binary'blob data3')", + }, data: [][]string{ {"1", "{\"key1\": \"val1\"}", "blob data"}, {"2", "{\"key2\": \"val2\"}", "blob data2"}, @@ -1573,8 +1575,10 @@ func TestPlayerPartialImages(t *testing.T) { } if runNoBlobTest { testCases = append(testCases, testCase{ - input: `update src set jd=JSON_SET(jd, '$.color', 'red') where id = 1`, - output: []string{"update dst set jd=JSON_INSERT(`jd`, _utf8mb4'$.color', _utf8mb4\"red\") where id=1"}, + input: `update src set jd=JSON_SET(jd, '$.color', 'red') where id = 1`, + output: []string{ + "update dst set jd=JSON_INSERT(`jd`, _utf8mb4'$.color', CAST(JSON_QUOTE(_utf8mb4'red') as JSON)) where id=1", + }, data: [][]string{ {"1", "{\"key1\": \"val1\", \"color\": \"red\"}", "blob data"}, {"2", "{\"key2\": \"val2\"}", "blob data2"}, @@ -1583,8 +1587,10 @@ func TestPlayerPartialImages(t *testing.T) { }) } else { testCases = append(testCases, testCase{ - input: `update src set jd=JSON_SET(jd, '$.color', 'red') where id = 1`, - output: []string{"update dst set jd=JSON_INSERT(`jd`, _utf8mb4'$.color', _utf8mb4\"red\"), bd=_binary'blob data' where id=1"}, + input: `update src set jd=JSON_SET(jd, '$.color', 'red') where id = 1`, + output: []string{ + "update dst set jd=JSON_INSERT(`jd`, _utf8mb4'$.color', CAST(JSON_QUOTE(_utf8mb4'red') as JSON)), bd=_binary'blob data' where id=1", + }, data: [][]string{ {"1", "{\"key1\": \"val1\", \"color\": \"red\"}", "blob data"}, {"2", "{\"key2\": \"val2\"}", "blob data2"}, @@ -1616,6 +1622,61 @@ func TestPlayerPartialImages(t *testing.T) { {"12", "{\"key2\": \"val2\"}", "newest blob data"}, }, }, + { + input: `update src set jd=JSON_SET(jd, '$.years', 5) where id = 1`, + output: []string{ + "update dst set jd=JSON_INSERT(`jd`, _utf8mb4'$.years', CAST(5 as JSON)) where id=1", + }, + data: [][]string{ + {"1", "{\"key1\": \"val1\", \"color\": \"red\", \"years\": 5}", "blob data"}, + {"3", "{\"key3\": \"val3\"}", "blob data3"}, + {"12", "{\"key2\": \"val2\"}", "newest blob data"}, + }, + }, + { + input: `update src set jd=JSON_SET(jd, '$.hobbies', JSON_ARRAY('skiing', 'video games', 'hiking')) where id = 1`, + output: []string{ + "update dst set jd=JSON_INSERT(`jd`, _utf8mb4'$.hobbies', JSON_ARRAY(_utf8mb4'skiing', _utf8mb4'video games', _utf8mb4'hiking')) where id=1", + }, + data: [][]string{ + {"1", "{\"key1\": \"val1\", \"color\": \"red\", \"years\": 5, \"hobbies\": [\"skiing\", \"video games\", \"hiking\"]}", "blob data"}, + {"3", "{\"key3\": \"val3\"}", "blob data3"}, + {"12", "{\"key2\": \"val2\"}", "newest blob data"}, + }, + }, + { + input: `update src set jd=JSON_SET(jd, '$.misc', '{"address":"1012 S Park", "town":"Hastings", "state":"MI"}') where id = 12`, + output: []string{ + "update dst set jd=JSON_INSERT(`jd`, _utf8mb4'$.misc', CAST(JSON_QUOTE(_utf8mb4'{\"address\":\"1012 S Park\", \"town\":\"Hastings\", \"state\":\"MI\"}') as JSON)) where id=12", + }, + data: [][]string{ + {"1", "{\"key1\": \"val1\", \"color\": \"red\", \"years\": 5, \"hobbies\": [\"skiing\", \"video games\", \"hiking\"]}", "blob data"}, + {"3", "{\"key3\": \"val3\"}", "blob data3"}, + {"12", "{\"key2\": \"val2\", \"misc\": \"{\\\"address\\\":\\\"1012 S Park\\\", \\\"town\\\":\\\"Hastings\\\", \\\"state\\\":\\\"MI\\\"}\"}", "newest blob data"}, + }, + }, + { + input: `update src set jd=JSON_SET(jd, '$.current', true) where id = 12`, + output: []string{ + "update dst set jd=JSON_INSERT(`jd`, _utf8mb4'$.current', CAST(_utf8mb4'true' as JSON)) where id=12", + }, + data: [][]string{ + {"1", "{\"key1\": \"val1\", \"color\": \"red\", \"years\": 5, \"hobbies\": [\"skiing\", \"video games\", \"hiking\"]}", "blob data"}, + {"3", "{\"key3\": \"val3\"}", "blob data3"}, + {"12", "{\"key2\": \"val2\", \"misc\": \"{\\\"address\\\":\\\"1012 S Park\\\", \\\"town\\\":\\\"Hastings\\\", \\\"state\\\":\\\"MI\\\"}\", \"current\": true}", "newest blob data"}, + }, + }, + { + input: `update src set jd=JSON_SET(jd, '$.idontknow', null) where id = 3`, + output: []string{ + "update dst set jd=JSON_INSERT(`jd`, _utf8mb4'$.idontknow', CAST(_utf8mb4'null' as JSON)) where id=3", + }, + data: [][]string{ + {"1", "{\"key1\": \"val1\", \"color\": \"red\", \"years\": 5, \"hobbies\": [\"skiing\", \"video games\", \"hiking\"]}", "blob data"}, + {"3", "{\"key3\": \"val3\", \"idontknow\": null}", "blob data3"}, + {"12", "{\"key2\": \"val2\", \"misc\": \"{\\\"address\\\":\\\"1012 S Park\\\", \\\"town\\\":\\\"Hastings\\\", \\\"state\\\":\\\"MI\\\"}\", \"current\": true}", "newest blob data"}, + }, + }, }...) if runNoBlobTest { testCases = append(testCases, testCase{ @@ -1630,9 +1691,9 @@ func TestPlayerPartialImages(t *testing.T) { "insert into dst(id,jd,bd) values (13,JSON_OBJECT(_utf8mb4'key3', _utf8mb4'val3'),_binary'blob data3')", }, data: [][]string{ - {"1", "{\"key1\": \"val1\", \"color\": \"red\"}", "blob data"}, - {"12", "{\"key2\": \"val2\"}", "newest blob data"}, - {"13", "{\"key3\": \"val3\"}", "blob data3"}, + {"1", "{\"key1\": \"val1\", \"color\": \"red\", \"years\": 5, \"hobbies\": [\"skiing\", \"video games\", \"hiking\"]}", "blob data"}, + {"12", "{\"key2\": \"val2\", \"misc\": \"{\\\"address\\\":\\\"1012 S Park\\\", \\\"town\\\":\\\"Hastings\\\", \\\"state\\\":\\\"MI\\\"}\", \"current\": true}", "newest blob data"}, + {"13", "{\"key3\": \"val3\", \"idontknow\": null}", "blob data3"}, }, }) } From 3a172c08597612336228c614913555e26a14529a Mon Sep 17 00:00:00 2001 From: Matt Lord Date: Sat, 21 Dec 2024 23:19:50 -0500 Subject: [PATCH 32/39] Simplify buffer allocation Signed-off-by: Matt Lord --- go/mysql/binlog/binlog_json.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/mysql/binlog/binlog_json.go b/go/mysql/binlog/binlog_json.go index 99845515823..e6b7d09819d 100644 --- a/go/mysql/binlog/binlog_json.go +++ b/go/mysql/binlog/binlog_json.go @@ -83,7 +83,7 @@ func ParseBinaryJSONDiff(data []byte) (sqltypes.Value, error) { // Reasonable estimate of the space we'll need to build the SQL // expression in order to try and avoid reallocations w/o // overallocating too much. - diff.Grow(int(float32(len(data)) * 1.25)) + diff.Grow(len(data) + 80) pos := 0 outer := false innerStr := "" From 7dc29b3edd1b3fc89f6b5258bba3736fabadb965 Mon Sep 17 00:00:00 2001 From: Matt Lord Date: Sat, 21 Dec 2024 23:43:56 -0500 Subject: [PATCH 33/39] Cover JSON null and literal null string in unit test case Signed-off-by: Matt Lord --- go/mysql/binlog/binlog_json.go | 5 ++--- .../tabletmanager/vreplication/vplayer_flaky_test.go | 6 +++--- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/go/mysql/binlog/binlog_json.go b/go/mysql/binlog/binlog_json.go index e6b7d09819d..30c80531a96 100644 --- a/go/mysql/binlog/binlog_json.go +++ b/go/mysql/binlog/binlog_json.go @@ -106,7 +106,7 @@ func ParseBinaryJSONDiff(data []byte) (sqltypes.Value, error) { // Can be a JSON null. js, err := ParseBinaryJSON(data) if err == nil && js.Type() == json.TypeNull { - return sqltypes.MakeTrusted(sqltypes.Expression, js.MarshalTo(nil)), nil + return sqltypes.MakeTrusted(sqltypes.Expression, js.MarshalSQLTo(nil)), nil } return sqltypes.Value{}, vterrors.Errorf(vtrpcpb.Code_INVALID_ARGUMENT, "invalid JSON diff operation: %d", opType) @@ -144,8 +144,7 @@ func ParseBinaryJSONDiff(data []byte) (sqltypes.Value, error) { "cannot read JSON diff value for path %q", path) } pos += valueLen - buf := value.MarshalSQLTo(nil) - diff.Write(buf) + diff.Write(value.MarshalSQLTo(nil)) diff.WriteByte(')') } diff --git a/go/vt/vttablet/tabletmanager/vreplication/vplayer_flaky_test.go b/go/vt/vttablet/tabletmanager/vreplication/vplayer_flaky_test.go index 872ba7a59b1..e697f6d2752 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/vplayer_flaky_test.go +++ b/go/vt/vttablet/tabletmanager/vreplication/vplayer_flaky_test.go @@ -1667,13 +1667,13 @@ func TestPlayerPartialImages(t *testing.T) { }, }, { - input: `update src set jd=JSON_SET(jd, '$.idontknow', null) where id = 3`, + input: `update src set jd=JSON_SET(jd, '$.idontknow', null, '$.idontknoweither', 'null') where id = 3`, output: []string{ - "update dst set jd=JSON_INSERT(`jd`, _utf8mb4'$.idontknow', CAST(_utf8mb4'null' as JSON)) where id=3", + "update dst set jd=JSON_INSERT(JSON_INSERT(`jd`, _utf8mb4'$.idontknow', CAST(_utf8mb4'null' as JSON)), _utf8mb4'$.idontknoweither', CAST(JSON_QUOTE(_utf8mb4'null') as JSON)) where id=3", }, data: [][]string{ {"1", "{\"key1\": \"val1\", \"color\": \"red\", \"years\": 5, \"hobbies\": [\"skiing\", \"video games\", \"hiking\"]}", "blob data"}, - {"3", "{\"key3\": \"val3\", \"idontknow\": null}", "blob data3"}, + {"3", "{\"key3\": \"val3\", \"idontknow\": null, \"idontknoweither\": \"null\"}", "blob data3"}, {"12", "{\"key2\": \"val2\", \"misc\": \"{\\\"address\\\":\\\"1012 S Park\\\", \\\"town\\\":\\\"Hastings\\\", \\\"state\\\":\\\"MI\\\"}\", \"current\": true}", "newest blob data"}, }, }, From 79d3f08467a381bb90e5fa69a9b8aba842d7097c Mon Sep 17 00:00:00 2001 From: Matt Lord Date: Sat, 21 Dec 2024 23:50:05 -0500 Subject: [PATCH 34/39] Update unit test for SQL conversion Signed-off-by: Matt Lord --- go/mysql/binlog_event_mysql56_test.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/go/mysql/binlog_event_mysql56_test.go b/go/mysql/binlog_event_mysql56_test.go index 161e48a50ae..ede2abece99 100644 --- a/go/mysql/binlog_event_mysql56_test.go +++ b/go/mysql/binlog_event_mysql56_test.go @@ -352,7 +352,7 @@ func TestMySQL56PartialUpdateRowsEvent(t *testing.T) { 97, 110, 97, 103, 101, 114, }, numRows: 5, - want: "JSON_INSERT(%s, _utf8mb4'$.role', _utf8mb4\"manager\")", + want: "JSON_INSERT(%s, _utf8mb4'$.role', CAST(JSON_QUOTE(_utf8mb4'manager') as JSON))", }, { // The mysqlbinlog -vvv --base64-output=decode-rows output for the following event: @@ -373,7 +373,7 @@ func TestMySQL56PartialUpdateRowsEvent(t *testing.T) { }, name: "REPLACE", numRows: 1, - want: "JSON_REPLACE(%s, _utf8mb4'$.role', _utf8mb4\"IC\")", + want: "JSON_REPLACE(%s, _utf8mb4'$.role', CAST(JSON_QUOTE(_utf8mb4'IC') as JSON))", }, { name: "REMOVE", @@ -486,7 +486,7 @@ func TestMySQL56PartialUpdateRowsEvent(t *testing.T) { 100, 97, 121, 2, 16, 36, 46, 102, 97, 118, 111, 114, 105, 116, 101, 95, 99, 111, 108, 111, 114, }, numRows: 5, - want: "JSON_REMOVE(JSON_REPLACE(%s, _utf8mb4'$.day', _utf8mb4\"monday\"), _utf8mb4'$.favorite_color')", + want: "JSON_REMOVE(JSON_REPLACE(%s, _utf8mb4'$.day', CAST(JSON_QUOTE(_utf8mb4'monday') as JSON)), _utf8mb4'$.favorite_color')", }, { name: "INSERT and REMOVE and REPLACE", @@ -514,7 +514,7 @@ func TestMySQL56PartialUpdateRowsEvent(t *testing.T) { 1, 7, 36, 46, 104, 111, 98, 98, 121, 8, 12, 6, 115, 107, 105, 105, 110, 103, }, numRows: 1, - want: "JSON_INSERT(JSON_REMOVE(JSON_REPLACE(%s, _utf8mb4'$.day', _utf8mb4\"tuesday\"), _utf8mb4'$.favorite_color'), _utf8mb4'$.hobby', _utf8mb4\"skiing\")", + want: "JSON_INSERT(JSON_REMOVE(JSON_REPLACE(%s, _utf8mb4'$.day', CAST(JSON_QUOTE(_utf8mb4'tuesday') as JSON)), _utf8mb4'$.favorite_color'), _utf8mb4'$.hobby', CAST(JSON_QUOTE(_utf8mb4'skiing') as JSON))", }, { name: "REPLACE with null", @@ -535,7 +535,7 @@ func TestMySQL56PartialUpdateRowsEvent(t *testing.T) { 109, 97, 105, 110, 46, 99, 111, 109, 13, 0, 0, 0, 0, 8, 36, 46, 115, 97, 108, 97, 114, 121, 2, 4, 0, }, numRows: 1, - want: "JSON_REPLACE(%s, _utf8mb4'$.salary', null)", + want: "JSON_REPLACE(%s, _utf8mb4'$.salary', CAST(_utf8mb4'null' as JSON))", }, { name: "REPLACE 2 paths", @@ -557,7 +557,7 @@ func TestMySQL56PartialUpdateRowsEvent(t *testing.T) { 105, 110, 46, 99, 111, 109, 27, 0, 0, 0, 0, 8, 36, 46, 115, 97, 108, 97, 114, 121, 3, 5, 110, 0, 0, 6, 36, 46, 114, 111, 108, 101, 4, 12, 2, 73, 67, }, numRows: 1, - want: "JSON_REPLACE(JSON_REPLACE(%s, _utf8mb4'$.salary', 110), _utf8mb4'$.role', _utf8mb4\"IC\")", + want: "JSON_REPLACE(JSON_REPLACE(%s, _utf8mb4'$.salary', CAST(110 as JSON)), _utf8mb4'$.role', CAST(JSON_QUOTE(_utf8mb4'IC') as JSON))", }, { name: "JSON null", @@ -624,7 +624,7 @@ func TestMySQL56PartialUpdateRowsEvent(t *testing.T) { 105, 110, 103, 115, 34, 44, 32, 34, 115, 116, 97, 116, 101, 34, 58, 34, 77, 73, 34, 125, }, numRows: 1, - want: "JSON_INSERT(%s, _utf8mb4'$.misc', _utf8mb4\"{\\\"address\\\":\\\"1012 S Park\\\", \\\"town\\\":\\\"Hastings\\\", \\\"state\\\":\\\"MI\\\"}\")", + want: "JSON_INSERT(%s, _utf8mb4'$.misc', CAST(JSON_QUOTE(_utf8mb4'{\"address\":\"1012 S Park\", \"town\":\"Hastings\", \"state\":\"MI\"}') as JSON))", }, { name: "JSON field not updated", From 5deec925173b87e5ee35f250badc4b084c38aa31 Mon Sep 17 00:00:00 2001 From: Matt Lord Date: Mon, 23 Dec 2024 12:31:32 -0500 Subject: [PATCH 35/39] Add a lot of comments to ParseBinaryJSONDiff Signed-off-by: Matt Lord --- go/mysql/binlog/binlog_json.go | 90 ++++++++++++++++++++++++++++++++-- 1 file changed, 87 insertions(+), 3 deletions(-) diff --git a/go/mysql/binlog/binlog_json.go b/go/mysql/binlog/binlog_json.go index 30c80531a96..d1ecf464081 100644 --- a/go/mysql/binlog/binlog_json.go +++ b/go/mysql/binlog/binlog_json.go @@ -73,9 +73,57 @@ func ParseBinaryJSON(data []byte) (*json.Value, error) { } // ParseBinaryJSONDiff provides the parsing function from the binary MySQL -// JSON diff representation to an SQL expression. +// JSON diff representation to an SQL expression. These diffs are included +// in the AFTER image of PartialUpdateRows events which exist in MySQL 8.0 +// and later when the --binlog-row-value-options=PARTIAL_JSON is used. You +// can read more about these here: +// https://dev.mysql.com/blog-archive/efficient-json-replication-in-mysql-8-0/ +// https://dev.mysql.com/worklog/task/?id=2955 +// https://github.com/mysql/mysql-server/blob/trunk/sql-common/json_diff.h +// https://github.com/mysql/mysql-server/blob/trunk/sql-common/json_diff.cc +// +// The binary format for the partial JSON column or JSON diff is: +// +--------+--------+--------+ +--------+ +// | length | diff_1 | diff_2 | ... | diff_N | +// +--------+--------+--------+ +--------+ +// +// Each diff_i represents a single JSON diff. It has the following +// format: +// +-----------+-------------+------+ +-------------+------+ +// | operation | path_length | path | ( | data_length | data | )? +// +-----------+-------------+------+ +-------------+------+ +// +// The fields are: +// +// 1. operation: a single byte containing the JSON diff operation. +// The possible values are defined by enum_json_diff_operation: +// REPLACE=0 +// INSERT=1 +// REMOVE=2 +// +// 2. path_length: an unsigned integer in net_field_length() format. +// +// 3. path: a string of 'path_length' bytes containing the JSON path +// of the update. +// +// 4. data_length: an unsigned integer in net_field_length() format. +// +// 5. data: a string of 'data_length' bytes containing the JSON +// document that will be inserted at the position specified by +// 'path'. +// +// data_length and data are omitted if and only if operation=REMOVE. +// +// Examples of the resulting SQL expression are: +// - "" for an empty diff when the column was not updated +// - "null" for a JSON null +// - "JSON_REMOVE(%s, _utf8mb4'$.salary')" for a REMOVE operation +// - "JSON_INSERT(%s, _utf8mb4'$.role', CAST(JSON_QUOTE(_utf8mb4'manager') as JSON))" for an INSERT operation +// - "JSON_INSERT(JSON_REMOVE(JSON_REPLACE(%s, _utf8mb4'$.day', CAST(JSON_QUOTE(_utf8mb4'tuesday') as JSON)), _utf8mb4'$.favorite_color'), _utf8mb4'$.hobby', CAST(JSON_QUOTE(_utf8mb4'skiing') as JSON))" for a more complex example func ParseBinaryJSONDiff(data []byte) (sqltypes.Value, error) { if len(data) == 0 { + // An empty diff is used as a way to elide the column from + // the AFTER image when it was not updated in the row event. return sqltypes.MakeTrusted(sqltypes.Expression, data), nil } @@ -88,10 +136,36 @@ func ParseBinaryJSONDiff(data []byte) (sqltypes.Value, error) { outer := false innerStr := "" + // Create the SQL expression from the data which will consist of + // a sequence of JSON_X(col/json, path[, value]) clauses where X + // is REPLACE, INSERT, or REMOVE. The data can also be a JSON + // null, which is a special case we handle here as well. We take + // a binary representation of a vector of JSON diffs, for example: + // (REPLACE, '$.a', '7') + // (REMOVE, '$.d[0]') + // (INSERT, '$.e', '"ee"') + // (INSERT, '$.f[1]', '"ff"') + // (INSERT, '$.g', '"gg"') + // And build an SQL expression from it: + // JSON_INSERT( + // JSON_INSERT( + // JSON_INSERT( + // JSON_REMOVE( + // JSON_REPLACE(col, '$.a', 7), + // '$.d[0]'), + // '$.e', 'ee'), + // '$.f[3]', 'ff'), + // '$.g', 'gg') for pos < len(data) { opType := jsonDiffOp(data[pos]) pos++ if outer { + // We process the bytes sequentially but build the SQL + // expression from the inner most function to the outer most + // and thus need to wrap any subsequent functions around the + // previous one(s). For example: + // The inner: JSON_REPLACE(%s, '$.a', 7) + // The outer: JSON_REMOVE(, '$.b') innerStr = diff.String() diff.Reset() } @@ -112,6 +186,7 @@ func ParseBinaryJSONDiff(data []byte) (sqltypes.Value, error) { "invalid JSON diff operation: %d", opType) } if outer { + // Wrap this outer function around the previous inner one(s). diff.WriteString(innerStr) diff.WriteString(", ") } else { // Only the inner most function has the field name @@ -119,13 +194,16 @@ func ParseBinaryJSONDiff(data []byte) (sqltypes.Value, error) { } outer = true + // Read the JSON document path that we want to operate on. pathLen, readTo := readVariableLength(data, pos) pos = readTo path := data[pos : pos+pathLen] pos += pathLen - // We have to specify the unicode character set for the strings we + // We have to specify the unicode character set for the path we // use in the expression as the connection can be using a different // character set (e.g. vreplication always uses set names binary). + // The generated path will look like this: + // _utf8mb4'$.role' diff.WriteString(sqlparser.Utf8mb4Str) diff.WriteByte('\'') diff.Write(path) @@ -136,16 +214,22 @@ func ParseBinaryJSONDiff(data []byte) (sqltypes.Value, error) { } diff.WriteString(", ") + // Read the JSON document value that we want to set. valueLen, readTo := readVariableLength(data, pos) pos = readTo + // The native JSON type and value that we want to + // set (string, number, object, array, null). value, err := ParseBinaryJSON(data[pos : pos+valueLen]) if err != nil { return sqltypes.Value{}, vterrors.Wrapf(err, "cannot read JSON diff value for path %q", path) } pos += valueLen + + // Generate the SQL clause for the JSON diff's value. For example: + // "CAST(JSON_QUOTE(_utf8mb4'manager') as JSON)" diff.Write(value.MarshalSQLTo(nil)) - diff.WriteByte(')') + diff.WriteByte(')') // Close the JSON function } return sqltypes.MakeTrusted(sqltypes.Expression, diff.Bytes()), nil From aeef8e93c56865ba0e04c08506c867266d62e71d Mon Sep 17 00:00:00 2001 From: Matt Lord Date: Mon, 23 Dec 2024 12:43:58 -0500 Subject: [PATCH 36/39] Comment nits Signed-off-by: Matt Lord --- go/mysql/binlog/binlog_json.go | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/go/mysql/binlog/binlog_json.go b/go/mysql/binlog/binlog_json.go index d1ecf464081..127a321f528 100644 --- a/go/mysql/binlog/binlog_json.go +++ b/go/mysql/binlog/binlog_json.go @@ -148,13 +148,14 @@ func ParseBinaryJSONDiff(data []byte) (sqltypes.Value, error) { // (INSERT, '$.g', '"gg"') // And build an SQL expression from it: // JSON_INSERT( - // JSON_INSERT( - // JSON_INSERT( - // JSON_REMOVE( - // JSON_REPLACE(col, '$.a', 7), - // '$.d[0]'), - // '$.e', 'ee'), - // '$.f[3]', 'ff'), + // JSON_INSERT( + // JSON_INSERT( + // JSON_REMOVE( + // JSON_REPLACE( + // col, '$.a', 7), + // '$.d[0]'), + // '$.e', 'ee'), + // '$.f[3]', 'ff'), // '$.g', 'gg') for pos < len(data) { opType := jsonDiffOp(data[pos]) @@ -164,8 +165,8 @@ func ParseBinaryJSONDiff(data []byte) (sqltypes.Value, error) { // expression from the inner most function to the outer most // and thus need to wrap any subsequent functions around the // previous one(s). For example: - // The inner: JSON_REPLACE(%s, '$.a', 7) - // The outer: JSON_REMOVE(, '$.b') + // - inner: JSON_REPLACE(%s, '$.a', 7) + // - outer: JSON_REMOVE(, '$.b') innerStr = diff.String() diff.Reset() } @@ -202,8 +203,7 @@ func ParseBinaryJSONDiff(data []byte) (sqltypes.Value, error) { // We have to specify the unicode character set for the path we // use in the expression as the connection can be using a different // character set (e.g. vreplication always uses set names binary). - // The generated path will look like this: - // _utf8mb4'$.role' + // The generated path will look like this: _utf8mb4'$.role' diff.WriteString(sqlparser.Utf8mb4Str) diff.WriteByte('\'') diff.Write(path) @@ -227,7 +227,7 @@ func ParseBinaryJSONDiff(data []byte) (sqltypes.Value, error) { pos += valueLen // Generate the SQL clause for the JSON diff's value. For example: - // "CAST(JSON_QUOTE(_utf8mb4'manager') as JSON)" + // CAST(JSON_QUOTE(_utf8mb4'manager') as JSON) diff.Write(value.MarshalSQLTo(nil)) diff.WriteByte(')') // Close the JSON function } From 77cb55e9c3b218a65ece6037593fde86e1e44b80 Mon Sep 17 00:00:00 2001 From: Matt Lord Date: Mon, 23 Dec 2024 13:10:22 -0500 Subject: [PATCH 37/39] Comment nits part II Signed-off-by: Matt Lord --- go/mysql/binlog/binlog_json.go | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/go/mysql/binlog/binlog_json.go b/go/mysql/binlog/binlog_json.go index 127a321f528..688e51d9aa0 100644 --- a/go/mysql/binlog/binlog_json.go +++ b/go/mysql/binlog/binlog_json.go @@ -214,19 +214,18 @@ func ParseBinaryJSONDiff(data []byte) (sqltypes.Value, error) { } diff.WriteString(", ") - // Read the JSON document value that we want to set. + // Read the value that we want to set. valueLen, readTo := readVariableLength(data, pos) pos = readTo - // The native JSON type and value that we want to - // set (string, number, object, array, null). + // Parse the native JSON type and its value that we want to set + // (string, number, object, array, null). value, err := ParseBinaryJSON(data[pos : pos+valueLen]) if err != nil { return sqltypes.Value{}, vterrors.Wrapf(err, "cannot read JSON diff value for path %q", path) } pos += valueLen - - // Generate the SQL clause for the JSON diff's value. For example: + // Generate the SQL clause for the JSON value. For example: // CAST(JSON_QUOTE(_utf8mb4'manager') as JSON) diff.Write(value.MarshalSQLTo(nil)) diff.WriteByte(')') // Close the JSON function From 27edecef52617bb9da01dc6981bf7dcc715a9e78 Mon Sep 17 00:00:00 2001 From: Matt Lord Date: Mon, 23 Dec 2024 13:29:05 -0500 Subject: [PATCH 38/39] Last comment nit (send halp) Signed-off-by: Matt Lord --- go/mysql/binlog/binlog_json.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/mysql/binlog/binlog_json.go b/go/mysql/binlog/binlog_json.go index 688e51d9aa0..51b4fef0ef8 100644 --- a/go/mysql/binlog/binlog_json.go +++ b/go/mysql/binlog/binlog_json.go @@ -209,7 +209,7 @@ func ParseBinaryJSONDiff(data []byte) (sqltypes.Value, error) { diff.Write(path) diff.WriteByte('\'') if opType == jsonDiffOpRemove { // No value for remove - diff.WriteByte(')') + diff.WriteByte(')') // Close the JSON function continue } From 3e3ab1d811989ce9d3fcf0d73642729680a17fd1 Mon Sep 17 00:00:00 2001 From: Matt Lord Date: Mon, 23 Dec 2024 14:22:32 -0500 Subject: [PATCH 39/39] Run all unit tests with binlog_row_value_options=PARTIAL_JSON This way we're testing this feature both with binlog-row-image=FULL and with binlog-row-image=NOBLOB. Signed-off-by: Matt Lord --- .../vreplication/framework_test.go | 13 +- .../vreplication/vplayer_flaky_test.go | 183 ++++++++++++------ 2 files changed, 127 insertions(+), 69 deletions(-) diff --git a/go/vt/vttablet/tabletmanager/vreplication/framework_test.go b/go/vt/vttablet/tabletmanager/vreplication/framework_test.go index c9c6451bbeb..9f2647a8770 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/framework_test.go +++ b/go/vt/vttablet/tabletmanager/vreplication/framework_test.go @@ -169,8 +169,7 @@ func setup(ctx context.Context) (func(), int) { var ( // We run unit tests twice, first with binlog_row_image=FULL, then with NOBLOB. runNoBlobTest = false - // When using MySQL 8.0 or later, we also set binlog_row_value_options=PARTIAL_JSON - // when using NOBLOB. + // When using MySQL 8.0 or later, we set binlog_row_value_options=PARTIAL_JSON. runPartialJSONTest = false ) @@ -183,7 +182,11 @@ func TestMain(m *testing.M) { exitCode := func() int { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - if err := utils.SetBinlogRowImageOptions("full", false, tempDir); err != nil { + // binlog-row-value-options=PARTIAL_JSON is only supported in MySQL 8.0 and later. + // We still run unit tests with MySQL 5.7, so we cannot add it to the cnf file + // when using 5.7 or mysqld will fail to start. + runPartialJSONTest = utils.CIDBPlatformIsMySQL8orLater() + if err := utils.SetBinlogRowImageOptions("full", runPartialJSONTest, tempDir); err != nil { panic(err) } defer utils.SetBinlogRowImageOptions("", false, tempDir) @@ -198,10 +201,6 @@ func TestMain(m *testing.M) { cancel() runNoBlobTest = true - // binlog-row-value-options=PARTIAL_JSON is only supported in MySQL 8.0 and later. - // We still run unit tests with MySQL 5.7, so we cannot add it to the cnf file - // when using 5.7 or mysqld will fail to start. - runPartialJSONTest = utils.CIDBPlatformIsMySQL8orLater() if err := utils.SetBinlogRowImageOptions("noblob", runPartialJSONTest, tempDir); err != nil { panic(err) } diff --git a/go/vt/vttablet/tabletmanager/vreplication/vplayer_flaky_test.go b/go/vt/vttablet/tabletmanager/vreplication/vplayer_flaky_test.go index e697f6d2752..fe96e47281c 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/vplayer_flaky_test.go +++ b/go/vt/vttablet/tabletmanager/vreplication/vplayer_flaky_test.go @@ -1622,80 +1622,139 @@ func TestPlayerPartialImages(t *testing.T) { {"12", "{\"key2\": \"val2\"}", "newest blob data"}, }, }, - { - input: `update src set jd=JSON_SET(jd, '$.years', 5) where id = 1`, - output: []string{ - "update dst set jd=JSON_INSERT(`jd`, _utf8mb4'$.years', CAST(5 as JSON)) where id=1", + }...) + if runNoBlobTest { + testCases = append(testCases, []testCase{ + { + input: `update src set jd=JSON_SET(jd, '$.years', 5) where id = 1`, + output: []string{ + "update dst set jd=JSON_INSERT(`jd`, _utf8mb4'$.years', CAST(5 as JSON)) where id=1", + }, + data: [][]string{ + {"1", "{\"key1\": \"val1\", \"color\": \"red\", \"years\": 5}", "blob data"}, + {"3", "{\"key3\": \"val3\"}", "blob data3"}, + {"12", "{\"key2\": \"val2\"}", "newest blob data"}, + }, }, - data: [][]string{ - {"1", "{\"key1\": \"val1\", \"color\": \"red\", \"years\": 5}", "blob data"}, - {"3", "{\"key3\": \"val3\"}", "blob data3"}, - {"12", "{\"key2\": \"val2\"}", "newest blob data"}, + { + input: `update src set jd=JSON_SET(jd, '$.hobbies', JSON_ARRAY('skiing', 'video games', 'hiking')) where id = 1`, + output: []string{ + "update dst set jd=JSON_INSERT(`jd`, _utf8mb4'$.hobbies', JSON_ARRAY(_utf8mb4'skiing', _utf8mb4'video games', _utf8mb4'hiking')) where id=1", + }, + data: [][]string{ + {"1", "{\"key1\": \"val1\", \"color\": \"red\", \"years\": 5, \"hobbies\": [\"skiing\", \"video games\", \"hiking\"]}", "blob data"}, + {"3", "{\"key3\": \"val3\"}", "blob data3"}, + {"12", "{\"key2\": \"val2\"}", "newest blob data"}, + }, }, - }, - { - input: `update src set jd=JSON_SET(jd, '$.hobbies', JSON_ARRAY('skiing', 'video games', 'hiking')) where id = 1`, - output: []string{ - "update dst set jd=JSON_INSERT(`jd`, _utf8mb4'$.hobbies', JSON_ARRAY(_utf8mb4'skiing', _utf8mb4'video games', _utf8mb4'hiking')) where id=1", + { + input: `update src set jd=JSON_SET(jd, '$.misc', '{"address":"1012 S Park", "town":"Hastings", "state":"MI"}') where id = 12`, + output: []string{ + "update dst set jd=JSON_INSERT(`jd`, _utf8mb4'$.misc', CAST(JSON_QUOTE(_utf8mb4'{\"address\":\"1012 S Park\", \"town\":\"Hastings\", \"state\":\"MI\"}') as JSON)) where id=12", + }, + data: [][]string{ + {"1", "{\"key1\": \"val1\", \"color\": \"red\", \"years\": 5, \"hobbies\": [\"skiing\", \"video games\", \"hiking\"]}", "blob data"}, + {"3", "{\"key3\": \"val3\"}", "blob data3"}, + {"12", "{\"key2\": \"val2\", \"misc\": \"{\\\"address\\\":\\\"1012 S Park\\\", \\\"town\\\":\\\"Hastings\\\", \\\"state\\\":\\\"MI\\\"}\"}", "newest blob data"}, + }, }, - data: [][]string{ - {"1", "{\"key1\": \"val1\", \"color\": \"red\", \"years\": 5, \"hobbies\": [\"skiing\", \"video games\", \"hiking\"]}", "blob data"}, - {"3", "{\"key3\": \"val3\"}", "blob data3"}, - {"12", "{\"key2\": \"val2\"}", "newest blob data"}, + { + input: `update src set jd=JSON_SET(jd, '$.current', true) where id = 12`, + output: []string{ + "update dst set jd=JSON_INSERT(`jd`, _utf8mb4'$.current', CAST(_utf8mb4'true' as JSON)) where id=12", + }, + data: [][]string{ + {"1", "{\"key1\": \"val1\", \"color\": \"red\", \"years\": 5, \"hobbies\": [\"skiing\", \"video games\", \"hiking\"]}", "blob data"}, + {"3", "{\"key3\": \"val3\"}", "blob data3"}, + {"12", "{\"key2\": \"val2\", \"misc\": \"{\\\"address\\\":\\\"1012 S Park\\\", \\\"town\\\":\\\"Hastings\\\", \\\"state\\\":\\\"MI\\\"}\", \"current\": true}", "newest blob data"}, + }, }, - }, - { - input: `update src set jd=JSON_SET(jd, '$.misc', '{"address":"1012 S Park", "town":"Hastings", "state":"MI"}') where id = 12`, - output: []string{ - "update dst set jd=JSON_INSERT(`jd`, _utf8mb4'$.misc', CAST(JSON_QUOTE(_utf8mb4'{\"address\":\"1012 S Park\", \"town\":\"Hastings\", \"state\":\"MI\"}') as JSON)) where id=12", + { + input: `update src set jd=JSON_SET(jd, '$.idontknow', null, '$.idontknoweither', 'null') where id = 3`, + output: []string{ + "update dst set jd=JSON_INSERT(JSON_INSERT(`jd`, _utf8mb4'$.idontknow', CAST(_utf8mb4'null' as JSON)), _utf8mb4'$.idontknoweither', CAST(JSON_QUOTE(_utf8mb4'null') as JSON)) where id=3", + }, + data: [][]string{ + {"1", "{\"key1\": \"val1\", \"color\": \"red\", \"years\": 5, \"hobbies\": [\"skiing\", \"video games\", \"hiking\"]}", "blob data"}, + {"3", "{\"key3\": \"val3\", \"idontknow\": null, \"idontknoweither\": \"null\"}", "blob data3"}, + {"12", "{\"key2\": \"val2\", \"misc\": \"{\\\"address\\\":\\\"1012 S Park\\\", \\\"town\\\":\\\"Hastings\\\", \\\"state\\\":\\\"MI\\\"}\", \"current\": true}", "newest blob data"}, + }, }, - data: [][]string{ - {"1", "{\"key1\": \"val1\", \"color\": \"red\", \"years\": 5, \"hobbies\": [\"skiing\", \"video games\", \"hiking\"]}", "blob data"}, - {"3", "{\"key3\": \"val3\"}", "blob data3"}, - {"12", "{\"key2\": \"val2\", \"misc\": \"{\\\"address\\\":\\\"1012 S Park\\\", \\\"town\\\":\\\"Hastings\\\", \\\"state\\\":\\\"MI\\\"}\"}", "newest blob data"}, + { + input: `update src set id = id+10 where id = 3`, + error: "binary log event missing a needed value for dst.bd due to not using binlog-row-image=FULL", }, - }, - { - input: `update src set jd=JSON_SET(jd, '$.current', true) where id = 12`, - output: []string{ - "update dst set jd=JSON_INSERT(`jd`, _utf8mb4'$.current', CAST(_utf8mb4'true' as JSON)) where id=12", + }...) + } else { + testCases = append(testCases, []testCase{ + { + input: `update src set jd=JSON_SET(jd, '$.years', 5) where id = 1`, + output: []string{ + "update dst set jd=JSON_INSERT(`jd`, _utf8mb4'$.years', CAST(5 as JSON)), bd=_binary'blob data' where id=1", + }, + data: [][]string{ + {"1", "{\"key1\": \"val1\", \"color\": \"red\", \"years\": 5}", "blob data"}, + {"3", "{\"key3\": \"val3\"}", "blob data3"}, + {"12", "{\"key2\": \"val2\"}", "newest blob data"}, + }, }, - data: [][]string{ - {"1", "{\"key1\": \"val1\", \"color\": \"red\", \"years\": 5, \"hobbies\": [\"skiing\", \"video games\", \"hiking\"]}", "blob data"}, - {"3", "{\"key3\": \"val3\"}", "blob data3"}, - {"12", "{\"key2\": \"val2\", \"misc\": \"{\\\"address\\\":\\\"1012 S Park\\\", \\\"town\\\":\\\"Hastings\\\", \\\"state\\\":\\\"MI\\\"}\", \"current\": true}", "newest blob data"}, + { + input: `update src set jd=JSON_SET(jd, '$.hobbies', JSON_ARRAY('skiing', 'video games', 'hiking')) where id = 1`, + output: []string{ + "update dst set jd=JSON_INSERT(`jd`, _utf8mb4'$.hobbies', JSON_ARRAY(_utf8mb4'skiing', _utf8mb4'video games', _utf8mb4'hiking')), bd=_binary'blob data' where id=1", + }, + data: [][]string{ + {"1", "{\"key1\": \"val1\", \"color\": \"red\", \"years\": 5, \"hobbies\": [\"skiing\", \"video games\", \"hiking\"]}", "blob data"}, + {"3", "{\"key3\": \"val3\"}", "blob data3"}, + {"12", "{\"key2\": \"val2\"}", "newest blob data"}, + }, }, - }, - { - input: `update src set jd=JSON_SET(jd, '$.idontknow', null, '$.idontknoweither', 'null') where id = 3`, - output: []string{ - "update dst set jd=JSON_INSERT(JSON_INSERT(`jd`, _utf8mb4'$.idontknow', CAST(_utf8mb4'null' as JSON)), _utf8mb4'$.idontknoweither', CAST(JSON_QUOTE(_utf8mb4'null') as JSON)) where id=3", + { + input: `update src set jd=JSON_SET(jd, '$.misc', '{"address":"1012 S Park", "town":"Hastings", "state":"MI"}') where id = 12`, + output: []string{ + "update dst set jd=JSON_INSERT(`jd`, _utf8mb4'$.misc', CAST(JSON_QUOTE(_utf8mb4'{\"address\":\"1012 S Park\", \"town\":\"Hastings\", \"state\":\"MI\"}') as JSON)), bd=_binary'newest blob data' where id=12", + }, + data: [][]string{ + {"1", "{\"key1\": \"val1\", \"color\": \"red\", \"years\": 5, \"hobbies\": [\"skiing\", \"video games\", \"hiking\"]}", "blob data"}, + {"3", "{\"key3\": \"val3\"}", "blob data3"}, + {"12", "{\"key2\": \"val2\", \"misc\": \"{\\\"address\\\":\\\"1012 S Park\\\", \\\"town\\\":\\\"Hastings\\\", \\\"state\\\":\\\"MI\\\"}\"}", "newest blob data"}, + }, }, - data: [][]string{ - {"1", "{\"key1\": \"val1\", \"color\": \"red\", \"years\": 5, \"hobbies\": [\"skiing\", \"video games\", \"hiking\"]}", "blob data"}, - {"3", "{\"key3\": \"val3\", \"idontknow\": null, \"idontknoweither\": \"null\"}", "blob data3"}, - {"12", "{\"key2\": \"val2\", \"misc\": \"{\\\"address\\\":\\\"1012 S Park\\\", \\\"town\\\":\\\"Hastings\\\", \\\"state\\\":\\\"MI\\\"}\", \"current\": true}", "newest blob data"}, + { + input: `update src set jd=JSON_SET(jd, '$.current', true) where id = 12`, + output: []string{ + "update dst set jd=JSON_INSERT(`jd`, _utf8mb4'$.current', CAST(_utf8mb4'true' as JSON)), bd=_binary'newest blob data' where id=12", + }, + data: [][]string{ + {"1", "{\"key1\": \"val1\", \"color\": \"red\", \"years\": 5, \"hobbies\": [\"skiing\", \"video games\", \"hiking\"]}", "blob data"}, + {"3", "{\"key3\": \"val3\"}", "blob data3"}, + {"12", "{\"key2\": \"val2\", \"misc\": \"{\\\"address\\\":\\\"1012 S Park\\\", \\\"town\\\":\\\"Hastings\\\", \\\"state\\\":\\\"MI\\\"}\", \"current\": true}", "newest blob data"}, + }, }, - }, - }...) - if runNoBlobTest { - testCases = append(testCases, testCase{ - input: `update src set id = id+10 where id = 3`, - error: "binary log event missing a needed value for dst.bd due to not using binlog-row-image=FULL", - }) - } else { - testCases = append(testCases, testCase{ - input: `update src set id = id+10 where id = 3`, - output: []string{ - "delete from dst where id=3", - "insert into dst(id,jd,bd) values (13,JSON_OBJECT(_utf8mb4'key3', _utf8mb4'val3'),_binary'blob data3')", + { + input: `update src set jd=JSON_SET(jd, '$.idontknow', null, '$.idontknoweither', 'null') where id = 3`, + output: []string{ + "update dst set jd=JSON_INSERT(JSON_INSERT(`jd`, _utf8mb4'$.idontknow', CAST(_utf8mb4'null' as JSON)), _utf8mb4'$.idontknoweither', CAST(JSON_QUOTE(_utf8mb4'null') as JSON)), bd=_binary'blob data3' where id=3", + }, + data: [][]string{ + {"1", "{\"key1\": \"val1\", \"color\": \"red\", \"years\": 5, \"hobbies\": [\"skiing\", \"video games\", \"hiking\"]}", "blob data"}, + {"3", "{\"key3\": \"val3\", \"idontknow\": null, \"idontknoweither\": \"null\"}", "blob data3"}, + {"12", "{\"key2\": \"val2\", \"misc\": \"{\\\"address\\\":\\\"1012 S Park\\\", \\\"town\\\":\\\"Hastings\\\", \\\"state\\\":\\\"MI\\\"}\", \"current\": true}", "newest blob data"}, + }, }, - data: [][]string{ - {"1", "{\"key1\": \"val1\", \"color\": \"red\", \"years\": 5, \"hobbies\": [\"skiing\", \"video games\", \"hiking\"]}", "blob data"}, - {"12", "{\"key2\": \"val2\", \"misc\": \"{\\\"address\\\":\\\"1012 S Park\\\", \\\"town\\\":\\\"Hastings\\\", \\\"state\\\":\\\"MI\\\"}\", \"current\": true}", "newest blob data"}, - {"13", "{\"key3\": \"val3\", \"idontknow\": null}", "blob data3"}, + { + input: `update src set id = id+10 where id = 3`, + output: []string{ + "delete from dst where id=3", + "insert into dst(id,jd,bd) values (13,JSON_OBJECT(_utf8mb4'idontknow', null, _utf8mb4'idontknoweither', _utf8mb4'null', _utf8mb4'key3', _utf8mb4'val3'),_binary'blob data3')", + }, + data: [][]string{ + {"1", "{\"key1\": \"val1\", \"color\": \"red\", \"years\": 5, \"hobbies\": [\"skiing\", \"video games\", \"hiking\"]}", "blob data"}, + {"12", "{\"key2\": \"val2\", \"misc\": \"{\\\"address\\\":\\\"1012 S Park\\\", \\\"town\\\":\\\"Hastings\\\", \\\"state\\\":\\\"MI\\\"}\", \"current\": true}", "newest blob data"}, + {"13", "{\"key3\": \"val3\", \"idontknow\": null, \"idontknoweither\": \"null\"}", "blob data3"}, + }, }, - }) + }...) } for _, tc := range testCases {