From 8f63ad58aed25fb075ffb97e4f10f1f930a97fc2 Mon Sep 17 00:00:00 2001 From: Tim Vaillancourt Date: Fri, 16 Feb 2024 23:21:43 +0100 Subject: [PATCH] Redact bind var value only Signed-off-by: Tim Vaillancourt --- go/streamlog/bind_variable.go | 8 ++++++-- go/vt/vtgate/logstats/logstats.go | 4 +--- go/vt/vtgate/logstats/logstats_test.go | 2 +- go/vt/vttablet/tabletserver/tabletenv/logstats.go | 15 ++++++--------- .../tabletserver/tabletenv/logstats_test.go | 2 +- 5 files changed, 15 insertions(+), 16 deletions(-) diff --git a/go/streamlog/bind_variable.go b/go/streamlog/bind_variable.go index c042bdf0569..4cb471f4724 100644 --- a/go/streamlog/bind_variable.go +++ b/go/streamlog/bind_variable.go @@ -74,10 +74,14 @@ func (bv *BindVariable) UnmarshalJSON(b []byte) error { // {"Type":"VARBINARY","Value":"FmtAtEq6S9Y="} func (bv BindVariable) MarshalJSON() ([]byte, error) { // convert querypb.Type integer to string and pass along Value. - return json.Marshal(map[string]interface{}{ + out := map[string]interface{}{ "Type": bv.Type.String(), "Value": bv.Value, - }) + } + if GetRedactDebugUIQueries() { + out["Value"] = nil + } + return json.Marshal(out) } // BindVariablesToProto converts a string-map of BindVariable to a string-map of *querypb.BindVariable. diff --git a/go/vt/vtgate/logstats/logstats.go b/go/vt/vtgate/logstats/logstats.go index 8da1ba42faf..3c260a88182 100644 --- a/go/vt/vtgate/logstats/logstats.go +++ b/go/vt/vtgate/logstats/logstats.go @@ -144,15 +144,13 @@ func (stats *LogStats) Logf(w io.Writer, params url.Values) error { }() formattedBindVars := "\"[REDACTED]\"" - if !streamlog.GetRedactDebugUIQueries() { + if !streamlog.GetRedactDebugUIQueries() && !streamlog.UseQueryLogJSONV2() { _, fullBindParams := params["full"] formattedBindVars = sqltypes.FormatBindVariables( streamlog.BindVariablesToProto(stats.BindVariables), fullBindParams, streamlog.GetQueryLogFormat() == streamlog.QueryLogFormatJSON, ) - } else { - stats.BindVariables = nil } // TODO: remove username here we fully enforce immediate caller id diff --git a/go/vt/vtgate/logstats/logstats_test.go b/go/vt/vtgate/logstats/logstats_test.go index 7bf3a57ea90..06389e3ee15 100644 --- a/go/vt/vtgate/logstats/logstats_test.go +++ b/go/vt/vtgate/logstats/logstats_test.go @@ -189,7 +189,7 @@ func TestLogStatsFormatJSONV2(t *testing.T) { streamlog.SetRedactDebugUIQueries(true) var buf bytes.Buffer assert.Nil(t, logStats.Logf(&buf, nil)) - assert.Equal(t, `{"RemoteAddr":"","Username":"","ImmediateCaller":"","EffectiveCaller":"","Method":"test","TabletType":"PRIMARY","StmtType":"select","SQL":"select * from testtable where name = :strVal and message = :bytesVal","StartTime":"2017-01-01T01:02:03Z","EndTime":"2017-01-01T01:02:04.000001234Z","ShardQueries":0,"RowsAffected":0,"RowsReturned":0,"PlanTime":0,"ExecuteTime":0,"CommitTime":0,"TablesUsed":["ks1.tbl1","ks2.tbl2"],"SessionUUID":"suuid","CachedPlan":false,"ActiveKeyspace":"db"}`, strings.TrimSpace(buf.String())) + assert.Equal(t, `{"RemoteAddr":"","Username":"","ImmediateCaller":"","EffectiveCaller":"","Method":"test","TabletType":"PRIMARY","StmtType":"select","SQL":"select * from testtable where name = :strVal and message = :bytesVal","BindVariables":{"bytesVal":{"Type":"VARBINARY","Value":null},"strVal":{"Type":"VARCHAR","Value":null}},"StartTime":"2017-01-01T01:02:03Z","EndTime":"2017-01-01T01:02:04.000001234Z","ShardQueries":0,"RowsAffected":0,"RowsReturned":0,"PlanTime":0,"ExecuteTime":0,"CommitTime":0,"TablesUsed":["ks1.tbl1","ks2.tbl2"],"SessionUUID":"suuid","CachedPlan":false,"ActiveKeyspace":"db"}`, strings.TrimSpace(buf.String())) assert.Nil(t, json.Unmarshal(buf.Bytes(), &cmpStats)) } } diff --git a/go/vt/vttablet/tabletserver/tabletenv/logstats.go b/go/vt/vttablet/tabletserver/tabletenv/logstats.go index 22f3aafef39..1d174bba52d 100644 --- a/go/vt/vttablet/tabletserver/tabletenv/logstats.go +++ b/go/vt/vttablet/tabletserver/tabletenv/logstats.go @@ -123,6 +123,9 @@ func (stats *LogStats) TotalTime() time.Duration { // RewrittenSQL returns a semicolon separated list of SQL statements // that were executed. func (stats *LogStats) RewrittenSQL() string { + if streamlog.GetRedactDebugUIQueries() { + return "[REDACTED]" + } return strings.Join(stats.rewrittenSqls, "; ") } @@ -193,20 +196,14 @@ func (stats *LogStats) Logf(w io.Writer, params url.Values) error { return nil } - rewrittenSQL := "[REDACTED]" formattedBindVars := "\"[REDACTED]\"" - - if !streamlog.GetRedactDebugUIQueries() { - rewrittenSQL = stats.RewrittenSQL() - + if !streamlog.GetRedactDebugUIQueries() && !streamlog.UseQueryLogJSONV2() { _, fullBindParams := params["full"] formattedBindVars = sqltypes.FormatBindVariables( streamlog.BindVariablesToProto(stats.BindVariables), fullBindParams, streamlog.GetQueryLogFormat() == streamlog.QueryLogFormatJSON, ) - } else { - stats.BindVariables = nil } // TODO: remove username here we fully enforce immediate caller id @@ -226,7 +223,7 @@ func (stats *LogStats) Logf(w io.Writer, params url.Values) error { LogStats: *stats, Username: username, CallInfo: callInfo, - RewrittenSQL: rewrittenSQL, + RewrittenSQL: stats.RewrittenSQL(), ResponseSize: stats.SizeOfResponse(), TotalTime: stats.TotalTime(), }) @@ -249,7 +246,7 @@ func (stats *LogStats) Logf(w io.Writer, params url.Values) error { stats.OriginalSQL, formattedBindVars, stats.NumberOfQueries, - rewrittenSQL, + stats.RewrittenSQL(), stats.FmtQuerySources(), stats.MysqlResponseTime.Seconds(), stats.WaitingForConnection.Seconds(), diff --git a/go/vt/vttablet/tabletserver/tabletenv/logstats_test.go b/go/vt/vttablet/tabletserver/tabletenv/logstats_test.go index 1997a96068a..ad793f29026 100644 --- a/go/vt/vttablet/tabletserver/tabletenv/logstats_test.go +++ b/go/vt/vttablet/tabletserver/tabletenv/logstats_test.go @@ -231,7 +231,7 @@ func TestLogStatsFormatJSONV2(t *testing.T) { streamlog.SetRedactDebugUIQueries(true) var buf bytes.Buffer assert.Nil(t, logStats.Logf(&buf, nil)) - assert.Equal(t, `{"CallInfo":"","Username":"","ImmediateCaller":"","EffectiveCaller":"","RewrittenSQL":"[REDACTED]","TotalTime":1000001234,"ResponseSize":1,"Method":"test","PlanType":"","OriginalSQL":"sql","RowsAffected":0,"NumberOfQueries":1,"StartTime":"2017-01-01T01:02:03Z","EndTime":"2017-01-01T01:02:04.000001234Z","MysqlResponseTime":0,"WaitingForConnection":0,"QuerySources":2,"TransactionID":12345,"ReservedID":0,"CachedPlan":false}`, strings.TrimSpace(buf.String())) + assert.Equal(t, `{"CallInfo":"","Username":"","ImmediateCaller":"","EffectiveCaller":"","RewrittenSQL":"[REDACTED]","TotalTime":1000001234,"ResponseSize":1,"Method":"test","PlanType":"","OriginalSQL":"sql","BindVariables":{"bytesVal":{"Type":"VARBINARY","Value":null},"intVal":{"Type":"INT64","Value":null}},"RowsAffected":0,"NumberOfQueries":1,"StartTime":"2017-01-01T01:02:03Z","EndTime":"2017-01-01T01:02:04.000001234Z","MysqlResponseTime":0,"WaitingForConnection":0,"QuerySources":2,"TransactionID":12345,"ReservedID":0,"CachedPlan":false}`, strings.TrimSpace(buf.String())) assert.Nil(t, json.Unmarshal(buf.Bytes(), &cmpStats)) } }