From d564200b8f8ca8f9cb4fc9c384a232849fd21e4f Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Thu, 14 Mar 2024 19:09:17 +0530 Subject: [PATCH 1/3] feat: add api end point to print the current database state Signed-off-by: Manan Gupta --- go/test/endtoend/vtorc/api/api_test.go | 10 ++++++++++ go/vt/vtorc/db/generate_base.go | 22 ++++++++++++++++++++++ go/vt/vtorc/inst/instance_dao.go | 17 +++++++++++++++++ go/vt/vtorc/inst/instance_dao_test.go | 16 ++++++++++++++++ go/vt/vtorc/server/api.go | 16 +++++++++++++++- 5 files changed, 80 insertions(+), 1 deletion(-) diff --git a/go/test/endtoend/vtorc/api/api_test.go b/go/test/endtoend/vtorc/api/api_test.go index 30e43dfc29a..6bed9ec0b77 100644 --- a/go/test/endtoend/vtorc/api/api_test.go +++ b/go/test/endtoend/vtorc/api/api_test.go @@ -93,6 +93,16 @@ func TestAPIEndpoints(t *testing.T) { return response != "null" }) + t.Run("Database State", func(t *testing.T) { + // Get database state + status, resp, err := utils.MakeAPICall(t, vtorc, "/api/database-state") + require.NoError(t, err) + assert.Equal(t, 200, status) + assert.Contains(t, resp, "alias:{zone1-0000000101 true}") + assert.Contains(t, resp, `Table vitess_keyspace +map[durability_policy:{none true} keyspace:{ks true} keyspace_type:{0 true}]`) + }) + t.Run("Disable Recoveries API", func(t *testing.T) { // Disable recoveries of VTOrc status, resp, err := utils.MakeAPICall(t, vtorc, "/api/disable-global-recoveries") diff --git a/go/vt/vtorc/db/generate_base.go b/go/vt/vtorc/db/generate_base.go index 73238802920..94daebbf7f0 100644 --- a/go/vt/vtorc/db/generate_base.go +++ b/go/vt/vtorc/db/generate_base.go @@ -16,6 +16,28 @@ package db +var TableNames = []string{ + "database_instance", + "audit", + "active_node", + "node_health", + "topology_recovery", + "database_instance_topology_history", + "candidate_database_instance", + "topology_failure_detection", + "blocked_topology_recovery", + "database_instance_last_analysis", + "database_instance_analysis_changelog", + "node_health_history", + "vtorc_db_deployments", + "global_recovery_disable", + "topology_recovery_steps", + "database_instance_stale_binlog_coordinates", + "vitess_tablet", + "vitess_keyspace", + "vitess_shard", +} + // vtorcBackend is a list of SQL statements required to build the vtorc backend var vtorcBackend = []string{ ` diff --git a/go/vt/vtorc/inst/instance_dao.go b/go/vt/vtorc/inst/instance_dao.go index 250d2bd6ba6..1a839f2ad58 100644 --- a/go/vt/vtorc/inst/instance_dao.go +++ b/go/vt/vtorc/inst/instance_dao.go @@ -1210,3 +1210,20 @@ func ExpireStaleInstanceBinlogCoordinates() error { } return ExecDBWriteFunc(writeFunc) } + +// SnapshotDatabaseState takes the snapshot of the database and returns it. +func SnapshotDatabaseState() (string, error) { + var finalOutput string + for _, tableName := range db.TableNames { + finalOutput += "Table " + tableName + "\n" + err := db.QueryVTOrc("select * from "+tableName, nil, func(rowMap sqlutils.RowMap) error { + finalOutput += fmt.Sprintf("%v\n", rowMap) + return nil + }) + if err != nil { + return "", err + } + finalOutput += "\n" + } + return finalOutput, nil +} diff --git a/go/vt/vtorc/inst/instance_dao_test.go b/go/vt/vtorc/inst/instance_dao_test.go index 549389f91fe..8e737b44fba 100644 --- a/go/vt/vtorc/inst/instance_dao_test.go +++ b/go/vt/vtorc/inst/instance_dao_test.go @@ -746,3 +746,19 @@ func waitForCacheInitialization() { time.Sleep(100 * time.Millisecond) } } + +func TestSnapshotDatabaseState(t *testing.T) { + // Clear the database after the test. The easiest way to do that is to run all the initialization commands again. + defer func() { + db.ClearVTOrcDatabase() + }() + + for _, query := range initialSQL { + _, err := db.ExecVTOrc(query) + require.NoError(t, err) + } + + snapshot, err := SnapshotDatabaseState() + require.NoError(t, err) + require.Contains(t, snapshot, `alias:{zone1-0000000112 true}`) +} diff --git a/go/vt/vtorc/server/api.go b/go/vt/vtorc/server/api.go index b0112e10add..9246c4c5ee1 100644 --- a/go/vt/vtorc/server/api.go +++ b/go/vt/vtorc/server/api.go @@ -45,6 +45,7 @@ const ( disableGlobalRecoveriesAPI = "/api/disable-global-recoveries" enableGlobalRecoveriesAPI = "/api/enable-global-recoveries" replicationAnalysisAPI = "/api/replication-analysis" + databaseStateAPI = "/api/database-state" healthAPI = "/debug/health" AggregatedDiscoveryMetricsAPI = "/api/aggregated-discovery-metrics" @@ -60,6 +61,7 @@ var ( disableGlobalRecoveriesAPI, enableGlobalRecoveriesAPI, replicationAnalysisAPI, + databaseStateAPI, healthAPI, AggregatedDiscoveryMetricsAPI, } @@ -86,6 +88,8 @@ func (v *vtorcAPI) ServeHTTP(response http.ResponseWriter, request *http.Request errantGTIDsAPIHandler(response, request) case replicationAnalysisAPI: replicationAnalysisAPIHandler(response, request) + case databaseStateAPI: + databaseStateAPIHandler(response) case AggregatedDiscoveryMetricsAPI: AggregatedDiscoveryMetricsAPIHandler(response, request) default: @@ -104,7 +108,7 @@ func getACLPermissionLevelForAPI(apiEndpoint string) string { return acl.ADMIN case replicationAnalysisAPI: return acl.MONITORING - case healthAPI: + case healthAPI, databaseStateAPI: return acl.MONITORING } return acl.ADMIN @@ -166,6 +170,16 @@ func errantGTIDsAPIHandler(response http.ResponseWriter, request *http.Request) returnAsJSON(response, http.StatusOK, instances) } +// databaseStateAPIHandler is the handler for the databaseStateAPI endpoint +func databaseStateAPIHandler(response http.ResponseWriter) { + snapshot, err := inst.SnapshotDatabaseState() + if err != nil { + http.Error(response, err.Error(), http.StatusInternalServerError) + return + } + writePlainTextResponse(response, snapshot, http.StatusOK) +} + // AggregatedDiscoveryMetricsAPIHandler is the handler for the discovery metrics endpoint func AggregatedDiscoveryMetricsAPIHandler(response http.ResponseWriter, request *http.Request) { // return metrics for last x seconds From a4f81912edcca0386319cb6247ef68d06993ce54 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Thu, 21 Mar 2024 12:16:51 +0530 Subject: [PATCH 2/3] feat: use json output instead of text Signed-off-by: Manan Gupta --- go/test/endtoend/vtorc/api/api_test.go | 5 ++--- go/vt/external/golib/sqlutils/sqlutils.go | 2 +- go/vt/vtorc/inst/instance_dao.go | 27 +++++++++++++++++------ go/vt/vtorc/inst/instance_dao_test.go | 6 ++--- go/vt/vtorc/server/api.go | 4 ++-- 5 files changed, 28 insertions(+), 16 deletions(-) diff --git a/go/test/endtoend/vtorc/api/api_test.go b/go/test/endtoend/vtorc/api/api_test.go index 6bed9ec0b77..3dcf2a26abf 100644 --- a/go/test/endtoend/vtorc/api/api_test.go +++ b/go/test/endtoend/vtorc/api/api_test.go @@ -98,9 +98,8 @@ func TestAPIEndpoints(t *testing.T) { status, resp, err := utils.MakeAPICall(t, vtorc, "/api/database-state") require.NoError(t, err) assert.Equal(t, 200, status) - assert.Contains(t, resp, "alias:{zone1-0000000101 true}") - assert.Contains(t, resp, `Table vitess_keyspace -map[durability_policy:{none true} keyspace:{ks true} keyspace_type:{0 true}]`) + assert.Contains(t, resp, `"alias":"zone1-0000000101"`) + assert.Contains(t, resp, `{"TableName":"vitess_keyspace","Rows":[{"durability_policy":"none","keyspace":"ks","keyspace_type":"0"}]}`) }) t.Run("Disable Recoveries API", func(t *testing.T) { diff --git a/go/vt/external/golib/sqlutils/sqlutils.go b/go/vt/external/golib/sqlutils/sqlutils.go index eb1cb8c8941..df6984b5634 100644 --- a/go/vt/external/golib/sqlutils/sqlutils.go +++ b/go/vt/external/golib/sqlutils/sqlutils.go @@ -40,7 +40,7 @@ type RowMap map[string]CellData // CellData is the result of a single (atomic) column in a single row type CellData sql.NullString -func (this *CellData) MarshalJSON() ([]byte, error) { +func (this CellData) MarshalJSON() ([]byte, error) { if this.Valid { return json.Marshal(this.String) } else { diff --git a/go/vt/vtorc/inst/instance_dao.go b/go/vt/vtorc/inst/instance_dao.go index 1a839f2ad58..7cbddf231d4 100644 --- a/go/vt/vtorc/inst/instance_dao.go +++ b/go/vt/vtorc/inst/instance_dao.go @@ -17,6 +17,7 @@ package inst import ( + "encoding/json" "errors" "fmt" "regexp" @@ -1211,19 +1212,31 @@ func ExpireStaleInstanceBinlogCoordinates() error { return ExecDBWriteFunc(writeFunc) } -// SnapshotDatabaseState takes the snapshot of the database and returns it. -func SnapshotDatabaseState() (string, error) { - var finalOutput string +// GetDatabaseState takes the snapshot of the database and returns it. +func GetDatabaseState() (string, error) { + type tableState struct { + TableName string + Rows []sqlutils.RowMap + } + + var dbState []tableState for _, tableName := range db.TableNames { - finalOutput += "Table " + tableName + "\n" + ts := tableState{ + TableName: tableName, + } err := db.QueryVTOrc("select * from "+tableName, nil, func(rowMap sqlutils.RowMap) error { - finalOutput += fmt.Sprintf("%v\n", rowMap) + ts.Rows = append(ts.Rows, rowMap) return nil }) if err != nil { return "", err } - finalOutput += "\n" + dbState = append(dbState, ts) } - return finalOutput, nil + jsonData, err := json.Marshal(dbState) + if err != nil { + return "", err + } + + return string(jsonData), nil } diff --git a/go/vt/vtorc/inst/instance_dao_test.go b/go/vt/vtorc/inst/instance_dao_test.go index 8e737b44fba..0cae4679093 100644 --- a/go/vt/vtorc/inst/instance_dao_test.go +++ b/go/vt/vtorc/inst/instance_dao_test.go @@ -747,7 +747,7 @@ func waitForCacheInitialization() { } } -func TestSnapshotDatabaseState(t *testing.T) { +func TestGetDatabaseState(t *testing.T) { // Clear the database after the test. The easiest way to do that is to run all the initialization commands again. defer func() { db.ClearVTOrcDatabase() @@ -758,7 +758,7 @@ func TestSnapshotDatabaseState(t *testing.T) { require.NoError(t, err) } - snapshot, err := SnapshotDatabaseState() + ds, err := GetDatabaseState() require.NoError(t, err) - require.Contains(t, snapshot, `alias:{zone1-0000000112 true}`) + require.Contains(t, ds, `"alias":"zone1-0000000112"`) } diff --git a/go/vt/vtorc/server/api.go b/go/vt/vtorc/server/api.go index 9246c4c5ee1..60fdf226e95 100644 --- a/go/vt/vtorc/server/api.go +++ b/go/vt/vtorc/server/api.go @@ -172,12 +172,12 @@ func errantGTIDsAPIHandler(response http.ResponseWriter, request *http.Request) // databaseStateAPIHandler is the handler for the databaseStateAPI endpoint func databaseStateAPIHandler(response http.ResponseWriter) { - snapshot, err := inst.SnapshotDatabaseState() + ds, err := inst.GetDatabaseState() if err != nil { http.Error(response, err.Error(), http.StatusInternalServerError) return } - writePlainTextResponse(response, snapshot, http.StatusOK) + writePlainTextResponse(response, ds, http.StatusOK) } // AggregatedDiscoveryMetricsAPIHandler is the handler for the discovery metrics endpoint From 1716eec50026a49cd570bd7ed0d58334e7c27865 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Thu, 21 Mar 2024 12:23:19 +0530 Subject: [PATCH 3/3] feat: indent the json output to be more readable Signed-off-by: Manan Gupta --- go/test/endtoend/vtorc/api/api_test.go | 13 +++++++++++-- go/vt/vtorc/inst/instance_dao.go | 2 +- go/vt/vtorc/inst/instance_dao_test.go | 2 +- 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/go/test/endtoend/vtorc/api/api_test.go b/go/test/endtoend/vtorc/api/api_test.go index 3dcf2a26abf..05b757a90ac 100644 --- a/go/test/endtoend/vtorc/api/api_test.go +++ b/go/test/endtoend/vtorc/api/api_test.go @@ -98,8 +98,17 @@ func TestAPIEndpoints(t *testing.T) { status, resp, err := utils.MakeAPICall(t, vtorc, "/api/database-state") require.NoError(t, err) assert.Equal(t, 200, status) - assert.Contains(t, resp, `"alias":"zone1-0000000101"`) - assert.Contains(t, resp, `{"TableName":"vitess_keyspace","Rows":[{"durability_policy":"none","keyspace":"ks","keyspace_type":"0"}]}`) + assert.Contains(t, resp, `"alias": "zone1-0000000101"`) + assert.Contains(t, resp, `{ + "TableName": "vitess_keyspace", + "Rows": [ + { + "durability_policy": "none", + "keyspace": "ks", + "keyspace_type": "0" + } + ] + },`) }) t.Run("Disable Recoveries API", func(t *testing.T) { diff --git a/go/vt/vtorc/inst/instance_dao.go b/go/vt/vtorc/inst/instance_dao.go index 7cbddf231d4..35b5d11bc95 100644 --- a/go/vt/vtorc/inst/instance_dao.go +++ b/go/vt/vtorc/inst/instance_dao.go @@ -1233,7 +1233,7 @@ func GetDatabaseState() (string, error) { } dbState = append(dbState, ts) } - jsonData, err := json.Marshal(dbState) + jsonData, err := json.MarshalIndent(dbState, "", "\t") if err != nil { return "", err } diff --git a/go/vt/vtorc/inst/instance_dao_test.go b/go/vt/vtorc/inst/instance_dao_test.go index 0cae4679093..c6020ec52d8 100644 --- a/go/vt/vtorc/inst/instance_dao_test.go +++ b/go/vt/vtorc/inst/instance_dao_test.go @@ -760,5 +760,5 @@ func TestGetDatabaseState(t *testing.T) { ds, err := GetDatabaseState() require.NoError(t, err) - require.Contains(t, ds, `"alias":"zone1-0000000112"`) + require.Contains(t, ds, `"alias": "zone1-0000000112"`) }