test/endtoend: add detected problems endtoend test

Signed-off-by: Amir Abushareb <[email protected]>

go/test/endtoend/vtorc/general/vtorc_test.go

@@ -28,6 +28,7 @@ import (
 	"vitess.io/vitess/go/test/endtoend/cluster"
 	"vitess.io/vitess/go/test/endtoend/vtorc/utils"
 	"vitess.io/vitess/go/vt/log"
+	"vitess.io/vitess/go/vt/vtorc/inst"
 	"vitess.io/vitess/go/vt/vtorc/logic"
 )
 
@@ -102,6 +103,7 @@ func TestKeyspaceShard(t *testing.T) {
 // 3. stop replication, let vtorc repair
 // 4. setup replication from non-primary, let vtorc repair
 // 5. make instance A replicates from B and B from A, wait for repair
+// 6. disable recoveries and make sure the detected problems are set correctly.
 func TestVTOrcRepairs(t *testing.T) {
 	defer utils.PrintVTOrcLogsOnFailure(t, clusterInfo.ClusterInstance)
 	defer cluster.PanicHandler(t)
@@ -224,6 +226,39 @@ func TestVTOrcRepairs(t *testing.T) {
 		utils.WaitForTabletType(t, replica, "drained")
 	})
 
+	t.Run("Sets DetectedProblems metric correctly", func(t *testing.T) {
+		// Since we're using a boolean metric here, disable recoveries for now.
+		status, _, err := utils.MakeAPICall(t, vtOrcProcess, "/api/disable-global-recoveries")
+		require.NoError(t, err)
+		require.Equal(t, 200, status)
+
+		// Make the current primary database read-only.
+		_, err = utils.RunSQL(t, "set global read_only=ON", curPrimary, "")
+		require.NoError(t, err)
+
+		// Wait for problems to be set.
+		utils.WaitForDetectedProblems(t, vtOrcProcess,
+			string(inst.PrimaryIsReadOnly),
+			curPrimary.Alias,
+			keyspace.Name,
+			shard0.Name,
+			1,
+		)
+
+		// Enable recoveries.
+		status, _, err = utils.MakeAPICall(t, vtOrcProcess, "/api/enable-global-recoveries")
+		require.NoError(t, err)
+		assert.Equal(t, 200, status)
+
+		// wait for detected problem to be cleared.
+		utils.WaitForDetectedProblems(t, vtOrcProcess,
+			string(inst.PrimaryIsReadOnly),
+			curPrimary.Alias,
+			keyspace.Name,
+			shard0.Name,
+			0,
+		)
+	})
 }
 
 func TestRepairAfterTER(t *testing.T) {

go/test/endtoend/vtorc/utils/utils.go

@@ -969,6 +969,39 @@ func WaitForSuccessfulRecoveryCount(t *testing.T, vtorcInstance *cluster.VTOrcPr
 	assert.EqualValues(t, countExpected, successCount)
 }
 
+// WaitForDetectedProblems waits until the given analysis code, alias, keyspace and shard count matches the count expected.
+func WaitForDetectedProblems(t *testing.T, vtorcInstance *cluster.VTOrcProcess, code, alias, ks, shard string, expect int) {
+	t.Helper()
+	key := strings.Join([]string{code, alias, ks, shard}, ".")
+	timeout := 15 * time.Second
+	startTime := time.Now()
+
+	for time.Since(startTime) < timeout {
+		vars := vtorcInstance.GetVars()
+		problems := vars["DetectedProblems"].(map[string]interface{})
+		actual := problems[key]
+		if actual == expect {
+			return
+		}
+		time.Sleep(time.Second)
+	}
+
+	vars := vtorcInstance.GetVars()
+	problems := vars["DetectedProblems"].(map[string]interface{})
+	actual, ok := problems[key]
+
+	assert.True(t, ok,
+		"The metric DetectedProblems[%s] should exist but does not (all problems: %+v)",
+		key, problems,
+	)
+
+	assert.EqualValues(t, expect, actual,
+		"The metric DetectedProblems[%s] should be %v but is %v (all problems: %+v)",
+		key, expect, actual,
+		problems,
+	)
+}
+
 // WaitForTabletType waits for the tablet to reach a certain type.
 func WaitForTabletType(t *testing.T, tablet *cluster.Vttablet, expectedTabletType string) {
 	t.Helper()