internal/controller: track reconciliation failure count

api/v1alpha1/atlasmigration_types.go

@@ -58,6 +58,8 @@ type (
 		ObservedHash string `json:"observed_hash"`
 		// LastApplied is the unix timestamp of the most recent successful versioned migration.
 		LastApplied int64 `json:"lastApplied"`
+		// Failed is the number of times the migration has failed.
+		Failed int `json:"failed"`
 	}
 	// AtlasMigrationSpec defines the desired state of AtlasMigration
 	AtlasMigrationSpec struct {
@@ -157,6 +159,7 @@ func (m *AtlasMigration) IsHashModified(hash string) bool {
 
 // SetReady sets the ready condition to true.
 func (m *AtlasMigration) SetReady(status AtlasMigrationStatus) {
+	status.Failed = 0
 	m.Status = status
 	meta.SetStatusCondition(&m.Status.Conditions, metav1.Condition{
 		Type:   readyCond,
@@ -167,6 +170,7 @@ func (m *AtlasMigration) SetReady(status AtlasMigrationStatus) {
 
 // SetNotReady sets the ready condition to false.
 func (m *AtlasMigration) SetNotReady(reason, message string) {
+	m.Status.Failed++
 	meta.SetStatusCondition(&m.Status.Conditions, metav1.Condition{
 		Type:    readyCond,
 		Status:  metav1.ConditionFalse,

api/v1alpha1/atlasschema_types.go

@@ -68,6 +68,8 @@ type (
 		// PlanLink is the link to the schema plan on the Atlas Cloud.
 		// +optional
 		PlanLink string `json:"planLink"`
+		// Failed is the number of times the schema has failed to apply.
+		Failed int `json:"failed"`
 	}
 	// AtlasSchemaSpec defines the desired state of AtlasSchema
 	AtlasSchemaSpec struct {
@@ -211,6 +213,7 @@ func (sc *AtlasSchema) SetReady(status AtlasSchemaStatus, report any) {
 	} else {
 		msg = "The schema has been applied successfully."
 	}
+	status.Failed = 0
 	sc.Status = status
 	meta.SetStatusCondition(&sc.Status.Conditions, metav1.Condition{
 		Type:    readyCond,
@@ -223,6 +226,7 @@ func (sc *AtlasSchema) SetReady(status AtlasSchemaStatus, report any) {
 // SetNotReady sets the Ready condition to false
 // with the given reason and message.
 func (sc *AtlasSchema) SetNotReady(reason, msg string) {
+	sc.Status.Failed++
 	meta.SetStatusCondition(&sc.Status.Conditions, metav1.Condition{
 		Type:    readyCond,
 		Status:  metav1.ConditionFalse,

charts/atlas-operator/templates/crds/crd.yaml

@@ -498,6 +498,9 @@ spec:
                   - type
                   type: object
                 type: array
+              failed:
+                description: Failed is the number of times the migration has failed.
+                type: integer
               lastApplied:
                 description: LastApplied is the unix timestamp of the most recent
                   successful versioned migration.
@@ -516,6 +519,7 @@ spec:
                   versioned migration.
                 type: string
             required:
+            - failed
             - lastApplied
             - observed_hash
             type: object
@@ -1079,6 +1083,10 @@ spec:
                   - type
                   type: object
                 type: array
+              failed:
+                description: Failed is the number of times the schema has failed to
+                  apply.
+                type: integer
               last_applied:
                 description: LastApplied is the unix timestamp of the most recent
                   successful schema apply operation.
@@ -1096,6 +1104,7 @@ spec:
                 description: PlanURL is the URL of the schema plan to apply.
                 type: string
             required:
+            - failed
             - last_applied
             - observed_hash
             type: object

config/crd/bases/db.atlasgo.io_atlasmigrations.yaml

@@ -512,6 +512,9 @@ spec:
                   - type
                   type: object
                 type: array
+              failed:
+                description: Failed is the number of times the migration has failed.
+                type: integer
               lastApplied:
                 description: LastApplied is the unix timestamp of the most recent
                   successful versioned migration.
@@ -530,6 +533,7 @@ spec:
                   versioned migration.
                 type: string
             required:
+            - failed
             - lastApplied
             - observed_hash
             type: object

config/crd/bases/db.atlasgo.io_atlasschemas.yaml

@@ -566,6 +566,10 @@ spec:
                   - type
                   type: object
                 type: array
+              failed:
+                description: Failed is the number of times the schema has failed to
+                  apply.
+                type: integer
               last_applied:
                 description: LastApplied is the unix timestamp of the most recent
                   successful schema apply operation.
@@ -583,6 +587,7 @@ spec:
                 description: PlanURL is the URL of the schema plan to apply.
                 type: string
             required:
+            - failed
             - last_applied
             - observed_hash
             type: object

internal/controller/atlasmigration_controller_test.go

@@ -1182,6 +1182,25 @@ env "kubernetes" {
 `, fileContent.String())
 }
 
+func TestMigration_RetriedCount(t *testing.T) {
+	tt := migrationCliTest(t)
+	tt.initDefaultAtlasMigration()
+	// First successful reconcile should reset failed count to 0
+	tt.r.Reconcile(context.Background(), migrationReq())
+	require.EqualValues(t, 0, tt.status().Failed)
+	// Second reconcile with bad SQL should increment failed count to 1
+	tt.addMigrationScript("20230412003627_bad_sql.sql", "BAD SQL")
+	tt.r.Reconcile(context.Background(), migrationReq())
+	require.EqualValues(t, 1, tt.status().Failed)
+	// Third reconcile with bad SQL should increment failed count to 2
+	tt.r.Reconcile(context.Background(), migrationReq())
+	require.EqualValues(t, 2, tt.status().Failed)
+	// Fifth reconcile with successful migration should reset failed count to 0
+	tt.removeMigrationScript("20230412003627_bad_sql.sql")
+	tt.r.Reconcile(context.Background(), migrationReq())
+	require.EqualValues(t, 0, tt.status().Failed)
+}
+
 func migrationObjmeta() metav1.ObjectMeta {
 	return metav1.ObjectMeta{
 		Name:      "atlas-migration",
@@ -1270,6 +1289,26 @@ func (t *migrationTest) addMigrationScript(name, content string) {
 	t.k8s.put(&cm)
 }
 
+func (t *migrationTest) removeMigrationScript(name string) {
+	// Get the current configmap
+	cm := corev1.ConfigMap{}
+	err := t.k8s.Get(context.Background(), types.NamespacedName{
+		Name:      "my-configmap",
+		Namespace: "default",
+	}, &cm)
+	require.NoError(t, err)
+
+	// Update the configmap
+	delete(cm.Data, name)
+
+	sum, err := must(memDir(cm.Data)).Checksum()
+	require.NoError(t, err)
+	atlasSum, err := sum.MarshalText()
+	require.NoError(t, err)
+	cm.Data[migrate.HashFileName] = string(atlasSum)
+	t.k8s.put(&cm)
+}
+
 func (t *migrationTest) initDefaultAtlasMigration() {
 	t.initDefaultMigrationDir()
 	t.initDefaultTokenSecret()

internal/controller/atlasschema_controller_test.go

@@ -166,6 +166,40 @@ func TestReconcile_Reconcile(t *testing.T) {
 	}, h.events())
 }
 
+func TestReconcile_RetriedCount(t *testing.T) {
+	tt := newTest(t)
+	tt.k8s.put(&dbv1alpha1.AtlasSchema{
+		ObjectMeta: objmeta(),
+		Spec:       dbv1alpha1.AtlasSchemaSpec{},
+	})
+	// First failed reconcile should increment the failed count to 1
+	tt.r.Reconcile(context.Background(), req())
+	require.EqualValues(t, 1, tt.state().Status.Failed)
+	// Second failed reconcile should increment the failed count to 2
+	tt.r.Reconcile(context.Background(), req())
+	require.EqualValues(t, 2, tt.state().Status.Failed)
+	tt.k8s.put(&dbv1alpha1.AtlasSchema{
+		ObjectMeta: objmeta(),
+		Spec: dbv1alpha1.AtlasSchemaSpec{
+			TargetSpec: dbv1alpha1.TargetSpec{URL: "sqlite://file2/?mode=memory"},
+			Schema:     dbv1alpha1.Schema{SQL: "CREATE TABLE foo(id INT PRIMARY KEY);"},
+		},
+		Status: dbv1alpha1.AtlasSchemaStatus{
+			Failed: 1,
+			Conditions: []metav1.Condition{
+				{
+					Type:   schemaReadyCond,
+					Status: metav1.ConditionFalse,
+				},
+			},
+		},
+	})
+	// Third successful reconcile should reset the failed count to 0
+	_, err := tt.r.Reconcile(context.Background(), req())
+	require.NoError(t, err)
+	require.EqualValues(t, 0, tt.state().Status.Failed)
+}
+
 func TestExtractData_CustomDevURL(t *testing.T) {
 	sc := conditionReconciling()
 	sc.Spec.DevURL = "mysql://dev"
@@ -795,6 +829,10 @@ func (t *test) cond() metav1.Condition {
 	return s.Status.Conditions[0]
 }
 
+func (t *test) state() *dbv1alpha1.AtlasSchema {
+	return t.k8s.state[req().NamespacedName].(*dbv1alpha1.AtlasSchema)
+}
+
 func (t *test) initDB(statement string) {
 	wd, err := atlasexec.NewWorkingDir()
 	require.NoError(t, err)