Fix version checker job update errors with field is immutable

.bazelrc

@@ -1,2 +1,2 @@
 build --workspace_status_command hack/build/print-workspace-status.sh
-test --test_output=errors --test_timeout=-1,-1,-1,2400
+test --test_output=errors --test_timeout=-1,-1,-1,3600

hack/bin/BUILD.bazel

@@ -42,6 +42,7 @@ genrule(
     name = "fetch_preflight",
     srcs = select({
         ":k8": ["@preflight_linux//file"],
+        ":m1": ["@preflight_linux//file"],
     }),
     outs = ["preflight"],
     cmd = "cp $(SRCS) $@",

pkg/actor/validate_version.go

@@ -86,7 +86,7 @@ func (v *versionChecker) Act(ctx context.Context, cluster *resource.Cluster, log
 
 	log.V(DEBUGLEVEL).Info("starting to check the crdb version of the container provided")
 
-	r := resource.NewManagedKubeResource(ctx, v.client, cluster, kube.AnnotatingPersister)
+	r := resource.NewManagedKubeResource(ctx, v.client, cluster, kube.RecreatingPersister)
 	owner := cluster.Unwrap()
 
 	// If the image.name is set use that value and do not check that the

pkg/kube/BUILD.bazel

@@ -11,6 +11,7 @@ go_library(
     importpath = "github.com/cockroachdb/cockroach-operator/pkg/kube",
     visibility = ["//visibility:public"],
     deps = [
+        "//pkg/ptr:go_default_library",
         "@com_github_banzaicloud_k8s_objectmatcher//patch:go_default_library",
         "@com_github_cenkalti_backoff//:go_default_library",
         "@com_github_cockroachdb_errors//:go_default_library",

pkg/kube/helpers.go

@@ -26,6 +26,7 @@ import (
 
 	"github.com/banzaicloud/k8s-objectmatcher/patch"
 	"github.com/cenkalti/backoff"
+	"github.com/cockroachdb/cockroach-operator/pkg/ptr"
 	"github.com/cockroachdb/errors"
 	"github.com/go-logr/logr"
 	"go.uber.org/zap/zapcore"
@@ -140,6 +141,12 @@ var AnnotatingPersister PersistFn = func(ctx context.Context, cl client.Client,
 	})
 }
 
+var RecreatingPersister PersistFn = func(ctx context.Context, cl client.Client, obj client.Object, f MutateFn) (upserted bool, err error) {
+	return ReCreateAnnotated(ctx, cl, obj, func() error {
+		return f()
+	})
+}
+
 // MutateFn is a function which mutates the existing object into it's desired state.
 type MutateFn func() error
 
@@ -190,6 +197,49 @@ func CreateOrUpdateAnnotated(ctx context.Context, c client.Client, obj client.Ob
 	return true, nil
 }
 
+func ReCreateAnnotated(ctx context.Context, c client.Client, obj client.Object, f MutateFn) (upserted bool, err error) {
+	key := client.ObjectKeyFromObject(obj)
+
+	if err := c.Get(ctx, key, obj); err == nil {
+		existing := obj.DeepCopyObject()
+		if err := mutate(f, key, obj); err != nil {
+			return false, err
+		}
+
+		changed, err := ObjectChanged(existing, obj)
+		if err != nil {
+			return false, err
+		}
+		if !changed {
+			return false, nil
+		}
+
+		dp := metav1.DeletePropagationForeground
+		if err := c.Delete(ctx, existing.(client.Object), &client.DeleteOptions{GracePeriodSeconds: ptr.Int64(0),
+			PropagationPolicy: &dp}); err != nil {
+			return false, err
+		}
+
+		return false, nil
+	} else if err != nil && !apierrors.IsNotFound(err) {
+		return false, err
+	}
+
+	if err := mutate(f, key, obj); err != nil {
+		return false, err
+	}
+
+	if err := annotator.SetLastAppliedAnnotation(obj); err != nil {
+		return false, err
+	}
+
+	if err := c.Create(ctx, obj); err != nil {
+		return false, err
+	}
+
+	return true, nil
+}
+
 func ObjectChanged(current, updated runtime.Object) (bool, error) {
 	opts := []patch.CalculateOption{
 		patch.IgnoreStatusFields(),