EVEREST-458 Restart everest-operator pod during the provisioning

cli-tests/tests/flow/pg-operator.spec.ts

@@ -12,7 +12,7 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
-import { test } from '@fixtures';
+import { test, expect } from '@fixtures';
 // eslint-disable-next-line import/no-extraneous-dependencies
 import { faker } from '@faker-js/faker';
 import { apiVerifyClusterExists } from '@support/backend';
@@ -62,10 +62,17 @@ test.describe('Everest CLI install operators', async () => {
     await apiVerifyClusterExists(request, clusterName);
 
     await test.step('re-run everest install operators command', async () => {
+      await page.waitForTimeout(60_000);
+      const operator = await cli.exec(`kubectl -n percona-everest get po | grep everest|awk {'print $1'}`);
+      await operator.assertSuccess();
+
       const out = await cli.everestExecSkipWizard(
         `install operators --operator.mongodb=false --operator.postgresql=true --operator.xtradb-cluster=true --backup.enable=0 --monitoring.enable=0 --name=${clusterName}`,
       );
+      const restartedOperator = await cli.exec(`kubectl -n percona-everest get po | grep everest|awk {'print $1'}`);
+      await restartedOperator.assertSuccess();
 
+      expect(operator.getStdOutLines()[0]).not.toEqual(restartedOperator.getStdOutLines()[0]);
       await out.assertSuccess();
       await out.outErrContainsNormalizedMany([
         'percona-xtradb-cluster-operator operator has been installed',

pkg/install/operators.go

@@ -735,7 +735,10 @@ func (o *Operators) provisionOperators(ctx context.Context) error {
 		return err
 	}
 
-	return o.installOperator(ctx, o.config.Channel.Everest, everestOperatorName)()
+	if err := o.installOperator(ctx, o.config.Channel.Everest, everestOperatorName)(); err != nil {
+		return err
+	}
+	return o.restartEverestOperatorPod(ctx)
 }
 
 func (o *Operators) installOperator(ctx context.Context, channel, operatorName string) func() error {
@@ -1002,3 +1005,7 @@ func (o *Operators) getServiceAccountKubeConfig(ctx context.Context) (string, er
 
 	return o.kubeClient.GenerateKubeConfigWithToken(everestServiceAccount, secret)
 }
+
+func (o *Operators) restartEverestOperatorPod(ctx context.Context) error {
+	return o.kubeClient.RestartEverestOperator(ctx, o.config.Namespace)
+}

pkg/kubernetes/client/client.go

@@ -490,6 +490,16 @@ func (c *Client) GetPods(
 	return c.clientset.CoreV1().Pods(namespace).List(ctx, options)
 }
 
+// ListPods lists pods.
+func (c *Client) ListPods(ctx context.Context, namespace string, options metav1.ListOptions) (*corev1.PodList, error) {
+	return c.clientset.CoreV1().Pods(namespace).List(ctx, options)
+}
+
+// DeletePod deletes a pod by given name in the given namespace.
+func (c *Client) DeletePod(ctx context.Context, namespace, name string) error {
+	return c.clientset.CoreV1().Pods(namespace).Delete(ctx, name, metav1.DeleteOptions{})
+}
+
 // GetNodes returns list of nodes.
 func (c *Client) GetNodes(ctx context.Context) (*corev1.NodeList, error) {
 	return c.clientset.CoreV1().Nodes().List(ctx, metav1.ListOptions{})

pkg/kubernetes/client/monitoring.go

@@ -23,8 +23,10 @@ import (
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	rbacv1 "k8s.io/api/rbac/v1"
+	k8serrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/discovery"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
@@ -39,6 +41,17 @@ func (c *Client) DeleteAllMonitoringResources(ctx context.Context, namespace str
 	if namespace == "" {
 		namespace = c.namespace
 	}
+	crd := &unstructured.Unstructured{}
+	crd.SetGroupVersionKind(schema.GroupVersionKind{
+		Group:   "apiextensions.k8s.io",
+		Kind:    "CustomResourceDefinition",
+		Version: "v1",
+	})
+	err = cl.Get(ctx, types.NamespacedName{Name: "vmagents.operator.victoriametrics.com"}, crd)
+	if err != nil && k8serrors.IsNotFound(err) {
+		// Nothing to do here. Monitoring does not exists in the cluster.
+		return nil
+	}
 
 	opts := []client.DeleteAllOfOption{
 		client.MatchingLabels{"everest.percona.com/type": "monitoring"},

pkg/kubernetes/kubernetes.go

@@ -89,8 +89,11 @@ const (
 	pollDuration = 300 * time.Second
 )
 
-// ErrEmptyVersionTag Got an empty version tag from GitHub API.
-var ErrEmptyVersionTag error = errors.New("got an empty version tag from Github")
+var (
+	// ErrEmptyVersionTag Got an empty version tag from GitHub API.
+	ErrEmptyVersionTag       error = errors.New("got an empty version tag from Github")
+	errNoEverestOperatorPods       = errors.New("no instances of everest-operator are running")
+)
 
 // Kubernetes is a client for Kubernetes.
 type Kubernetes struct {
@@ -832,3 +835,59 @@ func (k *Kubernetes) updateClusterRoleBindingNamespace(o map[string]interface{},
 
 	return nil
 }
+
+// RestartEverestOperator restarts everest pod.
+func (k *Kubernetes) RestartEverestOperator(ctx context.Context, namespace string) error {
+	var pod corev1.Pod
+	err := wait.PollUntilContextTimeout(ctx, 5*time.Second, time.Minute, true, func(ctx context.Context) (bool, error) {
+		p, err := k.getEverestOperatorPod(ctx, namespace)
+		if err != nil {
+			if errors.Is(err, errNoEverestOperatorPods) {
+				return false, nil
+			}
+			return false, err
+		}
+		pod = p
+		return true, nil
+	})
+	if err != nil {
+		return err
+	}
+	podUID := pod.UID
+	err = k.client.DeletePod(ctx, namespace, pod.Name)
+	if err != nil {
+		return err
+	}
+
+	return wait.PollUntilContextTimeout(ctx, 5*time.Second, time.Minute, true, func(ctx context.Context) (bool, error) {
+		pod, err := k.getEverestOperatorPod(ctx, namespace)
+		if err != nil {
+			return false, err
+		}
+		if podUID == pod.UID {
+			return false, nil
+		}
+		return pod.Status.Phase == corev1.PodRunning && pod.Status.ContainerStatuses[0].Ready, nil
+	})
+}
+
+func (k *Kubernetes) getEverestOperatorPod(ctx context.Context, namespace string) (corev1.Pod, error) {
+	podList, err := k.client.ListPods(ctx, namespace, metav1.ListOptions{
+		LabelSelector: metav1.FormatLabelSelector(&metav1.LabelSelector{
+			MatchLabels: map[string]string{
+				"app.kubernetes.io/name": "everest-operator",
+			},
+		}),
+		FieldSelector: "status.phase=Running",
+	})
+	if err != nil {
+		return corev1.Pod{}, err
+	}
+	if len(podList.Items) == 0 {
+		return corev1.Pod{}, errNoEverestOperatorPods
+	}
+	if len(podList.Items) > 1 {
+		return corev1.Pod{}, errors.New("multiple instances of everest-operator found")
+	}
+	return podList.Items[0], nil
+}