From 60d59ae5a8581857922a7adf4fa8200a5f1dc0d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ege=20G=C3=BCne=C5=9F?= Date: Wed, 2 Aug 2023 09:49:10 +0300 Subject: [PATCH] K8SPSMDB-926: Implement PiTR for physical restores --- clientcmd/clientcmd.go | 9 +- pkg/apis/psmdb/v1/psmdb_types.go | 42 +- pkg/controller/perconaservermongodb/backup.go | 2 +- .../perconaservermongodb/balancer.go | 4 +- .../perconaservermongodb/connections.go | 7 +- pkg/controller/perconaservermongodb/fcv.go | 6 +- pkg/controller/perconaservermongodb/mgo.go | 36 +- .../perconaservermongodb/psmdb_controller.go | 2 +- .../perconaservermongodb/secrets.go | 82 +-- pkg/controller/perconaservermongodb/smart.go | 2 +- pkg/controller/perconaservermongodb/users.go | 40 +- .../perconaservermongodb/version.go | 2 +- .../perconaservermongodbrestore/physical.go | 501 +++++++++++++----- 13 files changed, 475 insertions(+), 260 deletions(-) diff --git a/clientcmd/clientcmd.go b/clientcmd/clientcmd.go index 612869b0f0..06bbd99957 100644 --- a/clientcmd/clientcmd.go +++ b/clientcmd/clientcmd.go @@ -1,6 +1,7 @@ package clientcmd import ( + "context" "io" corev1 "k8s.io/api/core/v1" @@ -42,9 +43,9 @@ func NewClient() (*Client, error) { }, nil } -func (c *Client) Exec(pod *corev1.Pod, containerName string, command []string, stdin io.Reader, stdout, stderr io.Writer, tty bool) error { - // Prepare the API URL used to execute another process within the Pod. In - // this case, we'll run a remote shell. +func (c *Client) Exec(ctx context.Context, pod *corev1.Pod, containerName string, command []string, stdin io.Reader, stdout, stderr io.Writer, tty bool) error { + // Prepare the API URL used to execute another process within the Pod. + // In this case, we'll run a remote shell. req := c.client.RESTClient(). Post(). Namespace(pod.Namespace). @@ -66,7 +67,7 @@ func (c *Client) Exec(pod *corev1.Pod, containerName string, command []string, s } // Connect this process' std{in,out,err} to the remote shell process. - return exec.Stream(remotecommand.StreamOptions{ + return exec.StreamWithContext(ctx, remotecommand.StreamOptions{ Stdin: stdin, Stdout: stdout, Stderr: stderr, diff --git a/pkg/apis/psmdb/v1/psmdb_types.go b/pkg/apis/psmdb/v1/psmdb_types.go index c238758334..7b3de03de5 100644 --- a/pkg/apis/psmdb/v1/psmdb_types.go +++ b/pkg/apis/psmdb/v1/psmdb_types.go @@ -284,12 +284,6 @@ func (pmm *PMMSpec) HasSecret(secret *corev1.Secret) bool { return false } -const ( - PMMUserKey = "PMM_SERVER_USER" - PMMPasswordKey = "PMM_SERVER_PASSWORD" - PMMAPIKey = "PMM_SERVER_API_KEY" -) - func (spec *PMMSpec) ShouldUseAPIKeyAuth(secret *corev1.Secret) bool { if _, ok := secret.Data[PMMAPIKey]; !ok { _, okl := secret.Data[PMMUserKey] @@ -524,6 +518,10 @@ type ReplsetSpec struct { NonVoting NonVotingSpec `json:"nonvoting,omitempty"` } +func (r *ReplsetSpec) PodName(cr *PerconaServerMongoDB, idx int) string { + return fmt.Sprintf("%s-%s-%d", cr.Name, r.Name, idx) +} + func (r *ReplsetSpec) ServiceName(cr *PerconaServerMongoDB) string { return cr.Name + "-" + r.Name } @@ -899,6 +897,38 @@ const ( userPostfix = "-users" ) +const ( + PMMUserKey = "PMM_SERVER_USER" + PMMPasswordKey = "PMM_SERVER_PASSWORD" + PMMAPIKey = "PMM_SERVER_API_KEY" +) + +const ( + EnvMongoDBDatabaseAdminUser = "MONGODB_DATABASE_ADMIN_USER" + EnvMongoDBDatabaseAdminPassword = "MONGODB_DATABASE_ADMIN_PASSWORD" + EnvMongoDBClusterAdminUser = "MONGODB_CLUSTER_ADMIN_USER" + EnvMongoDBClusterAdminPassword = "MONGODB_CLUSTER_ADMIN_PASSWORD" + EnvMongoDBUserAdminUser = "MONGODB_USER_ADMIN_USER" + EnvMongoDBUserAdminPassword = "MONGODB_USER_ADMIN_PASSWORD" + EnvMongoDBBackupUser = "MONGODB_BACKUP_USER" + EnvMongoDBBackupPassword = "MONGODB_BACKUP_PASSWORD" + EnvMongoDBClusterMonitorUser = "MONGODB_CLUSTER_MONITOR_USER" + EnvMongoDBClusterMonitorPassword = "MONGODB_CLUSTER_MONITOR_PASSWORD" + EnvPMMServerUser = PMMUserKey + EnvPMMServerPassword = PMMPasswordKey + EnvPMMServerAPIKey = PMMAPIKey +) + +type UserRole string + +const ( + RoleDatabaseAdmin UserRole = "databaseAdmin" + RoleClusterAdmin UserRole = "clusterAdmin" + RoleUserAdmin UserRole = "userAdmin" + RoleClusterMonitor UserRole = "clusterMonitor" + RoleBackup UserRole = "backup" +) + func InternalUserSecretName(cr *PerconaServerMongoDB) string { return internalPrefix + cr.Name + userPostfix } diff --git a/pkg/controller/perconaservermongodb/backup.go b/pkg/controller/perconaservermongodb/backup.go index 72a5a16166..e3f8121d8e 100644 --- a/pkg/controller/perconaservermongodb/backup.go +++ b/pkg/controller/perconaservermongodb/backup.go @@ -516,7 +516,7 @@ func (r *ReconcilePerconaServerMongoDB) resyncPBMIfNeeded(ctx context.Context, c command := []string{"pbm", "config", "--force-resync"} log.Info("Starting PBM resync", "command", command) - err = r.clientcmd.Exec(pod, "backup-agent", command, nil, &stdoutBuffer, &stderrBuffer, false) + err = r.clientcmd.Exec(ctx, pod, "backup-agent", command, nil, &stdoutBuffer, &stderrBuffer, false) if err != nil { return errors.Wrapf(err, "start PBM resync: run %v", command) } diff --git a/pkg/controller/perconaservermongodb/balancer.go b/pkg/controller/perconaservermongodb/balancer.go index 21480d2633..46214601a4 100644 --- a/pkg/controller/perconaservermongodb/balancer.go +++ b/pkg/controller/perconaservermongodb/balancer.go @@ -75,7 +75,7 @@ func (r *ReconcilePerconaServerMongoDB) enableBalancerIfNeeded(ctx context.Conte } } - mongosSession, err := r.mongosClientWithRole(ctx, cr, roleClusterAdmin) + mongosSession, err := r.mongosClientWithRole(ctx, cr, api.RoleClusterAdmin) if err != nil { return errors.Wrap(err, "failed to get mongos connection") } @@ -121,7 +121,7 @@ func (r *ReconcilePerconaServerMongoDB) disableBalancer(ctx context.Context, cr return errors.Wrapf(err, "get mongos statefulset %s", msSts.Name) } - mongosSession, err := r.mongosClientWithRole(ctx, cr, roleClusterAdmin) + mongosSession, err := r.mongosClientWithRole(ctx, cr, api.RoleClusterAdmin) if err != nil { return errors.Wrap(err, "failed to get mongos connection") } diff --git a/pkg/controller/perconaservermongodb/connections.go b/pkg/controller/perconaservermongodb/connections.go index 9ceb62f7bf..31a78ea7ea 100644 --- a/pkg/controller/perconaservermongodb/connections.go +++ b/pkg/controller/perconaservermongodb/connections.go @@ -10,8 +10,7 @@ import ( "github.com/percona/percona-server-mongodb-operator/pkg/psmdb" ) -func (r *ReconcilePerconaServerMongoDB) mongoClientWithRole(ctx context.Context, cr *api.PerconaServerMongoDB, rs api.ReplsetSpec, - role UserRole) (*mgo.Client, error) { +func (r *ReconcilePerconaServerMongoDB) mongoClientWithRole(ctx context.Context, cr *api.PerconaServerMongoDB, rs api.ReplsetSpec, role api.UserRole) (*mgo.Client, error) { c, err := r.getInternalCredentials(ctx, cr, role) if err != nil { @@ -21,7 +20,7 @@ func (r *ReconcilePerconaServerMongoDB) mongoClientWithRole(ctx context.Context, return psmdb.MongoClient(ctx, r.client, cr, rs, c) } -func (r *ReconcilePerconaServerMongoDB) mongosClientWithRole(ctx context.Context, cr *api.PerconaServerMongoDB, role UserRole) (*mgo.Client, error) { +func (r *ReconcilePerconaServerMongoDB) mongosClientWithRole(ctx context.Context, cr *api.PerconaServerMongoDB, role api.UserRole) (*mgo.Client, error) { c, err := r.getInternalCredentials(ctx, cr, role) if err != nil { return nil, errors.Wrap(err, "failed to get credentials") @@ -30,7 +29,7 @@ func (r *ReconcilePerconaServerMongoDB) mongosClientWithRole(ctx context.Context return psmdb.MongosClient(ctx, r.client, cr, c) } -func (r *ReconcilePerconaServerMongoDB) standaloneClientWithRole(ctx context.Context, cr *api.PerconaServerMongoDB, role UserRole, host string) (*mgo.Client, error) { +func (r *ReconcilePerconaServerMongoDB) standaloneClientWithRole(ctx context.Context, cr *api.PerconaServerMongoDB, role api.UserRole, host string) (*mgo.Client, error) { c, err := r.getInternalCredentials(ctx, cr, role) if err != nil { return nil, errors.Wrap(err, "failed to get credentials") diff --git a/pkg/controller/perconaservermongodb/fcv.go b/pkg/controller/perconaservermongodb/fcv.go index 30dcfbcc84..4698afeb0a 100644 --- a/pkg/controller/perconaservermongodb/fcv.go +++ b/pkg/controller/perconaservermongodb/fcv.go @@ -13,7 +13,7 @@ import ( ) func (r *ReconcilePerconaServerMongoDB) getFCV(ctx context.Context, cr *api.PerconaServerMongoDB) (string, error) { - c, err := r.mongoClientWithRole(ctx, cr, *cr.Spec.Replsets[0], roleClusterAdmin) + c, err := r.mongoClientWithRole(ctx, cr, *cr.Spec.Replsets[0], api.RoleClusterAdmin) if err != nil { return "", errors.Wrap(err, "failed to get connection") } @@ -41,9 +41,9 @@ func (r *ReconcilePerconaServerMongoDB) setFCV(ctx context.Context, cr *api.Perc var connErr error if cr.Spec.Sharding.Enabled { - cli, connErr = r.mongosClientWithRole(ctx, cr, roleClusterAdmin) + cli, connErr = r.mongosClientWithRole(ctx, cr, api.RoleClusterAdmin) } else { - cli, connErr = r.mongoClientWithRole(ctx, cr, *cr.Spec.Replsets[0], roleClusterAdmin) + cli, connErr = r.mongoClientWithRole(ctx, cr, *cr.Spec.Replsets[0], api.RoleClusterAdmin) } if connErr != nil { diff --git a/pkg/controller/perconaservermongodb/mgo.go b/pkg/controller/perconaservermongodb/mgo.go index 6e17238c88..8cae5056ad 100644 --- a/pkg/controller/perconaservermongodb/mgo.go +++ b/pkg/controller/perconaservermongodb/mgo.go @@ -85,7 +85,7 @@ func (r *ReconcilePerconaServerMongoDB) reconcileCluster(ctx context.Context, cr } } - cli, err := r.mongoClientWithRole(ctx, cr, *replset, roleClusterAdmin) + cli, err := r.mongoClientWithRole(ctx, cr, *replset, api.RoleClusterAdmin) if err != nil { if cr.Spec.Unmanaged { return api.AppStateInit, nil @@ -169,7 +169,7 @@ func (r *ReconcilePerconaServerMongoDB) reconcileCluster(ctx context.Context, cr replset.ClusterRole == api.ClusterRoleShardSvr && len(mongosPods) > 0 { - mongosSession, err := r.mongosClientWithRole(ctx, cr, roleClusterAdmin) + mongosSession, err := r.mongosClientWithRole(ctx, cr, api.RoleClusterAdmin) if err != nil { return api.AppStateError, errors.Wrap(err, "failed to get mongos connection") } @@ -447,7 +447,7 @@ func (r *ReconcilePerconaServerMongoDB) removeRSFromShard(ctx context.Context, c return nil } - cli, err := r.mongosClientWithRole(ctx, cr, roleClusterAdmin) + cli, err := r.mongosClientWithRole(ctx, cr, api.RoleClusterAdmin) if err != nil { return errors.Errorf("failed to get mongos connection: %v", err) } @@ -491,7 +491,7 @@ func (r *ReconcilePerconaServerMongoDB) handleRsAddToShard(ctx context.Context, return errors.Wrapf(err, "get rsPod %s host", rspod.Name) } - cli, err := r.mongosClientWithRole(ctx, cr, roleClusterAdmin) + cli, err := r.mongosClientWithRole(ctx, cr, api.RoleClusterAdmin) if err != nil { return errors.Wrap(err, "failed to get mongos client") } @@ -531,7 +531,7 @@ func (r *ReconcilePerconaServerMongoDB) handleReplsetInit(ctx context.Context, c } var errb, outb bytes.Buffer - err = r.clientcmd.Exec(&pod, "mongod", []string{"mongod", "--version"}, nil, &outb, &errb, false) + err = r.clientcmd.Exec(ctx, &pod, "mongod", []string{"mongod", "--version"}, nil, &outb, &errb, false) if err != nil { return fmt.Errorf("exec --version: %v / %s / %s", err, outb.String(), errb.String()) } @@ -565,14 +565,14 @@ func (r *ReconcilePerconaServerMongoDB) handleReplsetInit(ctx context.Context, c errb.Reset() outb.Reset() - err = r.clientcmd.Exec(&pod, "mongod", cmd, nil, &outb, &errb, false) + err = r.clientcmd.Exec(ctx, &pod, "mongod", cmd, nil, &outb, &errb, false) if err != nil { return fmt.Errorf("exec rs.initiate: %v / %s / %s", err, outb.String(), errb.String()) } time.Sleep(time.Second * 5) - userAdmin, err := r.getInternalCredentials(ctx, cr, roleUserAdmin) + userAdmin, err := r.getInternalCredentials(ctx, cr, api.RoleUserAdmin) if err != nil { return errors.Wrap(err, "failed to get userAdmin credentials") } @@ -580,7 +580,7 @@ func (r *ReconcilePerconaServerMongoDB) handleReplsetInit(ctx context.Context, c cmd[2] = fmt.Sprintf(`%s --eval %s`, mongoCmd, mongoInitAdminUser(userAdmin.Username, userAdmin.Password)) errb.Reset() outb.Reset() - err = r.clientcmd.Exec(&pod, "mongod", cmd, nil, &outb, &errb, false) + err = r.clientcmd.Exec(ctx, &pod, "mongod", cmd, nil, &outb, &errb, false) if err != nil { return fmt.Errorf("exec add admin user: %v / %s / %s", err, outb.String(), errb.String()) } @@ -593,29 +593,29 @@ func (r *ReconcilePerconaServerMongoDB) handleReplsetInit(ctx context.Context, c return errNoRunningMongodContainers } -func getRoles(cr *api.PerconaServerMongoDB, role UserRole) []map[string]interface{} { +func getRoles(cr *api.PerconaServerMongoDB, role api.UserRole) []map[string]interface{} { roles := make([]map[string]interface{}, 0) switch role { - case roleDatabaseAdmin: + case api.RoleDatabaseAdmin: return []map[string]interface{}{ {"role": "readWriteAnyDatabase", "db": "admin"}, {"role": "readAnyDatabase", "db": "admin"}, {"role": "restore", "db": "admin"}, {"role": "backup", "db": "admin"}, {"role": "dbAdminAnyDatabase", "db": "admin"}, - {"role": string(roleClusterMonitor), "db": "admin"}, + {"role": string(api.RoleClusterMonitor), "db": "admin"}, } - case roleClusterMonitor: + case api.RoleClusterMonitor: if cr.CompareVersion("1.12.0") >= 0 { roles = []map[string]interface{}{ {"db": "admin", "role": "explainRole"}, {"db": "local", "role": "read"}, } } - case roleBackup: + case api.RoleBackup: roles = []map[string]interface{}{ {"db": "admin", "role": "readWrite"}, - {"db": "admin", "role": string(roleClusterMonitor)}, + {"db": "admin", "role": string(api.RoleClusterMonitor)}, {"db": "admin", "role": "restore"}, {"db": "admin", "role": "pbmAnyAction"}, } @@ -699,7 +699,7 @@ func compareRoles(x []map[string]interface{}, y []map[string]interface{}) bool { func (r *ReconcilePerconaServerMongoDB) createOrUpdateSystemUsers(ctx context.Context, cr *api.PerconaServerMongoDB, replset *api.ReplsetSpec) error { log := logf.FromContext(ctx) - cli, err := r.mongoClientWithRole(ctx, cr, *replset, roleUserAdmin) + cli, err := r.mongoClientWithRole(ctx, cr, *replset, api.RoleUserAdmin) if err != nil { return errors.Wrap(err, "failed to get mongo client") } @@ -741,9 +741,9 @@ func (r *ReconcilePerconaServerMongoDB) createOrUpdateSystemUsers(ctx context.Co return errors.Wrap(err, "create or update system role") } - users := []UserRole{roleClusterAdmin, roleClusterMonitor, roleBackup} + users := []api.UserRole{api.RoleClusterAdmin, api.RoleClusterMonitor, api.RoleBackup} if cr.CompareVersion("1.13.0") >= 0 { - users = append(users, roleDatabaseAdmin) + users = append(users, api.RoleDatabaseAdmin) } for _, role := range users { @@ -779,7 +779,7 @@ func (r *ReconcilePerconaServerMongoDB) recoverReplsetNoPrimary(ctx context.Cont return errors.Wrapf(err, "get mongo hostname for pod/%s", pod.Name) } - cli, err := r.standaloneClientWithRole(ctx, cr, roleClusterAdmin, host) + cli, err := r.standaloneClientWithRole(ctx, cr, api.RoleClusterAdmin, host) if err != nil { return errors.Wrap(err, "get standalone client") } diff --git a/pkg/controller/perconaservermongodb/psmdb_controller.go b/pkg/controller/perconaservermongodb/psmdb_controller.go index 5e5f56c415..1ac3dcf42a 100644 --- a/pkg/controller/perconaservermongodb/psmdb_controller.go +++ b/pkg/controller/perconaservermongodb/psmdb_controller.go @@ -666,7 +666,7 @@ func (r *ReconcilePerconaServerMongoDB) checkIfPossibleToRemove(ctx context.Cont "config": {}, } - client, err := r.mongoClientWithRole(ctx, cr, api.ReplsetSpec{Name: rsName}, roleClusterAdmin) + client, err := r.mongoClientWithRole(ctx, cr, api.ReplsetSpec{Name: rsName}, api.RoleClusterAdmin) if err != nil { return errors.Wrap(err, "dial:") } diff --git a/pkg/controller/perconaservermongodb/secrets.go b/pkg/controller/perconaservermongodb/secrets.go index 5e3297fd13..3c01d5f51d 100644 --- a/pkg/controller/perconaservermongodb/secrets.go +++ b/pkg/controller/perconaservermongodb/secrets.go @@ -16,43 +16,17 @@ import ( "github.com/percona/percona-server-mongodb-operator/pkg/psmdb/secret" ) -const ( - envMongoDBDatabaseAdminUser = "MONGODB_DATABASE_ADMIN_USER" - envMongoDBDatabaseAdminPassword = "MONGODB_DATABASE_ADMIN_PASSWORD" - envMongoDBClusterAdminUser = "MONGODB_CLUSTER_ADMIN_USER" - envMongoDBClusterAdminPassword = "MONGODB_CLUSTER_ADMIN_PASSWORD" - envMongoDBUserAdminUser = "MONGODB_USER_ADMIN_USER" - envMongoDBUserAdminPassword = "MONGODB_USER_ADMIN_PASSWORD" - envMongoDBBackupUser = "MONGODB_BACKUP_USER" - envMongoDBBackupPassword = "MONGODB_BACKUP_PASSWORD" - envMongoDBClusterMonitorUser = "MONGODB_CLUSTER_MONITOR_USER" - envMongoDBClusterMonitorPassword = "MONGODB_CLUSTER_MONITOR_PASSWORD" - envPMMServerUser = api.PMMUserKey - envPMMServerPassword = api.PMMPasswordKey - envPMMServerAPIKey = api.PMMAPIKey -) - -type UserRole string - -const ( - roleDatabaseAdmin UserRole = "databaseAdmin" - roleClusterAdmin UserRole = "clusterAdmin" - roleUserAdmin UserRole = "userAdmin" - roleClusterMonitor UserRole = "clusterMonitor" - roleBackup UserRole = "backup" -) - func (r *ReconcilePerconaServerMongoDB) getUserSecret(ctx context.Context, cr *api.PerconaServerMongoDB, name string) (corev1.Secret, error) { secrets := corev1.Secret{} err := r.client.Get(ctx, types.NamespacedName{Name: name, Namespace: cr.Namespace}, &secrets) return secrets, errors.Wrap(err, "get user secrets") } -func (r *ReconcilePerconaServerMongoDB) getInternalCredentials(ctx context.Context, cr *api.PerconaServerMongoDB, role UserRole) (psmdb.Credentials, error) { +func (r *ReconcilePerconaServerMongoDB) getInternalCredentials(ctx context.Context, cr *api.PerconaServerMongoDB, role api.UserRole) (psmdb.Credentials, error) { return r.getCredentials(ctx, cr, api.UserSecretName(cr), role) } -func (r *ReconcilePerconaServerMongoDB) getCredentials(ctx context.Context, cr *api.PerconaServerMongoDB, name string, role UserRole) (psmdb.Credentials, error) { +func (r *ReconcilePerconaServerMongoDB) getCredentials(ctx context.Context, cr *api.PerconaServerMongoDB, name string, role api.UserRole) (psmdb.Credentials, error) { creds := psmdb.Credentials{} usersSecret, err := r.getUserSecret(ctx, cr, name) if err != nil { @@ -60,21 +34,21 @@ func (r *ReconcilePerconaServerMongoDB) getCredentials(ctx context.Context, cr * } switch role { - case roleDatabaseAdmin: - creds.Username = string(usersSecret.Data[envMongoDBDatabaseAdminUser]) - creds.Password = string(usersSecret.Data[envMongoDBDatabaseAdminPassword]) - case roleClusterAdmin: - creds.Username = string(usersSecret.Data[envMongoDBClusterAdminUser]) - creds.Password = string(usersSecret.Data[envMongoDBClusterAdminPassword]) - case roleUserAdmin: - creds.Username = string(usersSecret.Data[envMongoDBUserAdminUser]) - creds.Password = string(usersSecret.Data[envMongoDBUserAdminPassword]) - case roleClusterMonitor: - creds.Username = string(usersSecret.Data[envMongoDBClusterMonitorUser]) - creds.Password = string(usersSecret.Data[envMongoDBClusterMonitorPassword]) - case roleBackup: - creds.Username = string(usersSecret.Data[envMongoDBBackupUser]) - creds.Password = string(usersSecret.Data[envMongoDBBackupPassword]) + case api.RoleDatabaseAdmin: + creds.Username = string(usersSecret.Data[api.EnvMongoDBDatabaseAdminUser]) + creds.Password = string(usersSecret.Data[api.EnvMongoDBDatabaseAdminPassword]) + case api.RoleClusterAdmin: + creds.Username = string(usersSecret.Data[api.EnvMongoDBClusterAdminUser]) + creds.Password = string(usersSecret.Data[api.EnvMongoDBClusterAdminPassword]) + case api.RoleUserAdmin: + creds.Username = string(usersSecret.Data[api.EnvMongoDBUserAdminUser]) + creds.Password = string(usersSecret.Data[api.EnvMongoDBUserAdminPassword]) + case api.RoleClusterMonitor: + creds.Username = string(usersSecret.Data[api.EnvMongoDBClusterMonitorUser]) + creds.Password = string(usersSecret.Data[api.EnvMongoDBClusterMonitorPassword]) + case api.RoleBackup: + creds.Username = string(usersSecret.Data[api.EnvMongoDBBackupUser]) + creds.Password = string(usersSecret.Data[api.EnvMongoDBBackupPassword]) default: return creds, errors.Errorf("not implemented for role: %s", role) } @@ -101,7 +75,7 @@ func (r *ReconcilePerconaServerMongoDB) reconcileUsersSecret(ctx context.Context return errors.Wrap(err, "failed to fill secret data") } if cr.CompareVersion("1.2.0") < 0 { - for _, v := range []string{envMongoDBClusterMonitorUser, envMongoDBClusterMonitorPassword} { + for _, v := range []string{api.EnvMongoDBClusterMonitorUser, api.EnvMongoDBClusterMonitorPassword} { escaped, ok := secretObj.Data[v+"_ESCAPED"] if !ok || url.QueryEscape(string(secretObj.Data[v])) != string(escaped) { secretObj.Data[v+"_ESCAPED"] = []byte(url.QueryEscape(string(secretObj.Data[v]))) @@ -145,20 +119,20 @@ func fillSecretData(cr *api.PerconaServerMongoDB, data map[string][]byte) (bool, data = make(map[string][]byte) } userMap := map[string]string{ - envMongoDBBackupUser: string(roleBackup), - envMongoDBClusterAdminUser: string(roleClusterAdmin), - envMongoDBClusterMonitorUser: string(roleClusterMonitor), - envMongoDBUserAdminUser: string(roleUserAdmin), + api.EnvMongoDBBackupUser: string(api.RoleBackup), + api.EnvMongoDBClusterAdminUser: string(api.RoleClusterAdmin), + api.EnvMongoDBClusterMonitorUser: string(api.RoleClusterMonitor), + api.EnvMongoDBUserAdminUser: string(api.RoleUserAdmin), } passKeys := []string{ - envMongoDBClusterAdminPassword, - envMongoDBUserAdminPassword, - envMongoDBBackupPassword, - envMongoDBClusterMonitorPassword, + api.EnvMongoDBClusterAdminPassword, + api.EnvMongoDBUserAdminPassword, + api.EnvMongoDBBackupPassword, + api.EnvMongoDBClusterMonitorPassword, } if cr.CompareVersion("1.13.0") >= 0 { - userMap[envMongoDBDatabaseAdminUser] = string(roleDatabaseAdmin) - passKeys = append(passKeys, envMongoDBDatabaseAdminPassword) + userMap[api.EnvMongoDBDatabaseAdminUser] = string(api.RoleDatabaseAdmin) + passKeys = append(passKeys, api.EnvMongoDBDatabaseAdminPassword) } changes := false diff --git a/pkg/controller/perconaservermongodb/smart.go b/pkg/controller/perconaservermongodb/smart.go index b3018ff25d..4dae854baa 100644 --- a/pkg/controller/perconaservermongodb/smart.go +++ b/pkg/controller/perconaservermongodb/smart.go @@ -116,7 +116,7 @@ func (r *ReconcilePerconaServerMongoDB) smartUpdate(ctx context.Context, cr *api } } - client, err := r.mongoClientWithRole(ctx, cr, *replset, roleClusterAdmin) + client, err := r.mongoClientWithRole(ctx, cr, *replset, api.RoleClusterAdmin) if err != nil { return fmt.Errorf("failed to get mongo client: %v", err) } diff --git a/pkg/controller/perconaservermongodb/users.go b/pkg/controller/perconaservermongodb/users.go index 6b4e7acbc9..a234d1c715 100644 --- a/pkg/controller/perconaservermongodb/users.go +++ b/pkg/controller/perconaservermongodb/users.go @@ -148,7 +148,7 @@ func (r *ReconcilePerconaServerMongoDB) killcontainer(ctx context.Context, pods err := retry.OnError(retry.DefaultBackoff, func(_ error) bool { return true }, func() error { stderrBuf := &bytes.Buffer{} - err := r.clientcmd.Exec(&pod, containerName, []string{"/bin/sh", "-c", "kill 1"}, nil, nil, stderrBuf, false) + err := r.clientcmd.Exec(ctx, &pod, containerName, []string{"/bin/sh", "-c", "kill 1"}, nil, nil, stderrBuf, false) if err != nil { return errors.Wrap(err, "exec command in pod") } @@ -196,7 +196,7 @@ func (su *systemUsers) add(nameKey, passKey string) (changed bool, err error) { bytes.Equal(su.newData[passKey], su.currData[passKey]) { return false, nil } - if nameKey == envPMMServerUser || passKey == envPMMServerAPIKey { + if nameKey == api.EnvPMMServerUser || passKey == api.EnvPMMServerAPIKey { return true, nil } su.users = append(su.users, systemUser{ @@ -226,31 +226,31 @@ func (r *ReconcilePerconaServerMongoDB) updateSysUsers(ctx context.Context, cr * } users := []user{ { - nameKey: envMongoDBClusterAdminUser, - passKey: envMongoDBClusterAdminPassword, + nameKey: api.EnvMongoDBClusterAdminUser, + passKey: api.EnvMongoDBClusterAdminPassword, }, { - nameKey: envMongoDBClusterMonitorUser, - passKey: envMongoDBClusterMonitorPassword, + nameKey: api.EnvMongoDBClusterMonitorUser, + passKey: api.EnvMongoDBClusterMonitorPassword, }, { - nameKey: envMongoDBBackupUser, - passKey: envMongoDBBackupPassword, + nameKey: api.EnvMongoDBBackupUser, + passKey: api.EnvMongoDBBackupPassword, }, // !!! UserAdmin always must be the last to update since we're using it for the mongo connection { - nameKey: envMongoDBUserAdminUser, - passKey: envMongoDBUserAdminPassword, + nameKey: api.EnvMongoDBUserAdminUser, + passKey: api.EnvMongoDBUserAdminPassword, }, } - if _, ok := currUsersSec.Data[envMongoDBDatabaseAdminUser]; cr.CompareVersion("1.13.0") >= 0 && ok { + if _, ok := currUsersSec.Data[api.EnvMongoDBDatabaseAdminUser]; cr.CompareVersion("1.13.0") >= 0 && ok { users = append([]user{ { - nameKey: envMongoDBDatabaseAdminUser, - passKey: envMongoDBDatabaseAdminPassword, + nameKey: api.EnvMongoDBDatabaseAdminUser, + passKey: api.EnvMongoDBDatabaseAdminPassword, }, }, users...) } @@ -259,15 +259,15 @@ func (r *ReconcilePerconaServerMongoDB) updateSysUsers(ctx context.Context, cr * if cr.Spec.PMM.ShouldUseAPIKeyAuth(newUsersSec) { users = append([]user{ { - nameKey: envPMMServerAPIKey, - passKey: envPMMServerAPIKey, + nameKey: api.EnvPMMServerAPIKey, + passKey: api.EnvPMMServerAPIKey, }, }, users...) } else { users = append([]user{ { - nameKey: envPMMServerUser, - passKey: envPMMServerPassword, + nameKey: api.EnvPMMServerUser, + passKey: api.EnvPMMServerPassword, }, }, users...) } @@ -281,9 +281,9 @@ func (r *ReconcilePerconaServerMongoDB) updateSysUsers(ctx context.Context, cr * if changed { switch u.nameKey { - case envMongoDBBackupUser: + case api.EnvMongoDBBackupUser: containers = append(containers, "backup-agent") - case envPMMServerUser, envPMMServerAPIKey: + case api.EnvPMMServerUser, api.EnvPMMServerAPIKey: containers = append(containers, "pmm-client") } } @@ -304,7 +304,7 @@ func (r *ReconcilePerconaServerMongoDB) updateUsers(ctx context.Context, cr *api for i := range repls { replset := repls[i] grp.Go(func() error { - client, err := r.mongoClientWithRole(gCtx, cr, *replset, roleUserAdmin) + client, err := r.mongoClientWithRole(gCtx, cr, *replset, api.RoleUserAdmin) if err != nil { return errors.Wrap(err, "dial:") } diff --git a/pkg/controller/perconaservermongodb/version.go b/pkg/controller/perconaservermongodb/version.go index 78534a45e9..f62a282907 100644 --- a/pkg/controller/perconaservermongodb/version.go +++ b/pkg/controller/perconaservermongodb/version.go @@ -430,7 +430,7 @@ func (r *ReconcilePerconaServerMongoDB) fetchVersionFromMongo(ctx context.Contex return nil } - session, err := r.mongoClientWithRole(ctx, cr, *replset, roleClusterAdmin) + session, err := r.mongoClientWithRole(ctx, cr, *replset, api.RoleClusterAdmin) if err != nil { return errors.Wrap(err, "dial") } diff --git a/pkg/controller/perconaservermongodbrestore/physical.go b/pkg/controller/perconaservermongodbrestore/physical.go index 956e30cd4c..3a19ed9151 100644 --- a/pkg/controller/perconaservermongodbrestore/physical.go +++ b/pkg/controller/perconaservermongodbrestore/physical.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "encoding/json" + "fmt" "strings" "time" @@ -26,6 +27,26 @@ import ( "github.com/percona/percona-server-mongodb-operator/version" ) +// reconcilePhysicalRestore performs a physical restore of a Percona Server for MongoDB from a backup. +// The function: +// 1. Fetches the associated MongoDB cluster. +// 2. Ensures the cluster is managed. +// 3. Verifies server and cluster versions. +// 4. Checks the cluster for incompatible configurations, such as arbiter nodes. +// 5. Handles restore configurations and preparations. +// 6. Checks and waits for readiness of necessary resources such as StatefulSets and MongoDB replica sets. +// 7. Initiates the physical restore using the Percona Backup Management (PBM) cli. +// 8. Monitors the restore status and updates the PerconaServerMongoDBRestore resource's state accordingly. +// 9. Upon successful restore, cleans up resources and ensures PBM configuration is resynchronized. +// +// Parameters: +// - ctx: The context of the operation. +// - cr: Represents the PerconaServerMongoDBRestore custom resource. +// - bcp: Represents the PerconaServerMongoDBBackup custom resource. +// +// Returns: +// - The status of the restore. +// - An error if encountered during the reconcile process. func (r *ReconcilePerconaServerMongoDBRestore) reconcilePhysicalRestore(ctx context.Context, cr *psmdbv1.PerconaServerMongoDBRestore, bcp *psmdbv1.PerconaServerMongoDBBackup) (psmdbv1.PerconaServerMongoDBRestoreStatus, error) { log := logf.FromContext(ctx) @@ -78,83 +99,122 @@ func (r *ReconcilePerconaServerMongoDBRestore) reconcilePhysicalRestore(ctx cont return status, errors.Wrap(err, "prepare statefulsets for physical restore") } - ready, err := r.checkIfStatefulSetsAreReadyForPhysicalRestore(ctx, cluster) + sfsReady, err := r.checkIfStatefulSetsAreReadyForPhysicalRestore(ctx, cluster) if err != nil { return status, errors.Wrap(err, "check if statefulsets are ready for physical restore") } - if (!ready && cr.Status.State != psmdbv1.RestoreStateRunning) || cr.Status.State == psmdbv1.RestoreStateNew { - log.Info("Waiting for statefulsets to be ready before restore", "ready", ready) + if (!sfsReady && cr.Status.State != psmdbv1.RestoreStateRunning) || cr.Status.State == psmdbv1.RestoreStateNew { + log.Info("Waiting for statefulsets to be ready before restore", "ready", sfsReady) return status, nil } - pod := corev1.Pod{} - if err := r.client.Get(ctx, types.NamespacedName{Name: cluster.Name + "-" + cluster.Spec.Replsets[0].Name + "-0", Namespace: cluster.Namespace}, &pod); err != nil { - return status, errors.Wrap(err, "get pod") + if cr.Status.State == psmdbv1.RestoreStateWaiting && sfsReady && cr.Spec.PITR != nil { + rsReady, err := r.checkIfReplsetsAreReadyForPhysicalRestore(ctx, cluster) + if err != nil { + return status, errors.Wrap(err, "check if replsets are ready for physical restore") + } + + if !rsReady { + if err := r.prepareReplsetsForPhysicalRestore(ctx, cluster); err != nil { + return status, errors.Wrap(err, "prepare replsets for physical restore") + } + + log.Info("Waiting for replsets to be ready before restore", "ready", rsReady) + return status, nil + } } stdoutBuf := &bytes.Buffer{} stderrBuf := &bytes.Buffer{} + replsets := cluster.Spec.Replsets + if cluster.Spec.Sharding.Enabled { + replsets = append(replsets, cluster.Spec.Sharding.ConfigsvrReplSet) + } + if cr.Status.State == psmdbv1.RestoreStateWaiting { - command := []string{"/opt/percona/pbm", "config", "--file", "/etc/pbm/pbm_config.yaml"} - log.Info("Set PBM configuration", "command", command) - if err := r.clientcmd.Exec(&pod, "mongod", command, nil, stdoutBuf, stderrBuf, false); err != nil { - return status, errors.Wrapf(err, "resync config stderr: %s stdout: %s", stderrBuf.String(), stdoutBuf.String()) - } - - ticker := time.NewTicker(5 * time.Second) - defer ticker.Stop() - - timeout := time.NewTimer(900 * time.Second) - defer timeout.Stop() - - outer: - for { - select { - case <-timeout.C: - return status, errors.Errorf("timeout while waiting PBM operation to finish") - case <-ticker.C: - err := retry.OnError(retry.DefaultBackoff, func(err error) bool { return strings.Contains(err.Error(), "No agent available") }, func() error { - stdoutBuf.Reset() - stderrBuf.Reset() - - command = []string{"/opt/percona/pbm", "status", "--out", "json"} - if err := r.clientcmd.Exec(&pod, "mongod", command, nil, stdoutBuf, stderrBuf, false); err != nil { - return errors.Wrapf(err, "get PBM status stderr: %s stdout: %s", stderrBuf.String(), stdoutBuf.String()) + for _, rs := range replsets { + pod := corev1.Pod{} + if err := r.client.Get(ctx, types.NamespacedName{Name: rs.PodName(cluster, 0), Namespace: cluster.Namespace}, &pod); err != nil { + return status, errors.Wrap(err, "get pod") + } + + log.V(1).Info("Checking PBM operations for replset", "replset", rs.Name, "pod", rs.PodName(cluster, 0)) + + command := []string{"/opt/percona/pbm", "config", "--file", "/etc/pbm/pbm_config.yaml"} + log.Info("Set PBM configuration", "command", command, "pod", pod.Name) + if err := r.clientcmd.Exec(ctx, &pod, "mongod", command, nil, stdoutBuf, stderrBuf, false); err != nil { + return status, errors.Wrapf(err, "resync config stderr: %s stdout: %s", stderrBuf.String(), stdoutBuf.String()) + } + + ticker := time.NewTicker(5 * time.Second) + defer ticker.Stop() + + timeout := time.NewTimer(900 * time.Second) + defer timeout.Stop() + + outer: + for { + select { + case <-timeout.C: + return status, errors.Errorf("timeout while waiting PBM operation to finish") + case <-ticker.C: + err := retry.OnError(retry.DefaultBackoff, func(err error) bool { return strings.Contains(err.Error(), "No agent available") }, func() error { + stdoutBuf.Reset() + stderrBuf.Reset() + + command = []string{"/opt/percona/pbm", "status", "--out", "json"} + if err := r.clientcmd.Exec(ctx, &pod, "mongod", command, nil, stdoutBuf, stderrBuf, false); err != nil { + return errors.Wrapf(err, "get PBM status stderr: %s stdout: %s", stderrBuf.String(), stdoutBuf.String()) + } + + log.V(1).Info("PBM status", "status", stdoutBuf.String()) + + return nil + }) + if err != nil { + return status, err } - return nil - }) - if err != nil { - return status, err - } + var pbmStatus struct { + Running struct { + Type string `json:"type,omitempty"` + OpId string `json:"opID,omitempty"` + } `json:"running"` + } - var pbmStatus struct { - Running struct { - Type string `json:"type,omitempty"` - OpId string `json:"opID,omitempty"` - } `json:"running"` - } + if err := json.Unmarshal(stdoutBuf.Bytes(), &pbmStatus); err != nil { + return status, errors.Wrap(err, "unmarshal PBM status output") + } - if err := json.Unmarshal(stdoutBuf.Bytes(), &pbmStatus); err != nil { - return status, errors.Wrap(err, "unmarshal PBM status output") - } + if len(pbmStatus.Running.OpId) == 0 { + break outer + } - if len(pbmStatus.Running.OpId) == 0 { - break outer + log.Info("Waiting for another PBM operation to finish", "type", pbmStatus.Running.Type, "opID", pbmStatus.Running.OpId) } - - log.Info("Waiting for another PBM operation to finish", "type", pbmStatus.Running.Type, "opID", pbmStatus.Running.OpId) } } - command = []string{"/opt/percona/pbm", "restore", bcp.Status.PBMname, "--out", "json"} - log.Info("Starting restore", "command", command) + var command []string + if cr.Spec.PITR != nil { + ts := cr.Spec.PITR.Date.Format("2006-01-02T15:04:05") + command = []string{"/opt/percona/pbm", "restore", "--base-snapshot", bcp.Status.PBMname, "--time", ts, "--out", "json"} + } else { + command = []string{"/opt/percona/pbm", "restore", bcp.Status.PBMname, "--out", "json"} + } + + pod := corev1.Pod{} + if err := r.client.Get(ctx, types.NamespacedName{Name: replsets[0].PodName(cluster, 0), Namespace: cluster.Namespace}, &pod); err != nil { + return status, errors.Wrap(err, "get pod") + } + + log.Info("Starting restore", "command", command, "pod", pod.Name) stdoutBuf.Reset() stderrBuf.Reset() - err := r.clientcmd.Exec(&pod, "mongod", command, nil, stdoutBuf, stderrBuf, false) + err := r.clientcmd.Exec(ctx, &pod, "mongod", command, nil, stdoutBuf, stderrBuf, false) if err != nil { return status, errors.Wrapf(err, "start restore stderr: %s stdout: %s", stderrBuf.String(), stdoutBuf.String()) } @@ -175,7 +235,9 @@ func (r *ReconcilePerconaServerMongoDBRestore) reconcilePhysicalRestore(ctx cont meta := pbm.BackupMeta{} - err = retry.OnError(retry.DefaultBackoff, func(err error) bool { return strings.Contains(err.Error(), "container is not created or running") }, func() error { + err = retry.OnError(retry.DefaultBackoff, func(err error) bool { + return strings.Contains(err.Error(), "container is not created or running") || strings.Contains(err.Error(), "describe restore: error dialing backend: No agent available") + }, func() error { stdoutBuf.Reset() stderrBuf.Reset() @@ -185,8 +247,14 @@ func (r *ReconcilePerconaServerMongoDBRestore) reconcilePhysicalRestore(ctx cont "--out", "json", } - log.V(1).Info("Check restore status", "command", command) - if err := r.clientcmd.Exec(&pod, "mongod", command, nil, stdoutBuf, stderrBuf, false); err != nil { + pod := corev1.Pod{} + if err := r.client.Get(ctx, types.NamespacedName{Name: replsets[0].PodName(cluster, 0), Namespace: cluster.Namespace}, &pod); err != nil { + return errors.Wrap(err, "get pod") + } + + log.V(1).Info("Check restore status", "command", command, "pod", pod.Name) + + if err := r.clientcmd.Exec(ctx, &pod, "mongod", command, nil, stdoutBuf, stderrBuf, false); err != nil { return errors.Wrap(err, "describe restore") } @@ -196,7 +264,6 @@ func (r *ReconcilePerconaServerMongoDBRestore) reconcilePhysicalRestore(ctx cont return status, err } - if err := json.Unmarshal(stdoutBuf.Bytes(), &meta); err != nil { return status, errors.Wrap(err, "unmarshal PBM describe-restore output") } @@ -211,6 +278,9 @@ func (r *ReconcilePerconaServerMongoDBRestore) reconcilePhysicalRestore(ctx cont return status, nil } } + case pbm.StatusError: + status.State = psmdbv1.RestoreStateError + status.Error = meta.Err case pbm.StatusRunning: status.State = psmdbv1.RestoreStateRunning case pbm.StatusDone: @@ -272,6 +342,112 @@ func (r *ReconcilePerconaServerMongoDBRestore) reconcilePhysicalRestore(ctx cont return status, nil } +// updateStatefulSetForPhysicalRestore updates the StatefulSet to prepare it for a physical restore of PerconaServerMongoDB. +// This involves: +// - Annotating the StatefulSet to prevent psmdb_controller reconciliation. +// - Adding an init container that installs necessary tools for backup and restore. +// - Removing the existing backup-agent container. +// - Appending a volume for backup configuration. +// - Adjusting the primary container's command, environment variables, and volume mounts for the restore process. +// It returns an error if there's any issue during the update or if the backup-agent container is not found. +func (r *ReconcilePerconaServerMongoDBRestore) updateStatefulSetForPhysicalRestore(ctx context.Context, cluster *psmdbv1.PerconaServerMongoDB, namespacedName types.NamespacedName) error { + log := logf.FromContext(ctx) + + sts := appsv1.StatefulSet{} + err := r.client.Get(ctx, namespacedName, &sts) + if err != nil { + return err + } + + // Annotating statefulset to stop reconciliation in psmdb_controller + sts.Annotations[psmdbv1.AnnotationRestoreInProgress] = "true" + + cmd := []string{ + "bash", "-c", + "install -D /usr/bin/pbm /opt/percona/pbm && install -D /usr/bin/pbm-agent /opt/percona/pbm-agent", + } + pbmInit := psmdb.EntrypointInitContainer( + cluster, + "pbm-init", + cluster.Spec.Backup.Image, + cluster.Spec.ImagePullPolicy, + cmd, + ) + sts.Spec.Template.Spec.InitContainers = append(sts.Spec.Template.Spec.InitContainers, pbmInit) + + // remove backup-agent container + pbmIdx := -1 + for idx, c := range sts.Spec.Template.Spec.Containers { + if c.Name == "backup-agent" { + pbmIdx = idx + break + } + } + if pbmIdx == -1 { + return errors.New("failed to find backup-agent container") + } + sts.Spec.Template.Spec.Containers = append(sts.Spec.Template.Spec.Containers[:pbmIdx], sts.Spec.Template.Spec.Containers[pbmIdx+1:]...) + + sts.Spec.Template.Spec.Volumes = append(sts.Spec.Template.Spec.Volumes, corev1.Volume{ + Name: "pbm-config", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: "pbm-config", + }, + }, + }) + sts.Spec.Template.Spec.Containers[0].VolumeMounts = append(sts.Spec.Template.Spec.Containers[0].VolumeMounts, corev1.VolumeMount{ + Name: "pbm-config", + MountPath: "/etc/pbm/", + ReadOnly: true, + }) + sts.Spec.Template.Spec.Containers[0].Command = []string{"/opt/percona/physical-restore-ps-entry.sh"} + sts.Spec.Template.Spec.Containers[0].Env = append(sts.Spec.Template.Spec.Containers[0].Env, []corev1.EnvVar{ + { + Name: "PBM_AGENT_MONGODB_USERNAME", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + Key: "MONGODB_BACKUP_USER", + LocalObjectReference: corev1.LocalObjectReference{ + Name: cluster.Spec.Secrets.Users, + }, + }, + }, + }, + { + Name: "PBM_AGENT_MONGODB_PASSWORD", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + Key: "MONGODB_BACKUP_PASSWORD", + LocalObjectReference: corev1.LocalObjectReference{ + Name: cluster.Spec.Secrets.Users, + }, + }, + }, + }, + { + Name: "POD_NAME", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "metadata.name", + }, + }, + }, + { + Name: "PBM_MONGODB_URI", + Value: "mongodb://$(PBM_AGENT_MONGODB_USERNAME):$(PBM_AGENT_MONGODB_PASSWORD)@$(POD_NAME)", + }, + }...) + + err = r.client.Update(ctx, &sts) + if err != nil { + return err + } + + log.Info("Updated statefulset", "name", namespacedName.Name) + return nil +} + func (r *ReconcilePerconaServerMongoDBRestore) prepareStatefulSetsForPhysicalRestore(ctx context.Context, cluster *psmdbv1.PerconaServerMongoDB) error { log := logf.FromContext(ctx) @@ -297,107 +473,140 @@ func (r *ReconcilePerconaServerMongoDBRestore) prepareStatefulSetsForPhysicalRes log.Info("Preparing statefulset for physical restore", "name", stsName) err = retry.RetryOnConflict(retry.DefaultRetry, func() error { - sts := appsv1.StatefulSet{} - nn := types.NamespacedName{Namespace: cluster.Namespace, Name: stsName} - err := r.client.Get(ctx, nn, &sts) - if err != nil { - return err - } + return r.updateStatefulSetForPhysicalRestore(ctx, cluster, types.NamespacedName{Namespace: cluster.Namespace, Name: stsName}) + }) + if err != nil { + return errors.Wrapf(err, "prepare statefulset %s for physical restore", stsName) + } + } - // Annotating statefulset to stop reconciliation in psmdb_controller - sts.Annotations[psmdbv1.AnnotationRestoreInProgress] = "true" + return nil +} - cmd := []string{ - "bash", "-c", - "install -D /usr/bin/pbm /opt/percona/pbm && install -D /usr/bin/pbm-agent /opt/percona/pbm-agent", - } - pbmInit := psmdb.EntrypointInitContainer( - cluster, - "pbm-init", - cluster.Spec.Backup.Image, - cluster.Spec.ImagePullPolicy, - cmd, - ) - sts.Spec.Template.Spec.InitContainers = append(sts.Spec.Template.Spec.InitContainers, pbmInit) - - // remove backup-agent container - pbmIdx := -1 - for idx, c := range sts.Spec.Template.Spec.Containers { - if c.Name == "backup-agent" { - pbmIdx = idx - break - } - } - if pbmIdx == -1 { - return errors.New("failed to find backup-agent container") +func (r *ReconcilePerconaServerMongoDBRestore) getUserCredentials(ctx context.Context, cluster *psmdbv1.PerconaServerMongoDB, role psmdbv1.UserRole) (psmdb.Credentials, error) { + creds := psmdb.Credentials{} + + usersSecret := corev1.Secret{} + err := r.client.Get(ctx, types.NamespacedName{Name: psmdbv1.UserSecretName(cluster), Namespace: cluster.Namespace}, &usersSecret) + if err != nil { + return creds, errors.Wrap(err, "get secret") + } + + switch role { + case psmdbv1.RoleDatabaseAdmin: + creds.Username = string(usersSecret.Data[psmdbv1.EnvMongoDBDatabaseAdminUser]) + creds.Password = string(usersSecret.Data[psmdbv1.EnvMongoDBDatabaseAdminPassword]) + case psmdbv1.RoleClusterAdmin: + creds.Username = string(usersSecret.Data[psmdbv1.EnvMongoDBClusterAdminUser]) + creds.Password = string(usersSecret.Data[psmdbv1.EnvMongoDBClusterAdminPassword]) + case psmdbv1.RoleUserAdmin: + creds.Username = string(usersSecret.Data[psmdbv1.EnvMongoDBUserAdminUser]) + creds.Password = string(usersSecret.Data[psmdbv1.EnvMongoDBUserAdminPassword]) + case psmdbv1.RoleClusterMonitor: + creds.Username = string(usersSecret.Data[psmdbv1.EnvMongoDBClusterMonitorUser]) + creds.Password = string(usersSecret.Data[psmdbv1.EnvMongoDBClusterMonitorPassword]) + case psmdbv1.RoleBackup: + creds.Username = string(usersSecret.Data[psmdbv1.EnvMongoDBBackupUser]) + creds.Password = string(usersSecret.Data[psmdbv1.EnvMongoDBBackupPassword]) + default: + return creds, errors.Errorf("not implemented for role: %s", role) + } + + return creds, nil +} + +func (r *ReconcilePerconaServerMongoDBRestore) runMongosh(ctx context.Context, cluster *psmdbv1.PerconaServerMongoDB, pod *corev1.Pod, eval string) (*bytes.Buffer, *bytes.Buffer, error) { + log := logf.FromContext(ctx) + + stdoutBuf := &bytes.Buffer{} + stderrBuf := &bytes.Buffer{} + + creds, err := r.getUserCredentials(ctx, cluster, psmdbv1.RoleClusterAdmin) + if err != nil { + return stdoutBuf, stderrBuf, errors.Wrapf(err, "get %s credentials", psmdbv1.RoleClusterAdmin) + } + + cmd := []string{"mongosh", "--quiet", "-u", creds.Username, "-p", creds.Password, "--eval", eval} + + log.V(1).Info("Running cmd in pod", "eval", eval, "pod", pod.Name) + if err := r.clientcmd.Exec(ctx, pod, "mongod", cmd, nil, stdoutBuf, stderrBuf, false); err != nil { + log.V(1).Info("Cmd failed", "stdout", stdoutBuf.String(), "stderr", stderrBuf.String()) + return stdoutBuf, stderrBuf, errors.Wrap(err, "cmd failed") + } + log.V(1).Info("Cmd succeeded", "stdout", stdoutBuf.String(), "stderr", stderrBuf.String()) + + return stdoutBuf, stderrBuf, nil +} + +func (r *ReconcilePerconaServerMongoDBRestore) prepareReplsetsForPhysicalRestore(ctx context.Context, cluster *psmdbv1.PerconaServerMongoDB) error { + log := logf.FromContext(ctx) + + replsets := cluster.Spec.Replsets + if cluster.Spec.Sharding.Enabled { + replsets = append(replsets, cluster.Spec.Sharding.ConfigsvrReplSet) + } + + jsTempl := "cfg = rs.config(); podZero = cfg.members.find(member => member.tags.podName === '%s'); podZero.priority += 1; rs.reconfig(cfg)" + + for _, rs := range replsets { + log.Info("Preparing replset for physical restore", "replset", rs.Name) + + podList, err := psmdb.GetRSPods(ctx, r.client, cluster, rs.Name, false) + if err != nil { + return errors.Wrapf(err, "get pods of replset %s", rs.Name) + } + + for _, pod := range podList.Items { + stdoutBuf, _, err := r.runMongosh(ctx, cluster, &pod, "db.isMaster().ismaster") + if err != nil { + continue } - sts.Spec.Template.Spec.Containers = append(sts.Spec.Template.Spec.Containers[:pbmIdx], sts.Spec.Template.Spec.Containers[pbmIdx+1:]...) - sts.Spec.Template.Spec.Volumes = append(sts.Spec.Template.Spec.Volumes, corev1.Volume{ - Name: "pbm-config", - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: "pbm-config", - }, - }, - }) - sts.Spec.Template.Spec.Containers[0].VolumeMounts = append(sts.Spec.Template.Spec.Containers[0].VolumeMounts, corev1.VolumeMount{ - Name: "pbm-config", - MountPath: "/etc/pbm/", - ReadOnly: true, - }) - sts.Spec.Template.Spec.Containers[0].Command = []string{"/opt/percona/physical-restore-ps-entry.sh"} - sts.Spec.Template.Spec.Containers[0].Env = append(sts.Spec.Template.Spec.Containers[0].Env, []corev1.EnvVar{ - { - Name: "PBM_AGENT_MONGODB_USERNAME", - ValueFrom: &corev1.EnvVarSource{ - SecretKeyRef: &corev1.SecretKeySelector{ - Key: "MONGODB_BACKUP_USER", - LocalObjectReference: corev1.LocalObjectReference{ - Name: cluster.Spec.Secrets.Users, - }, - }, - }, - }, - { - Name: "PBM_AGENT_MONGODB_PASSWORD", - ValueFrom: &corev1.EnvVarSource{ - SecretKeyRef: &corev1.SecretKeySelector{ - Key: "MONGODB_BACKUP_PASSWORD", - LocalObjectReference: corev1.LocalObjectReference{ - Name: cluster.Spec.Secrets.Users, - }, - }, - }, - }, - { - Name: "POD_NAME", - ValueFrom: &corev1.EnvVarSource{ - FieldRef: &corev1.ObjectFieldSelector{ - FieldPath: "metadata.name", - }, - }, - }, - { - Name: "PBM_MONGODB_URI", - Value: "mongodb://$(PBM_AGENT_MONGODB_USERNAME):$(PBM_AGENT_MONGODB_PASSWORD)@$(POD_NAME)", - }, - }...) + if strings.TrimSuffix(stdoutBuf.String(), "\n") != "true" { + log.V(1).Info("Skipping secondary pod", "pod", pod.Name) + continue + } - err = r.client.Update(ctx, &sts) + podZero := rs.PodName(cluster, 0) + _, _, err = r.runMongosh(ctx, cluster, &pod, fmt.Sprintf(jsTempl, podZero)) if err != nil { - return err + return errors.Wrapf(err, "make %s primary", podZero) } + } + } - log.Info("Updated statefulset", "name", stsName) - return nil - }) + return nil +} + +func (r *ReconcilePerconaServerMongoDBRestore) checkIfReplsetsAreReadyForPhysicalRestore(ctx context.Context, cluster *psmdbv1.PerconaServerMongoDB) (bool, error) { + log := logf.FromContext(ctx) + + replsets := cluster.Spec.Replsets + if cluster.Spec.Sharding.Enabled { + replsets = append(replsets, cluster.Spec.Sharding.ConfigsvrReplSet) + } + + for _, rs := range replsets { + log.Info("Checking if replset is ready for physical restore", "replset", rs.Name) + + podZero := rs.PodName(cluster, 0) + + pod := corev1.Pod{} + if err := r.client.Get(ctx, types.NamespacedName{Name: podZero, Namespace: cluster.Namespace}, &pod); err != nil { + return false, errors.Wrapf(err, "get pod %s", podZero) + } + + stdoutBuf, _, err := r.runMongosh(ctx, cluster, &pod, "db.isMaster().ismaster") if err != nil { - return errors.Wrapf(err, "prepare statefulset %s for physical restore", stsName) + return false, errors.Wrap(err, "check if pod zero is primary") + } + + if strings.TrimSuffix(stdoutBuf.String(), "\n") != "true" { + return false, nil } } - return nil + return true, nil } func (r *ReconcilePerconaServerMongoDBRestore) createPBMConfigSecret(ctx context.Context, cr *psmdbv1.PerconaServerMongoDBRestore, cluster *psmdbv1.PerconaServerMongoDB, bcp *psmdbv1.PerconaServerMongoDBBackup) error { @@ -423,6 +632,8 @@ func (r *ReconcilePerconaServerMongoDBRestore) createPBMConfigSecret(ctx context return errors.Wrap(err, "get PBM config") } + pbmConfig.PITR.Enabled = false + confBytes, err := yaml.Marshal(pbmConfig) if err != nil { return errors.Wrap(err, "marshal PBM config to yaml")