From 282394a89cd2f4874a3709848b6d8cae800cccde Mon Sep 17 00:00:00 2001 From: Andrii Dema Date: Tue, 23 Apr 2024 17:08:21 +0300 Subject: [PATCH 1/2] K8SPSMDB-1014: update cert-manager certs and issuers (#1383) * K8SPSMDB-1014: update cert-manager certs and issuers https://jira.percona.com/browse/K8SPSMDB-1014 * fix * Fix upgrade from `1.14.0` * fix test * fix merge * update `upgrade-consistency-sharded-tls` * fix `TestReconcileStatefulSet` * fix test * don't use dry controller for deleting deprecated issuer * rename variables * fix * check if sharding is enabled * rename `isAllStsHasLatestSSL` --------- Co-authored-by: Natalia Marukovich Co-authored-by: Viacheslav Sarzhan Co-authored-by: Tomislav Plavcic --- clientcmd/clientcmd.go | 20 +- .../certificate_some-name-ssl-internal.yml | 1 + .../compare/certificate_some-name-ssl.yml | 1 + .../issuer_some-name-psmdb-ca-issuer.yml | 1 + .../compare/issuer_some-name-psmdb-issuer.yml | 1 + .../statefulset_some-name-cfg-1150-oc.yml | 2 +- .../statefulset_some-name-cfg-1150.yml | 2 +- .../statefulset_some-name-cfg-1160-oc.yml | 2 +- .../statefulset_some-name-cfg-1160.yml | 2 +- .../statefulset_some-name-rs0-1150-oc.yml | 2 +- .../statefulset_some-name-rs0-1150.yml | 2 +- .../statefulset_some-name-rs0-1160-oc.yml | 2 +- .../statefulset_some-name-rs0-1160.yml | 2 +- e2e-tests/upgrade-consistency-sharded-tls/run | 45 +-- .../v1/perconaservermongodbrestore_types.go | 6 +- pkg/apis/psmdb/v1/psmdb_types.go | 8 + .../perconaservermongodb/psmdb_controller.go | 149 +++---- pkg/controller/perconaservermongodb/smart.go | 76 +++- pkg/controller/perconaservermongodb/ssl.go | 364 ++++++++++++++++-- .../perconaservermongodb/statefulset_test.go | 6 + .../perconaservermongodb/status_test.go | 10 +- .../perconaservermongodbbackup_controller.go | 24 +- .../perconaservermongodbrestore/logical.go | 2 +- .../perconaservermongodbrestore_controller.go | 4 +- .../perconaservermongodbrestore/physical.go | 2 +- pkg/psmdb/backup/pbm.go | 2 +- pkg/psmdb/mongos.go | 4 +- pkg/psmdb/statefulset.go | 4 +- pkg/psmdb/tls/certmanager.go | 151 +++++++- pkg/psmdb/tls/certmanager_test.go | 20 +- pkg/psmdb/tls/fake/certmanager.go | 67 ++++ pkg/psmdb/tls/pem.go | 60 +++ pkg/psmdb/tls/tls.go | 5 +- pkg/util/apply.go | 101 +++++ version/server.go | 12 +- 35 files changed, 914 insertions(+), 248 deletions(-) create mode 100644 pkg/psmdb/tls/fake/certmanager.go create mode 100644 pkg/psmdb/tls/pem.go create mode 100644 pkg/util/apply.go diff --git a/clientcmd/clientcmd.go b/clientcmd/clientcmd.go index 06bbd99957..83b50dc1ce 100644 --- a/clientcmd/clientcmd.go +++ b/clientcmd/clientcmd.go @@ -8,7 +8,6 @@ import ( "k8s.io/client-go/kubernetes/scheme" corev1client "k8s.io/client-go/kubernetes/typed/core/v1" restclient "k8s.io/client-go/rest" - "k8s.io/client-go/tools/clientcmd" "k8s.io/client-go/tools/remotecommand" ) @@ -17,29 +16,16 @@ type Client struct { restconfig *restclient.Config } -func NewClient() (*Client, error) { - // Instantiate loader for kubeconfig file. - kubeconfig := clientcmd.NewNonInteractiveDeferredLoadingClientConfig( - clientcmd.NewDefaultClientConfigLoadingRules(), - &clientcmd.ConfigOverrides{}, - ) - - // Get a rest.Config from the kubeconfig file. This will be passed into all - // the client objects we create. - restconfig, err := kubeconfig.ClientConfig() - if err != nil { - return nil, err - } - +func NewClient(config *restclient.Config) (*Client, error) { // Create a Kubernetes core/v1 client. - cl, err := corev1client.NewForConfig(restconfig) + cl, err := corev1client.NewForConfig(config) if err != nil { return nil, err } return &Client{ client: cl, - restconfig: restconfig, + restconfig: config, }, nil } diff --git a/e2e-tests/tls-issue-cert-manager/compare/certificate_some-name-ssl-internal.yml b/e2e-tests/tls-issue-cert-manager/compare/certificate_some-name-ssl-internal.yml index 5da1f33d73..192cc65056 100644 --- a/e2e-tests/tls-issue-cert-manager/compare/certificate_some-name-ssl-internal.yml +++ b/e2e-tests/tls-issue-cert-manager/compare/certificate_some-name-ssl-internal.yml @@ -1,6 +1,7 @@ apiVersion: cert-manager.io/v1 kind: Certificate metadata: + annotations: {} generation: 1 name: some-name-ssl-internal ownerReferences: diff --git a/e2e-tests/tls-issue-cert-manager/compare/certificate_some-name-ssl.yml b/e2e-tests/tls-issue-cert-manager/compare/certificate_some-name-ssl.yml index 485444ce02..514a5ba4bf 100644 --- a/e2e-tests/tls-issue-cert-manager/compare/certificate_some-name-ssl.yml +++ b/e2e-tests/tls-issue-cert-manager/compare/certificate_some-name-ssl.yml @@ -1,6 +1,7 @@ apiVersion: cert-manager.io/v1 kind: Certificate metadata: + annotations: {} generation: 1 name: some-name-ssl ownerReferences: diff --git a/e2e-tests/tls-issue-cert-manager/compare/issuer_some-name-psmdb-ca-issuer.yml b/e2e-tests/tls-issue-cert-manager/compare/issuer_some-name-psmdb-ca-issuer.yml index 1fc30a752e..b44c57cbd8 100644 --- a/e2e-tests/tls-issue-cert-manager/compare/issuer_some-name-psmdb-ca-issuer.yml +++ b/e2e-tests/tls-issue-cert-manager/compare/issuer_some-name-psmdb-ca-issuer.yml @@ -1,6 +1,7 @@ apiVersion: cert-manager.io/v1 kind: Issuer metadata: + annotations: {} generation: 1 name: some-name-psmdb-ca-issuer ownerReferences: diff --git a/e2e-tests/tls-issue-cert-manager/compare/issuer_some-name-psmdb-issuer.yml b/e2e-tests/tls-issue-cert-manager/compare/issuer_some-name-psmdb-issuer.yml index 28614ee5e2..6330059af5 100644 --- a/e2e-tests/tls-issue-cert-manager/compare/issuer_some-name-psmdb-issuer.yml +++ b/e2e-tests/tls-issue-cert-manager/compare/issuer_some-name-psmdb-issuer.yml @@ -1,6 +1,7 @@ apiVersion: cert-manager.io/v1 kind: Issuer metadata: + annotations: {} generation: 1 name: some-name-psmdb-issuer ownerReferences: diff --git a/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-cfg-1150-oc.yml b/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-cfg-1150-oc.yml index ffa28c009d..cdf4f58a37 100644 --- a/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-cfg-1150-oc.yml +++ b/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-cfg-1150-oc.yml @@ -2,7 +2,7 @@ apiVersion: apps/v1 kind: StatefulSet metadata: annotations: {} - generation: 7 + generation: 5 labels: app.kubernetes.io/component: cfg app.kubernetes.io/instance: some-name diff --git a/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-cfg-1150.yml b/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-cfg-1150.yml index 0d23cd3129..ef930dae59 100644 --- a/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-cfg-1150.yml +++ b/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-cfg-1150.yml @@ -2,7 +2,7 @@ apiVersion: apps/v1 kind: StatefulSet metadata: annotations: {} - generation: 7 + generation: 5 labels: app.kubernetes.io/component: cfg app.kubernetes.io/instance: some-name diff --git a/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-cfg-1160-oc.yml b/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-cfg-1160-oc.yml index 15e67406e5..90c8e42099 100644 --- a/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-cfg-1160-oc.yml +++ b/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-cfg-1160-oc.yml @@ -2,7 +2,7 @@ apiVersion: apps/v1 kind: StatefulSet metadata: annotations: {} - generation: 10 + generation: 8 labels: app.kubernetes.io/component: cfg app.kubernetes.io/instance: some-name diff --git a/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-cfg-1160.yml b/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-cfg-1160.yml index b46ac4a2cd..b8987ce42f 100644 --- a/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-cfg-1160.yml +++ b/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-cfg-1160.yml @@ -2,7 +2,7 @@ apiVersion: apps/v1 kind: StatefulSet metadata: annotations: {} - generation: 10 + generation: 8 labels: app.kubernetes.io/component: cfg app.kubernetes.io/instance: some-name diff --git a/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-rs0-1150-oc.yml b/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-rs0-1150-oc.yml index 57e2a6cd07..4fa22863e7 100644 --- a/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-rs0-1150-oc.yml +++ b/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-rs0-1150-oc.yml @@ -2,7 +2,7 @@ apiVersion: apps/v1 kind: StatefulSet metadata: annotations: {} - generation: 8 + generation: 5 labels: app.kubernetes.io/component: mongod app.kubernetes.io/instance: some-name diff --git a/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-rs0-1150.yml b/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-rs0-1150.yml index 3ee9b3294d..71cfbfcfbd 100644 --- a/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-rs0-1150.yml +++ b/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-rs0-1150.yml @@ -2,7 +2,7 @@ apiVersion: apps/v1 kind: StatefulSet metadata: annotations: {} - generation: 8 + generation: 5 labels: app.kubernetes.io/component: mongod app.kubernetes.io/instance: some-name diff --git a/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-rs0-1160-oc.yml b/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-rs0-1160-oc.yml index f099c7adb4..17ae8523cd 100644 --- a/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-rs0-1160-oc.yml +++ b/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-rs0-1160-oc.yml @@ -2,7 +2,7 @@ apiVersion: apps/v1 kind: StatefulSet metadata: annotations: {} - generation: 11 + generation: 8 labels: app.kubernetes.io/component: mongod app.kubernetes.io/instance: some-name diff --git a/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-rs0-1160.yml b/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-rs0-1160.yml index e214903219..bce22a650d 100644 --- a/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-rs0-1160.yml +++ b/e2e-tests/upgrade-consistency-sharded-tls/compare/statefulset_some-name-rs0-1160.yml @@ -2,7 +2,7 @@ apiVersion: apps/v1 kind: StatefulSet metadata: annotations: {} - generation: 11 + generation: 8 labels: app.kubernetes.io/component: mongod app.kubernetes.io/instance: some-name diff --git a/e2e-tests/upgrade-consistency-sharded-tls/run b/e2e-tests/upgrade-consistency-sharded-tls/run index 2d3e9e1c5b..8e6f949436 100755 --- a/e2e-tests/upgrade-consistency-sharded-tls/run +++ b/e2e-tests/upgrade-consistency-sharded-tls/run @@ -33,8 +33,8 @@ main() { compare_generation "1" "statefulset" "${CLUSTER}-rs0" compare_generation "1" "statefulset" "${CLUSTER}-cfg" - # TODO: uncomment when 1.14.0 will be removed, - # renewal doesn't work on "1.14.0" version + # Renewal doesn't work on "1.14.0" version + # #renew_certificate "some-name-ssl" #renew_certificate "some-name-ssl-internal" #wait_cluster @@ -46,29 +46,16 @@ main() { compare_kubectl statefulset/${CLUSTER}-cfg "-1140" desc 'test 1.15.0' - # workaround to switch to updated certificate structure - # more details: https://github.com/percona/percona-server-mongodb-operator/pull/1287 - # TODO: remove the workaround when 1.14.0 will be removed - stop_cluster $CLUSTER - - compare_generation "4" "statefulset" "${CLUSTER}-rs0" - compare_generation "3" "statefulset" "${CLUSTER}-cfg" - kubectl_bin patch psmdb "${CLUSTER}" --type=merge --patch '{ "spec": {"crVersion":"1.15.0"} }' # Wait for at least one reconciliation sleep 20 + desc 'check if Pod started' + wait_cluster - compare_generation "5" "statefulset" "${CLUSTER}-rs0" - compare_generation "4" "statefulset" "${CLUSTER}-cfg" - - kubectl_bin delete certificate "$CLUSTER"-ssl "$CLUSTER"-ssl-internal - kubectl_bin delete issuer "$CLUSTER-psmdb-ca" - kubectl_bin delete secret "$CLUSTER"-ssl "$CLUSTER"-ssl-internal - start_cluster $CLUSTER - compare_generation "6" "statefulset" "${CLUSTER}-rs0" - compare_generation "5" "statefulset" "${CLUSTER}-cfg" + compare_generation "3" "statefulset" "${CLUSTER}-rs0" + compare_generation "3" "statefulset" "${CLUSTER}-cfg" # Wait for at least one reconciliation sleep 20 @@ -78,14 +65,14 @@ main() { renew_certificate "some-name-ssl" sleep 20 wait_cluster - compare_generation "7" "statefulset" "${CLUSTER}-rs0" - compare_generation "6" "statefulset" "${CLUSTER}-cfg" + compare_generation "4" "statefulset" "${CLUSTER}-rs0" + compare_generation "4" "statefulset" "${CLUSTER}-cfg" renew_certificate "some-name-ssl-internal" sleep 20 wait_cluster - compare_generation "8" "statefulset" "${CLUSTER}-rs0" - compare_generation "7" "statefulset" "${CLUSTER}-cfg" + compare_generation "5" "statefulset" "${CLUSTER}-rs0" + compare_generation "5" "statefulset" "${CLUSTER}-cfg" desc 'check if service and statefulset created with expected config' compare_kubectl service/${CLUSTER}-rs0 "-1150" @@ -101,20 +88,20 @@ main() { sleep 20 desc 'check if Pod started' wait_cluster - compare_generation "9" "statefulset" "${CLUSTER}-rs0" - compare_generation "8" "statefulset" "${CLUSTER}-cfg" + compare_generation "6" "statefulset" "${CLUSTER}-rs0" + compare_generation "6" "statefulset" "${CLUSTER}-cfg" renew_certificate "some-name-ssl" sleep 20 wait_cluster - compare_generation "10" "statefulset" "${CLUSTER}-rs0" - compare_generation "9" "statefulset" "${CLUSTER}-cfg" + compare_generation "7" "statefulset" "${CLUSTER}-rs0" + compare_generation "7" "statefulset" "${CLUSTER}-cfg" renew_certificate "some-name-ssl-internal" sleep 20 wait_cluster - compare_generation "11" "statefulset" "${CLUSTER}-rs0" - compare_generation "10" "statefulset" "${CLUSTER}-cfg" + compare_generation "8" "statefulset" "${CLUSTER}-rs0" + compare_generation "8" "statefulset" "${CLUSTER}-cfg" desc 'check if service and statefulset created with expected config' compare_kubectl service/${CLUSTER}-rs0 "-1160" diff --git a/pkg/apis/psmdb/v1/perconaservermongodbrestore_types.go b/pkg/apis/psmdb/v1/perconaservermongodbrestore_types.go index ceed40d5f2..318c0cbc7c 100644 --- a/pkg/apis/psmdb/v1/perconaservermongodbrestore_types.go +++ b/pkg/apis/psmdb/v1/perconaservermongodbrestore_types.go @@ -157,4 +157,8 @@ var ( PITRestoreTypeLatest PITRestoreType = "latest" ) -const AnnotationRestoreInProgress = "percona.com/restore-in-progress" +const ( + AnnotationRestoreInProgress = "percona.com/restore-in-progress" + // AnnotationUpdateMongosFirst is an annotation used to force next smart update to be applied to mongos before mongod. + AnnotationUpdateMongosFirst = "percona.com/update-mongos-first" +) diff --git a/pkg/apis/psmdb/v1/psmdb_types.go b/pkg/apis/psmdb/v1/psmdb_types.go index deb92a41ff..2f71fc03c1 100644 --- a/pkg/apis/psmdb/v1/psmdb_types.go +++ b/pkg/apis/psmdb/v1/psmdb_types.go @@ -699,6 +699,14 @@ type SecretsSpec struct { LDAPSecret string `json:"ldapSecret,omitempty"` } +func SSLSecretName(cr *PerconaServerMongoDB) string { + return cr.Spec.Secrets.SSL +} + +func SSLInternalSecretName(cr *PerconaServerMongoDB) string { + return cr.Spec.Secrets.SSLInternal +} + type MongosSpec struct { MultiAZ `json:",inline"` diff --git a/pkg/controller/perconaservermongodb/psmdb_controller.go b/pkg/controller/perconaservermongodb/psmdb_controller.go index 311e81c3ef..6b43ca08c3 100644 --- a/pkg/controller/perconaservermongodb/psmdb_controller.go +++ b/pkg/controller/perconaservermongodb/psmdb_controller.go @@ -3,11 +3,8 @@ package perconaservermongodb import ( "context" "crypto/md5" - "encoding/base64" - "encoding/json" "fmt" "os" - "reflect" "strconv" "strings" "sync" @@ -25,6 +22,7 @@ import ( "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/rest" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/apiutil" "sigs.k8s.io/controller-runtime/pkg/client/config" @@ -40,6 +38,7 @@ import ( "github.com/percona/percona-server-mongodb-operator/pkg/psmdb" "github.com/percona/percona-server-mongodb-operator/pkg/psmdb/backup" "github.com/percona/percona-server-mongodb-operator/pkg/psmdb/secret" + "github.com/percona/percona-server-mongodb-operator/pkg/psmdb/tls" "github.com/percona/percona-server-mongodb-operator/pkg/util" "github.com/percona/percona-server-mongodb-operator/version" ) @@ -59,31 +58,33 @@ func Add(mgr manager.Manager) error { // newReconciler returns a new reconcile.Reconciler func newReconciler(mgr manager.Manager) (reconcile.Reconciler, error) { - sv, err := version.Server() + cli, err := clientcmd.NewClient(mgr.GetConfig()) if err != nil { - return nil, errors.Wrap(err, "get server version") + return nil, errors.Wrap(err, "create clientcmd") } - mgr.GetLogger().Info("server version", "platform", sv.Platform, "version", sv.Info) - - cli, err := clientcmd.NewClient() + sv, err := version.Server(cli) if err != nil { - return nil, errors.Wrap(err, "create clientcmd") + return nil, errors.Wrap(err, "get server version") } + mgr.GetLogger().Info("server version", "platform", sv.Platform, "version", sv.Info) + initImage, err := getOperatorPodImage(context.TODO()) if err != nil { return nil, errors.Wrap(err, "failed to get operator pod image") } return &ReconcilePerconaServerMongoDB{ - client: mgr.GetClient(), - scheme: mgr.GetScheme(), - serverVersion: sv, - reconcileIn: time.Second * 5, - crons: NewCronRegistry(), - lockers: newLockStore(), - newPBM: backup.NewPBM, + client: mgr.GetClient(), + scheme: mgr.GetScheme(), + serverVersion: sv, + reconcileIn: time.Second * 5, + crons: NewCronRegistry(), + lockers: newLockStore(), + newPBM: backup.NewPBM, + restConfig: mgr.GetConfig(), + newCertManagerCtrlFunc: tls.NewCertManagerController, initImage: initImage, @@ -164,8 +165,9 @@ var _ reconcile.Reconciler = &ReconcilePerconaServerMongoDB{} type ReconcilePerconaServerMongoDB struct { // This client, initialized using mgr.Client() above, is a split client // that reads objects from the cache and writes to the apiserver - client client.Client - scheme *runtime.Scheme + client client.Client + scheme *runtime.Scheme + restConfig *rest.Config crons CronRegistry clientcmd *clientcmd.Client @@ -173,6 +175,8 @@ type ReconcilePerconaServerMongoDB struct { reconcileIn time.Duration mongoClientProvider MongoClientProvider + newCertManagerCtrlFunc tls.NewCertManagerControllerFunc + newPBM backup.NewPBMFunc initImage string @@ -352,7 +356,7 @@ func (r *ReconcilePerconaServerMongoDB) Reconcile(ctx context.Context, request r err = r.reconcileSSL(ctx, cr) if err != nil { - err = errors.Errorf(`TLS secrets handler: "%v". Please create your TLS secret `+cr.Spec.Secrets.SSL+` manually or setup cert-manager correctly`, err) + err = errors.Errorf(`TLS secrets handler: "%v". Please create your TLS secret `+api.SSLSecretName(cr)+` manually or setup cert-manager correctly`, err) return reconcile.Result{}, err } @@ -1131,7 +1135,11 @@ func (r *ReconcilePerconaServerMongoDB) reconcileMongosStatefulset(ctx context.C return errors.Wrap(err, "failed to check running restores") } - if !uptodate || rstRunning { + mongosFirst, err := r.shouldUpdateMongosFirst(ctx, cr) + if err != nil { + return errors.Wrap(err, "should update mongos first") + } + if (!uptodate && !mongosFirst) || rstRunning { return nil } @@ -1181,6 +1189,9 @@ func (r *ReconcilePerconaServerMongoDB) reconcileMongosStatefulset(ctx context.C if cr.TLSEnabled() { sslAnn, err := r.sslAnnotation(ctx, cr) if err != nil { + if err == errTLSNotReady { + return nil + } return errors.Wrap(err, "failed to get ssl annotations") } if templateSpec.Annotations == nil { @@ -1325,24 +1336,28 @@ func ensurePVCs( return nil } +var errTLSNotReady = errors.New("waiting for TLS secret") + func (r *ReconcilePerconaServerMongoDB) sslAnnotation(ctx context.Context, cr *api.PerconaServerMongoDB) (map[string]string, error) { annotation := make(map[string]string) - is110 := cr.CompareVersion("1.1.0") >= 0 - if is110 { - sslHash, err := r.getTLSHash(ctx, cr, cr.Spec.Secrets.SSL) - if err != nil { - return nil, errors.Wrap(err, "get secret hash error") + sslHash, err := r.getTLSHash(ctx, cr, api.SSLSecretName(cr)) + if err != nil { + if k8serrors.IsNotFound(err) { + return nil, errTLSNotReady } - annotation["percona.com/ssl-hash"] = sslHash + return nil, errors.Wrap(err, "get secret hash error") + } + annotation["percona.com/ssl-hash"] = sslHash - sslInternalHash, err := r.getTLSHash(ctx, cr, cr.Spec.Secrets.SSLInternal) - if err != nil && !k8serrors.IsNotFound(err) { - return nil, errors.Wrap(err, "get secret hash error") - } else if err == nil { - annotation["percona.com/ssl-internal-hash"] = sslInternalHash + sslInternalHash, err := r.getTLSHash(ctx, cr, api.SSLInternalSecretName(cr)) + if err != nil { + if k8serrors.IsNotFound(err) { + return nil, errTLSNotReady } + return nil, errors.Wrap(err, "get secret hash error") } + annotation["percona.com/ssl-internal-hash"] = sslInternalHash return annotation, nil } @@ -1397,55 +1412,8 @@ func (r *ReconcilePerconaServerMongoDB) reconcilePDB(ctx context.Context, spec * } func (r *ReconcilePerconaServerMongoDB) createOrUpdate(ctx context.Context, obj client.Object) error { - if obj.GetAnnotations() == nil { - obj.SetAnnotations(make(map[string]string)) - } - - objAnnotations := obj.GetAnnotations() - delete(objAnnotations, "percona.com/last-config-hash") - obj.SetAnnotations(objAnnotations) - - hash, err := getObjectHash(obj) - if err != nil { - return errors.Wrap(err, "calculate object hash") - } - - objAnnotations = obj.GetAnnotations() - objAnnotations["percona.com/last-config-hash"] = hash - obj.SetAnnotations(objAnnotations) - - val := reflect.ValueOf(obj) - if val.Kind() == reflect.Ptr { - val = reflect.Indirect(val) - } - oldObject := reflect.New(val.Type()).Interface().(client.Object) - - err = r.client.Get(ctx, types.NamespacedName{ - Name: obj.GetName(), - Namespace: obj.GetNamespace(), - }, oldObject) - - if err != nil && !k8serrors.IsNotFound(err) { - return errors.Wrap(err, "get object") - } - - if k8serrors.IsNotFound(err) { - return r.client.Create(ctx, obj) - } - - if oldObject.GetAnnotations()["percona.com/last-config-hash"] != hash || - !util.MapEqual(oldObject.GetLabels(), obj.GetLabels()) || - !util.MapEqual(oldObject.GetAnnotations(), obj.GetAnnotations()) { - obj.SetResourceVersion(oldObject.GetResourceVersion()) - switch object := obj.(type) { - case *corev1.Service: - object.Spec.ClusterIP = oldObject.(*corev1.Service).Spec.ClusterIP - } - - return r.client.Update(ctx, obj) - } - - return nil + _, err := util.Apply(ctx, r.client, obj) + return err } func (r *ReconcilePerconaServerMongoDB) createOrUpdateSvc(ctx context.Context, cr *api.PerconaServerMongoDB, svc *corev1.Service, saveOldMeta bool) error { @@ -1497,27 +1465,6 @@ func setIgnoredLabels(cr *api.PerconaServerMongoDB, obj, oldObject client.Object obj.SetLabels(labels) } -func getObjectHash(obj client.Object) (string, error) { - var dataToMarshall interface{} - switch object := obj.(type) { - case *appsv1.StatefulSet: - dataToMarshall = object.Spec - case *appsv1.Deployment: - dataToMarshall = object.Spec - case *corev1.Service: - dataToMarshall = object.Spec - case *corev1.Secret: - dataToMarshall = object.Data - default: - dataToMarshall = obj - } - data, err := json.Marshal(dataToMarshall) - if err != nil { - return "", err - } - return base64.StdEncoding.EncodeToString(data), nil -} - func setControllerReference(owner client.Object, obj metav1.Object, scheme *runtime.Scheme) error { ownerRef, err := OwnerRef(owner, scheme) if err != nil { diff --git a/pkg/controller/perconaservermongodb/smart.go b/pkg/controller/perconaservermongodb/smart.go index bd2457d90e..fd93a1be71 100644 --- a/pkg/controller/perconaservermongodb/smart.go +++ b/pkg/controller/perconaservermongodb/smart.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "sort" + "strings" "time" "github.com/pkg/errors" @@ -12,6 +13,7 @@ import ( k8sErrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/util/retry" k8sclient "sigs.k8s.io/controller-runtime/pkg/client" logf "sigs.k8s.io/controller-runtime/pkg/log" @@ -21,8 +23,8 @@ import ( ) func (r *ReconcilePerconaServerMongoDB) smartUpdate(ctx context.Context, cr *api.PerconaServerMongoDB, sfs *appsv1.StatefulSet, - replset *api.ReplsetSpec) error { - + replset *api.ReplsetSpec, +) error { log := logf.FromContext(ctx) if replset.Size == 0 { return nil @@ -64,6 +66,14 @@ func (r *ReconcilePerconaServerMongoDB) smartUpdate(ctx context.Context, cr *api return nil } + mongosFirst, err := r.shouldUpdateMongosFirst(ctx, cr) + if err != nil { + return errors.Wrap(err, "should update mongos first") + } + if mongosFirst { + return nil + } + if cr.Spec.Sharding.Enabled && sfs.Name != cr.Name+"-"+api.ConfigReplSetName { cfgSfs := appsv1.StatefulSet{} err := r.client.Get(ctx, types.NamespacedName{Name: cr.Name + "-" + api.ConfigReplSetName, Namespace: cr.Namespace}, &cfgSfs) @@ -167,7 +177,14 @@ func (r *ReconcilePerconaServerMongoDB) smartUpdate(ctx context.Context, cr *api err = client.StepDown(ctx, 60, forceStepDown) if err != nil { - return errors.Wrap(err, "failed to do step down") + if strings.Contains(err.Error(), "No electable secondaries caught up") { + err = client.StepDown(ctx, 60, true) + if err != nil { + return errors.Wrap(err, "failed to do forced step down") + } + } else { + return errors.Wrap(err, "failed to do step down") + } } log.Info("apply changes to primary pod", "pod", primaryPod.Name) @@ -181,6 +198,56 @@ func (r *ReconcilePerconaServerMongoDB) smartUpdate(ctx context.Context, cr *api return nil } +func (r *ReconcilePerconaServerMongoDB) shouldUpdateMongosFirst(ctx context.Context, cr *api.PerconaServerMongoDB) (bool, error) { + if !cr.Spec.Sharding.Enabled { + return false, nil + } + + c := new(api.PerconaServerMongoDB) + if err := r.client.Get(ctx, types.NamespacedName{Name: cr.Name, Namespace: cr.Namespace}, c); err != nil { + return false, errors.Wrap(err, "failed to get cr") + } + + _, ok := c.Annotations[api.AnnotationUpdateMongosFirst] + return ok, nil +} + +func (r *ReconcilePerconaServerMongoDB) setUpdateMongosFirst(ctx context.Context, cr *api.PerconaServerMongoDB) error { + if !cr.Spec.Sharding.Enabled { + return nil + } + + return retry.RetryOnConflict(retry.DefaultBackoff, func() error { + c := new(api.PerconaServerMongoDB) + if err := r.client.Get(ctx, types.NamespacedName{Name: cr.Name, Namespace: cr.Namespace}, c); err != nil { + return err + } + + c.Annotations[api.AnnotationUpdateMongosFirst] = "true" + + return r.client.Update(ctx, c) + }) +} + +func (r *ReconcilePerconaServerMongoDB) unsetUpdateMongosFirst(ctx context.Context, cr *api.PerconaServerMongoDB) error { + if !cr.Spec.Sharding.Enabled { + return nil + } + return retry.RetryOnConflict(retry.DefaultBackoff, func() error { + c := new(api.PerconaServerMongoDB) + if err := r.client.Get(ctx, types.NamespacedName{Name: cr.Name, Namespace: cr.Namespace}, c); err != nil { + return err + } + if _, ok := c.Annotations[api.AnnotationUpdateMongosFirst]; !ok { + return nil + } + + delete(c.Annotations, api.AnnotationUpdateMongosFirst) + + return r.client.Update(ctx, c) + }) +} + func (r *ReconcilePerconaServerMongoDB) setPrimary(ctx context.Context, cr *api.PerconaServerMongoDB, rs *api.ReplsetSpec, expectedPrimary corev1.Pod) error { primary, err := r.isPodPrimary(ctx, cr, expectedPrimary, rs) if err != nil { @@ -348,6 +415,9 @@ func (r *ReconcilePerconaServerMongoDB) smartMongosUpdate(ctx context.Context, c return errors.Wrap(err, "failed to apply changes") } } + if err := r.unsetUpdateMongosFirst(ctx, cr); err != nil { + return errors.Wrap(err, "unset update mongos first") + } log.Info("smart update finished for mongos statefulset") return nil diff --git a/pkg/controller/perconaservermongodb/ssl.go b/pkg/controller/perconaservermongodb/ssl.go index 6232b85954..edb47fa253 100644 --- a/pkg/controller/perconaservermongodb/ssl.go +++ b/pkg/controller/perconaservermongodb/ssl.go @@ -1,17 +1,22 @@ package perconaservermongodb import ( + "bytes" "context" "github.com/pkg/errors" + appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" k8serr "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" logf "sigs.k8s.io/controller-runtime/pkg/log" api "github.com/percona/percona-server-mongodb-operator/pkg/apis/psmdb/v1" "github.com/percona/percona-server-mongodb-operator/pkg/psmdb/tls" + "github.com/percona/percona-server-mongodb-operator/pkg/util" ) func (r *ReconcilePerconaServerMongoDB) reconcileSSL(ctx context.Context, cr *api.PerconaServerMongoDB) error { @@ -24,20 +29,18 @@ func (r *ReconcilePerconaServerMongoDB) reconcileSSL(ctx context.Context, cr *ap errSecret := r.client.Get(ctx, types.NamespacedName{ Namespace: cr.Namespace, - Name: cr.Spec.Secrets.SSL, + Name: api.SSLSecretName(cr), }, &secretObj, ) errInternalSecret := r.client.Get(ctx, types.NamespacedName{ Namespace: cr.Namespace, - Name: cr.Spec.Secrets.SSL + "-internal", + Name: api.SSLInternalSecretName(cr), }, &secretInternalObj, ) - if errSecret == nil && errInternalSecret == nil { - return nil - } else if errSecret == nil && k8serr.IsNotFound(errInternalSecret) && !metav1.IsControlledBy(&secretObj, cr) { + if errSecret == nil && k8serr.IsNotFound(errInternalSecret) && !metav1.IsControlledBy(&secretObj, cr) { // don't create secret ssl-internal if secret ssl is not created by operator return nil } else if errSecret != nil && !k8serr.IsNotFound(errSecret) { @@ -46,57 +49,358 @@ func (r *ReconcilePerconaServerMongoDB) reconcileSSL(ctx context.Context, cr *ap return errors.Wrap(errInternalSecret, "get internal SSL secret") } - err := r.createSSLByCertManager(ctx, cr) + ok, err := r.isCertManagerInstalled(ctx, cr.Namespace) if err != nil { - logf.FromContext(ctx).Error(err, "issue cert with cert-manager") + return errors.Wrap(err, "check cert-manager") + } + if !ok { + if errSecret == nil && errInternalSecret == nil { + return nil + } err = r.createSSLManually(ctx, cr) if err != nil { return errors.Wrap(err, "create ssl manually") } + return nil + } + err = r.createSSLByCertManager(ctx, cr) + if err != nil { + return errors.Wrap(err, "create ssl by cert-manager") } return nil } +func (r *ReconcilePerconaServerMongoDB) isCertManagerInstalled(ctx context.Context, ns string) (bool, error) { + c := r.newCertManagerCtrlFunc(r.client, r.scheme, true) + err := c.Check(ctx, r.restConfig, ns) + if err != nil { + switch err { + case tls.ErrCertManagerNotFound: + return false, nil + case tls.ErrCertManagerNotReady: + return true, nil + } + return false, err + } + return true, nil +} + +func (r *ReconcilePerconaServerMongoDB) doAllStsHasLatestTLS(ctx context.Context, cr *api.PerconaServerMongoDB) (bool, error) { + sfsList := appsv1.StatefulSetList{} + if err := r.client.List(ctx, &sfsList, + &client.ListOptions{ + Namespace: cr.Namespace, + LabelSelector: labels.SelectorFromSet(map[string]string{ + "app.kubernetes.io/instance": cr.Name, + }), + }, + ); err != nil { + return false, errors.Wrap(err, "failed to get statefulset list") + } + + sslAnn, err := r.sslAnnotation(ctx, cr) + if err != nil { + if err == errTLSNotReady { + return false, nil + } + return false, errors.Wrap(err, "failed to get ssl annotations") + } + for _, sts := range sfsList.Items { + for k, v := range sslAnn { + if sts.Spec.Template.Annotations[k] != v { + return false, nil + } + } + } + return true, nil +} + func (r *ReconcilePerconaServerMongoDB) createSSLByCertManager(ctx context.Context, cr *api.PerconaServerMongoDB) error { - c := tls.NewCertManagerController(r.client, r.scheme) + log := logf.FromContext(ctx).WithName("createSSLByCertManager") + + dryController := r.newCertManagerCtrlFunc(r.client, r.scheme, true) + // checking if certificates will be updated + applyStatus, err := r.applyCertManagerCertificates(ctx, cr, dryController) + if err != nil { + return errors.Wrap(err, "apply cert-manager certificates") + } + + if applyStatus == util.ApplyStatusUnchanged { + // If we have merged the old CA and all sts are ready, + // we should recreate the secrets by deleting them. + uptodate, err := r.isAllSfsUpToDate(ctx, cr) + if err != nil { + return errors.Wrap(err, "check sfs") + } + // These sts should also have latest tls secrets + hasSSL, err := r.doAllStsHasLatestTLS(ctx, cr) + if err != nil { + return errors.Wrap(err, "has ssl") + } + if uptodate && hasSSL { + secretNames := []string{ + api.SSLInternalSecretName(cr), + api.SSLSecretName(cr), + } + // We should be sure that old CA is merged. + // mergeNewCA will delete old secrets if they are not needed. + for _, name := range secretNames { + _, err := r.getSecret(ctx, cr, name+"-old") + if client.IgnoreNotFound(err) != nil { + return errors.Wrap(err, "get secret") + } + if err != nil { + continue + } + log.Info("Old secret exists, merging ca", "secret", name+"-old") + if err := r.mergeNewCA(ctx, cr); err != nil { + return errors.Wrap(err, "update secrets with old ones") + } + return nil + } + + caSecret, err := r.getSecret(ctx, cr, tls.CACertificateSecretName(cr)) + if err != nil { + if k8serr.IsNotFound(err) { + return nil + } + return errors.Wrap(err, "failed to get ca secret") + } + + for _, name := range secretNames { + secret, err := r.getSecret(ctx, cr, name) + if err != nil { + if k8serr.IsNotFound(err) { + continue + } + return errors.Wrap(err, "get secret") + } + + if bytes.Equal(secret.Data["ca.crt"], caSecret.Data["ca.crt"]) { + continue + } + + // Mongos pods will only accept the first part of the CA. + // After the secret recreation, all mongod pods will have the last part of the CA + // and mongos won't be able to connect to them. + // So we should update the mongos pods before the mongod pods. + if err := r.setUpdateMongosFirst(ctx, cr); err != nil { + return errors.Wrap(err, "set update mongos first") + } + + log.Info("CA is not up to date. Recreating secret", "secret", secret.Name) + if err := r.client.Delete(ctx, secret); err != nil { + return err + } + } + } + + return nil + } + + log.Info("updating cert-manager certificates") + + if err := r.updateCertManagerCerts(ctx, cr); err != nil { + return errors.Wrap(err, "update cert mangager certs") + } + + c := r.newCertManagerCtrlFunc(r.client, r.scheme, false) + if cr.CompareVersion("1.15.0") >= 0 { + if err := c.DeleteDeprecatedIssuerIfExists(ctx, cr); err != nil { + return errors.Wrap(err, "delete deprecated issuer") + } + } + return nil +} + +func (r *ReconcilePerconaServerMongoDB) getSecret(ctx context.Context, cr *api.PerconaServerMongoDB, name string) (*corev1.Secret, error) { + secret := new(corev1.Secret) + err := r.client.Get(ctx, + types.NamespacedName{ + Namespace: cr.Namespace, + Name: name, + }, + secret, + ) + if err != nil { + return nil, err + } + return secret, nil +} +func (r *ReconcilePerconaServerMongoDB) updateCertManagerCerts(ctx context.Context, cr *api.PerconaServerMongoDB) error { + log := logf.FromContext(ctx) + + secrets := []string{ + api.SSLSecretName(cr), + api.SSLInternalSecretName(cr), + } + log.Info("Creating old secrets") + for _, name := range secrets { + secret, err := r.getSecret(ctx, cr, name) + if err != nil { + if k8serr.IsNotFound(err) { + continue + } + return errors.Wrap(err, "get secret") + } + newSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: secret.Name + "-old", + Namespace: secret.Namespace, + }, + Data: secret.Data, + } + + if err := r.client.Create(ctx, newSecret); err != nil { + return errors.Wrap(err, "create secret") + } + } + + c := r.newCertManagerCtrlFunc(r.client, r.scheme, false) + log.Info("applying new certificates") + if _, err := r.applyCertManagerCertificates(ctx, cr, c); err != nil { + return errors.Wrap(err, "failed to apply cert-manager certificates") + } + + log.Info("migrating new ca") + if err := r.mergeNewCA(ctx, cr); err != nil { + return errors.Wrap(err, "update secrets with old ones") + } + return nil +} + +// mergeNewCA overwrites current ssl secrets with the old ones, but merges ca.crt from the current secret +func (r *ReconcilePerconaServerMongoDB) mergeNewCA(ctx context.Context, cr *api.PerconaServerMongoDB) error { + log := logf.FromContext(ctx) + c := tls.NewCertManagerController(r.client, r.scheme, false) + // In versions 1.14.0 and below, these secrets contained different ca.crt + oldCA, err := c.GetMergedCA(ctx, cr, []string{ + api.SSLInternalSecretName(cr) + "-old", + api.SSLSecretName(cr) + "-old", + }) + if err != nil { + return errors.Wrap(err, "get old ca") + } + if len(oldCA) == 0 { + return nil + } + + secretNames := []string{ + api.SSLInternalSecretName(cr), + api.SSLSecretName(cr), + } + + newCA, err := c.GetMergedCA(ctx, cr, secretNames) + if err != nil { + return errors.Wrap(err, "get new ca") + } + + for _, secretName := range secretNames { + secret, err := r.getSecret(ctx, cr, secretName) + if err != nil { + if k8serr.IsNotFound(err) { + continue + } + return errors.Wrap(err, "get ca secret") + } + oldSecret, err := r.getSecret(ctx, cr, secretName+"-old") + if err != nil { + if k8serr.IsNotFound(err) { + continue + } + return errors.Wrap(err, "get ca secret") + } + + mergedCA, err := tls.MergePEM(oldCA, newCA) + if err != nil { + return errors.Wrap(err, "failed to merge ca") + } + + // If secret was already updated, we should delete the old one + if bytes.Equal(mergedCA, secret.Data["ca.crt"]) { + if err := r.client.Delete(ctx, oldSecret); err != nil { + return err + } + log.Info("new ca is already in secret, deleting old secret") + continue + } + + secret.Data = oldSecret.Data + secret.Data["ca.crt"] = mergedCA + + if err := r.client.Update(ctx, secret); err != nil { + return errors.Wrap(err, "update ca secret") + } + } + + return nil +} + +func (r *ReconcilePerconaServerMongoDB) applyCertManagerCertificates(ctx context.Context, cr *api.PerconaServerMongoDB, c tls.CertManagerController) (util.ApplyStatus, error) { + applyStatus := util.ApplyStatusUnchanged + applyFunc := func(f func() (util.ApplyStatus, error)) error { + status, err := f() + if err != nil { + return err + } + if status != util.ApplyStatusUnchanged { + applyStatus = status + } + return nil + } if cr.CompareVersion("1.15.0") >= 0 { - err := c.CreateCAIssuer(ctx, cr) - if err != nil && !k8serr.IsAlreadyExists(err) { - return errors.Wrap(err, "create ca issuer") + err := applyFunc(func() (util.ApplyStatus, error) { + return c.ApplyCAIssuer(ctx, cr) + }) + if err != nil { + return "", errors.Wrap(err, "apply ca issuer") } - err = c.CreateCACertificate(ctx, cr) - if err != nil && !k8serr.IsAlreadyExists(err) { - return errors.Wrap(err, "create ca certificate") + err = applyFunc(func() (util.ApplyStatus, error) { + return c.ApplyCACertificate(ctx, cr) + }) + if err != nil { + return "", errors.Wrap(err, "create ca certificate") } err = c.WaitForCerts(ctx, cr, tls.CACertificateSecretName(cr)) if err != nil { - return errors.Wrap(err, "failed to wait for ca cert") + return "", errors.Wrap(err, "failed to wait for ca cert") } } - err := c.CreateIssuer(ctx, cr) - if err != nil && !k8serr.IsAlreadyExists(err) { - return errors.Wrap(err, "create issuer") + err := applyFunc(func() (util.ApplyStatus, error) { + return c.ApplyIssuer(ctx, cr) + }) + if err != nil { + return "", errors.Wrap(err, "create issuer") } - err = c.CreateCertificate(ctx, cr, false) - if err != nil && !k8serr.IsAlreadyExists(err) { - return errors.Wrap(err, "create certificate") + err = applyFunc(func() (util.ApplyStatus, error) { + return c.ApplyCertificate(ctx, cr, false) + }) + if err != nil { + return "", errors.Wrap(err, "create certificate") } - if tls.CertificateSecretName(cr, false) == tls.CertificateSecretName(cr, true) { - return c.WaitForCerts(ctx, cr, tls.CertificateSecretName(cr, false)) - } + secretNames := []string{tls.CertificateSecretName(cr, false)} - err = c.CreateCertificate(ctx, cr, true) - if err != nil && !k8serr.IsAlreadyExists(err) { - return errors.Wrap(err, "create certificate") + if tls.CertificateSecretName(cr, false) != tls.CertificateSecretName(cr, true) { + err = applyFunc(func() (util.ApplyStatus, error) { + return c.ApplyCertificate(ctx, cr, true) + }) + if err != nil { + return "", errors.Wrap(err, "create certificate") + } + secretNames = append(secretNames, tls.CertificateSecretName(cr, true)) } - return c.WaitForCerts(ctx, cr, tls.CertificateSecretName(cr, false), tls.CertificateSecretName(cr, true)) + err = c.WaitForCerts(ctx, cr, secretNames...) + if err != nil { + return "", errors.Wrap(err, "failed to wait for certs") + } + return applyStatus, nil } func (r *ReconcilePerconaServerMongoDB) createSSLManually(ctx context.Context, cr *api.PerconaServerMongoDB) error { @@ -119,7 +423,7 @@ func (r *ReconcilePerconaServerMongoDB) createSSLManually(ctx context.Context, c secretObj := corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ - Name: cr.Spec.Secrets.SSL, + Name: api.SSLSecretName(cr), Namespace: cr.Namespace, OwnerReferences: ownerReferences, }, @@ -140,7 +444,7 @@ func (r *ReconcilePerconaServerMongoDB) createSSLManually(ctx context.Context, c data["tls.key"] = key secretObjInternal := corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ - Name: cr.Spec.Secrets.SSLInternal, + Name: api.SSLInternalSecretName(cr), Namespace: cr.Namespace, OwnerReferences: ownerReferences, }, diff --git a/pkg/controller/perconaservermongodb/statefulset_test.go b/pkg/controller/perconaservermongodb/statefulset_test.go index 5ee77575ba..ea961150c3 100644 --- a/pkg/controller/perconaservermongodb/statefulset_test.go +++ b/pkg/controller/perconaservermongodb/statefulset_test.go @@ -90,6 +90,11 @@ func TestReconcileStatefulSet(t *testing.T) { Name: crName + "-ssl", Namespace: tt.cr.Namespace, }, + }, &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: crName + "-ssl-internal", + Namespace: tt.cr.Namespace, + }, }) rs := tt.cr.Spec.Replset(tt.rsName) @@ -156,6 +161,7 @@ func compareSts(t *testing.T, got, want *appsv1.StatefulSet) { compareSpec := func(got, want appsv1.StatefulSetSpec) { delete(got.Template.Annotations, "percona.com/ssl-hash") + delete(got.Template.Annotations, "percona.com/ssl-internal-hash") gotBytes, err := yaml.Marshal(got) if err != nil { t.Fatalf("error marshaling got: %v", err) diff --git a/pkg/controller/perconaservermongodb/status_test.go b/pkg/controller/perconaservermongodb/status_test.go index 3fab61531c..33cab8284b 100644 --- a/pkg/controller/perconaservermongodb/status_test.go +++ b/pkg/controller/perconaservermongodb/status_test.go @@ -16,6 +16,7 @@ import ( api "github.com/percona/percona-server-mongodb-operator/pkg/apis/psmdb/v1" fakeBackup "github.com/percona/percona-server-mongodb-operator/pkg/psmdb/backup/fake" + faketls "github.com/percona/percona-server-mongodb-operator/pkg/psmdb/tls/fake" "github.com/percona/percona-server-mongodb-operator/version" ) @@ -38,10 +39,11 @@ func buildFakeClient(objs ...client.Object) *ReconcilePerconaServerMongoDB { cl := fake.NewClientBuilder().WithScheme(s).WithObjects(objs...).WithStatusSubresource(objs...).Build() return &ReconcilePerconaServerMongoDB{ - client: cl, - scheme: s, - lockers: newLockStore(), - newPBM: fakeBackup.NewPBM, + client: cl, + scheme: s, + lockers: newLockStore(), + newPBM: fakeBackup.NewPBM, + newCertManagerCtrlFunc: faketls.NewCertManagerController, } } diff --git a/pkg/controller/perconaservermongodbbackup/perconaservermongodbbackup_controller.go b/pkg/controller/perconaservermongodbbackup/perconaservermongodbbackup_controller.go index 2f62e88e0e..e973b81372 100644 --- a/pkg/controller/perconaservermongodbbackup/perconaservermongodbbackup_controller.go +++ b/pkg/controller/perconaservermongodbbackup/perconaservermongodbbackup_controller.go @@ -29,6 +29,7 @@ import ( "github.com/percona/percona-backup-mongodb/pbm/storage" "github.com/percona/percona-backup-mongodb/pbm/storage/azure" "github.com/percona/percona-backup-mongodb/pbm/storage/s3" + "github.com/percona/percona-server-mongodb-operator/clientcmd" psmdbv1 "github.com/percona/percona-server-mongodb-operator/pkg/apis/psmdb/v1" "github.com/percona/percona-server-mongodb-operator/pkg/psmdb/backup" "github.com/percona/percona-server-mongodb-operator/version" @@ -42,16 +43,26 @@ import ( // Add creates a new PerconaServerMongoDBBackup Controller and adds it to the Manager. The Manager will set fields on the Controller // and Start it when the Manager is Started. func Add(mgr manager.Manager) error { - return add(mgr, newReconciler(mgr)) + r, err := newReconciler(mgr) + if err != nil { + return err + } + return add(mgr, r) } // newReconciler returns a new reconcile.Reconciler -func newReconciler(mgr manager.Manager) reconcile.Reconciler { +func newReconciler(mgr manager.Manager) (reconcile.Reconciler, error) { + cli, err := clientcmd.NewClient(mgr.GetConfig()) + if err != nil { + return nil, errors.Wrap(err, "create clientcmd") + } + return &ReconcilePerconaServerMongoDBBackup{ client: mgr.GetClient(), scheme: mgr.GetScheme(), newPBMFunc: backup.NewPBM, - } + clientcmd: cli, + }, nil } // add adds a new Controller to mgr with r as the reconcile.Reconciler @@ -86,8 +97,9 @@ var _ reconcile.Reconciler = &ReconcilePerconaServerMongoDBBackup{} type ReconcilePerconaServerMongoDBBackup struct { // This client, initialized using mgr.Client() above, is a split client // that reads objects from the cache and writes to the apiserver - client client.Client - scheme *runtime.Scheme + client client.Client + scheme *runtime.Scheme + clientcmd *clientcmd.Client newPBMFunc backup.NewPBMFunc } @@ -155,7 +167,7 @@ func (r *ReconcilePerconaServerMongoDBBackup) Reconcile(ctx context.Context, req if cluster != nil { var svr *version.ServerVersion - svr, err = version.Server() + svr, err = version.Server(r.clientcmd) if err != nil { return rr, errors.Wrapf(err, "fetch server version") } diff --git a/pkg/controller/perconaservermongodbrestore/logical.go b/pkg/controller/perconaservermongodbrestore/logical.go index 644d622b5a..25bcf7f476 100644 --- a/pkg/controller/perconaservermongodbrestore/logical.go +++ b/pkg/controller/perconaservermongodbrestore/logical.go @@ -35,7 +35,7 @@ func (r *ReconcilePerconaServerMongoDBRestore) reconcileLogicalRestore(ctx conte return status, errors.New("cluster is unmanaged") } - svr, err := version.Server() + svr, err := version.Server(r.clientcmd) if err != nil { return status, errors.Wrapf(err, "fetch server version") } diff --git a/pkg/controller/perconaservermongodbrestore/perconaservermongodbrestore_controller.go b/pkg/controller/perconaservermongodbrestore/perconaservermongodbrestore_controller.go index 04fbbdf480..f037f7fda0 100644 --- a/pkg/controller/perconaservermongodbrestore/perconaservermongodbrestore_controller.go +++ b/pkg/controller/perconaservermongodbrestore/perconaservermongodbrestore_controller.go @@ -41,7 +41,7 @@ func Add(mgr manager.Manager) error { // newReconciler returns a new reconcile.Reconciler func newReconciler(mgr manager.Manager) (reconcile.Reconciler, error) { - cli, err := clientcmd.NewClient() + cli, err := clientcmd.NewClient(mgr.GetConfig()) if err != nil { return nil, errors.Wrap(err, "create clientcmd") } @@ -237,7 +237,7 @@ func (r *ReconcilePerconaServerMongoDBRestore) getBackup(ctx context.Context, cr } func (r *ReconcilePerconaServerMongoDBRestore) updateStatus(ctx context.Context, cr *psmdbv1.PerconaServerMongoDBRestore) error { - var backoff = wait.Backoff{ + backoff := wait.Backoff{ Steps: 5, Duration: 500 * time.Millisecond, Factor: 5.0, diff --git a/pkg/controller/perconaservermongodbrestore/physical.go b/pkg/controller/perconaservermongodbrestore/physical.go index 33414ded24..c9b4cd8b2f 100644 --- a/pkg/controller/perconaservermongodbrestore/physical.go +++ b/pkg/controller/perconaservermongodbrestore/physical.go @@ -44,7 +44,7 @@ func (r *ReconcilePerconaServerMongoDBRestore) reconcilePhysicalRestore(ctx cont return status, errors.New("cluster is unmanaged") } - svr, err := version.Server() + svr, err := version.Server(r.clientcmd) if err != nil { return status, errors.Wrapf(err, "fetch server version") } diff --git a/pkg/psmdb/backup/pbm.go b/pkg/psmdb/backup/pbm.go index c91e56fda1..9bcefbc787 100644 --- a/pkg/psmdb/backup/pbm.go +++ b/pkg/psmdb/backup/pbm.go @@ -105,7 +105,7 @@ func getMongoUri(ctx context.Context, k8sclient client.Client, cr *api.PerconaSe // certificates of the cluster, we need to copy them to operator pod. // This is especially important if the user passes custom config to set // net.tls.mode to requireTLS. - sslSecret, err := getSecret(ctx, k8sclient, cr.Namespace, cr.Spec.Secrets.SSL) + sslSecret, err := getSecret(ctx, k8sclient, cr.Namespace, api.SSLSecretName(cr)) if err != nil { return "", errors.Wrap(err, "get ssl secret") } diff --git a/pkg/psmdb/mongos.go b/pkg/psmdb/mongos.go index 4e9b9e1b69..aff6671051 100644 --- a/pkg/psmdb/mongos.go +++ b/pkg/psmdb/mongos.go @@ -303,7 +303,7 @@ func volumes(cr *api.PerconaServerMongoDB, configSource VolumeSourceType) []core Name: "ssl", VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ - SecretName: cr.Spec.Secrets.SSL, + SecretName: api.SSLSecretName(cr), Optional: &fvar, DefaultMode: &secretFileMode, }, @@ -313,7 +313,7 @@ func volumes(cr *api.PerconaServerMongoDB, configSource VolumeSourceType) []core Name: "ssl-internal", VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ - SecretName: cr.Spec.Secrets.SSLInternal, + SecretName: api.SSLInternalSecretName(cr), Optional: &tvar, DefaultMode: &secretFileMode, }, diff --git a/pkg/psmdb/statefulset.go b/pkg/psmdb/statefulset.go index ab8fbc1072..0a57df94b4 100644 --- a/pkg/psmdb/statefulset.go +++ b/pkg/psmdb/statefulset.go @@ -173,7 +173,7 @@ func StatefulSpec(ctx context.Context, cr *api.PerconaServerMongoDB, replset *ap Name: "ssl", VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ - SecretName: cr.Spec.Secrets.SSL, + SecretName: api.SSLSecretName(cr), Optional: &cr.Spec.UnsafeConf, DefaultMode: &secretFileMode, }, @@ -191,7 +191,7 @@ func StatefulSpec(ctx context.Context, cr *api.PerconaServerMongoDB, replset *ap Name: "ssl-internal", VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ - SecretName: cr.Spec.Secrets.SSLInternal, + SecretName: api.SSLInternalSecretName(cr), Optional: &t, DefaultMode: &secretFileMode, }, diff --git a/pkg/psmdb/tls/certmanager.go b/pkg/psmdb/tls/certmanager.go index a07ac05a6e..1fa13b7be4 100644 --- a/pkg/psmdb/tls/certmanager.go +++ b/pkg/psmdb/tls/certmanager.go @@ -6,30 +6,60 @@ import ( cm "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1" cmmeta "github.com/cert-manager/cert-manager/pkg/apis/meta/v1" + "github.com/cert-manager/cert-manager/pkg/util/cmapichecker" "github.com/pkg/errors" corev1 "k8s.io/api/core/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/rest" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + logf "sigs.k8s.io/controller-runtime/pkg/log" api "github.com/percona/percona-server-mongodb-operator/pkg/apis/psmdb/v1" + "github.com/percona/percona-server-mongodb-operator/pkg/util" ) -type CertManagerController struct { +type CertManagerController interface { + ApplyIssuer(ctx context.Context, cr *api.PerconaServerMongoDB) (util.ApplyStatus, error) + ApplyCAIssuer(ctx context.Context, cr *api.PerconaServerMongoDB) (util.ApplyStatus, error) + ApplyCertificate(ctx context.Context, cr *api.PerconaServerMongoDB, internal bool) (util.ApplyStatus, error) + ApplyCACertificate(ctx context.Context, cr *api.PerconaServerMongoDB) (util.ApplyStatus, error) + DeleteDeprecatedIssuerIfExists(ctx context.Context, cr *api.PerconaServerMongoDB) error + WaitForCerts(ctx context.Context, cr *api.PerconaServerMongoDB, secretsList ...string) error + GetMergedCA(ctx context.Context, cr *api.PerconaServerMongoDB, secretNames []string) ([]byte, error) + Check(ctx context.Context, config *rest.Config, ns string) error + IsDryRun() bool + GetClient() client.Client +} + +type certManagerController struct { cl client.Client scheme *runtime.Scheme + dryRun bool } -func NewCertManagerController(cl client.Client, scheme *runtime.Scheme) *CertManagerController { - return &CertManagerController{ +var _ CertManagerController = new(certManagerController) + +type NewCertManagerControllerFunc func(cl client.Client, scheme *runtime.Scheme, dryRun bool) CertManagerController + +func NewCertManagerController(cl client.Client, scheme *runtime.Scheme, dryRun bool) CertManagerController { + if dryRun { + cl = client.NewDryRunClient(cl) + } + return &certManagerController{ cl: cl, scheme: scheme, + dryRun: dryRun, } } +func (c *certManagerController) IsDryRun() bool { + return c.dryRun +} + func certificateName(cr *api.PerconaServerMongoDB, internal bool) string { if internal { return cr.Name + "-ssl-internal" @@ -39,10 +69,14 @@ func certificateName(cr *api.PerconaServerMongoDB, internal bool) string { func CertificateSecretName(cr *api.PerconaServerMongoDB, internal bool) string { if internal { - return cr.Spec.Secrets.SSLInternal + return api.SSLInternalSecretName(cr) } - return cr.Spec.Secrets.SSL + return api.SSLSecretName(cr) +} + +func deprecatedIssuerName(cr *api.PerconaServerMongoDB) string { + return cr.Name + "-psmdb-ca" } func issuerName(cr *api.PerconaServerMongoDB) string { @@ -51,7 +85,7 @@ func issuerName(cr *api.PerconaServerMongoDB) string { } if cr.CompareVersion("1.15.0") < 0 { - return cr.Name + "-psmdb-ca" + return deprecatedIssuerName(cr) } return cr.Name + "-psmdb-issuer" @@ -65,14 +99,34 @@ func CACertificateSecretName(cr *api.PerconaServerMongoDB) string { return cr.Name + "-ca-cert" } -func (c *CertManagerController) create(ctx context.Context, cr *api.PerconaServerMongoDB, obj client.Object) error { +func (c *certManagerController) DeleteDeprecatedIssuerIfExists(ctx context.Context, cr *api.PerconaServerMongoDB) error { + issuer := new(cm.Issuer) + err := c.cl.Get(ctx, types.NamespacedName{ + Name: deprecatedIssuerName(cr), + Namespace: cr.Namespace, + }, issuer) + if err != nil { + if k8serrors.IsNotFound(err) { + return nil + } + return err + } + return c.cl.Delete(ctx, issuer) +} + +func (c *certManagerController) createOrUpdate(ctx context.Context, cr *api.PerconaServerMongoDB, obj client.Object) (util.ApplyStatus, error) { if err := controllerutil.SetControllerReference(cr, obj, c.scheme); err != nil { - return errors.Wrap(err, "set controller reference") + return "", errors.Wrap(err, "set controller reference") } - return c.cl.Create(ctx, obj) + + status, err := util.Apply(ctx, c.cl, obj) + if err != nil { + return "", errors.Wrap(err, "create or update") + } + return status, nil } -func (c *CertManagerController) CreateIssuer(ctx context.Context, cr *api.PerconaServerMongoDB) error { +func (c *certManagerController) ApplyIssuer(ctx context.Context, cr *api.PerconaServerMongoDB) (util.ApplyStatus, error) { issuer := &cm.Issuer{ ObjectMeta: metav1.ObjectMeta{ Name: issuerName(cr), @@ -95,10 +149,10 @@ func (c *CertManagerController) CreateIssuer(ctx context.Context, cr *api.Percon } } - return c.create(ctx, cr, issuer) + return c.createOrUpdate(ctx, cr, issuer) } -func (c *CertManagerController) CreateCAIssuer(ctx context.Context, cr *api.PerconaServerMongoDB) error { +func (c *certManagerController) ApplyCAIssuer(ctx context.Context, cr *api.PerconaServerMongoDB) (util.ApplyStatus, error) { issuer := &cm.Issuer{ ObjectMeta: metav1.ObjectMeta{ Name: caIssuerName(cr), @@ -111,10 +165,10 @@ func (c *CertManagerController) CreateCAIssuer(ctx context.Context, cr *api.Perc }, } - return c.create(ctx, cr, issuer) + return c.createOrUpdate(ctx, cr, issuer) } -func (c *CertManagerController) CreateCertificate(ctx context.Context, cr *api.PerconaServerMongoDB, internal bool) error { +func (c *certManagerController) ApplyCertificate(ctx context.Context, cr *api.PerconaServerMongoDB, internal bool) (util.ApplyStatus, error) { issuerKind := cm.IssuerKind issuerGroup := "" if cr.CompareVersion("1.16.0") >= 0 && cr.Spec.TLS != nil && cr.Spec.TLS.IssuerConf != nil { @@ -149,10 +203,35 @@ func (c *CertManagerController) CreateCertificate(ctx context.Context, cr *api.P }, } - return c.create(ctx, cr, certificate) + return c.createOrUpdate(ctx, cr, certificate) } -func (c *CertManagerController) CreateCACertificate(ctx context.Context, cr *api.PerconaServerMongoDB) error { +var ( + ErrCertManagerNotFound = errors.New("cert-manager not found") + ErrCertManagerNotReady = errors.New("cert-manager not ready") +) + +func (c *certManagerController) Check(ctx context.Context, config *rest.Config, ns string) error { + log := logf.FromContext(ctx) + checker, err := cmapichecker.New(config, c.scheme, ns) + if err != nil { + return err + } + err = checker.Check(ctx) + if err != nil { + switch cmapichecker.TranslateToSimpleError(err) { + case cmapichecker.ErrCertManagerCRDsNotFound: + return ErrCertManagerNotFound + case cmapichecker.ErrWebhookCertificateFailure, cmapichecker.ErrWebhookServiceFailure, cmapichecker.ErrWebhookDeploymentFailure: + log.Info("cert-manager is not ready", "error", cmapichecker.TranslateToSimpleError(err)) + return ErrCertManagerNotReady + } + return err + } + return nil +} + +func (c *certManagerController) ApplyCACertificate(ctx context.Context, cr *api.PerconaServerMongoDB) (util.ApplyStatus, error) { cert := &cm.Certificate{ ObjectMeta: metav1.ObjectMeta{ Name: CACertificateSecretName(cr), @@ -171,10 +250,13 @@ func (c *CertManagerController) CreateCACertificate(ctx context.Context, cr *api }, } - return c.create(ctx, cr, cert) + return c.createOrUpdate(ctx, cr, cert) } -func (c *CertManagerController) WaitForCerts(ctx context.Context, cr *api.PerconaServerMongoDB, secretsList ...string) error { +func (c *certManagerController) WaitForCerts(ctx context.Context, cr *api.PerconaServerMongoDB, secretsList ...string) error { + if c.dryRun { + return nil + } ticker := time.NewTicker(1 * time.Second) timeoutTimer := time.NewTimer(30 * time.Second) defer timeoutTimer.Stop() @@ -211,3 +293,36 @@ func (c *CertManagerController) WaitForCerts(ctx context.Context, cr *api.Percon } } } + +// GetMergedCA returns merged CA from provided secrets. Result will not contain PEM duplicates. +func (c *certManagerController) GetMergedCA(ctx context.Context, cr *api.PerconaServerMongoDB, secretNames []string) ([]byte, error) { + mergedCA := []byte{} + + for _, secretName := range secretNames { + secret := new(corev1.Secret) + err := c.cl.Get(ctx, types.NamespacedName{ + Name: secretName, + Namespace: cr.Namespace, + }, secret) + if err != nil { + if k8serrors.IsNotFound(err) { + continue + } + return nil, errors.Wrap(err, "get old ssl secret") + } + if len(mergedCA) == 0 { + mergedCA = secret.Data["ca.crt"] + continue + } + + mergedCA, err = MergePEM(mergedCA, secret.Data["ca.crt"]) + if err != nil { + return nil, errors.Wrap(err, "merge old ssl and ssl internal secret") + } + } + return mergedCA, nil +} + +func (c *certManagerController) GetClient() client.Client { + return c.cl +} diff --git a/pkg/psmdb/tls/certmanager_test.go b/pkg/psmdb/tls/certmanager_test.go index 5ca7e9d3da..18fec27ad8 100644 --- a/pkg/psmdb/tls/certmanager_test.go +++ b/pkg/psmdb/tls/certmanager_test.go @@ -37,11 +37,11 @@ func TestCreateIssuer(t *testing.T) { issuer := &cm.Issuer{} t.Run("Create issuer with custom name", func(t *testing.T) { - if err := r.CreateIssuer(ctx, cr); err != nil { + if _, err := r.ApplyIssuer(ctx, cr); err != nil { t.Fatal(err) } - err := r.cl.Get(ctx, types.NamespacedName{Namespace: "psmdb", Name: customIssuerName}, issuer) + err := r.GetClient().Get(ctx, types.NamespacedName{Namespace: "psmdb", Name: customIssuerName}, issuer) if err != nil { t.Fatal(err) } @@ -53,11 +53,11 @@ func TestCreateIssuer(t *testing.T) { t.Run("Create issuer with default name", func(t *testing.T) { cr.Spec.CRVersion = "1.15.0" - if err := r.CreateIssuer(ctx, cr); err != nil { + if _, err := r.ApplyIssuer(ctx, cr); err != nil { t.Fatal(err) } - err := r.cl.Get(ctx, types.NamespacedName{Namespace: "psmdb", Name: issuerName(cr)}, issuer) + err := r.GetClient().Get(ctx, types.NamespacedName{Namespace: "psmdb", Name: issuerName(cr)}, issuer) if err != nil { t.Fatal(err) } @@ -97,11 +97,11 @@ func TestCreateCertificate(t *testing.T) { cert := &cm.Certificate{} t.Run("Create certificate with custom issuer name", func(t *testing.T) { - if err := r.CreateCertificate(ctx, cr, false); err != nil { + if _, err := r.ApplyCertificate(ctx, cr, false); err != nil { t.Fatal(err) } - err := r.cl.Get(ctx, types.NamespacedName{Namespace: "psmdb", Name: certificateName(cr, false)}, cert) + err := r.GetClient().Get(ctx, types.NamespacedName{Namespace: "psmdb", Name: certificateName(cr, false)}, cert) if err != nil { t.Fatal(err) } @@ -115,11 +115,11 @@ func TestCreateCertificate(t *testing.T) { cr.Name = "psmdb-mock-1" cr.Spec.CRVersion = "1.15.0" - if err := r.CreateCertificate(ctx, cr, false); err != nil { + if _, err := r.ApplyCertificate(ctx, cr, false); err != nil { t.Fatal(err) } - err := r.cl.Get(ctx, types.NamespacedName{Namespace: "psmdb", Name: certificateName(cr, false)}, cert) + err := r.GetClient().Get(ctx, types.NamespacedName{Namespace: "psmdb", Name: certificateName(cr, false)}, cert) if err != nil { t.Fatal(err) } @@ -131,7 +131,7 @@ func TestCreateCertificate(t *testing.T) { } // creates a fake client to mock API calls with the mock objects -func buildFakeClient(objs ...client.Object) *CertManagerController { +func buildFakeClient(objs ...client.Object) CertManagerController { s := scheme.Scheme s.AddKnownTypes(api.SchemeGroupVersion, @@ -142,7 +142,7 @@ func buildFakeClient(objs ...client.Object) *CertManagerController { cl := fake.NewClientBuilder().WithScheme(s).WithObjects(objs...).WithStatusSubresource(objs...).Build() - return &CertManagerController{ + return &certManagerController{ cl: cl, scheme: s, } diff --git a/pkg/psmdb/tls/fake/certmanager.go b/pkg/psmdb/tls/fake/certmanager.go new file mode 100644 index 0000000000..b1b59a6dbc --- /dev/null +++ b/pkg/psmdb/tls/fake/certmanager.go @@ -0,0 +1,67 @@ +package fake + +import ( + "context" + + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client" + + api "github.com/percona/percona-server-mongodb-operator/pkg/apis/psmdb/v1" + "github.com/percona/percona-server-mongodb-operator/pkg/psmdb/tls" + "github.com/percona/percona-server-mongodb-operator/pkg/util" +) + +type fakeCertManagerController struct { + dryRun bool + cl client.Client +} + +var _ tls.CertManagerController = new(fakeCertManagerController) + +func NewCertManagerController(cl client.Client, scheme *runtime.Scheme, dryRun bool) tls.CertManagerController { + return &fakeCertManagerController{ + dryRun: dryRun, + cl: cl, + } +} + +func (c *fakeCertManagerController) ApplyIssuer(ctx context.Context, cr *api.PerconaServerMongoDB) (util.ApplyStatus, error) { + return util.ApplyStatusUnchanged, nil +} + +func (c *fakeCertManagerController) ApplyCAIssuer(ctx context.Context, cr *api.PerconaServerMongoDB) (util.ApplyStatus, error) { + return util.ApplyStatusUnchanged, nil +} + +func (c *fakeCertManagerController) ApplyCertificate(ctx context.Context, cr *api.PerconaServerMongoDB, internal bool) (util.ApplyStatus, error) { + return util.ApplyStatusUnchanged, nil +} + +func (c *fakeCertManagerController) ApplyCACertificate(ctx context.Context, cr *api.PerconaServerMongoDB) (util.ApplyStatus, error) { + return util.ApplyStatusUnchanged, nil +} + +func (c *fakeCertManagerController) DeleteDeprecatedIssuerIfExists(ctx context.Context, cr *api.PerconaServerMongoDB) error { + return nil +} + +func (c *fakeCertManagerController) WaitForCerts(ctx context.Context, cr *api.PerconaServerMongoDB, secretsList ...string) error { + return nil +} + +func (c *fakeCertManagerController) GetMergedCA(ctx context.Context, cr *api.PerconaServerMongoDB, secretNames []string) ([]byte, error) { + return nil, nil +} + +func (c *fakeCertManagerController) Check(ctx context.Context, config *rest.Config, ns string) error { + return tls.ErrCertManagerNotFound +} + +func (c *fakeCertManagerController) IsDryRun() bool { + return false +} + +func (c *fakeCertManagerController) GetClient() client.Client { + return c.cl +} diff --git a/pkg/psmdb/tls/pem.go b/pkg/psmdb/tls/pem.go new file mode 100644 index 0000000000..cfe86eef72 --- /dev/null +++ b/pkg/psmdb/tls/pem.go @@ -0,0 +1,60 @@ +package tls + +import ( + "bytes" + "encoding/pem" + "reflect" +) + +func decodePEMList(data []byte) []*pem.Block { + blocks := []*pem.Block{} + rest := data + for { + var p *pem.Block + p, rest = pem.Decode(rest) + if p == nil { + break + } + blocks = append(blocks, p) + } + return blocks +} + +func mergePEMBlocks(result []*pem.Block, toMerge []*pem.Block) ([]*pem.Block, error) { + for _, block := range toMerge { + if !hasBlock(result, block) { + result = append(result, block) + } + } + return result, nil +} + +func hasBlock(data []*pem.Block, block *pem.Block) bool { + for _, b := range data { + if bytes.Equal(b.Bytes, block.Bytes) && + reflect.DeepEqual(b.Headers, block.Headers) && + b.Type == block.Type { + return true + } + } + return false +} + +func MergePEM(target []byte, toMerge ...[]byte) ([]byte, error) { + var err error + targetBlocks := decodePEMList(target) + for _, mergeData := range toMerge { + mergeBlocks := decodePEMList(mergeData) + targetBlocks, err = mergePEMBlocks(targetBlocks, mergeBlocks) + if err != nil { + return nil, err + } + } + + ca := []byte{} + for _, block := range targetBlocks { + ca = append(ca, pem.EncodeToMemory(block)...) + } + + return ca, nil +} diff --git a/pkg/psmdb/tls/tls.go b/pkg/psmdb/tls/tls.go index 17fbfbd187..d96e7b306c 100644 --- a/pkg/psmdb/tls/tls.go +++ b/pkg/psmdb/tls/tls.go @@ -110,10 +110,7 @@ func Issue(hosts []string) (caCert []byte, tlsCert []byte, tlsKey []byte, err er // Config returns tls.Config to be used in mongo.Config func Config(ctx context.Context, k8sclient client.Client, cr *api.PerconaServerMongoDB) (tls.Config, error) { - secretName := cr.Spec.Secrets.SSL - if len(secretName) == 0 { - secretName = cr.Name + "-ssl" - } + secretName := api.SSLSecretName(cr) certSecret := &corev1.Secret{} err := k8sclient.Get(ctx, types.NamespacedName{ Name: secretName, diff --git a/pkg/util/apply.go b/pkg/util/apply.go new file mode 100644 index 0000000000..0881445fba --- /dev/null +++ b/pkg/util/apply.go @@ -0,0 +1,101 @@ +package util + +import ( + "context" + "encoding/base64" + "encoding/json" + "reflect" + + cm "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1" + "github.com/pkg/errors" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +type ApplyStatus string + +const ( + ApplyStatusCreated ApplyStatus = "created" + ApplyStatusUpdated ApplyStatus = "updated" + ApplyStatusUnchanged ApplyStatus = "unchanged" +) + +func Apply(ctx context.Context, cl client.Client, obj client.Object) (ApplyStatus, error) { + if obj.GetAnnotations() == nil { + obj.SetAnnotations(make(map[string]string)) + } + + objAnnotations := obj.GetAnnotations() + delete(objAnnotations, "percona.com/last-config-hash") + obj.SetAnnotations(objAnnotations) + + hash, err := getObjectHash(obj) + if err != nil { + return "", errors.Wrap(err, "calculate object hash") + } + + objAnnotations = obj.GetAnnotations() + objAnnotations["percona.com/last-config-hash"] = hash + obj.SetAnnotations(objAnnotations) + + val := reflect.ValueOf(obj) + if val.Kind() == reflect.Ptr { + val = reflect.Indirect(val) + } + oldObject := reflect.New(val.Type()).Interface().(client.Object) + + err = cl.Get(ctx, types.NamespacedName{ + Name: obj.GetName(), + Namespace: obj.GetNamespace(), + }, oldObject) + + if err != nil && !k8serrors.IsNotFound(err) { + return "", errors.Wrap(err, "get object") + } + + if k8serrors.IsNotFound(err) { + return ApplyStatusCreated, cl.Create(ctx, obj) + } + + if oldObject.GetAnnotations()["percona.com/last-config-hash"] != hash || + !MapEqual(oldObject.GetLabels(), obj.GetLabels()) || + !MapEqual(oldObject.GetAnnotations(), obj.GetAnnotations()) { + obj.SetResourceVersion(oldObject.GetResourceVersion()) + switch object := obj.(type) { + case *corev1.Service: + object.Spec.ClusterIP = oldObject.(*corev1.Service).Spec.ClusterIP + } + + return ApplyStatusUpdated, cl.Update(ctx, obj) + } + + return ApplyStatusUnchanged, nil +} + +func getObjectHash(obj client.Object) (string, error) { + var dataToMarshall interface{} + switch object := obj.(type) { + case *appsv1.StatefulSet: + dataToMarshall = object.Spec + case *appsv1.Deployment: + dataToMarshall = object.Spec + case *corev1.Service: + dataToMarshall = object.Spec + case *corev1.Secret: + dataToMarshall = object.Data + case *cm.Certificate: + dataToMarshall = object.Spec + case *cm.Issuer: + dataToMarshall = object.Spec + default: + dataToMarshall = obj + } + data, err := json.Marshal(dataToMarshall) + if err != nil { + return "", err + } + return base64.StdEncoding.EncodeToString(data), nil +} diff --git a/version/server.go b/version/server.go index 8879088fbd..b1cf2030f7 100644 --- a/version/server.go +++ b/version/server.go @@ -3,7 +3,6 @@ package version import ( "context" "encoding/json" - "fmt" "sync" k8sversion "k8s.io/apimachinery/pkg/version" @@ -33,14 +32,14 @@ var ( // Server returns server version and platform (k8s|oc) // it performs API requests for the first invocation and then returns "cached" value -func Server() (*ServerVersion, error) { +func Server(cl *clientcmd.Client) (*ServerVersion, error) { mx.Lock() defer mx.Unlock() if cVersion != nil { return cVersion, nil } - v, err := GetServer() + v, err := GetServer(cl) if err != nil { return nil, err } @@ -51,15 +50,12 @@ func Server() (*ServerVersion, error) { } // GetServer make request to platform server and returns server version and platform (k8s|oc) -func GetServer() (*ServerVersion, error) { - cl, err := clientcmd.NewClient() - if err != nil { - return nil, fmt.Errorf("create REST client: %v", err) - } +func GetServer(cl *clientcmd.Client) (*ServerVersion, error) { client := cl.REST() version := &ServerVersion{} // oc 3.9 + var err error version.Info, err = probeAPI("/version/openshift", client) if err == nil { version.Platform = PlatformOpenshift From 028e8c36af67eb9319aadec8fa3f5c6a03787d8d Mon Sep 17 00:00:00 2001 From: Serge Date: Wed, 24 Apr 2024 13:52:12 +0300 Subject: [PATCH 2/2] K8SPSMDB-1003: Kubernetes node zone/region tag (#1360) * K8SPSMDB-1003 - kubernetes node tags zone/region Add kubernetes node tags zone/region to the monogo nodes. * Remove worning message if we do not have special permission. * fix test * fix cross-site test * fix image * update test * delete unsused * update cross-site test * fix PR comments * fix * fix * fix * fix --------- Co-authored-by: Viacheslav Sarzhan Co-authored-by: Natalia Marukovich Co-authored-by: Natalia Marukovich Co-authored-by: Inel Pandzic --- deploy/cw-bundle.yaml | 8 +++++ deploy/cw-rbac.yaml | 8 +++++ e2e-tests/cross-site-sharded/run | 36 +++++++++++++++++++--- e2e-tests/serviceless-external-nodes/run | 2 ++ pkg/controller/perconaservermongodb/mgo.go | 30 ++++++++++++------ pkg/psmdb/getters.go | 18 +++++++++++ 6 files changed, 88 insertions(+), 14 deletions(-) diff --git a/deploy/cw-bundle.yaml b/deploy/cw-bundle.yaml index 0561191dd9..73817307f0 100644 --- a/deploy/cw-bundle.yaml +++ b/deploy/cw-bundle.yaml @@ -18336,6 +18336,14 @@ rules: - update - patch - delete +- apiGroups: + - "" + resources: + - nodes + verbs: + - get + - list + - watch - apiGroups: - "" resources: diff --git a/deploy/cw-rbac.yaml b/deploy/cw-rbac.yaml index 8fd62dcd59..c311436f9d 100644 --- a/deploy/cw-rbac.yaml +++ b/deploy/cw-rbac.yaml @@ -35,6 +35,14 @@ rules: - update - patch - delete +- apiGroups: + - "" + resources: + - nodes + verbs: + - get + - list + - watch - apiGroups: - "" resources: diff --git a/e2e-tests/cross-site-sharded/run b/e2e-tests/cross-site-sharded/run index 290c2507a7..47c688f7f3 100755 --- a/e2e-tests/cross-site-sharded/run +++ b/e2e-tests/cross-site-sharded/run @@ -13,6 +13,26 @@ unset OPERATOR_NS main_cluster="cross-site-sharded-main" replica_cluster="cross-site-sharded-replica" +wait_for_members() { + local endpoint="$1" + local rsName="$2" + local nodes_amount=0 + until [[ ${nodes_amount} == 6 ]]; do + nodes_amount=$(run_mongos 'rs.conf().members.length' "clusterAdmin:clusterAdmin123456@$endpoint" "mongodb" ":27017" \ + | egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|bye' \ + | $sed -re 's/ObjectId\("[0-9a-f]+"\)//; s/-[0-9]+.svc/-xxx.svc/') + + echo "waiting for all members to be configured in ${rsName}" + let retry+=1 + if [ $retry -ge 15 ]; then + echo "Max retry count $retry reached. something went wrong with mongo cluster. Config for endpoint $endpoint has $nodes_amount but expected 6." + exit 1 + fi + echo -n . + sleep 10 + done +} + desc "create main cluster" create_infra "$namespace" @@ -118,7 +138,10 @@ sleep 30 desc "create replica PSMDB cluster $cluster" apply_cluster "$test_dir/conf/${replica_cluster}.yml" -sleep 300 + +wait_for_running $replica_cluster-rs0 3 "false" +wait_for_running $replica_cluster-rs1 3 "false" +wait_for_running $replica_cluster-cfg 3 "false" replica_cfg_0_endpoint=$(get_service_ip cross-site-sharded-replica-cfg-0 'cfg') replica_cfg_1_endpoint=$(get_service_ip cross-site-sharded-replica-cfg-1 'cfg') @@ -141,7 +164,10 @@ kubectl_bin patch psmdb ${main_cluster} --type=merge --patch '{ } }' -sleep 60 +wait_for_members $replica_cfg_0_endpoint cfg +wait_for_members $replica_rs0_0_endpoint rs0 +wait_for_members $replica_rs1_0_endpoint rs1 + kubectl_bin config set-context $(kubectl_bin config current-context) --namespace="$replica_namespace" desc 'check if all 3 Pods started' @@ -165,8 +191,8 @@ compare_mongos_cmd "find" "myApp:myPass@$main_cluster-mongos.$namespace" desc 'test failover' kubectl_bin config set-context $(kubectl_bin config current-context) --namespace="$namespace" + kubectl_bin delete psmdb $main_cluster -sleep 60 desc 'run disaster recovery script for replset: cfg' run_script_mongos "${test_dir}/disaster_recovery.js" "clusterAdmin:clusterAdmin123456@$replica_cfg_0_endpoint" "mongodb" ":27017" @@ -180,7 +206,9 @@ run_script_mongos "${test_dir}/disaster_recovery.js" "clusterAdmin:clusterAdmin1 desc 'make replica cluster managed' kubectl_bin config set-context $(kubectl_bin config current-context) --namespace="$replica_namespace" kubectl_bin patch psmdb ${replica_cluster} --type=merge --patch '{"spec":{"unmanaged": false}}' -sleep 120 + +wait_for_running $replica_cluster-rs0 3 +wait_for_running $replica_cluster-cfg 3 desc "check failover status" compare_mongos_cmd "find" "myApp:myPass@$replica_cluster-mongos.$replica_namespace" diff --git a/e2e-tests/serviceless-external-nodes/run b/e2e-tests/serviceless-external-nodes/run index c34e1c16cd..a207ee4c25 100755 --- a/e2e-tests/serviceless-external-nodes/run +++ b/e2e-tests/serviceless-external-nodes/run @@ -46,6 +46,8 @@ cat $tmp_dir/psmdb.yaml \ wait_cluster_consistency ${cluster} +# waiting the config will be ready. +sleep 30 run_mongo 'rs.status().members.forEach(function(z){printjson(z.name);printjson(z.stateStr); })' "clusterAdmin:clusterAdmin123456@${cluster}-rs0-0.${cluster}-rs0.${namespace}" "mongodb" | egrep -v 'I NETWORK|W NETWORK|Error saving history file|Percona Server for MongoDB|connecting to:|Unable to reach primary for set|Implicit session:|versions do not match|Error saving history file:|bye' >"$tmp_dir/rs.txt" cat "${test_dir}/compare/rs.txt" \ diff --git a/pkg/controller/perconaservermongodb/mgo.go b/pkg/controller/perconaservermongodb/mgo.go index 872468ab62..679b2a55c5 100644 --- a/pkg/controller/perconaservermongodb/mgo.go +++ b/pkg/controller/perconaservermongodb/mgo.go @@ -19,6 +19,7 @@ import ( api "github.com/percona/percona-server-mongodb-operator/pkg/apis/psmdb/v1" "github.com/percona/percona-server-mongodb-operator/pkg/psmdb" "github.com/percona/percona-server-mongodb-operator/pkg/psmdb/mongo" + "github.com/percona/percona-server-mongodb-operator/pkg/util" ) var errReplsetLimit = fmt.Errorf("maximum replset member (%d) count reached", mongo.MaxMembers) @@ -267,6 +268,20 @@ func (r *ReconcilePerconaServerMongoDB) updateConfigMembers(ctx context.Context, return 0, fmt.Errorf("get host for pod %s: %v", pod.Name, err) } + nodeLabels := mongo.ReplsetTags{ + "nodeName": pod.Spec.NodeName, + "podName": pod.Name, + "serviceName": cr.Name, + } + + labels, err := psmdb.GetNodeLabels(ctx, r.client, cr, pod) + if err == nil { + nodeLabels = util.MapMerge(nodeLabels, mongo.ReplsetTags{ + "region": labels[corev1.LabelTopologyRegion], + "zone": labels[corev1.LabelTopologyZone], + }) + } + member := mongo.ConfigMember{ ID: key, Host: host, @@ -293,16 +308,11 @@ func (r *ReconcilePerconaServerMongoDB) updateConfigMembers(ctx context.Context, member.ArbiterOnly = true member.Priority = 0 case "mongod", "cfg": - member.Tags = mongo.ReplsetTags{ - "podName": pod.Name, - "serviceName": cr.Name, - } + member.Tags = nodeLabels case "nonVoting": - member.Tags = mongo.ReplsetTags{ - "podName": pod.Name, - "serviceName": cr.Name, - "nonVoting": "true", - } + member.Tags = util.MapMerge(mongo.ReplsetTags{ + "nonVoting": "true", + }, nodeLabels) member.Priority = 0 member.Votes = 0 } @@ -597,7 +607,7 @@ func (r *ReconcilePerconaServerMongoDB) handleReplsetInit(ctx context.Context, c "sh", "-c", fmt.Sprintf( ` - cat <<-EOF | %s + cat <<-EOF | %s rs.initiate( { _id: '%s', diff --git a/pkg/psmdb/getters.go b/pkg/psmdb/getters.go index cc70127705..652ebe9feb 100644 --- a/pkg/psmdb/getters.go +++ b/pkg/psmdb/getters.go @@ -3,6 +3,7 @@ package psmdb import ( "context" "sort" + "time" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" @@ -165,3 +166,20 @@ func GetExportedServices(ctx context.Context, cl client.Client, cr *api.PerconaS return seList, nil } + +func GetNodeLabels(ctx context.Context, cl client.Client, cr *api.PerconaServerMongoDB, pod corev1.Pod) (map[string]string, error) { + // Set a timeout for the request, to avoid hanging forever + ctx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + node := &corev1.Node{} + + err := cl.Get(ctx, client.ObjectKey{ + Name: pod.Spec.NodeName, + }, node) + if err != nil { + return nil, errors.Wrapf(err, "failed to get node %s", pod.Spec.NodeName) + } + + return node.Labels, nil +}