K8SPSMDB-956: fix problems with TLS certificate renewal

cmd/manager/main.go

@@ -11,8 +11,8 @@ import (
 	// to ensure that exec-entrypoint and run can make use of them.
 	_ "k8s.io/client-go/plugin/pkg/client/auth"
 
+	certmgrscheme "github.com/cert-manager/cert-manager/pkg/client/clientset/versioned/scheme"
 	"github.com/go-logr/logr"
-	certmgrscheme "github.com/jetstack/cert-manager/pkg/client/clientset/versioned/scheme"
 	uzap "go.uber.org/zap"
 	"go.uber.org/zap/zapcore"
 	k8sruntime "k8s.io/apimachinery/pkg/runtime"

e2e-tests/conf/cmctl.yml

@@ -0,0 +1,28 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: cmctl
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      name: cmctl
+  template:
+    metadata:
+      labels:
+        name: cmctl
+    spec:
+      serviceAccountName: cmctl
+      containers:
+        - name: cmctl
+          image: debian
+          imagePullPolicy: Always
+          command:
+          - /bin/bash
+          - -c
+          - |
+            apt-get update && apt-get install -y curl \
+            && curl -fsSL -o cmctl.tar.gz https://github.com/cert-manager/cert-manager/releases/latest/download/cmctl-linux-amd64.tar.gz \
+            && tar xzf cmctl.tar.gz \
+            && sleep 100500
+      restartPolicy: Always

e2e-tests/functions

@@ -16,7 +16,7 @@ SKIP_BACKUPS_TO_AWS_GCP_AZURE=${SKIP_BACKUPS_TO_AWS_GCP_AZURE:-1}
 PMM_SERVER_VER=${PMM_SERVER_VER:-"9.9.9"}
 IMAGE_PMM_SERVER_REPO=${IMAGE_PMM_SERVER_REPO:-"perconalab/pmm-server"}
 IMAGE_PMM_SERVER_TAG=${IMAGE_PMM_SERVER_TAG:-"dev-latest"}
-CERT_MANAGER_VER="1.8.0"
+CERT_MANAGER_VER="1.12.3"
 tmp_dir=$(mktemp -d)
 sed=$(which gsed || which sed)
 date=$(which gdate || which date)
@@ -845,8 +845,9 @@ deploy_cert_manager() {
 
 	kubectl_bin create namespace cert-manager || :
 	kubectl_bin label namespace cert-manager certmanager.k8s.io/disable-validation=true || :
-	kubectl_bin apply -f "https://github.com/jetstack/cert-manager/releases/download/v${CERT_MANAGER_VER}/cert-manager.yaml" --validate=false || : 2>/dev/null
-	sleep 30
+	kubectl_bin apply -f "https://github.com/cert-manager/cert-manager/releases/download/v${CERT_MANAGER_VER}/cert-manager.yaml" --validate=false || : 2>/dev/null
+	kubectl_bin -n cert-manager wait pod -l app.kubernetes.io/instance=cert-manager --for=condition=ready
+	sleep 120
 }
 
 delete_crd() {
@@ -891,7 +892,7 @@ destroy() {
 
 	delete_crd
 
-	kubectl_bin delete -f "https://github.com/jetstack/cert-manager/releases/download/v${CERT_MANAGER_VER}/cert-manager.yaml" 2>/dev/null || :
+	kubectl_bin delete -f "https://github.com/cert-manager/cert-manager/releases/download/v${CERT_MANAGER_VER}/cert-manager.yaml" 2>/dev/null || :
 	if [ -n "$OPENSHIFT" ]; then
 		oc delete --grace-period=0 --force=true project "$namespace" &
 		if [ -n "$OPERATOR_NS" ]; then

e2e-tests/run-distro.csv

@@ -19,6 +19,7 @@ pitr-physical
 recover-no-primary
 rs-shard-migration
 scaling
+tls-issue-cert-manager
 upgrade
 upgrade-sharded
 users

e2e-tests/run-minikube.csv

@@ -15,6 +15,7 @@ scheduled-backup
 security-context
 self-healing-chaos
 smart-update
+tls-issue-cert-manager
 upgrade-consistency
 upgrade-consistency-sharded
 users

e2e-tests/run-pr.csv

@@ -34,6 +34,7 @@ service-per-pod
 serviceless-external-nodes
 smart-update
 storage
+tls-issue-cert-manager
 upgrade
 upgrade-consistency
 upgrade-consistency-sharded

e2e-tests/run-release.csv

@@ -35,6 +35,7 @@ service-per-pod
 serviceless-external-nodes
 smart-update
 storage
+tls-issue-cert-manager
 upgrade
 upgrade-consistency
 upgrade-consistency-sharded

e2e-tests/tls-issue-cert-manager/compare/certificate_some-name-ssl-internal.yml

@@ -0,0 +1,47 @@
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  generation: 1
+  name: some-name-ssl-internal
+  ownerReferences:
+    - blockOwnerDeletion: true
+      controller: true
+      kind: PerconaServerMongoDB
+      name: some-name
+spec:
+  commonName: some-name
+  dnsNames:
+    - localhost
+    - some-name-rs0
+    - some-name-rs0.NAME_SPACE
+    - some-name-rs0.NAME_SPACE.svc.cluster.local
+    - '*.some-name-rs0'
+    - '*.some-name-rs0.NAME_SPACE'
+    - '*.some-name-rs0.NAME_SPACE.svc.cluster.local'
+    - some-name-rs0.NAME_SPACE.svc.clusterset.local
+    - '*.some-name-rs0.NAME_SPACE.svc.clusterset.local'
+    - '*.NAME_SPACE.svc.clusterset.local'
+    - some-name-mongos
+    - some-name-mongos.NAME_SPACE
+    - some-name-mongos.NAME_SPACE.svc.cluster.local
+    - '*.some-name-mongos'
+    - '*.some-name-mongos.NAME_SPACE'
+    - '*.some-name-mongos.NAME_SPACE.svc.cluster.local'
+    - some-name-cfg
+    - some-name-cfg.NAME_SPACE
+    - some-name-cfg.NAME_SPACE.svc.cluster.local
+    - '*.some-name-cfg'
+    - '*.some-name-cfg.NAME_SPACE'
+    - '*.some-name-cfg.NAME_SPACE.svc.cluster.local'
+    - some-name-mongos.NAME_SPACE.svc.clusterset.local
+    - '*.some-name-mongos.NAME_SPACE.svc.clusterset.local'
+    - some-name-cfg.NAME_SPACE.svc.clusterset.local
+    - '*.some-name-cfg.NAME_SPACE.svc.clusterset.local'
+  duration: 2160h0m0s
+  issuerRef:
+    kind: Issuer
+    name: some-name-psmdb-issuer
+  secretName: some-name-ssl-internal
+  subject:
+    organizations:
+      - PSMDB

e2e-tests/tls-issue-cert-manager/compare/certificate_some-name-ssl.yml

@@ -0,0 +1,47 @@
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  generation: 1
+  name: some-name-ssl
+  ownerReferences:
+    - blockOwnerDeletion: true
+      controller: true
+      kind: PerconaServerMongoDB
+      name: some-name
+spec:
+  commonName: some-name
+  dnsNames:
+    - localhost
+    - some-name-rs0
+    - some-name-rs0.NAME_SPACE
+    - some-name-rs0.NAME_SPACE.svc.cluster.local
+    - '*.some-name-rs0'
+    - '*.some-name-rs0.NAME_SPACE'
+    - '*.some-name-rs0.NAME_SPACE.svc.cluster.local'
+    - some-name-rs0.NAME_SPACE.svc.clusterset.local
+    - '*.some-name-rs0.NAME_SPACE.svc.clusterset.local'
+    - '*.NAME_SPACE.svc.clusterset.local'
+    - some-name-mongos
+    - some-name-mongos.NAME_SPACE
+    - some-name-mongos.NAME_SPACE.svc.cluster.local
+    - '*.some-name-mongos'
+    - '*.some-name-mongos.NAME_SPACE'
+    - '*.some-name-mongos.NAME_SPACE.svc.cluster.local'
+    - some-name-cfg
+    - some-name-cfg.NAME_SPACE
+    - some-name-cfg.NAME_SPACE.svc.cluster.local
+    - '*.some-name-cfg'
+    - '*.some-name-cfg.NAME_SPACE'
+    - '*.some-name-cfg.NAME_SPACE.svc.cluster.local'
+    - some-name-mongos.NAME_SPACE.svc.clusterset.local
+    - '*.some-name-mongos.NAME_SPACE.svc.clusterset.local'
+    - some-name-cfg.NAME_SPACE.svc.clusterset.local
+    - '*.some-name-cfg.NAME_SPACE.svc.clusterset.local'
+  duration: 2160h0m0s
+  issuerRef:
+    kind: Issuer
+    name: some-name-psmdb-issuer
+  secretName: some-name-ssl
+  subject:
+    organizations:
+      - PSMDB

e2e-tests/tls-issue-cert-manager/compare/issuer_some-name-psmdb-ca-issuer.yml

@@ -0,0 +1,12 @@
+apiVersion: cert-manager.io/v1
+kind: Issuer
+metadata:
+  generation: 1
+  name: some-name-psmdb-ca-issuer
+  ownerReferences:
+    - blockOwnerDeletion: true
+      controller: true
+      kind: PerconaServerMongoDB
+      name: some-name
+spec:
+  selfSigned: {}

e2e-tests/tls-issue-cert-manager/compare/issuer_some-name-psmdb-issuer.yml

@@ -0,0 +1,13 @@
+apiVersion: cert-manager.io/v1
+kind: Issuer
+metadata:
+  generation: 1
+  name: some-name-psmdb-issuer
+  ownerReferences:
+    - blockOwnerDeletion: true
+      controller: true
+      kind: PerconaServerMongoDB
+      name: some-name
+spec:
+  ca:
+    secretName: some-name-ca-cert

e2e-tests/tls-issue-cert-manager/conf/some-name.yml

@@ -0,0 +1,45 @@
+apiVersion: psmdb.percona.com/v1
+kind: PerconaServerMongoDB
+metadata:
+  name: some-name
+spec:
+  #platform: openshift
+  image:
+  imagePullPolicy: Always
+  backup:
+    enabled: false
+  replsets:
+  - name: rs0
+    affinity:
+      antiAffinityTopologyKey: none
+    resources:
+      limits:
+        cpu: 500m
+        memory: 1G
+      requests:
+        cpu: 100m
+        memory: 0.1G
+    volumeSpec:
+      persistentVolumeClaim:
+        resources:
+          requests:
+            storage: 1Gi
+    expose:
+      enabled: false
+      exposeType: ClusterIP
+    size: 3
+  sharding:
+    enabled: true
+    configsvrReplSet:
+      size: 3
+      volumeSpec:
+        persistentVolumeClaim:
+          resources:
+            requests:
+              storage: 3Gi
+      expose:
+        enabled: false
+    mongos:
+      size: 3
+  secrets:
+    users: some-users

e2e-tests/tls-issue-cert-manager/run

@@ -0,0 +1,121 @@
+#!/bin/bash
+
+set -o errexit
+
+test_dir=$(realpath $(dirname $0))
+. "${test_dir}/../functions"
+set_debug
+
+renew-certificate() {
+	certificate="$1"
+
+	desc "renew $certificate"
+
+	local pod_name
+	pod_name=$(kubectl_bin get pods --selector=name=cmctl -o 'jsonpath={.items[].metadata.name}')
+
+	local revision
+	revision=$(kubectl_bin get certificate "$certificate" -o 'jsonpath={.status.revision}')
+
+	kubectl_bin exec "$pod_name" -- ./cmctl renew "$certificate"
+
+	# wait for new revision
+	for i in {1..10}; do
+		local new_revision
+		new_revision=$(kubectl_bin get certificate "$certificate" -o 'jsonpath={.status.revision}')
+		if [ "$((revision + 1))" == "$new_revision" ]; then
+			break
+		fi
+		sleep 1
+	done
+}
+
+check_tls_secret() {
+	local secret_name=$1
+	check_secret_data_key "$secret_name" 'ca.crt'
+	check_secret_data_key "$secret_name" 'tls.crt'
+	check_secret_data_key "$secret_name" 'tls.key'
+}
+
+check_secret_data_key() {
+	local secret_name=$1
+	local data_key=$2
+	local secret_data
+
+	secret_data=$(kubectl_bin get "secrets/${secret_name}" -o json | jq ".data[\"${data_key}\"]")
+	if [ -z "$secret_data" ]; then
+		exit 1
+	fi
+}
+
+deploy_cmctl() {
+	local service_account="cmctl"
+
+	$sed -e "s/percona-server-mongodb-operator/$service_account/g" "${src_dir}/deploy/rbac.yaml" \
+		| yq '(select(.rules).rules[] | select(contains({"apiGroups": ["cert-manager.io"]}))).resources += "certificates/status"' \
+		| kubectl_bin apply -f -
+	kubectl_bin apply -f "$conf_dir/cmctl.yml"
+}
+
+main() {
+	create_infra "$namespace"
+	deploy_cert_manager
+
+	desc 'create secrets and start client'
+	kubectl_bin apply -f "$conf_dir/secrets.yml"
+	kubectl_bin apply -f "$conf_dir/client_with_tls.yml"
+	deploy_cmctl
+
+	cluster="some-name"
+	desc "create first PSMDB cluster $cluster"
+	apply_cluster "$test_dir/conf/$cluster.yml"
+
+	desc 'check if all Pods started'
+	wait_for_running $cluster-rs0 3
+	wait_for_running $cluster-cfg 3 "false"
+	wait_for_running $cluster-mongos 3
+
+	desc 'check if certificates issued with certmanager'
+	check_tls_secret "$cluster-ssl"
+
+	desc 'check if CA issuer created'
+	compare_kubectl issuer/$cluster-psmdb-ca-issuer
+
+	desc 'check if issuer created'
+	compare_kubectl issuer/$cluster-psmdb-issuer
+
+	desc 'check if certificate issued'
+	compare_kubectl certificate/$cluster-ssl
+
+	desc 'check if internal certificate issued'
+	compare_kubectl certificate/$cluster-ssl-internal
+
+	renew-certificate "some-name-ssl"
+	sleep 10
+	wait_for_running $cluster-rs0 3
+	wait_for_running $cluster-cfg 3 "false"
+	wait_for_running $cluster-mongos 3
+
+	renew-certificate "some-name-ssl-internal"
+	sleep 10
+	wait_for_running $cluster-rs0 3
+	wait_for_running $cluster-cfg 3 "false"
+	wait_for_running $cluster-mongos 3
+
+	desc 'check if CA issuer created'
+	compare_kubectl issuer/$cluster-psmdb-ca-issuer
+
+	desc 'check if issuer created'
+	compare_kubectl issuer/$cluster-psmdb-issuer
+
+	desc 'check if certificate issued'
+	compare_kubectl certificate/$cluster-ssl
+
+	desc 'check if internal certificate issued'
+	compare_kubectl certificate/$cluster-ssl-internal
+
+	destroy "$namespace"
+	desc 'test passed'
+}
+
+main