From cb4a80b554c08e2ba013da207897c5e6085c00c7 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Sun, 20 Jun 2021 18:45:30 +0200
Subject: [PATCH 1/7] Transition to Bitnami mariadb-galera chart

---
 .gitignore                                    |   1 +
 00namespace.yml                               |   5 -
 10conf-d.yml                                  | 158 ------------------
 20mariadb-service.yml                         |  26 ---
 21mariadb-ready-service.yml                   |  14 --
 50mariadb.yml                                 | 121 --------------
 README.md                                     |  68 +-------
 base/kustomization.yaml                       |  40 +++++
 .../mysql-service.yaml                        |   5 +-
 kustomization.yaml                            |   6 -
 test.sh                                       |  48 ++++++
 ...orageclass-ssd-regional-europewest1cd.yaml |  11 --
 variants/gke/gke-storageclass-ssd.yml         |   9 -
 variants/gke/kustomization.yaml               |   7 -
 variants/gke/volume-claims.yaml               |  16 --
 variants/namespace/kustomization.yaml         |  23 ++-
 variants/scale-1-ephemeral/ephemeral.yaml     |   5 +-
 variants/scale-1-ephemeral/kustomization.yaml |   6 +-
 variants/scale-1/kustomization.yaml           |  14 +-
 variants/scale-1/wsrep-1.yaml                 |  12 --
 variants/scale-2/kustomization.yaml           |  14 +-
 variants/scale-2/replicas-2.yaml              |   7 -
 variants/scale-2/wsrep-2.yaml                 |  12 --
 23 files changed, 141 insertions(+), 487 deletions(-)
 delete mode 100644 00namespace.yml
 delete mode 100644 10conf-d.yml
 delete mode 100644 20mariadb-service.yml
 delete mode 100644 21mariadb-ready-service.yml
 delete mode 100644 50mariadb.yml
 create mode 100644 base/kustomization.yaml
 rename 30mysql-service.yml => base/mysql-service.yaml (57%)
 delete mode 100644 kustomization.yaml
 create mode 100755 test.sh
 delete mode 100644 variants/gke/gke-storageclass-ssd-regional-europewest1cd.yaml
 delete mode 100644 variants/gke/gke-storageclass-ssd.yml
 delete mode 100644 variants/gke/kustomization.yaml
 delete mode 100644 variants/gke/volume-claims.yaml
 delete mode 100644 variants/scale-1/wsrep-1.yaml
 delete mode 100644 variants/scale-2/replicas-2.yaml
 delete mode 100644 variants/scale-2/wsrep-2.yaml

diff --git a/.gitignore b/.gitignore
index 1269488..3a72626 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1 +1,2 @@
 data
+kubeconfig
diff --git a/00namespace.yml b/00namespace.yml
deleted file mode 100644
index b792c1b..0000000
--- a/00namespace.yml
+++ /dev/null
@@ -1,5 +0,0 @@
----
-apiVersion: v1
-kind: Namespace
-metadata:
-  name: mysql
diff --git a/10conf-d.yml b/10conf-d.yml
deleted file mode 100644
index 39a2700..0000000
--- a/10conf-d.yml
+++ /dev/null
@@ -1,158 +0,0 @@
-kind: ConfigMap
-metadata:
-  name: conf-d
-  namespace: mysql
-apiVersion: v1
-data:
-  datadir.cnf: |
-    [mysqld]
-    datadir=/data/db
-  galera.cnf: |
-    #
-    # * Galera-related settings
-    #
-    # https://mariadb.com/kb/en/mariadb/galera-cluster-system-variables/
-    #
-    [galera]
-    # Mandatory settings
-    wsrep_on=ON
-    wsrep_provider="/usr/lib/galera/libgalera_smm.so"
-    #init-new-cluster#wsrep_new_cluster=TRUE
-    #init-recover#wsrep_recover=TRUE
-    binlog_format=ROW
-    default_storage_engine=InnoDB
-    innodb_autoinc_lock_mode=2
-    #init-wsrep#wsrep_cluster_address="gcomm://mariadb-0.mariadb,mariadb-1.mariadb,mariadb-2.mariadb"
-    wsrep-sst-method=rsync
-
-    #
-    # Allow server to accept connections on all interfaces.
-    #
-    bind-address=0.0.0.0
-    #
-    # Optional setting
-    #wsrep_slave_threads=1
-    #innodb_flush_log_at_trx_commit=0
-  utf8.cnf: |
-    [client-server]
-
-    # This will be passed to all mysql clients
-    [client]
-    default-character-set = utf8mb4
-
-    # The MySQL server
-    [mysqld]
-    character-set-server = utf8mb4
-    collation-server = utf8mb4_unicode_ci
-    init-connect='SET NAMES utf8mb4'
-
-    [mysql]
-    default-character-set = utf8mb4
-
-    [mysqldump]
-
-    # This group is only read by MariaDB servers, not by MySQL.
-    # If you use the same .cnf file for MySQL and MariaDB,
-    # you can put MariaDB-only options here
-    [mariadb]
-
-    [mariadb-10.1]
-
-    [mariadb-10.2]
-  init.sh: |
-    #!/bin/bash
-    set -x
-    [ "$(pwd)" != "/etc/mysql/conf.d" ] && cp * /etc/mysql/conf.d/
-
-    HOST_ID=${HOSTNAME##*-}
-
-    STATEFULSET_SERVICE=$(dnsdomainname -d)
-    POD_FQDN=$(dnsdomainname -A)
-
-    echo "This is pod $HOST_ID ($POD_FQDN) for statefulset $STATEFULSET_SERVICE"
-
-    [ -z "$WSREP_CLUSTER_ADDRESS" ] && echo "Missing WSREP_CLUSTER_ADDRESS env" && exit 1
-    sed -i "s|^#init-wsrep#.*|wsrep_cluster_address=$WSREP_CLUSTER_ADDRESS|" /etc/mysql/conf.d/galera.cnf
-
-    [ -z "$DATADIR" ] && exit "Missing DATADIR variable" && exit 1
-
-    SUGGEST_EXEC_COMMAND="kubectl --namespace=$POD_NAMESPACE exec -c init-config $POD_NAME --"
-
-    function wsrepNewCluster {
-      sed -i 's|^#init-new-cluster#||' /etc/mysql/conf.d/galera.cnf
-    }
-
-    function wsrepRecover {
-      sed -i 's|^#init-recover#||' /etc/mysql/conf.d/galera.cnf
-    }
-
-    function wsrepForceBootstrap {
-      sed -i 's|safe_to_bootstrap: 0|safe_to_bootstrap: 1|' /data/db/grastate.dat
-    }
-
-    [[ $STATEFULSET_SERVICE = mariadb.* ]] || echo "WARNING: unexpected service name $STATEFULSET_SERVICE, Peer detection below may fail falsely."
-
-    if [ $HOST_ID -eq 0 ]; then
-      echo "This is the 1st statefulset pod. Checking if the statefulset is down ..."
-      getent hosts mariadb-ready
-      [ $? -eq 2 ] && {
-        # https://github.com/docker-library/mariadb/commit/f76084f0f9dc13f29cce48c727440eb79b4e92fa#diff-b0fa4b30392406b32de6b8ffe36e290dR80
-        if [ ! -d "$DATADIR/mysql" ]; then
-          echo "No database in $DATADIR; configuring $POD_NAME for initial start"
-          wsrepNewCluster
-        else
-          set +x
-          echo "----- ACTION REQUIRED -----"
-          echo "No peers found, but data exists. To start in wsrep_new_cluster mode, run:"
-          echo "  $SUGGEST_EXEC_COMMAND touch /tmp/confirm-new-cluster"
-          echo "Or to start in recovery mode, to see replication state, run:"
-          echo "  $SUGGEST_EXEC_COMMAND touch /tmp/confirm-recover"
-          echo "Or to force bootstrap on this node, potentially losing writes, run:"
-          echo "  $SUGGEST_EXEC_COMMAND touch /tmp/confirm-force-bootstrap"
-          #echo "    NOTE This bypasses the following warning from new cluster mode:"
-          #echo "    It may not be safe to bootstrap the cluster from this node. It was not the last one to leave the cluster and may not contain all the updates. To force cluster bootstrap with this node, edit the grastate.dat file manually and set safe_to_bootstrap to 1 ."
-          echo "Or to try a regular start (for example after recovery + manual intervention), run:"
-          echo "  $SUGGEST_EXEC_COMMAND touch /tmp/confirm-resume"
-          if [ ! -z "$AUTO_RECOVERY_MODE" ]; then
-            echo "The AUTO_RECOVERY_MODE env was set to $AUTO_RECOVERY_MODE, will trigger that choice"
-            touch /tmp/$AUTO_RECOVERY_MODE
-          else
-            echo "Waiting for response ..."
-          fi
-          while [ ! -f /tmp/confirm-resume ]; do
-            if [ "$AUTO_NEW_CLUSTER" = "true" ]; then
-              echo "The AUTO_NEW_CLUSTER env was set to $AUTO_NEW_CLUSTER, will proceed without confirmation"
-              echo "NOTE this env is deprecated, use AUTO_RECOVERY_MODE instead"
-              wsrepNewCluster
-              touch /tmp/confirm-resume
-            elif [ -f /tmp/confirm-new-cluster ]; then
-              echo "Confirmation received. Resuming new cluster start ..."
-              wsrepNewCluster
-              touch /tmp/confirm-resume
-            elif [ -f /tmp/confirm-force-bootstrap ]; then
-              echo "Forcing bootstrap on this node ..."
-              wsrepForceBootstrap
-              touch /tmp/confirm-new-cluster
-            elif [ -f /tmp/confirm-recover ]; then
-              echo "Confirmation received. Resuming in recovery mode."
-              echo "Note: to start the other pods you need to edit OrderedReady and add a command: --wsrep-recover"
-              wsrepRecover
-              touch /tmp/confirm-resume
-            fi
-            sleep 1
-          done
-          rm /tmp/confirm-*
-          set -x
-        fi
-      }
-    else
-      getent hosts mariadb-ready
-      [ $? -eq 2 ] && {
-        echo "This is NOT the 1st statefulset pod. Must not go up as primary."
-        echo "Found no ready pods. Will exit to trigger a crash loop back off."
-        exit 1
-      }
-    fi
-
-    # https://github.com/docker-library/mariadb/blob/master/10.2/docker-entrypoint.sh#L62
-    mysqld --verbose --help --log-bin-index="$(mktemp -u)" | tee /tmp/mariadb-start-config | grep -e ^version -e ^datadir -e ^wsrep -e ^binlog -e ^character-set -e ^collation
diff --git a/20mariadb-service.yml b/20mariadb-service.yml
deleted file mode 100644
index c9d5f94..0000000
--- a/20mariadb-service.yml
+++ /dev/null
@@ -1,26 +0,0 @@
-# the "Headless Service, used to control the network domain"
----
-apiVersion: v1
-kind: Service
-metadata:
-  name: mariadb
-  namespace: mysql
-  annotations:
-    service.alpha.kubernetes.io/tolerate-unready-endpoints: "true"
-spec:
-  publishNotReadyAddresses: true
-  clusterIP: None
-  selector:
-    app: mariadb
-  ports:
-    - port: 3306
-      name: mysql
-    - port: 4444
-      name: sst
-    - port: 4567
-      name: replication
-    - protocol: UDP
-      port: 4567
-      name: replicationudp
-    - port: 4568
-      name: ist
diff --git a/21mariadb-ready-service.yml b/21mariadb-ready-service.yml
deleted file mode 100644
index 6895dbd..0000000
--- a/21mariadb-ready-service.yml
+++ /dev/null
@@ -1,14 +0,0 @@
-# Used with `getent hosts` to check how many pods that are ready
----
-apiVersion: v1
-kind: Service
-metadata:
-  name: mariadb-ready
-  namespace: mysql
-  annotations:
-    service.alpha.kubernetes.io/tolerate-unready-endpoints: "false"
-spec:
-  publishNotReadyAddresses: false
-  clusterIP: None
-  selector:
-    app: mariadb
diff --git a/50mariadb.yml b/50mariadb.yml
deleted file mode 100644
index 9a1843d..0000000
--- a/50mariadb.yml
+++ /dev/null
@@ -1,121 +0,0 @@
-apiVersion: apps/v1
-kind: StatefulSet
-metadata:
-  name: mariadb
-  namespace: mysql
-spec:
-  selector:
-    matchLabels:
-      app: mariadb
-  serviceName: "mariadb"
-  replicas: 3
-  podManagementPolicy: Parallel
-  template:
-    metadata:
-      labels:
-        app: mariadb
-      annotations:
-        prometheus.io/scrape: "true"
-        prometheus.io/port: "9104"
-    spec:
-      terminationGracePeriodSeconds: 30
-      initContainers:
-      - name: init-config
-        command: ['/bin/bash', '/etc/mysql/conf.d-configmap/init.sh']
-        env:
-        - name: K8S_NODE_NAME
-          valueFrom:
-            fieldRef:
-              fieldPath: spec.nodeName
-        - name: POD_NAME
-          valueFrom:
-            fieldRef:
-              fieldPath: metadata.name
-        - name: POD_NAMESPACE
-          valueFrom:
-            fieldRef:
-              fieldPath: metadata.namespace
-        - name: DATADIR
-          value: /data/db
-        - name: AUTO_RECOVERY_MODE
-          value: confirm-force-bootstrap
-        - name: WSREP_CLUSTER_ADDRESS
-          value: "gcomm://mariadb-0.mariadb,mariadb-1.mariadb,mariadb-2.mariadb"
-        workingDir: /etc/mysql/conf.d-configmap
-        volumeMounts:
-        - name: mysql
-          mountPath: /data
-        - name: conf-readonly
-          mountPath: /etc/mysql/conf.d-configmap
-        - name: conf
-          mountPath: /etc/mysql/conf.d
-        - name: initdb
-          mountPath: /docker-entrypoint-initdb.d
-        image: mariadb:10.2.36-bionic@sha256:b7be3ade3d5441c79b5c8a9cf2c2269f14bf420876a06def7d50e1763f042238
-      containers:
-      - name: mariadb
-        image: mariadb:10.2.36-bionic@sha256:b7be3ade3d5441c79b5c8a9cf2c2269f14bf420876a06def7d50e1763f042238
-        ports:
-        - containerPort: 3306
-          name: mysql
-        - containerPort: 4444
-          name: sst
-        - containerPort: 4567
-          name: replication
-        - containerPort: 4567
-          protocol: UDP
-          name: replicationudp
-        - containerPort: 4568
-          name: ist
-        env:
-        - name: MYSQL_ROOT_HOST
-          value: "localhost"
-        - name: MYSQL_ALLOW_EMPTY_PASSWORD
-          value: "yes"
-        - name: MYSQL_INITDB_SKIP_TZINFO
-          value: "yes"
-        readinessProbe:
-          exec:
-            command:
-            - /bin/sh
-            - -ec
-            - test ON = "$(mysql -e "SHOW STATUS LIKE 'wsrep_ready';" -N -B | sed 's/wsrep_ready\t//')"
-          initialDelaySeconds: 30
-        livenessProbe:
-          exec:
-            command:
-            - mysql
-            - -e
-            - "SHOW DATABASES;"
-          # might need to be tweaked for large initial state transfers
-          initialDelaySeconds: 60
-        volumeMounts:
-        - name: mysql
-          mountPath: /data
-        - name: conf
-          mountPath: /etc/mysql/conf.d
-        - name: initdb
-          mountPath: /docker-entrypoint-initdb.d
-      - name: metrics
-        image: prom/mysqld-exporter:v0.12.1@sha256:9fe9938c4ac9216cc24005144338f14fac4f604f139b481cc541bead008db3c1
-        env:
-        - name: DATA_SOURCE_NAME
-          value: root@(localhost:3306)/
-        ports:
-        - containerPort: 9104
-      volumes:
-      - name: conf
-        emptyDir: {}
-      - name: conf-readonly
-        configMap:
-          name: conf-d
-      - name: initdb
-        emptyDir: {}
-  volumeClaimTemplates:
-  - metadata:
-      name: mysql
-    spec:
-      accessModes: [ "ReadWriteOnce" ]
-      resources:
-        requests:
-          storage: 1Gi
diff --git a/README.md b/README.md
index 2a23044..759aa9d 100644
--- a/README.md
+++ b/README.md
@@ -1,67 +1,7 @@
 # MySQL with automatic failover on Kubernetes
 
-This is a galera cluster setup, with plain manifests.
-We actually use it in production, though with modest loads.
+RETIRED See [v2.1.0](https://github.com/Yolean/kubernetes-mysql-cluster/tree/v2.1.0) for the original setup.
 
-## Get started
-
-First create a storage class `mysql-data`. See exampels in `./configure/`.
-You might also want to edit the volume size request, at the bottom of `./50mariadb.yml`.
-
-Then: `kubectl apply -f .`.
-
-### Cluster Health
-
-Readiness and liveness probes will only assert client-level health of individual pods.
-Watch logs for "sst" or "Quorum results", or run this quick check:
-```
-for i in 0 1 2; do kubectl -n mysql exec mariadb-$i -- mysql -e "SHOW STATUS LIKE 'wsrep_cluster_size';" -N; done
-```
-
-Port 9104 exposes plaintext metris in [Prometheus](https://prometheus.io/docs/concepts/data_model/) scrape format.
-```
-# with kubectl -n mysql port-forward mariadb-0 9104:9104
-$ curl -s http://localhost:9104/metrics | grep ^mysql_global_status_wsrep_cluster_size
-mysql_global_status_wsrep_cluster_size 3
-```
-
-A reasonable alert is on `mysql_global_status_wsrep_cluster_size` staying below the desired number of replicas.
-
-### Cluster un-health
-
-We need to assume a couple of things here. First and foremost:
-Production clusters are configured so that the statefulset pods do not go down together.
-
- * Pods are properly spread across nodes.
- * Nodes are spread across multiple availability zones.
-
-Let's also assume that there is monitoring.
-Any `wsrep_cluster_size` issue (see above), or absence of `wsrep_cluster_size`
-should lead to a human being paged.
-
-Rarity combined with manual attention means that this statefulset can/should avoid
-attempts at automatic [recovery](http://galeracluster.com/documentation-webpages/pcrecovery.html).
-The reason for that being: we can't test for failure modes properly,
-as they depend on the Kubernetes setup.
-Automation may appoint the wrong leader - losing writes -
-or cause split-brain situations.
-
-We can however support detection in the init script.
-
-It's normal operations to scale down to two instances
-- actually one instance, but nodes should be considered ephemeral so don't do that -
-and up to any number of replicas.
-
-### phpMyAdmin
-
-Carefully consider the security implications before you create this. Note that it uses a non-official image.
-
-```
-kubectl apply -f myadmin/
-```
-
-PhpMyAdmin has a login page where you need a mysql user. To allow login (with full access) create a user with your choice of password:
-
-```
-kubectl -n mysql exec mariadb-0 -- mysql -e "CREATE USER 'phpmyadmin'@'%' IDENTIFIED BY 'my-admin-pw'; GRANT ALL ON *.* TO 'phpmyadmin'@'%' WITH GRANT OPTION;"
-```
+We've now adopted the
+[Bitnami mariadb-galera helm chart](https://github.com/bitnami/charts/tree/master/bitnami/mariadb-galera)
+via [unhelm](https://github.com/Yolean/unhelm/tree/master/mysql) so we can use [Kustomize](https://kubernetes.io/docs/reference/generated/kubectl/kubectl-commands#kustomize).
diff --git a/base/kustomization.yaml b/base/kustomization.yaml
new file mode 100644
index 0000000..4b5670c
--- /dev/null
+++ b/base/kustomization.yaml
@@ -0,0 +1,40 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+# See also the override below
+namespace: mysqltest
+
+resources:
+- ../../unhelm/mysql/mariadb-galera/
+- ./mysql-service.yaml
+
+secretGenerator:
+- name: mariadb-auth
+  literals:
+  - mariadb-root-password=TESTROOT
+  - mariadb-galera-mariabackup-password=TESTBACKUP
+  - mariadb-password=TESTMARIADB
+
+images:
+- name: docker.io/bitnami/mariadb-galera
+  # 10.5.10-debian-10-r26
+  digest: sha256:c8be8e01d6e5a594e1caa257ef0c1ce42d4a66e74202b093606c68fa8e6383b9
+- name: docker.io/bitnami/mysqld-exporter
+  # 0.13.0-debian-10-r12
+  digest: sha256:6851348a52762a77fec89a7bc101e3984a1a7ec5322ffd350bc81a9f3d6e4d11
+
+patchesStrategicMerge:
+  # for namespace to match the value at the top of this file
+- |-
+  apiVersion: apps/v1
+  kind: StatefulSet
+  metadata:
+    name: ystack-mariadb-galera
+  spec:
+    template:
+      spec:
+        containers:
+        - name: "mariadb-galera"
+          env:
+          - name: MARIADB_GALERA_CLUSTER_ADDRESS
+            value: "gcomm://ystack-mariadb-galera-headless.mysqltest.svc.cluster.local"
diff --git a/30mysql-service.yml b/base/mysql-service.yaml
similarity index 57%
rename from 30mysql-service.yml
rename to base/mysql-service.yaml
index dee62dc..9b9c88d 100644
--- a/30mysql-service.yml
+++ b/base/mysql-service.yaml
@@ -1,12 +1,11 @@
----
 apiVersion: v1
 kind: Service
 metadata:
   name: mysql
-  namespace: mysql
 spec:
   ports:
   - port: 3306
     name: mysql
   selector:
-    app: mariadb
+    app.kubernetes.io/name: mariadb-galera
+    app.kubernetes.io/instance: ystack
diff --git a/kustomization.yaml b/kustomization.yaml
deleted file mode 100644
index 176a01a..0000000
--- a/kustomization.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-resources:
-- 10conf-d.yml
-- 20mariadb-service.yml
-- 21mariadb-ready-service.yml
-- 30mysql-service.yml
-- 50mariadb.yml
diff --git a/test.sh b/test.sh
new file mode 100755
index 0000000..d2917e4
--- /dev/null
+++ b/test.sh
@@ -0,0 +1,48 @@
+#!/usr/bin/env bash
+[ -z "$DEBUG" ] || set -x
+set -eo pipefail
+DIR="$(dirname $0)"
+
+export KUBECONFIG=$DIR/kubeconfig
+
+show_cluster_size() {
+  kubectl -n mysqltest exec -c mariadb-galera ystack-mariadb-galera-0 -- mysql -u root "-pTESTROOT" -N -e "SHOW STATUS LIKE 'wsrep_cluster_size';" 
+}
+
+k3d cluster create mysqltest --agents 3 --agents-memory 512M
+
+kubectl create namespace mysqltest
+
+kubectl -n mysqltest apply -k ./base
+sleep 1
+kubectl -n mysqltest wait --timeout=120s --for=condition=Ready pod/ystack-mariadb-galera-0
+
+kubectl -n mysqltest exec -c mariadb-galera ystack-mariadb-galera-0 -- mysql -u root "-pTESTROOT" -N -e "SHOW STATUS LIKE 'wsrep_cluster_size';"
+
+kubectl -n mysqltest scale --replicas=2 statefulset/ystack-mariadb-galera
+kubectl -n mysqltest rollout status statefulset/ystack-mariadb-galera
+
+show_cluster_size
+
+kubectl -n mysqltest scale --replicas=4 statefulset/ystack-mariadb-galera
+kubectl -n mysqltest rollout status statefulset/ystack-mariadb-galera
+
+show_cluster_size
+
+kubectl -n mysqltest delete --wait=false pod/ystack-mariadb-galera-1
+sleep 1
+show_cluster_size
+kubectl -n mysqltest wait --for=condition=Ready --timeout=60s pod/ystack-mariadb-galera-1
+show_cluster_size
+
+# With curl from somewhere we wouldn't need a pod name for show_cluster_size
+# curl -s http://ystack-mariadb-galera-metrics.mysqltest:9104/metrics | grep cluster_size
+#kubectl -n mysqltest delete --wait=false pod/ystack-mariadb-galera-0
+kubectl -n mysqltest delete --wait=false pod/ystack-mariadb-galera-2
+kubectl -n mysqltest delete --wait=false pod/ystack-mariadb-galera-1
+sleep 1
+show_cluster_size
+kubectl -n mysqltest wait --for=condition=Ready --timeout=60s pod/ystack-mariadb-galera-1
+show_cluster_size
+
+k3d cluster delete mysqltest
diff --git a/variants/gke/gke-storageclass-ssd-regional-europewest1cd.yaml b/variants/gke/gke-storageclass-ssd-regional-europewest1cd.yaml
deleted file mode 100644
index c5bd899..0000000
--- a/variants/gke/gke-storageclass-ssd-regional-europewest1cd.yaml
+++ /dev/null
@@ -1,11 +0,0 @@
-kind: StorageClass
-apiVersion: storage.k8s.io/v1
-metadata:
-  name: mysql-data
-provisioner: kubernetes.io/gce-pd
-reclaimPolicy: Retain
-allowVolumeExpansion: true
-parameters:
-  type: pd-ssd
-  replication-type: regional-pd
-  zones: europe-west1-c, europe-west1-d
diff --git a/variants/gke/gke-storageclass-ssd.yml b/variants/gke/gke-storageclass-ssd.yml
deleted file mode 100644
index cd4852f..0000000
--- a/variants/gke/gke-storageclass-ssd.yml
+++ /dev/null
@@ -1,9 +0,0 @@
-kind: StorageClass
-apiVersion: storage.k8s.io/v1
-metadata:
-  name: mysql-data
-provisioner: kubernetes.io/gce-pd
-reclaimPolicy: Retain
-allowVolumeExpansion: true
-parameters:
-  type: pd-ssd
diff --git a/variants/gke/kustomization.yaml b/variants/gke/kustomization.yaml
deleted file mode 100644
index 50b758f..0000000
--- a/variants/gke/kustomization.yaml
+++ /dev/null
@@ -1,7 +0,0 @@
-bases:
-- github.com/Yolean/kubernetes-mysql-cluster?ref=009cad0
-resources:
-- gke-storageclass-ssd.yml
-#- gke-storageclass-ssd-regional-europewest1cd.yaml
-patchesStrategicMerge:
-- volume-claims.yaml
diff --git a/variants/gke/volume-claims.yaml b/variants/gke/volume-claims.yaml
deleted file mode 100644
index 0de97f6..0000000
--- a/variants/gke/volume-claims.yaml
+++ /dev/null
@@ -1,16 +0,0 @@
----
-apiVersion: apps/v1
-kind: StatefulSet
-metadata:
-  name: mariadb
-  namespace: mysql
-spec:
-  volumeClaimTemplates:
-  - metadata:
-      name: mysql
-    spec:
-      accessModes: [ "ReadWriteOnce" ]
-      storageClassName: mysql-data
-      resources:
-        requests:
-          storage: 1Gi
diff --git a/variants/namespace/kustomization.yaml b/variants/namespace/kustomization.yaml
index 16d767e..ea7a52d 100644
--- a/variants/namespace/kustomization.yaml
+++ b/variants/namespace/kustomization.yaml
@@ -1,3 +1,22 @@
-bases:
-- github.com/Yolean/kubernetes-mysql-cluster?ref=549e804
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+- ../../base
+
 namespace: analytics
+
+patchesStrategicMerge:
+- |-
+  apiVersion: apps/v1
+  kind: StatefulSet
+  metadata:
+    name: ystack-mariadb-galera
+  spec:
+    template:
+      spec:
+        containers:
+        - name: "mariadb-galera"
+          env:
+          - name: MARIADB_GALERA_CLUSTER_ADDRESS
+            value: "gcomm://ystack-mariadb-galera-headless.analytics.svc.cluster.local"
diff --git a/variants/scale-1-ephemeral/ephemeral.yaml b/variants/scale-1-ephemeral/ephemeral.yaml
index c2f015b..228fd35 100644
--- a/variants/scale-1-ephemeral/ephemeral.yaml
+++ b/variants/scale-1-ephemeral/ephemeral.yaml
@@ -1,12 +1,11 @@
 apiVersion: apps/v1
 kind: StatefulSet
 metadata:
-  name: mariadb
-  namespace: mysql
+  name: ystack-mariadb-galera
 spec:
   template:
     spec:
       volumes:
-      - name: mysql
+      - name: data
         emptyDir: {}
   volumeClaimTemplates: []
diff --git a/variants/scale-1-ephemeral/kustomization.yaml b/variants/scale-1-ephemeral/kustomization.yaml
index 5498756..542e2ed 100644
--- a/variants/scale-1-ephemeral/kustomization.yaml
+++ b/variants/scale-1-ephemeral/kustomization.yaml
@@ -1,4 +1,8 @@
-bases:
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
 - ../scale-1
+
 patchesStrategicMerge:
 - ephemeral.yaml
diff --git a/variants/scale-1/kustomization.yaml b/variants/scale-1/kustomization.yaml
index 6e1fa9a..ed8a5b8 100644
--- a/variants/scale-1/kustomization.yaml
+++ b/variants/scale-1/kustomization.yaml
@@ -1,5 +1,9 @@
-bases:
-- github.com/Yolean/kubernetes-mysql-cluster?ref=a9e6956000a31c973d183d8c318d6828cd73c26c
-patchesStrategicMerge:
-- replicas-1.yaml
-- wsrep-1.yaml
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+- ../../base
+
+replicas:
+- name: ystack-mariadb-galera
+  count: 1
diff --git a/variants/scale-1/wsrep-1.yaml b/variants/scale-1/wsrep-1.yaml
deleted file mode 100644
index 95a61ef..0000000
--- a/variants/scale-1/wsrep-1.yaml
+++ /dev/null
@@ -1,12 +0,0 @@
-apiVersion: apps/v1
-kind: StatefulSet
-metadata:
-  name: mariadb
-spec:
-  template:
-    spec:
-      initContainers:
-      - name: init-config
-        env:
-        - name: WSREP_CLUSTER_ADDRESS
-          value: "gcomm://mariadb-0.mariadb"
diff --git a/variants/scale-2/kustomization.yaml b/variants/scale-2/kustomization.yaml
index 2f7540f..b7aa659 100644
--- a/variants/scale-2/kustomization.yaml
+++ b/variants/scale-2/kustomization.yaml
@@ -1,5 +1,9 @@
-bases:
-- github.com/Yolean/kubernetes-mysql-cluster?ref=8c4439f
-patchesStrategicMerge:
-- replicas-2.yaml
-- wsrep-2.yaml
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+- ../../base
+
+replicas:
+- name: ystack-mariadb-galera
+  count: 2
diff --git a/variants/scale-2/replicas-2.yaml b/variants/scale-2/replicas-2.yaml
deleted file mode 100644
index e57c624..0000000
--- a/variants/scale-2/replicas-2.yaml
+++ /dev/null
@@ -1,7 +0,0 @@
-apiVersion: apps/v1
-kind: StatefulSet
-metadata:
-  name: mariadb
-  namespace: mysql
-spec:
-  replicas: 2
diff --git a/variants/scale-2/wsrep-2.yaml b/variants/scale-2/wsrep-2.yaml
deleted file mode 100644
index fc09f0e..0000000
--- a/variants/scale-2/wsrep-2.yaml
+++ /dev/null
@@ -1,12 +0,0 @@
-apiVersion: apps/v1
-kind: StatefulSet
-metadata:
-  name: mariadb
-spec:
-  template:
-    spec:
-      initContainers:
-      - name: init-config
-        env:
-        - name: WSREP_CLUSTER_ADDRESS
-          value: "gcomm://mariadb-0.mariadb,mariadb-1.mariadb"

From 63149b78adbbf1cff8ab8355d5ba0fefbd31e567 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Sun, 20 Jun 2021 21:15:55 +0200
Subject: [PATCH 2/7] Always use immutable refs. This is the one with utf8mb4.

---
 base/kustomization.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/base/kustomization.yaml b/base/kustomization.yaml
index 4b5670c..3a49701 100644
--- a/base/kustomization.yaml
+++ b/base/kustomization.yaml
@@ -5,7 +5,7 @@ kind: Kustomization
 namespace: mysqltest
 
 resources:
-- ../../unhelm/mysql/mariadb-galera/
+- github.com/Yolean/unhelm/mysql/mariadb-galera?ref=e9bf2b43b34ff4698d84bd9afa766ad6b2d35809
 - ./mysql-service.yaml
 
 secretGenerator:

From a868fca77da8cc85dca4e2f48412aae86e6fdcab Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Sun, 20 Jun 2021 21:16:15 +0200
Subject: [PATCH 3/7] The managed-by label seemed misleading

---
 base/kustomization.yaml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/base/kustomization.yaml b/base/kustomization.yaml
index 3a49701..63129b3 100644
--- a/base/kustomization.yaml
+++ b/base/kustomization.yaml
@@ -4,6 +4,9 @@ kind: Kustomization
 # See also the override below
 namespace: mysqltest
 
+commonLabels:
+  app.kubernetes.io/managed-by: unhelm
+
 resources:
 - github.com/Yolean/unhelm/mysql/mariadb-galera?ref=e9bf2b43b34ff4698d84bd9afa766ad6b2d35809
 - ./mysql-service.yaml

From b70d36c730bbc5f0a4720242966ce868cbbd8002 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Mon, 21 Jun 2021 08:43:02 +0200
Subject: [PATCH 4/7] Experiments with Parallel pod management

---
 base-bootstrap-force/bootstrap-force.yaml | 12 ++++++++++++
 base-bootstrap-force/kustomization.yaml   |  8 ++++++++
 base-bootstrap/bootstrap-1-replica.yaml   | 19 +++++++++++++++++++
 base-bootstrap/kustomization.yaml         |  8 ++++++++
 base/bootstrap-no.yaml                    | 19 +++++++++++++++++++
 base/kustomization.yaml                   | 15 ++++++++++-----
 test.sh                                   | 20 +++++++++++++++++---
 variants/scale-1/replicas-1.yaml          |  7 -------
 8 files changed, 93 insertions(+), 15 deletions(-)
 create mode 100644 base-bootstrap-force/bootstrap-force.yaml
 create mode 100644 base-bootstrap-force/kustomization.yaml
 create mode 100644 base-bootstrap/bootstrap-1-replica.yaml
 create mode 100644 base-bootstrap/kustomization.yaml
 create mode 100644 base/bootstrap-no.yaml
 delete mode 100644 variants/scale-1/replicas-1.yaml

diff --git a/base-bootstrap-force/bootstrap-force.yaml b/base-bootstrap-force/bootstrap-force.yaml
new file mode 100644
index 0000000..e686533
--- /dev/null
+++ b/base-bootstrap-force/bootstrap-force.yaml
@@ -0,0 +1,12 @@
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: ystack-mariadb-galera
+spec:
+  template:
+    spec:
+      containers:
+      - name: "mariadb-galera"
+        env:
+        - name: MARIADB_GALERA_FORCE_SAFETOBOOTSTRAP
+          value: "yes"
diff --git a/base-bootstrap-force/kustomization.yaml b/base-bootstrap-force/kustomization.yaml
new file mode 100644
index 0000000..f1b0f22
--- /dev/null
+++ b/base-bootstrap-force/kustomization.yaml
@@ -0,0 +1,8 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+- ../base-bootstrap
+
+patchesStrategicMerge:
+- ./bootstrap-force.yaml
diff --git a/base-bootstrap/bootstrap-1-replica.yaml b/base-bootstrap/bootstrap-1-replica.yaml
new file mode 100644
index 0000000..d765f5c
--- /dev/null
+++ b/base-bootstrap/bootstrap-1-replica.yaml
@@ -0,0 +1,19 @@
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: ystack-mariadb-galera
+spec:
+  replicas: 1
+  template:
+    spec:
+      containers:
+      - name: "mariadb-galera"
+        # https://github.com/bitnami/charts/blob/70822dd1aea385cad908462be5fc1004ef8b5e07/bitnami/mariadb-galera/templates/statefulset.yaml#L68
+        command:
+        - /opt/bitnami/scripts/mariadb-galera/entrypoint.sh
+        - /opt/bitnami/scripts/mariadb-galera/run.sh
+        env:
+        - name: MARIADB_GALERA_CLUSTER_BOOTSTRAP
+          value: "yes"
+        - name: MARIADB_GALERA_FORCE_SAFETOBOOTSTRAP
+          value: "no"
diff --git a/base-bootstrap/kustomization.yaml b/base-bootstrap/kustomization.yaml
new file mode 100644
index 0000000..1d9b092
--- /dev/null
+++ b/base-bootstrap/kustomization.yaml
@@ -0,0 +1,8 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+- ../base
+
+patchesStrategicMerge:
+- ./bootstrap-1-replica.yaml
diff --git a/base/bootstrap-no.yaml b/base/bootstrap-no.yaml
new file mode 100644
index 0000000..67eda7d
--- /dev/null
+++ b/base/bootstrap-no.yaml
@@ -0,0 +1,19 @@
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: ystack-mariadb-galera
+spec:
+  template:
+    spec:
+      containers:
+      - name: "mariadb-galera"
+        # https://github.com/bitnami/charts/blob/70822dd1aea385cad908462be5fc1004ef8b5e07/bitnami/mariadb-galera/templates/statefulset.yaml#L68
+        command:
+        - /opt/bitnami/scripts/mariadb-galera/entrypoint.sh
+        - /opt/bitnami/scripts/mariadb-galera/run.sh
+        env:
+        # WIP this seems to have no effect, a new cluster will start to split-brain anyway
+        - name: MARIADB_GALERA_CLUSTER_BOOTSTRAP
+          value: "no"
+        - name: MARIADB_GALERA_FORCE_SAFETOBOOTSTRAP
+          value: "no"
diff --git a/base/kustomization.yaml b/base/kustomization.yaml
index 63129b3..8cab939 100644
--- a/base/kustomization.yaml
+++ b/base/kustomization.yaml
@@ -8,7 +8,7 @@ commonLabels:
   app.kubernetes.io/managed-by: unhelm
 
 resources:
-- github.com/Yolean/unhelm/mysql/mariadb-galera?ref=e9bf2b43b34ff4698d84bd9afa766ad6b2d35809
+- github.com/Yolean/unhelm/mysql/mariadb-galera?ref=b28227ecaf5003bb3aa4114fc84844667060b962
 - ./mysql-service.yaml
 
 secretGenerator:
@@ -20,13 +20,18 @@ secretGenerator:
 
 images:
 - name: docker.io/bitnami/mariadb-galera
-  # 10.5.10-debian-10-r26
-  digest: sha256:c8be8e01d6e5a594e1caa257ef0c1ce42d4a66e74202b093606c68fa8e6383b9
+  # 10.5.10-debian-10-r30
+  digest: sha256:3dc89271c5051abfaa33f0540b802a75f869570fa8b7ed7f5e7e6178df4b5653
 - name: docker.io/bitnami/mysqld-exporter
-  # 0.13.0-debian-10-r12
-  digest: sha256:6851348a52762a77fec89a7bc101e3984a1a7ec5322ffd350bc81a9f3d6e4d11
+  # 0.13.0-debian-10-r16
+  digest: sha256:55eb650f79f8c6b4936890261f6e4f966940c2e2eaadd4374b93dbe372a8dbd3
+
+replicas:
+- name: ystack-mariadb-galera
+  count: 3
 
 patchesStrategicMerge:
+- ./bootstrap-no.yaml
   # for namespace to match the value at the top of this file
 - |-
   apiVersion: apps/v1
diff --git a/test.sh b/test.sh
index d2917e4..574ef8a 100755
--- a/test.sh
+++ b/test.sh
@@ -13,11 +13,18 @@ k3d cluster create mysqltest --agents 3 --agents-memory 512M
 
 kubectl create namespace mysqltest
 
+# With >1 replicas this leads to split brain despite ./base/bootstrap-no.yaml
+#kubectl -n mysqltest apply -k ./base
+
+kubectl -n mysqltest apply -k ./base-bootstrap
+kubectl -n mysqltest rollout status statefulset/ystack-mariadb-galera
+
+show_cluster_size
+
 kubectl -n mysqltest apply -k ./base
-sleep 1
-kubectl -n mysqltest wait --timeout=120s --for=condition=Ready pod/ystack-mariadb-galera-0
+kubectl -n mysqltest rollout status statefulset/ystack-mariadb-galera
 
-kubectl -n mysqltest exec -c mariadb-galera ystack-mariadb-galera-0 -- mysql -u root "-pTESTROOT" -N -e "SHOW STATUS LIKE 'wsrep_cluster_size';"
+show_cluster_size
 
 kubectl -n mysqltest scale --replicas=2 statefulset/ystack-mariadb-galera
 kubectl -n mysqltest rollout status statefulset/ystack-mariadb-galera
@@ -45,4 +52,11 @@ show_cluster_size
 kubectl -n mysqltest wait --for=condition=Ready --timeout=60s pod/ystack-mariadb-galera-1
 show_cluster_size
 
+echo "From zero we expect \"base\" to fail to start"
+
+kubectl -n mysqltest scale --replicas=0 statefulset/ystack-mariadb-galera
+kubectl -n mysqltest rollout status statefulset/ystack-mariadb-galera
+kubectl -n mysqltest apply -k ./base
+kubectl -n mysqltest rollout status statefulset/ystack-mariadb-galera
+
 k3d cluster delete mysqltest
diff --git a/variants/scale-1/replicas-1.yaml b/variants/scale-1/replicas-1.yaml
deleted file mode 100644
index 2d5a351..0000000
--- a/variants/scale-1/replicas-1.yaml
+++ /dev/null
@@ -1,7 +0,0 @@
-apiVersion: apps/v1
-kind: StatefulSet
-metadata:
-  name: mariadb
-  namespace: mysql
-spec:
-  replicas: 1

From 7811624bc4da0e9fa55f24fb499b6ea63cbec3f3 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Mon, 21 Jun 2021 17:09:55 +0200
Subject: [PATCH 5/7] Tests the proposed recovery path

---
 base/kustomization.yaml |  1 +
 base/podmonitor.yaml    | 15 +++++++++++++++
 test.sh                 | 28 ++++++++++++++++++++++++++++
 3 files changed, 44 insertions(+)
 create mode 100644 base/podmonitor.yaml

diff --git a/base/kustomization.yaml b/base/kustomization.yaml
index 8cab939..bbea6fc 100644
--- a/base/kustomization.yaml
+++ b/base/kustomization.yaml
@@ -10,6 +10,7 @@ commonLabels:
 resources:
 - github.com/Yolean/unhelm/mysql/mariadb-galera?ref=b28227ecaf5003bb3aa4114fc84844667060b962
 - ./mysql-service.yaml
+- ./podmonitor.yaml
 
 secretGenerator:
 - name: mariadb-auth
diff --git a/base/podmonitor.yaml b/base/podmonitor.yaml
new file mode 100644
index 0000000..7768922
--- /dev/null
+++ b/base/podmonitor.yaml
@@ -0,0 +1,15 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PodMonitor
+metadata:
+  name: mariadb-galera
+  labels:
+    prometheus: now
+spec:
+  jobLabel: app.kubernetes.io/name
+  namespaceSelector:
+    any: false
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: mariadb-galera
+  podMetricsEndpoints:
+  - port: metrics
diff --git a/test.sh b/test.sh
index 574ef8a..531a814 100755
--- a/test.sh
+++ b/test.sh
@@ -11,6 +11,16 @@ show_cluster_size() {
 
 k3d cluster create mysqltest --agents 3 --agents-memory 512M
 
+# https://github.com/Yolean/ystack/blob/master/bin/y-cluster-assert-install
+ctx=""
+OPERATOR_VERSION=5555f492df250168657b72bb8cb60bec071de71f
+KUBERNETES_ASSERT_VERSION=cb66d46758654b819d0d4402857122dca1884bcb
+kubectl $ctx create namespace monitoring
+kubectl $ctx -n default apply -f https://github.com/prometheus-operator/prometheus-operator/raw/$OPERATOR_VERSION/bundle.yaml
+kubectl wait $ctx -n default --for=condition=Ready pod -l app.kubernetes.io/name=prometheus-operator
+kubectl $ctx -n monitoring apply -k github.com/Yolean/kubernetes-assert/example-small?ref=$KUBERNETES_ASSERT_VERSION
+kubectl wait $ctx -n monitoring --for=condition=Ready pod --all
+
 kubectl create namespace mysqltest
 
 # With >1 replicas this leads to split brain despite ./base/bootstrap-no.yaml
@@ -57,6 +67,24 @@ echo "From zero we expect \"base\" to fail to start"
 kubectl -n mysqltest scale --replicas=0 statefulset/ystack-mariadb-galera
 kubectl -n mysqltest rollout status statefulset/ystack-mariadb-galera
 kubectl -n mysqltest apply -k ./base
+kubectl -n mysqltest rollout status --timeout=30s statefulset/ystack-mariadb-galera || echo "Timeout is expected"
+kubectl -n mysqltest get pods
+echo "Prometheus will now report absent(mysql_global_status_wsrep_cluster_size) == 1"
+
+kubectl -n mysqltest apply -k ./base-bootstrap
+sleep 5
+kubectl -n mysqltest get pods
+kubectl -n mysqltest logs -c mariadb-galera ystack-mariadb-galera-0
+
+echo "Using bootstrap-force, assuming that pod 0 has the latest writes"
+echo "To bootstrap from a different node use the helm chart"
+kubectl -n mysqltest apply -k ./base-bootstrap-force
 kubectl -n mysqltest rollout status statefulset/ystack-mariadb-galera
+show_cluster_size
+
+echo "Upon bootstrap success, apply the regular base to prevent more bootstrapping"
+kubectl -n mysqltest apply -k ./base
+kubectl -n mysqltest rollout status statefulset/ystack-mariadb-galera
+show_cluster_size
 
 k3d cluster delete mysqltest

From dbafd9bb4f077a0a6d8e1a1a6ebc5ab98750e08a Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Mon, 21 Jun 2021 17:30:14 +0200
Subject: [PATCH 6/7] We can now override namespace without env patches

---
 base/kustomization.yaml |  5 ++---
 test.sh                 | 10 +++++-----
 2 files changed, 7 insertions(+), 8 deletions(-)

diff --git a/base/kustomization.yaml b/base/kustomization.yaml
index bbea6fc..1965e0b 100644
--- a/base/kustomization.yaml
+++ b/base/kustomization.yaml
@@ -1,7 +1,6 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 
-# See also the override below
 namespace: mysqltest
 
 commonLabels:
@@ -33,7 +32,7 @@ replicas:
 
 patchesStrategicMerge:
 - ./bootstrap-no.yaml
-  # for namespace to match the value at the top of this file
+  # The helm chart depends on the namespace for galera cluster address, but that's not necessary
 - |-
   apiVersion: apps/v1
   kind: StatefulSet
@@ -46,4 +45,4 @@ patchesStrategicMerge:
         - name: "mariadb-galera"
           env:
           - name: MARIADB_GALERA_CLUSTER_ADDRESS
-            value: "gcomm://ystack-mariadb-galera-headless.mysqltest.svc.cluster.local"
+            value: "gcomm://ystack-mariadb-galera-headless"
diff --git a/test.sh b/test.sh
index 531a814..feb3ed6 100755
--- a/test.sh
+++ b/test.sh
@@ -62,27 +62,27 @@ show_cluster_size
 kubectl -n mysqltest wait --for=condition=Ready --timeout=60s pod/ystack-mariadb-galera-1
 show_cluster_size
 
-echo "From zero we expect \"base\" to fail to start"
+echo "# From zero we expect \"base\" to fail to start"
 
 kubectl -n mysqltest scale --replicas=0 statefulset/ystack-mariadb-galera
 kubectl -n mysqltest rollout status statefulset/ystack-mariadb-galera
 kubectl -n mysqltest apply -k ./base
 kubectl -n mysqltest rollout status --timeout=30s statefulset/ystack-mariadb-galera || echo "Timeout is expected"
 kubectl -n mysqltest get pods
-echo "Prometheus will now report absent(mysql_global_status_wsrep_cluster_size) == 1"
+echo "# Prometheus will now report absent(mysql_global_status_wsrep_cluster_size) == 1"
 
 kubectl -n mysqltest apply -k ./base-bootstrap
 sleep 5
 kubectl -n mysqltest get pods
 kubectl -n mysqltest logs -c mariadb-galera ystack-mariadb-galera-0
 
-echo "Using bootstrap-force, assuming that pod 0 has the latest writes"
-echo "To bootstrap from a different node use the helm chart"
+echo "# Using bootstrap-force, assuming that pod 0 has the latest writes"
+echo "# To bootstrap from a different node use the helm chart"
 kubectl -n mysqltest apply -k ./base-bootstrap-force
 kubectl -n mysqltest rollout status statefulset/ystack-mariadb-galera
 show_cluster_size
 
-echo "Upon bootstrap success, apply the regular base to prevent more bootstrapping"
+echo "# Upon bootstrap success, apply the regular base to prevent more bootstrapping"
 kubectl -n mysqltest apply -k ./base
 kubectl -n mysqltest rollout status statefulset/ystack-mariadb-galera
 show_cluster_size

From 1cbc2d7b55c40d0ac56785762fbaf3d3b0918c96 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Mon, 21 Jun 2021 17:39:50 +0200
Subject: [PATCH 7/7] With plugin_load_add=auth_pam commented out

---
 base/kustomization.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/base/kustomization.yaml b/base/kustomization.yaml
index 1965e0b..2370669 100644
--- a/base/kustomization.yaml
+++ b/base/kustomization.yaml
@@ -7,7 +7,7 @@ commonLabels:
   app.kubernetes.io/managed-by: unhelm
 
 resources:
-- github.com/Yolean/unhelm/mysql/mariadb-galera?ref=b28227ecaf5003bb3aa4114fc84844667060b962
+- github.com/Yolean/unhelm/mysql/mariadb-galera?ref=606228e73ae4ca7d9fd0ec2820515a07b2b25cbd
 - ./mysql-service.yaml
 - ./podmonitor.yaml