diff --git a/.gitignore b/.gitignore index 1269488..3a72626 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ data +kubeconfig diff --git a/00namespace.yml b/00namespace.yml deleted file mode 100644 index b792c1b..0000000 --- a/00namespace.yml +++ /dev/null @@ -1,5 +0,0 @@ ---- -apiVersion: v1 -kind: Namespace -metadata: - name: mysql diff --git a/10conf-d.yml b/10conf-d.yml deleted file mode 100644 index 39a2700..0000000 --- a/10conf-d.yml +++ /dev/null @@ -1,158 +0,0 @@ -kind: ConfigMap -metadata: - name: conf-d - namespace: mysql -apiVersion: v1 -data: - datadir.cnf: | - [mysqld] - datadir=/data/db - galera.cnf: | - # - # * Galera-related settings - # - # https://mariadb.com/kb/en/mariadb/galera-cluster-system-variables/ - # - [galera] - # Mandatory settings - wsrep_on=ON - wsrep_provider="/usr/lib/galera/libgalera_smm.so" - #init-new-cluster#wsrep_new_cluster=TRUE - #init-recover#wsrep_recover=TRUE - binlog_format=ROW - default_storage_engine=InnoDB - innodb_autoinc_lock_mode=2 - #init-wsrep#wsrep_cluster_address="gcomm://mariadb-0.mariadb,mariadb-1.mariadb,mariadb-2.mariadb" - wsrep-sst-method=rsync - - # - # Allow server to accept connections on all interfaces. - # - bind-address=0.0.0.0 - # - # Optional setting - #wsrep_slave_threads=1 - #innodb_flush_log_at_trx_commit=0 - utf8.cnf: | - [client-server] - - # This will be passed to all mysql clients - [client] - default-character-set = utf8mb4 - - # The MySQL server - [mysqld] - character-set-server = utf8mb4 - collation-server = utf8mb4_unicode_ci - init-connect='SET NAMES utf8mb4' - - [mysql] - default-character-set = utf8mb4 - - [mysqldump] - - # This group is only read by MariaDB servers, not by MySQL. - # If you use the same .cnf file for MySQL and MariaDB, - # you can put MariaDB-only options here - [mariadb] - - [mariadb-10.1] - - [mariadb-10.2] - init.sh: | - #!/bin/bash - set -x - [ "$(pwd)" != "/etc/mysql/conf.d" ] && cp * /etc/mysql/conf.d/ - - HOST_ID=${HOSTNAME##*-} - - STATEFULSET_SERVICE=$(dnsdomainname -d) - POD_FQDN=$(dnsdomainname -A) - - echo "This is pod $HOST_ID ($POD_FQDN) for statefulset $STATEFULSET_SERVICE" - - [ -z "$WSREP_CLUSTER_ADDRESS" ] && echo "Missing WSREP_CLUSTER_ADDRESS env" && exit 1 - sed -i "s|^#init-wsrep#.*|wsrep_cluster_address=$WSREP_CLUSTER_ADDRESS|" /etc/mysql/conf.d/galera.cnf - - [ -z "$DATADIR" ] && exit "Missing DATADIR variable" && exit 1 - - SUGGEST_EXEC_COMMAND="kubectl --namespace=$POD_NAMESPACE exec -c init-config $POD_NAME --" - - function wsrepNewCluster { - sed -i 's|^#init-new-cluster#||' /etc/mysql/conf.d/galera.cnf - } - - function wsrepRecover { - sed -i 's|^#init-recover#||' /etc/mysql/conf.d/galera.cnf - } - - function wsrepForceBootstrap { - sed -i 's|safe_to_bootstrap: 0|safe_to_bootstrap: 1|' /data/db/grastate.dat - } - - [[ $STATEFULSET_SERVICE = mariadb.* ]] || echo "WARNING: unexpected service name $STATEFULSET_SERVICE, Peer detection below may fail falsely." - - if [ $HOST_ID -eq 0 ]; then - echo "This is the 1st statefulset pod. Checking if the statefulset is down ..." - getent hosts mariadb-ready - [ $? -eq 2 ] && { - # https://github.com/docker-library/mariadb/commit/f76084f0f9dc13f29cce48c727440eb79b4e92fa#diff-b0fa4b30392406b32de6b8ffe36e290dR80 - if [ ! -d "$DATADIR/mysql" ]; then - echo "No database in $DATADIR; configuring $POD_NAME for initial start" - wsrepNewCluster - else - set +x - echo "----- ACTION REQUIRED -----" - echo "No peers found, but data exists. To start in wsrep_new_cluster mode, run:" - echo " $SUGGEST_EXEC_COMMAND touch /tmp/confirm-new-cluster" - echo "Or to start in recovery mode, to see replication state, run:" - echo " $SUGGEST_EXEC_COMMAND touch /tmp/confirm-recover" - echo "Or to force bootstrap on this node, potentially losing writes, run:" - echo " $SUGGEST_EXEC_COMMAND touch /tmp/confirm-force-bootstrap" - #echo " NOTE This bypasses the following warning from new cluster mode:" - #echo " It may not be safe to bootstrap the cluster from this node. It was not the last one to leave the cluster and may not contain all the updates. To force cluster bootstrap with this node, edit the grastate.dat file manually and set safe_to_bootstrap to 1 ." - echo "Or to try a regular start (for example after recovery + manual intervention), run:" - echo " $SUGGEST_EXEC_COMMAND touch /tmp/confirm-resume" - if [ ! -z "$AUTO_RECOVERY_MODE" ]; then - echo "The AUTO_RECOVERY_MODE env was set to $AUTO_RECOVERY_MODE, will trigger that choice" - touch /tmp/$AUTO_RECOVERY_MODE - else - echo "Waiting for response ..." - fi - while [ ! -f /tmp/confirm-resume ]; do - if [ "$AUTO_NEW_CLUSTER" = "true" ]; then - echo "The AUTO_NEW_CLUSTER env was set to $AUTO_NEW_CLUSTER, will proceed without confirmation" - echo "NOTE this env is deprecated, use AUTO_RECOVERY_MODE instead" - wsrepNewCluster - touch /tmp/confirm-resume - elif [ -f /tmp/confirm-new-cluster ]; then - echo "Confirmation received. Resuming new cluster start ..." - wsrepNewCluster - touch /tmp/confirm-resume - elif [ -f /tmp/confirm-force-bootstrap ]; then - echo "Forcing bootstrap on this node ..." - wsrepForceBootstrap - touch /tmp/confirm-new-cluster - elif [ -f /tmp/confirm-recover ]; then - echo "Confirmation received. Resuming in recovery mode." - echo "Note: to start the other pods you need to edit OrderedReady and add a command: --wsrep-recover" - wsrepRecover - touch /tmp/confirm-resume - fi - sleep 1 - done - rm /tmp/confirm-* - set -x - fi - } - else - getent hosts mariadb-ready - [ $? -eq 2 ] && { - echo "This is NOT the 1st statefulset pod. Must not go up as primary." - echo "Found no ready pods. Will exit to trigger a crash loop back off." - exit 1 - } - fi - - # https://github.com/docker-library/mariadb/blob/master/10.2/docker-entrypoint.sh#L62 - mysqld --verbose --help --log-bin-index="$(mktemp -u)" | tee /tmp/mariadb-start-config | grep -e ^version -e ^datadir -e ^wsrep -e ^binlog -e ^character-set -e ^collation diff --git a/20mariadb-service.yml b/20mariadb-service.yml deleted file mode 100644 index c9d5f94..0000000 --- a/20mariadb-service.yml +++ /dev/null @@ -1,26 +0,0 @@ -# the "Headless Service, used to control the network domain" ---- -apiVersion: v1 -kind: Service -metadata: - name: mariadb - namespace: mysql - annotations: - service.alpha.kubernetes.io/tolerate-unready-endpoints: "true" -spec: - publishNotReadyAddresses: true - clusterIP: None - selector: - app: mariadb - ports: - - port: 3306 - name: mysql - - port: 4444 - name: sst - - port: 4567 - name: replication - - protocol: UDP - port: 4567 - name: replicationudp - - port: 4568 - name: ist diff --git a/21mariadb-ready-service.yml b/21mariadb-ready-service.yml deleted file mode 100644 index 6895dbd..0000000 --- a/21mariadb-ready-service.yml +++ /dev/null @@ -1,14 +0,0 @@ -# Used with `getent hosts` to check how many pods that are ready ---- -apiVersion: v1 -kind: Service -metadata: - name: mariadb-ready - namespace: mysql - annotations: - service.alpha.kubernetes.io/tolerate-unready-endpoints: "false" -spec: - publishNotReadyAddresses: false - clusterIP: None - selector: - app: mariadb diff --git a/50mariadb.yml b/50mariadb.yml deleted file mode 100644 index 9a1843d..0000000 --- a/50mariadb.yml +++ /dev/null @@ -1,121 +0,0 @@ -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: mariadb - namespace: mysql -spec: - selector: - matchLabels: - app: mariadb - serviceName: "mariadb" - replicas: 3 - podManagementPolicy: Parallel - template: - metadata: - labels: - app: mariadb - annotations: - prometheus.io/scrape: "true" - prometheus.io/port: "9104" - spec: - terminationGracePeriodSeconds: 30 - initContainers: - - name: init-config - command: ['/bin/bash', '/etc/mysql/conf.d-configmap/init.sh'] - env: - - name: K8S_NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: DATADIR - value: /data/db - - name: AUTO_RECOVERY_MODE - value: confirm-force-bootstrap - - name: WSREP_CLUSTER_ADDRESS - value: "gcomm://mariadb-0.mariadb,mariadb-1.mariadb,mariadb-2.mariadb" - workingDir: /etc/mysql/conf.d-configmap - volumeMounts: - - name: mysql - mountPath: /data - - name: conf-readonly - mountPath: /etc/mysql/conf.d-configmap - - name: conf - mountPath: /etc/mysql/conf.d - - name: initdb - mountPath: /docker-entrypoint-initdb.d - image: mariadb:10.2.36-bionic@sha256:b7be3ade3d5441c79b5c8a9cf2c2269f14bf420876a06def7d50e1763f042238 - containers: - - name: mariadb - image: mariadb:10.2.36-bionic@sha256:b7be3ade3d5441c79b5c8a9cf2c2269f14bf420876a06def7d50e1763f042238 - ports: - - containerPort: 3306 - name: mysql - - containerPort: 4444 - name: sst - - containerPort: 4567 - name: replication - - containerPort: 4567 - protocol: UDP - name: replicationudp - - containerPort: 4568 - name: ist - env: - - name: MYSQL_ROOT_HOST - value: "localhost" - - name: MYSQL_ALLOW_EMPTY_PASSWORD - value: "yes" - - name: MYSQL_INITDB_SKIP_TZINFO - value: "yes" - readinessProbe: - exec: - command: - - /bin/sh - - -ec - - test ON = "$(mysql -e "SHOW STATUS LIKE 'wsrep_ready';" -N -B | sed 's/wsrep_ready\t//')" - initialDelaySeconds: 30 - livenessProbe: - exec: - command: - - mysql - - -e - - "SHOW DATABASES;" - # might need to be tweaked for large initial state transfers - initialDelaySeconds: 60 - volumeMounts: - - name: mysql - mountPath: /data - - name: conf - mountPath: /etc/mysql/conf.d - - name: initdb - mountPath: /docker-entrypoint-initdb.d - - name: metrics - image: prom/mysqld-exporter:v0.12.1@sha256:9fe9938c4ac9216cc24005144338f14fac4f604f139b481cc541bead008db3c1 - env: - - name: DATA_SOURCE_NAME - value: root@(localhost:3306)/ - ports: - - containerPort: 9104 - volumes: - - name: conf - emptyDir: {} - - name: conf-readonly - configMap: - name: conf-d - - name: initdb - emptyDir: {} - volumeClaimTemplates: - - metadata: - name: mysql - spec: - accessModes: [ "ReadWriteOnce" ] - resources: - requests: - storage: 1Gi diff --git a/README.md b/README.md index 2a23044..759aa9d 100644 --- a/README.md +++ b/README.md @@ -1,67 +1,7 @@ # MySQL with automatic failover on Kubernetes -This is a galera cluster setup, with plain manifests. -We actually use it in production, though with modest loads. +RETIRED See [v2.1.0](https://github.com/Yolean/kubernetes-mysql-cluster/tree/v2.1.0) for the original setup. -## Get started - -First create a storage class `mysql-data`. See exampels in `./configure/`. -You might also want to edit the volume size request, at the bottom of `./50mariadb.yml`. - -Then: `kubectl apply -f .`. - -### Cluster Health - -Readiness and liveness probes will only assert client-level health of individual pods. -Watch logs for "sst" or "Quorum results", or run this quick check: -``` -for i in 0 1 2; do kubectl -n mysql exec mariadb-$i -- mysql -e "SHOW STATUS LIKE 'wsrep_cluster_size';" -N; done -``` - -Port 9104 exposes plaintext metris in [Prometheus](https://prometheus.io/docs/concepts/data_model/) scrape format. -``` -# with kubectl -n mysql port-forward mariadb-0 9104:9104 -$ curl -s http://localhost:9104/metrics | grep ^mysql_global_status_wsrep_cluster_size -mysql_global_status_wsrep_cluster_size 3 -``` - -A reasonable alert is on `mysql_global_status_wsrep_cluster_size` staying below the desired number of replicas. - -### Cluster un-health - -We need to assume a couple of things here. First and foremost: -Production clusters are configured so that the statefulset pods do not go down together. - - * Pods are properly spread across nodes. - * Nodes are spread across multiple availability zones. - -Let's also assume that there is monitoring. -Any `wsrep_cluster_size` issue (see above), or absence of `wsrep_cluster_size` -should lead to a human being paged. - -Rarity combined with manual attention means that this statefulset can/should avoid -attempts at automatic [recovery](http://galeracluster.com/documentation-webpages/pcrecovery.html). -The reason for that being: we can't test for failure modes properly, -as they depend on the Kubernetes setup. -Automation may appoint the wrong leader - losing writes - -or cause split-brain situations. - -We can however support detection in the init script. - -It's normal operations to scale down to two instances -- actually one instance, but nodes should be considered ephemeral so don't do that - -and up to any number of replicas. - -### phpMyAdmin - -Carefully consider the security implications before you create this. Note that it uses a non-official image. - -``` -kubectl apply -f myadmin/ -``` - -PhpMyAdmin has a login page where you need a mysql user. To allow login (with full access) create a user with your choice of password: - -``` -kubectl -n mysql exec mariadb-0 -- mysql -e "CREATE USER 'phpmyadmin'@'%' IDENTIFIED BY 'my-admin-pw'; GRANT ALL ON *.* TO 'phpmyadmin'@'%' WITH GRANT OPTION;" -``` +We've now adopted the +[Bitnami mariadb-galera helm chart](https://github.com/bitnami/charts/tree/master/bitnami/mariadb-galera) +via [unhelm](https://github.com/Yolean/unhelm/tree/master/mysql) so we can use [Kustomize](https://kubernetes.io/docs/reference/generated/kubectl/kubectl-commands#kustomize). diff --git a/base-bootstrap-force/bootstrap-force.yaml b/base-bootstrap-force/bootstrap-force.yaml new file mode 100644 index 0000000..e686533 --- /dev/null +++ b/base-bootstrap-force/bootstrap-force.yaml @@ -0,0 +1,12 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: ystack-mariadb-galera +spec: + template: + spec: + containers: + - name: "mariadb-galera" + env: + - name: MARIADB_GALERA_FORCE_SAFETOBOOTSTRAP + value: "yes" diff --git a/base-bootstrap-force/kustomization.yaml b/base-bootstrap-force/kustomization.yaml new file mode 100644 index 0000000..f1b0f22 --- /dev/null +++ b/base-bootstrap-force/kustomization.yaml @@ -0,0 +1,8 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: +- ../base-bootstrap + +patchesStrategicMerge: +- ./bootstrap-force.yaml diff --git a/base-bootstrap/bootstrap-1-replica.yaml b/base-bootstrap/bootstrap-1-replica.yaml new file mode 100644 index 0000000..d765f5c --- /dev/null +++ b/base-bootstrap/bootstrap-1-replica.yaml @@ -0,0 +1,19 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: ystack-mariadb-galera +spec: + replicas: 1 + template: + spec: + containers: + - name: "mariadb-galera" + # https://github.com/bitnami/charts/blob/70822dd1aea385cad908462be5fc1004ef8b5e07/bitnami/mariadb-galera/templates/statefulset.yaml#L68 + command: + - /opt/bitnami/scripts/mariadb-galera/entrypoint.sh + - /opt/bitnami/scripts/mariadb-galera/run.sh + env: + - name: MARIADB_GALERA_CLUSTER_BOOTSTRAP + value: "yes" + - name: MARIADB_GALERA_FORCE_SAFETOBOOTSTRAP + value: "no" diff --git a/base-bootstrap/kustomization.yaml b/base-bootstrap/kustomization.yaml new file mode 100644 index 0000000..1d9b092 --- /dev/null +++ b/base-bootstrap/kustomization.yaml @@ -0,0 +1,8 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: +- ../base + +patchesStrategicMerge: +- ./bootstrap-1-replica.yaml diff --git a/base/bootstrap-no.yaml b/base/bootstrap-no.yaml new file mode 100644 index 0000000..67eda7d --- /dev/null +++ b/base/bootstrap-no.yaml @@ -0,0 +1,19 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: ystack-mariadb-galera +spec: + template: + spec: + containers: + - name: "mariadb-galera" + # https://github.com/bitnami/charts/blob/70822dd1aea385cad908462be5fc1004ef8b5e07/bitnami/mariadb-galera/templates/statefulset.yaml#L68 + command: + - /opt/bitnami/scripts/mariadb-galera/entrypoint.sh + - /opt/bitnami/scripts/mariadb-galera/run.sh + env: + # WIP this seems to have no effect, a new cluster will start to split-brain anyway + - name: MARIADB_GALERA_CLUSTER_BOOTSTRAP + value: "no" + - name: MARIADB_GALERA_FORCE_SAFETOBOOTSTRAP + value: "no" diff --git a/base/kustomization.yaml b/base/kustomization.yaml new file mode 100644 index 0000000..2370669 --- /dev/null +++ b/base/kustomization.yaml @@ -0,0 +1,48 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: mysqltest + +commonLabels: + app.kubernetes.io/managed-by: unhelm + +resources: +- github.com/Yolean/unhelm/mysql/mariadb-galera?ref=606228e73ae4ca7d9fd0ec2820515a07b2b25cbd +- ./mysql-service.yaml +- ./podmonitor.yaml + +secretGenerator: +- name: mariadb-auth + literals: + - mariadb-root-password=TESTROOT + - mariadb-galera-mariabackup-password=TESTBACKUP + - mariadb-password=TESTMARIADB + +images: +- name: docker.io/bitnami/mariadb-galera + # 10.5.10-debian-10-r30 + digest: sha256:3dc89271c5051abfaa33f0540b802a75f869570fa8b7ed7f5e7e6178df4b5653 +- name: docker.io/bitnami/mysqld-exporter + # 0.13.0-debian-10-r16 + digest: sha256:55eb650f79f8c6b4936890261f6e4f966940c2e2eaadd4374b93dbe372a8dbd3 + +replicas: +- name: ystack-mariadb-galera + count: 3 + +patchesStrategicMerge: +- ./bootstrap-no.yaml + # The helm chart depends on the namespace for galera cluster address, but that's not necessary +- |- + apiVersion: apps/v1 + kind: StatefulSet + metadata: + name: ystack-mariadb-galera + spec: + template: + spec: + containers: + - name: "mariadb-galera" + env: + - name: MARIADB_GALERA_CLUSTER_ADDRESS + value: "gcomm://ystack-mariadb-galera-headless" diff --git a/30mysql-service.yml b/base/mysql-service.yaml similarity index 57% rename from 30mysql-service.yml rename to base/mysql-service.yaml index dee62dc..9b9c88d 100644 --- a/30mysql-service.yml +++ b/base/mysql-service.yaml @@ -1,12 +1,11 @@ ---- apiVersion: v1 kind: Service metadata: name: mysql - namespace: mysql spec: ports: - port: 3306 name: mysql selector: - app: mariadb + app.kubernetes.io/name: mariadb-galera + app.kubernetes.io/instance: ystack diff --git a/base/podmonitor.yaml b/base/podmonitor.yaml new file mode 100644 index 0000000..7768922 --- /dev/null +++ b/base/podmonitor.yaml @@ -0,0 +1,15 @@ +apiVersion: monitoring.coreos.com/v1 +kind: PodMonitor +metadata: + name: mariadb-galera + labels: + prometheus: now +spec: + jobLabel: app.kubernetes.io/name + namespaceSelector: + any: false + selector: + matchLabels: + app.kubernetes.io/name: mariadb-galera + podMetricsEndpoints: + - port: metrics diff --git a/kustomization.yaml b/kustomization.yaml deleted file mode 100644 index 176a01a..0000000 --- a/kustomization.yaml +++ /dev/null @@ -1,6 +0,0 @@ -resources: -- 10conf-d.yml -- 20mariadb-service.yml -- 21mariadb-ready-service.yml -- 30mysql-service.yml -- 50mariadb.yml diff --git a/test.sh b/test.sh new file mode 100755 index 0000000..feb3ed6 --- /dev/null +++ b/test.sh @@ -0,0 +1,90 @@ +#!/usr/bin/env bash +[ -z "$DEBUG" ] || set -x +set -eo pipefail +DIR="$(dirname $0)" + +export KUBECONFIG=$DIR/kubeconfig + +show_cluster_size() { + kubectl -n mysqltest exec -c mariadb-galera ystack-mariadb-galera-0 -- mysql -u root "-pTESTROOT" -N -e "SHOW STATUS LIKE 'wsrep_cluster_size';" +} + +k3d cluster create mysqltest --agents 3 --agents-memory 512M + +# https://github.com/Yolean/ystack/blob/master/bin/y-cluster-assert-install +ctx="" +OPERATOR_VERSION=5555f492df250168657b72bb8cb60bec071de71f +KUBERNETES_ASSERT_VERSION=cb66d46758654b819d0d4402857122dca1884bcb +kubectl $ctx create namespace monitoring +kubectl $ctx -n default apply -f https://github.com/prometheus-operator/prometheus-operator/raw/$OPERATOR_VERSION/bundle.yaml +kubectl wait $ctx -n default --for=condition=Ready pod -l app.kubernetes.io/name=prometheus-operator +kubectl $ctx -n monitoring apply -k github.com/Yolean/kubernetes-assert/example-small?ref=$KUBERNETES_ASSERT_VERSION +kubectl wait $ctx -n monitoring --for=condition=Ready pod --all + +kubectl create namespace mysqltest + +# With >1 replicas this leads to split brain despite ./base/bootstrap-no.yaml +#kubectl -n mysqltest apply -k ./base + +kubectl -n mysqltest apply -k ./base-bootstrap +kubectl -n mysqltest rollout status statefulset/ystack-mariadb-galera + +show_cluster_size + +kubectl -n mysqltest apply -k ./base +kubectl -n mysqltest rollout status statefulset/ystack-mariadb-galera + +show_cluster_size + +kubectl -n mysqltest scale --replicas=2 statefulset/ystack-mariadb-galera +kubectl -n mysqltest rollout status statefulset/ystack-mariadb-galera + +show_cluster_size + +kubectl -n mysqltest scale --replicas=4 statefulset/ystack-mariadb-galera +kubectl -n mysqltest rollout status statefulset/ystack-mariadb-galera + +show_cluster_size + +kubectl -n mysqltest delete --wait=false pod/ystack-mariadb-galera-1 +sleep 1 +show_cluster_size +kubectl -n mysqltest wait --for=condition=Ready --timeout=60s pod/ystack-mariadb-galera-1 +show_cluster_size + +# With curl from somewhere we wouldn't need a pod name for show_cluster_size +# curl -s http://ystack-mariadb-galera-metrics.mysqltest:9104/metrics | grep cluster_size +#kubectl -n mysqltest delete --wait=false pod/ystack-mariadb-galera-0 +kubectl -n mysqltest delete --wait=false pod/ystack-mariadb-galera-2 +kubectl -n mysqltest delete --wait=false pod/ystack-mariadb-galera-1 +sleep 1 +show_cluster_size +kubectl -n mysqltest wait --for=condition=Ready --timeout=60s pod/ystack-mariadb-galera-1 +show_cluster_size + +echo "# From zero we expect \"base\" to fail to start" + +kubectl -n mysqltest scale --replicas=0 statefulset/ystack-mariadb-galera +kubectl -n mysqltest rollout status statefulset/ystack-mariadb-galera +kubectl -n mysqltest apply -k ./base +kubectl -n mysqltest rollout status --timeout=30s statefulset/ystack-mariadb-galera || echo "Timeout is expected" +kubectl -n mysqltest get pods +echo "# Prometheus will now report absent(mysql_global_status_wsrep_cluster_size) == 1" + +kubectl -n mysqltest apply -k ./base-bootstrap +sleep 5 +kubectl -n mysqltest get pods +kubectl -n mysqltest logs -c mariadb-galera ystack-mariadb-galera-0 + +echo "# Using bootstrap-force, assuming that pod 0 has the latest writes" +echo "# To bootstrap from a different node use the helm chart" +kubectl -n mysqltest apply -k ./base-bootstrap-force +kubectl -n mysqltest rollout status statefulset/ystack-mariadb-galera +show_cluster_size + +echo "# Upon bootstrap success, apply the regular base to prevent more bootstrapping" +kubectl -n mysqltest apply -k ./base +kubectl -n mysqltest rollout status statefulset/ystack-mariadb-galera +show_cluster_size + +k3d cluster delete mysqltest diff --git a/variants/gke/gke-storageclass-ssd-regional-europewest1cd.yaml b/variants/gke/gke-storageclass-ssd-regional-europewest1cd.yaml deleted file mode 100644 index c5bd899..0000000 --- a/variants/gke/gke-storageclass-ssd-regional-europewest1cd.yaml +++ /dev/null @@ -1,11 +0,0 @@ -kind: StorageClass -apiVersion: storage.k8s.io/v1 -metadata: - name: mysql-data -provisioner: kubernetes.io/gce-pd -reclaimPolicy: Retain -allowVolumeExpansion: true -parameters: - type: pd-ssd - replication-type: regional-pd - zones: europe-west1-c, europe-west1-d diff --git a/variants/gke/gke-storageclass-ssd.yml b/variants/gke/gke-storageclass-ssd.yml deleted file mode 100644 index cd4852f..0000000 --- a/variants/gke/gke-storageclass-ssd.yml +++ /dev/null @@ -1,9 +0,0 @@ -kind: StorageClass -apiVersion: storage.k8s.io/v1 -metadata: - name: mysql-data -provisioner: kubernetes.io/gce-pd -reclaimPolicy: Retain -allowVolumeExpansion: true -parameters: - type: pd-ssd diff --git a/variants/gke/kustomization.yaml b/variants/gke/kustomization.yaml deleted file mode 100644 index 50b758f..0000000 --- a/variants/gke/kustomization.yaml +++ /dev/null @@ -1,7 +0,0 @@ -bases: -- github.com/Yolean/kubernetes-mysql-cluster?ref=009cad0 -resources: -- gke-storageclass-ssd.yml -#- gke-storageclass-ssd-regional-europewest1cd.yaml -patchesStrategicMerge: -- volume-claims.yaml diff --git a/variants/gke/volume-claims.yaml b/variants/gke/volume-claims.yaml deleted file mode 100644 index 0de97f6..0000000 --- a/variants/gke/volume-claims.yaml +++ /dev/null @@ -1,16 +0,0 @@ ---- -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: mariadb - namespace: mysql -spec: - volumeClaimTemplates: - - metadata: - name: mysql - spec: - accessModes: [ "ReadWriteOnce" ] - storageClassName: mysql-data - resources: - requests: - storage: 1Gi diff --git a/variants/namespace/kustomization.yaml b/variants/namespace/kustomization.yaml index 16d767e..ea7a52d 100644 --- a/variants/namespace/kustomization.yaml +++ b/variants/namespace/kustomization.yaml @@ -1,3 +1,22 @@ -bases: -- github.com/Yolean/kubernetes-mysql-cluster?ref=549e804 +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: +- ../../base + namespace: analytics + +patchesStrategicMerge: +- |- + apiVersion: apps/v1 + kind: StatefulSet + metadata: + name: ystack-mariadb-galera + spec: + template: + spec: + containers: + - name: "mariadb-galera" + env: + - name: MARIADB_GALERA_CLUSTER_ADDRESS + value: "gcomm://ystack-mariadb-galera-headless.analytics.svc.cluster.local" diff --git a/variants/scale-1-ephemeral/ephemeral.yaml b/variants/scale-1-ephemeral/ephemeral.yaml index c2f015b..228fd35 100644 --- a/variants/scale-1-ephemeral/ephemeral.yaml +++ b/variants/scale-1-ephemeral/ephemeral.yaml @@ -1,12 +1,11 @@ apiVersion: apps/v1 kind: StatefulSet metadata: - name: mariadb - namespace: mysql + name: ystack-mariadb-galera spec: template: spec: volumes: - - name: mysql + - name: data emptyDir: {} volumeClaimTemplates: [] diff --git a/variants/scale-1-ephemeral/kustomization.yaml b/variants/scale-1-ephemeral/kustomization.yaml index 5498756..542e2ed 100644 --- a/variants/scale-1-ephemeral/kustomization.yaml +++ b/variants/scale-1-ephemeral/kustomization.yaml @@ -1,4 +1,8 @@ -bases: +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: - ../scale-1 + patchesStrategicMerge: - ephemeral.yaml diff --git a/variants/scale-1/kustomization.yaml b/variants/scale-1/kustomization.yaml index 6e1fa9a..ed8a5b8 100644 --- a/variants/scale-1/kustomization.yaml +++ b/variants/scale-1/kustomization.yaml @@ -1,5 +1,9 @@ -bases: -- github.com/Yolean/kubernetes-mysql-cluster?ref=a9e6956000a31c973d183d8c318d6828cd73c26c -patchesStrategicMerge: -- replicas-1.yaml -- wsrep-1.yaml +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: +- ../../base + +replicas: +- name: ystack-mariadb-galera + count: 1 diff --git a/variants/scale-1/replicas-1.yaml b/variants/scale-1/replicas-1.yaml deleted file mode 100644 index 2d5a351..0000000 --- a/variants/scale-1/replicas-1.yaml +++ /dev/null @@ -1,7 +0,0 @@ -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: mariadb - namespace: mysql -spec: - replicas: 1 diff --git a/variants/scale-1/wsrep-1.yaml b/variants/scale-1/wsrep-1.yaml deleted file mode 100644 index 95a61ef..0000000 --- a/variants/scale-1/wsrep-1.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: mariadb -spec: - template: - spec: - initContainers: - - name: init-config - env: - - name: WSREP_CLUSTER_ADDRESS - value: "gcomm://mariadb-0.mariadb" diff --git a/variants/scale-2/kustomization.yaml b/variants/scale-2/kustomization.yaml index 2f7540f..b7aa659 100644 --- a/variants/scale-2/kustomization.yaml +++ b/variants/scale-2/kustomization.yaml @@ -1,5 +1,9 @@ -bases: -- github.com/Yolean/kubernetes-mysql-cluster?ref=8c4439f -patchesStrategicMerge: -- replicas-2.yaml -- wsrep-2.yaml +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: +- ../../base + +replicas: +- name: ystack-mariadb-galera + count: 2 diff --git a/variants/scale-2/replicas-2.yaml b/variants/scale-2/replicas-2.yaml deleted file mode 100644 index e57c624..0000000 --- a/variants/scale-2/replicas-2.yaml +++ /dev/null @@ -1,7 +0,0 @@ -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: mariadb - namespace: mysql -spec: - replicas: 2 diff --git a/variants/scale-2/wsrep-2.yaml b/variants/scale-2/wsrep-2.yaml deleted file mode 100644 index fc09f0e..0000000 --- a/variants/scale-2/wsrep-2.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: mariadb -spec: - template: - spec: - initContainers: - - name: init-config - env: - - name: WSREP_CLUSTER_ADDRESS - value: "gcomm://mariadb-0.mariadb,mariadb-1.mariadb"