+
MysqlctldSpec configures the local mysqlctld gRPC server within a tablet.
+ +| Field | +Description | +
|---|---|
+resources
+
+
+Kubernetes core/v1.ResourceRequirements
+
+
+ |
+
+ Resources specify the compute resources to allocate for just the MySQL Control Daemon. + |
+
+extraFlags
+
+map[string]string
+
+ |
+
+ ExtraFlags can optionally be used to override default flags set by the +operator, or pass additional flags to mysqlctld. All entries must be +key-value string pairs of the form “flag”: “value”. The flag name should +not have any prefix (just “flag”, not “-flag”). To set a boolean flag, +set the string value to either “true” or “false”. + |
+
diff --git a/pkg/apis/planetscale/v2/vitessshard_types.go b/pkg/apis/planetscale/v2/vitessshard_types.go index 615e423f..c243484c 100644 --- a/pkg/apis/planetscale/v2/vitessshard_types.go +++ b/pkg/apis/planetscale/v2/vitessshard_types.go @@ -315,6 +315,19 @@ type MysqldSpec struct { ConfigOverrides string `json:"configOverrides,omitempty"` } +// MysqlctldSpec configures the local mysqlctld gRPC server within a tablet. +type MysqlctldSpec struct { + // Resources specify the compute resources to allocate for just the MySQL Control Daemon. + Resources corev1.ResourceRequirements `json:"resources"` + + // ExtraFlags can optionally be used to override default flags set by the + // operator, or pass additional flags to mysqlctld. All entries must be + // key-value string pairs of the form "flag": "value". The flag name should + // not have any prefix (just "flag", not "-flag"). To set a boolean flag, + // set the string value to either "true" or "false". + ExtraFlags map[string]string `json:"extraFlags,omitempty"` +} + // MysqldExporterSpec configures the local MySQL exporter within a tablet. type MysqldExporterSpec struct { // Resources specify the compute resources to allocate for just the MySQL Exporter. diff --git a/pkg/apis/planetscale/v2/zz_generated.deepcopy.go b/pkg/apis/planetscale/v2/zz_generated.deepcopy.go index 4db30177..66ac030b 100644 --- a/pkg/apis/planetscale/v2/zz_generated.deepcopy.go +++ b/pkg/apis/planetscale/v2/zz_generated.deepcopy.go @@ -397,6 +397,29 @@ func (in *LockserverStatus) DeepCopy() *LockserverStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *MysqlctldSpec) DeepCopyInto(out *MysqlctldSpec) { + *out = *in + in.Resources.DeepCopyInto(&out.Resources) + if in.ExtraFlags != nil { + in, out := &in.ExtraFlags, &out.ExtraFlags + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MysqlctldSpec. +func (in *MysqlctldSpec) DeepCopy() *MysqlctldSpec { + if in == nil { + return nil + } + out := new(MysqlctldSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *MysqldExporterSpec) DeepCopyInto(out *MysqldExporterSpec) { *out = *in diff --git a/pkg/operator/vttablet/env_vars.go b/pkg/operator/vttablet/env_vars.go index 3e3189c5..34c1b6da 100644 --- a/pkg/operator/vttablet/env_vars.go +++ b/pkg/operator/vttablet/env_vars.go @@ -18,10 +18,11 @@ package vttablet import ( "fmt" + "os" "strings" - "planetscale.dev/vitess-operator/pkg/operator/lazy" corev1 "k8s.io/api/core/v1" + "planetscale.dev/vitess-operator/pkg/operator/lazy" ) func init() { @@ -40,6 +41,10 @@ func init() { Name: "EXTRA_MY_CNF", Value: strings.Join(extraMyCnf.Get(spec), ":"), }, + // The mysqlbinlog binary is NOT in the vendor mysql container images + // so we copy it to /mnt/vt/bin when initializing the mysqld container + // so that mysqlctl can find it. + {Name: "PATH", Value: fmt.Sprintf("%s:%s", os.Getenv("PATH"), vtBinPath)}, } }) diff --git a/pkg/operator/vttablet/mysqlctld.go b/pkg/operator/vttablet/mysqlctld.go index 442aaf6d..d204ca06 100644 --- a/pkg/operator/vttablet/mysqlctld.go +++ b/pkg/operator/vttablet/mysqlctld.go @@ -29,6 +29,7 @@ const ( vtRootInitScript = `set -ex mkdir -p /mnt/vt/bin cp --no-clobber /vt/bin/mysqlctld /mnt/vt/bin/ +cp --no-clobber /usr/bin/mysqlbinlog /mnt/vt/bin/ mkdir -p /mnt/vt/config if [[ -d /vt/config/mycnf ]]; then cp --no-clobber -R /vt/config/mycnf /mnt/vt/config/ @@ -68,7 +69,7 @@ func init() { securityContext := &corev1.SecurityContext{} if planetscalev2.DefaultVitessRunAsUser >= 0 { - securityContext.RunAsUser = pointer.Int64Ptr(planetscalev2.DefaultVitessRunAsUser) + securityContext.RunAsUser = pointer.Int64(planetscalev2.DefaultVitessRunAsUser) } // Use an init container to copy only the files we need from the Vitess image. diff --git a/pkg/operator/vttablet/pod.go b/pkg/operator/vttablet/pod.go index 9dcc25f1..f3e2fa01 100644 --- a/pkg/operator/vttablet/pod.go +++ b/pkg/operator/vttablet/pod.go @@ -87,6 +87,23 @@ func UpdatePod(obj *corev1.Pod, spec *Spec) { key = strings.TrimLeft(key, "-") vttabletAllFlags[key] = value } + // Ensure that binary logs are restored to/from a location that all containers + // in the pod can access if no location was explicitly provided. + if _, ok := vttabletAllFlags["builtinbackup-incremental-restore-path"]; !ok { + // This flag only exists in 2.12.0 and later as it includes vitess + // 20.0 as a dependency. + /* + parts := strings.Split(version.Version, ".") + if len(parts) > 0 { + major, _ := strconv.Atoi(parts[0]) + minor, _ := strconv.Atoi(parts[1]) + if major >= 2 && minor >= 12 { + vttabletAllFlags["builtinbackup-incremental-restore-path"] = vtDataRootPath + } + } + */ + vttabletAllFlags["builtinbackup-incremental-restore-path"] = vtDataRootPath + } mysql.UpdateMySQLServerVersion(vttabletAllFlags, spec.Images.Mysqld.Image()) // Compute all operator-generated env vars first. @@ -108,7 +125,7 @@ func UpdatePod(obj *corev1.Pod, spec *Spec) { securityContext := &corev1.SecurityContext{} if planetscalev2.DefaultVitessRunAsUser >= 0 { - securityContext.RunAsUser = pointer.Int64Ptr(planetscalev2.DefaultVitessRunAsUser) + securityContext.RunAsUser = pointer.Int64(planetscalev2.DefaultVitessRunAsUser) } vttabletLifecycle := &spec.Vttablet.Lifecycle @@ -315,13 +332,13 @@ func UpdatePod(obj *corev1.Pod, spec *Spec) { obj.Spec.SecurityContext = &corev1.PodSecurityContext{} } if planetscalev2.DefaultVitessFSGroup >= 0 { - obj.Spec.SecurityContext.FSGroup = pointer.Int64Ptr(planetscalev2.DefaultVitessFSGroup) + obj.Spec.SecurityContext.FSGroup = pointer.Int64(planetscalev2.DefaultVitessFSGroup) } if spec.Vttablet.TerminationGracePeriodSeconds != nil { obj.Spec.TerminationGracePeriodSeconds = spec.Vttablet.TerminationGracePeriodSeconds } else { - obj.Spec.TerminationGracePeriodSeconds = pointer.Int64Ptr(defaultTerminationGracePeriodSeconds) + obj.Spec.TerminationGracePeriodSeconds = pointer.Int64(defaultTerminationGracePeriodSeconds) } // In both the case of the user injecting their own affinity and the default, we diff --git a/pkg/operator/vttablet/vtbackup_pod.go b/pkg/operator/vttablet/vtbackup_pod.go index 5823cdb1..8e8b3a1e 100644 --- a/pkg/operator/vttablet/vtbackup_pod.go +++ b/pkg/operator/vttablet/vtbackup_pod.go @@ -35,6 +35,7 @@ const ( vtbackupInitScript = `set -ex mkdir -p /mnt/vt/bin cp --no-clobber /vt/bin/vtbackup /mnt/vt/bin/ +cp --no-clobber /usr/bin/mysqlbinlog /mnt/vt/bin/ mkdir -p /mnt/vt/config if [[ -d /vt/config/mycnf ]]; then cp --no-clobber -R /vt/config/mycnf /mnt/vt/config/ @@ -115,6 +116,12 @@ func NewBackupPod(key client.ObjectKey, backupSpec *BackupSpec, mysqldImage stri MountPath: sslCertsPath, SubPath: "certs", }, + { + Name: vtRootVolumeName, + ReadOnly: true, + MountPath: vtBinPath, + SubPath: "bin", + }, } volumeMounts = append(volumeMounts, mysqldVolumeMounts.Get(tabletSpec)...) volumeMounts = append(volumeMounts, tabletVolumeMounts.Get(tabletSpec)...) @@ -122,11 +129,11 @@ func NewBackupPod(key client.ObjectKey, backupSpec *BackupSpec, mysqldImage stri podSecurityContext := &corev1.PodSecurityContext{} if planetscalev2.DefaultVitessFSGroup >= 0 { - podSecurityContext.FSGroup = pointer.Int64Ptr(planetscalev2.DefaultVitessFSGroup) + podSecurityContext.FSGroup = pointer.Int64(planetscalev2.DefaultVitessFSGroup) } securityContext := &corev1.SecurityContext{} if planetscalev2.DefaultVitessRunAsUser >= 0 { - securityContext.RunAsUser = pointer.Int64Ptr(planetscalev2.DefaultVitessRunAsUser) + securityContext.RunAsUser = pointer.Int64(planetscalev2.DefaultVitessRunAsUser) } var containerResources corev1.ResourceRequirements @@ -134,6 +141,11 @@ func NewBackupPod(key client.ObjectKey, backupSpec *BackupSpec, mysqldImage stri update.ResourceRequirements(&containerResources, &tabletSpec.Mysqld.Resources) vtbackupAllFlags := vtbackupFlags.Get(backupSpec) + // Ensure that binary logs are restored to/from a location that all containers + // in the pod can access if no location was explicitly provided. + if _, ok := vtbackupAllFlags["builtinbackup-incremental-restore-path"]; !ok { + vtbackupAllFlags["builtinbackup-incremental-restore-path"] = vtDataRootPath + } mysql.UpdateMySQLServerVersion(vtbackupAllFlags, mysqldImage) pod := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ diff --git a/test/endtoend/backup_restore_test.sh b/test/endtoend/backup_restore_test.sh index a9c48b9f..5332382f 100755 --- a/test/endtoend/backup_restore_test.sh +++ b/test/endtoend/backup_restore_test.sh @@ -112,6 +112,7 @@ verifyVtGateVersion "21.0.0" checkSemiSyncSetup takeBackup "commerce/-" verifyListBackupsOutput +restoreBackup "$(vtctldclient GetTablets --keyspace commerce --tablet-type replica --shard '-' | head -1 | awk '{print $1}')" takedownShard resurrectShard checkSemiSyncSetup diff --git a/test/endtoend/utils.sh b/test/endtoend/utils.sh index bf685730..9d04d481 100644 --- a/test/endtoend/utils.sh +++ b/test/endtoend/utils.sh @@ -83,24 +83,83 @@ function removeBackupFiles() { # takeBackup: # $1: keyspace-shard for which the backup needs to be taken +declare INCREMENTAL_RESTORE_POS="" function takeBackup() { keyspaceShard=$1 initialBackupCount=$(kubectl get vtb --no-headers | wc -l) finalBackupCount=$((initialBackupCount+1)) - # issue the backupShard command to vtctldclient - vtctldclient BackupShard "$keyspaceShard" + # Issue the BackupShard command to vtctldclient. + vtctldclient BackupShard "${keyspaceShard}" for i in {1..600} ; do out=$(kubectl get vtb --no-headers | wc -l) - echo "$out" | grep "$finalBackupCount" > /dev/null 2>&1 - if [[ $? -eq 0 ]]; then - echo "Backup created" + if echo "${out}" | grep -c "${finalBackupCount}" >/dev/null; then + echo "Full backup created" + break + fi + sleep 3 + done + + # Now perform an incremental backup. + insertWithRetry + pos=$(mysql -sN -e "select @@global.gtid_executed") + INCREMENTAL_RESTORE_POS="MySQL56/${pos//\\n/}" + sleep 2 + echo "Backup position is $INCREMENTAL_RESTORE_POS" + sleep 2 + insertWithRetry + + vtctldclient BackupShard --incremental-from-pos=auto "${keyspaceShard}" + let finalBackupCount=${finalBackupCount}+1 + + for i in {1..600} ; do + out=$(kubectl get vtb --no-headers | wc -l) + if echo "${out}" | grep -c "${finalBackupCount}" >/dev/null; then + echo "Incremental backup created" + return 0 + fi + sleep 3 + done + + echo -e "ERROR: Backups not created - ${out}. ${backupCount} backups expected." + exit 1 +} + +# restoreBackup: +# $1: tablet alias for which the backup needs to be restored +function restoreBackup() { + tabletAlias=$1 + if [[ -z "${tabletAlias}" ]]; then + echo "Tablet alias not provided as restore target" + exit 1 + fi + + # Issue the PITR restore command to vtctldclient. + # This should restore the last full backup, followed by applying the + # binary logs to reach the desired timestamp. + echo "Listing backups" + vtctldclient GetBackups "$keyspaceShard" + echo "End listing backups" + echo "Restoring tablet ${tabletAlias} to position ${INCREMENTAL_RESTORE_POS}" + if ! vtctldclient RestoreFromBackup --restore-to-pos "${INCREMENTAL_RESTORE_POS}" "${tabletAlias}"; then + echo "ERROR: failed to perform incremental restore" + exit 1 + fi + + cell="${tabletAlias%-*}" + uid="${tabletAlias##*-}" + + for i in {1..600} ; do + out=$(kubectl get pods --no-headers -l "planetscale.com/cell=${cell},planetscale.com/tablet-uid=${uid}" | grep "Running" | wc -l) + if echo "$out" | grep -c "1" >/dev/null; then + echo "Tablet ${tabletAlias} restore complete" return 0 fi sleep 3 done - echo -e "ERROR: Backup not created - $out. $backupCount backups expected." + + echo -e "ERROR: restored tablet ${tabletAlias} did not become healthy after the restore." exit 1 } @@ -220,7 +279,9 @@ function waitForKeyspaceToBeServing() { nb_of_replica=$3 for i in {1..600} ; do out=$(mysql --table --execute="show vitess_tablets") + echo "Serving output: ${out}" numtablets=$(echo "$out" | grep -E "$ks(.*)$shard(.*)PRIMARY(.*)SERVING|$ks(.*)$shard(.*)REPLICA(.*)SERVING" | wc -l) + echo "Number of serving tablets: ${numtablets}" if [[ $numtablets -ge $((nb_of_replica+1)) ]]; then echo "Shard $ks/$shard is serving" return