Skip to content

[release-15.0] vtgate/engine: Fix race condition in join logic (#14435) #14110

[release-15.0] vtgate/engine: Fix race condition in join logic (#14435)

[release-15.0] vtgate/engine: Fix race condition in join logic (#14435) #14110

name: Upgrade Downgrade Testing - Backups - Manual - Next Release
on:
push:
pull_request:
concurrency:
group: format('{0}-{1}', ${{ github.ref }}, 'Upgrade Downgrade Testing - Backups - Manual - Next Release')
cancel-in-progress: true
jobs:
get_next_release:
if: always()
name: Get Previous Release - Backups - Manual - Next Release
runs-on: ubuntu-20.04
outputs:
next_release: ${{ steps.output-next-release-ref.outputs.next_release_ref }}
steps:
- name: Check out to HEAD
uses: actions/checkout@v3
with:
fetch-depth: 0
- name: Set output with latest release branch
id: output-next-release-ref
run: |
next_release_ref=$(./tools/get_next_release.sh ${{github.base_ref}} ${{github.ref}})
echo $next_release_ref
echo "next_release_ref=${next_release_ref}" >> $GITHUB_OUTPUT
# This job usually execute in ± 20 minutes
upgrade_downgrade_test_manual:
timeout-minutes: 40
if: always() && (needs.get_next_release.result == 'success')
name: Run Upgrade Downgrade Test - Backups - Manual - Next Release
runs-on: ubuntu-20.04
needs:
- get_next_release
steps:
- name: Check if workflow needs to be skipped
id: skip-workflow
run: |
skip='false'
if [[ "${{github.event.pull_request}}" == "" ]] && [[ "${{github.ref}}" != "refs/heads/main" ]] && [[ ! "${{github.ref}}" =~ ^refs/heads/release-[0-9]+\.[0-9]$ ]] && [[ ! "${{github.ref}}" =~ "refs/tags/.*" ]]; then
skip='true'
fi
if [[ "${{needs.get_next_release.outputs.next_release}}" == "" ]]; then
skip='true'
fi
echo Skip ${skip}
echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT
# Checkout to this build's commit
- name: Checkout to commit's code
if: steps.skip-workflow.outputs.skip-workflow == 'false'
uses: actions/checkout@v3
- name: Check for changes in relevant files
if: steps.skip-workflow.outputs.skip-workflow == 'false'
uses: frouioui/paths-filter@main
id: changes
with:
token: ''
filters: |
end_to_end:
- 'go/**'
- 'go/**/*.go'
- 'test.go'
- 'Makefile'
- 'build.env'
- 'go.sum'
- 'go.mod'
- 'proto/*.proto'
- 'tools/**'
- 'config/**'
- 'bootstrap.sh'
- '.github/workflows/upgrade_downgrade_test_backups_manual_next_release.yml'
- name: Set up Go
if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
uses: actions/setup-go@v3
with:
go-version: 1.20.10
- name: Set up python
if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
uses: actions/setup-python@v4
- name: Tune the OS
if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
run: |
echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range
- name: Get base dependencies
if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
run: |
sudo DEBIAN_FRONTEND="noninteractive" apt-get update
# Uninstall any nextly installed MySQL first
sudo systemctl stop apparmor
sudo DEBIAN_FRONTEND="noninteractive" apt-get remove -y --purge mysql-server mysql-client mysql-common
sudo apt-get -y autoremove
sudo apt-get -y autoclean
sudo deluser mysql
sudo rm -rf /var/lib/mysql
sudo rm -rf /etc/mysql
# Install MySQL 8.0
####
## Temporarily pin the MySQL version at 8.0.29 as Vitess 14.0.1 does not have the fix to support
## backups of 8.0.30+. See: https://github.com/vitessio/vitess/pull/10847
## TODO: remove this pin once the above fixes are included in a v14 release (will be in v14.0.2) OR
## Vitess 16.0.0-SNAPSHOT becomes the dev version on vitessio/main
#sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29
#wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb
#echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections
#sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config*
#sudo apt-get update
#sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y mysql-server mysql-client
####
wget -c https://cdn.mysql.com/archives/mysql-8.0/mysql-common_8.0.28-1ubuntu20.04_amd64.deb \
https://cdn.mysql.com/archives/mysql-8.0/mysql-community-client-core_8.0.28-1ubuntu20.04_amd64.deb \
https://cdn.mysql.com/archives/mysql-8.0/mysql-community-client-plugins_8.0.28-1ubuntu20.04_amd64.deb \
https://cdn.mysql.com/archives/mysql-8.0/mysql-client_8.0.28-1ubuntu20.04_amd64.deb \
https://cdn.mysql.com/archives/mysql-8.0/mysql-community-server-core_8.0.28-1ubuntu20.04_amd64.deb \
https://cdn.mysql.com/archives/mysql-8.0/mysql-community-server_8.0.28-1ubuntu20.04_amd64.deb \
https://cdn.mysql.com/archives/mysql-8.0/mysql-community-client_8.0.28-1ubuntu20.04_amd64.deb
sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y ./mysql-*.deb
# Install everything else we need, and configure
sudo apt-get install -y make unzip g++ etcd curl git wget eatmydata grep
sudo service mysql stop
sudo service etcd stop
sudo bash -c "echo '/usr/sbin/mysqld { }' > /etc/apparmor.d/usr.sbin.mysqld" # https://bugs.launchpad.net/ubuntu/+source/mariadb-10.1/+bug/1806263
sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/
sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld || echo "could not remove mysqld profile"
# install JUnit report formatter
go install github.com/vitessio/go-junit-report@HEAD
wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb
sudo apt-get install -y gnupg2
sudo dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb
sudo apt-get update
sudo apt-get install percona-xtrabackup-24
# Checkout to the next release of Vitess
- name: Checkout to the other version's code (${{ needs.get_next_release.outputs.next_release }})
if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
uses: actions/checkout@v3
with:
ref: ${{ needs.get_next_release.outputs.next_release }}
- name: Get dependencies for the next release
if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
run: |
go mod download
- name: Building next release's binaries
if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
timeout-minutes: 5
run: |
source build.env
make build
mkdir -p /tmp/vitess-build-other/
cp -R bin /tmp/vitess-build-other/
rm -Rf bin/*
# Checkout to this build's commit
- name: Checkout to commit's code
if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
uses: actions/checkout@v3
- name: Get dependencies for this commit
if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
run: |
go mod download
- name: Run make minimaltools
if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
run: |
make minimaltools
- name: Building the binaries for this commit
if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
timeout-minutes: 5
run: |
source build.env
make build
mkdir -p /tmp/vitess-build-current/
cp -R bin /tmp/vitess-build-current/
# We create a sharded Vitess cluster following the local example.
# We also insert a few rows in our three tables.
- name: Create the example Vitess cluster with all components using version N
if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
timeout-minutes: 5
run: |
source build.env ; cd examples/local
./backups/start_cluster.sh
# Taking a backup
- name: Take a backup of all the shards
if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
timeout-minutes: 2
run: |
source build.env ; cd examples/local
./backups/take_backups.sh
# We insert more data in every table after the backup.
# When we restore the backup made in the next step, we do not want to see the rows we are about to insert now.
# The initial number of rows for each table is:
# - customer: 5
# - product: 2
# - corder: 5
# We shall see the same number of rows after restoring the backup.
- name: Insert more data after the backup
if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
run: |
source build.env ; cd examples/local ; source ./env.sh
echo "insert into customer(email) values('[email protected]');" | mysql
echo "insert into product(sku, description, price) values('SKU-1009', 'description', 89);" | mysql
echo "insert into corder(customer_id, sku, price) values(1, 'SKU-1009', 100);" | mysql
# Stop all the tablets and remove their data
- name: Stop tablets
if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
timeout-minutes: 5
run: |
source build.env ; cd examples/local
./backups/stop_tablets.sh
# We downgrade: we use the version N+1 of vttablet
- name: Downgrade - Swap binaries, use VTTablet N+1
if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
run: |
source build.env
rm -f $PWD/bin/vttablet
cp /tmp/vitess-build-other/bin/vttablet $PWD/bin/vttablet
vttablet --version
# Starting the tablets again, they will automatically start restoring the last backup.
- name: Start new tablets and restore
if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
timeout-minutes: 5
run: |
source build.env ; cd examples/local
./backups/restart_tablets.sh
# give enough time to the tablets to restore the backup
sleep 60
# Count the number of rows in each table to make sure the restoration is successful.
- name: Assert the number of rows in every table
if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
run: |
source build.env ; cd examples/local ; source ./env.sh
echo "select count(sku) from product;" | mysql 2>&1| grep 2
echo "select count(email) from customer;" | mysql 2>&1| grep 5
echo "select count(sku) from corder;" | mysql 2>&1| grep 5
# We insert one more row in every table.
- name: Insert more rows in the tables
if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
run: |
source build.env ; cd examples/local ; source ./env.sh
echo "insert into customer(email) values('[email protected]');" | mysql
echo "insert into product(sku, description, price) values('SKU-1011', 'description', 111);" | mysql
echo "insert into corder(customer_id, sku, price) values(1, 'SKU-1011', 111);" | mysql
# Taking a second backup of the cluster.
- name: Take a second backup of all the shards
if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
timeout-minutes: 2
run: |
source build.env ; cd examples/local
./backups/take_backups.sh
# Stopping the tablets so we can perform the upgrade.
- name: Stop tablets
if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
timeout-minutes: 5
run: |
source build.env ; cd examples/local
./backups/stop_tablets.sh
# We upgrade: we swap binaries and use the version N of the tablet.
- name: Upgrade - Swap binaries, use VTTablet N
if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
run: |
source build.env
rm -f $PWD/bin/vttablet
cp /tmp/vitess-build-current/bin/vttablet $PWD/bin/vttablet
vttablet --version
# Starting the tablets again and restoring the next backup.
- name: Start new tablets and restore
if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
timeout-minutes: 5
run: |
source build.env ; cd examples/local
./backups/restart_tablets.sh
# give enough time to the tablets to restore the backup
sleep 60
# We count the number of rows in every table to check that the restore step was successful.
- name: Assert the number of rows in every table
if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
run: |
source build.env ; cd examples/local ; source ./env.sh
echo "select count(sku) from product;" | mysql 2>&1| grep 3
echo "select count(email) from customer;" | mysql 2>&1| grep 6
echo "select count(sku) from corder;" | mysql 2>&1| grep 6
- name: Stop the Vitess cluster
if: always() && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
run: |
source build.env ; cd examples/local
./401_teardown.sh || true