diff --git a/.github/docker/cluster_test_12/Dockerfile b/.github/docker/cluster_test_12/Dockerfile deleted file mode 100644 index c13bafe0d87..00000000000 --- a/.github/docker/cluster_test_12/Dockerfile +++ /dev/null @@ -1,34 +0,0 @@ -# DO NOT MODIFY: THIS FILE IS GENERATED USING "make generate_ci_workflows" - -ARG bootstrap_version=10.2 -ARG image="vitess/bootstrap:${bootstrap_version}-mysql57" - -FROM "${image}" - -USER root - -# Re-copy sources from working tree -RUN rm -rf /vt/src/vitess.io/vitess/* -COPY . /vt/src/vitess.io/vitess - -# Set the working directory -WORKDIR /vt/src/vitess.io/vitess - -# Fix permissions -RUN chown -R vitess:vitess /vt - -USER vitess - -# Set environment variables -ENV VTROOT /vt/src/vitess.io/vitess -# Set the vtdataroot such that it uses the volume mount -ENV VTDATAROOT /vt/vtdataroot - -# create the vtdataroot directory -RUN mkdir -p $VTDATAROOT - -# install goimports -RUN go install golang.org/x/tools/cmd/goimports@latest - -# sleep for 50 minutes -CMD sleep 3000 diff --git a/.github/docker/cluster_test_18/Dockerfile b/.github/docker/cluster_test_18/Dockerfile deleted file mode 100644 index 29b6317b381..00000000000 --- a/.github/docker/cluster_test_18/Dockerfile +++ /dev/null @@ -1,37 +0,0 @@ -# DO NOT MODIFY: THIS FILE IS GENERATED USING "make generate_ci_workflows" - -ARG bootstrap_version=10.2 -ARG image="vitess/bootstrap:${bootstrap_version}-mysql57" - -FROM "${image}" - -USER root - -# Re-copy sources from working tree -RUN rm -rf /vt/src/vitess.io/vitess/* -COPY . /vt/src/vitess.io/vitess - -# Set the working directory -WORKDIR /vt/src/vitess.io/vitess - -# Fix permissions -RUN chown -R vitess:vitess /vt - -USER vitess - -# Set environment variables -ENV VTROOT /vt/src/vitess.io/vitess -# Set the vtdataroot such that it uses the volume mount -ENV VTDATAROOT /vt/vtdataroot - -# create the vtdataroot directory -RUN mkdir -p $VTDATAROOT - -# install goimports -RUN go install golang.org/x/tools/cmd/goimports@latest - -# make tools -RUN make tools - -# sleep for 50 minutes -CMD sleep 3000 diff --git a/.github/workflows/cluster_endtoend_12.yml b/.github/workflows/cluster_endtoend_12.yml index b6f2cab4d40..2208b4218d9 100644 --- a/.github/workflows/cluster_endtoend_12.yml +++ b/.github/workflows/cluster_endtoend_12.yml @@ -9,7 +9,7 @@ concurrency: jobs: build: name: Run endtoend tests on Cluster (12) - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - name: Check if workflow needs to be skipped @@ -59,6 +59,8 @@ jobs: - name: Tune the OS if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf @@ -67,10 +69,22 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + # Setup MySQL 8.0 + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata - sudo service mysql stop + # Install everything else we need, and configure + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get update + sudo service etcd stop + echo 'mysql version: ' + sudo mysql --version + sudo service mysql stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download @@ -80,7 +94,7 @@ jobs: - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. diff --git a/.github/workflows/cluster_endtoend_13.yml b/.github/workflows/cluster_endtoend_13.yml index fe2a8bef2e5..c41d4b1d27c 100644 --- a/.github/workflows/cluster_endtoend_13.yml +++ b/.github/workflows/cluster_endtoend_13.yml @@ -9,7 +9,7 @@ concurrency: jobs: build: name: Run endtoend tests on Cluster (13) - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - name: Check if workflow needs to be skipped @@ -59,6 +59,8 @@ jobs: - name: Tune the OS if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf @@ -67,10 +69,22 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + # Setup MySQL 8.0 + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata - sudo service mysql stop + # Install everything else we need, and configure + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get update + sudo service etcd stop + echo 'mysql version: ' + sudo mysql --version + sudo service mysql stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download @@ -80,7 +94,7 @@ jobs: - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. diff --git a/.github/workflows/cluster_endtoend_15.yml b/.github/workflows/cluster_endtoend_15.yml index c5057e1fb46..2d7846ce852 100644 --- a/.github/workflows/cluster_endtoend_15.yml +++ b/.github/workflows/cluster_endtoend_15.yml @@ -9,7 +9,7 @@ concurrency: jobs: build: name: Run endtoend tests on Cluster (15) - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - name: Check if workflow needs to be skipped @@ -59,6 +59,8 @@ jobs: - name: Tune the OS if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf @@ -67,10 +69,22 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + # Setup MySQL 8.0 + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata - sudo service mysql stop + # Install everything else we need, and configure + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get update + sudo service etcd stop + echo 'mysql version: ' + sudo mysql --version + sudo service mysql stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download @@ -80,7 +94,7 @@ jobs: - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. diff --git a/.github/workflows/cluster_endtoend_18.yml b/.github/workflows/cluster_endtoend_18.yml index e6962e4a9fa..5555f64fa12 100644 --- a/.github/workflows/cluster_endtoend_18.yml +++ b/.github/workflows/cluster_endtoend_18.yml @@ -9,7 +9,7 @@ concurrency: jobs: build: name: Run endtoend tests on Cluster (18) - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - name: Check if workflow needs to be skipped @@ -59,6 +59,8 @@ jobs: - name: Tune the OS if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf @@ -67,10 +69,22 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + # Setup MySQL 8.0 + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata - sudo service mysql stop + # Install everything else we need, and configure + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get update + sudo service etcd stop + echo 'mysql version: ' + sudo mysql --version + sudo service mysql stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download @@ -85,7 +99,7 @@ jobs: - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. diff --git a/.github/workflows/cluster_endtoend_19.yml b/.github/workflows/cluster_endtoend_19.yml index 09868998736..b700c8c333e 100644 --- a/.github/workflows/cluster_endtoend_19.yml +++ b/.github/workflows/cluster_endtoend_19.yml @@ -9,7 +9,7 @@ concurrency: jobs: build: name: Run endtoend tests on Cluster (19) - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - name: Check if workflow needs to be skipped @@ -59,6 +59,8 @@ jobs: - name: Tune the OS if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf @@ -67,10 +69,22 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + # Setup MySQL 8.0 + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata - sudo service mysql stop + # Install everything else we need, and configure + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get update + sudo service etcd stop + echo 'mysql version: ' + sudo mysql --version + sudo service mysql stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download @@ -80,7 +94,7 @@ jobs: - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. diff --git a/.github/workflows/cluster_endtoend_21.yml b/.github/workflows/cluster_endtoend_21.yml index b83b12fdd11..8e6220bc3e8 100644 --- a/.github/workflows/cluster_endtoend_21.yml +++ b/.github/workflows/cluster_endtoend_21.yml @@ -9,7 +9,7 @@ concurrency: jobs: build: name: Run endtoend tests on Cluster (21) - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - name: Check if workflow needs to be skipped @@ -59,6 +59,8 @@ jobs: - name: Tune the OS if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf @@ -67,10 +69,22 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + # Setup MySQL 8.0 + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata - sudo service mysql stop + # Install everything else we need, and configure + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get update + sudo service etcd stop + echo 'mysql version: ' + sudo mysql --version + sudo service mysql stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download @@ -80,7 +94,7 @@ jobs: - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. diff --git a/.github/workflows/cluster_endtoend_22.yml b/.github/workflows/cluster_endtoend_22.yml index 1b8172e480f..5ce247f6cd1 100644 --- a/.github/workflows/cluster_endtoend_22.yml +++ b/.github/workflows/cluster_endtoend_22.yml @@ -9,7 +9,7 @@ concurrency: jobs: build: name: Run endtoend tests on Cluster (22) - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - name: Check if workflow needs to be skipped @@ -59,6 +59,8 @@ jobs: - name: Tune the OS if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf @@ -67,10 +69,22 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + # Setup MySQL 8.0 + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata - sudo service mysql stop + # Install everything else we need, and configure + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get update + sudo service etcd stop + echo 'mysql version: ' + sudo mysql --version + sudo service mysql stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download @@ -80,7 +94,7 @@ jobs: - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. diff --git a/.github/workflows/cluster_endtoend_24.yml b/.github/workflows/cluster_endtoend_24.yml index 8e8d6690cac..842a1f98121 100644 --- a/.github/workflows/cluster_endtoend_24.yml +++ b/.github/workflows/cluster_endtoend_24.yml @@ -9,7 +9,7 @@ concurrency: jobs: build: name: Run endtoend tests on Cluster (24) - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - name: Check if workflow needs to be skipped @@ -59,6 +59,8 @@ jobs: - name: Tune the OS if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf @@ -67,10 +69,22 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + # Setup MySQL 8.0 + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata - sudo service mysql stop + # Install everything else we need, and configure + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get update + sudo service etcd stop + echo 'mysql version: ' + sudo mysql --version + sudo service mysql stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download @@ -85,7 +99,7 @@ jobs: - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. diff --git a/.github/workflows/cluster_endtoend_ers_prs_newfeatures_heavy.yml b/.github/workflows/cluster_endtoend_ers_prs_newfeatures_heavy.yml index f90cc12e552..cc9d91d8dab 100644 --- a/.github/workflows/cluster_endtoend_ers_prs_newfeatures_heavy.yml +++ b/.github/workflows/cluster_endtoend_ers_prs_newfeatures_heavy.yml @@ -9,7 +9,7 @@ concurrency: jobs: build: name: Run endtoend tests on Cluster (ers_prs_newfeatures_heavy) - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - name: Check if workflow needs to be skipped @@ -59,6 +59,8 @@ jobs: - name: Tune the OS if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf @@ -67,10 +69,22 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + # Setup MySQL 8.0 + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata - sudo service mysql stop + # Install everything else we need, and configure + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get update + sudo service etcd stop + echo 'mysql version: ' + sudo mysql --version + sudo service mysql stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download @@ -80,7 +94,7 @@ jobs: - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. @@ -93,10 +107,11 @@ jobs: sudo sysctl -w net.ipv4.ip_local_port_range="22768 61999" # Increase our open file descriptor limit as we could hit this ulimit -n 65536 - cat <<-EOF>>./config/mycnf/mysql57.cnf + cat <<-EOF>>./config/mycnf/mysql80.cnf innodb_buffer_pool_dump_at_shutdown=OFF + innodb_buffer_pool_in_core_file=OFF innodb_buffer_pool_load_at_startup=OFF - innodb_buffer_pool_size=64M + innodb_buffer_pool_size=16M innodb_doublewrite=OFF innodb_flush_log_at_trx_commit=0 innodb_flush_method=O_DIRECT @@ -107,6 +122,7 @@ jobs: performance_schema=OFF slow-query-log=OFF EOF + make embed_config # run the tests however you normally do, then produce a JUnit XML file eatmydata -- go run test.go -docker=false -follow -shard ers_prs_newfeatures_heavy diff --git a/.github/workflows/cluster_endtoend_mysql57.yml b/.github/workflows/cluster_endtoend_mysql57.yml new file mode 100644 index 00000000000..5e3e7f3c1af --- /dev/null +++ b/.github/workflows/cluster_endtoend_mysql57.yml @@ -0,0 +1,117 @@ +# DO NOT MODIFY: THIS FILE IS GENERATED USING "make generate_ci_workflows" + +name: Cluster (mysql57) +on: [push, pull_request] +concurrency: + group: format('{0}-{1}', ${{ github.ref }}, 'Cluster (mysql57)') + cancel-in-progress: true + +jobs: + build: + name: Run endtoend tests on Cluster (mysql57) + runs-on: ubuntu-20.04 + timeout-minutes: 45 + + steps: + - name: Check if workflow needs to be skipped + id: skip-workflow + run: | + skip='false' + if [[ "${{github.event.pull_request}}" == "" ]] && [[ "${{github.ref}}" != "refs/heads/main" ]] && [[ ! "${{github.ref}}" =~ ^refs/heads/release-[0-9]+\.[0-9]$ ]] && [[ ! "${{github.ref}}" =~ "refs/tags/.*" ]]; then + skip='true' + fi + echo Skip ${skip} + echo "::set-output name=skip-workflow::${skip}" + + - name: Check out code + if: steps.skip-workflow.outputs.skip-workflow == 'false' + uses: actions/checkout@v2 + + - name: Check for changes in relevant files + if: steps.skip-workflow.outputs.skip-workflow == 'false' + uses: frouioui/paths-filter@main + id: changes + with: + token: '' + filters: | + end_to_end: + - 'go/**/*.go' + - 'test.go' + - 'Makefile' + - 'build.env' + - 'go.sum' + - 'go.mod' + - 'proto/*.proto' + - 'tools/**' + - 'config/**' + - 'bootstrap.sh' + - '.github/workflows/cluster_endtoend_mysql57.yml' + + - name: Set up Go + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/setup-go@v2 + with: + go-version: 1.18.7 + + - name: Set up python + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/setup-python@v2 + + - name: Tune the OS + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range + # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio + echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf + sudo sysctl -p /etc/sysctl.conf + + - name: Get dependencies + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + sudo apt-get update + + # Uninstall any previously installed MySQL first + sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ + sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + + sudo systemctl stop apparmor + sudo DEBIAN_FRONTEND="noninteractive" apt-get remove -y --purge mysql-server mysql-client mysql-common + sudo apt-get -y autoremove + sudo apt-get -y autoclean + sudo deluser mysql + sudo rm -rf /var/lib/mysql + sudo rm -rf /etc/mysql + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.14-1_all.deb + # Bionic packages are still compatible for Focal since there's no MySQL 5.7 + # packages for Focal. + echo mysql-apt-config mysql-apt-config/repo-codename select bionic | sudo debconf-set-selections + echo mysql-apt-config mysql-apt-config/select-server select mysql-5.7 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* + sudo apt-get update + sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y mysql-client=5.7* mysql-community-server=5.7* mysql-server=5.7* + + # Install everything else we need, and configure + sudo apt-get install -y make unzip g++ etcd curl git wget eatmydata + sudo service mysql stop + sudo service etcd stop + + # install JUnit report formatter + go install github.com/vitessio/go-junit-report@HEAD + + - name: Run cluster endtoend test + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 45 + run: | + # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file + # which musn't be more than 107 characters long. + export VTDATAROOT="/tmp/" + source build.env + + set -x + + # run the tests however you normally do, then produce a JUnit XML file + eatmydata -- go run test.go -docker=false -follow -shard mysql57 diff --git a/.github/workflows/cluster_endtoend_mysql80.yml b/.github/workflows/cluster_endtoend_mysql80.yml index 07dcc0075bc..13cb3d93689 100644 --- a/.github/workflows/cluster_endtoend_mysql80.yml +++ b/.github/workflows/cluster_endtoend_mysql80.yml @@ -59,6 +59,8 @@ jobs: - name: Tune the OS if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf @@ -67,19 +69,22 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Get key to latest MySQL repo sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 - # Setup MySQL 8.0 wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update - # Install everything else we need, and configure sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils - sudo service mysql stop + sudo apt-get update + sudo service etcd stop + echo 'mysql version: ' + sudo mysql --version + sudo service mysql stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download @@ -89,7 +94,7 @@ jobs: - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. diff --git a/.github/workflows/cluster_endtoend_onlineddl_declarative.yml b/.github/workflows/cluster_endtoend_onlineddl_declarative.yml index 8f1fa23d7cc..24373aadad1 100644 --- a/.github/workflows/cluster_endtoend_onlineddl_declarative.yml +++ b/.github/workflows/cluster_endtoend_onlineddl_declarative.yml @@ -9,7 +9,7 @@ concurrency: jobs: build: name: Run endtoend tests on Cluster (onlineddl_declarative) - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - name: Check if workflow needs to be skipped @@ -59,6 +59,8 @@ jobs: - name: Tune the OS if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf @@ -67,10 +69,22 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + # Setup MySQL 8.0 + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata - sudo service mysql stop + # Install everything else we need, and configure + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get update + sudo service etcd stop + echo 'mysql version: ' + sudo mysql --version + sudo service mysql stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download @@ -80,7 +94,7 @@ jobs: - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. diff --git a/.github/workflows/cluster_endtoend_onlineddl_declarative_mysql80.yml b/.github/workflows/cluster_endtoend_onlineddl_declarative_mysql57.yml similarity index 77% rename from .github/workflows/cluster_endtoend_onlineddl_declarative_mysql80.yml rename to .github/workflows/cluster_endtoend_onlineddl_declarative_mysql57.yml index 15a7576c00c..985ed57ec0e 100644 --- a/.github/workflows/cluster_endtoend_onlineddl_declarative_mysql80.yml +++ b/.github/workflows/cluster_endtoend_onlineddl_declarative_mysql57.yml @@ -1,15 +1,16 @@ # DO NOT MODIFY: THIS FILE IS GENERATED USING "make generate_ci_workflows" -name: Cluster (onlineddl_declarative) mysql80 +name: Cluster (onlineddl_declarative) mysql57 on: [push, pull_request] concurrency: - group: format('{0}-{1}', ${{ github.ref }}, 'Cluster (onlineddl_declarative) mysql80') + group: format('{0}-{1}', ${{ github.ref }}, 'Cluster (onlineddl_declarative) mysql57') cancel-in-progress: true jobs: build: - name: Run endtoend tests on Cluster (onlineddl_declarative) mysql80 + name: Run endtoend tests on Cluster (onlineddl_declarative) mysql57 runs-on: ubuntu-20.04 + timeout-minutes: 45 steps: - name: Check if workflow needs to be skipped @@ -44,7 +45,7 @@ jobs: - 'tools/**' - 'config/**' - 'bootstrap.sh' - - '.github/workflows/cluster_endtoend_onlineddl_declarative_mysql80.yml' + - '.github/workflows/cluster_endtoend_onlineddl_declarative_mysql57.yml' - name: Set up Go if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' @@ -67,29 +68,43 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + sudo apt-get update + + # Uninstall any previously installed MySQL first + sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ + sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + + sudo systemctl stop apparmor + sudo DEBIAN_FRONTEND="noninteractive" apt-get remove -y --purge mysql-server mysql-client mysql-common + sudo apt-get -y autoremove + sudo apt-get -y autoclean + sudo deluser mysql + sudo rm -rf /var/lib/mysql + sudo rm -rf /etc/mysql + # Get key to latest MySQL repo sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 - # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb - echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.14-1_all.deb + # Bionic packages are still compatible for Focal since there's no MySQL 5.7 + # packages for Focal. + echo mysql-apt-config mysql-apt-config/repo-codename select bionic | sudo debconf-set-selections + echo mysql-apt-config mysql-apt-config/select-server select mysql-5.7 | sudo debconf-set-selections sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update + sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y mysql-client=5.7* mysql-community-server=5.7* mysql-server=5.7* # Install everything else we need, and configure - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get install -y make unzip g++ etcd curl git wget eatmydata sudo service mysql stop sudo service etcd stop - sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ - sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld - go mod download # install JUnit report formatter go install github.com/vitessio/go-junit-report@HEAD - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. diff --git a/.github/workflows/cluster_endtoend_onlineddl_ghost.yml b/.github/workflows/cluster_endtoend_onlineddl_ghost.yml index fad7d4407b6..44df92c038f 100644 --- a/.github/workflows/cluster_endtoend_onlineddl_ghost.yml +++ b/.github/workflows/cluster_endtoend_onlineddl_ghost.yml @@ -9,7 +9,7 @@ concurrency: jobs: build: name: Run endtoend tests on Cluster (onlineddl_ghost) - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - name: Check if workflow needs to be skipped @@ -59,6 +59,8 @@ jobs: - name: Tune the OS if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf @@ -67,10 +69,22 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + # Setup MySQL 8.0 + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata - sudo service mysql stop + # Install everything else we need, and configure + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get update + sudo service etcd stop + echo 'mysql version: ' + sudo mysql --version + sudo service mysql stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download @@ -80,7 +94,7 @@ jobs: - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. diff --git a/.github/workflows/cluster_endtoend_onlineddl_ghost_mysql80.yml b/.github/workflows/cluster_endtoend_onlineddl_ghost_mysql57.yml similarity index 76% rename from .github/workflows/cluster_endtoend_onlineddl_ghost_mysql80.yml rename to .github/workflows/cluster_endtoend_onlineddl_ghost_mysql57.yml index 8b481e162c9..a6b9ae2dd5b 100644 --- a/.github/workflows/cluster_endtoend_onlineddl_ghost_mysql80.yml +++ b/.github/workflows/cluster_endtoend_onlineddl_ghost_mysql57.yml @@ -1,15 +1,16 @@ # DO NOT MODIFY: THIS FILE IS GENERATED USING "make generate_ci_workflows" -name: Cluster (onlineddl_ghost) mysql80 +name: Cluster (onlineddl_ghost) mysql57 on: [push, pull_request] concurrency: - group: format('{0}-{1}', ${{ github.ref }}, 'Cluster (onlineddl_ghost) mysql80') + group: format('{0}-{1}', ${{ github.ref }}, 'Cluster (onlineddl_ghost) mysql57') cancel-in-progress: true jobs: build: - name: Run endtoend tests on Cluster (onlineddl_ghost) mysql80 + name: Run endtoend tests on Cluster (onlineddl_ghost) mysql57 runs-on: ubuntu-20.04 + timeout-minutes: 45 steps: - name: Check if workflow needs to be skipped @@ -44,7 +45,7 @@ jobs: - 'tools/**' - 'config/**' - 'bootstrap.sh' - - '.github/workflows/cluster_endtoend_onlineddl_ghost_mysql80.yml' + - '.github/workflows/cluster_endtoend_onlineddl_ghost_mysql57.yml' - name: Set up Go if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' @@ -67,29 +68,43 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + sudo apt-get update + + # Uninstall any previously installed MySQL first + sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ + sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + + sudo systemctl stop apparmor + sudo DEBIAN_FRONTEND="noninteractive" apt-get remove -y --purge mysql-server mysql-client mysql-common + sudo apt-get -y autoremove + sudo apt-get -y autoclean + sudo deluser mysql + sudo rm -rf /var/lib/mysql + sudo rm -rf /etc/mysql + # Get key to latest MySQL repo sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 - # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb - echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.14-1_all.deb + # Bionic packages are still compatible for Focal since there's no MySQL 5.7 + # packages for Focal. + echo mysql-apt-config mysql-apt-config/repo-codename select bionic | sudo debconf-set-selections + echo mysql-apt-config mysql-apt-config/select-server select mysql-5.7 | sudo debconf-set-selections sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update + sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y mysql-client=5.7* mysql-community-server=5.7* mysql-server=5.7* # Install everything else we need, and configure - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get install -y make unzip g++ etcd curl git wget eatmydata sudo service mysql stop sudo service etcd stop - sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ - sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld - go mod download # install JUnit report formatter go install github.com/vitessio/go-junit-report@HEAD - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. diff --git a/.github/workflows/cluster_endtoend_onlineddl_revert.yml b/.github/workflows/cluster_endtoend_onlineddl_revert.yml index 3016f1f3f67..dde4adf570d 100644 --- a/.github/workflows/cluster_endtoend_onlineddl_revert.yml +++ b/.github/workflows/cluster_endtoend_onlineddl_revert.yml @@ -9,7 +9,7 @@ concurrency: jobs: build: name: Run endtoend tests on Cluster (onlineddl_revert) - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - name: Check if workflow needs to be skipped @@ -59,6 +59,8 @@ jobs: - name: Tune the OS if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf @@ -67,10 +69,22 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + # Setup MySQL 8.0 + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata - sudo service mysql stop + # Install everything else we need, and configure + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get update + sudo service etcd stop + echo 'mysql version: ' + sudo mysql --version + sudo service mysql stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download @@ -80,7 +94,7 @@ jobs: - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. diff --git a/.github/workflows/cluster_endtoend_onlineddl_revert_mysql80.yml b/.github/workflows/cluster_endtoend_onlineddl_revert_mysql57.yml similarity index 77% rename from .github/workflows/cluster_endtoend_onlineddl_revert_mysql80.yml rename to .github/workflows/cluster_endtoend_onlineddl_revert_mysql57.yml index 681dc3ac78d..e74bbd7eba5 100644 --- a/.github/workflows/cluster_endtoend_onlineddl_revert_mysql80.yml +++ b/.github/workflows/cluster_endtoend_onlineddl_revert_mysql57.yml @@ -1,15 +1,16 @@ # DO NOT MODIFY: THIS FILE IS GENERATED USING "make generate_ci_workflows" -name: Cluster (onlineddl_revert) mysql80 +name: Cluster (onlineddl_revert) mysql57 on: [push, pull_request] concurrency: - group: format('{0}-{1}', ${{ github.ref }}, 'Cluster (onlineddl_revert) mysql80') + group: format('{0}-{1}', ${{ github.ref }}, 'Cluster (onlineddl_revert) mysql57') cancel-in-progress: true jobs: build: - name: Run endtoend tests on Cluster (onlineddl_revert) mysql80 + name: Run endtoend tests on Cluster (onlineddl_revert) mysql57 runs-on: ubuntu-20.04 + timeout-minutes: 45 steps: - name: Check if workflow needs to be skipped @@ -44,7 +45,7 @@ jobs: - 'tools/**' - 'config/**' - 'bootstrap.sh' - - '.github/workflows/cluster_endtoend_onlineddl_revert_mysql80.yml' + - '.github/workflows/cluster_endtoend_onlineddl_revert_mysql57.yml' - name: Set up Go if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' @@ -67,29 +68,43 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + sudo apt-get update + + # Uninstall any previously installed MySQL first + sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ + sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + + sudo systemctl stop apparmor + sudo DEBIAN_FRONTEND="noninteractive" apt-get remove -y --purge mysql-server mysql-client mysql-common + sudo apt-get -y autoremove + sudo apt-get -y autoclean + sudo deluser mysql + sudo rm -rf /var/lib/mysql + sudo rm -rf /etc/mysql + # Get key to latest MySQL repo sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 - # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb - echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.14-1_all.deb + # Bionic packages are still compatible for Focal since there's no MySQL 5.7 + # packages for Focal. + echo mysql-apt-config mysql-apt-config/repo-codename select bionic | sudo debconf-set-selections + echo mysql-apt-config mysql-apt-config/select-server select mysql-5.7 | sudo debconf-set-selections sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update + sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y mysql-client=5.7* mysql-community-server=5.7* mysql-server=5.7* # Install everything else we need, and configure - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get install -y make unzip g++ etcd curl git wget eatmydata sudo service mysql stop sudo service etcd stop - sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ - sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld - go mod download # install JUnit report formatter go install github.com/vitessio/go-junit-report@HEAD - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. diff --git a/.github/workflows/cluster_endtoend_onlineddl_revertible.yml b/.github/workflows/cluster_endtoend_onlineddl_revertible.yml index 7768a3f919c..66c9c5b8894 100644 --- a/.github/workflows/cluster_endtoend_onlineddl_revertible.yml +++ b/.github/workflows/cluster_endtoend_onlineddl_revertible.yml @@ -9,7 +9,7 @@ concurrency: jobs: build: name: Run endtoend tests on Cluster (onlineddl_revertible) - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - name: Check if workflow needs to be skipped @@ -59,6 +59,8 @@ jobs: - name: Tune the OS if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf @@ -67,10 +69,22 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + # Setup MySQL 8.0 + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata - sudo service mysql stop + # Install everything else we need, and configure + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get update + sudo service etcd stop + echo 'mysql version: ' + sudo mysql --version + sudo service mysql stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download @@ -80,7 +94,7 @@ jobs: - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. diff --git a/.github/workflows/cluster_endtoend_onlineddl_revertible_mysql80.yml b/.github/workflows/cluster_endtoend_onlineddl_revertible_mysql57.yml similarity index 77% rename from .github/workflows/cluster_endtoend_onlineddl_revertible_mysql80.yml rename to .github/workflows/cluster_endtoend_onlineddl_revertible_mysql57.yml index 83df8d6d82b..3fcb0ebddfc 100644 --- a/.github/workflows/cluster_endtoend_onlineddl_revertible_mysql80.yml +++ b/.github/workflows/cluster_endtoend_onlineddl_revertible_mysql57.yml @@ -1,15 +1,16 @@ # DO NOT MODIFY: THIS FILE IS GENERATED USING "make generate_ci_workflows" -name: Cluster (onlineddl_revertible) mysql80 +name: Cluster (onlineddl_revertible) mysql57 on: [push, pull_request] concurrency: - group: format('{0}-{1}', ${{ github.ref }}, 'Cluster (onlineddl_revertible) mysql80') + group: format('{0}-{1}', ${{ github.ref }}, 'Cluster (onlineddl_revertible) mysql57') cancel-in-progress: true jobs: build: - name: Run endtoend tests on Cluster (onlineddl_revertible) mysql80 + name: Run endtoend tests on Cluster (onlineddl_revertible) mysql57 runs-on: ubuntu-20.04 + timeout-minutes: 45 steps: - name: Check if workflow needs to be skipped @@ -44,7 +45,7 @@ jobs: - 'tools/**' - 'config/**' - 'bootstrap.sh' - - '.github/workflows/cluster_endtoend_onlineddl_revertible_mysql80.yml' + - '.github/workflows/cluster_endtoend_onlineddl_revertible_mysql57.yml' - name: Set up Go if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' @@ -67,29 +68,43 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + sudo apt-get update + + # Uninstall any previously installed MySQL first + sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ + sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + + sudo systemctl stop apparmor + sudo DEBIAN_FRONTEND="noninteractive" apt-get remove -y --purge mysql-server mysql-client mysql-common + sudo apt-get -y autoremove + sudo apt-get -y autoclean + sudo deluser mysql + sudo rm -rf /var/lib/mysql + sudo rm -rf /etc/mysql + # Get key to latest MySQL repo sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 - # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb - echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.14-1_all.deb + # Bionic packages are still compatible for Focal since there's no MySQL 5.7 + # packages for Focal. + echo mysql-apt-config mysql-apt-config/repo-codename select bionic | sudo debconf-set-selections + echo mysql-apt-config mysql-apt-config/select-server select mysql-5.7 | sudo debconf-set-selections sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update + sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y mysql-client=5.7* mysql-community-server=5.7* mysql-server=5.7* # Install everything else we need, and configure - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get install -y make unzip g++ etcd curl git wget eatmydata sudo service mysql stop sudo service etcd stop - sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ - sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld - go mod download # install JUnit report formatter go install github.com/vitessio/go-junit-report@HEAD - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. diff --git a/.github/workflows/cluster_endtoend_onlineddl_scheduler.yml b/.github/workflows/cluster_endtoend_onlineddl_scheduler.yml index c622a8d0271..cfeecd23b89 100644 --- a/.github/workflows/cluster_endtoend_onlineddl_scheduler.yml +++ b/.github/workflows/cluster_endtoend_onlineddl_scheduler.yml @@ -9,7 +9,7 @@ concurrency: jobs: build: name: Run endtoend tests on Cluster (onlineddl_scheduler) - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - name: Check if workflow needs to be skipped @@ -59,6 +59,8 @@ jobs: - name: Tune the OS if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf @@ -67,10 +69,22 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + # Setup MySQL 8.0 + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata - sudo service mysql stop + # Install everything else we need, and configure + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get update + sudo service etcd stop + echo 'mysql version: ' + sudo mysql --version + sudo service mysql stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download @@ -80,7 +94,7 @@ jobs: - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. diff --git a/.github/workflows/cluster_endtoend_onlineddl_scheduler_mysql80.yml b/.github/workflows/cluster_endtoend_onlineddl_scheduler_mysql57.yml similarity index 77% rename from .github/workflows/cluster_endtoend_onlineddl_scheduler_mysql80.yml rename to .github/workflows/cluster_endtoend_onlineddl_scheduler_mysql57.yml index cfe5084093f..356c929af6f 100644 --- a/.github/workflows/cluster_endtoend_onlineddl_scheduler_mysql80.yml +++ b/.github/workflows/cluster_endtoend_onlineddl_scheduler_mysql57.yml @@ -1,15 +1,16 @@ # DO NOT MODIFY: THIS FILE IS GENERATED USING "make generate_ci_workflows" -name: Cluster (onlineddl_scheduler) mysql80 +name: Cluster (onlineddl_scheduler) mysql57 on: [push, pull_request] concurrency: - group: format('{0}-{1}', ${{ github.ref }}, 'Cluster (onlineddl_scheduler) mysql80') + group: format('{0}-{1}', ${{ github.ref }}, 'Cluster (onlineddl_scheduler) mysql57') cancel-in-progress: true jobs: build: - name: Run endtoend tests on Cluster (onlineddl_scheduler) mysql80 + name: Run endtoend tests on Cluster (onlineddl_scheduler) mysql57 runs-on: ubuntu-20.04 + timeout-minutes: 45 steps: - name: Check if workflow needs to be skipped @@ -44,7 +45,7 @@ jobs: - 'tools/**' - 'config/**' - 'bootstrap.sh' - - '.github/workflows/cluster_endtoend_onlineddl_scheduler_mysql80.yml' + - '.github/workflows/cluster_endtoend_onlineddl_scheduler_mysql57.yml' - name: Set up Go if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' @@ -67,29 +68,43 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + sudo apt-get update + + # Uninstall any previously installed MySQL first + sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ + sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + + sudo systemctl stop apparmor + sudo DEBIAN_FRONTEND="noninteractive" apt-get remove -y --purge mysql-server mysql-client mysql-common + sudo apt-get -y autoremove + sudo apt-get -y autoclean + sudo deluser mysql + sudo rm -rf /var/lib/mysql + sudo rm -rf /etc/mysql + # Get key to latest MySQL repo sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 - # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb - echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.14-1_all.deb + # Bionic packages are still compatible for Focal since there's no MySQL 5.7 + # packages for Focal. + echo mysql-apt-config mysql-apt-config/repo-codename select bionic | sudo debconf-set-selections + echo mysql-apt-config mysql-apt-config/select-server select mysql-5.7 | sudo debconf-set-selections sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update + sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y mysql-client=5.7* mysql-community-server=5.7* mysql-server=5.7* # Install everything else we need, and configure - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get install -y make unzip g++ etcd curl git wget eatmydata sudo service mysql stop sudo service etcd stop - sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ - sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld - go mod download # install JUnit report formatter go install github.com/vitessio/go-junit-report@HEAD - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. diff --git a/.github/workflows/cluster_endtoend_onlineddl_singleton.yml b/.github/workflows/cluster_endtoend_onlineddl_singleton.yml index bef4bf4b9d4..daeaed40636 100644 --- a/.github/workflows/cluster_endtoend_onlineddl_singleton.yml +++ b/.github/workflows/cluster_endtoend_onlineddl_singleton.yml @@ -9,7 +9,7 @@ concurrency: jobs: build: name: Run endtoend tests on Cluster (onlineddl_singleton) - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - name: Check if workflow needs to be skipped @@ -59,6 +59,8 @@ jobs: - name: Tune the OS if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf @@ -67,10 +69,22 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + # Setup MySQL 8.0 + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata - sudo service mysql stop + # Install everything else we need, and configure + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get update + sudo service etcd stop + echo 'mysql version: ' + sudo mysql --version + sudo service mysql stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download @@ -80,7 +94,7 @@ jobs: - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. diff --git a/.github/workflows/cluster_endtoend_onlineddl_singleton_mysql80.yml b/.github/workflows/cluster_endtoend_onlineddl_singleton_mysql57.yml similarity index 77% rename from .github/workflows/cluster_endtoend_onlineddl_singleton_mysql80.yml rename to .github/workflows/cluster_endtoend_onlineddl_singleton_mysql57.yml index 19e8c27129d..2bd8edcc553 100644 --- a/.github/workflows/cluster_endtoend_onlineddl_singleton_mysql80.yml +++ b/.github/workflows/cluster_endtoend_onlineddl_singleton_mysql57.yml @@ -1,15 +1,16 @@ # DO NOT MODIFY: THIS FILE IS GENERATED USING "make generate_ci_workflows" -name: Cluster (onlineddl_singleton) mysql80 +name: Cluster (onlineddl_singleton) mysql57 on: [push, pull_request] concurrency: - group: format('{0}-{1}', ${{ github.ref }}, 'Cluster (onlineddl_singleton) mysql80') + group: format('{0}-{1}', ${{ github.ref }}, 'Cluster (onlineddl_singleton) mysql57') cancel-in-progress: true jobs: build: - name: Run endtoend tests on Cluster (onlineddl_singleton) mysql80 + name: Run endtoend tests on Cluster (onlineddl_singleton) mysql57 runs-on: ubuntu-20.04 + timeout-minutes: 45 steps: - name: Check if workflow needs to be skipped @@ -44,7 +45,7 @@ jobs: - 'tools/**' - 'config/**' - 'bootstrap.sh' - - '.github/workflows/cluster_endtoend_onlineddl_singleton_mysql80.yml' + - '.github/workflows/cluster_endtoend_onlineddl_singleton_mysql57.yml' - name: Set up Go if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' @@ -67,29 +68,43 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + sudo apt-get update + + # Uninstall any previously installed MySQL first + sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ + sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + + sudo systemctl stop apparmor + sudo DEBIAN_FRONTEND="noninteractive" apt-get remove -y --purge mysql-server mysql-client mysql-common + sudo apt-get -y autoremove + sudo apt-get -y autoclean + sudo deluser mysql + sudo rm -rf /var/lib/mysql + sudo rm -rf /etc/mysql + # Get key to latest MySQL repo sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 - # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb - echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.14-1_all.deb + # Bionic packages are still compatible for Focal since there's no MySQL 5.7 + # packages for Focal. + echo mysql-apt-config mysql-apt-config/repo-codename select bionic | sudo debconf-set-selections + echo mysql-apt-config mysql-apt-config/select-server select mysql-5.7 | sudo debconf-set-selections sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update + sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y mysql-client=5.7* mysql-community-server=5.7* mysql-server=5.7* # Install everything else we need, and configure - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get install -y make unzip g++ etcd curl git wget eatmydata sudo service mysql stop sudo service etcd stop - sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ - sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld - go mod download # install JUnit report formatter go install github.com/vitessio/go-junit-report@HEAD - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. diff --git a/.github/workflows/cluster_endtoend_onlineddl_vrepl.yml b/.github/workflows/cluster_endtoend_onlineddl_vrepl.yml index 7d1dd17e58b..3b290b76b3c 100644 --- a/.github/workflows/cluster_endtoend_onlineddl_vrepl.yml +++ b/.github/workflows/cluster_endtoend_onlineddl_vrepl.yml @@ -9,7 +9,7 @@ concurrency: jobs: build: name: Run endtoend tests on Cluster (onlineddl_vrepl) - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - name: Check if workflow needs to be skipped @@ -59,6 +59,8 @@ jobs: - name: Tune the OS if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf @@ -67,10 +69,22 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + # Setup MySQL 8.0 + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata - sudo service mysql stop + # Install everything else we need, and configure + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get update + sudo service etcd stop + echo 'mysql version: ' + sudo mysql --version + sudo service mysql stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download @@ -80,7 +94,7 @@ jobs: - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. diff --git a/.github/workflows/cluster_endtoend_onlineddl_vrepl_mysql80.yml b/.github/workflows/cluster_endtoend_onlineddl_vrepl_mysql57.yml similarity index 76% rename from .github/workflows/cluster_endtoend_onlineddl_vrepl_mysql80.yml rename to .github/workflows/cluster_endtoend_onlineddl_vrepl_mysql57.yml index c3511ffb4fd..4ee40ffde1d 100644 --- a/.github/workflows/cluster_endtoend_onlineddl_vrepl_mysql80.yml +++ b/.github/workflows/cluster_endtoend_onlineddl_vrepl_mysql57.yml @@ -1,15 +1,16 @@ # DO NOT MODIFY: THIS FILE IS GENERATED USING "make generate_ci_workflows" -name: Cluster (onlineddl_vrepl) mysql80 +name: Cluster (onlineddl_vrepl) mysql57 on: [push, pull_request] concurrency: - group: format('{0}-{1}', ${{ github.ref }}, 'Cluster (onlineddl_vrepl) mysql80') + group: format('{0}-{1}', ${{ github.ref }}, 'Cluster (onlineddl_vrepl) mysql57') cancel-in-progress: true jobs: build: - name: Run endtoend tests on Cluster (onlineddl_vrepl) mysql80 + name: Run endtoend tests on Cluster (onlineddl_vrepl) mysql57 runs-on: ubuntu-20.04 + timeout-minutes: 45 steps: - name: Check if workflow needs to be skipped @@ -44,7 +45,7 @@ jobs: - 'tools/**' - 'config/**' - 'bootstrap.sh' - - '.github/workflows/cluster_endtoend_onlineddl_vrepl_mysql80.yml' + - '.github/workflows/cluster_endtoend_onlineddl_vrepl_mysql57.yml' - name: Set up Go if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' @@ -67,29 +68,43 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + sudo apt-get update + + # Uninstall any previously installed MySQL first + sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ + sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + + sudo systemctl stop apparmor + sudo DEBIAN_FRONTEND="noninteractive" apt-get remove -y --purge mysql-server mysql-client mysql-common + sudo apt-get -y autoremove + sudo apt-get -y autoclean + sudo deluser mysql + sudo rm -rf /var/lib/mysql + sudo rm -rf /etc/mysql + # Get key to latest MySQL repo sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 - # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb - echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.14-1_all.deb + # Bionic packages are still compatible for Focal since there's no MySQL 5.7 + # packages for Focal. + echo mysql-apt-config mysql-apt-config/repo-codename select bionic | sudo debconf-set-selections + echo mysql-apt-config mysql-apt-config/select-server select mysql-5.7 | sudo debconf-set-selections sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update + sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y mysql-client=5.7* mysql-community-server=5.7* mysql-server=5.7* # Install everything else we need, and configure - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get install -y make unzip g++ etcd curl git wget eatmydata sudo service mysql stop sudo service etcd stop - sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ - sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld - go mod download # install JUnit report formatter go install github.com/vitessio/go-junit-report@HEAD - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. diff --git a/.github/workflows/cluster_endtoend_onlineddl_vrepl_stress.yml b/.github/workflows/cluster_endtoend_onlineddl_vrepl_stress.yml index 19ff2664b5e..65239436d1e 100644 --- a/.github/workflows/cluster_endtoend_onlineddl_vrepl_stress.yml +++ b/.github/workflows/cluster_endtoend_onlineddl_vrepl_stress.yml @@ -9,7 +9,7 @@ concurrency: jobs: build: name: Run endtoend tests on Cluster (onlineddl_vrepl_stress) - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - name: Check if workflow needs to be skipped @@ -59,6 +59,8 @@ jobs: - name: Tune the OS if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf @@ -67,10 +69,22 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + # Setup MySQL 8.0 + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata - sudo service mysql stop + # Install everything else we need, and configure + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get update + sudo service etcd stop + echo 'mysql version: ' + sudo mysql --version + sudo service mysql stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download @@ -80,7 +94,7 @@ jobs: - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. diff --git a/.github/workflows/cluster_endtoend_onlineddl_vrepl_stress_mysql80.yml b/.github/workflows/cluster_endtoend_onlineddl_vrepl_stress_mysql57.yml similarity index 77% rename from .github/workflows/cluster_endtoend_onlineddl_vrepl_stress_mysql80.yml rename to .github/workflows/cluster_endtoend_onlineddl_vrepl_stress_mysql57.yml index a6c4f4af583..414ddc30b04 100644 --- a/.github/workflows/cluster_endtoend_onlineddl_vrepl_stress_mysql80.yml +++ b/.github/workflows/cluster_endtoend_onlineddl_vrepl_stress_mysql57.yml @@ -1,15 +1,16 @@ # DO NOT MODIFY: THIS FILE IS GENERATED USING "make generate_ci_workflows" -name: Cluster (onlineddl_vrepl_stress) mysql80 +name: Cluster (onlineddl_vrepl_stress) mysql57 on: [push, pull_request] concurrency: - group: format('{0}-{1}', ${{ github.ref }}, 'Cluster (onlineddl_vrepl_stress) mysql80') + group: format('{0}-{1}', ${{ github.ref }}, 'Cluster (onlineddl_vrepl_stress) mysql57') cancel-in-progress: true jobs: build: - name: Run endtoend tests on Cluster (onlineddl_vrepl_stress) mysql80 + name: Run endtoend tests on Cluster (onlineddl_vrepl_stress) mysql57 runs-on: ubuntu-20.04 + timeout-minutes: 45 steps: - name: Check if workflow needs to be skipped @@ -44,7 +45,7 @@ jobs: - 'tools/**' - 'config/**' - 'bootstrap.sh' - - '.github/workflows/cluster_endtoend_onlineddl_vrepl_stress_mysql80.yml' + - '.github/workflows/cluster_endtoend_onlineddl_vrepl_stress_mysql57.yml' - name: Set up Go if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' @@ -67,29 +68,43 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + sudo apt-get update + + # Uninstall any previously installed MySQL first + sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ + sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + + sudo systemctl stop apparmor + sudo DEBIAN_FRONTEND="noninteractive" apt-get remove -y --purge mysql-server mysql-client mysql-common + sudo apt-get -y autoremove + sudo apt-get -y autoclean + sudo deluser mysql + sudo rm -rf /var/lib/mysql + sudo rm -rf /etc/mysql + # Get key to latest MySQL repo sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 - # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb - echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.14-1_all.deb + # Bionic packages are still compatible for Focal since there's no MySQL 5.7 + # packages for Focal. + echo mysql-apt-config mysql-apt-config/repo-codename select bionic | sudo debconf-set-selections + echo mysql-apt-config mysql-apt-config/select-server select mysql-5.7 | sudo debconf-set-selections sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update + sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y mysql-client=5.7* mysql-community-server=5.7* mysql-server=5.7* # Install everything else we need, and configure - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get install -y make unzip g++ etcd curl git wget eatmydata sudo service mysql stop sudo service etcd stop - sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ - sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld - go mod download # install JUnit report formatter go install github.com/vitessio/go-junit-report@HEAD - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. diff --git a/.github/workflows/cluster_endtoend_onlineddl_vrepl_stress_suite.yml b/.github/workflows/cluster_endtoend_onlineddl_vrepl_stress_suite.yml index f3a9ced8c80..8d95fa3d6cd 100644 --- a/.github/workflows/cluster_endtoend_onlineddl_vrepl_stress_suite.yml +++ b/.github/workflows/cluster_endtoend_onlineddl_vrepl_stress_suite.yml @@ -9,7 +9,7 @@ concurrency: jobs: build: name: Run endtoend tests on Cluster (onlineddl_vrepl_stress_suite) - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - name: Check if workflow needs to be skipped @@ -59,6 +59,8 @@ jobs: - name: Tune the OS if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf @@ -67,10 +69,22 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + # Setup MySQL 8.0 + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata - sudo service mysql stop + # Install everything else we need, and configure + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get update + sudo service etcd stop + echo 'mysql version: ' + sudo mysql --version + sudo service mysql stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download @@ -80,7 +94,7 @@ jobs: - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. diff --git a/.github/workflows/cluster_endtoend_onlineddl_vrepl_stress_suite_mysql80.yml b/.github/workflows/cluster_endtoend_onlineddl_vrepl_stress_suite_mysql57.yml similarity index 76% rename from .github/workflows/cluster_endtoend_onlineddl_vrepl_stress_suite_mysql80.yml rename to .github/workflows/cluster_endtoend_onlineddl_vrepl_stress_suite_mysql57.yml index 2235e028127..64467b70579 100644 --- a/.github/workflows/cluster_endtoend_onlineddl_vrepl_stress_suite_mysql80.yml +++ b/.github/workflows/cluster_endtoend_onlineddl_vrepl_stress_suite_mysql57.yml @@ -1,15 +1,16 @@ # DO NOT MODIFY: THIS FILE IS GENERATED USING "make generate_ci_workflows" -name: Cluster (onlineddl_vrepl_stress_suite) mysql80 +name: Cluster (onlineddl_vrepl_stress_suite) mysql57 on: [push, pull_request] concurrency: - group: format('{0}-{1}', ${{ github.ref }}, 'Cluster (onlineddl_vrepl_stress_suite) mysql80') + group: format('{0}-{1}', ${{ github.ref }}, 'Cluster (onlineddl_vrepl_stress_suite) mysql57') cancel-in-progress: true jobs: build: - name: Run endtoend tests on Cluster (onlineddl_vrepl_stress_suite) mysql80 + name: Run endtoend tests on Cluster (onlineddl_vrepl_stress_suite) mysql57 runs-on: ubuntu-20.04 + timeout-minutes: 45 steps: - name: Check if workflow needs to be skipped @@ -44,7 +45,7 @@ jobs: - 'tools/**' - 'config/**' - 'bootstrap.sh' - - '.github/workflows/cluster_endtoend_onlineddl_vrepl_stress_suite_mysql80.yml' + - '.github/workflows/cluster_endtoend_onlineddl_vrepl_stress_suite_mysql57.yml' - name: Set up Go if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' @@ -67,29 +68,43 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + sudo apt-get update + + # Uninstall any previously installed MySQL first + sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ + sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + + sudo systemctl stop apparmor + sudo DEBIAN_FRONTEND="noninteractive" apt-get remove -y --purge mysql-server mysql-client mysql-common + sudo apt-get -y autoremove + sudo apt-get -y autoclean + sudo deluser mysql + sudo rm -rf /var/lib/mysql + sudo rm -rf /etc/mysql + # Get key to latest MySQL repo sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 - # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb - echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.14-1_all.deb + # Bionic packages are still compatible for Focal since there's no MySQL 5.7 + # packages for Focal. + echo mysql-apt-config mysql-apt-config/repo-codename select bionic | sudo debconf-set-selections + echo mysql-apt-config mysql-apt-config/select-server select mysql-5.7 | sudo debconf-set-selections sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update + sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y mysql-client=5.7* mysql-community-server=5.7* mysql-server=5.7* # Install everything else we need, and configure - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get install -y make unzip g++ etcd curl git wget eatmydata sudo service mysql stop sudo service etcd stop - sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ - sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld - go mod download # install JUnit report formatter go install github.com/vitessio/go-junit-report@HEAD - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. diff --git a/.github/workflows/cluster_endtoend_onlineddl_vrepl_suite.yml b/.github/workflows/cluster_endtoend_onlineddl_vrepl_suite.yml index 6c019a034ef..c41f9963005 100644 --- a/.github/workflows/cluster_endtoend_onlineddl_vrepl_suite.yml +++ b/.github/workflows/cluster_endtoend_onlineddl_vrepl_suite.yml @@ -9,7 +9,7 @@ concurrency: jobs: build: name: Run endtoend tests on Cluster (onlineddl_vrepl_suite) - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - name: Check if workflow needs to be skipped @@ -59,6 +59,8 @@ jobs: - name: Tune the OS if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf @@ -67,10 +69,22 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + # Setup MySQL 8.0 + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata - sudo service mysql stop + # Install everything else we need, and configure + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get update + sudo service etcd stop + echo 'mysql version: ' + sudo mysql --version + sudo service mysql stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download @@ -80,7 +94,7 @@ jobs: - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. diff --git a/.github/workflows/cluster_endtoend_onlineddl_vrepl_suite_mysql80.yml b/.github/workflows/cluster_endtoend_onlineddl_vrepl_suite_mysql57.yml similarity index 77% rename from .github/workflows/cluster_endtoend_onlineddl_vrepl_suite_mysql80.yml rename to .github/workflows/cluster_endtoend_onlineddl_vrepl_suite_mysql57.yml index 014f206b350..98fe7ff03d9 100644 --- a/.github/workflows/cluster_endtoend_onlineddl_vrepl_suite_mysql80.yml +++ b/.github/workflows/cluster_endtoend_onlineddl_vrepl_suite_mysql57.yml @@ -1,15 +1,16 @@ # DO NOT MODIFY: THIS FILE IS GENERATED USING "make generate_ci_workflows" -name: Cluster (onlineddl_vrepl_suite) mysql80 +name: Cluster (onlineddl_vrepl_suite) mysql57 on: [push, pull_request] concurrency: - group: format('{0}-{1}', ${{ github.ref }}, 'Cluster (onlineddl_vrepl_suite) mysql80') + group: format('{0}-{1}', ${{ github.ref }}, 'Cluster (onlineddl_vrepl_suite) mysql57') cancel-in-progress: true jobs: build: - name: Run endtoend tests on Cluster (onlineddl_vrepl_suite) mysql80 + name: Run endtoend tests on Cluster (onlineddl_vrepl_suite) mysql57 runs-on: ubuntu-20.04 + timeout-minutes: 45 steps: - name: Check if workflow needs to be skipped @@ -44,7 +45,7 @@ jobs: - 'tools/**' - 'config/**' - 'bootstrap.sh' - - '.github/workflows/cluster_endtoend_onlineddl_vrepl_suite_mysql80.yml' + - '.github/workflows/cluster_endtoend_onlineddl_vrepl_suite_mysql57.yml' - name: Set up Go if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' @@ -67,29 +68,43 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + sudo apt-get update + + # Uninstall any previously installed MySQL first + sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ + sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + + sudo systemctl stop apparmor + sudo DEBIAN_FRONTEND="noninteractive" apt-get remove -y --purge mysql-server mysql-client mysql-common + sudo apt-get -y autoremove + sudo apt-get -y autoclean + sudo deluser mysql + sudo rm -rf /var/lib/mysql + sudo rm -rf /etc/mysql + # Get key to latest MySQL repo sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 - # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb - echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.14-1_all.deb + # Bionic packages are still compatible for Focal since there's no MySQL 5.7 + # packages for Focal. + echo mysql-apt-config mysql-apt-config/repo-codename select bionic | sudo debconf-set-selections + echo mysql-apt-config mysql-apt-config/select-server select mysql-5.7 | sudo debconf-set-selections sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update + sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y mysql-client=5.7* mysql-community-server=5.7* mysql-server=5.7* # Install everything else we need, and configure - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get install -y make unzip g++ etcd curl git wget eatmydata sudo service mysql stop sudo service etcd stop - sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ - sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld - go mod download # install JUnit report formatter go install github.com/vitessio/go-junit-report@HEAD - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. diff --git a/.github/workflows/cluster_endtoend_resharding.yml b/.github/workflows/cluster_endtoend_resharding.yml index 003d3afbd19..73d1f214b9b 100644 --- a/.github/workflows/cluster_endtoend_resharding.yml +++ b/.github/workflows/cluster_endtoend_resharding.yml @@ -9,7 +9,7 @@ concurrency: jobs: build: name: Run endtoend tests on Cluster (resharding) - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - name: Check if workflow needs to be skipped @@ -59,6 +59,8 @@ jobs: - name: Tune the OS if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf @@ -67,10 +69,22 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + # Setup MySQL 8.0 + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata - sudo service mysql stop + # Install everything else we need, and configure + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get update + sudo service etcd stop + echo 'mysql version: ' + sudo mysql --version + sudo service mysql stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download @@ -80,7 +94,7 @@ jobs: - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. diff --git a/.github/workflows/cluster_endtoend_resharding_bytes.yml b/.github/workflows/cluster_endtoend_resharding_bytes.yml index ad6d19948a0..a7e9934f930 100644 --- a/.github/workflows/cluster_endtoend_resharding_bytes.yml +++ b/.github/workflows/cluster_endtoend_resharding_bytes.yml @@ -9,7 +9,7 @@ concurrency: jobs: build: name: Run endtoend tests on Cluster (resharding_bytes) - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - name: Check if workflow needs to be skipped @@ -59,6 +59,8 @@ jobs: - name: Tune the OS if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf @@ -67,10 +69,22 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + # Setup MySQL 8.0 + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata - sudo service mysql stop + # Install everything else we need, and configure + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get update + sudo service etcd stop + echo 'mysql version: ' + sudo mysql --version + sudo service mysql stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download @@ -80,7 +94,7 @@ jobs: - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. diff --git a/.github/workflows/cluster_endtoend_schemadiff_vrepl.yml b/.github/workflows/cluster_endtoend_schemadiff_vrepl.yml index 39f8669b6ed..493f319cb61 100644 --- a/.github/workflows/cluster_endtoend_schemadiff_vrepl.yml +++ b/.github/workflows/cluster_endtoend_schemadiff_vrepl.yml @@ -9,7 +9,7 @@ concurrency: jobs: build: name: Run endtoend tests on Cluster (schemadiff_vrepl) - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - name: Check if workflow needs to be skipped @@ -59,6 +59,8 @@ jobs: - name: Tune the OS if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf @@ -67,10 +69,22 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + # Setup MySQL 8.0 + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata - sudo service mysql stop + # Install everything else we need, and configure + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get update + sudo service etcd stop + echo 'mysql version: ' + sudo mysql --version + sudo service mysql stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download @@ -80,7 +94,7 @@ jobs: - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. diff --git a/.github/workflows/cluster_endtoend_schemadiff_vrepl_mysql80.yml b/.github/workflows/cluster_endtoend_schemadiff_vrepl_mysql57.yml similarity index 77% rename from .github/workflows/cluster_endtoend_schemadiff_vrepl_mysql80.yml rename to .github/workflows/cluster_endtoend_schemadiff_vrepl_mysql57.yml index b6101f6510f..cbf62731051 100644 --- a/.github/workflows/cluster_endtoend_schemadiff_vrepl_mysql80.yml +++ b/.github/workflows/cluster_endtoend_schemadiff_vrepl_mysql57.yml @@ -1,15 +1,16 @@ # DO NOT MODIFY: THIS FILE IS GENERATED USING "make generate_ci_workflows" -name: Cluster (schemadiff_vrepl) mysql80 +name: Cluster (schemadiff_vrepl) mysql57 on: [push, pull_request] concurrency: - group: format('{0}-{1}', ${{ github.ref }}, 'Cluster (schemadiff_vrepl) mysql80') + group: format('{0}-{1}', ${{ github.ref }}, 'Cluster (schemadiff_vrepl) mysql57') cancel-in-progress: true jobs: build: - name: Run endtoend tests on Cluster (schemadiff_vrepl) mysql80 + name: Run endtoend tests on Cluster (schemadiff_vrepl) mysql57 runs-on: ubuntu-20.04 + timeout-minutes: 45 steps: - name: Check if workflow needs to be skipped @@ -44,7 +45,7 @@ jobs: - 'tools/**' - 'config/**' - 'bootstrap.sh' - - '.github/workflows/cluster_endtoend_schemadiff_vrepl_mysql80.yml' + - '.github/workflows/cluster_endtoend_schemadiff_vrepl_mysql57.yml' - name: Set up Go if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' @@ -67,29 +68,43 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + sudo apt-get update + + # Uninstall any previously installed MySQL first + sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ + sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + + sudo systemctl stop apparmor + sudo DEBIAN_FRONTEND="noninteractive" apt-get remove -y --purge mysql-server mysql-client mysql-common + sudo apt-get -y autoremove + sudo apt-get -y autoclean + sudo deluser mysql + sudo rm -rf /var/lib/mysql + sudo rm -rf /etc/mysql + # Get key to latest MySQL repo sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 - # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb - echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.14-1_all.deb + # Bionic packages are still compatible for Focal since there's no MySQL 5.7 + # packages for Focal. + echo mysql-apt-config mysql-apt-config/repo-codename select bionic | sudo debconf-set-selections + echo mysql-apt-config mysql-apt-config/select-server select mysql-5.7 | sudo debconf-set-selections sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update + sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y mysql-client=5.7* mysql-community-server=5.7* mysql-server=5.7* # Install everything else we need, and configure - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get install -y make unzip g++ etcd curl git wget eatmydata sudo service mysql stop sudo service etcd stop - sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ - sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld - go mod download # install JUnit report formatter go install github.com/vitessio/go-junit-report@HEAD - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. diff --git a/.github/workflows/cluster_endtoend_shardedrecovery_stress_verticalsplit_heavy.yml b/.github/workflows/cluster_endtoend_shardedrecovery_stress_verticalsplit_heavy.yml index cab0296a653..7ee32da49aa 100644 --- a/.github/workflows/cluster_endtoend_shardedrecovery_stress_verticalsplit_heavy.yml +++ b/.github/workflows/cluster_endtoend_shardedrecovery_stress_verticalsplit_heavy.yml @@ -9,7 +9,7 @@ concurrency: jobs: build: name: Run endtoend tests on Cluster (shardedrecovery_stress_verticalsplit_heavy) - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - name: Check if workflow needs to be skipped @@ -59,6 +59,8 @@ jobs: - name: Tune the OS if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf @@ -67,10 +69,22 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + # Setup MySQL 8.0 + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata - sudo service mysql stop + # Install everything else we need, and configure + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get update + sudo service etcd stop + echo 'mysql version: ' + sudo mysql --version + sudo service mysql stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download @@ -80,7 +94,7 @@ jobs: - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. @@ -93,10 +107,11 @@ jobs: sudo sysctl -w net.ipv4.ip_local_port_range="22768 61999" # Increase our open file descriptor limit as we could hit this ulimit -n 65536 - cat <<-EOF>>./config/mycnf/mysql57.cnf + cat <<-EOF>>./config/mycnf/mysql80.cnf innodb_buffer_pool_dump_at_shutdown=OFF + innodb_buffer_pool_in_core_file=OFF innodb_buffer_pool_load_at_startup=OFF - innodb_buffer_pool_size=64M + innodb_buffer_pool_size=16M innodb_doublewrite=OFF innodb_flush_log_at_trx_commit=0 innodb_flush_method=O_DIRECT @@ -107,6 +122,7 @@ jobs: performance_schema=OFF slow-query-log=OFF EOF + make embed_config # run the tests however you normally do, then produce a JUnit XML file eatmydata -- go run test.go -docker=false -follow -shard shardedrecovery_stress_verticalsplit_heavy diff --git a/.github/workflows/cluster_endtoend_tabletmanager_consul.yml b/.github/workflows/cluster_endtoend_tabletmanager_consul.yml index 283eba2ceb2..8d27eeec8a0 100644 --- a/.github/workflows/cluster_endtoend_tabletmanager_consul.yml +++ b/.github/workflows/cluster_endtoend_tabletmanager_consul.yml @@ -9,7 +9,7 @@ concurrency: jobs: build: name: Run endtoend tests on Cluster (tabletmanager_consul) - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - name: Check if workflow needs to be skipped @@ -59,6 +59,8 @@ jobs: - name: Tune the OS if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf @@ -67,10 +69,22 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + # Setup MySQL 8.0 + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata - sudo service mysql stop + # Install everything else we need, and configure + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get update + sudo service etcd stop + echo 'mysql version: ' + sudo mysql --version + sudo service mysql stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download @@ -85,7 +99,7 @@ jobs: - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. diff --git a/.github/workflows/cluster_endtoend_tabletmanager_tablegc.yml b/.github/workflows/cluster_endtoend_tabletmanager_tablegc.yml index 341e4bc3df1..e3ecedf8c7f 100644 --- a/.github/workflows/cluster_endtoend_tabletmanager_tablegc.yml +++ b/.github/workflows/cluster_endtoend_tabletmanager_tablegc.yml @@ -9,7 +9,7 @@ concurrency: jobs: build: name: Run endtoend tests on Cluster (tabletmanager_tablegc) - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - name: Check if workflow needs to be skipped @@ -59,6 +59,8 @@ jobs: - name: Tune the OS if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf @@ -67,10 +69,22 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + # Setup MySQL 8.0 + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata - sudo service mysql stop + # Install everything else we need, and configure + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get update + sudo service etcd stop + echo 'mysql version: ' + sudo mysql --version + sudo service mysql stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download @@ -80,7 +94,7 @@ jobs: - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. diff --git a/.github/workflows/cluster_endtoend_tabletmanager_tablegc_mysql80.yml b/.github/workflows/cluster_endtoend_tabletmanager_tablegc_mysql57.yml similarity index 77% rename from .github/workflows/cluster_endtoend_tabletmanager_tablegc_mysql80.yml rename to .github/workflows/cluster_endtoend_tabletmanager_tablegc_mysql57.yml index 3c756410546..50337c243c8 100644 --- a/.github/workflows/cluster_endtoend_tabletmanager_tablegc_mysql80.yml +++ b/.github/workflows/cluster_endtoend_tabletmanager_tablegc_mysql57.yml @@ -1,15 +1,16 @@ # DO NOT MODIFY: THIS FILE IS GENERATED USING "make generate_ci_workflows" -name: Cluster (tabletmanager_tablegc) mysql80 +name: Cluster (tabletmanager_tablegc) mysql57 on: [push, pull_request] concurrency: - group: format('{0}-{1}', ${{ github.ref }}, 'Cluster (tabletmanager_tablegc) mysql80') + group: format('{0}-{1}', ${{ github.ref }}, 'Cluster (tabletmanager_tablegc) mysql57') cancel-in-progress: true jobs: build: - name: Run endtoend tests on Cluster (tabletmanager_tablegc) mysql80 + name: Run endtoend tests on Cluster (tabletmanager_tablegc) mysql57 runs-on: ubuntu-20.04 + timeout-minutes: 45 steps: - name: Check if workflow needs to be skipped @@ -44,7 +45,7 @@ jobs: - 'tools/**' - 'config/**' - 'bootstrap.sh' - - '.github/workflows/cluster_endtoend_tabletmanager_tablegc_mysql80.yml' + - '.github/workflows/cluster_endtoend_tabletmanager_tablegc_mysql57.yml' - name: Set up Go if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' @@ -67,29 +68,43 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + sudo apt-get update + + # Uninstall any previously installed MySQL first + sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ + sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + + sudo systemctl stop apparmor + sudo DEBIAN_FRONTEND="noninteractive" apt-get remove -y --purge mysql-server mysql-client mysql-common + sudo apt-get -y autoremove + sudo apt-get -y autoclean + sudo deluser mysql + sudo rm -rf /var/lib/mysql + sudo rm -rf /etc/mysql + # Get key to latest MySQL repo sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 - # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb - echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.14-1_all.deb + # Bionic packages are still compatible for Focal since there's no MySQL 5.7 + # packages for Focal. + echo mysql-apt-config mysql-apt-config/repo-codename select bionic | sudo debconf-set-selections + echo mysql-apt-config mysql-apt-config/select-server select mysql-5.7 | sudo debconf-set-selections sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update + sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y mysql-client=5.7* mysql-community-server=5.7* mysql-server=5.7* # Install everything else we need, and configure - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get install -y make unzip g++ etcd curl git wget eatmydata sudo service mysql stop sudo service etcd stop - sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ - sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld - go mod download # install JUnit report formatter go install github.com/vitessio/go-junit-report@HEAD - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. diff --git a/.github/workflows/cluster_endtoend_tabletmanager_throttler.yml b/.github/workflows/cluster_endtoend_tabletmanager_throttler.yml index ee4377e811c..3dce3c93745 100644 --- a/.github/workflows/cluster_endtoend_tabletmanager_throttler.yml +++ b/.github/workflows/cluster_endtoend_tabletmanager_throttler.yml @@ -9,7 +9,7 @@ concurrency: jobs: build: name: Run endtoend tests on Cluster (tabletmanager_throttler) - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - name: Check if workflow needs to be skipped @@ -59,6 +59,8 @@ jobs: - name: Tune the OS if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf @@ -67,10 +69,22 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + # Setup MySQL 8.0 + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata - sudo service mysql stop + # Install everything else we need, and configure + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get update + sudo service etcd stop + echo 'mysql version: ' + sudo mysql --version + sudo service mysql stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download @@ -80,7 +94,7 @@ jobs: - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. diff --git a/.github/workflows/cluster_endtoend_tabletmanager_throttler_custom_config.yml b/.github/workflows/cluster_endtoend_tabletmanager_throttler_custom_config.yml index 487a59cf073..901843d2416 100644 --- a/.github/workflows/cluster_endtoend_tabletmanager_throttler_custom_config.yml +++ b/.github/workflows/cluster_endtoend_tabletmanager_throttler_custom_config.yml @@ -9,7 +9,7 @@ concurrency: jobs: build: name: Run endtoend tests on Cluster (tabletmanager_throttler_custom_config) - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - name: Check if workflow needs to be skipped @@ -59,6 +59,8 @@ jobs: - name: Tune the OS if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf @@ -67,10 +69,22 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + # Setup MySQL 8.0 + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata - sudo service mysql stop + # Install everything else we need, and configure + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get update + sudo service etcd stop + echo 'mysql version: ' + sudo mysql --version + sudo service mysql stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download @@ -80,7 +94,7 @@ jobs: - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. diff --git a/.github/workflows/cluster_endtoend_topo_connection_cache.yml b/.github/workflows/cluster_endtoend_topo_connection_cache.yml index ce2b8ffc693..de097b16326 100644 --- a/.github/workflows/cluster_endtoend_topo_connection_cache.yml +++ b/.github/workflows/cluster_endtoend_topo_connection_cache.yml @@ -9,7 +9,7 @@ concurrency: jobs: build: name: Run endtoend tests on Cluster (topo_connection_cache) - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - name: Check if workflow needs to be skipped @@ -59,6 +59,8 @@ jobs: - name: Tune the OS if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf @@ -67,10 +69,22 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + # Setup MySQL 8.0 + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata - sudo service mysql stop + # Install everything else we need, and configure + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get update + sudo service etcd stop + echo 'mysql version: ' + sudo mysql --version + sudo service mysql stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download @@ -80,7 +94,7 @@ jobs: - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. diff --git a/.github/workflows/cluster_endtoend_vreplication_across_db_versions.yml b/.github/workflows/cluster_endtoend_vreplication_across_db_versions.yml index ff34a6f1b03..5d418533bf3 100644 --- a/.github/workflows/cluster_endtoend_vreplication_across_db_versions.yml +++ b/.github/workflows/cluster_endtoend_vreplication_across_db_versions.yml @@ -59,6 +59,8 @@ jobs: - name: Tune the OS if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf @@ -67,19 +69,22 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Get key to latest MySQL repo sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 - # Setup MySQL 8.0 wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update - # Install everything else we need, and configure sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils - sudo service mysql stop + sudo apt-get update + sudo service etcd stop + echo 'mysql version: ' + sudo mysql --version + sudo service mysql stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download @@ -89,7 +94,7 @@ jobs: - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. @@ -106,7 +111,7 @@ jobs: innodb_buffer_pool_dump_at_shutdown=OFF innodb_buffer_pool_in_core_file=OFF innodb_buffer_pool_load_at_startup=OFF - innodb_buffer_pool_size=64M + innodb_buffer_pool_size=16M innodb_doublewrite=OFF innodb_flush_log_at_trx_commit=0 innodb_flush_method=O_DIRECT @@ -117,6 +122,7 @@ jobs: performance_schema=OFF slow-query-log=OFF EOF + make embed_config # run the tests however you normally do, then produce a JUnit XML file eatmydata -- go run test.go -docker=false -follow -shard vreplication_across_db_versions diff --git a/.github/workflows/cluster_endtoend_vreplication_basic.yml b/.github/workflows/cluster_endtoend_vreplication_basic.yml index ac56390c03b..50be8f8fa36 100644 --- a/.github/workflows/cluster_endtoend_vreplication_basic.yml +++ b/.github/workflows/cluster_endtoend_vreplication_basic.yml @@ -9,7 +9,7 @@ concurrency: jobs: build: name: Run endtoend tests on Cluster (vreplication_basic) - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - name: Check if workflow needs to be skipped @@ -59,6 +59,8 @@ jobs: - name: Tune the OS if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf @@ -67,10 +69,22 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + # Setup MySQL 8.0 + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata - sudo service mysql stop + # Install everything else we need, and configure + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get update + sudo service etcd stop + echo 'mysql version: ' + sudo mysql --version + sudo service mysql stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download @@ -80,7 +94,7 @@ jobs: - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. @@ -93,10 +107,11 @@ jobs: sudo sysctl -w net.ipv4.ip_local_port_range="22768 61999" # Increase our open file descriptor limit as we could hit this ulimit -n 65536 - cat <<-EOF>>./config/mycnf/mysql57.cnf + cat <<-EOF>>./config/mycnf/mysql80.cnf innodb_buffer_pool_dump_at_shutdown=OFF + innodb_buffer_pool_in_core_file=OFF innodb_buffer_pool_load_at_startup=OFF - innodb_buffer_pool_size=64M + innodb_buffer_pool_size=16M innodb_doublewrite=OFF innodb_flush_log_at_trx_commit=0 innodb_flush_method=O_DIRECT @@ -107,6 +122,7 @@ jobs: performance_schema=OFF slow-query-log=OFF EOF + make embed_config # run the tests however you normally do, then produce a JUnit XML file eatmydata -- go run test.go -docker=false -follow -shard vreplication_basic diff --git a/.github/workflows/cluster_endtoend_vreplication_cellalias.yml b/.github/workflows/cluster_endtoend_vreplication_cellalias.yml index 56e9b585c71..c780684644c 100644 --- a/.github/workflows/cluster_endtoend_vreplication_cellalias.yml +++ b/.github/workflows/cluster_endtoend_vreplication_cellalias.yml @@ -9,7 +9,7 @@ concurrency: jobs: build: name: Run endtoend tests on Cluster (vreplication_cellalias) - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - name: Check if workflow needs to be skipped @@ -59,6 +59,8 @@ jobs: - name: Tune the OS if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf @@ -67,10 +69,22 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + # Setup MySQL 8.0 + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata - sudo service mysql stop + # Install everything else we need, and configure + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get update + sudo service etcd stop + echo 'mysql version: ' + sudo mysql --version + sudo service mysql stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download @@ -80,7 +94,7 @@ jobs: - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. @@ -93,10 +107,11 @@ jobs: sudo sysctl -w net.ipv4.ip_local_port_range="22768 61999" # Increase our open file descriptor limit as we could hit this ulimit -n 65536 - cat <<-EOF>>./config/mycnf/mysql57.cnf + cat <<-EOF>>./config/mycnf/mysql80.cnf innodb_buffer_pool_dump_at_shutdown=OFF + innodb_buffer_pool_in_core_file=OFF innodb_buffer_pool_load_at_startup=OFF - innodb_buffer_pool_size=64M + innodb_buffer_pool_size=16M innodb_doublewrite=OFF innodb_flush_log_at_trx_commit=0 innodb_flush_method=O_DIRECT @@ -107,6 +122,7 @@ jobs: performance_schema=OFF slow-query-log=OFF EOF + make embed_config # run the tests however you normally do, then produce a JUnit XML file eatmydata -- go run test.go -docker=false -follow -shard vreplication_cellalias diff --git a/.github/workflows/cluster_endtoend_vreplication_migrate.yml b/.github/workflows/cluster_endtoend_vreplication_migrate.yml index 871caa461ac..5537e802c1e 100644 --- a/.github/workflows/cluster_endtoend_vreplication_migrate.yml +++ b/.github/workflows/cluster_endtoend_vreplication_migrate.yml @@ -9,7 +9,7 @@ concurrency: jobs: build: name: Run endtoend tests on Cluster (vreplication_migrate) - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - name: Check if workflow needs to be skipped @@ -59,6 +59,8 @@ jobs: - name: Tune the OS if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf @@ -67,10 +69,22 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + # Setup MySQL 8.0 + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata - sudo service mysql stop + # Install everything else we need, and configure + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get update + sudo service etcd stop + echo 'mysql version: ' + sudo mysql --version + sudo service mysql stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download @@ -80,7 +94,7 @@ jobs: - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. @@ -93,10 +107,11 @@ jobs: sudo sysctl -w net.ipv4.ip_local_port_range="22768 61999" # Increase our open file descriptor limit as we could hit this ulimit -n 65536 - cat <<-EOF>>./config/mycnf/mysql57.cnf + cat <<-EOF>>./config/mycnf/mysql80.cnf innodb_buffer_pool_dump_at_shutdown=OFF + innodb_buffer_pool_in_core_file=OFF innodb_buffer_pool_load_at_startup=OFF - innodb_buffer_pool_size=64M + innodb_buffer_pool_size=16M innodb_doublewrite=OFF innodb_flush_log_at_trx_commit=0 innodb_flush_method=O_DIRECT @@ -107,6 +122,7 @@ jobs: performance_schema=OFF slow-query-log=OFF EOF + make embed_config # run the tests however you normally do, then produce a JUnit XML file eatmydata -- go run test.go -docker=false -follow -shard vreplication_migrate diff --git a/.github/workflows/cluster_endtoend_vreplication_migrate_vdiff2_convert_tz.yml b/.github/workflows/cluster_endtoend_vreplication_migrate_vdiff2_convert_tz.yml new file mode 100644 index 00000000000..6d5a08b2457 --- /dev/null +++ b/.github/workflows/cluster_endtoend_vreplication_migrate_vdiff2_convert_tz.yml @@ -0,0 +1,146 @@ +# DO NOT MODIFY: THIS FILE IS GENERATED USING "make generate_ci_workflows" + +name: Cluster (vreplication_migrate_vdiff2_convert_tz) +on: [push, pull_request] +concurrency: + group: format('{0}-{1}', ${{ github.ref }}, 'Cluster (vreplication_migrate_vdiff2_convert_tz)') + cancel-in-progress: true + +env: + LAUNCHABLE_ORGANIZATION: "vitess" + LAUNCHABLE_WORKSPACE: "vitess-app" + GITHUB_PR_HEAD_SHA: "${{ github.event.pull_request.head.sha }}" + +jobs: + build: + name: Run endtoend tests on Cluster (vreplication_migrate_vdiff2_convert_tz) + runs-on: ubuntu-20.04 + + steps: + - name: Check if workflow needs to be skipped + id: skip-workflow + run: | + skip='false' + if [[ "${{github.event.pull_request}}" == "" ]] && [[ "${{github.ref}}" != "refs/heads/main" ]] && [[ ! "${{github.ref}}" =~ ^refs/heads/release-[0-9]+\.[0-9]$ ]] && [[ ! "${{github.ref}}" =~ "refs/tags/.*" ]]; then + skip='true' + fi + echo Skip ${skip} + echo "::set-output name=skip-workflow::${skip}" + + - name: Check out code + if: steps.skip-workflow.outputs.skip-workflow == 'false' + uses: actions/checkout@v2 + + - name: Check for changes in relevant files + if: steps.skip-workflow.outputs.skip-workflow == 'false' + uses: frouioui/paths-filter@main + id: changes + with: + token: '' + filters: | + end_to_end: + - 'go/**/*.go' + - 'test.go' + - 'Makefile' + - 'build.env' + - 'go.[sumod]' + - 'proto/*.proto' + - 'tools/**' + - 'config/**' + - 'bootstrap.sh' + - '.github/workflows/cluster_endtoend_vreplication_migrate_vdiff2_convert_tz.yml' + + - name: Set up Go + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/setup-go@v2 + with: + go-version: 1.19.3 + + - name: Set up python + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/setup-python@v2 + + - name: Tune the OS + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range + # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio + echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf + sudo sysctl -p /etc/sysctl.conf + + - name: Get dependencies + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + # Setup Percona Server for MySQL 8.0 + sudo apt-get update + sudo apt-get install -y lsb-release gnupg2 curl + wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb + sudo percona-release setup ps80 + sudo apt-get update + + # Install everything else we need, and configure + sudo apt-get install -y percona-server-server percona-server-client make unzip g++ etcd git wget eatmydata xz-utils + sudo service mysql stop + sudo service etcd stop + sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ + sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + go mod download + + # install JUnit report formatter + go install github.com/vitessio/go-junit-report@HEAD + + - name: Setup launchable dependencies + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + # Get Launchable CLI installed. If you can, make it a part of the builder image to speed things up + pip3 install --user launchable~=1.0 > /dev/null + + # verify that launchable setup is all correct. + launchable verify || true + + # Tell Launchable about the build you are producing and testing + launchable record build --name "$GITHUB_RUN_ID" --source . + + - name: Run cluster endtoend test + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 45 + run: | + # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file + # which musn't be more than 107 characters long. + export VTDATAROOT="/tmp/" + source build.env + + set -x + + # Increase our local ephemeral port range as we could exhaust this + sudo sysctl -w net.ipv4.ip_local_port_range="22768 61999" + # Increase our open file descriptor limit as we could hit this + ulimit -n 65536 + cat <<-EOF>>./config/mycnf/mysql80.cnf + innodb_buffer_pool_dump_at_shutdown=OFF + innodb_buffer_pool_in_core_file=OFF + innodb_buffer_pool_load_at_startup=OFF + innodb_buffer_pool_size=64M + innodb_doublewrite=OFF + innodb_flush_log_at_trx_commit=0 + innodb_flush_method=O_DIRECT + innodb_numa_interleave=ON + innodb_adaptive_hash_index=OFF + sync_binlog=0 + sync_relay_log=0 + performance_schema=OFF + slow-query-log=OFF + EOF + + # run the tests however you normally do, then produce a JUnit XML file + eatmydata -- go run test.go -docker=false -follow -shard vreplication_migrate_vdiff2_convert_tz | tee -a output.txt | go-junit-report -set-exit-code > report.xml + + - name: Print test output and Record test result in launchable + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + run: | + # send recorded tests to launchable + launchable record tests --build "$GITHUB_RUN_ID" go-test . || true + + # print test output + cat output.txt diff --git a/.github/workflows/cluster_endtoend_vreplication_multicell.yml b/.github/workflows/cluster_endtoend_vreplication_multicell.yml index c38f084dbe1..7e8c33ee621 100644 --- a/.github/workflows/cluster_endtoend_vreplication_multicell.yml +++ b/.github/workflows/cluster_endtoend_vreplication_multicell.yml @@ -9,7 +9,7 @@ concurrency: jobs: build: name: Run endtoend tests on Cluster (vreplication_multicell) - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - name: Check if workflow needs to be skipped @@ -59,6 +59,8 @@ jobs: - name: Tune the OS if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf @@ -67,10 +69,22 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + # Setup MySQL 8.0 + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata - sudo service mysql stop + # Install everything else we need, and configure + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get update + sudo service etcd stop + echo 'mysql version: ' + sudo mysql --version + sudo service mysql stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download @@ -80,7 +94,7 @@ jobs: - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. @@ -93,10 +107,11 @@ jobs: sudo sysctl -w net.ipv4.ip_local_port_range="22768 61999" # Increase our open file descriptor limit as we could hit this ulimit -n 65536 - cat <<-EOF>>./config/mycnf/mysql57.cnf + cat <<-EOF>>./config/mycnf/mysql80.cnf innodb_buffer_pool_dump_at_shutdown=OFF + innodb_buffer_pool_in_core_file=OFF innodb_buffer_pool_load_at_startup=OFF - innodb_buffer_pool_size=64M + innodb_buffer_pool_size=16M innodb_doublewrite=OFF innodb_flush_log_at_trx_commit=0 innodb_flush_method=O_DIRECT @@ -107,6 +122,7 @@ jobs: performance_schema=OFF slow-query-log=OFF EOF + make embed_config # run the tests however you normally do, then produce a JUnit XML file eatmydata -- go run test.go -docker=false -follow -shard vreplication_multicell diff --git a/.github/workflows/cluster_endtoend_vreplication_v2.yml b/.github/workflows/cluster_endtoend_vreplication_v2.yml index 398e4f23d77..e30f49896d1 100644 --- a/.github/workflows/cluster_endtoend_vreplication_v2.yml +++ b/.github/workflows/cluster_endtoend_vreplication_v2.yml @@ -9,7 +9,7 @@ concurrency: jobs: build: name: Run endtoend tests on Cluster (vreplication_v2) - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - name: Check if workflow needs to be skipped @@ -59,6 +59,8 @@ jobs: - name: Tune the OS if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf @@ -67,10 +69,22 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + # Setup MySQL 8.0 + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata - sudo service mysql stop + # Install everything else we need, and configure + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get update + sudo service etcd stop + echo 'mysql version: ' + sudo mysql --version + sudo service mysql stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download @@ -80,7 +94,7 @@ jobs: - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. @@ -93,10 +107,11 @@ jobs: sudo sysctl -w net.ipv4.ip_local_port_range="22768 61999" # Increase our open file descriptor limit as we could hit this ulimit -n 65536 - cat <<-EOF>>./config/mycnf/mysql57.cnf + cat <<-EOF>>./config/mycnf/mysql80.cnf innodb_buffer_pool_dump_at_shutdown=OFF + innodb_buffer_pool_in_core_file=OFF innodb_buffer_pool_load_at_startup=OFF - innodb_buffer_pool_size=64M + innodb_buffer_pool_size=16M innodb_doublewrite=OFF innodb_flush_log_at_trx_commit=0 innodb_flush_method=O_DIRECT @@ -107,6 +122,7 @@ jobs: performance_schema=OFF slow-query-log=OFF EOF + make embed_config # run the tests however you normally do, then produce a JUnit XML file eatmydata -- go run test.go -docker=false -follow -shard vreplication_v2 diff --git a/.github/workflows/cluster_endtoend_vstream_failover.yml b/.github/workflows/cluster_endtoend_vstream_failover.yml index ae8c35816d9..19f9dcf7a34 100644 --- a/.github/workflows/cluster_endtoend_vstream_failover.yml +++ b/.github/workflows/cluster_endtoend_vstream_failover.yml @@ -9,7 +9,7 @@ concurrency: jobs: build: name: Run endtoend tests on Cluster (vstream_failover) - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - name: Check if workflow needs to be skipped @@ -59,6 +59,8 @@ jobs: - name: Tune the OS if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf @@ -67,10 +69,22 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + # Setup MySQL 8.0 + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata - sudo service mysql stop + # Install everything else we need, and configure + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get update + sudo service etcd stop + echo 'mysql version: ' + sudo mysql --version + sudo service mysql stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download @@ -80,7 +94,7 @@ jobs: - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. diff --git a/.github/workflows/cluster_endtoend_vstream_stoponreshard_false.yml b/.github/workflows/cluster_endtoend_vstream_stoponreshard_false.yml index 09c64a014a9..920cbc063d3 100644 --- a/.github/workflows/cluster_endtoend_vstream_stoponreshard_false.yml +++ b/.github/workflows/cluster_endtoend_vstream_stoponreshard_false.yml @@ -9,7 +9,7 @@ concurrency: jobs: build: name: Run endtoend tests on Cluster (vstream_stoponreshard_false) - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - name: Check if workflow needs to be skipped @@ -59,6 +59,8 @@ jobs: - name: Tune the OS if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf @@ -67,10 +69,22 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + # Setup MySQL 8.0 + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata - sudo service mysql stop + # Install everything else we need, and configure + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get update + sudo service etcd stop + echo 'mysql version: ' + sudo mysql --version + sudo service mysql stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download @@ -80,7 +94,7 @@ jobs: - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. diff --git a/.github/workflows/cluster_endtoend_vstream_stoponreshard_true.yml b/.github/workflows/cluster_endtoend_vstream_stoponreshard_true.yml index d2783576595..4018ff284bd 100644 --- a/.github/workflows/cluster_endtoend_vstream_stoponreshard_true.yml +++ b/.github/workflows/cluster_endtoend_vstream_stoponreshard_true.yml @@ -9,7 +9,7 @@ concurrency: jobs: build: name: Run endtoend tests on Cluster (vstream_stoponreshard_true) - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - name: Check if workflow needs to be skipped @@ -59,6 +59,8 @@ jobs: - name: Tune the OS if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf @@ -67,10 +69,22 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + # Setup MySQL 8.0 + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata - sudo service mysql stop + # Install everything else we need, and configure + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get update + sudo service etcd stop + echo 'mysql version: ' + sudo mysql --version + sudo service mysql stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download @@ -80,7 +94,7 @@ jobs: - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. diff --git a/.github/workflows/cluster_endtoend_vstream_with_keyspaces_to_watch.yml b/.github/workflows/cluster_endtoend_vstream_with_keyspaces_to_watch.yml index f1fe80941d6..bca9e639c83 100644 --- a/.github/workflows/cluster_endtoend_vstream_with_keyspaces_to_watch.yml +++ b/.github/workflows/cluster_endtoend_vstream_with_keyspaces_to_watch.yml @@ -9,7 +9,7 @@ concurrency: jobs: build: name: Run endtoend tests on Cluster (vstream_with_keyspaces_to_watch) - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - name: Check if workflow needs to be skipped @@ -59,6 +59,8 @@ jobs: - name: Tune the OS if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf @@ -67,10 +69,22 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + # Setup MySQL 8.0 + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata - sudo service mysql stop + # Install everything else we need, and configure + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get update + sudo service etcd stop + echo 'mysql version: ' + sudo mysql --version + sudo service mysql stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download @@ -80,7 +94,7 @@ jobs: - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. diff --git a/.github/workflows/cluster_endtoend_vtbackup_transform.yml b/.github/workflows/cluster_endtoend_vtbackup_transform.yml new file mode 100644 index 00000000000..db8f001732a --- /dev/null +++ b/.github/workflows/cluster_endtoend_vtbackup_transform.yml @@ -0,0 +1,126 @@ +# DO NOT MODIFY: THIS FILE IS GENERATED USING "make generate_ci_workflows" + +name: Cluster (vtbackup_transform) +on: [push, pull_request] +concurrency: + group: format('{0}-{1}', ${{ github.ref }}, 'Cluster (vtbackup_transform)') + cancel-in-progress: true + +env: + LAUNCHABLE_ORGANIZATION: "vitess" + LAUNCHABLE_WORKSPACE: "vitess-app" + GITHUB_PR_HEAD_SHA: "${{ github.event.pull_request.head.sha }}" + +jobs: + build: + name: Run endtoend tests on Cluster (vtbackup_transform) + runs-on: ubuntu-20.04 + + steps: + - name: Check if workflow needs to be skipped + id: skip-workflow + run: | + skip='false' + if [[ "${{github.event.pull_request}}" == "" ]] && [[ "${{github.ref}}" != "refs/heads/main" ]] && [[ ! "${{github.ref}}" =~ ^refs/heads/release-[0-9]+\.[0-9]$ ]] && [[ ! "${{github.ref}}" =~ "refs/tags/.*" ]]; then + skip='true' + fi + echo Skip ${skip} + echo "::set-output name=skip-workflow::${skip}" + + - name: Check out code + if: steps.skip-workflow.outputs.skip-workflow == 'false' + uses: actions/checkout@v2 + + - name: Check for changes in relevant files + if: steps.skip-workflow.outputs.skip-workflow == 'false' + uses: frouioui/paths-filter@main + id: changes + with: + token: '' + filters: | + end_to_end: + - 'go/**/*.go' + - 'test.go' + - 'Makefile' + - 'build.env' + - 'go.[sumod]' + - 'proto/*.proto' + - 'tools/**' + - 'config/**' + - 'bootstrap.sh' + - '.github/workflows/cluster_endtoend_vtbackup_transform.yml' + + - name: Set up Go + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/setup-go@v2 + with: + go-version: 1.19.3 + + - name: Set up python + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/setup-python@v2 + + - name: Tune the OS + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range + # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio + echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf + sudo sysctl -p /etc/sysctl.conf + + - name: Get dependencies + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + # Setup Percona Server for MySQL 8.0 + sudo apt-get update + sudo apt-get install -y lsb-release gnupg2 curl + wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb + sudo percona-release setup ps80 + sudo apt-get update + + # Install everything else we need, and configure + sudo apt-get install -y percona-server-server percona-server-client make unzip g++ etcd git wget eatmydata xz-utils + sudo service mysql stop + sudo service etcd stop + sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ + sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + go mod download + + # install JUnit report formatter + go install github.com/vitessio/go-junit-report@HEAD + + - name: Setup launchable dependencies + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + # Get Launchable CLI installed. If you can, make it a part of the builder image to speed things up + pip3 install --user launchable~=1.0 > /dev/null + + # verify that launchable setup is all correct. + launchable verify || true + + # Tell Launchable about the build you are producing and testing + launchable record build --name "$GITHUB_RUN_ID" --source . + + - name: Run cluster endtoend test + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 45 + run: | + # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file + # which musn't be more than 107 characters long. + export VTDATAROOT="/tmp/" + source build.env + + set -x + + # run the tests however you normally do, then produce a JUnit XML file + eatmydata -- go run test.go -docker=false -follow -shard vtbackup_transform | tee -a output.txt | go-junit-report -set-exit-code > report.xml + + - name: Print test output and Record test result in launchable + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + run: | + # send recorded tests to launchable + launchable record tests --build "$GITHUB_RUN_ID" go-test . || true + + # print test output + cat output.txt diff --git a/.github/workflows/cluster_endtoend_vtctlbackup_sharded_clustertest_heavy.yml b/.github/workflows/cluster_endtoend_vtctlbackup_sharded_clustertest_heavy.yml index c9dfdfb01c6..db2754f7eb6 100644 --- a/.github/workflows/cluster_endtoend_vtctlbackup_sharded_clustertest_heavy.yml +++ b/.github/workflows/cluster_endtoend_vtctlbackup_sharded_clustertest_heavy.yml @@ -9,7 +9,7 @@ concurrency: jobs: build: name: Run endtoend tests on Cluster (vtctlbackup_sharded_clustertest_heavy) - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - name: Check if workflow needs to be skipped @@ -59,6 +59,8 @@ jobs: - name: Tune the OS if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf @@ -67,10 +69,22 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + # Setup MySQL 8.0 + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata - sudo service mysql stop + # Install everything else we need, and configure + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get update + sudo service etcd stop + echo 'mysql version: ' + sudo mysql --version + sudo service mysql stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download @@ -80,7 +94,7 @@ jobs: - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. @@ -93,10 +107,11 @@ jobs: sudo sysctl -w net.ipv4.ip_local_port_range="22768 61999" # Increase our open file descriptor limit as we could hit this ulimit -n 65536 - cat <<-EOF>>./config/mycnf/mysql57.cnf + cat <<-EOF>>./config/mycnf/mysql80.cnf innodb_buffer_pool_dump_at_shutdown=OFF + innodb_buffer_pool_in_core_file=OFF innodb_buffer_pool_load_at_startup=OFF - innodb_buffer_pool_size=64M + innodb_buffer_pool_size=16M innodb_doublewrite=OFF innodb_flush_log_at_trx_commit=0 innodb_flush_method=O_DIRECT @@ -107,6 +122,7 @@ jobs: performance_schema=OFF slow-query-log=OFF EOF + make embed_config # run the tests however you normally do, then produce a JUnit XML file eatmydata -- go run test.go -docker=false -follow -shard vtctlbackup_sharded_clustertest_heavy diff --git a/.github/workflows/cluster_endtoend_vtgate_concurrentdml.yml b/.github/workflows/cluster_endtoend_vtgate_concurrentdml.yml index 951526d6266..85a23db62b9 100644 --- a/.github/workflows/cluster_endtoend_vtgate_concurrentdml.yml +++ b/.github/workflows/cluster_endtoend_vtgate_concurrentdml.yml @@ -9,7 +9,7 @@ concurrency: jobs: build: name: Run endtoend tests on Cluster (vtgate_concurrentdml) - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - name: Check if workflow needs to be skipped @@ -59,6 +59,8 @@ jobs: - name: Tune the OS if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf @@ -67,10 +69,22 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + # Setup MySQL 8.0 + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata - sudo service mysql stop + # Install everything else we need, and configure + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get update + sudo service etcd stop + echo 'mysql version: ' + sudo mysql --version + sudo service mysql stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download @@ -80,7 +94,7 @@ jobs: - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. diff --git a/.github/workflows/cluster_endtoend_vtgate_gen4.yml b/.github/workflows/cluster_endtoend_vtgate_gen4.yml index 91d65db4eea..4b180c20964 100644 --- a/.github/workflows/cluster_endtoend_vtgate_gen4.yml +++ b/.github/workflows/cluster_endtoend_vtgate_gen4.yml @@ -9,7 +9,7 @@ concurrency: jobs: build: name: Run endtoend tests on Cluster (vtgate_gen4) - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - name: Check if workflow needs to be skipped @@ -59,6 +59,8 @@ jobs: - name: Tune the OS if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf @@ -67,10 +69,22 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + # Setup MySQL 8.0 + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata - sudo service mysql stop + # Install everything else we need, and configure + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get update + sudo service etcd stop + echo 'mysql version: ' + sudo mysql --version + sudo service mysql stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download @@ -80,7 +94,7 @@ jobs: - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. diff --git a/.github/workflows/cluster_endtoend_vtgate_general_heavy.yml b/.github/workflows/cluster_endtoend_vtgate_general_heavy.yml index ebbc3781054..6182bd869d2 100644 --- a/.github/workflows/cluster_endtoend_vtgate_general_heavy.yml +++ b/.github/workflows/cluster_endtoend_vtgate_general_heavy.yml @@ -9,7 +9,7 @@ concurrency: jobs: build: name: Run endtoend tests on Cluster (vtgate_general_heavy) - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - name: Check if workflow needs to be skipped @@ -59,6 +59,8 @@ jobs: - name: Tune the OS if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf @@ -67,10 +69,22 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + # Setup MySQL 8.0 + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata - sudo service mysql stop + # Install everything else we need, and configure + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get update + sudo service etcd stop + echo 'mysql version: ' + sudo mysql --version + sudo service mysql stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download @@ -80,7 +94,7 @@ jobs: - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. @@ -93,10 +107,11 @@ jobs: sudo sysctl -w net.ipv4.ip_local_port_range="22768 61999" # Increase our open file descriptor limit as we could hit this ulimit -n 65536 - cat <<-EOF>>./config/mycnf/mysql57.cnf + cat <<-EOF>>./config/mycnf/mysql80.cnf innodb_buffer_pool_dump_at_shutdown=OFF + innodb_buffer_pool_in_core_file=OFF innodb_buffer_pool_load_at_startup=OFF - innodb_buffer_pool_size=64M + innodb_buffer_pool_size=16M innodb_doublewrite=OFF innodb_flush_log_at_trx_commit=0 innodb_flush_method=O_DIRECT @@ -107,6 +122,7 @@ jobs: performance_schema=OFF slow-query-log=OFF EOF + make embed_config # run the tests however you normally do, then produce a JUnit XML file eatmydata -- go run test.go -docker=false -follow -shard vtgate_general_heavy diff --git a/.github/workflows/cluster_endtoend_vtgate_godriver.yml b/.github/workflows/cluster_endtoend_vtgate_godriver.yml index bf70cf3fa08..7d7ab5bb63f 100644 --- a/.github/workflows/cluster_endtoend_vtgate_godriver.yml +++ b/.github/workflows/cluster_endtoend_vtgate_godriver.yml @@ -9,7 +9,7 @@ concurrency: jobs: build: name: Run endtoend tests on Cluster (vtgate_godriver) - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - name: Check if workflow needs to be skipped @@ -59,6 +59,8 @@ jobs: - name: Tune the OS if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf @@ -67,10 +69,22 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + # Setup MySQL 8.0 + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata - sudo service mysql stop + # Install everything else we need, and configure + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get update + sudo service etcd stop + echo 'mysql version: ' + sudo mysql --version + sudo service mysql stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download @@ -80,7 +94,7 @@ jobs: - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. diff --git a/.github/workflows/cluster_endtoend_vtgate_partial_keyspace.yml b/.github/workflows/cluster_endtoend_vtgate_partial_keyspace.yml new file mode 100644 index 00000000000..153b8ff0330 --- /dev/null +++ b/.github/workflows/cluster_endtoend_vtgate_partial_keyspace.yml @@ -0,0 +1,126 @@ +# DO NOT MODIFY: THIS FILE IS GENERATED USING "make generate_ci_workflows" + +name: Cluster (vtgate_partial_keyspace) +on: [push, pull_request] +concurrency: + group: format('{0}-{1}', ${{ github.ref }}, 'Cluster (vtgate_partial_keyspace)') + cancel-in-progress: true + +env: + LAUNCHABLE_ORGANIZATION: "vitess" + LAUNCHABLE_WORKSPACE: "vitess-app" + GITHUB_PR_HEAD_SHA: "${{ github.event.pull_request.head.sha }}" + +jobs: + build: + name: Run endtoend tests on Cluster (vtgate_partial_keyspace) + runs-on: ubuntu-20.04 + + steps: + - name: Check if workflow needs to be skipped + id: skip-workflow + run: | + skip='false' + if [[ "${{github.event.pull_request}}" == "" ]] && [[ "${{github.ref}}" != "refs/heads/main" ]] && [[ ! "${{github.ref}}" =~ ^refs/heads/release-[0-9]+\.[0-9]$ ]] && [[ ! "${{github.ref}}" =~ "refs/tags/.*" ]]; then + skip='true' + fi + echo Skip ${skip} + echo "::set-output name=skip-workflow::${skip}" + + - name: Check out code + if: steps.skip-workflow.outputs.skip-workflow == 'false' + uses: actions/checkout@v2 + + - name: Check for changes in relevant files + if: steps.skip-workflow.outputs.skip-workflow == 'false' + uses: frouioui/paths-filter@main + id: changes + with: + token: '' + filters: | + end_to_end: + - 'go/**/*.go' + - 'test.go' + - 'Makefile' + - 'build.env' + - 'go.[sumod]' + - 'proto/*.proto' + - 'tools/**' + - 'config/**' + - 'bootstrap.sh' + - '.github/workflows/cluster_endtoend_vtgate_partial_keyspace.yml' + + - name: Set up Go + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/setup-go@v2 + with: + go-version: 1.19.3 + + - name: Set up python + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/setup-python@v2 + + - name: Tune the OS + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range + # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio + echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf + sudo sysctl -p /etc/sysctl.conf + + - name: Get dependencies + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + # Setup Percona Server for MySQL 8.0 + sudo apt-get update + sudo apt-get install -y lsb-release gnupg2 curl + wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb + sudo percona-release setup ps80 + sudo apt-get update + + # Install everything else we need, and configure + sudo apt-get install -y percona-server-server percona-server-client make unzip g++ etcd git wget eatmydata xz-utils + sudo service mysql stop + sudo service etcd stop + sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ + sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + go mod download + + # install JUnit report formatter + go install github.com/vitessio/go-junit-report@HEAD + + - name: Setup launchable dependencies + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + # Get Launchable CLI installed. If you can, make it a part of the builder image to speed things up + pip3 install --user launchable~=1.0 > /dev/null + + # verify that launchable setup is all correct. + launchable verify || true + + # Tell Launchable about the build you are producing and testing + launchable record build --name "$GITHUB_RUN_ID" --source . + + - name: Run cluster endtoend test + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 45 + run: | + # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file + # which musn't be more than 107 characters long. + export VTDATAROOT="/tmp/" + source build.env + + set -x + + # run the tests however you normally do, then produce a JUnit XML file + eatmydata -- go run test.go -docker=false -follow -shard vtgate_partial_keyspace -partial-keyspace=true | tee -a output.txt | go-junit-report -set-exit-code > report.xml + + - name: Print test output and Record test result in launchable + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + run: | + # send recorded tests to launchable + launchable record tests --build "$GITHUB_RUN_ID" go-test . || true + + # print test output + cat output.txt diff --git a/.github/workflows/cluster_endtoend_vtgate_queries.yml b/.github/workflows/cluster_endtoend_vtgate_queries.yml index 6a79e226003..9ba1daba13b 100644 --- a/.github/workflows/cluster_endtoend_vtgate_queries.yml +++ b/.github/workflows/cluster_endtoend_vtgate_queries.yml @@ -9,7 +9,7 @@ concurrency: jobs: build: name: Run endtoend tests on Cluster (vtgate_queries) - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - name: Check if workflow needs to be skipped @@ -59,6 +59,8 @@ jobs: - name: Tune the OS if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf @@ -67,10 +69,22 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + # Setup MySQL 8.0 + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata - sudo service mysql stop + # Install everything else we need, and configure + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get update + sudo service etcd stop + echo 'mysql version: ' + sudo mysql --version + sudo service mysql stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download @@ -80,7 +94,7 @@ jobs: - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. diff --git a/.github/workflows/cluster_endtoend_vtgate_readafterwrite.yml b/.github/workflows/cluster_endtoend_vtgate_readafterwrite.yml index 564e84d9130..b593bb20584 100644 --- a/.github/workflows/cluster_endtoend_vtgate_readafterwrite.yml +++ b/.github/workflows/cluster_endtoend_vtgate_readafterwrite.yml @@ -9,7 +9,7 @@ concurrency: jobs: build: name: Run endtoend tests on Cluster (vtgate_readafterwrite) - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - name: Check if workflow needs to be skipped @@ -59,6 +59,8 @@ jobs: - name: Tune the OS if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf @@ -67,10 +69,22 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + # Setup MySQL 8.0 + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata - sudo service mysql stop + # Install everything else we need, and configure + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get update + sudo service etcd stop + echo 'mysql version: ' + sudo mysql --version + sudo service mysql stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download @@ -80,7 +94,7 @@ jobs: - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. diff --git a/.github/workflows/cluster_endtoend_vtgate_reservedconn.yml b/.github/workflows/cluster_endtoend_vtgate_reservedconn.yml index 6dbec0b1cf2..35cef4e0f47 100644 --- a/.github/workflows/cluster_endtoend_vtgate_reservedconn.yml +++ b/.github/workflows/cluster_endtoend_vtgate_reservedconn.yml @@ -9,7 +9,7 @@ concurrency: jobs: build: name: Run endtoend tests on Cluster (vtgate_reservedconn) - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - name: Check if workflow needs to be skipped @@ -59,6 +59,8 @@ jobs: - name: Tune the OS if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf @@ -67,10 +69,22 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + # Setup MySQL 8.0 + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata - sudo service mysql stop + # Install everything else we need, and configure + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get update + sudo service etcd stop + echo 'mysql version: ' + sudo mysql --version + sudo service mysql stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download @@ -80,7 +94,7 @@ jobs: - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. diff --git a/.github/workflows/cluster_endtoend_vtgate_schema.yml b/.github/workflows/cluster_endtoend_vtgate_schema.yml index 6856e446b24..68643475489 100644 --- a/.github/workflows/cluster_endtoend_vtgate_schema.yml +++ b/.github/workflows/cluster_endtoend_vtgate_schema.yml @@ -9,7 +9,7 @@ concurrency: jobs: build: name: Run endtoend tests on Cluster (vtgate_schema) - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - name: Check if workflow needs to be skipped @@ -59,6 +59,8 @@ jobs: - name: Tune the OS if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf @@ -67,10 +69,22 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + # Setup MySQL 8.0 + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata - sudo service mysql stop + # Install everything else we need, and configure + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get update + sudo service etcd stop + echo 'mysql version: ' + sudo mysql --version + sudo service mysql stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download @@ -80,7 +94,7 @@ jobs: - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. diff --git a/.github/workflows/cluster_endtoend_vtgate_schema_tracker.yml b/.github/workflows/cluster_endtoend_vtgate_schema_tracker.yml index 3405a4f993d..7a2471ed763 100644 --- a/.github/workflows/cluster_endtoend_vtgate_schema_tracker.yml +++ b/.github/workflows/cluster_endtoend_vtgate_schema_tracker.yml @@ -9,7 +9,7 @@ concurrency: jobs: build: name: Run endtoend tests on Cluster (vtgate_schema_tracker) - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - name: Check if workflow needs to be skipped @@ -59,6 +59,8 @@ jobs: - name: Tune the OS if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf @@ -67,10 +69,22 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + # Setup MySQL 8.0 + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata - sudo service mysql stop + # Install everything else we need, and configure + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get update + sudo service etcd stop + echo 'mysql version: ' + sudo mysql --version + sudo service mysql stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download @@ -80,7 +94,7 @@ jobs: - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. diff --git a/.github/workflows/cluster_endtoend_vtgate_tablet_healthcheck_cache.yml b/.github/workflows/cluster_endtoend_vtgate_tablet_healthcheck_cache.yml index 17f64de524f..95b89589c20 100644 --- a/.github/workflows/cluster_endtoend_vtgate_tablet_healthcheck_cache.yml +++ b/.github/workflows/cluster_endtoend_vtgate_tablet_healthcheck_cache.yml @@ -9,7 +9,7 @@ concurrency: jobs: build: name: Run endtoend tests on Cluster (vtgate_tablet_healthcheck_cache) - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - name: Check if workflow needs to be skipped @@ -59,6 +59,8 @@ jobs: - name: Tune the OS if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf @@ -67,10 +69,22 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + # Setup MySQL 8.0 + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata - sudo service mysql stop + # Install everything else we need, and configure + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get update + sudo service etcd stop + echo 'mysql version: ' + sudo mysql --version + sudo service mysql stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download @@ -80,7 +94,7 @@ jobs: - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. diff --git a/.github/workflows/cluster_endtoend_vtgate_topo.yml b/.github/workflows/cluster_endtoend_vtgate_topo.yml index 455b4fbd6ab..00dee5a229c 100644 --- a/.github/workflows/cluster_endtoend_vtgate_topo.yml +++ b/.github/workflows/cluster_endtoend_vtgate_topo.yml @@ -9,7 +9,7 @@ concurrency: jobs: build: name: Run endtoend tests on Cluster (vtgate_topo) - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - name: Check if workflow needs to be skipped @@ -59,6 +59,8 @@ jobs: - name: Tune the OS if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf @@ -67,10 +69,22 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + # Setup MySQL 8.0 + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata - sudo service mysql stop + # Install everything else we need, and configure + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get update + sudo service etcd stop + echo 'mysql version: ' + sudo mysql --version + sudo service mysql stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download @@ -80,7 +94,7 @@ jobs: - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. diff --git a/.github/workflows/cluster_endtoend_vtgate_topo_consul.yml b/.github/workflows/cluster_endtoend_vtgate_topo_consul.yml index beb213b8df3..c4120c553eb 100644 --- a/.github/workflows/cluster_endtoend_vtgate_topo_consul.yml +++ b/.github/workflows/cluster_endtoend_vtgate_topo_consul.yml @@ -9,7 +9,7 @@ concurrency: jobs: build: name: Run endtoend tests on Cluster (vtgate_topo_consul) - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - name: Check if workflow needs to be skipped @@ -59,6 +59,8 @@ jobs: - name: Tune the OS if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf @@ -67,10 +69,22 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + # Setup MySQL 8.0 + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata - sudo service mysql stop + # Install everything else we need, and configure + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get update + sudo service etcd stop + echo 'mysql version: ' + sudo mysql --version + sudo service mysql stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download @@ -85,7 +99,7 @@ jobs: - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. diff --git a/.github/workflows/cluster_endtoend_vtgate_topo_etcd.yml b/.github/workflows/cluster_endtoend_vtgate_topo_etcd.yml index 16cc9e85756..3e505fe37d5 100644 --- a/.github/workflows/cluster_endtoend_vtgate_topo_etcd.yml +++ b/.github/workflows/cluster_endtoend_vtgate_topo_etcd.yml @@ -9,7 +9,7 @@ concurrency: jobs: build: name: Run endtoend tests on Cluster (vtgate_topo_etcd) - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - name: Check if workflow needs to be skipped @@ -59,6 +59,8 @@ jobs: - name: Tune the OS if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf @@ -67,10 +69,22 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + # Setup MySQL 8.0 + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata - sudo service mysql stop + # Install everything else we need, and configure + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get update + sudo service etcd stop + echo 'mysql version: ' + sudo mysql --version + sudo service mysql stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download @@ -80,7 +94,7 @@ jobs: - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. diff --git a/.github/workflows/cluster_endtoend_vtgate_transaction.yml b/.github/workflows/cluster_endtoend_vtgate_transaction.yml index e8b321d01e6..da75648ced8 100644 --- a/.github/workflows/cluster_endtoend_vtgate_transaction.yml +++ b/.github/workflows/cluster_endtoend_vtgate_transaction.yml @@ -9,7 +9,7 @@ concurrency: jobs: build: name: Run endtoend tests on Cluster (vtgate_transaction) - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - name: Check if workflow needs to be skipped @@ -59,6 +59,8 @@ jobs: - name: Tune the OS if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf @@ -67,10 +69,22 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + # Setup MySQL 8.0 + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata - sudo service mysql stop + # Install everything else we need, and configure + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get update + sudo service etcd stop + echo 'mysql version: ' + sudo mysql --version + sudo service mysql stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download @@ -80,7 +94,7 @@ jobs: - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. diff --git a/.github/workflows/cluster_endtoend_vtgate_unsharded.yml b/.github/workflows/cluster_endtoend_vtgate_unsharded.yml index 7e70cc12215..b7cea310587 100644 --- a/.github/workflows/cluster_endtoend_vtgate_unsharded.yml +++ b/.github/workflows/cluster_endtoend_vtgate_unsharded.yml @@ -9,7 +9,7 @@ concurrency: jobs: build: name: Run endtoend tests on Cluster (vtgate_unsharded) - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - name: Check if workflow needs to be skipped @@ -59,6 +59,8 @@ jobs: - name: Tune the OS if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf @@ -67,10 +69,22 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + # Setup MySQL 8.0 + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata - sudo service mysql stop + # Install everything else we need, and configure + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get update + sudo service etcd stop + echo 'mysql version: ' + sudo mysql --version + sudo service mysql stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download @@ -80,7 +94,7 @@ jobs: - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. diff --git a/.github/workflows/cluster_endtoend_vtgate_vindex_heavy.yml b/.github/workflows/cluster_endtoend_vtgate_vindex_heavy.yml index 81af9209d08..b9e642a7f9c 100644 --- a/.github/workflows/cluster_endtoend_vtgate_vindex_heavy.yml +++ b/.github/workflows/cluster_endtoend_vtgate_vindex_heavy.yml @@ -9,7 +9,7 @@ concurrency: jobs: build: name: Run endtoend tests on Cluster (vtgate_vindex_heavy) - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - name: Check if workflow needs to be skipped @@ -59,6 +59,8 @@ jobs: - name: Tune the OS if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf @@ -67,10 +69,22 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + # Setup MySQL 8.0 + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata - sudo service mysql stop + # Install everything else we need, and configure + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get update + sudo service etcd stop + echo 'mysql version: ' + sudo mysql --version + sudo service mysql stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download @@ -80,7 +94,7 @@ jobs: - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. @@ -93,10 +107,11 @@ jobs: sudo sysctl -w net.ipv4.ip_local_port_range="22768 61999" # Increase our open file descriptor limit as we could hit this ulimit -n 65536 - cat <<-EOF>>./config/mycnf/mysql57.cnf + cat <<-EOF>>./config/mycnf/mysql80.cnf innodb_buffer_pool_dump_at_shutdown=OFF + innodb_buffer_pool_in_core_file=OFF innodb_buffer_pool_load_at_startup=OFF - innodb_buffer_pool_size=64M + innodb_buffer_pool_size=16M innodb_doublewrite=OFF innodb_flush_log_at_trx_commit=0 innodb_flush_method=O_DIRECT @@ -107,6 +122,7 @@ jobs: performance_schema=OFF slow-query-log=OFF EOF + make embed_config # run the tests however you normally do, then produce a JUnit XML file eatmydata -- go run test.go -docker=false -follow -shard vtgate_vindex_heavy diff --git a/.github/workflows/cluster_endtoend_vtgate_vschema.yml b/.github/workflows/cluster_endtoend_vtgate_vschema.yml index 0653c05bf77..b4ad9aae717 100644 --- a/.github/workflows/cluster_endtoend_vtgate_vschema.yml +++ b/.github/workflows/cluster_endtoend_vtgate_vschema.yml @@ -9,7 +9,7 @@ concurrency: jobs: build: name: Run endtoend tests on Cluster (vtgate_vschema) - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - name: Check if workflow needs to be skipped @@ -59,6 +59,8 @@ jobs: - name: Tune the OS if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf @@ -67,10 +69,22 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + # Setup MySQL 8.0 + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata - sudo service mysql stop + # Install everything else we need, and configure + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get update + sudo service etcd stop + echo 'mysql version: ' + sudo mysql --version + sudo service mysql stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download @@ -80,7 +94,7 @@ jobs: - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. diff --git a/.github/workflows/cluster_endtoend_vtorc.yml b/.github/workflows/cluster_endtoend_vtorc.yml index 9c00b0713c2..45472eeb8ca 100644 --- a/.github/workflows/cluster_endtoend_vtorc.yml +++ b/.github/workflows/cluster_endtoend_vtorc.yml @@ -9,7 +9,7 @@ concurrency: jobs: build: name: Run endtoend tests on Cluster (vtorc) - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - name: Check if workflow needs to be skipped @@ -59,6 +59,8 @@ jobs: - name: Tune the OS if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf @@ -67,10 +69,22 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + # Setup MySQL 8.0 + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata - sudo service mysql stop + # Install everything else we need, and configure + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get update + sudo service etcd stop + echo 'mysql version: ' + sudo mysql --version + sudo service mysql stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download @@ -80,7 +94,7 @@ jobs: - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. diff --git a/.github/workflows/cluster_endtoend_vtorc_8.0.yml b/.github/workflows/cluster_endtoend_vtorc_8.0.yml index f1719c59d31..7d02851ed58 100644 --- a/.github/workflows/cluster_endtoend_vtorc_8.0.yml +++ b/.github/workflows/cluster_endtoend_vtorc_8.0.yml @@ -59,6 +59,8 @@ jobs: - name: Tune the OS if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf @@ -67,19 +69,22 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Get key to latest MySQL repo sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 - # Setup MySQL 8.0 wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update - # Install everything else we need, and configure sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils - sudo service mysql stop + sudo apt-get update + sudo service etcd stop + echo 'mysql version: ' + sudo mysql --version + sudo service mysql stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download @@ -89,7 +94,7 @@ jobs: - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. diff --git a/.github/workflows/cluster_endtoend_worker_vault_heavy.yml b/.github/workflows/cluster_endtoend_worker_vault_heavy.yml index f8a1a8ed364..d6e8641c67d 100644 --- a/.github/workflows/cluster_endtoend_worker_vault_heavy.yml +++ b/.github/workflows/cluster_endtoend_worker_vault_heavy.yml @@ -9,7 +9,7 @@ concurrency: jobs: build: name: Run endtoend tests on Cluster (worker_vault_heavy) - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - name: Check if workflow needs to be skipped @@ -59,6 +59,8 @@ jobs: - name: Tune the OS if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf @@ -67,10 +69,22 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + # Setup MySQL 8.0 + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata - sudo service mysql stop + # Install everything else we need, and configure + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get update + sudo service etcd stop + echo 'mysql version: ' + sudo mysql --version + sudo service mysql stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download @@ -80,7 +94,7 @@ jobs: - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. @@ -93,10 +107,11 @@ jobs: sudo sysctl -w net.ipv4.ip_local_port_range="22768 61999" # Increase our open file descriptor limit as we could hit this ulimit -n 65536 - cat <<-EOF>>./config/mycnf/mysql57.cnf + cat <<-EOF>>./config/mycnf/mysql80.cnf innodb_buffer_pool_dump_at_shutdown=OFF + innodb_buffer_pool_in_core_file=OFF innodb_buffer_pool_load_at_startup=OFF - innodb_buffer_pool_size=64M + innodb_buffer_pool_size=16M innodb_doublewrite=OFF innodb_flush_log_at_trx_commit=0 innodb_flush_method=O_DIRECT @@ -107,6 +122,7 @@ jobs: performance_schema=OFF slow-query-log=OFF EOF + make embed_config # run the tests however you normally do, then produce a JUnit XML file eatmydata -- go run test.go -docker=false -follow -shard worker_vault_heavy diff --git a/.github/workflows/cluster_endtoend_xb_backup.yml b/.github/workflows/cluster_endtoend_xb_backup.yml index 8a88ffbe12e..9a1a7f1a1cd 100644 --- a/.github/workflows/cluster_endtoend_xb_backup.yml +++ b/.github/workflows/cluster_endtoend_xb_backup.yml @@ -14,7 +14,7 @@ env: jobs: build: name: Run endtoend tests on Cluster (xb_backup) - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - name: Check if workflow needs to be skipped @@ -64,6 +64,8 @@ jobs: - name: Tune the OS if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf @@ -72,10 +74,22 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + + # Setup Percona Server for MySQL 8.0 sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata - sudo service mysql stop + sudo apt-get install -y lsb-release gnupg2 curl + wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb + sudo percona-release setup ps80 + sudo apt-get update + + # Install everything else we need, and configure + sudo apt-get install -y percona-server-server percona-server-client make unzip g++ etcd git wget eatmydata xz-utils + sudo service etcd stop + echo 'mysql version: ' + sudo mysql --version + sudo service mysql stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download @@ -83,21 +97,11 @@ jobs: # install JUnit report formatter go install github.com/vitessio/go-junit-report@HEAD - wget "https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb" - sudo apt-get install -y gnupg2 - sudo dpkg -i "percona-release_latest.$(lsb_release -sc)_all.deb" - sudo apt-get update - if [[ -n $XTRABACKUP_VERSION ]]; then - debfile="percona-xtrabackup-24_$XTRABACKUP_VERSION.$(lsb_release -sc)_amd64.deb" - wget "https://repo.percona.com/pxb-24/apt/pool/main/p/percona-xtrabackup-24/$debfile" - sudo apt install -y "./$debfile" - else - sudo apt-get install -y percona-xtrabackup-24 - fi - + sudo apt-get install percona-xtrabackup-80 lz4 + - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. diff --git a/.github/workflows/cluster_endtoend_xb_backup_mysql57.yml b/.github/workflows/cluster_endtoend_xb_backup_mysql57.yml new file mode 100644 index 00000000000..9474985c529 --- /dev/null +++ b/.github/workflows/cluster_endtoend_xb_backup_mysql57.yml @@ -0,0 +1,156 @@ +# DO NOT MODIFY: THIS FILE IS GENERATED USING "make generate_ci_workflows" + +name: Cluster (xb_backup) mysql57 +on: [push, pull_request] +concurrency: + group: format('{0}-{1}', ${{ github.ref }}, 'Cluster (xb_backup) mysql57') + cancel-in-progress: true + +env: + LAUNCHABLE_ORGANIZATION: "vitess" + LAUNCHABLE_WORKSPACE: "vitess-app" + GITHUB_PR_HEAD_SHA: "${{ github.event.pull_request.head.sha }}" + + # This is used if we need to pin the xtrabackup version used in tests. + # If this is NOT set then the latest version available will be used. + #XTRABACKUP_VERSION: "2.4.24-1" + +jobs: + build: + name: Run endtoend tests on Cluster (xb_backup) mysql57 + runs-on: ubuntu-20.04 + + steps: + - name: Check if workflow needs to be skipped + id: skip-workflow + run: | + skip='false' + if [[ "${{github.event.pull_request}}" == "" ]] && [[ "${{github.ref}}" != "refs/heads/main" ]] && [[ ! "${{github.ref}}" =~ ^refs/heads/release-[0-9]+\.[0-9]$ ]] && [[ ! "${{github.ref}}" =~ "refs/tags/.*" ]]; then + skip='true' + fi + echo Skip ${skip} + echo "::set-output name=skip-workflow::${skip}" + + - name: Check out code + if: steps.skip-workflow.outputs.skip-workflow == 'false' + uses: actions/checkout@v2 + + - name: Check for changes in relevant files + if: steps.skip-workflow.outputs.skip-workflow == 'false' + uses: frouioui/paths-filter@main + id: changes + with: + token: '' + filters: | + end_to_end: + - 'go/**/*.go' + - 'test.go' + - 'Makefile' + - 'build.env' + - 'go.[sumod]' + - 'proto/*.proto' + - 'tools/**' + - 'config/**' + - 'bootstrap.sh' + - '.github/workflows/cluster_endtoend_xb_backup_mysql57.yml' + + - name: Set up Go + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/setup-go@v2 + with: + go-version: 1.19.3 + + - name: Set up python + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/setup-python@v2 + + - name: Tune the OS + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range + # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio + echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf + sudo sysctl -p /etc/sysctl.conf + + - name: Get dependencies + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + sudo apt-get update + + # Uninstall any previously installed MySQL first + sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ + sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + + sudo systemctl stop apparmor + sudo DEBIAN_FRONTEND="noninteractive" apt-get remove -y --purge mysql-server mysql-client mysql-common + sudo apt-get -y autoremove + sudo apt-get -y autoclean + sudo deluser mysql + sudo rm -rf /var/lib/mysql + sudo rm -rf /etc/mysql + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.14-1_all.deb + # Bionic packages are still compatible for Focal since there's no MySQL 5.7 + # packages for Focal. + echo mysql-apt-config mysql-apt-config/repo-codename select bionic | sudo debconf-set-selections + echo mysql-apt-config mysql-apt-config/select-server select mysql-5.7 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* + sudo apt-get update + sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y mysql-client=5.7* mysql-community-server=5.7* mysql-server=5.7* + + sudo apt-get install -y make unzip g++ etcd curl git wget eatmydata + sudo service mysql stop + sudo service etcd stop + + # install JUnit report formatter + go install github.com/vitessio/go-junit-report@HEAD + + wget "https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb" + sudo apt-get install -y gnupg2 + sudo dpkg -i "percona-release_latest.$(lsb_release -sc)_all.deb" + sudo apt-get update + if [[ -n $XTRABACKUP_VERSION ]]; then + debfile="percona-xtrabackup-24_$XTRABACKUP_VERSION.$(lsb_release -sc)_amd64.deb" + wget "https://repo.percona.com/pxb-24/apt/pool/main/p/percona-xtrabackup-24/$debfile" + sudo apt install -y "./$debfile" + else + sudo apt-get install -y percona-xtrabackup-24 + fi + + - name: Setup launchable dependencies + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + # Get Launchable CLI installed. If you can, make it a part of the builder image to speed things up + pip3 install --user launchable~=1.0 > /dev/null + + # verify that launchable setup is all correct. + launchable verify || true + + # Tell Launchable about the build you are producing and testing + launchable record build --name "$GITHUB_RUN_ID" --source . + + - name: Run cluster endtoend test + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 45 + run: | + # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file + # which musn't be more than 107 characters long. + export VTDATAROOT="/tmp/" + source build.env + + set -x + + # run the tests however you normally do, then produce a JUnit XML file + eatmydata -- go run test.go -docker=false -follow -shard xb_backup | tee -a output.txt | go-junit-report -set-exit-code > report.xml + + - name: Print test output and Record test result in launchable + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + run: | + # send recorded tests to launchable + launchable record tests --build "$GITHUB_RUN_ID" go-test . || true + + # print test output + cat output.txt diff --git a/.github/workflows/cluster_endtoend_xb_recovery.yml b/.github/workflows/cluster_endtoend_xb_recovery.yml index 10b5492e6cb..0b083fc183a 100644 --- a/.github/workflows/cluster_endtoend_xb_recovery.yml +++ b/.github/workflows/cluster_endtoend_xb_recovery.yml @@ -14,7 +14,7 @@ env: jobs: build: name: Run endtoend tests on Cluster (xb_recovery) - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - name: Check if workflow needs to be skipped @@ -64,6 +64,8 @@ jobs: - name: Tune the OS if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf @@ -72,10 +74,22 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + + # Setup Percona Server for MySQL 8.0 sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata - sudo service mysql stop + sudo apt-get install -y lsb-release gnupg2 curl + wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb + sudo percona-release setup ps80 + sudo apt-get update + + # Install everything else we need, and configure + sudo apt-get install -y percona-server-server percona-server-client make unzip g++ etcd git wget eatmydata xz-utils + sudo service etcd stop + echo 'mysql version: ' + sudo mysql --version + sudo service mysql stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download @@ -83,21 +97,11 @@ jobs: # install JUnit report formatter go install github.com/vitessio/go-junit-report@HEAD - wget "https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb" - sudo apt-get install -y gnupg2 - sudo dpkg -i "percona-release_latest.$(lsb_release -sc)_all.deb" - sudo apt-get update - if [[ -n $XTRABACKUP_VERSION ]]; then - debfile="percona-xtrabackup-24_$XTRABACKUP_VERSION.$(lsb_release -sc)_amd64.deb" - wget "https://repo.percona.com/pxb-24/apt/pool/main/p/percona-xtrabackup-24/$debfile" - sudo apt install -y "./$debfile" - else - sudo apt-get install -y percona-xtrabackup-24 - fi - + sudo apt-get install percona-xtrabackup-80 lz4 + - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. diff --git a/.github/workflows/cluster_endtoend_xb_recovery_mysql57.yml b/.github/workflows/cluster_endtoend_xb_recovery_mysql57.yml new file mode 100644 index 00000000000..c8ac3a0530a --- /dev/null +++ b/.github/workflows/cluster_endtoend_xb_recovery_mysql57.yml @@ -0,0 +1,156 @@ +# DO NOT MODIFY: THIS FILE IS GENERATED USING "make generate_ci_workflows" + +name: Cluster (xb_recovery) mysql57 +on: [push, pull_request] +concurrency: + group: format('{0}-{1}', ${{ github.ref }}, 'Cluster (xb_recovery) mysql57') + cancel-in-progress: true + +env: + LAUNCHABLE_ORGANIZATION: "vitess" + LAUNCHABLE_WORKSPACE: "vitess-app" + GITHUB_PR_HEAD_SHA: "${{ github.event.pull_request.head.sha }}" + + # This is used if we need to pin the xtrabackup version used in tests. + # If this is NOT set then the latest version available will be used. + #XTRABACKUP_VERSION: "2.4.24-1" + +jobs: + build: + name: Run endtoend tests on Cluster (xb_recovery) mysql57 + runs-on: ubuntu-20.04 + + steps: + - name: Check if workflow needs to be skipped + id: skip-workflow + run: | + skip='false' + if [[ "${{github.event.pull_request}}" == "" ]] && [[ "${{github.ref}}" != "refs/heads/main" ]] && [[ ! "${{github.ref}}" =~ ^refs/heads/release-[0-9]+\.[0-9]$ ]] && [[ ! "${{github.ref}}" =~ "refs/tags/.*" ]]; then + skip='true' + fi + echo Skip ${skip} + echo "::set-output name=skip-workflow::${skip}" + + - name: Check out code + if: steps.skip-workflow.outputs.skip-workflow == 'false' + uses: actions/checkout@v2 + + - name: Check for changes in relevant files + if: steps.skip-workflow.outputs.skip-workflow == 'false' + uses: frouioui/paths-filter@main + id: changes + with: + token: '' + filters: | + end_to_end: + - 'go/**/*.go' + - 'test.go' + - 'Makefile' + - 'build.env' + - 'go.[sumod]' + - 'proto/*.proto' + - 'tools/**' + - 'config/**' + - 'bootstrap.sh' + - '.github/workflows/cluster_endtoend_xb_recovery_mysql57.yml' + + - name: Set up Go + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/setup-go@v2 + with: + go-version: 1.19.3 + + - name: Set up python + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/setup-python@v2 + + - name: Tune the OS + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range + # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio + echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf + sudo sysctl -p /etc/sysctl.conf + + - name: Get dependencies + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + sudo apt-get update + + # Uninstall any previously installed MySQL first + sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ + sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + + sudo systemctl stop apparmor + sudo DEBIAN_FRONTEND="noninteractive" apt-get remove -y --purge mysql-server mysql-client mysql-common + sudo apt-get -y autoremove + sudo apt-get -y autoclean + sudo deluser mysql + sudo rm -rf /var/lib/mysql + sudo rm -rf /etc/mysql + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.14-1_all.deb + # Bionic packages are still compatible for Focal since there's no MySQL 5.7 + # packages for Focal. + echo mysql-apt-config mysql-apt-config/repo-codename select bionic | sudo debconf-set-selections + echo mysql-apt-config mysql-apt-config/select-server select mysql-5.7 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* + sudo apt-get update + sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y mysql-client=5.7* mysql-community-server=5.7* mysql-server=5.7* + + sudo apt-get install -y make unzip g++ etcd curl git wget eatmydata + sudo service mysql stop + sudo service etcd stop + + # install JUnit report formatter + go install github.com/vitessio/go-junit-report@HEAD + + wget "https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb" + sudo apt-get install -y gnupg2 + sudo dpkg -i "percona-release_latest.$(lsb_release -sc)_all.deb" + sudo apt-get update + if [[ -n $XTRABACKUP_VERSION ]]; then + debfile="percona-xtrabackup-24_$XTRABACKUP_VERSION.$(lsb_release -sc)_amd64.deb" + wget "https://repo.percona.com/pxb-24/apt/pool/main/p/percona-xtrabackup-24/$debfile" + sudo apt install -y "./$debfile" + else + sudo apt-get install -y percona-xtrabackup-24 + fi + + - name: Setup launchable dependencies + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + # Get Launchable CLI installed. If you can, make it a part of the builder image to speed things up + pip3 install --user launchable~=1.0 > /dev/null + + # verify that launchable setup is all correct. + launchable verify || true + + # Tell Launchable about the build you are producing and testing + launchable record build --name "$GITHUB_RUN_ID" --source . + + - name: Run cluster endtoend test + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 45 + run: | + # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file + # which musn't be more than 107 characters long. + export VTDATAROOT="/tmp/" + source build.env + + set -x + + # run the tests however you normally do, then produce a JUnit XML file + eatmydata -- go run test.go -docker=false -follow -shard xb_recovery | tee -a output.txt | go-junit-report -set-exit-code > report.xml + + - name: Print test output and Record test result in launchable + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + run: | + # send recorded tests to launchable + launchable record tests --build "$GITHUB_RUN_ID" go-test . || true + + # print test output + cat output.txt diff --git a/.github/workflows/unit_test_mariadb103.yml b/.github/workflows/unit_test_mariadb103.yml index dfe77c60288..9a27205f0cb 100644 --- a/.github/workflows/unit_test_mariadb103.yml +++ b/.github/workflows/unit_test_mariadb103.yml @@ -8,7 +8,7 @@ concurrency: jobs: test: - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - name: Check if workflow needs to be skipped diff --git a/.github/workflows/unit_test_mysql57.yml b/.github/workflows/unit_test_mysql57.yml index ea3a2917d78..be6f1fa37c0 100644 --- a/.github/workflows/unit_test_mysql57.yml +++ b/.github/workflows/unit_test_mysql57.yml @@ -8,7 +8,7 @@ concurrency: jobs: test: - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - name: Check if workflow needs to be skipped diff --git a/.github/workflows/unit_test_mysql80.yml b/.github/workflows/unit_test_mysql80.yml index 77df9083e54..c4b694f7a4e 100644 --- a/.github/workflows/unit_test_mysql80.yml +++ b/.github/workflows/unit_test_mysql80.yml @@ -8,7 +8,7 @@ concurrency: jobs: test: - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - name: Check if workflow needs to be skipped diff --git a/.github/workflows/upgrade_downgrade_test_backups_e2e_next_release.yml b/.github/workflows/upgrade_downgrade_test_backups_e2e_next_release.yml index f77f6f98ca6..2dad2c79771 100644 --- a/.github/workflows/upgrade_downgrade_test_backups_e2e_next_release.yml +++ b/.github/workflows/upgrade_downgrade_test_backups_e2e_next_release.yml @@ -32,7 +32,7 @@ jobs: timeout-minutes: 60 if: always() && needs.get_next_release.result == 'success' name: Run Upgrade Downgrade Test - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 needs: - get_next_release diff --git a/go/mysql/collations/coercion.go b/go/mysql/collations/coercion.go index 0a1bbcb63e6..edd1a6289ea 100644 --- a/go/mysql/collations/coercion.go +++ b/go/mysql/collations/coercion.go @@ -37,13 +37,13 @@ func init() { // // The rules for assigning a Coercibility value to an expression are as follows: // -// - An explicit COLLATE clause has a coercibility of 0 (not coercible at all). -// - The concatenation of two strings with different collations has a coercibility of 1. -// - The collation of a column or a stored routine parameter or local variable has a coercibility of 2. -// - A “system constant” (the string returned by functions such as USER() or VERSION()) has a coercibility of 3. -// - The collation of a literal has a coercibility of 4. -// - The collation of a numeric or temporal value has a coercibility of 5. -// - NULL or an expression that is derived from NULL has a coercibility of 6. +// - An explicit COLLATE clause has a coercibility of 0 (not coercible at all). +// - The concatenation of two strings with different collations has a coercibility of 1. +// - The collation of a column or a stored routine parameter or local variable has a coercibility of 2. +// - A “system constant” (the string returned by functions such as USER() or VERSION()) has a coercibility of 3. +// - The collation of a literal has a coercibility of 4. +// - The collation of a numeric or temporal value has a coercibility of 5. +// - NULL or an expression that is derived from NULL has a coercibility of 6. // // According to the MySQL documentation, Coercibility is an actual word of the English // language, although the Vitess maintainers disagree with this assessment. diff --git a/go/mysql/collations/wildcard.go b/go/mysql/collations/wildcard.go index 6b38eb1a19f..9925042214e 100644 --- a/go/mysql/collations/wildcard.go +++ b/go/mysql/collations/wildcard.go @@ -83,9 +83,9 @@ func (emptyMatcher) Match(in []byte) bool { // fastMatcher is an implementation of WildcardPattern that uses a collation's Collate method // to perform wildcard matching. // It is returned: -// - when the wildcard pattern has no wildcard characters at all -// - when the wildcard pattern has a single '%' (patternMatchMany) and it is the very last -// character of the pattern (in this case, we set isPrefix to true to use prefix-match collation) +// - when the wildcard pattern has no wildcard characters at all +// - when the wildcard pattern has a single '%' (patternMatchMany) and it is the very last +// character of the pattern (in this case, we set isPrefix to true to use prefix-match collation) type fastMatcher struct { collate func(left, right []byte, isPrefix bool) int pattern []byte diff --git a/go/test/endtoend/backup/mysqlctld/backup_mysqlctld_test.go b/go/test/endtoend/backup/mysqlctld/backup_mysqlctld_test.go index 2ee11103c72..4b97f5ff895 100644 --- a/go/test/endtoend/backup/mysqlctld/backup_mysqlctld_test.go +++ b/go/test/endtoend/backup/mysqlctld/backup_mysqlctld_test.go @@ -24,5 +24,5 @@ import ( // TestBackupMysqlctld - tests the backup using mysqlctld. func TestBackupMysqlctld(t *testing.T) { - backup.TestBackup(t, backup.Mysqlctld, "", 0) + backup.TestBackup(t, backup.Mysqlctld, "xbstream", 0) } diff --git a/go/test/endtoend/backup/transform/backup_transform_utils.go b/go/test/endtoend/backup/transform/backup_transform_utils.go index 62b52260a60..e04f6826397 100644 --- a/go/test/endtoend/backup/transform/backup_transform_utils.go +++ b/go/test/endtoend/backup/transform/backup_transform_utils.go @@ -91,7 +91,7 @@ func TestMainSetup(m *testing.M, useMysqlctld bool) { initDb, _ := os.ReadFile(path.Join(os.Getenv("VTROOT"), "/config/init_db.sql")) sql := string(initDb) newInitDBFile = path.Join(localCluster.TmpDirectory, "init_db_with_passwords.sql") - sql = sql + initialsharding.GetPasswordUpdateSQL(localCluster) + sql = sql + initialsharding.GetPasswordUpdateSQL() os.WriteFile(newInitDBFile, []byte(sql), 0666) extraArgs := []string{"--db-credentials-file", dbCredentialFile} diff --git a/go/test/endtoend/backup/vtbackup/main_test.go b/go/test/endtoend/backup/vtbackup/main_test.go index d8e474b7336..5f994948639 100644 --- a/go/test/endtoend/backup/vtbackup/main_test.go +++ b/go/test/endtoend/backup/vtbackup/main_test.go @@ -91,7 +91,7 @@ func TestMain(m *testing.M) { initDb, _ := os.ReadFile(path.Join(os.Getenv("VTROOT"), "/config/init_db.sql")) sql := string(initDb) newInitDBFile = path.Join(localCluster.TmpDirectory, "init_db_with_passwords.sql") - sql = sql + initialsharding.GetPasswordUpdateSQL(localCluster) + sql = sql + initialsharding.GetPasswordUpdateSQL() err = os.WriteFile(newInitDBFile, []byte(sql), 0666) if err != nil { return 1, err @@ -117,12 +117,13 @@ func TestMain(m *testing.M) { tablet.MysqlctlProcess = *cluster.MysqlCtlProcessInstance(tablet.TabletUID, tablet.MySQLPort, localCluster.TmpDirectory) tablet.MysqlctlProcess.InitDBFile = newInitDBFile tablet.MysqlctlProcess.ExtraArgs = extraArgs - if proc, err := tablet.MysqlctlProcess.StartProcess(); err != nil { + proc, err := tablet.MysqlctlProcess.StartProcess() + if err != nil { return 1, err - } else { - // ignore golint warning, we need the else block to use proc - mysqlProcs = append(mysqlProcs, proc) } + // ignore golint warning, we need the else block to use proc + mysqlProcs = append(mysqlProcs, proc) + } for _, proc := range mysqlProcs { if err := proc.Wait(); err != nil { diff --git a/go/test/endtoend/backup/vtctlbackup/backup_test.go b/go/test/endtoend/backup/vtctlbackup/backup_test.go index 6f233eafeda..d7b3da30f9f 100644 --- a/go/test/endtoend/backup/vtctlbackup/backup_test.go +++ b/go/test/endtoend/backup/vtctlbackup/backup_test.go @@ -22,5 +22,5 @@ import ( // TestBackupMain - main tests backup using vtctl commands func TestBackupMain(t *testing.T) { - TestBackup(t, Backup, "", 0) + TestBackup(t, Backup, "xbstream", 0) } diff --git a/go/test/endtoend/backup/vtctlbackup/backup_utils.go b/go/test/endtoend/backup/vtctlbackup/backup_utils.go index f4e2e92a9ac..eae0c306c31 100644 --- a/go/test/endtoend/backup/vtctlbackup/backup_utils.go +++ b/go/test/endtoend/backup/vtctlbackup/backup_utils.go @@ -44,6 +44,7 @@ const ( XtraBackup = iota Backup Mysqlctld + timeout = time.Duration(60 * time.Second) ) var ( @@ -107,7 +108,7 @@ func LaunchCluster(setupType int, streamMode string, stripes int) (int, error) { initDb, _ := os.ReadFile(path.Join(os.Getenv("VTROOT"), "/config/init_db.sql")) sql := string(initDb) newInitDBFile = path.Join(localCluster.TmpDirectory, "init_db_with_passwords.sql") - sql = sql + initialsharding.GetPasswordUpdateSQL(localCluster) + sql = sql + initialsharding.GetPasswordUpdateSQL() err = os.WriteFile(newInitDBFile, []byte(sql), 0666) if err != nil { return 1, err @@ -214,8 +215,28 @@ func TearDownCluster() { } // TestBackup runs all the backup tests -func TestBackup(t *testing.T, setupType int, streamMode string, stripes int) { - +func TestBackup(t *testing.T, setupType int, streamMode string, stripes int) error { + verStr, err := mysqlctl.GetVersionString() + require.NoError(t, err) + _, vers, err := mysqlctl.ParseVersionString(verStr) + require.NoError(t, err) + switch streamMode { + case "xbstream": + if vers.Major < 8 { + t.Logf("Skipping xtrabackup tests with --xtrabackup_stream_mode=xbstream as those are only tested on XtraBackup/MySQL 8.0+") + return nil + } + case "", "tar": // streaming method of tar is the default for the vttablet --xtrabackup_stream_mode flag + // XtraBackup 8.0 must be used with MySQL 8.0 and it no longer supports tar as a stream method: + // https://docs.percona.com/percona-xtrabackup/2.4/innobackupex/streaming_backups_innobackupex.html + // https://docs.percona.com/percona-xtrabackup/8.0/xtrabackup_bin/backup.streaming.html + if vers.Major > 5 { + t.Logf("Skipping xtrabackup tests with --xtrabackup_stream_mode=tar as tar is no longer a streaming option in XtraBackup 8.0") + return nil + } + default: + require.FailNow(t, fmt.Sprintf("Unsupported xtrabackup stream mode: %s", streamMode)) + } testMethods := []struct { name string method func(t *testing.T) @@ -268,7 +289,7 @@ func TestBackup(t *testing.T, setupType int, streamMode string, stripes int) { for _, test := range testMethods { t.Run(test.name, test.method) } - + return nil } type restoreMethod func(t *testing.T, tablet *cluster.Vttablet) @@ -282,12 +303,12 @@ type restoreMethod func(t *testing.T, tablet *cluster.Vttablet) // 7. insert more data on the primary // 8. take another backup // 9. verify that we now have 2 backups -// 10. do a PRS to make the original primary a replica so that we can do a restore there -// 11. Delete+teardown the new primary so that we can restore the first backup on the original +// 10. do a PRS to make the original primary a replica so that we can do a restore there +// 11. Delete+teardown the new primary so that we can restore the first backup on the original // primary to confirm we don't have the data from #7 -// 12. restore first backup on the original primary tablet using the first backup timstamp -// 13. verify that don't have the data added after the first backup -// 14. remove the backups +// 12. restore first backup on the original primary tablet using the first backup timstamp +// 13. verify that don't have the data added after the first backup +// 14. remove the backups func primaryBackup(t *testing.T) { verifyInitialReplication(t) @@ -301,7 +322,7 @@ func primaryBackup(t *testing.T) { require.Nil(t, err) // We'll restore this on the primary later to test restores using a backup timestamp - firstBackupTimestamp := time.Now().Format(mysqlctl.BackupTimestampFormat) + firstBackupTimestamp := time.Now().UTC().Format(mysqlctl.BackupTimestampFormat) backups := localCluster.VerifyBackupCount(t, shardKsName, 1) assert.Contains(t, backups[0], primary.Alias) @@ -357,10 +378,10 @@ func primaryBackup(t *testing.T) { require.Nil(t, err) } -// Test a primary and replica from the same backup. +// Test a primary and replica from the same backup. // -// Check that a replica and primary both restored from the same backup -// can replicate successfully. +// Check that a replica and primary both restored from the same backup +// can replicate successfully. func primaryReplicaSameBackup(t *testing.T) { // insert data on primary, wait for replica to get it verifyInitialReplication(t) @@ -375,7 +396,7 @@ func primaryReplicaSameBackup(t *testing.T) { // now bring up the other replica, letting it restore from backup. restoreWaitForBackup(t, "replica") - err = replica2.VttabletProcess.WaitForTabletStatusesForTimeout([]string{"SERVING"}, 25*time.Second) + err = replica2.VttabletProcess.WaitForTabletStatusesForTimeout([]string{"SERVING"}, timeout) require.Nil(t, err) // check the new replica has the data @@ -428,14 +449,13 @@ func restoreOldPrimaryInPlace(t *testing.T) { // Test that a former primary replicates correctly after being restored. // -//- Take a backup. +// - Take a backup. // - Reparent from old primary to new primary. // - Force old primary to restore from a previous backup using restore_method. // -//Args: -//restore_method: function accepting one parameter of type tablet.Tablet, -//this function is called to force a restore on the provided tablet -// +// Args: +// restore_method: function accepting one parameter of type tablet.Tablet, +// this function is called to force a restore on the provided tablet func testRestoreOldPrimary(t *testing.T, method restoreMethod) { // insert data on primary, wait for replica to get it verifyInitialReplication(t) @@ -584,21 +604,19 @@ func terminatedRestore(t *testing.T) { stopAllTablets() } -//test_backup will: +// test_backup will: // - create a shard with primary and replica1 only -//- run InitShardPrimary -//- bring up tablet_replica2 concurrently, telling it to wait for a backup -//- insert some data -//- take a backup +// - run InitShardPrimary +// - bring up tablet_replica2 concurrently, telling it to wait for a backup +// - insert some data +// - take a backup // - insert more data on the primary -//- wait for tablet_replica2 to become SERVING -//- check all data is right (before+after backup data) -//- list the backup, remove it -// -//Args: -//tablet_type: 'replica' or 'rdonly'. -// +// - wait for tablet_replica2 to become SERVING +// - check all data is right (before+after backup data) +// - list the backup, remove it // +// Args: +// tablet_type: 'replica' or 'rdonly'. func vtctlBackup(t *testing.T, tabletType string) { restoreWaitForBackup(t, tabletType) verifyInitialReplication(t) @@ -611,7 +629,7 @@ func vtctlBackup(t *testing.T, tabletType string) { _, err = primary.VttabletProcess.QueryTablet("insert into vt_insert_test (msg) values ('test2')", keyspaceName, true) require.Nil(t, err) - err = replica2.VttabletProcess.WaitForTabletStatusesForTimeout([]string{"SERVING"}, 25*time.Second) + err = replica2.VttabletProcess.WaitForTabletStatusesForTimeout([]string{"SERVING"}, timeout) require.Nil(t, err) cluster.VerifyRowsInTablet(t, replica2, keyspaceName, 2) @@ -674,7 +692,7 @@ func verifyRestoreTablet(t *testing.T, tablet *cluster.Vttablet, status string) err := tablet.VttabletProcess.Setup() require.Nil(t, err) if status != "" { - err = tablet.VttabletProcess.WaitForTabletStatusesForTimeout([]string{status}, 25*time.Second) + err = tablet.VttabletProcess.WaitForTabletStatusesForTimeout([]string{status}, timeout) require.Nil(t, err) } // We restart replication here because semi-sync will not be set correctly on tablet startup since diff --git a/go/test/endtoend/cluster/cluster_process.go b/go/test/endtoend/cluster/cluster_process.go index b17f90e57c5..a2fa8f4523f 100644 --- a/go/test/endtoend/cluster/cluster_process.go +++ b/go/test/endtoend/cluster/cluster_process.go @@ -22,7 +22,6 @@ import ( "flag" "fmt" "io" - "io/ioutil" "math/rand" "net" "os" @@ -1155,11 +1154,11 @@ func getCoveragePath(fileName string) string { // PrintMysqlctlLogFiles prints all the log files associated with the mysqlctl binary func (cluster *LocalProcessCluster) PrintMysqlctlLogFiles() { logDir := cluster.TmpDirectory - files, _ := ioutil.ReadDir(logDir) + files, _ := os.ReadDir(logDir) for _, fileInfo := range files { if !fileInfo.IsDir() && strings.Contains(fileInfo.Name(), "mysqlctl") { log.Errorf("Printing the log file - " + fileInfo.Name()) - logOut, _ := ioutil.ReadFile(path.Join(logDir, fileInfo.Name())) + logOut, _ := os.ReadFile(path.Join(logDir, fileInfo.Name())) log.Errorf(string(logOut)) } } diff --git a/go/test/endtoend/cluster/mysqlctl_process.go b/go/test/endtoend/cluster/mysqlctl_process.go index 97ccaf80f86..4bc42abcd9e 100644 --- a/go/test/endtoend/cluster/mysqlctl_process.go +++ b/go/test/endtoend/cluster/mysqlctl_process.go @@ -118,6 +118,10 @@ ssl_key={{.Dir}}/server-001-key.pem tmpProcess.Env = append(tmpProcess.Env, "EXTRA_MY_CNF="+extraMyCNF) tmpProcess.Env = append(tmpProcess.Env, "VTDATAROOT="+os.Getenv("VTDATAROOT")) + tmpProcess.Env = append(tmpProcess.Env, "VT_MYSQL_ROOT="+os.Getenv("VT_MYSQL_ROOT")) + tmpProcess.Env = append(tmpProcess.Env, "xtrabackup_root_path="+os.Getenv("xtrabackup_root_path")) + tmpProcess.Env = append(tmpProcess.Env, "MYSQL_FLAVOR="+os.Getenv("MYSQL_FLAVOR")) + tmpProcess.Env = append(tmpProcess.Env, "VT_ROOT="+os.Getenv("VT_ROOT")) } tmpProcess.Args = append(tmpProcess.Args, "init", "--", diff --git a/go/test/endtoend/cluster/vtgate_process.go b/go/test/endtoend/cluster/vtgate_process.go index fa0c1d54d91..95807b8c5ce 100644 --- a/go/test/endtoend/cluster/vtgate_process.go +++ b/go/test/endtoend/cluster/vtgate_process.go @@ -20,7 +20,6 @@ import ( "encoding/json" "fmt" "io" - "io/ioutil" "net/http" "os" "os/exec" @@ -317,7 +316,7 @@ func (vtgate *VtgateProcess) ReadVSchema() (*interface{}, error) { return nil, err } defer resp.Body.Close() - res, err := ioutil.ReadAll(resp.Body) + res, err := io.ReadAll(resp.Body) if err != nil { return nil, err } diff --git a/go/test/endtoend/clustertest/vtgate_test.go b/go/test/endtoend/clustertest/vtgate_test.go index 1d888d9abee..2f72682a391 100644 --- a/go/test/endtoend/clustertest/vtgate_test.go +++ b/go/test/endtoend/clustertest/vtgate_test.go @@ -5,7 +5,7 @@ Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 + http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, diff --git a/go/test/endtoend/recovery/pitr/shardedpitr_test.go b/go/test/endtoend/recovery/pitr/shardedpitr_test.go index 9d4c0691b6d..ee39fe67310 100644 --- a/go/test/endtoend/recovery/pitr/shardedpitr_test.go +++ b/go/test/endtoend/recovery/pitr/shardedpitr_test.go @@ -39,13 +39,15 @@ var ( var ( clusterInstance *cluster.LocalProcessCluster - - primary *cluster.Vttablet - replica *cluster.Vttablet - shard0Primary *cluster.Vttablet - shard0Replica *cluster.Vttablet - shard1Primary *cluster.Vttablet - shard1Replica *cluster.Vttablet + primary *cluster.Vttablet + replica1 *cluster.Vttablet + replica2 *cluster.Vttablet + shard0Primary *cluster.Vttablet + shard0Replica1 *cluster.Vttablet + shard0Replica2 *cluster.Vttablet + shard1Primary *cluster.Vttablet + shard1Replica1 *cluster.Vttablet + shard1Replica2 *cluster.Vttablet cell = "zone1" hostname = "localhost" @@ -127,10 +129,10 @@ func TestPITRRecovery(t *testing.T) { insertRow(t, 1, "prd-1", false) insertRow(t, 2, "prd-2", false) - cluster.VerifyRowsInTabletForTable(t, replica, keyspaceName, 2, "product") + cluster.VerifyRowsInTabletForTable(t, replica1, keyspaceName, 2, "product") // backup the replica - err = clusterInstance.VtctlclientProcess.ExecuteCommand("Backup", replica.Alias) + err = clusterInstance.VtctlclientProcess.ExecuteCommand("Backup", replica1.Alias) require.NoError(t, err) // check that the backup shows up in the listing @@ -166,14 +168,14 @@ func TestPITRRecovery(t *testing.T) { } // wait till all the shards have required data - cluster.VerifyRowsInTabletForTable(t, shard0Replica, keyspaceName, 6, "product") - cluster.VerifyRowsInTabletForTable(t, shard1Replica, keyspaceName, 4, "product") + cluster.VerifyRowsInTabletForTable(t, shard0Replica1, keyspaceName, 6, "product") + cluster.VerifyRowsInTabletForTable(t, shard1Replica1, keyspaceName, 4, "product") // take the backup (to simulate the regular backup) - err = clusterInstance.VtctlclientProcess.ExecuteCommand("Backup", shard0Replica.Alias) + err = clusterInstance.VtctlclientProcess.ExecuteCommand("Backup", shard0Replica1.Alias) require.NoError(t, err) // take the backup (to simulate the regular backup) - err = clusterInstance.VtctlclientProcess.ExecuteCommand("Backup", shard1Replica.Alias) + err = clusterInstance.VtctlclientProcess.ExecuteCommand("Backup", shard1Replica1.Alias) require.NoError(t, err) backups, err := clusterInstance.ListBackups(keyspaceName + "/-80") @@ -313,9 +315,9 @@ func performResharding(t *testing.T) { require.NoError(t, err) // remove the original tablets in the original shard - removeTablets(t, []*cluster.Vttablet{primary, replica}) + removeTablets(t, []*cluster.Vttablet{primary, replica1, replica2}) - for _, tablet := range []*cluster.Vttablet{replica} { + for _, tablet := range []*cluster.Vttablet{replica1, replica2} { err = clusterInstance.VtctlclientProcess.ExecuteCommand("DeleteTablet", tablet.Alias) require.NoError(t, err) } @@ -394,15 +396,18 @@ func initializeCluster(t *testing.T) { // Defining all the tablets primary = clusterInstance.NewVttabletInstance("replica", 0, "") - replica = clusterInstance.NewVttabletInstance("replica", 0, "") + replica1 = clusterInstance.NewVttabletInstance("replica", 0, "") + replica2 = clusterInstance.NewVttabletInstance("replica", 0, "") shard0Primary = clusterInstance.NewVttabletInstance("replica", 0, "") - shard0Replica = clusterInstance.NewVttabletInstance("replica", 0, "") + shard0Replica1 = clusterInstance.NewVttabletInstance("replica", 0, "") + shard0Replica2 = clusterInstance.NewVttabletInstance("replica", 0, "") shard1Primary = clusterInstance.NewVttabletInstance("replica", 0, "") - shard1Replica = clusterInstance.NewVttabletInstance("replica", 0, "") + shard1Replica1 = clusterInstance.NewVttabletInstance("replica", 0, "") + shard1Replica2 = clusterInstance.NewVttabletInstance("replica", 0, "") - shard.Vttablets = []*cluster.Vttablet{primary, replica} - shard0.Vttablets = []*cluster.Vttablet{shard0Primary, shard0Replica} - shard1.Vttablets = []*cluster.Vttablet{shard1Primary, shard1Replica} + shard.Vttablets = []*cluster.Vttablet{primary, replica1, replica2} + shard0.Vttablets = []*cluster.Vttablet{shard0Primary, shard0Replica1, shard0Replica2} + shard1.Vttablets = []*cluster.Vttablet{shard1Primary, shard1Replica1, shard1Replica2} clusterInstance.VtTabletExtraArgs = append(clusterInstance.VtTabletExtraArgs, commonTabletArg...) clusterInstance.VtTabletExtraArgs = append(clusterInstance.VtTabletExtraArgs, "--restore_from_backup", "--enable_semi_sync") @@ -436,14 +441,16 @@ func initializeCluster(t *testing.T) { "FLUSH PRIVILEGES;", } - for _, tablet := range []*cluster.Vttablet{primary, replica, shard0Primary, shard0Replica, shard1Primary, shard1Replica} { - for _, query := range queryCmds { - _, err = tablet.VttabletProcess.QueryTablet(query, keyspace.Name, false) + for _, shard := range clusterInstance.Keyspaces[0].Shards { + for _, tablet := range shard.Vttablets { + for _, query := range queryCmds { + _, err = tablet.VttabletProcess.QueryTablet(query, keyspace.Name, false) + require.NoError(t, err) + } + + err = tablet.VttabletProcess.Setup() require.NoError(t, err) } - - err = tablet.VttabletProcess.Setup() - require.NoError(t, err) } err = clusterInstance.VtctlclientProcess.InitShardPrimary(keyspaceName, shard.Name, cell, primary.TabletUID) diff --git a/go/test/endtoend/recovery/unshardedrecovery/recovery.go b/go/test/endtoend/recovery/unshardedrecovery/recovery.go index 0f85ad5131b..1b5a42e86f3 100644 --- a/go/test/endtoend/recovery/unshardedrecovery/recovery.go +++ b/go/test/endtoend/recovery/unshardedrecovery/recovery.go @@ -96,7 +96,7 @@ func TestMainImpl(m *testing.M) { initDb, _ := os.ReadFile(path.Join(os.Getenv("VTROOT"), "/config/init_db.sql")) sql := string(initDb) newInitDBFile = path.Join(localCluster.TmpDirectory, "init_db_with_passwords.sql") - sql = sql + initialsharding.GetPasswordUpdateSQL(localCluster) + sql = sql + initialsharding.GetPasswordUpdateSQL() // https://github.com/vitessio/vitess/issues/8315 oldAlterTableMode := ` SET GLOBAL old_alter_table = ON; diff --git a/go/test/endtoend/sharding/initialsharding/sharding_util.go b/go/test/endtoend/sharding/initialsharding/sharding_util.go index 567ffb67c99..d0094304477 100644 --- a/go/test/endtoend/sharding/initialsharding/sharding_util.go +++ b/go/test/endtoend/sharding/initialsharding/sharding_util.go @@ -646,7 +646,7 @@ func writeInitDBFile() { initDb, _ := os.ReadFile(path.Join(os.Getenv("VTROOT"), "/config/init_db.sql")) sql := string(initDb) newInitDbFile = path.Join(ClusterInstance.TmpDirectory, "init_db_with_passwords.sql") - sql = sql + GetPasswordUpdateSQL(ClusterInstance) + ` + sql = sql + GetPasswordUpdateSQL() + ` # connecting through a port requires 127.0.0.1 # --host=localhost will connect through socket CREATE USER 'vt_dba'@'127.0.0.1' IDENTIFIED BY 'VtDbaPass'; @@ -695,45 +695,17 @@ func WriteDbCredentialToTmp(tmpDir string) string { } // GetPasswordUpdateSQL returns the sql for password update -func GetPasswordUpdateSQL(localCluster *cluster.LocalProcessCluster) string { +func GetPasswordUpdateSQL() string { pwdChangeCmd := ` # Set real passwords for all users. - UPDATE mysql.user SET %s = PASSWORD('RootPass') - WHERE User = 'root' AND Host = 'localhost'; - UPDATE mysql.user SET %s = PASSWORD('VtDbaPass') - WHERE User = 'vt_dba' AND Host = 'localhost'; - UPDATE mysql.user SET %s = PASSWORD('VtAppPass') - WHERE User = 'vt_app' AND Host = 'localhost'; - UPDATE mysql.user SET %s = PASSWORD('VtAllprivsPass') - WHERE User = 'vt_allprivs' AND Host = 'localhost'; - UPDATE mysql.user SET %s = PASSWORD('VtReplPass') - WHERE User = 'vt_repl' AND Host = '%%'; - UPDATE mysql.user SET %s = PASSWORD('VtFilteredPass') - WHERE User = 'vt_filtered' AND Host = 'localhost'; + # Set real passwords for all users. + ALTER USER 'root'@'localhost' IDENTIFIED BY 'RootPass'; + ALTER USER 'vt_dba'@'localhost' IDENTIFIED BY 'VtDbaPass'; + ALTER USER 'vt_app'@'localhost' IDENTIFIED BY 'VtAppPass'; + ALTER USER 'vt_allprivs'@'localhost' IDENTIFIED BY 'VtAllprivsPass'; + ALTER USER 'vt_repl'@'%' IDENTIFIED BY 'VtReplPass'; + ALTER USER 'vt_filtered'@'localhost' IDENTIFIED BY 'VtFilteredPass'; FLUSH PRIVILEGES; ` - pwdCol, _ := getPasswordField(localCluster) - return fmt.Sprintf(pwdChangeCmd, pwdCol, pwdCol, pwdCol, pwdCol, pwdCol, pwdCol) -} - -// getPasswordField Determines which column is used for user passwords in this MySQL version. -func getPasswordField(localCluster *cluster.LocalProcessCluster) (pwdCol string, err error) { - tablet := &cluster.Vttablet{ - Type: "relpica", - TabletUID: 100, - MySQLPort: 15000, - MysqlctlProcess: *cluster.MysqlCtlProcessInstance(100, 15000, localCluster.TmpDirectory), - } - if err = tablet.MysqlctlProcess.Start(); err != nil { - return "", err - } - tablet.VttabletProcess = cluster.VttabletProcessInstance(tablet.HTTPPort, tablet.GrpcPort, tablet.TabletUID, "", "", "", 0, tablet.Type, localCluster.TopoPort, "", "", nil, false, localCluster.DefaultCharset) - result, err := tablet.VttabletProcess.QueryTablet("select password from mysql.user limit 0", "", false) - if err == nil && len(result.Rows) > 0 { - return "password", nil - } - tablet.MysqlctlProcess.Stop() - os.RemoveAll(path.Join(tablet.VttabletProcess.Directory)) - return "authentication_string", nil - + return pwdChangeCmd } diff --git a/go/test/endtoend/tabletgateway/buffer/buffer_test_helpers.go b/go/test/endtoend/tabletgateway/buffer/buffer_test_helpers.go index bfaaa2e9ac4..19c0f88ed43 100644 --- a/go/test/endtoend/tabletgateway/buffer/buffer_test_helpers.go +++ b/go/test/endtoend/tabletgateway/buffer/buffer_test_helpers.go @@ -67,7 +67,7 @@ const ( updateRowID = 2 ) -//threadParams is set of params passed into read and write threads +// threadParams is set of params passed into read and write threads type threadParams struct { quit bool rpcs int // Number of queries successfully executed. diff --git a/go/test/endtoend/tabletgateway/vtgate_test.go b/go/test/endtoend/tabletgateway/vtgate_test.go index 59f5a43791e..a3876b259f3 100644 --- a/go/test/endtoend/tabletgateway/vtgate_test.go +++ b/go/test/endtoend/tabletgateway/vtgate_test.go @@ -5,7 +5,7 @@ Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 + http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, diff --git a/go/test/endtoend/tabletmanager/commands_test.go b/go/test/endtoend/tabletmanager/commands_test.go index 276d25fc9dc..493282c0c2e 100644 --- a/go/test/endtoend/tabletmanager/commands_test.go +++ b/go/test/endtoend/tabletmanager/commands_test.go @@ -21,23 +21,26 @@ import ( "encoding/json" "fmt" "reflect" + "regexp" + "strconv" "testing" "time" - "vitess.io/vitess/go/test/endtoend/utils" - - "github.com/stretchr/testify/require" "github.com/tidwall/gjson" + "vitess.io/vitess/go/test/endtoend/utils" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "vitess.io/vitess/go/mysql" "vitess.io/vitess/go/test/endtoend/cluster" ) var ( - getSchemaT1Results = "CREATE TABLE `t1` (\n `id` bigint(20) NOT NULL,\n `value` varchar(16) DEFAULT NULL,\n PRIMARY KEY (`id`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8" - getSchemaV1Results = fmt.Sprintf("CREATE ALGORITHM=UNDEFINED DEFINER=`%s`@`%s` SQL SECURITY DEFINER VIEW {{.DatabaseName}}.`v1` AS select {{.DatabaseName}}.`t1`.`id` AS `id`,{{.DatabaseName}}.`t1`.`value` AS `value` from {{.DatabaseName}}.`t1`", username, hostname) + getSchemaT1Results57 = "CREATE TABLE `t1` (\n `id` bigint(20) NOT NULL,\n `value` varchar(16) DEFAULT NULL,\n PRIMARY KEY (`id`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8" + getSchemaT1Results80 = "CREATE TABLE `t1` (\n `id` bigint NOT NULL,\n `value` varchar(16) DEFAULT NULL,\n PRIMARY KEY (`id`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb3" + getSchemaV1Results = fmt.Sprintf("CREATE ALGORITHM=UNDEFINED DEFINER=`%s`@`%s` SQL SECURITY DEFINER VIEW {{.DatabaseName}}.`v1` AS select {{.DatabaseName}}.`t1`.`id` AS `id`,{{.DatabaseName}}.`t1`.`value` AS `value` from {{.DatabaseName}}.`t1`", username, hostname) ) // TabletCommands tests the basic tablet commands @@ -232,10 +235,45 @@ func TestGetSchema(t *testing.T) { fmt.Sprintf("%s-%d", clusterInstance.Cell, primaryTablet.TabletUID)) require.Nil(t, err) + ctx := context.Background() + conn, err := mysql.Connect(ctx, &primaryTabletParams) + require.Nil(t, err) + fmt.Printf("%s", conn.ServerVersion) + major, _, _ := parseVersionString(conn.ServerVersion) + require.LessOrEqual(t, 0, major) + t1Create := gjson.Get(res, "table_definitions.#(name==\"t1\").schema") - assert.Equal(t, getSchemaT1Results, t1Create.String()) + if major <= 5 { + assert.Equal(t, getSchemaT1Results57, t1Create.String()) + } else { + assert.Equal(t, getSchemaT1Results80, t1Create.String()) + } v1Create := gjson.Get(res, "table_definitions.#(name==\"v1\").schema") assert.Equal(t, getSchemaV1Results, v1Create.String()) + +} + +// ParseVersionString parses the output of mysqld --version into a flavor and version +func parseVersionString(version string) (int, int, int) { + versionRegex := regexp.MustCompile(`([0-9]+)\.([0-9]+)\.([0-9]+)`) + v := versionRegex.FindStringSubmatch(version) + if len(v) != 4 { + return 0, 0, 0 + } + major, err := strconv.Atoi(string(v[1])) + if err != nil { + return 0, 0, 0 + } + minor, err := strconv.Atoi(string(v[2])) + if err != nil { + return 0, 0, 0 + } + patch, err := strconv.Atoi(string(v[3])) + if err != nil { + return 0, 0, 0 + } + + return major, minor, patch } func assertNodeCount(t *testing.T, result string, want int) { diff --git a/go/test/endtoend/tabletmanager/custom_rule_topo_test.go b/go/test/endtoend/tabletmanager/custom_rule_topo_test.go index 2a7ed957ac6..0d68c7a2521 100644 --- a/go/test/endtoend/tabletmanager/custom_rule_topo_test.go +++ b/go/test/endtoend/tabletmanager/custom_rule_topo_test.go @@ -5,7 +5,7 @@ Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 + http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, diff --git a/go/test/endtoend/tabletmanager/primary/tablet_test.go b/go/test/endtoend/tabletmanager/primary/tablet_test.go index a7001393833..a87990e828b 100644 --- a/go/test/endtoend/tabletmanager/primary/tablet_test.go +++ b/go/test/endtoend/tabletmanager/primary/tablet_test.go @@ -5,7 +5,7 @@ Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 + http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, diff --git a/go/test/endtoend/tabletmanager/qps_test.go b/go/test/endtoend/tabletmanager/qps_test.go index 4dbdcffb0bd..066e56ac576 100644 --- a/go/test/endtoend/tabletmanager/qps_test.go +++ b/go/test/endtoend/tabletmanager/qps_test.go @@ -5,7 +5,7 @@ Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 + http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, diff --git a/go/test/endtoend/tabletmanager/tablet_security_policy_test.go b/go/test/endtoend/tabletmanager/tablet_security_policy_test.go index 85b1e36f8c9..a2e1e8bd987 100644 --- a/go/test/endtoend/tabletmanager/tablet_security_policy_test.go +++ b/go/test/endtoend/tabletmanager/tablet_security_policy_test.go @@ -5,7 +5,7 @@ Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 + http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, diff --git a/go/test/endtoend/tabletmanager/throttler/throttler_test.go b/go/test/endtoend/tabletmanager/throttler/throttler_test.go index d87f4d3abaa..dfd03299063 100644 --- a/go/test/endtoend/tabletmanager/throttler/throttler_test.go +++ b/go/test/endtoend/tabletmanager/throttler/throttler_test.go @@ -5,7 +5,7 @@ Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 + http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, diff --git a/go/test/endtoend/tabletmanager/throttler_custom_config/throttler_test.go b/go/test/endtoend/tabletmanager/throttler_custom_config/throttler_test.go index 3c21c4d9c5e..422309e0452 100644 --- a/go/test/endtoend/tabletmanager/throttler_custom_config/throttler_test.go +++ b/go/test/endtoend/tabletmanager/throttler_custom_config/throttler_test.go @@ -5,7 +5,7 @@ Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 + http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, diff --git a/go/test/endtoend/topoconncache/main_test.go b/go/test/endtoend/topoconncache/main_test.go index c8d71027bcf..7b02ac6fe9c 100644 --- a/go/test/endtoend/topoconncache/main_test.go +++ b/go/test/endtoend/topoconncache/main_test.go @@ -90,7 +90,9 @@ var ( shard2 cluster.Shard ) -/* This end-to-end test validates the cache fix in topo/server.go inside ConnForCell. +/* + This end-to-end test validates the cache fix in topo/server.go inside ConnForCell. + The issue was, if we delete and add back a cell with same name but at different path, the map of cells in server.go returned the connection object of the previous cell instead of newly-created one with the same name. diff --git a/go/test/endtoend/utils/mysql.go b/go/test/endtoend/utils/mysql.go index 22b6d7ddc09..0f373f929a8 100644 --- a/go/test/endtoend/utils/mysql.go +++ b/go/test/endtoend/utils/mysql.go @@ -68,7 +68,7 @@ func NewMySQL(cluster *cluster.LocalProcessCluster, dbName string, schemaSQL ... } for _, sql := range schemaSQL { - err = prepareMySQLWithSchema(err, params, sql) + err = prepareMySQLWithSchema(params, sql) if err != nil { return mysql.ConnParams{}, nil, err } @@ -119,7 +119,7 @@ func initMysqld(mysqld *mysqlctl.Mysqld, mycnf *mysqlctl.Mycnf, initSQLFile stri return nil } -func prepareMySQLWithSchema(err error, params mysql.ConnParams, sql string) error { +func prepareMySQLWithSchema(params mysql.ConnParams, sql string) error { ctx := context.Background() conn, err := mysql.Connect(ctx, ¶ms) if err != nil { diff --git a/go/test/endtoend/vreplication/config_test.go b/go/test/endtoend/vreplication/config_test.go index dd566e35f5b..95246f5d849 100644 --- a/go/test/endtoend/vreplication/config_test.go +++ b/go/test/endtoend/vreplication/config_test.go @@ -21,12 +21,14 @@ package vreplication // MySQL 5.7+ and MariaDB 10.2+ to ensure that vreplication still works everywhere and the // permissive sql_mode now used in vreplication causes no unwanted side effects. // The customer table also tests two important things: -// 1. Composite or multi-column primary keys -// 2. PKs that contain an ENUM column +// 1. Composite or multi-column primary keys +// 2. PKs that contain an ENUM column +// // The Lead and Lead-1 tables also allows us to test several things: -// 1. Mixed case identifiers -// 2. Column and table names with special characters in them, namely a dash -// 3. Identifiers using reserved words, as lead is a reserved word in MySQL 8.0+ (https://dev.mysql.com/doc/refman/8.0/en/keywords.html) +// 1. Mixed case identifiers +// 2. Column and table names with special characters in them, namely a dash +// 3. Identifiers using reserved words, as lead is a reserved word in MySQL 8.0+ (https://dev.mysql.com/doc/refman/8.0/en/keywords.html) +// // The internal table _vt_PURGE_4f9194b43b2011eb8a0104ed332e05c2_20221210194431 should be ignored by vreplication // The db_order_test table is used to ensure vreplication and vdiff work well with complex non-integer PKs, even across DB versions. var ( diff --git a/go/test/endtoend/vreplication/vreplication_test.go b/go/test/endtoend/vreplication/vreplication_test.go index bcdda81ac8a..a3a88a7fc7f 100644 --- a/go/test/endtoend/vreplication/vreplication_test.go +++ b/go/test/endtoend/vreplication/vreplication_test.go @@ -159,8 +159,8 @@ func TestVreplicationCopyThrottling(t *testing.T) { } func TestBasicVreplicationWorkflow(t *testing.T) { - sourceKsOpts["DBTypeVersion"] = "mysql-5.7" - targetKsOpts["DBTypeVersion"] = "mysql-5.7" + sourceKsOpts["DBTypeVersion"] = "mysql-8.0" + targetKsOpts["DBTypeVersion"] = "mysql-8.0" testBasicVreplicationWorkflow(t) } @@ -1178,6 +1178,7 @@ func dropSources(t *testing.T, ksWorkflow string) { // This allows us to confirm two behaviors: // 1. MoveTables blocks on starting its first copy phase until we rollback // 2. All other workflows continue to work w/o issue with this MVCC history in place (not used yet) +// // Returns a db connection used for the transaction which you can use for follow-up // work, such as rolling it back directly or using the releaseInnoDBRowHistory call. func generateInnoDBRowHistory(t *testing.T, sourceKS string, neededTrxHistory int64) *mysql.Conn { diff --git a/go/test/endtoend/vtgate/queries/aggregation/aggregation_test.go b/go/test/endtoend/vtgate/queries/aggregation/aggregation_test.go index 8467cb30abc..a4088f567f3 100644 --- a/go/test/endtoend/vtgate/queries/aggregation/aggregation_test.go +++ b/go/test/endtoend/vtgate/queries/aggregation/aggregation_test.go @@ -61,9 +61,9 @@ func TestAggregateTypes(t *testing.T) { // TODO (@frouioui): following assertions produce different results between MySQL and Vitess // their differences are ignored for now. Fix it. // `ascii(val1)` returns an `INT64` on Vitess, and `INT32` on MySQL - utils.AssertMatches(t, mcmp.VtConn, "select ascii(val1) as a, count(*) from aggr_test group by a", `[[INT64(65) INT64(1)] [INT64(69) INT64(1)] [INT64(97) INT64(1)] [INT64(98) INT64(1)] [INT64(99) INT64(2)] [INT64(100) INT64(1)] [INT64(101) INT64(1)]]`) - utils.AssertMatches(t, mcmp.VtConn, "select ascii(val1) as a, count(*) from aggr_test group by a order by a", `[[INT64(65) INT64(1)] [INT64(69) INT64(1)] [INT64(97) INT64(1)] [INT64(98) INT64(1)] [INT64(99) INT64(2)] [INT64(100) INT64(1)] [INT64(101) INT64(1)]]`) - utils.AssertMatches(t, mcmp.VtConn, "select ascii(val1) as a, count(*) from aggr_test group by a order by 2, a", `[[INT64(65) INT64(1)] [INT64(69) INT64(1)] [INT64(97) INT64(1)] [INT64(98) INT64(1)] [INT64(100) INT64(1)] [INT64(101) INT64(1)] [INT64(99) INT64(2)]]`) + utils.AssertMatches(t, mcmp.VtConn, "select ascii(val1) as a, count(*) from aggr_test group by a", `[[INT32(65) INT64(1)] [INT32(69) INT64(1)] [INT32(97) INT64(1)] [INT32(98) INT64(1)] [INT32(99) INT64(2)] [INT32(100) INT64(1)] [INT32(101) INT64(1)]]`) + utils.AssertMatches(t, mcmp.VtConn, "select ascii(val1) as a, count(*) from aggr_test group by a order by a", `[[INT32(65) INT64(1)] [INT32(69) INT64(1)] [INT32(97) INT64(1)] [INT32(98) INT64(1)] [INT32(99) INT64(2)] [INT32(100) INT64(1)] [INT32(101) INT64(1)]]`) + utils.AssertMatches(t, mcmp.VtConn, "select ascii(val1) as a, count(*) from aggr_test group by a order by 2, a", `[[INT32(65) INT64(1)] [INT32(69) INT64(1)] [INT32(97) INT64(1)] [INT32(98) INT64(1)] [INT32(100) INT64(1)] [INT32(101) INT64(1)] [INT32(99) INT64(2)]]`) mcmp.AssertMatches("select val1 as a, count(*) from aggr_test group by a", `[[VARCHAR("a") INT64(2)] [VARCHAR("b") INT64(1)] [VARCHAR("c") INT64(2)] [VARCHAR("d") INT64(1)] [VARCHAR("e") INT64(2)]]`) mcmp.AssertMatches("select val1 as a, count(*) from aggr_test group by a order by a", `[[VARCHAR("a") INT64(2)] [VARCHAR("b") INT64(1)] [VARCHAR("c") INT64(2)] [VARCHAR("d") INT64(1)] [VARCHAR("e") INT64(2)]]`) diff --git a/go/test/endtoend/vtgate/queries/aggregation/main_test.go b/go/test/endtoend/vtgate/queries/aggregation/main_test.go index 93968e861f7..084b2332791 100644 --- a/go/test/endtoend/vtgate/queries/aggregation/main_test.go +++ b/go/test/endtoend/vtgate/queries/aggregation/main_test.go @@ -63,14 +63,19 @@ func TestMain(m *testing.M) { SchemaSQL: schemaSQL, VSchema: vschema, } - clusterInstance.VtGateExtraArgs = []string{"--schema_change_signal"} - clusterInstance.VtTabletExtraArgs = []string{"--queryserver-config-schema-change-signal", "--queryserver-config-schema-change-signal-interval", "0.1"} + + clusterInstance.VtTabletExtraArgs = append(clusterInstance.VtTabletExtraArgs, + "--enable_query_plan_field_caching=false", + "--queryserver-config-schema-change-signal", + "--queryserver-config-schema-change-signal-interval", "0.1") err = clusterInstance.StartKeyspace(*keyspace, []string{"-80", "80-"}, 0, false) if err != nil { return 1 } - clusterInstance.VtGateExtraArgs = append(clusterInstance.VtGateExtraArgs, "--enable_system_settings=true") + clusterInstance.VtGateExtraArgs = append(clusterInstance.VtGateExtraArgs, + "--schema_change_signal", + "--enable_system_settings=true") // Start vtgate err = clusterInstance.StartVtgate() if err != nil { diff --git a/go/test/endtoend/vtgate/queries/informationschema/informationschema_test.go b/go/test/endtoend/vtgate/queries/informationschema/informationschema_test.go index 140cb46a0a8..a973c6e2337 100644 --- a/go/test/endtoend/vtgate/queries/informationschema/informationschema_test.go +++ b/go/test/endtoend/vtgate/queries/informationschema/informationschema_test.go @@ -106,7 +106,8 @@ func TestFKConstraintUsingInformationSchema(t *testing.T) { defer closer() query := "select fk.referenced_table_name as to_table, fk.referenced_column_name as primary_key, fk.column_name as `column`, fk.constraint_name as name, rc.update_rule as on_update, rc.delete_rule as on_delete from information_schema.referential_constraints as rc join information_schema.key_column_usage as fk on fk.constraint_schema = rc.constraint_schema and fk.constraint_name = rc.constraint_name where fk.referenced_column_name is not null and fk.table_schema = database() and fk.table_name = 't7_fk' and rc.constraint_schema = database() and rc.table_name = 't7_fk'" - mcmp.AssertMatches(query, `[[VARCHAR("t7_xxhash") VARCHAR("uid") VARCHAR("t7_uid") VARCHAR("t7_fk_ibfk_1") VARCHAR("CASCADE") VARCHAR("SET NULL")]]`) + mcmp.AssertMatches(query, + `[[VARBINARY("t7_xxhash") VARCHAR("uid") VARCHAR("t7_uid") VARCHAR("t7_fk_ibfk_1") BINARY("CASCADE") BINARY("SET NULL")]]`) } func TestConnectWithSystemSchema(t *testing.T) { diff --git a/go/test/endtoend/vtgate/unsharded/main_test.go b/go/test/endtoend/vtgate/unsharded/main_test.go index 7249f8c1c2f..f1bfeecae82 100644 --- a/go/test/endtoend/vtgate/unsharded/main_test.go +++ b/go/test/endtoend/vtgate/unsharded/main_test.go @@ -297,7 +297,7 @@ func TestDDLUnsharded(t *testing.T) { utils.AssertMatches(t, conn, "select * from v1", `[[INT64(3) INT64(0) INT64(3) VARCHAR("a")] [INT64(30) INT64(10) INT64(30) VARCHAR("ac")] [INT64(300) INT64(100) INT64(300) VARCHAR("abc")]]`) utils.Exec(t, conn, `drop view v1`) utils.Exec(t, conn, `drop table tempt1`) - utils.AssertMatches(t, conn, "show tables", `[[VARCHAR("allDefaults")] [VARCHAR("t1")]]`) + utils.AssertMatchesOneOf(t, conn, "show tables", `[[VARCHAR("allDefaults")] [VARCHAR("t1")]]`, `[[VARBINARY("allDefaults")] [VARBINARY("t1")]]`) } func TestCallProcedure(t *testing.T) { @@ -464,7 +464,7 @@ func TestFloatValueDefault(t *testing.T) { want57 := `[[VARCHAR("test_float_default") VARCHAR("pos_f") TEXT("2.1")] [VARCHAR("test_float_default") VARCHAR("neg_f") TEXT("-2.1")]]` want80 := `[[VARBINARY("test_float_default") VARCHAR("pos_f") BLOB("2.1")] [VARBINARY("test_float_default") VARCHAR("neg_f") BLOB("-2.1")]]` - query := "select table_name, column_name, column_default from information_schema.columns where table_name = 'test_float_default'" + query := "select table_name, column_name, column_default from information_schema.columns where table_name = 'test_float_default' order by column_default desc" utils.AssertMatchesOneOf(t, conn, query, want57, want80) } diff --git a/go/test/endtoend/vtorc/utils/utils.go b/go/test/endtoend/vtorc/utils/utils.go index b6020918b6a..96075aef3c0 100644 --- a/go/test/endtoend/vtorc/utils/utils.go +++ b/go/test/endtoend/vtorc/utils/utils.go @@ -19,7 +19,7 @@ package utils import ( "context" "fmt" - "io/ioutil" + "io" "net/http" "os" "os/exec" @@ -727,7 +727,7 @@ func MakeAPICall(t *testing.T, url string) (status int, response string) { res, err := http.Get(url) require.NoError(t, err) defer res.Body.Close() - bodyBytes, err := ioutil.ReadAll(res.Body) + bodyBytes, err := io.ReadAll(res.Body) require.NoError(t, err) body := string(bodyBytes) return res.StatusCode, body diff --git a/go/vt/mysqlctl/mysqld.go b/go/vt/mysqlctl/mysqld.go index 52699917df0..574c6380096 100644 --- a/go/vt/mysqlctl/mysqld.go +++ b/go/vt/mysqlctl/mysqld.go @@ -165,6 +165,7 @@ func NewMysqld(dbcfgs *dbconfigs.DBConfigs) *Mysqld { os.Getenv("VTROOT"), vtenvMysqlRoot, os.Getenv("MYSQL_FLAVOR")) + log.Errorf(message) panic(message) } } diff --git a/go/vt/mysqlctl/rice_box_test.go b/go/vt/mysqlctl/rice_box_test.go index e3126c44a35..c228c4ba5d3 100644 --- a/go/vt/mysqlctl/rice_box_test.go +++ b/go/vt/mysqlctl/rice_box_test.go @@ -18,7 +18,6 @@ package mysqlctl import ( "fmt" - "io/ioutil" "os" "path/filepath" "strings" @@ -55,7 +54,7 @@ func TestMySQLCtlRiceBox(t *testing.T) { require.NoError(t, err) require.NotNil(t, riceContents) - diskBytes, err := ioutil.ReadFile(fmt.Sprintf("%s/%s", mySQLCtlRiceBoxLocation, fileName)) + diskBytes, err := os.ReadFile(fmt.Sprintf("%s/%s", mySQLCtlRiceBoxLocation, fileName)) diskContents := string(diskBytes) require.NoError(t, err) require.Equal(t, riceContents, diskContents) diff --git a/go/vt/servenv/grpc_server.go b/go/vt/servenv/grpc_server.go index f25fff45b97..3b3d7659d61 100644 --- a/go/vt/servenv/grpc_server.go +++ b/go/vt/servenv/grpc_server.go @@ -48,11 +48,12 @@ import ( // Clients register servers, based on service map: // // servenv.RegisterGRPCFlags() -// servenv.OnRun(func() { -// if servenv.GRPCCheckServiceMap("XXX") { -// pb.RegisterXXX(servenv.GRPCServer, XXX) -// } -// } +// +// servenv.OnRun(func() { +// if servenv.GRPCCheckServiceMap("XXX") { +// pb.RegisterXXX(servenv.GRPCServer, XXX) +// } +// } // // Note servenv.GRPCServer can only be used in servenv.OnRun, // and not before, as it is initialized right before calling OnRun. diff --git a/go/vt/vtctld/vtctld_test.go b/go/vt/vtctld/vtctld_test.go index 437cc924d20..dec0e2d742d 100644 --- a/go/vt/vtctld/vtctld_test.go +++ b/go/vt/vtctld/vtctld_test.go @@ -18,7 +18,7 @@ package vtctld import ( "flag" - "io/ioutil" + "io" "net/http" "net/http/httptest" "testing" @@ -37,7 +37,7 @@ func TestWebApp(t *testing.T) { defer res.Body.Close() - data, err := ioutil.ReadAll(res.Body) + data, err := io.ReadAll(res.Body) assert.NoError(t, err) assert.Contains(t, string(data), "") } @@ -56,7 +56,7 @@ func TestWebAppDisabled(t *testing.T) { defer res.Body.Close() - data, err := ioutil.ReadAll(res.Body) + data, err := io.ReadAll(res.Body) assert.NoError(t, err) assert.Equal(t, "404 page not found\n", string(data)) } diff --git a/go/vt/workflow/parallel_runner.go b/go/vt/workflow/parallel_runner.go index 3292d2a1cac..9b8b4b0d193 100644 --- a/go/vt/workflow/parallel_runner.go +++ b/go/vt/workflow/parallel_runner.go @@ -430,7 +430,7 @@ func VerifyAllTasksDone(ctx context.Context, ts *topo.Server, uuid string) error return verifyAllTasksState(ctx, ts, uuid, workflowpb.TaskState_TaskDone) } -//verifyAllTasksState verifies that all tasks are in taskState. Only for tests purposes. +// verifyAllTasksState verifies that all tasks are in taskState. Only for tests purposes. func verifyAllTasksState(ctx context.Context, ts *topo.Server, uuid string, taskState workflowpb.TaskState) error { checkpoint, err := checkpoint(ctx, ts, uuid) if err != nil { diff --git a/test/ci_workflow_gen.go b/test/ci_workflow_gen.go index 800254b5db2..ac26cbfafcb 100644 --- a/test/ci_workflow_gen.go +++ b/test/ci_workflow_gen.go @@ -33,7 +33,7 @@ const ( mysql80 mysqlVersion = "mysql80" mariadb103 mysqlVersion = "mariadb103" - defaultMySQLVersion mysqlVersion = mysql57 + defaultMySQLVersion = mysql80 ) type mysqlVersions []mysqlVersion @@ -82,7 +82,6 @@ var ( "22", "worker_vault_heavy", "24", - "26", "vstream_failover", "vstream_stoponreshard_true", "vstream_stoponreshard_false", @@ -121,6 +120,7 @@ var ( "xb_recovery", "resharding", "resharding_bytes", + "mysql57", "mysql80", "vreplication_across_db_versions", "vreplication_multicell", @@ -155,7 +155,7 @@ type clusterTest struct { Name, Shard, Platform string FileName string MakeTools, InstallXtraBackup bool - Ubuntu20, Docker bool + Docker bool LimitResourceUsage bool } @@ -174,12 +174,8 @@ func clusterMySQLVersions(clusterName string) mysqlVersions { return allMySQLVersions case clusterName == "tabletmanager_tablegc": return allMySQLVersions - case clusterName == "mysql80": - return []mysqlVersion{mysql80} - case clusterName == "vtorc_8.0": - return []mysqlVersion{mysql80} - case clusterName == "vreplication_across_db_versions": - return []mysqlVersion{mysql80} + case clusterName == "mysql57": + return []mysqlVersion{mysql57} default: return defaultMySQLVersions } @@ -271,10 +267,15 @@ func generateSelfHostedClusterWorkflows() error { clusters := canonnizeList(clusterSelfHostedList) for _, cluster := range clusters { for _, mysqlVersion := range clusterMySQLVersions(cluster) { - directoryName := fmt.Sprintf("cluster_test_%s", cluster) + mysqlVersionIndicator := "" + if mysqlVersion != defaultMySQLVersion && len(clusterMySQLVersions(cluster)) > 1 { + mysqlVersionIndicator = "_" + string(mysqlVersion) + } + + directoryName := fmt.Sprintf("cluster_test_%s%s", cluster, mysqlVersionIndicator) test := &selfHostedTest{ - Name: fmt.Sprintf("Cluster (%s)", cluster), - ImageName: fmt.Sprintf("cluster_test_%s", cluster), + Name: fmt.Sprintf("Cluster (%s)(%s)", cluster, mysqlVersion), + ImageName: fmt.Sprintf("cluster_test_%s%s", cluster, mysqlVersionIndicator), Platform: "mysql57", directoryName: directoryName, Dockerfile: fmt.Sprintf("./.github/docker/%s/Dockerfile", directoryName), @@ -299,11 +300,6 @@ func generateSelfHostedClusterWorkflows() error { if mysqlVersion == mysql80 { test.Platform = string(mysql80) } - mysqlVersionIndicator := "" - if mysqlVersion != defaultMySQLVersion && len(clusterMySQLVersions(cluster)) > 1 { - mysqlVersionIndicator = "_" + string(mysqlVersion) - } - err := setupTestDockerFile(test) if err != nil { return err @@ -342,9 +338,8 @@ func generateClusterWorkflows(list []string, tpl string) { break } } - if mysqlVersion == mysql80 { - test.Ubuntu20 = true - test.Platform = string(mysql80) + if mysqlVersion == mysql57 { + test.Platform = string(mysql57) } if strings.HasPrefix(cluster, "vreplication") || strings.HasSuffix(cluster, "heavy") { test.LimitResourceUsage = true diff --git a/test/config.json b/test/config.json index 7dfcc44bf98..67be2465ec6 100644 --- a/test/config.json +++ b/test/config.json @@ -420,17 +420,6 @@ "site_test" ] }, - "pitrtls": { - "File": "unused.go", - "Args": ["vitess.io/vitess/go/test/endtoend/recovery/pitrtls"], - "Command": [], - "Manual": false, - "Shard": "26", - "RetryMax": 1, - "Tags": [ - "site_test" - ] - }, "recovery": { "File": "unused.go", "Args": ["vitess.io/vitess/go/test/endtoend/recovery/unshardedrecovery"], @@ -500,7 +489,7 @@ }, "sharded_recovery": { "File": "unused.go", - "Args": ["vitess.io/vitess/go/test/endtoend/recovery/shardedrecovery"], + "Args": ["vitess.io/vitess/go/test/endtoend/recovery/shardedrecovery", "-timeout", "20m"], "Command": [], "Manual": false, "Shard": "shardedrecovery_stress_verticalsplit_heavy", @@ -509,7 +498,7 @@ }, "stress": { "File": "unused.go", - "Args": ["vitess.io/vitess/go/test/endtoend/stress"], + "Args": ["vitess.io/vitess/go/test/endtoend/stress", "-timeout", "20m"], "Command": [], "Manual": false, "Shard": "shardedrecovery_stress_verticalsplit_heavy", @@ -624,7 +613,7 @@ }, "vertical_split": { "File": "unused.go", - "Args": ["vitess.io/vitess/go/test/endtoend/sharding/verticalsplit"], + "Args": ["vitess.io/vitess/go/test/endtoend/sharding/verticalsplit", "-timeout", "20m"], "Command": [], "Manual": false, "Shard": "shardedrecovery_stress_verticalsplit_heavy", diff --git a/test/templates/cluster_endtoend_test.tpl b/test/templates/cluster_endtoend_test.tpl index d2d4d6b75f1..716694df224 100644 --- a/test/templates/cluster_endtoend_test.tpl +++ b/test/templates/cluster_endtoend_test.tpl @@ -14,7 +14,7 @@ env: jobs: build: name: Run endtoend tests on {{.Name}} - {{if .Ubuntu20}}runs-on: ubuntu-20.04{{else}}runs-on: ubuntu-18.04{{end}} + runs-on: ubuntu-20.04 steps: - name: Check if workflow needs to be skipped @@ -64,6 +64,8 @@ jobs: - name: Tune the OS if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf @@ -72,10 +74,36 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + {{if .InstallXtraBackup}} + + # Setup Percona Server for MySQL 8.0 sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata - sudo service mysql stop + sudo apt-get install -y lsb-release gnupg2 curl + wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb + sudo percona-release setup ps80 + sudo apt-get update + + # Install everything else we need, and configure + sudo apt-get install -y percona-server-server percona-server-client make unzip g++ etcd git wget eatmydata xz-utils + {{else}} + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + # Setup MySQL 8.0 + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* + sudo apt-get update + # Install everything else we need, and configure + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get update + {{end}} + sudo service etcd stop + echo 'mysql version: ' + sudo mysql --version + sudo service mysql stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download @@ -84,19 +112,7 @@ jobs: go install github.com/vitessio/go-junit-report@HEAD {{if .InstallXtraBackup}} - - wget "https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb" - sudo apt-get install -y gnupg2 - sudo dpkg -i "percona-release_latest.$(lsb_release -sc)_all.deb" - sudo apt-get update - if [[ -n $XTRABACKUP_VERSION ]]; then - debfile="percona-xtrabackup-24_$XTRABACKUP_VERSION.$(lsb_release -sc)_amd64.deb" - wget "https://repo.percona.com/pxb-24/apt/pool/main/p/percona-xtrabackup-24/$debfile" - sudo apt install -y "./$debfile" - else - sudo apt-get install -y percona-xtrabackup-24 - fi - + sudo apt-get install percona-xtrabackup-80 lz4 {{end}} {{if .MakeTools}} @@ -110,7 +126,7 @@ jobs: - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. @@ -124,10 +140,11 @@ jobs: sudo sysctl -w net.ipv4.ip_local_port_range="22768 61999" # Increase our open file descriptor limit as we could hit this ulimit -n 65536 - cat <<-EOF>>./config/mycnf/mysql57.cnf + cat <<-EOF>>./config/mycnf/mysql80.cnf innodb_buffer_pool_dump_at_shutdown=OFF + innodb_buffer_pool_in_core_file=OFF innodb_buffer_pool_load_at_startup=OFF - innodb_buffer_pool_size=64M + innodb_buffer_pool_size=16M innodb_doublewrite=OFF innodb_flush_log_at_trx_commit=0 innodb_flush_method=O_DIRECT @@ -138,6 +155,7 @@ jobs: performance_schema=OFF slow-query-log=OFF EOF + make embed_config {{end}} # run the tests however you normally do, then produce a JUnit XML file diff --git a/test/templates/cluster_endtoend_test_docker.tpl b/test/templates/cluster_endtoend_test_docker.tpl index 38316965d2a..75b2c0c8c22 100644 --- a/test/templates/cluster_endtoend_test_docker.tpl +++ b/test/templates/cluster_endtoend_test_docker.tpl @@ -4,7 +4,7 @@ on: [push, pull_request] jobs: build: name: Run endtoend tests on {{.Name}} - {{if .Ubuntu20}}runs-on: ubuntu-20.04{{else}}runs-on: ubuntu-latest{{end}} + runs-on: ubuntu-20.04 steps: - name: Check if workflow needs to be skipped diff --git a/test/templates/cluster_endtoend_test_mysql80.tpl b/test/templates/cluster_endtoend_test_mysql57.tpl similarity index 75% rename from test/templates/cluster_endtoend_test_mysql80.tpl rename to test/templates/cluster_endtoend_test_mysql57.tpl index a53a2cbf82f..e225ee214b9 100644 --- a/test/templates/cluster_endtoend_test_mysql80.tpl +++ b/test/templates/cluster_endtoend_test_mysql57.tpl @@ -7,7 +7,8 @@ concurrency: jobs: build: name: Run endtoend tests on {{.Name}} - {{if .Ubuntu20}}runs-on: ubuntu-20.04{{else}}runs-on: ubuntu-18.04{{end}} + runs-on: ubuntu-20.04 + timeout-minutes: 45 steps: - name: Check if workflow needs to be skipped @@ -65,33 +66,53 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + sudo apt-get update + + # Uninstall any previously installed MySQL first + sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ + sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + + sudo systemctl stop apparmor + sudo DEBIAN_FRONTEND="noninteractive" apt-get remove -y --purge mysql-server mysql-client mysql-common + sudo apt-get -y autoremove + sudo apt-get -y autoclean + sudo deluser mysql + sudo rm -rf /var/lib/mysql + sudo rm -rf /etc/mysql + # Get key to latest MySQL repo sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 - # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb - echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.14-1_all.deb + # Bionic packages are still compatible for Focal since there's no MySQL 5.7 + # packages for Focal. + echo mysql-apt-config mysql-apt-config/repo-codename select bionic | sudo debconf-set-selections + echo mysql-apt-config mysql-apt-config/select-server select mysql-5.7 | sudo debconf-set-selections sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update + sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y mysql-client=5.7* mysql-community-server=5.7* mysql-server=5.7* # Install everything else we need, and configure - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get install -y make unzip g++ etcd curl git wget eatmydata sudo service mysql stop sudo service etcd stop - sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ - sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld - go mod download # install JUnit report formatter go install github.com/vitessio/go-junit-report@HEAD {{if .InstallXtraBackup}} - wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb + wget "https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb" sudo apt-get install -y gnupg2 - sudo dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb + sudo dpkg -i "percona-release_latest.$(lsb_release -sc)_all.deb" sudo apt-get update - sudo apt-get install percona-xtrabackup-24 + if [[ -n $XTRABACKUP_VERSION ]]; then + debfile="percona-xtrabackup-24_$XTRABACKUP_VERSION.$(lsb_release -sc)_amd64.deb" + wget "https://repo.percona.com/pxb-24/apt/pool/main/p/percona-xtrabackup-24/$debfile" + sudo apt install -y "./$debfile" + else + sudo apt-get install -y percona-xtrabackup-24 + fi {{end}} @@ -106,7 +127,7 @@ jobs: - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. diff --git a/test/templates/unit_test.tpl b/test/templates/unit_test.tpl index 1d1f34b271b..9bb7757c5e6 100644 --- a/test/templates/unit_test.tpl +++ b/test/templates/unit_test.tpl @@ -6,7 +6,7 @@ concurrency: jobs: test: - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - name: Check if workflow needs to be skipped