From d3a8b39afb8407772b0a383f537c69bf13ec04a2 Mon Sep 17 00:00:00 2001 From: Rameez Sajwani Date: Wed, 30 Nov 2022 15:36:42 -0800 Subject: [PATCH] Move to MySQL 8.0 as the default template generation (#11153) (#11769) * Move towards MySQL 8.0 as the default template generation (#11153) * Move towards MySQL 8.0 as the default template generation This upgrades the remaining things to Ubuntu 20.04 and makes MySQL 8.0 the default we run tests against. We still have tests for MySQL 5.7 but those are now explicitly opted into. This should finish up the Ubuntu 20.04 upgrade and also makes things easier for the future when we need to upgrade again. Signed-off-by: Dirkjan Bussink * CI: rename shard vtorc_8.0 to vtorc_5.7, change expected test output for 8.0 Signed-off-by: deepthi * CI: increase timeout for 8.0 tests on the actual test step from 30 to 45 mins Signed-off-by: deepthi * CI: increase timeout to 45 minutes for mysql57 tests too. We really only need this for vtorc, but I've made the change to the template so all tests get it. Signed-off-by: deepthi * CI: fix vtorc test to work with both 5.7 and 8.0 Signed-off-by: deepthi * CI: missed docker flag in mysql57 template, one more fix to vtorc test Signed-off-by: deepthi * removing spaces from pb file Signed-off-by: Rameez Sajwani * removing spaces in pb file part 2 Signed-off-by: Rameez Sajwani Signed-off-by: Dirkjan Bussink Signed-off-by: deepthi Signed-off-by: Rameez Sajwani Co-authored-by: deepthi Co-authored-by: Rameez Sajwani * changing update statement to alter Signed-off-by: Rameez Sajwani * Adding mysql 8.0.25 Signed-off-by: Rameez Sajwani * fix junit Signed-off-by: Rameez Sajwani * fixing failures Signed-off-by: Rameez Sajwani * fixing more unit tests Signed-off-by: Rameez Sajwani * fix selfhost test Signed-off-by: Rameez Sajwani * Fixing selfhosted workflows Signed-off-by: Rameez Sajwani * fixing vreplication_v2 Signed-off-by: Rameez Sajwani * removing cluster 12 and 18 from selfhosted Signed-off-by: Rameez Sajwani * manual fixing some workflows Signed-off-by: Rameez Sajwani * making vreplication_v2 exactly like upstream Signed-off-by: Rameez Sajwani * Moving all the workflow to latest mysql version (vanilla) Signed-off-by: Rameez Sajwani * remove port restrictions Signed-off-by: Rameez Sajwani * tmp fix for vreplication_v2 Signed-off-by: Rameez Sajwani * making vreplication_v2 to previous version Signed-off-by: Rameez Sajwani * correcting typo Signed-off-by: Rameez Sajwani * mvoing vreplication_v2 to selfhost Signed-off-by: Rameez Sajwani * change docker image to mysql80 Signed-off-by: Rameez Sajwani * removing limited source from vrepl Signed-off-by: Rameez Sajwani * revert back last commit changes Signed-off-by: Rameez Sajwani * test: fix aggregation test Signed-off-by: Harshit Gangal * removing ptrtls Signed-off-by: Rameez Sajwani * remove pitrTls tests Signed-off-by: Rameez Sajwani * changing source/destination to mysql8.0 for vreplication Signed-off-by: Rameez Sajwani * remove unwanted workflows Signed-off-by: Rameez Sajwani * adding timeout to stress test Signed-off-by: Rameez Sajwani * feeback through code review Signed-off-by: Rameez Sajwani Signed-off-by: Dirkjan Bussink Signed-off-by: deepthi Signed-off-by: Rameez Sajwani Signed-off-by: Harshit Gangal Co-authored-by: Dirkjan Bussink Co-authored-by: deepthi Co-authored-by: Harshit Gangal --- .github/docker/cluster_test_12/Dockerfile | 34 ---- .github/docker/cluster_test_18/Dockerfile | 37 ----- .github/workflows/cluster_endtoend_12.yml | 22 ++- .github/workflows/cluster_endtoend_13.yml | 22 ++- .github/workflows/cluster_endtoend_15.yml | 22 ++- .github/workflows/cluster_endtoend_18.yml | 22 ++- .github/workflows/cluster_endtoend_19.yml | 22 ++- .github/workflows/cluster_endtoend_21.yml | 22 ++- .github/workflows/cluster_endtoend_22.yml | 22 ++- .github/workflows/cluster_endtoend_24.yml | 22 ++- ...ter_endtoend_ers_prs_newfeatures_heavy.yml | 28 +++- .../workflows/cluster_endtoend_mysql57.yml | 117 +++++++++++++ .../workflows/cluster_endtoend_mysql80.yml | 13 +- ...cluster_endtoend_onlineddl_declarative.yml | 22 ++- ...ndtoend_onlineddl_declarative_mysql57.yml} | 39 +++-- .../cluster_endtoend_onlineddl_ghost.yml | 22 ++- ...ster_endtoend_onlineddl_ghost_mysql57.yml} | 39 +++-- .../cluster_endtoend_onlineddl_revert.yml | 22 ++- ...ter_endtoend_onlineddl_revert_mysql57.yml} | 39 +++-- .../cluster_endtoend_onlineddl_revertible.yml | 22 ++- ...endtoend_onlineddl_revertible_mysql57.yml} | 39 +++-- .../cluster_endtoend_onlineddl_scheduler.yml | 22 ++- ..._endtoend_onlineddl_scheduler_mysql57.yml} | 39 +++-- .../cluster_endtoend_onlineddl_singleton.yml | 22 ++- ..._endtoend_onlineddl_singleton_mysql57.yml} | 39 +++-- .../cluster_endtoend_onlineddl_vrepl.yml | 22 ++- ...ster_endtoend_onlineddl_vrepl_mysql57.yml} | 39 +++-- ...luster_endtoend_onlineddl_vrepl_stress.yml | 22 ++- ...dtoend_onlineddl_vrepl_stress_mysql57.yml} | 39 +++-- ..._endtoend_onlineddl_vrepl_stress_suite.yml | 22 ++- ..._onlineddl_vrepl_stress_suite_mysql57.yml} | 39 +++-- ...cluster_endtoend_onlineddl_vrepl_suite.yml | 22 ++- ...ndtoend_onlineddl_vrepl_suite_mysql57.yml} | 39 +++-- .../workflows/cluster_endtoend_resharding.yml | 22 ++- .../cluster_endtoend_resharding_bytes.yml | 22 ++- .../cluster_endtoend_schemadiff_vrepl.yml | 22 ++- ...ter_endtoend_schemadiff_vrepl_mysql57.yml} | 39 +++-- ...dedrecovery_stress_verticalsplit_heavy.yml | 28 +++- .../cluster_endtoend_tabletmanager_consul.yml | 22 ++- ...cluster_endtoend_tabletmanager_tablegc.yml | 22 ++- ...ndtoend_tabletmanager_tablegc_mysql57.yml} | 39 +++-- ...uster_endtoend_tabletmanager_throttler.yml | 22 ++- ..._tabletmanager_throttler_custom_config.yml | 22 ++- ...cluster_endtoend_topo_connection_cache.yml | 22 ++- ...dtoend_vreplication_across_db_versions.yml | 16 +- .../cluster_endtoend_vreplication_basic.yml | 28 +++- ...luster_endtoend_vreplication_cellalias.yml | 28 +++- .../cluster_endtoend_vreplication_migrate.yml | 28 +++- ...vreplication_migrate_vdiff2_convert_tz.yml | 146 ++++++++++++++++ ...luster_endtoend_vreplication_multicell.yml | 28 +++- .../cluster_endtoend_vreplication_v2.yml | 28 +++- .../cluster_endtoend_vstream_failover.yml | 22 ++- ...r_endtoend_vstream_stoponreshard_false.yml | 22 ++- ...er_endtoend_vstream_stoponreshard_true.yml | 22 ++- ...dtoend_vstream_with_keyspaces_to_watch.yml | 22 ++- .../cluster_endtoend_vtbackup_transform.yml | 126 ++++++++++++++ ..._vtctlbackup_sharded_clustertest_heavy.yml | 28 +++- .../cluster_endtoend_vtgate_concurrentdml.yml | 22 ++- .../cluster_endtoend_vtgate_gen4.yml | 22 ++- .../cluster_endtoend_vtgate_general_heavy.yml | 28 +++- .../cluster_endtoend_vtgate_godriver.yml | 22 ++- ...uster_endtoend_vtgate_partial_keyspace.yml | 126 ++++++++++++++ .../cluster_endtoend_vtgate_queries.yml | 22 ++- ...cluster_endtoend_vtgate_readafterwrite.yml | 22 ++- .../cluster_endtoend_vtgate_reservedconn.yml | 22 ++- .../cluster_endtoend_vtgate_schema.yml | 22 ++- ...cluster_endtoend_vtgate_schema_tracker.yml | 22 ++- ...dtoend_vtgate_tablet_healthcheck_cache.yml | 22 ++- .../cluster_endtoend_vtgate_topo.yml | 22 ++- .../cluster_endtoend_vtgate_topo_consul.yml | 22 ++- .../cluster_endtoend_vtgate_topo_etcd.yml | 22 ++- .../cluster_endtoend_vtgate_transaction.yml | 22 ++- .../cluster_endtoend_vtgate_unsharded.yml | 22 ++- .../cluster_endtoend_vtgate_vindex_heavy.yml | 28 +++- .../cluster_endtoend_vtgate_vschema.yml | 22 ++- .github/workflows/cluster_endtoend_vtorc.yml | 22 ++- .../workflows/cluster_endtoend_vtorc_8.0.yml | 13 +- .../cluster_endtoend_worker_vault_heavy.yml | 28 +++- .../workflows/cluster_endtoend_xb_backup.yml | 36 ++-- .../cluster_endtoend_xb_backup_mysql57.yml | 156 ++++++++++++++++++ .../cluster_endtoend_xb_recovery.yml | 36 ++-- .../cluster_endtoend_xb_recovery_mysql57.yml | 156 ++++++++++++++++++ .github/workflows/unit_test_mariadb103.yml | 2 +- .github/workflows/unit_test_mysql57.yml | 2 +- .github/workflows/unit_test_mysql80.yml | 2 +- ...owngrade_test_backups_e2e_next_release.yml | 2 +- go/mysql/collations/coercion.go | 14 +- go/mysql/collations/wildcard.go | 6 +- .../backup/mysqlctld/backup_mysqlctld_test.go | 2 +- .../transform/backup_transform_utils.go | 2 +- go/test/endtoend/backup/vtbackup/main_test.go | 11 +- .../backup/vtctlbackup/backup_test.go | 2 +- .../backup/vtctlbackup/backup_utils.go | 84 ++++++---- go/test/endtoend/cluster/cluster_process.go | 5 +- go/test/endtoend/cluster/mysqlctl_process.go | 4 + go/test/endtoend/cluster/vtgate_process.go | 3 +- go/test/endtoend/clustertest/vtgate_test.go | 2 +- .../recovery/pitr/shardedpitr_test.go | 61 ++++--- .../recovery/unshardedrecovery/recovery.go | 2 +- .../sharding/initialsharding/sharding_util.go | 48 ++---- .../buffer/buffer_test_helpers.go | 2 +- go/test/endtoend/tabletgateway/vtgate_test.go | 2 +- .../endtoend/tabletmanager/commands_test.go | 50 +++++- .../tabletmanager/custom_rule_topo_test.go | 2 +- .../tabletmanager/primary/tablet_test.go | 2 +- go/test/endtoend/tabletmanager/qps_test.go | 2 +- .../tablet_security_policy_test.go | 2 +- .../tabletmanager/throttler/throttler_test.go | 2 +- .../throttler_custom_config/throttler_test.go | 2 +- go/test/endtoend/topoconncache/main_test.go | 4 +- go/test/endtoend/utils/mysql.go | 4 +- go/test/endtoend/vreplication/config_test.go | 12 +- .../vreplication/vreplication_test.go | 5 +- .../queries/aggregation/aggregation_test.go | 6 +- .../vtgate/queries/aggregation/main_test.go | 11 +- .../informationschema_test.go | 3 +- .../endtoend/vtgate/unsharded/main_test.go | 4 +- go/test/endtoend/vtorc/utils/utils.go | 4 +- go/vt/mysqlctl/mysqld.go | 1 + go/vt/mysqlctl/rice_box_test.go | 3 +- go/vt/servenv/grpc_server.go | 11 +- go/vt/vtctld/vtctld_test.go | 6 +- go/vt/workflow/parallel_runner.go | 2 +- test/ci_workflow_gen.go | 35 ++-- test/config.json | 17 +- test/templates/cluster_endtoend_test.tpl | 56 ++++--- .../cluster_endtoend_test_docker.tpl | 2 +- ....tpl => cluster_endtoend_test_mysql57.tpl} | 45 +++-- test/templates/unit_test.tpl | 2 +- 129 files changed, 2603 insertions(+), 750 deletions(-) delete mode 100644 .github/docker/cluster_test_12/Dockerfile delete mode 100644 .github/docker/cluster_test_18/Dockerfile create mode 100644 .github/workflows/cluster_endtoend_mysql57.yml rename .github/workflows/{cluster_endtoend_onlineddl_declarative_mysql80.yml => cluster_endtoend_onlineddl_declarative_mysql57.yml} (77%) rename .github/workflows/{cluster_endtoend_onlineddl_ghost_mysql80.yml => cluster_endtoend_onlineddl_ghost_mysql57.yml} (76%) rename .github/workflows/{cluster_endtoend_onlineddl_revert_mysql80.yml => cluster_endtoend_onlineddl_revert_mysql57.yml} (77%) rename .github/workflows/{cluster_endtoend_onlineddl_revertible_mysql80.yml => cluster_endtoend_onlineddl_revertible_mysql57.yml} (77%) rename .github/workflows/{cluster_endtoend_onlineddl_scheduler_mysql80.yml => cluster_endtoend_onlineddl_scheduler_mysql57.yml} (77%) rename .github/workflows/{cluster_endtoend_onlineddl_singleton_mysql80.yml => cluster_endtoend_onlineddl_singleton_mysql57.yml} (77%) rename .github/workflows/{cluster_endtoend_onlineddl_vrepl_mysql80.yml => cluster_endtoend_onlineddl_vrepl_mysql57.yml} (76%) rename .github/workflows/{cluster_endtoend_onlineddl_vrepl_stress_mysql80.yml => cluster_endtoend_onlineddl_vrepl_stress_mysql57.yml} (77%) rename .github/workflows/{cluster_endtoend_onlineddl_vrepl_stress_suite_mysql80.yml => cluster_endtoend_onlineddl_vrepl_stress_suite_mysql57.yml} (76%) rename .github/workflows/{cluster_endtoend_onlineddl_vrepl_suite_mysql80.yml => cluster_endtoend_onlineddl_vrepl_suite_mysql57.yml} (77%) rename .github/workflows/{cluster_endtoend_schemadiff_vrepl_mysql80.yml => cluster_endtoend_schemadiff_vrepl_mysql57.yml} (77%) rename .github/workflows/{cluster_endtoend_tabletmanager_tablegc_mysql80.yml => cluster_endtoend_tabletmanager_tablegc_mysql57.yml} (77%) create mode 100644 .github/workflows/cluster_endtoend_vreplication_migrate_vdiff2_convert_tz.yml create mode 100644 .github/workflows/cluster_endtoend_vtbackup_transform.yml create mode 100644 .github/workflows/cluster_endtoend_vtgate_partial_keyspace.yml create mode 100644 .github/workflows/cluster_endtoend_xb_backup_mysql57.yml create mode 100644 .github/workflows/cluster_endtoend_xb_recovery_mysql57.yml rename test/templates/{cluster_endtoend_test_mysql80.tpl => cluster_endtoend_test_mysql57.tpl} (75%) diff --git a/.github/docker/cluster_test_12/Dockerfile b/.github/docker/cluster_test_12/Dockerfile deleted file mode 100644 index c13bafe0d87..00000000000 --- a/.github/docker/cluster_test_12/Dockerfile +++ /dev/null @@ -1,34 +0,0 @@ -# DO NOT MODIFY: THIS FILE IS GENERATED USING "make generate_ci_workflows" - -ARG bootstrap_version=10.2 -ARG image="vitess/bootstrap:${bootstrap_version}-mysql57" - -FROM "${image}" - -USER root - -# Re-copy sources from working tree -RUN rm -rf /vt/src/vitess.io/vitess/* -COPY . /vt/src/vitess.io/vitess - -# Set the working directory -WORKDIR /vt/src/vitess.io/vitess - -# Fix permissions -RUN chown -R vitess:vitess /vt - -USER vitess - -# Set environment variables -ENV VTROOT /vt/src/vitess.io/vitess -# Set the vtdataroot such that it uses the volume mount -ENV VTDATAROOT /vt/vtdataroot - -# create the vtdataroot directory -RUN mkdir -p $VTDATAROOT - -# install goimports -RUN go install golang.org/x/tools/cmd/goimports@latest - -# sleep for 50 minutes -CMD sleep 3000 diff --git a/.github/docker/cluster_test_18/Dockerfile b/.github/docker/cluster_test_18/Dockerfile deleted file mode 100644 index 29b6317b381..00000000000 --- a/.github/docker/cluster_test_18/Dockerfile +++ /dev/null @@ -1,37 +0,0 @@ -# DO NOT MODIFY: THIS FILE IS GENERATED USING "make generate_ci_workflows" - -ARG bootstrap_version=10.2 -ARG image="vitess/bootstrap:${bootstrap_version}-mysql57" - -FROM "${image}" - -USER root - -# Re-copy sources from working tree -RUN rm -rf /vt/src/vitess.io/vitess/* -COPY . /vt/src/vitess.io/vitess - -# Set the working directory -WORKDIR /vt/src/vitess.io/vitess - -# Fix permissions -RUN chown -R vitess:vitess /vt - -USER vitess - -# Set environment variables -ENV VTROOT /vt/src/vitess.io/vitess -# Set the vtdataroot such that it uses the volume mount -ENV VTDATAROOT /vt/vtdataroot - -# create the vtdataroot directory -RUN mkdir -p $VTDATAROOT - -# install goimports -RUN go install golang.org/x/tools/cmd/goimports@latest - -# make tools -RUN make tools - -# sleep for 50 minutes -CMD sleep 3000 diff --git a/.github/workflows/cluster_endtoend_12.yml b/.github/workflows/cluster_endtoend_12.yml index b6f2cab4d40..2208b4218d9 100644 --- a/.github/workflows/cluster_endtoend_12.yml +++ b/.github/workflows/cluster_endtoend_12.yml @@ -9,7 +9,7 @@ concurrency: jobs: build: name: Run endtoend tests on Cluster (12) - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - name: Check if workflow needs to be skipped @@ -59,6 +59,8 @@ jobs: - name: Tune the OS if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf @@ -67,10 +69,22 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + # Setup MySQL 8.0 + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata - sudo service mysql stop + # Install everything else we need, and configure + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get update + sudo service etcd stop + echo 'mysql version: ' + sudo mysql --version + sudo service mysql stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download @@ -80,7 +94,7 @@ jobs: - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. diff --git a/.github/workflows/cluster_endtoend_13.yml b/.github/workflows/cluster_endtoend_13.yml index fe2a8bef2e5..c41d4b1d27c 100644 --- a/.github/workflows/cluster_endtoend_13.yml +++ b/.github/workflows/cluster_endtoend_13.yml @@ -9,7 +9,7 @@ concurrency: jobs: build: name: Run endtoend tests on Cluster (13) - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - name: Check if workflow needs to be skipped @@ -59,6 +59,8 @@ jobs: - name: Tune the OS if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf @@ -67,10 +69,22 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + # Setup MySQL 8.0 + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata - sudo service mysql stop + # Install everything else we need, and configure + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get update + sudo service etcd stop + echo 'mysql version: ' + sudo mysql --version + sudo service mysql stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download @@ -80,7 +94,7 @@ jobs: - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. diff --git a/.github/workflows/cluster_endtoend_15.yml b/.github/workflows/cluster_endtoend_15.yml index c5057e1fb46..2d7846ce852 100644 --- a/.github/workflows/cluster_endtoend_15.yml +++ b/.github/workflows/cluster_endtoend_15.yml @@ -9,7 +9,7 @@ concurrency: jobs: build: name: Run endtoend tests on Cluster (15) - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - name: Check if workflow needs to be skipped @@ -59,6 +59,8 @@ jobs: - name: Tune the OS if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf @@ -67,10 +69,22 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + # Setup MySQL 8.0 + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata - sudo service mysql stop + # Install everything else we need, and configure + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get update + sudo service etcd stop + echo 'mysql version: ' + sudo mysql --version + sudo service mysql stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download @@ -80,7 +94,7 @@ jobs: - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. diff --git a/.github/workflows/cluster_endtoend_18.yml b/.github/workflows/cluster_endtoend_18.yml index e6962e4a9fa..5555f64fa12 100644 --- a/.github/workflows/cluster_endtoend_18.yml +++ b/.github/workflows/cluster_endtoend_18.yml @@ -9,7 +9,7 @@ concurrency: jobs: build: name: Run endtoend tests on Cluster (18) - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - name: Check if workflow needs to be skipped @@ -59,6 +59,8 @@ jobs: - name: Tune the OS if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf @@ -67,10 +69,22 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + # Setup MySQL 8.0 + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata - sudo service mysql stop + # Install everything else we need, and configure + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get update + sudo service etcd stop + echo 'mysql version: ' + sudo mysql --version + sudo service mysql stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download @@ -85,7 +99,7 @@ jobs: - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. diff --git a/.github/workflows/cluster_endtoend_19.yml b/.github/workflows/cluster_endtoend_19.yml index 09868998736..b700c8c333e 100644 --- a/.github/workflows/cluster_endtoend_19.yml +++ b/.github/workflows/cluster_endtoend_19.yml @@ -9,7 +9,7 @@ concurrency: jobs: build: name: Run endtoend tests on Cluster (19) - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - name: Check if workflow needs to be skipped @@ -59,6 +59,8 @@ jobs: - name: Tune the OS if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf @@ -67,10 +69,22 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + # Setup MySQL 8.0 + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata - sudo service mysql stop + # Install everything else we need, and configure + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get update + sudo service etcd stop + echo 'mysql version: ' + sudo mysql --version + sudo service mysql stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download @@ -80,7 +94,7 @@ jobs: - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. diff --git a/.github/workflows/cluster_endtoend_21.yml b/.github/workflows/cluster_endtoend_21.yml index b83b12fdd11..8e6220bc3e8 100644 --- a/.github/workflows/cluster_endtoend_21.yml +++ b/.github/workflows/cluster_endtoend_21.yml @@ -9,7 +9,7 @@ concurrency: jobs: build: name: Run endtoend tests on Cluster (21) - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - name: Check if workflow needs to be skipped @@ -59,6 +59,8 @@ jobs: - name: Tune the OS if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf @@ -67,10 +69,22 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + # Setup MySQL 8.0 + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata - sudo service mysql stop + # Install everything else we need, and configure + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get update + sudo service etcd stop + echo 'mysql version: ' + sudo mysql --version + sudo service mysql stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download @@ -80,7 +94,7 @@ jobs: - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. diff --git a/.github/workflows/cluster_endtoend_22.yml b/.github/workflows/cluster_endtoend_22.yml index 1b8172e480f..5ce247f6cd1 100644 --- a/.github/workflows/cluster_endtoend_22.yml +++ b/.github/workflows/cluster_endtoend_22.yml @@ -9,7 +9,7 @@ concurrency: jobs: build: name: Run endtoend tests on Cluster (22) - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - name: Check if workflow needs to be skipped @@ -59,6 +59,8 @@ jobs: - name: Tune the OS if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf @@ -67,10 +69,22 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + # Setup MySQL 8.0 + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata - sudo service mysql stop + # Install everything else we need, and configure + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get update + sudo service etcd stop + echo 'mysql version: ' + sudo mysql --version + sudo service mysql stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download @@ -80,7 +94,7 @@ jobs: - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. diff --git a/.github/workflows/cluster_endtoend_24.yml b/.github/workflows/cluster_endtoend_24.yml index 8e8d6690cac..842a1f98121 100644 --- a/.github/workflows/cluster_endtoend_24.yml +++ b/.github/workflows/cluster_endtoend_24.yml @@ -9,7 +9,7 @@ concurrency: jobs: build: name: Run endtoend tests on Cluster (24) - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - name: Check if workflow needs to be skipped @@ -59,6 +59,8 @@ jobs: - name: Tune the OS if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf @@ -67,10 +69,22 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + # Setup MySQL 8.0 + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata - sudo service mysql stop + # Install everything else we need, and configure + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get update + sudo service etcd stop + echo 'mysql version: ' + sudo mysql --version + sudo service mysql stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download @@ -85,7 +99,7 @@ jobs: - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. diff --git a/.github/workflows/cluster_endtoend_ers_prs_newfeatures_heavy.yml b/.github/workflows/cluster_endtoend_ers_prs_newfeatures_heavy.yml index f90cc12e552..cc9d91d8dab 100644 --- a/.github/workflows/cluster_endtoend_ers_prs_newfeatures_heavy.yml +++ b/.github/workflows/cluster_endtoend_ers_prs_newfeatures_heavy.yml @@ -9,7 +9,7 @@ concurrency: jobs: build: name: Run endtoend tests on Cluster (ers_prs_newfeatures_heavy) - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - name: Check if workflow needs to be skipped @@ -59,6 +59,8 @@ jobs: - name: Tune the OS if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf @@ -67,10 +69,22 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + # Setup MySQL 8.0 + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata - sudo service mysql stop + # Install everything else we need, and configure + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get update + sudo service etcd stop + echo 'mysql version: ' + sudo mysql --version + sudo service mysql stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download @@ -80,7 +94,7 @@ jobs: - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. @@ -93,10 +107,11 @@ jobs: sudo sysctl -w net.ipv4.ip_local_port_range="22768 61999" # Increase our open file descriptor limit as we could hit this ulimit -n 65536 - cat <<-EOF>>./config/mycnf/mysql57.cnf + cat <<-EOF>>./config/mycnf/mysql80.cnf innodb_buffer_pool_dump_at_shutdown=OFF + innodb_buffer_pool_in_core_file=OFF innodb_buffer_pool_load_at_startup=OFF - innodb_buffer_pool_size=64M + innodb_buffer_pool_size=16M innodb_doublewrite=OFF innodb_flush_log_at_trx_commit=0 innodb_flush_method=O_DIRECT @@ -107,6 +122,7 @@ jobs: performance_schema=OFF slow-query-log=OFF EOF + make embed_config # run the tests however you normally do, then produce a JUnit XML file eatmydata -- go run test.go -docker=false -follow -shard ers_prs_newfeatures_heavy diff --git a/.github/workflows/cluster_endtoend_mysql57.yml b/.github/workflows/cluster_endtoend_mysql57.yml new file mode 100644 index 00000000000..5e3e7f3c1af --- /dev/null +++ b/.github/workflows/cluster_endtoend_mysql57.yml @@ -0,0 +1,117 @@ +# DO NOT MODIFY: THIS FILE IS GENERATED USING "make generate_ci_workflows" + +name: Cluster (mysql57) +on: [push, pull_request] +concurrency: + group: format('{0}-{1}', ${{ github.ref }}, 'Cluster (mysql57)') + cancel-in-progress: true + +jobs: + build: + name: Run endtoend tests on Cluster (mysql57) + runs-on: ubuntu-20.04 + timeout-minutes: 45 + + steps: + - name: Check if workflow needs to be skipped + id: skip-workflow + run: | + skip='false' + if [[ "${{github.event.pull_request}}" == "" ]] && [[ "${{github.ref}}" != "refs/heads/main" ]] && [[ ! "${{github.ref}}" =~ ^refs/heads/release-[0-9]+\.[0-9]$ ]] && [[ ! "${{github.ref}}" =~ "refs/tags/.*" ]]; then + skip='true' + fi + echo Skip ${skip} + echo "::set-output name=skip-workflow::${skip}" + + - name: Check out code + if: steps.skip-workflow.outputs.skip-workflow == 'false' + uses: actions/checkout@v2 + + - name: Check for changes in relevant files + if: steps.skip-workflow.outputs.skip-workflow == 'false' + uses: frouioui/paths-filter@main + id: changes + with: + token: '' + filters: | + end_to_end: + - 'go/**/*.go' + - 'test.go' + - 'Makefile' + - 'build.env' + - 'go.sum' + - 'go.mod' + - 'proto/*.proto' + - 'tools/**' + - 'config/**' + - 'bootstrap.sh' + - '.github/workflows/cluster_endtoend_mysql57.yml' + + - name: Set up Go + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/setup-go@v2 + with: + go-version: 1.18.7 + + - name: Set up python + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/setup-python@v2 + + - name: Tune the OS + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range + # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio + echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf + sudo sysctl -p /etc/sysctl.conf + + - name: Get dependencies + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + sudo apt-get update + + # Uninstall any previously installed MySQL first + sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ + sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + + sudo systemctl stop apparmor + sudo DEBIAN_FRONTEND="noninteractive" apt-get remove -y --purge mysql-server mysql-client mysql-common + sudo apt-get -y autoremove + sudo apt-get -y autoclean + sudo deluser mysql + sudo rm -rf /var/lib/mysql + sudo rm -rf /etc/mysql + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.14-1_all.deb + # Bionic packages are still compatible for Focal since there's no MySQL 5.7 + # packages for Focal. + echo mysql-apt-config mysql-apt-config/repo-codename select bionic | sudo debconf-set-selections + echo mysql-apt-config mysql-apt-config/select-server select mysql-5.7 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* + sudo apt-get update + sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y mysql-client=5.7* mysql-community-server=5.7* mysql-server=5.7* + + # Install everything else we need, and configure + sudo apt-get install -y make unzip g++ etcd curl git wget eatmydata + sudo service mysql stop + sudo service etcd stop + + # install JUnit report formatter + go install github.com/vitessio/go-junit-report@HEAD + + - name: Run cluster endtoend test + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 45 + run: | + # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file + # which musn't be more than 107 characters long. + export VTDATAROOT="/tmp/" + source build.env + + set -x + + # run the tests however you normally do, then produce a JUnit XML file + eatmydata -- go run test.go -docker=false -follow -shard mysql57 diff --git a/.github/workflows/cluster_endtoend_mysql80.yml b/.github/workflows/cluster_endtoend_mysql80.yml index 07dcc0075bc..13cb3d93689 100644 --- a/.github/workflows/cluster_endtoend_mysql80.yml +++ b/.github/workflows/cluster_endtoend_mysql80.yml @@ -59,6 +59,8 @@ jobs: - name: Tune the OS if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf @@ -67,19 +69,22 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Get key to latest MySQL repo sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 - # Setup MySQL 8.0 wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update - # Install everything else we need, and configure sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils - sudo service mysql stop + sudo apt-get update + sudo service etcd stop + echo 'mysql version: ' + sudo mysql --version + sudo service mysql stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download @@ -89,7 +94,7 @@ jobs: - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. diff --git a/.github/workflows/cluster_endtoend_onlineddl_declarative.yml b/.github/workflows/cluster_endtoend_onlineddl_declarative.yml index 8f1fa23d7cc..24373aadad1 100644 --- a/.github/workflows/cluster_endtoend_onlineddl_declarative.yml +++ b/.github/workflows/cluster_endtoend_onlineddl_declarative.yml @@ -9,7 +9,7 @@ concurrency: jobs: build: name: Run endtoend tests on Cluster (onlineddl_declarative) - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - name: Check if workflow needs to be skipped @@ -59,6 +59,8 @@ jobs: - name: Tune the OS if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf @@ -67,10 +69,22 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + # Setup MySQL 8.0 + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata - sudo service mysql stop + # Install everything else we need, and configure + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get update + sudo service etcd stop + echo 'mysql version: ' + sudo mysql --version + sudo service mysql stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download @@ -80,7 +94,7 @@ jobs: - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. diff --git a/.github/workflows/cluster_endtoend_onlineddl_declarative_mysql80.yml b/.github/workflows/cluster_endtoend_onlineddl_declarative_mysql57.yml similarity index 77% rename from .github/workflows/cluster_endtoend_onlineddl_declarative_mysql80.yml rename to .github/workflows/cluster_endtoend_onlineddl_declarative_mysql57.yml index 15a7576c00c..985ed57ec0e 100644 --- a/.github/workflows/cluster_endtoend_onlineddl_declarative_mysql80.yml +++ b/.github/workflows/cluster_endtoend_onlineddl_declarative_mysql57.yml @@ -1,15 +1,16 @@ # DO NOT MODIFY: THIS FILE IS GENERATED USING "make generate_ci_workflows" -name: Cluster (onlineddl_declarative) mysql80 +name: Cluster (onlineddl_declarative) mysql57 on: [push, pull_request] concurrency: - group: format('{0}-{1}', ${{ github.ref }}, 'Cluster (onlineddl_declarative) mysql80') + group: format('{0}-{1}', ${{ github.ref }}, 'Cluster (onlineddl_declarative) mysql57') cancel-in-progress: true jobs: build: - name: Run endtoend tests on Cluster (onlineddl_declarative) mysql80 + name: Run endtoend tests on Cluster (onlineddl_declarative) mysql57 runs-on: ubuntu-20.04 + timeout-minutes: 45 steps: - name: Check if workflow needs to be skipped @@ -44,7 +45,7 @@ jobs: - 'tools/**' - 'config/**' - 'bootstrap.sh' - - '.github/workflows/cluster_endtoend_onlineddl_declarative_mysql80.yml' + - '.github/workflows/cluster_endtoend_onlineddl_declarative_mysql57.yml' - name: Set up Go if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' @@ -67,29 +68,43 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + sudo apt-get update + + # Uninstall any previously installed MySQL first + sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ + sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + + sudo systemctl stop apparmor + sudo DEBIAN_FRONTEND="noninteractive" apt-get remove -y --purge mysql-server mysql-client mysql-common + sudo apt-get -y autoremove + sudo apt-get -y autoclean + sudo deluser mysql + sudo rm -rf /var/lib/mysql + sudo rm -rf /etc/mysql + # Get key to latest MySQL repo sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 - # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb - echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.14-1_all.deb + # Bionic packages are still compatible for Focal since there's no MySQL 5.7 + # packages for Focal. + echo mysql-apt-config mysql-apt-config/repo-codename select bionic | sudo debconf-set-selections + echo mysql-apt-config mysql-apt-config/select-server select mysql-5.7 | sudo debconf-set-selections sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update + sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y mysql-client=5.7* mysql-community-server=5.7* mysql-server=5.7* # Install everything else we need, and configure - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get install -y make unzip g++ etcd curl git wget eatmydata sudo service mysql stop sudo service etcd stop - sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ - sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld - go mod download # install JUnit report formatter go install github.com/vitessio/go-junit-report@HEAD - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. diff --git a/.github/workflows/cluster_endtoend_onlineddl_ghost.yml b/.github/workflows/cluster_endtoend_onlineddl_ghost.yml index fad7d4407b6..44df92c038f 100644 --- a/.github/workflows/cluster_endtoend_onlineddl_ghost.yml +++ b/.github/workflows/cluster_endtoend_onlineddl_ghost.yml @@ -9,7 +9,7 @@ concurrency: jobs: build: name: Run endtoend tests on Cluster (onlineddl_ghost) - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - name: Check if workflow needs to be skipped @@ -59,6 +59,8 @@ jobs: - name: Tune the OS if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf @@ -67,10 +69,22 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + # Setup MySQL 8.0 + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata - sudo service mysql stop + # Install everything else we need, and configure + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get update + sudo service etcd stop + echo 'mysql version: ' + sudo mysql --version + sudo service mysql stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download @@ -80,7 +94,7 @@ jobs: - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. diff --git a/.github/workflows/cluster_endtoend_onlineddl_ghost_mysql80.yml b/.github/workflows/cluster_endtoend_onlineddl_ghost_mysql57.yml similarity index 76% rename from .github/workflows/cluster_endtoend_onlineddl_ghost_mysql80.yml rename to .github/workflows/cluster_endtoend_onlineddl_ghost_mysql57.yml index 8b481e162c9..a6b9ae2dd5b 100644 --- a/.github/workflows/cluster_endtoend_onlineddl_ghost_mysql80.yml +++ b/.github/workflows/cluster_endtoend_onlineddl_ghost_mysql57.yml @@ -1,15 +1,16 @@ # DO NOT MODIFY: THIS FILE IS GENERATED USING "make generate_ci_workflows" -name: Cluster (onlineddl_ghost) mysql80 +name: Cluster (onlineddl_ghost) mysql57 on: [push, pull_request] concurrency: - group: format('{0}-{1}', ${{ github.ref }}, 'Cluster (onlineddl_ghost) mysql80') + group: format('{0}-{1}', ${{ github.ref }}, 'Cluster (onlineddl_ghost) mysql57') cancel-in-progress: true jobs: build: - name: Run endtoend tests on Cluster (onlineddl_ghost) mysql80 + name: Run endtoend tests on Cluster (onlineddl_ghost) mysql57 runs-on: ubuntu-20.04 + timeout-minutes: 45 steps: - name: Check if workflow needs to be skipped @@ -44,7 +45,7 @@ jobs: - 'tools/**' - 'config/**' - 'bootstrap.sh' - - '.github/workflows/cluster_endtoend_onlineddl_ghost_mysql80.yml' + - '.github/workflows/cluster_endtoend_onlineddl_ghost_mysql57.yml' - name: Set up Go if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' @@ -67,29 +68,43 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + sudo apt-get update + + # Uninstall any previously installed MySQL first + sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ + sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + + sudo systemctl stop apparmor + sudo DEBIAN_FRONTEND="noninteractive" apt-get remove -y --purge mysql-server mysql-client mysql-common + sudo apt-get -y autoremove + sudo apt-get -y autoclean + sudo deluser mysql + sudo rm -rf /var/lib/mysql + sudo rm -rf /etc/mysql + # Get key to latest MySQL repo sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 - # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb - echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.14-1_all.deb + # Bionic packages are still compatible for Focal since there's no MySQL 5.7 + # packages for Focal. + echo mysql-apt-config mysql-apt-config/repo-codename select bionic | sudo debconf-set-selections + echo mysql-apt-config mysql-apt-config/select-server select mysql-5.7 | sudo debconf-set-selections sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update + sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y mysql-client=5.7* mysql-community-server=5.7* mysql-server=5.7* # Install everything else we need, and configure - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get install -y make unzip g++ etcd curl git wget eatmydata sudo service mysql stop sudo service etcd stop - sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ - sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld - go mod download # install JUnit report formatter go install github.com/vitessio/go-junit-report@HEAD - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. diff --git a/.github/workflows/cluster_endtoend_onlineddl_revert.yml b/.github/workflows/cluster_endtoend_onlineddl_revert.yml index 3016f1f3f67..dde4adf570d 100644 --- a/.github/workflows/cluster_endtoend_onlineddl_revert.yml +++ b/.github/workflows/cluster_endtoend_onlineddl_revert.yml @@ -9,7 +9,7 @@ concurrency: jobs: build: name: Run endtoend tests on Cluster (onlineddl_revert) - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - name: Check if workflow needs to be skipped @@ -59,6 +59,8 @@ jobs: - name: Tune the OS if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf @@ -67,10 +69,22 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + # Setup MySQL 8.0 + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata - sudo service mysql stop + # Install everything else we need, and configure + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get update + sudo service etcd stop + echo 'mysql version: ' + sudo mysql --version + sudo service mysql stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download @@ -80,7 +94,7 @@ jobs: - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. diff --git a/.github/workflows/cluster_endtoend_onlineddl_revert_mysql80.yml b/.github/workflows/cluster_endtoend_onlineddl_revert_mysql57.yml similarity index 77% rename from .github/workflows/cluster_endtoend_onlineddl_revert_mysql80.yml rename to .github/workflows/cluster_endtoend_onlineddl_revert_mysql57.yml index 681dc3ac78d..e74bbd7eba5 100644 --- a/.github/workflows/cluster_endtoend_onlineddl_revert_mysql80.yml +++ b/.github/workflows/cluster_endtoend_onlineddl_revert_mysql57.yml @@ -1,15 +1,16 @@ # DO NOT MODIFY: THIS FILE IS GENERATED USING "make generate_ci_workflows" -name: Cluster (onlineddl_revert) mysql80 +name: Cluster (onlineddl_revert) mysql57 on: [push, pull_request] concurrency: - group: format('{0}-{1}', ${{ github.ref }}, 'Cluster (onlineddl_revert) mysql80') + group: format('{0}-{1}', ${{ github.ref }}, 'Cluster (onlineddl_revert) mysql57') cancel-in-progress: true jobs: build: - name: Run endtoend tests on Cluster (onlineddl_revert) mysql80 + name: Run endtoend tests on Cluster (onlineddl_revert) mysql57 runs-on: ubuntu-20.04 + timeout-minutes: 45 steps: - name: Check if workflow needs to be skipped @@ -44,7 +45,7 @@ jobs: - 'tools/**' - 'config/**' - 'bootstrap.sh' - - '.github/workflows/cluster_endtoend_onlineddl_revert_mysql80.yml' + - '.github/workflows/cluster_endtoend_onlineddl_revert_mysql57.yml' - name: Set up Go if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' @@ -67,29 +68,43 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + sudo apt-get update + + # Uninstall any previously installed MySQL first + sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ + sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + + sudo systemctl stop apparmor + sudo DEBIAN_FRONTEND="noninteractive" apt-get remove -y --purge mysql-server mysql-client mysql-common + sudo apt-get -y autoremove + sudo apt-get -y autoclean + sudo deluser mysql + sudo rm -rf /var/lib/mysql + sudo rm -rf /etc/mysql + # Get key to latest MySQL repo sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 - # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb - echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.14-1_all.deb + # Bionic packages are still compatible for Focal since there's no MySQL 5.7 + # packages for Focal. + echo mysql-apt-config mysql-apt-config/repo-codename select bionic | sudo debconf-set-selections + echo mysql-apt-config mysql-apt-config/select-server select mysql-5.7 | sudo debconf-set-selections sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update + sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y mysql-client=5.7* mysql-community-server=5.7* mysql-server=5.7* # Install everything else we need, and configure - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get install -y make unzip g++ etcd curl git wget eatmydata sudo service mysql stop sudo service etcd stop - sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ - sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld - go mod download # install JUnit report formatter go install github.com/vitessio/go-junit-report@HEAD - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. diff --git a/.github/workflows/cluster_endtoend_onlineddl_revertible.yml b/.github/workflows/cluster_endtoend_onlineddl_revertible.yml index 7768a3f919c..66c9c5b8894 100644 --- a/.github/workflows/cluster_endtoend_onlineddl_revertible.yml +++ b/.github/workflows/cluster_endtoend_onlineddl_revertible.yml @@ -9,7 +9,7 @@ concurrency: jobs: build: name: Run endtoend tests on Cluster (onlineddl_revertible) - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - name: Check if workflow needs to be skipped @@ -59,6 +59,8 @@ jobs: - name: Tune the OS if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf @@ -67,10 +69,22 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + # Setup MySQL 8.0 + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata - sudo service mysql stop + # Install everything else we need, and configure + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get update + sudo service etcd stop + echo 'mysql version: ' + sudo mysql --version + sudo service mysql stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download @@ -80,7 +94,7 @@ jobs: - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. diff --git a/.github/workflows/cluster_endtoend_onlineddl_revertible_mysql80.yml b/.github/workflows/cluster_endtoend_onlineddl_revertible_mysql57.yml similarity index 77% rename from .github/workflows/cluster_endtoend_onlineddl_revertible_mysql80.yml rename to .github/workflows/cluster_endtoend_onlineddl_revertible_mysql57.yml index 83df8d6d82b..3fcb0ebddfc 100644 --- a/.github/workflows/cluster_endtoend_onlineddl_revertible_mysql80.yml +++ b/.github/workflows/cluster_endtoend_onlineddl_revertible_mysql57.yml @@ -1,15 +1,16 @@ # DO NOT MODIFY: THIS FILE IS GENERATED USING "make generate_ci_workflows" -name: Cluster (onlineddl_revertible) mysql80 +name: Cluster (onlineddl_revertible) mysql57 on: [push, pull_request] concurrency: - group: format('{0}-{1}', ${{ github.ref }}, 'Cluster (onlineddl_revertible) mysql80') + group: format('{0}-{1}', ${{ github.ref }}, 'Cluster (onlineddl_revertible) mysql57') cancel-in-progress: true jobs: build: - name: Run endtoend tests on Cluster (onlineddl_revertible) mysql80 + name: Run endtoend tests on Cluster (onlineddl_revertible) mysql57 runs-on: ubuntu-20.04 + timeout-minutes: 45 steps: - name: Check if workflow needs to be skipped @@ -44,7 +45,7 @@ jobs: - 'tools/**' - 'config/**' - 'bootstrap.sh' - - '.github/workflows/cluster_endtoend_onlineddl_revertible_mysql80.yml' + - '.github/workflows/cluster_endtoend_onlineddl_revertible_mysql57.yml' - name: Set up Go if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' @@ -67,29 +68,43 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + sudo apt-get update + + # Uninstall any previously installed MySQL first + sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ + sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + + sudo systemctl stop apparmor + sudo DEBIAN_FRONTEND="noninteractive" apt-get remove -y --purge mysql-server mysql-client mysql-common + sudo apt-get -y autoremove + sudo apt-get -y autoclean + sudo deluser mysql + sudo rm -rf /var/lib/mysql + sudo rm -rf /etc/mysql + # Get key to latest MySQL repo sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 - # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb - echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.14-1_all.deb + # Bionic packages are still compatible for Focal since there's no MySQL 5.7 + # packages for Focal. + echo mysql-apt-config mysql-apt-config/repo-codename select bionic | sudo debconf-set-selections + echo mysql-apt-config mysql-apt-config/select-server select mysql-5.7 | sudo debconf-set-selections sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update + sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y mysql-client=5.7* mysql-community-server=5.7* mysql-server=5.7* # Install everything else we need, and configure - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get install -y make unzip g++ etcd curl git wget eatmydata sudo service mysql stop sudo service etcd stop - sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ - sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld - go mod download # install JUnit report formatter go install github.com/vitessio/go-junit-report@HEAD - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. diff --git a/.github/workflows/cluster_endtoend_onlineddl_scheduler.yml b/.github/workflows/cluster_endtoend_onlineddl_scheduler.yml index c622a8d0271..cfeecd23b89 100644 --- a/.github/workflows/cluster_endtoend_onlineddl_scheduler.yml +++ b/.github/workflows/cluster_endtoend_onlineddl_scheduler.yml @@ -9,7 +9,7 @@ concurrency: jobs: build: name: Run endtoend tests on Cluster (onlineddl_scheduler) - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - name: Check if workflow needs to be skipped @@ -59,6 +59,8 @@ jobs: - name: Tune the OS if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf @@ -67,10 +69,22 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + # Setup MySQL 8.0 + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata - sudo service mysql stop + # Install everything else we need, and configure + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get update + sudo service etcd stop + echo 'mysql version: ' + sudo mysql --version + sudo service mysql stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download @@ -80,7 +94,7 @@ jobs: - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. diff --git a/.github/workflows/cluster_endtoend_onlineddl_scheduler_mysql80.yml b/.github/workflows/cluster_endtoend_onlineddl_scheduler_mysql57.yml similarity index 77% rename from .github/workflows/cluster_endtoend_onlineddl_scheduler_mysql80.yml rename to .github/workflows/cluster_endtoend_onlineddl_scheduler_mysql57.yml index cfe5084093f..356c929af6f 100644 --- a/.github/workflows/cluster_endtoend_onlineddl_scheduler_mysql80.yml +++ b/.github/workflows/cluster_endtoend_onlineddl_scheduler_mysql57.yml @@ -1,15 +1,16 @@ # DO NOT MODIFY: THIS FILE IS GENERATED USING "make generate_ci_workflows" -name: Cluster (onlineddl_scheduler) mysql80 +name: Cluster (onlineddl_scheduler) mysql57 on: [push, pull_request] concurrency: - group: format('{0}-{1}', ${{ github.ref }}, 'Cluster (onlineddl_scheduler) mysql80') + group: format('{0}-{1}', ${{ github.ref }}, 'Cluster (onlineddl_scheduler) mysql57') cancel-in-progress: true jobs: build: - name: Run endtoend tests on Cluster (onlineddl_scheduler) mysql80 + name: Run endtoend tests on Cluster (onlineddl_scheduler) mysql57 runs-on: ubuntu-20.04 + timeout-minutes: 45 steps: - name: Check if workflow needs to be skipped @@ -44,7 +45,7 @@ jobs: - 'tools/**' - 'config/**' - 'bootstrap.sh' - - '.github/workflows/cluster_endtoend_onlineddl_scheduler_mysql80.yml' + - '.github/workflows/cluster_endtoend_onlineddl_scheduler_mysql57.yml' - name: Set up Go if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' @@ -67,29 +68,43 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + sudo apt-get update + + # Uninstall any previously installed MySQL first + sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ + sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + + sudo systemctl stop apparmor + sudo DEBIAN_FRONTEND="noninteractive" apt-get remove -y --purge mysql-server mysql-client mysql-common + sudo apt-get -y autoremove + sudo apt-get -y autoclean + sudo deluser mysql + sudo rm -rf /var/lib/mysql + sudo rm -rf /etc/mysql + # Get key to latest MySQL repo sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 - # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb - echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.14-1_all.deb + # Bionic packages are still compatible for Focal since there's no MySQL 5.7 + # packages for Focal. + echo mysql-apt-config mysql-apt-config/repo-codename select bionic | sudo debconf-set-selections + echo mysql-apt-config mysql-apt-config/select-server select mysql-5.7 | sudo debconf-set-selections sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update + sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y mysql-client=5.7* mysql-community-server=5.7* mysql-server=5.7* # Install everything else we need, and configure - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get install -y make unzip g++ etcd curl git wget eatmydata sudo service mysql stop sudo service etcd stop - sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ - sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld - go mod download # install JUnit report formatter go install github.com/vitessio/go-junit-report@HEAD - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. diff --git a/.github/workflows/cluster_endtoend_onlineddl_singleton.yml b/.github/workflows/cluster_endtoend_onlineddl_singleton.yml index bef4bf4b9d4..daeaed40636 100644 --- a/.github/workflows/cluster_endtoend_onlineddl_singleton.yml +++ b/.github/workflows/cluster_endtoend_onlineddl_singleton.yml @@ -9,7 +9,7 @@ concurrency: jobs: build: name: Run endtoend tests on Cluster (onlineddl_singleton) - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - name: Check if workflow needs to be skipped @@ -59,6 +59,8 @@ jobs: - name: Tune the OS if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf @@ -67,10 +69,22 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + # Setup MySQL 8.0 + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata - sudo service mysql stop + # Install everything else we need, and configure + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get update + sudo service etcd stop + echo 'mysql version: ' + sudo mysql --version + sudo service mysql stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download @@ -80,7 +94,7 @@ jobs: - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. diff --git a/.github/workflows/cluster_endtoend_onlineddl_singleton_mysql80.yml b/.github/workflows/cluster_endtoend_onlineddl_singleton_mysql57.yml similarity index 77% rename from .github/workflows/cluster_endtoend_onlineddl_singleton_mysql80.yml rename to .github/workflows/cluster_endtoend_onlineddl_singleton_mysql57.yml index 19e8c27129d..2bd8edcc553 100644 --- a/.github/workflows/cluster_endtoend_onlineddl_singleton_mysql80.yml +++ b/.github/workflows/cluster_endtoend_onlineddl_singleton_mysql57.yml @@ -1,15 +1,16 @@ # DO NOT MODIFY: THIS FILE IS GENERATED USING "make generate_ci_workflows" -name: Cluster (onlineddl_singleton) mysql80 +name: Cluster (onlineddl_singleton) mysql57 on: [push, pull_request] concurrency: - group: format('{0}-{1}', ${{ github.ref }}, 'Cluster (onlineddl_singleton) mysql80') + group: format('{0}-{1}', ${{ github.ref }}, 'Cluster (onlineddl_singleton) mysql57') cancel-in-progress: true jobs: build: - name: Run endtoend tests on Cluster (onlineddl_singleton) mysql80 + name: Run endtoend tests on Cluster (onlineddl_singleton) mysql57 runs-on: ubuntu-20.04 + timeout-minutes: 45 steps: - name: Check if workflow needs to be skipped @@ -44,7 +45,7 @@ jobs: - 'tools/**' - 'config/**' - 'bootstrap.sh' - - '.github/workflows/cluster_endtoend_onlineddl_singleton_mysql80.yml' + - '.github/workflows/cluster_endtoend_onlineddl_singleton_mysql57.yml' - name: Set up Go if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' @@ -67,29 +68,43 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + sudo apt-get update + + # Uninstall any previously installed MySQL first + sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ + sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + + sudo systemctl stop apparmor + sudo DEBIAN_FRONTEND="noninteractive" apt-get remove -y --purge mysql-server mysql-client mysql-common + sudo apt-get -y autoremove + sudo apt-get -y autoclean + sudo deluser mysql + sudo rm -rf /var/lib/mysql + sudo rm -rf /etc/mysql + # Get key to latest MySQL repo sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 - # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb - echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.14-1_all.deb + # Bionic packages are still compatible for Focal since there's no MySQL 5.7 + # packages for Focal. + echo mysql-apt-config mysql-apt-config/repo-codename select bionic | sudo debconf-set-selections + echo mysql-apt-config mysql-apt-config/select-server select mysql-5.7 | sudo debconf-set-selections sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update + sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y mysql-client=5.7* mysql-community-server=5.7* mysql-server=5.7* # Install everything else we need, and configure - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get install -y make unzip g++ etcd curl git wget eatmydata sudo service mysql stop sudo service etcd stop - sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ - sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld - go mod download # install JUnit report formatter go install github.com/vitessio/go-junit-report@HEAD - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. diff --git a/.github/workflows/cluster_endtoend_onlineddl_vrepl.yml b/.github/workflows/cluster_endtoend_onlineddl_vrepl.yml index 7d1dd17e58b..3b290b76b3c 100644 --- a/.github/workflows/cluster_endtoend_onlineddl_vrepl.yml +++ b/.github/workflows/cluster_endtoend_onlineddl_vrepl.yml @@ -9,7 +9,7 @@ concurrency: jobs: build: name: Run endtoend tests on Cluster (onlineddl_vrepl) - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - name: Check if workflow needs to be skipped @@ -59,6 +59,8 @@ jobs: - name: Tune the OS if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf @@ -67,10 +69,22 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + # Setup MySQL 8.0 + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata - sudo service mysql stop + # Install everything else we need, and configure + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get update + sudo service etcd stop + echo 'mysql version: ' + sudo mysql --version + sudo service mysql stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download @@ -80,7 +94,7 @@ jobs: - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. diff --git a/.github/workflows/cluster_endtoend_onlineddl_vrepl_mysql80.yml b/.github/workflows/cluster_endtoend_onlineddl_vrepl_mysql57.yml similarity index 76% rename from .github/workflows/cluster_endtoend_onlineddl_vrepl_mysql80.yml rename to .github/workflows/cluster_endtoend_onlineddl_vrepl_mysql57.yml index c3511ffb4fd..4ee40ffde1d 100644 --- a/.github/workflows/cluster_endtoend_onlineddl_vrepl_mysql80.yml +++ b/.github/workflows/cluster_endtoend_onlineddl_vrepl_mysql57.yml @@ -1,15 +1,16 @@ # DO NOT MODIFY: THIS FILE IS GENERATED USING "make generate_ci_workflows" -name: Cluster (onlineddl_vrepl) mysql80 +name: Cluster (onlineddl_vrepl) mysql57 on: [push, pull_request] concurrency: - group: format('{0}-{1}', ${{ github.ref }}, 'Cluster (onlineddl_vrepl) mysql80') + group: format('{0}-{1}', ${{ github.ref }}, 'Cluster (onlineddl_vrepl) mysql57') cancel-in-progress: true jobs: build: - name: Run endtoend tests on Cluster (onlineddl_vrepl) mysql80 + name: Run endtoend tests on Cluster (onlineddl_vrepl) mysql57 runs-on: ubuntu-20.04 + timeout-minutes: 45 steps: - name: Check if workflow needs to be skipped @@ -44,7 +45,7 @@ jobs: - 'tools/**' - 'config/**' - 'bootstrap.sh' - - '.github/workflows/cluster_endtoend_onlineddl_vrepl_mysql80.yml' + - '.github/workflows/cluster_endtoend_onlineddl_vrepl_mysql57.yml' - name: Set up Go if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' @@ -67,29 +68,43 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + sudo apt-get update + + # Uninstall any previously installed MySQL first + sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ + sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + + sudo systemctl stop apparmor + sudo DEBIAN_FRONTEND="noninteractive" apt-get remove -y --purge mysql-server mysql-client mysql-common + sudo apt-get -y autoremove + sudo apt-get -y autoclean + sudo deluser mysql + sudo rm -rf /var/lib/mysql + sudo rm -rf /etc/mysql + # Get key to latest MySQL repo sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 - # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb - echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.14-1_all.deb + # Bionic packages are still compatible for Focal since there's no MySQL 5.7 + # packages for Focal. + echo mysql-apt-config mysql-apt-config/repo-codename select bionic | sudo debconf-set-selections + echo mysql-apt-config mysql-apt-config/select-server select mysql-5.7 | sudo debconf-set-selections sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update + sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y mysql-client=5.7* mysql-community-server=5.7* mysql-server=5.7* # Install everything else we need, and configure - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get install -y make unzip g++ etcd curl git wget eatmydata sudo service mysql stop sudo service etcd stop - sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ - sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld - go mod download # install JUnit report formatter go install github.com/vitessio/go-junit-report@HEAD - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. diff --git a/.github/workflows/cluster_endtoend_onlineddl_vrepl_stress.yml b/.github/workflows/cluster_endtoend_onlineddl_vrepl_stress.yml index 19ff2664b5e..65239436d1e 100644 --- a/.github/workflows/cluster_endtoend_onlineddl_vrepl_stress.yml +++ b/.github/workflows/cluster_endtoend_onlineddl_vrepl_stress.yml @@ -9,7 +9,7 @@ concurrency: jobs: build: name: Run endtoend tests on Cluster (onlineddl_vrepl_stress) - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - name: Check if workflow needs to be skipped @@ -59,6 +59,8 @@ jobs: - name: Tune the OS if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf @@ -67,10 +69,22 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + # Setup MySQL 8.0 + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata - sudo service mysql stop + # Install everything else we need, and configure + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get update + sudo service etcd stop + echo 'mysql version: ' + sudo mysql --version + sudo service mysql stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download @@ -80,7 +94,7 @@ jobs: - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. diff --git a/.github/workflows/cluster_endtoend_onlineddl_vrepl_stress_mysql80.yml b/.github/workflows/cluster_endtoend_onlineddl_vrepl_stress_mysql57.yml similarity index 77% rename from .github/workflows/cluster_endtoend_onlineddl_vrepl_stress_mysql80.yml rename to .github/workflows/cluster_endtoend_onlineddl_vrepl_stress_mysql57.yml index a6c4f4af583..414ddc30b04 100644 --- a/.github/workflows/cluster_endtoend_onlineddl_vrepl_stress_mysql80.yml +++ b/.github/workflows/cluster_endtoend_onlineddl_vrepl_stress_mysql57.yml @@ -1,15 +1,16 @@ # DO NOT MODIFY: THIS FILE IS GENERATED USING "make generate_ci_workflows" -name: Cluster (onlineddl_vrepl_stress) mysql80 +name: Cluster (onlineddl_vrepl_stress) mysql57 on: [push, pull_request] concurrency: - group: format('{0}-{1}', ${{ github.ref }}, 'Cluster (onlineddl_vrepl_stress) mysql80') + group: format('{0}-{1}', ${{ github.ref }}, 'Cluster (onlineddl_vrepl_stress) mysql57') cancel-in-progress: true jobs: build: - name: Run endtoend tests on Cluster (onlineddl_vrepl_stress) mysql80 + name: Run endtoend tests on Cluster (onlineddl_vrepl_stress) mysql57 runs-on: ubuntu-20.04 + timeout-minutes: 45 steps: - name: Check if workflow needs to be skipped @@ -44,7 +45,7 @@ jobs: - 'tools/**' - 'config/**' - 'bootstrap.sh' - - '.github/workflows/cluster_endtoend_onlineddl_vrepl_stress_mysql80.yml' + - '.github/workflows/cluster_endtoend_onlineddl_vrepl_stress_mysql57.yml' - name: Set up Go if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' @@ -67,29 +68,43 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + sudo apt-get update + + # Uninstall any previously installed MySQL first + sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ + sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + + sudo systemctl stop apparmor + sudo DEBIAN_FRONTEND="noninteractive" apt-get remove -y --purge mysql-server mysql-client mysql-common + sudo apt-get -y autoremove + sudo apt-get -y autoclean + sudo deluser mysql + sudo rm -rf /var/lib/mysql + sudo rm -rf /etc/mysql + # Get key to latest MySQL repo sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 - # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb - echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.14-1_all.deb + # Bionic packages are still compatible for Focal since there's no MySQL 5.7 + # packages for Focal. + echo mysql-apt-config mysql-apt-config/repo-codename select bionic | sudo debconf-set-selections + echo mysql-apt-config mysql-apt-config/select-server select mysql-5.7 | sudo debconf-set-selections sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update + sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y mysql-client=5.7* mysql-community-server=5.7* mysql-server=5.7* # Install everything else we need, and configure - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get install -y make unzip g++ etcd curl git wget eatmydata sudo service mysql stop sudo service etcd stop - sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ - sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld - go mod download # install JUnit report formatter go install github.com/vitessio/go-junit-report@HEAD - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. diff --git a/.github/workflows/cluster_endtoend_onlineddl_vrepl_stress_suite.yml b/.github/workflows/cluster_endtoend_onlineddl_vrepl_stress_suite.yml index f3a9ced8c80..8d95fa3d6cd 100644 --- a/.github/workflows/cluster_endtoend_onlineddl_vrepl_stress_suite.yml +++ b/.github/workflows/cluster_endtoend_onlineddl_vrepl_stress_suite.yml @@ -9,7 +9,7 @@ concurrency: jobs: build: name: Run endtoend tests on Cluster (onlineddl_vrepl_stress_suite) - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - name: Check if workflow needs to be skipped @@ -59,6 +59,8 @@ jobs: - name: Tune the OS if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf @@ -67,10 +69,22 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + # Setup MySQL 8.0 + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata - sudo service mysql stop + # Install everything else we need, and configure + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get update + sudo service etcd stop + echo 'mysql version: ' + sudo mysql --version + sudo service mysql stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download @@ -80,7 +94,7 @@ jobs: - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. diff --git a/.github/workflows/cluster_endtoend_onlineddl_vrepl_stress_suite_mysql80.yml b/.github/workflows/cluster_endtoend_onlineddl_vrepl_stress_suite_mysql57.yml similarity index 76% rename from .github/workflows/cluster_endtoend_onlineddl_vrepl_stress_suite_mysql80.yml rename to .github/workflows/cluster_endtoend_onlineddl_vrepl_stress_suite_mysql57.yml index 2235e028127..64467b70579 100644 --- a/.github/workflows/cluster_endtoend_onlineddl_vrepl_stress_suite_mysql80.yml +++ b/.github/workflows/cluster_endtoend_onlineddl_vrepl_stress_suite_mysql57.yml @@ -1,15 +1,16 @@ # DO NOT MODIFY: THIS FILE IS GENERATED USING "make generate_ci_workflows" -name: Cluster (onlineddl_vrepl_stress_suite) mysql80 +name: Cluster (onlineddl_vrepl_stress_suite) mysql57 on: [push, pull_request] concurrency: - group: format('{0}-{1}', ${{ github.ref }}, 'Cluster (onlineddl_vrepl_stress_suite) mysql80') + group: format('{0}-{1}', ${{ github.ref }}, 'Cluster (onlineddl_vrepl_stress_suite) mysql57') cancel-in-progress: true jobs: build: - name: Run endtoend tests on Cluster (onlineddl_vrepl_stress_suite) mysql80 + name: Run endtoend tests on Cluster (onlineddl_vrepl_stress_suite) mysql57 runs-on: ubuntu-20.04 + timeout-minutes: 45 steps: - name: Check if workflow needs to be skipped @@ -44,7 +45,7 @@ jobs: - 'tools/**' - 'config/**' - 'bootstrap.sh' - - '.github/workflows/cluster_endtoend_onlineddl_vrepl_stress_suite_mysql80.yml' + - '.github/workflows/cluster_endtoend_onlineddl_vrepl_stress_suite_mysql57.yml' - name: Set up Go if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' @@ -67,29 +68,43 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + sudo apt-get update + + # Uninstall any previously installed MySQL first + sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ + sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + + sudo systemctl stop apparmor + sudo DEBIAN_FRONTEND="noninteractive" apt-get remove -y --purge mysql-server mysql-client mysql-common + sudo apt-get -y autoremove + sudo apt-get -y autoclean + sudo deluser mysql + sudo rm -rf /var/lib/mysql + sudo rm -rf /etc/mysql + # Get key to latest MySQL repo sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 - # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb - echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.14-1_all.deb + # Bionic packages are still compatible for Focal since there's no MySQL 5.7 + # packages for Focal. + echo mysql-apt-config mysql-apt-config/repo-codename select bionic | sudo debconf-set-selections + echo mysql-apt-config mysql-apt-config/select-server select mysql-5.7 | sudo debconf-set-selections sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update + sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y mysql-client=5.7* mysql-community-server=5.7* mysql-server=5.7* # Install everything else we need, and configure - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get install -y make unzip g++ etcd curl git wget eatmydata sudo service mysql stop sudo service etcd stop - sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ - sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld - go mod download # install JUnit report formatter go install github.com/vitessio/go-junit-report@HEAD - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. diff --git a/.github/workflows/cluster_endtoend_onlineddl_vrepl_suite.yml b/.github/workflows/cluster_endtoend_onlineddl_vrepl_suite.yml index 6c019a034ef..c41f9963005 100644 --- a/.github/workflows/cluster_endtoend_onlineddl_vrepl_suite.yml +++ b/.github/workflows/cluster_endtoend_onlineddl_vrepl_suite.yml @@ -9,7 +9,7 @@ concurrency: jobs: build: name: Run endtoend tests on Cluster (onlineddl_vrepl_suite) - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - name: Check if workflow needs to be skipped @@ -59,6 +59,8 @@ jobs: - name: Tune the OS if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf @@ -67,10 +69,22 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + # Setup MySQL 8.0 + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata - sudo service mysql stop + # Install everything else we need, and configure + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get update + sudo service etcd stop + echo 'mysql version: ' + sudo mysql --version + sudo service mysql stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download @@ -80,7 +94,7 @@ jobs: - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. diff --git a/.github/workflows/cluster_endtoend_onlineddl_vrepl_suite_mysql80.yml b/.github/workflows/cluster_endtoend_onlineddl_vrepl_suite_mysql57.yml similarity index 77% rename from .github/workflows/cluster_endtoend_onlineddl_vrepl_suite_mysql80.yml rename to .github/workflows/cluster_endtoend_onlineddl_vrepl_suite_mysql57.yml index 014f206b350..98fe7ff03d9 100644 --- a/.github/workflows/cluster_endtoend_onlineddl_vrepl_suite_mysql80.yml +++ b/.github/workflows/cluster_endtoend_onlineddl_vrepl_suite_mysql57.yml @@ -1,15 +1,16 @@ # DO NOT MODIFY: THIS FILE IS GENERATED USING "make generate_ci_workflows" -name: Cluster (onlineddl_vrepl_suite) mysql80 +name: Cluster (onlineddl_vrepl_suite) mysql57 on: [push, pull_request] concurrency: - group: format('{0}-{1}', ${{ github.ref }}, 'Cluster (onlineddl_vrepl_suite) mysql80') + group: format('{0}-{1}', ${{ github.ref }}, 'Cluster (onlineddl_vrepl_suite) mysql57') cancel-in-progress: true jobs: build: - name: Run endtoend tests on Cluster (onlineddl_vrepl_suite) mysql80 + name: Run endtoend tests on Cluster (onlineddl_vrepl_suite) mysql57 runs-on: ubuntu-20.04 + timeout-minutes: 45 steps: - name: Check if workflow needs to be skipped @@ -44,7 +45,7 @@ jobs: - 'tools/**' - 'config/**' - 'bootstrap.sh' - - '.github/workflows/cluster_endtoend_onlineddl_vrepl_suite_mysql80.yml' + - '.github/workflows/cluster_endtoend_onlineddl_vrepl_suite_mysql57.yml' - name: Set up Go if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' @@ -67,29 +68,43 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + sudo apt-get update + + # Uninstall any previously installed MySQL first + sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ + sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + + sudo systemctl stop apparmor + sudo DEBIAN_FRONTEND="noninteractive" apt-get remove -y --purge mysql-server mysql-client mysql-common + sudo apt-get -y autoremove + sudo apt-get -y autoclean + sudo deluser mysql + sudo rm -rf /var/lib/mysql + sudo rm -rf /etc/mysql + # Get key to latest MySQL repo sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 - # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb - echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.14-1_all.deb + # Bionic packages are still compatible for Focal since there's no MySQL 5.7 + # packages for Focal. + echo mysql-apt-config mysql-apt-config/repo-codename select bionic | sudo debconf-set-selections + echo mysql-apt-config mysql-apt-config/select-server select mysql-5.7 | sudo debconf-set-selections sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update + sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y mysql-client=5.7* mysql-community-server=5.7* mysql-server=5.7* # Install everything else we need, and configure - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get install -y make unzip g++ etcd curl git wget eatmydata sudo service mysql stop sudo service etcd stop - sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ - sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld - go mod download # install JUnit report formatter go install github.com/vitessio/go-junit-report@HEAD - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. diff --git a/.github/workflows/cluster_endtoend_resharding.yml b/.github/workflows/cluster_endtoend_resharding.yml index 003d3afbd19..73d1f214b9b 100644 --- a/.github/workflows/cluster_endtoend_resharding.yml +++ b/.github/workflows/cluster_endtoend_resharding.yml @@ -9,7 +9,7 @@ concurrency: jobs: build: name: Run endtoend tests on Cluster (resharding) - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - name: Check if workflow needs to be skipped @@ -59,6 +59,8 @@ jobs: - name: Tune the OS if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf @@ -67,10 +69,22 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + # Setup MySQL 8.0 + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata - sudo service mysql stop + # Install everything else we need, and configure + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get update + sudo service etcd stop + echo 'mysql version: ' + sudo mysql --version + sudo service mysql stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download @@ -80,7 +94,7 @@ jobs: - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. diff --git a/.github/workflows/cluster_endtoend_resharding_bytes.yml b/.github/workflows/cluster_endtoend_resharding_bytes.yml index ad6d19948a0..a7e9934f930 100644 --- a/.github/workflows/cluster_endtoend_resharding_bytes.yml +++ b/.github/workflows/cluster_endtoend_resharding_bytes.yml @@ -9,7 +9,7 @@ concurrency: jobs: build: name: Run endtoend tests on Cluster (resharding_bytes) - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - name: Check if workflow needs to be skipped @@ -59,6 +59,8 @@ jobs: - name: Tune the OS if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf @@ -67,10 +69,22 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + # Setup MySQL 8.0 + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata - sudo service mysql stop + # Install everything else we need, and configure + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get update + sudo service etcd stop + echo 'mysql version: ' + sudo mysql --version + sudo service mysql stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download @@ -80,7 +94,7 @@ jobs: - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. diff --git a/.github/workflows/cluster_endtoend_schemadiff_vrepl.yml b/.github/workflows/cluster_endtoend_schemadiff_vrepl.yml index 39f8669b6ed..493f319cb61 100644 --- a/.github/workflows/cluster_endtoend_schemadiff_vrepl.yml +++ b/.github/workflows/cluster_endtoend_schemadiff_vrepl.yml @@ -9,7 +9,7 @@ concurrency: jobs: build: name: Run endtoend tests on Cluster (schemadiff_vrepl) - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - name: Check if workflow needs to be skipped @@ -59,6 +59,8 @@ jobs: - name: Tune the OS if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf @@ -67,10 +69,22 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + # Setup MySQL 8.0 + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata - sudo service mysql stop + # Install everything else we need, and configure + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get update + sudo service etcd stop + echo 'mysql version: ' + sudo mysql --version + sudo service mysql stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download @@ -80,7 +94,7 @@ jobs: - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. diff --git a/.github/workflows/cluster_endtoend_schemadiff_vrepl_mysql80.yml b/.github/workflows/cluster_endtoend_schemadiff_vrepl_mysql57.yml similarity index 77% rename from .github/workflows/cluster_endtoend_schemadiff_vrepl_mysql80.yml rename to .github/workflows/cluster_endtoend_schemadiff_vrepl_mysql57.yml index b6101f6510f..cbf62731051 100644 --- a/.github/workflows/cluster_endtoend_schemadiff_vrepl_mysql80.yml +++ b/.github/workflows/cluster_endtoend_schemadiff_vrepl_mysql57.yml @@ -1,15 +1,16 @@ # DO NOT MODIFY: THIS FILE IS GENERATED USING "make generate_ci_workflows" -name: Cluster (schemadiff_vrepl) mysql80 +name: Cluster (schemadiff_vrepl) mysql57 on: [push, pull_request] concurrency: - group: format('{0}-{1}', ${{ github.ref }}, 'Cluster (schemadiff_vrepl) mysql80') + group: format('{0}-{1}', ${{ github.ref }}, 'Cluster (schemadiff_vrepl) mysql57') cancel-in-progress: true jobs: build: - name: Run endtoend tests on Cluster (schemadiff_vrepl) mysql80 + name: Run endtoend tests on Cluster (schemadiff_vrepl) mysql57 runs-on: ubuntu-20.04 + timeout-minutes: 45 steps: - name: Check if workflow needs to be skipped @@ -44,7 +45,7 @@ jobs: - 'tools/**' - 'config/**' - 'bootstrap.sh' - - '.github/workflows/cluster_endtoend_schemadiff_vrepl_mysql80.yml' + - '.github/workflows/cluster_endtoend_schemadiff_vrepl_mysql57.yml' - name: Set up Go if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' @@ -67,29 +68,43 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + sudo apt-get update + + # Uninstall any previously installed MySQL first + sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ + sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + + sudo systemctl stop apparmor + sudo DEBIAN_FRONTEND="noninteractive" apt-get remove -y --purge mysql-server mysql-client mysql-common + sudo apt-get -y autoremove + sudo apt-get -y autoclean + sudo deluser mysql + sudo rm -rf /var/lib/mysql + sudo rm -rf /etc/mysql + # Get key to latest MySQL repo sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 - # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb - echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.14-1_all.deb + # Bionic packages are still compatible for Focal since there's no MySQL 5.7 + # packages for Focal. + echo mysql-apt-config mysql-apt-config/repo-codename select bionic | sudo debconf-set-selections + echo mysql-apt-config mysql-apt-config/select-server select mysql-5.7 | sudo debconf-set-selections sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update + sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y mysql-client=5.7* mysql-community-server=5.7* mysql-server=5.7* # Install everything else we need, and configure - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get install -y make unzip g++ etcd curl git wget eatmydata sudo service mysql stop sudo service etcd stop - sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ - sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld - go mod download # install JUnit report formatter go install github.com/vitessio/go-junit-report@HEAD - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. diff --git a/.github/workflows/cluster_endtoend_shardedrecovery_stress_verticalsplit_heavy.yml b/.github/workflows/cluster_endtoend_shardedrecovery_stress_verticalsplit_heavy.yml index cab0296a653..7ee32da49aa 100644 --- a/.github/workflows/cluster_endtoend_shardedrecovery_stress_verticalsplit_heavy.yml +++ b/.github/workflows/cluster_endtoend_shardedrecovery_stress_verticalsplit_heavy.yml @@ -9,7 +9,7 @@ concurrency: jobs: build: name: Run endtoend tests on Cluster (shardedrecovery_stress_verticalsplit_heavy) - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - name: Check if workflow needs to be skipped @@ -59,6 +59,8 @@ jobs: - name: Tune the OS if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf @@ -67,10 +69,22 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + # Setup MySQL 8.0 + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata - sudo service mysql stop + # Install everything else we need, and configure + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get update + sudo service etcd stop + echo 'mysql version: ' + sudo mysql --version + sudo service mysql stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download @@ -80,7 +94,7 @@ jobs: - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. @@ -93,10 +107,11 @@ jobs: sudo sysctl -w net.ipv4.ip_local_port_range="22768 61999" # Increase our open file descriptor limit as we could hit this ulimit -n 65536 - cat <<-EOF>>./config/mycnf/mysql57.cnf + cat <<-EOF>>./config/mycnf/mysql80.cnf innodb_buffer_pool_dump_at_shutdown=OFF + innodb_buffer_pool_in_core_file=OFF innodb_buffer_pool_load_at_startup=OFF - innodb_buffer_pool_size=64M + innodb_buffer_pool_size=16M innodb_doublewrite=OFF innodb_flush_log_at_trx_commit=0 innodb_flush_method=O_DIRECT @@ -107,6 +122,7 @@ jobs: performance_schema=OFF slow-query-log=OFF EOF + make embed_config # run the tests however you normally do, then produce a JUnit XML file eatmydata -- go run test.go -docker=false -follow -shard shardedrecovery_stress_verticalsplit_heavy diff --git a/.github/workflows/cluster_endtoend_tabletmanager_consul.yml b/.github/workflows/cluster_endtoend_tabletmanager_consul.yml index 283eba2ceb2..8d27eeec8a0 100644 --- a/.github/workflows/cluster_endtoend_tabletmanager_consul.yml +++ b/.github/workflows/cluster_endtoend_tabletmanager_consul.yml @@ -9,7 +9,7 @@ concurrency: jobs: build: name: Run endtoend tests on Cluster (tabletmanager_consul) - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - name: Check if workflow needs to be skipped @@ -59,6 +59,8 @@ jobs: - name: Tune the OS if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf @@ -67,10 +69,22 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + # Setup MySQL 8.0 + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata - sudo service mysql stop + # Install everything else we need, and configure + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get update + sudo service etcd stop + echo 'mysql version: ' + sudo mysql --version + sudo service mysql stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download @@ -85,7 +99,7 @@ jobs: - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. diff --git a/.github/workflows/cluster_endtoend_tabletmanager_tablegc.yml b/.github/workflows/cluster_endtoend_tabletmanager_tablegc.yml index 341e4bc3df1..e3ecedf8c7f 100644 --- a/.github/workflows/cluster_endtoend_tabletmanager_tablegc.yml +++ b/.github/workflows/cluster_endtoend_tabletmanager_tablegc.yml @@ -9,7 +9,7 @@ concurrency: jobs: build: name: Run endtoend tests on Cluster (tabletmanager_tablegc) - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - name: Check if workflow needs to be skipped @@ -59,6 +59,8 @@ jobs: - name: Tune the OS if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf @@ -67,10 +69,22 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + # Setup MySQL 8.0 + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata - sudo service mysql stop + # Install everything else we need, and configure + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get update + sudo service etcd stop + echo 'mysql version: ' + sudo mysql --version + sudo service mysql stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download @@ -80,7 +94,7 @@ jobs: - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. diff --git a/.github/workflows/cluster_endtoend_tabletmanager_tablegc_mysql80.yml b/.github/workflows/cluster_endtoend_tabletmanager_tablegc_mysql57.yml similarity index 77% rename from .github/workflows/cluster_endtoend_tabletmanager_tablegc_mysql80.yml rename to .github/workflows/cluster_endtoend_tabletmanager_tablegc_mysql57.yml index 3c756410546..50337c243c8 100644 --- a/.github/workflows/cluster_endtoend_tabletmanager_tablegc_mysql80.yml +++ b/.github/workflows/cluster_endtoend_tabletmanager_tablegc_mysql57.yml @@ -1,15 +1,16 @@ # DO NOT MODIFY: THIS FILE IS GENERATED USING "make generate_ci_workflows" -name: Cluster (tabletmanager_tablegc) mysql80 +name: Cluster (tabletmanager_tablegc) mysql57 on: [push, pull_request] concurrency: - group: format('{0}-{1}', ${{ github.ref }}, 'Cluster (tabletmanager_tablegc) mysql80') + group: format('{0}-{1}', ${{ github.ref }}, 'Cluster (tabletmanager_tablegc) mysql57') cancel-in-progress: true jobs: build: - name: Run endtoend tests on Cluster (tabletmanager_tablegc) mysql80 + name: Run endtoend tests on Cluster (tabletmanager_tablegc) mysql57 runs-on: ubuntu-20.04 + timeout-minutes: 45 steps: - name: Check if workflow needs to be skipped @@ -44,7 +45,7 @@ jobs: - 'tools/**' - 'config/**' - 'bootstrap.sh' - - '.github/workflows/cluster_endtoend_tabletmanager_tablegc_mysql80.yml' + - '.github/workflows/cluster_endtoend_tabletmanager_tablegc_mysql57.yml' - name: Set up Go if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' @@ -67,29 +68,43 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + sudo apt-get update + + # Uninstall any previously installed MySQL first + sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ + sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + + sudo systemctl stop apparmor + sudo DEBIAN_FRONTEND="noninteractive" apt-get remove -y --purge mysql-server mysql-client mysql-common + sudo apt-get -y autoremove + sudo apt-get -y autoclean + sudo deluser mysql + sudo rm -rf /var/lib/mysql + sudo rm -rf /etc/mysql + # Get key to latest MySQL repo sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 - # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb - echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.14-1_all.deb + # Bionic packages are still compatible for Focal since there's no MySQL 5.7 + # packages for Focal. + echo mysql-apt-config mysql-apt-config/repo-codename select bionic | sudo debconf-set-selections + echo mysql-apt-config mysql-apt-config/select-server select mysql-5.7 | sudo debconf-set-selections sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update + sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y mysql-client=5.7* mysql-community-server=5.7* mysql-server=5.7* # Install everything else we need, and configure - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get install -y make unzip g++ etcd curl git wget eatmydata sudo service mysql stop sudo service etcd stop - sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ - sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld - go mod download # install JUnit report formatter go install github.com/vitessio/go-junit-report@HEAD - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. diff --git a/.github/workflows/cluster_endtoend_tabletmanager_throttler.yml b/.github/workflows/cluster_endtoend_tabletmanager_throttler.yml index ee4377e811c..3dce3c93745 100644 --- a/.github/workflows/cluster_endtoend_tabletmanager_throttler.yml +++ b/.github/workflows/cluster_endtoend_tabletmanager_throttler.yml @@ -9,7 +9,7 @@ concurrency: jobs: build: name: Run endtoend tests on Cluster (tabletmanager_throttler) - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - name: Check if workflow needs to be skipped @@ -59,6 +59,8 @@ jobs: - name: Tune the OS if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf @@ -67,10 +69,22 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + # Setup MySQL 8.0 + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata - sudo service mysql stop + # Install everything else we need, and configure + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get update + sudo service etcd stop + echo 'mysql version: ' + sudo mysql --version + sudo service mysql stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download @@ -80,7 +94,7 @@ jobs: - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. diff --git a/.github/workflows/cluster_endtoend_tabletmanager_throttler_custom_config.yml b/.github/workflows/cluster_endtoend_tabletmanager_throttler_custom_config.yml index 487a59cf073..901843d2416 100644 --- a/.github/workflows/cluster_endtoend_tabletmanager_throttler_custom_config.yml +++ b/.github/workflows/cluster_endtoend_tabletmanager_throttler_custom_config.yml @@ -9,7 +9,7 @@ concurrency: jobs: build: name: Run endtoend tests on Cluster (tabletmanager_throttler_custom_config) - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - name: Check if workflow needs to be skipped @@ -59,6 +59,8 @@ jobs: - name: Tune the OS if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf @@ -67,10 +69,22 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + # Setup MySQL 8.0 + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata - sudo service mysql stop + # Install everything else we need, and configure + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get update + sudo service etcd stop + echo 'mysql version: ' + sudo mysql --version + sudo service mysql stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download @@ -80,7 +94,7 @@ jobs: - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. diff --git a/.github/workflows/cluster_endtoend_topo_connection_cache.yml b/.github/workflows/cluster_endtoend_topo_connection_cache.yml index ce2b8ffc693..de097b16326 100644 --- a/.github/workflows/cluster_endtoend_topo_connection_cache.yml +++ b/.github/workflows/cluster_endtoend_topo_connection_cache.yml @@ -9,7 +9,7 @@ concurrency: jobs: build: name: Run endtoend tests on Cluster (topo_connection_cache) - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - name: Check if workflow needs to be skipped @@ -59,6 +59,8 @@ jobs: - name: Tune the OS if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf @@ -67,10 +69,22 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + # Setup MySQL 8.0 + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata - sudo service mysql stop + # Install everything else we need, and configure + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get update + sudo service etcd stop + echo 'mysql version: ' + sudo mysql --version + sudo service mysql stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download @@ -80,7 +94,7 @@ jobs: - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. diff --git a/.github/workflows/cluster_endtoend_vreplication_across_db_versions.yml b/.github/workflows/cluster_endtoend_vreplication_across_db_versions.yml index ff34a6f1b03..5d418533bf3 100644 --- a/.github/workflows/cluster_endtoend_vreplication_across_db_versions.yml +++ b/.github/workflows/cluster_endtoend_vreplication_across_db_versions.yml @@ -59,6 +59,8 @@ jobs: - name: Tune the OS if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf @@ -67,19 +69,22 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Get key to latest MySQL repo sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 - # Setup MySQL 8.0 wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update - # Install everything else we need, and configure sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils - sudo service mysql stop + sudo apt-get update + sudo service etcd stop + echo 'mysql version: ' + sudo mysql --version + sudo service mysql stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download @@ -89,7 +94,7 @@ jobs: - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. @@ -106,7 +111,7 @@ jobs: innodb_buffer_pool_dump_at_shutdown=OFF innodb_buffer_pool_in_core_file=OFF innodb_buffer_pool_load_at_startup=OFF - innodb_buffer_pool_size=64M + innodb_buffer_pool_size=16M innodb_doublewrite=OFF innodb_flush_log_at_trx_commit=0 innodb_flush_method=O_DIRECT @@ -117,6 +122,7 @@ jobs: performance_schema=OFF slow-query-log=OFF EOF + make embed_config # run the tests however you normally do, then produce a JUnit XML file eatmydata -- go run test.go -docker=false -follow -shard vreplication_across_db_versions diff --git a/.github/workflows/cluster_endtoend_vreplication_basic.yml b/.github/workflows/cluster_endtoend_vreplication_basic.yml index ac56390c03b..50be8f8fa36 100644 --- a/.github/workflows/cluster_endtoend_vreplication_basic.yml +++ b/.github/workflows/cluster_endtoend_vreplication_basic.yml @@ -9,7 +9,7 @@ concurrency: jobs: build: name: Run endtoend tests on Cluster (vreplication_basic) - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - name: Check if workflow needs to be skipped @@ -59,6 +59,8 @@ jobs: - name: Tune the OS if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf @@ -67,10 +69,22 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + # Setup MySQL 8.0 + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata - sudo service mysql stop + # Install everything else we need, and configure + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get update + sudo service etcd stop + echo 'mysql version: ' + sudo mysql --version + sudo service mysql stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download @@ -80,7 +94,7 @@ jobs: - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. @@ -93,10 +107,11 @@ jobs: sudo sysctl -w net.ipv4.ip_local_port_range="22768 61999" # Increase our open file descriptor limit as we could hit this ulimit -n 65536 - cat <<-EOF>>./config/mycnf/mysql57.cnf + cat <<-EOF>>./config/mycnf/mysql80.cnf innodb_buffer_pool_dump_at_shutdown=OFF + innodb_buffer_pool_in_core_file=OFF innodb_buffer_pool_load_at_startup=OFF - innodb_buffer_pool_size=64M + innodb_buffer_pool_size=16M innodb_doublewrite=OFF innodb_flush_log_at_trx_commit=0 innodb_flush_method=O_DIRECT @@ -107,6 +122,7 @@ jobs: performance_schema=OFF slow-query-log=OFF EOF + make embed_config # run the tests however you normally do, then produce a JUnit XML file eatmydata -- go run test.go -docker=false -follow -shard vreplication_basic diff --git a/.github/workflows/cluster_endtoend_vreplication_cellalias.yml b/.github/workflows/cluster_endtoend_vreplication_cellalias.yml index 56e9b585c71..c780684644c 100644 --- a/.github/workflows/cluster_endtoend_vreplication_cellalias.yml +++ b/.github/workflows/cluster_endtoend_vreplication_cellalias.yml @@ -9,7 +9,7 @@ concurrency: jobs: build: name: Run endtoend tests on Cluster (vreplication_cellalias) - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - name: Check if workflow needs to be skipped @@ -59,6 +59,8 @@ jobs: - name: Tune the OS if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf @@ -67,10 +69,22 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + # Setup MySQL 8.0 + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata - sudo service mysql stop + # Install everything else we need, and configure + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get update + sudo service etcd stop + echo 'mysql version: ' + sudo mysql --version + sudo service mysql stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download @@ -80,7 +94,7 @@ jobs: - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. @@ -93,10 +107,11 @@ jobs: sudo sysctl -w net.ipv4.ip_local_port_range="22768 61999" # Increase our open file descriptor limit as we could hit this ulimit -n 65536 - cat <<-EOF>>./config/mycnf/mysql57.cnf + cat <<-EOF>>./config/mycnf/mysql80.cnf innodb_buffer_pool_dump_at_shutdown=OFF + innodb_buffer_pool_in_core_file=OFF innodb_buffer_pool_load_at_startup=OFF - innodb_buffer_pool_size=64M + innodb_buffer_pool_size=16M innodb_doublewrite=OFF innodb_flush_log_at_trx_commit=0 innodb_flush_method=O_DIRECT @@ -107,6 +122,7 @@ jobs: performance_schema=OFF slow-query-log=OFF EOF + make embed_config # run the tests however you normally do, then produce a JUnit XML file eatmydata -- go run test.go -docker=false -follow -shard vreplication_cellalias diff --git a/.github/workflows/cluster_endtoend_vreplication_migrate.yml b/.github/workflows/cluster_endtoend_vreplication_migrate.yml index 871caa461ac..5537e802c1e 100644 --- a/.github/workflows/cluster_endtoend_vreplication_migrate.yml +++ b/.github/workflows/cluster_endtoend_vreplication_migrate.yml @@ -9,7 +9,7 @@ concurrency: jobs: build: name: Run endtoend tests on Cluster (vreplication_migrate) - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - name: Check if workflow needs to be skipped @@ -59,6 +59,8 @@ jobs: - name: Tune the OS if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf @@ -67,10 +69,22 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + # Setup MySQL 8.0 + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata - sudo service mysql stop + # Install everything else we need, and configure + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get update + sudo service etcd stop + echo 'mysql version: ' + sudo mysql --version + sudo service mysql stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download @@ -80,7 +94,7 @@ jobs: - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. @@ -93,10 +107,11 @@ jobs: sudo sysctl -w net.ipv4.ip_local_port_range="22768 61999" # Increase our open file descriptor limit as we could hit this ulimit -n 65536 - cat <<-EOF>>./config/mycnf/mysql57.cnf + cat <<-EOF>>./config/mycnf/mysql80.cnf innodb_buffer_pool_dump_at_shutdown=OFF + innodb_buffer_pool_in_core_file=OFF innodb_buffer_pool_load_at_startup=OFF - innodb_buffer_pool_size=64M + innodb_buffer_pool_size=16M innodb_doublewrite=OFF innodb_flush_log_at_trx_commit=0 innodb_flush_method=O_DIRECT @@ -107,6 +122,7 @@ jobs: performance_schema=OFF slow-query-log=OFF EOF + make embed_config # run the tests however you normally do, then produce a JUnit XML file eatmydata -- go run test.go -docker=false -follow -shard vreplication_migrate diff --git a/.github/workflows/cluster_endtoend_vreplication_migrate_vdiff2_convert_tz.yml b/.github/workflows/cluster_endtoend_vreplication_migrate_vdiff2_convert_tz.yml new file mode 100644 index 00000000000..6d5a08b2457 --- /dev/null +++ b/.github/workflows/cluster_endtoend_vreplication_migrate_vdiff2_convert_tz.yml @@ -0,0 +1,146 @@ +# DO NOT MODIFY: THIS FILE IS GENERATED USING "make generate_ci_workflows" + +name: Cluster (vreplication_migrate_vdiff2_convert_tz) +on: [push, pull_request] +concurrency: + group: format('{0}-{1}', ${{ github.ref }}, 'Cluster (vreplication_migrate_vdiff2_convert_tz)') + cancel-in-progress: true + +env: + LAUNCHABLE_ORGANIZATION: "vitess" + LAUNCHABLE_WORKSPACE: "vitess-app" + GITHUB_PR_HEAD_SHA: "${{ github.event.pull_request.head.sha }}" + +jobs: + build: + name: Run endtoend tests on Cluster (vreplication_migrate_vdiff2_convert_tz) + runs-on: ubuntu-20.04 + + steps: + - name: Check if workflow needs to be skipped + id: skip-workflow + run: | + skip='false' + if [[ "${{github.event.pull_request}}" == "" ]] && [[ "${{github.ref}}" != "refs/heads/main" ]] && [[ ! "${{github.ref}}" =~ ^refs/heads/release-[0-9]+\.[0-9]$ ]] && [[ ! "${{github.ref}}" =~ "refs/tags/.*" ]]; then + skip='true' + fi + echo Skip ${skip} + echo "::set-output name=skip-workflow::${skip}" + + - name: Check out code + if: steps.skip-workflow.outputs.skip-workflow == 'false' + uses: actions/checkout@v2 + + - name: Check for changes in relevant files + if: steps.skip-workflow.outputs.skip-workflow == 'false' + uses: frouioui/paths-filter@main + id: changes + with: + token: '' + filters: | + end_to_end: + - 'go/**/*.go' + - 'test.go' + - 'Makefile' + - 'build.env' + - 'go.[sumod]' + - 'proto/*.proto' + - 'tools/**' + - 'config/**' + - 'bootstrap.sh' + - '.github/workflows/cluster_endtoend_vreplication_migrate_vdiff2_convert_tz.yml' + + - name: Set up Go + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/setup-go@v2 + with: + go-version: 1.19.3 + + - name: Set up python + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/setup-python@v2 + + - name: Tune the OS + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range + # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio + echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf + sudo sysctl -p /etc/sysctl.conf + + - name: Get dependencies + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + # Setup Percona Server for MySQL 8.0 + sudo apt-get update + sudo apt-get install -y lsb-release gnupg2 curl + wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb + sudo percona-release setup ps80 + sudo apt-get update + + # Install everything else we need, and configure + sudo apt-get install -y percona-server-server percona-server-client make unzip g++ etcd git wget eatmydata xz-utils + sudo service mysql stop + sudo service etcd stop + sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ + sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + go mod download + + # install JUnit report formatter + go install github.com/vitessio/go-junit-report@HEAD + + - name: Setup launchable dependencies + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + # Get Launchable CLI installed. If you can, make it a part of the builder image to speed things up + pip3 install --user launchable~=1.0 > /dev/null + + # verify that launchable setup is all correct. + launchable verify || true + + # Tell Launchable about the build you are producing and testing + launchable record build --name "$GITHUB_RUN_ID" --source . + + - name: Run cluster endtoend test + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 45 + run: | + # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file + # which musn't be more than 107 characters long. + export VTDATAROOT="/tmp/" + source build.env + + set -x + + # Increase our local ephemeral port range as we could exhaust this + sudo sysctl -w net.ipv4.ip_local_port_range="22768 61999" + # Increase our open file descriptor limit as we could hit this + ulimit -n 65536 + cat <<-EOF>>./config/mycnf/mysql80.cnf + innodb_buffer_pool_dump_at_shutdown=OFF + innodb_buffer_pool_in_core_file=OFF + innodb_buffer_pool_load_at_startup=OFF + innodb_buffer_pool_size=64M + innodb_doublewrite=OFF + innodb_flush_log_at_trx_commit=0 + innodb_flush_method=O_DIRECT + innodb_numa_interleave=ON + innodb_adaptive_hash_index=OFF + sync_binlog=0 + sync_relay_log=0 + performance_schema=OFF + slow-query-log=OFF + EOF + + # run the tests however you normally do, then produce a JUnit XML file + eatmydata -- go run test.go -docker=false -follow -shard vreplication_migrate_vdiff2_convert_tz | tee -a output.txt | go-junit-report -set-exit-code > report.xml + + - name: Print test output and Record test result in launchable + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + run: | + # send recorded tests to launchable + launchable record tests --build "$GITHUB_RUN_ID" go-test . || true + + # print test output + cat output.txt diff --git a/.github/workflows/cluster_endtoend_vreplication_multicell.yml b/.github/workflows/cluster_endtoend_vreplication_multicell.yml index c38f084dbe1..7e8c33ee621 100644 --- a/.github/workflows/cluster_endtoend_vreplication_multicell.yml +++ b/.github/workflows/cluster_endtoend_vreplication_multicell.yml @@ -9,7 +9,7 @@ concurrency: jobs: build: name: Run endtoend tests on Cluster (vreplication_multicell) - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - name: Check if workflow needs to be skipped @@ -59,6 +59,8 @@ jobs: - name: Tune the OS if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf @@ -67,10 +69,22 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + # Setup MySQL 8.0 + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata - sudo service mysql stop + # Install everything else we need, and configure + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get update + sudo service etcd stop + echo 'mysql version: ' + sudo mysql --version + sudo service mysql stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download @@ -80,7 +94,7 @@ jobs: - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. @@ -93,10 +107,11 @@ jobs: sudo sysctl -w net.ipv4.ip_local_port_range="22768 61999" # Increase our open file descriptor limit as we could hit this ulimit -n 65536 - cat <<-EOF>>./config/mycnf/mysql57.cnf + cat <<-EOF>>./config/mycnf/mysql80.cnf innodb_buffer_pool_dump_at_shutdown=OFF + innodb_buffer_pool_in_core_file=OFF innodb_buffer_pool_load_at_startup=OFF - innodb_buffer_pool_size=64M + innodb_buffer_pool_size=16M innodb_doublewrite=OFF innodb_flush_log_at_trx_commit=0 innodb_flush_method=O_DIRECT @@ -107,6 +122,7 @@ jobs: performance_schema=OFF slow-query-log=OFF EOF + make embed_config # run the tests however you normally do, then produce a JUnit XML file eatmydata -- go run test.go -docker=false -follow -shard vreplication_multicell diff --git a/.github/workflows/cluster_endtoend_vreplication_v2.yml b/.github/workflows/cluster_endtoend_vreplication_v2.yml index 398e4f23d77..e30f49896d1 100644 --- a/.github/workflows/cluster_endtoend_vreplication_v2.yml +++ b/.github/workflows/cluster_endtoend_vreplication_v2.yml @@ -9,7 +9,7 @@ concurrency: jobs: build: name: Run endtoend tests on Cluster (vreplication_v2) - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - name: Check if workflow needs to be skipped @@ -59,6 +59,8 @@ jobs: - name: Tune the OS if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf @@ -67,10 +69,22 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + # Setup MySQL 8.0 + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata - sudo service mysql stop + # Install everything else we need, and configure + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get update + sudo service etcd stop + echo 'mysql version: ' + sudo mysql --version + sudo service mysql stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download @@ -80,7 +94,7 @@ jobs: - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. @@ -93,10 +107,11 @@ jobs: sudo sysctl -w net.ipv4.ip_local_port_range="22768 61999" # Increase our open file descriptor limit as we could hit this ulimit -n 65536 - cat <<-EOF>>./config/mycnf/mysql57.cnf + cat <<-EOF>>./config/mycnf/mysql80.cnf innodb_buffer_pool_dump_at_shutdown=OFF + innodb_buffer_pool_in_core_file=OFF innodb_buffer_pool_load_at_startup=OFF - innodb_buffer_pool_size=64M + innodb_buffer_pool_size=16M innodb_doublewrite=OFF innodb_flush_log_at_trx_commit=0 innodb_flush_method=O_DIRECT @@ -107,6 +122,7 @@ jobs: performance_schema=OFF slow-query-log=OFF EOF + make embed_config # run the tests however you normally do, then produce a JUnit XML file eatmydata -- go run test.go -docker=false -follow -shard vreplication_v2 diff --git a/.github/workflows/cluster_endtoend_vstream_failover.yml b/.github/workflows/cluster_endtoend_vstream_failover.yml index ae8c35816d9..19f9dcf7a34 100644 --- a/.github/workflows/cluster_endtoend_vstream_failover.yml +++ b/.github/workflows/cluster_endtoend_vstream_failover.yml @@ -9,7 +9,7 @@ concurrency: jobs: build: name: Run endtoend tests on Cluster (vstream_failover) - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - name: Check if workflow needs to be skipped @@ -59,6 +59,8 @@ jobs: - name: Tune the OS if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf @@ -67,10 +69,22 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + # Setup MySQL 8.0 + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata - sudo service mysql stop + # Install everything else we need, and configure + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get update + sudo service etcd stop + echo 'mysql version: ' + sudo mysql --version + sudo service mysql stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download @@ -80,7 +94,7 @@ jobs: - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. diff --git a/.github/workflows/cluster_endtoend_vstream_stoponreshard_false.yml b/.github/workflows/cluster_endtoend_vstream_stoponreshard_false.yml index 09c64a014a9..920cbc063d3 100644 --- a/.github/workflows/cluster_endtoend_vstream_stoponreshard_false.yml +++ b/.github/workflows/cluster_endtoend_vstream_stoponreshard_false.yml @@ -9,7 +9,7 @@ concurrency: jobs: build: name: Run endtoend tests on Cluster (vstream_stoponreshard_false) - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - name: Check if workflow needs to be skipped @@ -59,6 +59,8 @@ jobs: - name: Tune the OS if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf @@ -67,10 +69,22 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + # Setup MySQL 8.0 + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata - sudo service mysql stop + # Install everything else we need, and configure + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get update + sudo service etcd stop + echo 'mysql version: ' + sudo mysql --version + sudo service mysql stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download @@ -80,7 +94,7 @@ jobs: - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. diff --git a/.github/workflows/cluster_endtoend_vstream_stoponreshard_true.yml b/.github/workflows/cluster_endtoend_vstream_stoponreshard_true.yml index d2783576595..4018ff284bd 100644 --- a/.github/workflows/cluster_endtoend_vstream_stoponreshard_true.yml +++ b/.github/workflows/cluster_endtoend_vstream_stoponreshard_true.yml @@ -9,7 +9,7 @@ concurrency: jobs: build: name: Run endtoend tests on Cluster (vstream_stoponreshard_true) - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - name: Check if workflow needs to be skipped @@ -59,6 +59,8 @@ jobs: - name: Tune the OS if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf @@ -67,10 +69,22 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + # Setup MySQL 8.0 + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata - sudo service mysql stop + # Install everything else we need, and configure + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get update + sudo service etcd stop + echo 'mysql version: ' + sudo mysql --version + sudo service mysql stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download @@ -80,7 +94,7 @@ jobs: - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. diff --git a/.github/workflows/cluster_endtoend_vstream_with_keyspaces_to_watch.yml b/.github/workflows/cluster_endtoend_vstream_with_keyspaces_to_watch.yml index f1fe80941d6..bca9e639c83 100644 --- a/.github/workflows/cluster_endtoend_vstream_with_keyspaces_to_watch.yml +++ b/.github/workflows/cluster_endtoend_vstream_with_keyspaces_to_watch.yml @@ -9,7 +9,7 @@ concurrency: jobs: build: name: Run endtoend tests on Cluster (vstream_with_keyspaces_to_watch) - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - name: Check if workflow needs to be skipped @@ -59,6 +59,8 @@ jobs: - name: Tune the OS if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf @@ -67,10 +69,22 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + # Setup MySQL 8.0 + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata - sudo service mysql stop + # Install everything else we need, and configure + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get update + sudo service etcd stop + echo 'mysql version: ' + sudo mysql --version + sudo service mysql stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download @@ -80,7 +94,7 @@ jobs: - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. diff --git a/.github/workflows/cluster_endtoend_vtbackup_transform.yml b/.github/workflows/cluster_endtoend_vtbackup_transform.yml new file mode 100644 index 00000000000..db8f001732a --- /dev/null +++ b/.github/workflows/cluster_endtoend_vtbackup_transform.yml @@ -0,0 +1,126 @@ +# DO NOT MODIFY: THIS FILE IS GENERATED USING "make generate_ci_workflows" + +name: Cluster (vtbackup_transform) +on: [push, pull_request] +concurrency: + group: format('{0}-{1}', ${{ github.ref }}, 'Cluster (vtbackup_transform)') + cancel-in-progress: true + +env: + LAUNCHABLE_ORGANIZATION: "vitess" + LAUNCHABLE_WORKSPACE: "vitess-app" + GITHUB_PR_HEAD_SHA: "${{ github.event.pull_request.head.sha }}" + +jobs: + build: + name: Run endtoend tests on Cluster (vtbackup_transform) + runs-on: ubuntu-20.04 + + steps: + - name: Check if workflow needs to be skipped + id: skip-workflow + run: | + skip='false' + if [[ "${{github.event.pull_request}}" == "" ]] && [[ "${{github.ref}}" != "refs/heads/main" ]] && [[ ! "${{github.ref}}" =~ ^refs/heads/release-[0-9]+\.[0-9]$ ]] && [[ ! "${{github.ref}}" =~ "refs/tags/.*" ]]; then + skip='true' + fi + echo Skip ${skip} + echo "::set-output name=skip-workflow::${skip}" + + - name: Check out code + if: steps.skip-workflow.outputs.skip-workflow == 'false' + uses: actions/checkout@v2 + + - name: Check for changes in relevant files + if: steps.skip-workflow.outputs.skip-workflow == 'false' + uses: frouioui/paths-filter@main + id: changes + with: + token: '' + filters: | + end_to_end: + - 'go/**/*.go' + - 'test.go' + - 'Makefile' + - 'build.env' + - 'go.[sumod]' + - 'proto/*.proto' + - 'tools/**' + - 'config/**' + - 'bootstrap.sh' + - '.github/workflows/cluster_endtoend_vtbackup_transform.yml' + + - name: Set up Go + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/setup-go@v2 + with: + go-version: 1.19.3 + + - name: Set up python + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/setup-python@v2 + + - name: Tune the OS + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range + # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio + echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf + sudo sysctl -p /etc/sysctl.conf + + - name: Get dependencies + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + # Setup Percona Server for MySQL 8.0 + sudo apt-get update + sudo apt-get install -y lsb-release gnupg2 curl + wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb + sudo percona-release setup ps80 + sudo apt-get update + + # Install everything else we need, and configure + sudo apt-get install -y percona-server-server percona-server-client make unzip g++ etcd git wget eatmydata xz-utils + sudo service mysql stop + sudo service etcd stop + sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ + sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + go mod download + + # install JUnit report formatter + go install github.com/vitessio/go-junit-report@HEAD + + - name: Setup launchable dependencies + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + # Get Launchable CLI installed. If you can, make it a part of the builder image to speed things up + pip3 install --user launchable~=1.0 > /dev/null + + # verify that launchable setup is all correct. + launchable verify || true + + # Tell Launchable about the build you are producing and testing + launchable record build --name "$GITHUB_RUN_ID" --source . + + - name: Run cluster endtoend test + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 45 + run: | + # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file + # which musn't be more than 107 characters long. + export VTDATAROOT="/tmp/" + source build.env + + set -x + + # run the tests however you normally do, then produce a JUnit XML file + eatmydata -- go run test.go -docker=false -follow -shard vtbackup_transform | tee -a output.txt | go-junit-report -set-exit-code > report.xml + + - name: Print test output and Record test result in launchable + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + run: | + # send recorded tests to launchable + launchable record tests --build "$GITHUB_RUN_ID" go-test . || true + + # print test output + cat output.txt diff --git a/.github/workflows/cluster_endtoend_vtctlbackup_sharded_clustertest_heavy.yml b/.github/workflows/cluster_endtoend_vtctlbackup_sharded_clustertest_heavy.yml index c9dfdfb01c6..db2754f7eb6 100644 --- a/.github/workflows/cluster_endtoend_vtctlbackup_sharded_clustertest_heavy.yml +++ b/.github/workflows/cluster_endtoend_vtctlbackup_sharded_clustertest_heavy.yml @@ -9,7 +9,7 @@ concurrency: jobs: build: name: Run endtoend tests on Cluster (vtctlbackup_sharded_clustertest_heavy) - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - name: Check if workflow needs to be skipped @@ -59,6 +59,8 @@ jobs: - name: Tune the OS if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf @@ -67,10 +69,22 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + # Setup MySQL 8.0 + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata - sudo service mysql stop + # Install everything else we need, and configure + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get update + sudo service etcd stop + echo 'mysql version: ' + sudo mysql --version + sudo service mysql stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download @@ -80,7 +94,7 @@ jobs: - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. @@ -93,10 +107,11 @@ jobs: sudo sysctl -w net.ipv4.ip_local_port_range="22768 61999" # Increase our open file descriptor limit as we could hit this ulimit -n 65536 - cat <<-EOF>>./config/mycnf/mysql57.cnf + cat <<-EOF>>./config/mycnf/mysql80.cnf innodb_buffer_pool_dump_at_shutdown=OFF + innodb_buffer_pool_in_core_file=OFF innodb_buffer_pool_load_at_startup=OFF - innodb_buffer_pool_size=64M + innodb_buffer_pool_size=16M innodb_doublewrite=OFF innodb_flush_log_at_trx_commit=0 innodb_flush_method=O_DIRECT @@ -107,6 +122,7 @@ jobs: performance_schema=OFF slow-query-log=OFF EOF + make embed_config # run the tests however you normally do, then produce a JUnit XML file eatmydata -- go run test.go -docker=false -follow -shard vtctlbackup_sharded_clustertest_heavy diff --git a/.github/workflows/cluster_endtoend_vtgate_concurrentdml.yml b/.github/workflows/cluster_endtoend_vtgate_concurrentdml.yml index 951526d6266..85a23db62b9 100644 --- a/.github/workflows/cluster_endtoend_vtgate_concurrentdml.yml +++ b/.github/workflows/cluster_endtoend_vtgate_concurrentdml.yml @@ -9,7 +9,7 @@ concurrency: jobs: build: name: Run endtoend tests on Cluster (vtgate_concurrentdml) - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - name: Check if workflow needs to be skipped @@ -59,6 +59,8 @@ jobs: - name: Tune the OS if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf @@ -67,10 +69,22 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + # Setup MySQL 8.0 + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata - sudo service mysql stop + # Install everything else we need, and configure + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get update + sudo service etcd stop + echo 'mysql version: ' + sudo mysql --version + sudo service mysql stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download @@ -80,7 +94,7 @@ jobs: - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. diff --git a/.github/workflows/cluster_endtoend_vtgate_gen4.yml b/.github/workflows/cluster_endtoend_vtgate_gen4.yml index 91d65db4eea..4b180c20964 100644 --- a/.github/workflows/cluster_endtoend_vtgate_gen4.yml +++ b/.github/workflows/cluster_endtoend_vtgate_gen4.yml @@ -9,7 +9,7 @@ concurrency: jobs: build: name: Run endtoend tests on Cluster (vtgate_gen4) - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - name: Check if workflow needs to be skipped @@ -59,6 +59,8 @@ jobs: - name: Tune the OS if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf @@ -67,10 +69,22 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + # Setup MySQL 8.0 + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata - sudo service mysql stop + # Install everything else we need, and configure + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get update + sudo service etcd stop + echo 'mysql version: ' + sudo mysql --version + sudo service mysql stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download @@ -80,7 +94,7 @@ jobs: - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. diff --git a/.github/workflows/cluster_endtoend_vtgate_general_heavy.yml b/.github/workflows/cluster_endtoend_vtgate_general_heavy.yml index ebbc3781054..6182bd869d2 100644 --- a/.github/workflows/cluster_endtoend_vtgate_general_heavy.yml +++ b/.github/workflows/cluster_endtoend_vtgate_general_heavy.yml @@ -9,7 +9,7 @@ concurrency: jobs: build: name: Run endtoend tests on Cluster (vtgate_general_heavy) - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - name: Check if workflow needs to be skipped @@ -59,6 +59,8 @@ jobs: - name: Tune the OS if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf @@ -67,10 +69,22 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + # Setup MySQL 8.0 + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata - sudo service mysql stop + # Install everything else we need, and configure + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get update + sudo service etcd stop + echo 'mysql version: ' + sudo mysql --version + sudo service mysql stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download @@ -80,7 +94,7 @@ jobs: - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. @@ -93,10 +107,11 @@ jobs: sudo sysctl -w net.ipv4.ip_local_port_range="22768 61999" # Increase our open file descriptor limit as we could hit this ulimit -n 65536 - cat <<-EOF>>./config/mycnf/mysql57.cnf + cat <<-EOF>>./config/mycnf/mysql80.cnf innodb_buffer_pool_dump_at_shutdown=OFF + innodb_buffer_pool_in_core_file=OFF innodb_buffer_pool_load_at_startup=OFF - innodb_buffer_pool_size=64M + innodb_buffer_pool_size=16M innodb_doublewrite=OFF innodb_flush_log_at_trx_commit=0 innodb_flush_method=O_DIRECT @@ -107,6 +122,7 @@ jobs: performance_schema=OFF slow-query-log=OFF EOF + make embed_config # run the tests however you normally do, then produce a JUnit XML file eatmydata -- go run test.go -docker=false -follow -shard vtgate_general_heavy diff --git a/.github/workflows/cluster_endtoend_vtgate_godriver.yml b/.github/workflows/cluster_endtoend_vtgate_godriver.yml index bf70cf3fa08..7d7ab5bb63f 100644 --- a/.github/workflows/cluster_endtoend_vtgate_godriver.yml +++ b/.github/workflows/cluster_endtoend_vtgate_godriver.yml @@ -9,7 +9,7 @@ concurrency: jobs: build: name: Run endtoend tests on Cluster (vtgate_godriver) - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - name: Check if workflow needs to be skipped @@ -59,6 +59,8 @@ jobs: - name: Tune the OS if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf @@ -67,10 +69,22 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + # Setup MySQL 8.0 + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata - sudo service mysql stop + # Install everything else we need, and configure + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get update + sudo service etcd stop + echo 'mysql version: ' + sudo mysql --version + sudo service mysql stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download @@ -80,7 +94,7 @@ jobs: - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. diff --git a/.github/workflows/cluster_endtoend_vtgate_partial_keyspace.yml b/.github/workflows/cluster_endtoend_vtgate_partial_keyspace.yml new file mode 100644 index 00000000000..153b8ff0330 --- /dev/null +++ b/.github/workflows/cluster_endtoend_vtgate_partial_keyspace.yml @@ -0,0 +1,126 @@ +# DO NOT MODIFY: THIS FILE IS GENERATED USING "make generate_ci_workflows" + +name: Cluster (vtgate_partial_keyspace) +on: [push, pull_request] +concurrency: + group: format('{0}-{1}', ${{ github.ref }}, 'Cluster (vtgate_partial_keyspace)') + cancel-in-progress: true + +env: + LAUNCHABLE_ORGANIZATION: "vitess" + LAUNCHABLE_WORKSPACE: "vitess-app" + GITHUB_PR_HEAD_SHA: "${{ github.event.pull_request.head.sha }}" + +jobs: + build: + name: Run endtoend tests on Cluster (vtgate_partial_keyspace) + runs-on: ubuntu-20.04 + + steps: + - name: Check if workflow needs to be skipped + id: skip-workflow + run: | + skip='false' + if [[ "${{github.event.pull_request}}" == "" ]] && [[ "${{github.ref}}" != "refs/heads/main" ]] && [[ ! "${{github.ref}}" =~ ^refs/heads/release-[0-9]+\.[0-9]$ ]] && [[ ! "${{github.ref}}" =~ "refs/tags/.*" ]]; then + skip='true' + fi + echo Skip ${skip} + echo "::set-output name=skip-workflow::${skip}" + + - name: Check out code + if: steps.skip-workflow.outputs.skip-workflow == 'false' + uses: actions/checkout@v2 + + - name: Check for changes in relevant files + if: steps.skip-workflow.outputs.skip-workflow == 'false' + uses: frouioui/paths-filter@main + id: changes + with: + token: '' + filters: | + end_to_end: + - 'go/**/*.go' + - 'test.go' + - 'Makefile' + - 'build.env' + - 'go.[sumod]' + - 'proto/*.proto' + - 'tools/**' + - 'config/**' + - 'bootstrap.sh' + - '.github/workflows/cluster_endtoend_vtgate_partial_keyspace.yml' + + - name: Set up Go + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/setup-go@v2 + with: + go-version: 1.19.3 + + - name: Set up python + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/setup-python@v2 + + - name: Tune the OS + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range + # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio + echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf + sudo sysctl -p /etc/sysctl.conf + + - name: Get dependencies + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + # Setup Percona Server for MySQL 8.0 + sudo apt-get update + sudo apt-get install -y lsb-release gnupg2 curl + wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb + sudo percona-release setup ps80 + sudo apt-get update + + # Install everything else we need, and configure + sudo apt-get install -y percona-server-server percona-server-client make unzip g++ etcd git wget eatmydata xz-utils + sudo service mysql stop + sudo service etcd stop + sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ + sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + go mod download + + # install JUnit report formatter + go install github.com/vitessio/go-junit-report@HEAD + + - name: Setup launchable dependencies + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + # Get Launchable CLI installed. If you can, make it a part of the builder image to speed things up + pip3 install --user launchable~=1.0 > /dev/null + + # verify that launchable setup is all correct. + launchable verify || true + + # Tell Launchable about the build you are producing and testing + launchable record build --name "$GITHUB_RUN_ID" --source . + + - name: Run cluster endtoend test + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 45 + run: | + # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file + # which musn't be more than 107 characters long. + export VTDATAROOT="/tmp/" + source build.env + + set -x + + # run the tests however you normally do, then produce a JUnit XML file + eatmydata -- go run test.go -docker=false -follow -shard vtgate_partial_keyspace -partial-keyspace=true | tee -a output.txt | go-junit-report -set-exit-code > report.xml + + - name: Print test output and Record test result in launchable + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + run: | + # send recorded tests to launchable + launchable record tests --build "$GITHUB_RUN_ID" go-test . || true + + # print test output + cat output.txt diff --git a/.github/workflows/cluster_endtoend_vtgate_queries.yml b/.github/workflows/cluster_endtoend_vtgate_queries.yml index 6a79e226003..9ba1daba13b 100644 --- a/.github/workflows/cluster_endtoend_vtgate_queries.yml +++ b/.github/workflows/cluster_endtoend_vtgate_queries.yml @@ -9,7 +9,7 @@ concurrency: jobs: build: name: Run endtoend tests on Cluster (vtgate_queries) - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - name: Check if workflow needs to be skipped @@ -59,6 +59,8 @@ jobs: - name: Tune the OS if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf @@ -67,10 +69,22 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + # Setup MySQL 8.0 + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata - sudo service mysql stop + # Install everything else we need, and configure + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get update + sudo service etcd stop + echo 'mysql version: ' + sudo mysql --version + sudo service mysql stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download @@ -80,7 +94,7 @@ jobs: - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. diff --git a/.github/workflows/cluster_endtoend_vtgate_readafterwrite.yml b/.github/workflows/cluster_endtoend_vtgate_readafterwrite.yml index 564e84d9130..b593bb20584 100644 --- a/.github/workflows/cluster_endtoend_vtgate_readafterwrite.yml +++ b/.github/workflows/cluster_endtoend_vtgate_readafterwrite.yml @@ -9,7 +9,7 @@ concurrency: jobs: build: name: Run endtoend tests on Cluster (vtgate_readafterwrite) - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - name: Check if workflow needs to be skipped @@ -59,6 +59,8 @@ jobs: - name: Tune the OS if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf @@ -67,10 +69,22 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + # Setup MySQL 8.0 + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata - sudo service mysql stop + # Install everything else we need, and configure + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get update + sudo service etcd stop + echo 'mysql version: ' + sudo mysql --version + sudo service mysql stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download @@ -80,7 +94,7 @@ jobs: - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. diff --git a/.github/workflows/cluster_endtoend_vtgate_reservedconn.yml b/.github/workflows/cluster_endtoend_vtgate_reservedconn.yml index 6dbec0b1cf2..35cef4e0f47 100644 --- a/.github/workflows/cluster_endtoend_vtgate_reservedconn.yml +++ b/.github/workflows/cluster_endtoend_vtgate_reservedconn.yml @@ -9,7 +9,7 @@ concurrency: jobs: build: name: Run endtoend tests on Cluster (vtgate_reservedconn) - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - name: Check if workflow needs to be skipped @@ -59,6 +59,8 @@ jobs: - name: Tune the OS if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf @@ -67,10 +69,22 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + # Setup MySQL 8.0 + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata - sudo service mysql stop + # Install everything else we need, and configure + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get update + sudo service etcd stop + echo 'mysql version: ' + sudo mysql --version + sudo service mysql stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download @@ -80,7 +94,7 @@ jobs: - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. diff --git a/.github/workflows/cluster_endtoend_vtgate_schema.yml b/.github/workflows/cluster_endtoend_vtgate_schema.yml index 6856e446b24..68643475489 100644 --- a/.github/workflows/cluster_endtoend_vtgate_schema.yml +++ b/.github/workflows/cluster_endtoend_vtgate_schema.yml @@ -9,7 +9,7 @@ concurrency: jobs: build: name: Run endtoend tests on Cluster (vtgate_schema) - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - name: Check if workflow needs to be skipped @@ -59,6 +59,8 @@ jobs: - name: Tune the OS if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf @@ -67,10 +69,22 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + # Setup MySQL 8.0 + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata - sudo service mysql stop + # Install everything else we need, and configure + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get update + sudo service etcd stop + echo 'mysql version: ' + sudo mysql --version + sudo service mysql stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download @@ -80,7 +94,7 @@ jobs: - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. diff --git a/.github/workflows/cluster_endtoend_vtgate_schema_tracker.yml b/.github/workflows/cluster_endtoend_vtgate_schema_tracker.yml index 3405a4f993d..7a2471ed763 100644 --- a/.github/workflows/cluster_endtoend_vtgate_schema_tracker.yml +++ b/.github/workflows/cluster_endtoend_vtgate_schema_tracker.yml @@ -9,7 +9,7 @@ concurrency: jobs: build: name: Run endtoend tests on Cluster (vtgate_schema_tracker) - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - name: Check if workflow needs to be skipped @@ -59,6 +59,8 @@ jobs: - name: Tune the OS if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf @@ -67,10 +69,22 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + # Setup MySQL 8.0 + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata - sudo service mysql stop + # Install everything else we need, and configure + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get update + sudo service etcd stop + echo 'mysql version: ' + sudo mysql --version + sudo service mysql stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download @@ -80,7 +94,7 @@ jobs: - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. diff --git a/.github/workflows/cluster_endtoend_vtgate_tablet_healthcheck_cache.yml b/.github/workflows/cluster_endtoend_vtgate_tablet_healthcheck_cache.yml index 17f64de524f..95b89589c20 100644 --- a/.github/workflows/cluster_endtoend_vtgate_tablet_healthcheck_cache.yml +++ b/.github/workflows/cluster_endtoend_vtgate_tablet_healthcheck_cache.yml @@ -9,7 +9,7 @@ concurrency: jobs: build: name: Run endtoend tests on Cluster (vtgate_tablet_healthcheck_cache) - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - name: Check if workflow needs to be skipped @@ -59,6 +59,8 @@ jobs: - name: Tune the OS if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf @@ -67,10 +69,22 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + # Setup MySQL 8.0 + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata - sudo service mysql stop + # Install everything else we need, and configure + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get update + sudo service etcd stop + echo 'mysql version: ' + sudo mysql --version + sudo service mysql stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download @@ -80,7 +94,7 @@ jobs: - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. diff --git a/.github/workflows/cluster_endtoend_vtgate_topo.yml b/.github/workflows/cluster_endtoend_vtgate_topo.yml index 455b4fbd6ab..00dee5a229c 100644 --- a/.github/workflows/cluster_endtoend_vtgate_topo.yml +++ b/.github/workflows/cluster_endtoend_vtgate_topo.yml @@ -9,7 +9,7 @@ concurrency: jobs: build: name: Run endtoend tests on Cluster (vtgate_topo) - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - name: Check if workflow needs to be skipped @@ -59,6 +59,8 @@ jobs: - name: Tune the OS if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf @@ -67,10 +69,22 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + # Setup MySQL 8.0 + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata - sudo service mysql stop + # Install everything else we need, and configure + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get update + sudo service etcd stop + echo 'mysql version: ' + sudo mysql --version + sudo service mysql stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download @@ -80,7 +94,7 @@ jobs: - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. diff --git a/.github/workflows/cluster_endtoend_vtgate_topo_consul.yml b/.github/workflows/cluster_endtoend_vtgate_topo_consul.yml index beb213b8df3..c4120c553eb 100644 --- a/.github/workflows/cluster_endtoend_vtgate_topo_consul.yml +++ b/.github/workflows/cluster_endtoend_vtgate_topo_consul.yml @@ -9,7 +9,7 @@ concurrency: jobs: build: name: Run endtoend tests on Cluster (vtgate_topo_consul) - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - name: Check if workflow needs to be skipped @@ -59,6 +59,8 @@ jobs: - name: Tune the OS if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf @@ -67,10 +69,22 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + # Setup MySQL 8.0 + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata - sudo service mysql stop + # Install everything else we need, and configure + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get update + sudo service etcd stop + echo 'mysql version: ' + sudo mysql --version + sudo service mysql stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download @@ -85,7 +99,7 @@ jobs: - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. diff --git a/.github/workflows/cluster_endtoend_vtgate_topo_etcd.yml b/.github/workflows/cluster_endtoend_vtgate_topo_etcd.yml index 16cc9e85756..3e505fe37d5 100644 --- a/.github/workflows/cluster_endtoend_vtgate_topo_etcd.yml +++ b/.github/workflows/cluster_endtoend_vtgate_topo_etcd.yml @@ -9,7 +9,7 @@ concurrency: jobs: build: name: Run endtoend tests on Cluster (vtgate_topo_etcd) - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - name: Check if workflow needs to be skipped @@ -59,6 +59,8 @@ jobs: - name: Tune the OS if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf @@ -67,10 +69,22 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + # Setup MySQL 8.0 + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata - sudo service mysql stop + # Install everything else we need, and configure + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get update + sudo service etcd stop + echo 'mysql version: ' + sudo mysql --version + sudo service mysql stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download @@ -80,7 +94,7 @@ jobs: - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. diff --git a/.github/workflows/cluster_endtoend_vtgate_transaction.yml b/.github/workflows/cluster_endtoend_vtgate_transaction.yml index e8b321d01e6..da75648ced8 100644 --- a/.github/workflows/cluster_endtoend_vtgate_transaction.yml +++ b/.github/workflows/cluster_endtoend_vtgate_transaction.yml @@ -9,7 +9,7 @@ concurrency: jobs: build: name: Run endtoend tests on Cluster (vtgate_transaction) - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - name: Check if workflow needs to be skipped @@ -59,6 +59,8 @@ jobs: - name: Tune the OS if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf @@ -67,10 +69,22 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + # Setup MySQL 8.0 + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata - sudo service mysql stop + # Install everything else we need, and configure + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get update + sudo service etcd stop + echo 'mysql version: ' + sudo mysql --version + sudo service mysql stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download @@ -80,7 +94,7 @@ jobs: - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. diff --git a/.github/workflows/cluster_endtoend_vtgate_unsharded.yml b/.github/workflows/cluster_endtoend_vtgate_unsharded.yml index 7e70cc12215..b7cea310587 100644 --- a/.github/workflows/cluster_endtoend_vtgate_unsharded.yml +++ b/.github/workflows/cluster_endtoend_vtgate_unsharded.yml @@ -9,7 +9,7 @@ concurrency: jobs: build: name: Run endtoend tests on Cluster (vtgate_unsharded) - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - name: Check if workflow needs to be skipped @@ -59,6 +59,8 @@ jobs: - name: Tune the OS if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf @@ -67,10 +69,22 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + # Setup MySQL 8.0 + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata - sudo service mysql stop + # Install everything else we need, and configure + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get update + sudo service etcd stop + echo 'mysql version: ' + sudo mysql --version + sudo service mysql stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download @@ -80,7 +94,7 @@ jobs: - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. diff --git a/.github/workflows/cluster_endtoend_vtgate_vindex_heavy.yml b/.github/workflows/cluster_endtoend_vtgate_vindex_heavy.yml index 81af9209d08..b9e642a7f9c 100644 --- a/.github/workflows/cluster_endtoend_vtgate_vindex_heavy.yml +++ b/.github/workflows/cluster_endtoend_vtgate_vindex_heavy.yml @@ -9,7 +9,7 @@ concurrency: jobs: build: name: Run endtoend tests on Cluster (vtgate_vindex_heavy) - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - name: Check if workflow needs to be skipped @@ -59,6 +59,8 @@ jobs: - name: Tune the OS if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf @@ -67,10 +69,22 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + # Setup MySQL 8.0 + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata - sudo service mysql stop + # Install everything else we need, and configure + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get update + sudo service etcd stop + echo 'mysql version: ' + sudo mysql --version + sudo service mysql stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download @@ -80,7 +94,7 @@ jobs: - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. @@ -93,10 +107,11 @@ jobs: sudo sysctl -w net.ipv4.ip_local_port_range="22768 61999" # Increase our open file descriptor limit as we could hit this ulimit -n 65536 - cat <<-EOF>>./config/mycnf/mysql57.cnf + cat <<-EOF>>./config/mycnf/mysql80.cnf innodb_buffer_pool_dump_at_shutdown=OFF + innodb_buffer_pool_in_core_file=OFF innodb_buffer_pool_load_at_startup=OFF - innodb_buffer_pool_size=64M + innodb_buffer_pool_size=16M innodb_doublewrite=OFF innodb_flush_log_at_trx_commit=0 innodb_flush_method=O_DIRECT @@ -107,6 +122,7 @@ jobs: performance_schema=OFF slow-query-log=OFF EOF + make embed_config # run the tests however you normally do, then produce a JUnit XML file eatmydata -- go run test.go -docker=false -follow -shard vtgate_vindex_heavy diff --git a/.github/workflows/cluster_endtoend_vtgate_vschema.yml b/.github/workflows/cluster_endtoend_vtgate_vschema.yml index 0653c05bf77..b4ad9aae717 100644 --- a/.github/workflows/cluster_endtoend_vtgate_vschema.yml +++ b/.github/workflows/cluster_endtoend_vtgate_vschema.yml @@ -9,7 +9,7 @@ concurrency: jobs: build: name: Run endtoend tests on Cluster (vtgate_vschema) - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - name: Check if workflow needs to be skipped @@ -59,6 +59,8 @@ jobs: - name: Tune the OS if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf @@ -67,10 +69,22 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + # Setup MySQL 8.0 + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata - sudo service mysql stop + # Install everything else we need, and configure + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get update + sudo service etcd stop + echo 'mysql version: ' + sudo mysql --version + sudo service mysql stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download @@ -80,7 +94,7 @@ jobs: - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. diff --git a/.github/workflows/cluster_endtoend_vtorc.yml b/.github/workflows/cluster_endtoend_vtorc.yml index 9c00b0713c2..45472eeb8ca 100644 --- a/.github/workflows/cluster_endtoend_vtorc.yml +++ b/.github/workflows/cluster_endtoend_vtorc.yml @@ -9,7 +9,7 @@ concurrency: jobs: build: name: Run endtoend tests on Cluster (vtorc) - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - name: Check if workflow needs to be skipped @@ -59,6 +59,8 @@ jobs: - name: Tune the OS if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf @@ -67,10 +69,22 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + # Setup MySQL 8.0 + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata - sudo service mysql stop + # Install everything else we need, and configure + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get update + sudo service etcd stop + echo 'mysql version: ' + sudo mysql --version + sudo service mysql stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download @@ -80,7 +94,7 @@ jobs: - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. diff --git a/.github/workflows/cluster_endtoend_vtorc_8.0.yml b/.github/workflows/cluster_endtoend_vtorc_8.0.yml index f1719c59d31..7d02851ed58 100644 --- a/.github/workflows/cluster_endtoend_vtorc_8.0.yml +++ b/.github/workflows/cluster_endtoend_vtorc_8.0.yml @@ -59,6 +59,8 @@ jobs: - name: Tune the OS if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf @@ -67,19 +69,22 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Get key to latest MySQL repo sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 - # Setup MySQL 8.0 wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update - # Install everything else we need, and configure sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils - sudo service mysql stop + sudo apt-get update + sudo service etcd stop + echo 'mysql version: ' + sudo mysql --version + sudo service mysql stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download @@ -89,7 +94,7 @@ jobs: - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. diff --git a/.github/workflows/cluster_endtoend_worker_vault_heavy.yml b/.github/workflows/cluster_endtoend_worker_vault_heavy.yml index f8a1a8ed364..d6e8641c67d 100644 --- a/.github/workflows/cluster_endtoend_worker_vault_heavy.yml +++ b/.github/workflows/cluster_endtoend_worker_vault_heavy.yml @@ -9,7 +9,7 @@ concurrency: jobs: build: name: Run endtoend tests on Cluster (worker_vault_heavy) - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - name: Check if workflow needs to be skipped @@ -59,6 +59,8 @@ jobs: - name: Tune the OS if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf @@ -67,10 +69,22 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + # Setup MySQL 8.0 + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata - sudo service mysql stop + # Install everything else we need, and configure + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get update + sudo service etcd stop + echo 'mysql version: ' + sudo mysql --version + sudo service mysql stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download @@ -80,7 +94,7 @@ jobs: - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. @@ -93,10 +107,11 @@ jobs: sudo sysctl -w net.ipv4.ip_local_port_range="22768 61999" # Increase our open file descriptor limit as we could hit this ulimit -n 65536 - cat <<-EOF>>./config/mycnf/mysql57.cnf + cat <<-EOF>>./config/mycnf/mysql80.cnf innodb_buffer_pool_dump_at_shutdown=OFF + innodb_buffer_pool_in_core_file=OFF innodb_buffer_pool_load_at_startup=OFF - innodb_buffer_pool_size=64M + innodb_buffer_pool_size=16M innodb_doublewrite=OFF innodb_flush_log_at_trx_commit=0 innodb_flush_method=O_DIRECT @@ -107,6 +122,7 @@ jobs: performance_schema=OFF slow-query-log=OFF EOF + make embed_config # run the tests however you normally do, then produce a JUnit XML file eatmydata -- go run test.go -docker=false -follow -shard worker_vault_heavy diff --git a/.github/workflows/cluster_endtoend_xb_backup.yml b/.github/workflows/cluster_endtoend_xb_backup.yml index 8a88ffbe12e..9a1a7f1a1cd 100644 --- a/.github/workflows/cluster_endtoend_xb_backup.yml +++ b/.github/workflows/cluster_endtoend_xb_backup.yml @@ -14,7 +14,7 @@ env: jobs: build: name: Run endtoend tests on Cluster (xb_backup) - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - name: Check if workflow needs to be skipped @@ -64,6 +64,8 @@ jobs: - name: Tune the OS if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf @@ -72,10 +74,22 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + + # Setup Percona Server for MySQL 8.0 sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata - sudo service mysql stop + sudo apt-get install -y lsb-release gnupg2 curl + wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb + sudo percona-release setup ps80 + sudo apt-get update + + # Install everything else we need, and configure + sudo apt-get install -y percona-server-server percona-server-client make unzip g++ etcd git wget eatmydata xz-utils + sudo service etcd stop + echo 'mysql version: ' + sudo mysql --version + sudo service mysql stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download @@ -83,21 +97,11 @@ jobs: # install JUnit report formatter go install github.com/vitessio/go-junit-report@HEAD - wget "https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb" - sudo apt-get install -y gnupg2 - sudo dpkg -i "percona-release_latest.$(lsb_release -sc)_all.deb" - sudo apt-get update - if [[ -n $XTRABACKUP_VERSION ]]; then - debfile="percona-xtrabackup-24_$XTRABACKUP_VERSION.$(lsb_release -sc)_amd64.deb" - wget "https://repo.percona.com/pxb-24/apt/pool/main/p/percona-xtrabackup-24/$debfile" - sudo apt install -y "./$debfile" - else - sudo apt-get install -y percona-xtrabackup-24 - fi - + sudo apt-get install percona-xtrabackup-80 lz4 + - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. diff --git a/.github/workflows/cluster_endtoend_xb_backup_mysql57.yml b/.github/workflows/cluster_endtoend_xb_backup_mysql57.yml new file mode 100644 index 00000000000..9474985c529 --- /dev/null +++ b/.github/workflows/cluster_endtoend_xb_backup_mysql57.yml @@ -0,0 +1,156 @@ +# DO NOT MODIFY: THIS FILE IS GENERATED USING "make generate_ci_workflows" + +name: Cluster (xb_backup) mysql57 +on: [push, pull_request] +concurrency: + group: format('{0}-{1}', ${{ github.ref }}, 'Cluster (xb_backup) mysql57') + cancel-in-progress: true + +env: + LAUNCHABLE_ORGANIZATION: "vitess" + LAUNCHABLE_WORKSPACE: "vitess-app" + GITHUB_PR_HEAD_SHA: "${{ github.event.pull_request.head.sha }}" + + # This is used if we need to pin the xtrabackup version used in tests. + # If this is NOT set then the latest version available will be used. + #XTRABACKUP_VERSION: "2.4.24-1" + +jobs: + build: + name: Run endtoend tests on Cluster (xb_backup) mysql57 + runs-on: ubuntu-20.04 + + steps: + - name: Check if workflow needs to be skipped + id: skip-workflow + run: | + skip='false' + if [[ "${{github.event.pull_request}}" == "" ]] && [[ "${{github.ref}}" != "refs/heads/main" ]] && [[ ! "${{github.ref}}" =~ ^refs/heads/release-[0-9]+\.[0-9]$ ]] && [[ ! "${{github.ref}}" =~ "refs/tags/.*" ]]; then + skip='true' + fi + echo Skip ${skip} + echo "::set-output name=skip-workflow::${skip}" + + - name: Check out code + if: steps.skip-workflow.outputs.skip-workflow == 'false' + uses: actions/checkout@v2 + + - name: Check for changes in relevant files + if: steps.skip-workflow.outputs.skip-workflow == 'false' + uses: frouioui/paths-filter@main + id: changes + with: + token: '' + filters: | + end_to_end: + - 'go/**/*.go' + - 'test.go' + - 'Makefile' + - 'build.env' + - 'go.[sumod]' + - 'proto/*.proto' + - 'tools/**' + - 'config/**' + - 'bootstrap.sh' + - '.github/workflows/cluster_endtoend_xb_backup_mysql57.yml' + + - name: Set up Go + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/setup-go@v2 + with: + go-version: 1.19.3 + + - name: Set up python + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/setup-python@v2 + + - name: Tune the OS + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range + # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio + echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf + sudo sysctl -p /etc/sysctl.conf + + - name: Get dependencies + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + sudo apt-get update + + # Uninstall any previously installed MySQL first + sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ + sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + + sudo systemctl stop apparmor + sudo DEBIAN_FRONTEND="noninteractive" apt-get remove -y --purge mysql-server mysql-client mysql-common + sudo apt-get -y autoremove + sudo apt-get -y autoclean + sudo deluser mysql + sudo rm -rf /var/lib/mysql + sudo rm -rf /etc/mysql + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.14-1_all.deb + # Bionic packages are still compatible for Focal since there's no MySQL 5.7 + # packages for Focal. + echo mysql-apt-config mysql-apt-config/repo-codename select bionic | sudo debconf-set-selections + echo mysql-apt-config mysql-apt-config/select-server select mysql-5.7 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* + sudo apt-get update + sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y mysql-client=5.7* mysql-community-server=5.7* mysql-server=5.7* + + sudo apt-get install -y make unzip g++ etcd curl git wget eatmydata + sudo service mysql stop + sudo service etcd stop + + # install JUnit report formatter + go install github.com/vitessio/go-junit-report@HEAD + + wget "https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb" + sudo apt-get install -y gnupg2 + sudo dpkg -i "percona-release_latest.$(lsb_release -sc)_all.deb" + sudo apt-get update + if [[ -n $XTRABACKUP_VERSION ]]; then + debfile="percona-xtrabackup-24_$XTRABACKUP_VERSION.$(lsb_release -sc)_amd64.deb" + wget "https://repo.percona.com/pxb-24/apt/pool/main/p/percona-xtrabackup-24/$debfile" + sudo apt install -y "./$debfile" + else + sudo apt-get install -y percona-xtrabackup-24 + fi + + - name: Setup launchable dependencies + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + # Get Launchable CLI installed. If you can, make it a part of the builder image to speed things up + pip3 install --user launchable~=1.0 > /dev/null + + # verify that launchable setup is all correct. + launchable verify || true + + # Tell Launchable about the build you are producing and testing + launchable record build --name "$GITHUB_RUN_ID" --source . + + - name: Run cluster endtoend test + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 45 + run: | + # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file + # which musn't be more than 107 characters long. + export VTDATAROOT="/tmp/" + source build.env + + set -x + + # run the tests however you normally do, then produce a JUnit XML file + eatmydata -- go run test.go -docker=false -follow -shard xb_backup | tee -a output.txt | go-junit-report -set-exit-code > report.xml + + - name: Print test output and Record test result in launchable + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + run: | + # send recorded tests to launchable + launchable record tests --build "$GITHUB_RUN_ID" go-test . || true + + # print test output + cat output.txt diff --git a/.github/workflows/cluster_endtoend_xb_recovery.yml b/.github/workflows/cluster_endtoend_xb_recovery.yml index 10b5492e6cb..0b083fc183a 100644 --- a/.github/workflows/cluster_endtoend_xb_recovery.yml +++ b/.github/workflows/cluster_endtoend_xb_recovery.yml @@ -14,7 +14,7 @@ env: jobs: build: name: Run endtoend tests on Cluster (xb_recovery) - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - name: Check if workflow needs to be skipped @@ -64,6 +64,8 @@ jobs: - name: Tune the OS if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf @@ -72,10 +74,22 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + + # Setup Percona Server for MySQL 8.0 sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata - sudo service mysql stop + sudo apt-get install -y lsb-release gnupg2 curl + wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb + sudo percona-release setup ps80 + sudo apt-get update + + # Install everything else we need, and configure + sudo apt-get install -y percona-server-server percona-server-client make unzip g++ etcd git wget eatmydata xz-utils + sudo service etcd stop + echo 'mysql version: ' + sudo mysql --version + sudo service mysql stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download @@ -83,21 +97,11 @@ jobs: # install JUnit report formatter go install github.com/vitessio/go-junit-report@HEAD - wget "https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb" - sudo apt-get install -y gnupg2 - sudo dpkg -i "percona-release_latest.$(lsb_release -sc)_all.deb" - sudo apt-get update - if [[ -n $XTRABACKUP_VERSION ]]; then - debfile="percona-xtrabackup-24_$XTRABACKUP_VERSION.$(lsb_release -sc)_amd64.deb" - wget "https://repo.percona.com/pxb-24/apt/pool/main/p/percona-xtrabackup-24/$debfile" - sudo apt install -y "./$debfile" - else - sudo apt-get install -y percona-xtrabackup-24 - fi - + sudo apt-get install percona-xtrabackup-80 lz4 + - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. diff --git a/.github/workflows/cluster_endtoend_xb_recovery_mysql57.yml b/.github/workflows/cluster_endtoend_xb_recovery_mysql57.yml new file mode 100644 index 00000000000..c8ac3a0530a --- /dev/null +++ b/.github/workflows/cluster_endtoend_xb_recovery_mysql57.yml @@ -0,0 +1,156 @@ +# DO NOT MODIFY: THIS FILE IS GENERATED USING "make generate_ci_workflows" + +name: Cluster (xb_recovery) mysql57 +on: [push, pull_request] +concurrency: + group: format('{0}-{1}', ${{ github.ref }}, 'Cluster (xb_recovery) mysql57') + cancel-in-progress: true + +env: + LAUNCHABLE_ORGANIZATION: "vitess" + LAUNCHABLE_WORKSPACE: "vitess-app" + GITHUB_PR_HEAD_SHA: "${{ github.event.pull_request.head.sha }}" + + # This is used if we need to pin the xtrabackup version used in tests. + # If this is NOT set then the latest version available will be used. + #XTRABACKUP_VERSION: "2.4.24-1" + +jobs: + build: + name: Run endtoend tests on Cluster (xb_recovery) mysql57 + runs-on: ubuntu-20.04 + + steps: + - name: Check if workflow needs to be skipped + id: skip-workflow + run: | + skip='false' + if [[ "${{github.event.pull_request}}" == "" ]] && [[ "${{github.ref}}" != "refs/heads/main" ]] && [[ ! "${{github.ref}}" =~ ^refs/heads/release-[0-9]+\.[0-9]$ ]] && [[ ! "${{github.ref}}" =~ "refs/tags/.*" ]]; then + skip='true' + fi + echo Skip ${skip} + echo "::set-output name=skip-workflow::${skip}" + + - name: Check out code + if: steps.skip-workflow.outputs.skip-workflow == 'false' + uses: actions/checkout@v2 + + - name: Check for changes in relevant files + if: steps.skip-workflow.outputs.skip-workflow == 'false' + uses: frouioui/paths-filter@main + id: changes + with: + token: '' + filters: | + end_to_end: + - 'go/**/*.go' + - 'test.go' + - 'Makefile' + - 'build.env' + - 'go.[sumod]' + - 'proto/*.proto' + - 'tools/**' + - 'config/**' + - 'bootstrap.sh' + - '.github/workflows/cluster_endtoend_xb_recovery_mysql57.yml' + + - name: Set up Go + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/setup-go@v2 + with: + go-version: 1.19.3 + + - name: Set up python + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/setup-python@v2 + + - name: Tune the OS + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range + # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio + echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf + sudo sysctl -p /etc/sysctl.conf + + - name: Get dependencies + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + sudo apt-get update + + # Uninstall any previously installed MySQL first + sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ + sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + + sudo systemctl stop apparmor + sudo DEBIAN_FRONTEND="noninteractive" apt-get remove -y --purge mysql-server mysql-client mysql-common + sudo apt-get -y autoremove + sudo apt-get -y autoclean + sudo deluser mysql + sudo rm -rf /var/lib/mysql + sudo rm -rf /etc/mysql + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.14-1_all.deb + # Bionic packages are still compatible for Focal since there's no MySQL 5.7 + # packages for Focal. + echo mysql-apt-config mysql-apt-config/repo-codename select bionic | sudo debconf-set-selections + echo mysql-apt-config mysql-apt-config/select-server select mysql-5.7 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* + sudo apt-get update + sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y mysql-client=5.7* mysql-community-server=5.7* mysql-server=5.7* + + sudo apt-get install -y make unzip g++ etcd curl git wget eatmydata + sudo service mysql stop + sudo service etcd stop + + # install JUnit report formatter + go install github.com/vitessio/go-junit-report@HEAD + + wget "https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb" + sudo apt-get install -y gnupg2 + sudo dpkg -i "percona-release_latest.$(lsb_release -sc)_all.deb" + sudo apt-get update + if [[ -n $XTRABACKUP_VERSION ]]; then + debfile="percona-xtrabackup-24_$XTRABACKUP_VERSION.$(lsb_release -sc)_amd64.deb" + wget "https://repo.percona.com/pxb-24/apt/pool/main/p/percona-xtrabackup-24/$debfile" + sudo apt install -y "./$debfile" + else + sudo apt-get install -y percona-xtrabackup-24 + fi + + - name: Setup launchable dependencies + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + # Get Launchable CLI installed. If you can, make it a part of the builder image to speed things up + pip3 install --user launchable~=1.0 > /dev/null + + # verify that launchable setup is all correct. + launchable verify || true + + # Tell Launchable about the build you are producing and testing + launchable record build --name "$GITHUB_RUN_ID" --source . + + - name: Run cluster endtoend test + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 45 + run: | + # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file + # which musn't be more than 107 characters long. + export VTDATAROOT="/tmp/" + source build.env + + set -x + + # run the tests however you normally do, then produce a JUnit XML file + eatmydata -- go run test.go -docker=false -follow -shard xb_recovery | tee -a output.txt | go-junit-report -set-exit-code > report.xml + + - name: Print test output and Record test result in launchable + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + run: | + # send recorded tests to launchable + launchable record tests --build "$GITHUB_RUN_ID" go-test . || true + + # print test output + cat output.txt diff --git a/.github/workflows/unit_test_mariadb103.yml b/.github/workflows/unit_test_mariadb103.yml index dfe77c60288..9a27205f0cb 100644 --- a/.github/workflows/unit_test_mariadb103.yml +++ b/.github/workflows/unit_test_mariadb103.yml @@ -8,7 +8,7 @@ concurrency: jobs: test: - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - name: Check if workflow needs to be skipped diff --git a/.github/workflows/unit_test_mysql57.yml b/.github/workflows/unit_test_mysql57.yml index ea3a2917d78..be6f1fa37c0 100644 --- a/.github/workflows/unit_test_mysql57.yml +++ b/.github/workflows/unit_test_mysql57.yml @@ -8,7 +8,7 @@ concurrency: jobs: test: - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - name: Check if workflow needs to be skipped diff --git a/.github/workflows/unit_test_mysql80.yml b/.github/workflows/unit_test_mysql80.yml index 77df9083e54..c4b694f7a4e 100644 --- a/.github/workflows/unit_test_mysql80.yml +++ b/.github/workflows/unit_test_mysql80.yml @@ -8,7 +8,7 @@ concurrency: jobs: test: - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - name: Check if workflow needs to be skipped diff --git a/.github/workflows/upgrade_downgrade_test_backups_e2e_next_release.yml b/.github/workflows/upgrade_downgrade_test_backups_e2e_next_release.yml index f77f6f98ca6..2dad2c79771 100644 --- a/.github/workflows/upgrade_downgrade_test_backups_e2e_next_release.yml +++ b/.github/workflows/upgrade_downgrade_test_backups_e2e_next_release.yml @@ -32,7 +32,7 @@ jobs: timeout-minutes: 60 if: always() && needs.get_next_release.result == 'success' name: Run Upgrade Downgrade Test - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 needs: - get_next_release diff --git a/go/mysql/collations/coercion.go b/go/mysql/collations/coercion.go index 0a1bbcb63e6..edd1a6289ea 100644 --- a/go/mysql/collations/coercion.go +++ b/go/mysql/collations/coercion.go @@ -37,13 +37,13 @@ func init() { // // The rules for assigning a Coercibility value to an expression are as follows: // -// - An explicit COLLATE clause has a coercibility of 0 (not coercible at all). -// - The concatenation of two strings with different collations has a coercibility of 1. -// - The collation of a column or a stored routine parameter or local variable has a coercibility of 2. -// - A “system constant” (the string returned by functions such as USER() or VERSION()) has a coercibility of 3. -// - The collation of a literal has a coercibility of 4. -// - The collation of a numeric or temporal value has a coercibility of 5. -// - NULL or an expression that is derived from NULL has a coercibility of 6. +// - An explicit COLLATE clause has a coercibility of 0 (not coercible at all). +// - The concatenation of two strings with different collations has a coercibility of 1. +// - The collation of a column or a stored routine parameter or local variable has a coercibility of 2. +// - A “system constant” (the string returned by functions such as USER() or VERSION()) has a coercibility of 3. +// - The collation of a literal has a coercibility of 4. +// - The collation of a numeric or temporal value has a coercibility of 5. +// - NULL or an expression that is derived from NULL has a coercibility of 6. // // According to the MySQL documentation, Coercibility is an actual word of the English // language, although the Vitess maintainers disagree with this assessment. diff --git a/go/mysql/collations/wildcard.go b/go/mysql/collations/wildcard.go index 6b38eb1a19f..9925042214e 100644 --- a/go/mysql/collations/wildcard.go +++ b/go/mysql/collations/wildcard.go @@ -83,9 +83,9 @@ func (emptyMatcher) Match(in []byte) bool { // fastMatcher is an implementation of WildcardPattern that uses a collation's Collate method // to perform wildcard matching. // It is returned: -// - when the wildcard pattern has no wildcard characters at all -// - when the wildcard pattern has a single '%' (patternMatchMany) and it is the very last -// character of the pattern (in this case, we set isPrefix to true to use prefix-match collation) +// - when the wildcard pattern has no wildcard characters at all +// - when the wildcard pattern has a single '%' (patternMatchMany) and it is the very last +// character of the pattern (in this case, we set isPrefix to true to use prefix-match collation) type fastMatcher struct { collate func(left, right []byte, isPrefix bool) int pattern []byte diff --git a/go/test/endtoend/backup/mysqlctld/backup_mysqlctld_test.go b/go/test/endtoend/backup/mysqlctld/backup_mysqlctld_test.go index 2ee11103c72..4b97f5ff895 100644 --- a/go/test/endtoend/backup/mysqlctld/backup_mysqlctld_test.go +++ b/go/test/endtoend/backup/mysqlctld/backup_mysqlctld_test.go @@ -24,5 +24,5 @@ import ( // TestBackupMysqlctld - tests the backup using mysqlctld. func TestBackupMysqlctld(t *testing.T) { - backup.TestBackup(t, backup.Mysqlctld, "", 0) + backup.TestBackup(t, backup.Mysqlctld, "xbstream", 0) } diff --git a/go/test/endtoend/backup/transform/backup_transform_utils.go b/go/test/endtoend/backup/transform/backup_transform_utils.go index 62b52260a60..e04f6826397 100644 --- a/go/test/endtoend/backup/transform/backup_transform_utils.go +++ b/go/test/endtoend/backup/transform/backup_transform_utils.go @@ -91,7 +91,7 @@ func TestMainSetup(m *testing.M, useMysqlctld bool) { initDb, _ := os.ReadFile(path.Join(os.Getenv("VTROOT"), "/config/init_db.sql")) sql := string(initDb) newInitDBFile = path.Join(localCluster.TmpDirectory, "init_db_with_passwords.sql") - sql = sql + initialsharding.GetPasswordUpdateSQL(localCluster) + sql = sql + initialsharding.GetPasswordUpdateSQL() os.WriteFile(newInitDBFile, []byte(sql), 0666) extraArgs := []string{"--db-credentials-file", dbCredentialFile} diff --git a/go/test/endtoend/backup/vtbackup/main_test.go b/go/test/endtoend/backup/vtbackup/main_test.go index d8e474b7336..5f994948639 100644 --- a/go/test/endtoend/backup/vtbackup/main_test.go +++ b/go/test/endtoend/backup/vtbackup/main_test.go @@ -91,7 +91,7 @@ func TestMain(m *testing.M) { initDb, _ := os.ReadFile(path.Join(os.Getenv("VTROOT"), "/config/init_db.sql")) sql := string(initDb) newInitDBFile = path.Join(localCluster.TmpDirectory, "init_db_with_passwords.sql") - sql = sql + initialsharding.GetPasswordUpdateSQL(localCluster) + sql = sql + initialsharding.GetPasswordUpdateSQL() err = os.WriteFile(newInitDBFile, []byte(sql), 0666) if err != nil { return 1, err @@ -117,12 +117,13 @@ func TestMain(m *testing.M) { tablet.MysqlctlProcess = *cluster.MysqlCtlProcessInstance(tablet.TabletUID, tablet.MySQLPort, localCluster.TmpDirectory) tablet.MysqlctlProcess.InitDBFile = newInitDBFile tablet.MysqlctlProcess.ExtraArgs = extraArgs - if proc, err := tablet.MysqlctlProcess.StartProcess(); err != nil { + proc, err := tablet.MysqlctlProcess.StartProcess() + if err != nil { return 1, err - } else { - // ignore golint warning, we need the else block to use proc - mysqlProcs = append(mysqlProcs, proc) } + // ignore golint warning, we need the else block to use proc + mysqlProcs = append(mysqlProcs, proc) + } for _, proc := range mysqlProcs { if err := proc.Wait(); err != nil { diff --git a/go/test/endtoend/backup/vtctlbackup/backup_test.go b/go/test/endtoend/backup/vtctlbackup/backup_test.go index 6f233eafeda..d7b3da30f9f 100644 --- a/go/test/endtoend/backup/vtctlbackup/backup_test.go +++ b/go/test/endtoend/backup/vtctlbackup/backup_test.go @@ -22,5 +22,5 @@ import ( // TestBackupMain - main tests backup using vtctl commands func TestBackupMain(t *testing.T) { - TestBackup(t, Backup, "", 0) + TestBackup(t, Backup, "xbstream", 0) } diff --git a/go/test/endtoend/backup/vtctlbackup/backup_utils.go b/go/test/endtoend/backup/vtctlbackup/backup_utils.go index f4e2e92a9ac..eae0c306c31 100644 --- a/go/test/endtoend/backup/vtctlbackup/backup_utils.go +++ b/go/test/endtoend/backup/vtctlbackup/backup_utils.go @@ -44,6 +44,7 @@ const ( XtraBackup = iota Backup Mysqlctld + timeout = time.Duration(60 * time.Second) ) var ( @@ -107,7 +108,7 @@ func LaunchCluster(setupType int, streamMode string, stripes int) (int, error) { initDb, _ := os.ReadFile(path.Join(os.Getenv("VTROOT"), "/config/init_db.sql")) sql := string(initDb) newInitDBFile = path.Join(localCluster.TmpDirectory, "init_db_with_passwords.sql") - sql = sql + initialsharding.GetPasswordUpdateSQL(localCluster) + sql = sql + initialsharding.GetPasswordUpdateSQL() err = os.WriteFile(newInitDBFile, []byte(sql), 0666) if err != nil { return 1, err @@ -214,8 +215,28 @@ func TearDownCluster() { } // TestBackup runs all the backup tests -func TestBackup(t *testing.T, setupType int, streamMode string, stripes int) { - +func TestBackup(t *testing.T, setupType int, streamMode string, stripes int) error { + verStr, err := mysqlctl.GetVersionString() + require.NoError(t, err) + _, vers, err := mysqlctl.ParseVersionString(verStr) + require.NoError(t, err) + switch streamMode { + case "xbstream": + if vers.Major < 8 { + t.Logf("Skipping xtrabackup tests with --xtrabackup_stream_mode=xbstream as those are only tested on XtraBackup/MySQL 8.0+") + return nil + } + case "", "tar": // streaming method of tar is the default for the vttablet --xtrabackup_stream_mode flag + // XtraBackup 8.0 must be used with MySQL 8.0 and it no longer supports tar as a stream method: + // https://docs.percona.com/percona-xtrabackup/2.4/innobackupex/streaming_backups_innobackupex.html + // https://docs.percona.com/percona-xtrabackup/8.0/xtrabackup_bin/backup.streaming.html + if vers.Major > 5 { + t.Logf("Skipping xtrabackup tests with --xtrabackup_stream_mode=tar as tar is no longer a streaming option in XtraBackup 8.0") + return nil + } + default: + require.FailNow(t, fmt.Sprintf("Unsupported xtrabackup stream mode: %s", streamMode)) + } testMethods := []struct { name string method func(t *testing.T) @@ -268,7 +289,7 @@ func TestBackup(t *testing.T, setupType int, streamMode string, stripes int) { for _, test := range testMethods { t.Run(test.name, test.method) } - + return nil } type restoreMethod func(t *testing.T, tablet *cluster.Vttablet) @@ -282,12 +303,12 @@ type restoreMethod func(t *testing.T, tablet *cluster.Vttablet) // 7. insert more data on the primary // 8. take another backup // 9. verify that we now have 2 backups -// 10. do a PRS to make the original primary a replica so that we can do a restore there -// 11. Delete+teardown the new primary so that we can restore the first backup on the original +// 10. do a PRS to make the original primary a replica so that we can do a restore there +// 11. Delete+teardown the new primary so that we can restore the first backup on the original // primary to confirm we don't have the data from #7 -// 12. restore first backup on the original primary tablet using the first backup timstamp -// 13. verify that don't have the data added after the first backup -// 14. remove the backups +// 12. restore first backup on the original primary tablet using the first backup timstamp +// 13. verify that don't have the data added after the first backup +// 14. remove the backups func primaryBackup(t *testing.T) { verifyInitialReplication(t) @@ -301,7 +322,7 @@ func primaryBackup(t *testing.T) { require.Nil(t, err) // We'll restore this on the primary later to test restores using a backup timestamp - firstBackupTimestamp := time.Now().Format(mysqlctl.BackupTimestampFormat) + firstBackupTimestamp := time.Now().UTC().Format(mysqlctl.BackupTimestampFormat) backups := localCluster.VerifyBackupCount(t, shardKsName, 1) assert.Contains(t, backups[0], primary.Alias) @@ -357,10 +378,10 @@ func primaryBackup(t *testing.T) { require.Nil(t, err) } -// Test a primary and replica from the same backup. +// Test a primary and replica from the same backup. // -// Check that a replica and primary both restored from the same backup -// can replicate successfully. +// Check that a replica and primary both restored from the same backup +// can replicate successfully. func primaryReplicaSameBackup(t *testing.T) { // insert data on primary, wait for replica to get it verifyInitialReplication(t) @@ -375,7 +396,7 @@ func primaryReplicaSameBackup(t *testing.T) { // now bring up the other replica, letting it restore from backup. restoreWaitForBackup(t, "replica") - err = replica2.VttabletProcess.WaitForTabletStatusesForTimeout([]string{"SERVING"}, 25*time.Second) + err = replica2.VttabletProcess.WaitForTabletStatusesForTimeout([]string{"SERVING"}, timeout) require.Nil(t, err) // check the new replica has the data @@ -428,14 +449,13 @@ func restoreOldPrimaryInPlace(t *testing.T) { // Test that a former primary replicates correctly after being restored. // -//- Take a backup. +// - Take a backup. // - Reparent from old primary to new primary. // - Force old primary to restore from a previous backup using restore_method. // -//Args: -//restore_method: function accepting one parameter of type tablet.Tablet, -//this function is called to force a restore on the provided tablet -// +// Args: +// restore_method: function accepting one parameter of type tablet.Tablet, +// this function is called to force a restore on the provided tablet func testRestoreOldPrimary(t *testing.T, method restoreMethod) { // insert data on primary, wait for replica to get it verifyInitialReplication(t) @@ -584,21 +604,19 @@ func terminatedRestore(t *testing.T) { stopAllTablets() } -//test_backup will: +// test_backup will: // - create a shard with primary and replica1 only -//- run InitShardPrimary -//- bring up tablet_replica2 concurrently, telling it to wait for a backup -//- insert some data -//- take a backup +// - run InitShardPrimary +// - bring up tablet_replica2 concurrently, telling it to wait for a backup +// - insert some data +// - take a backup // - insert more data on the primary -//- wait for tablet_replica2 to become SERVING -//- check all data is right (before+after backup data) -//- list the backup, remove it -// -//Args: -//tablet_type: 'replica' or 'rdonly'. -// +// - wait for tablet_replica2 to become SERVING +// - check all data is right (before+after backup data) +// - list the backup, remove it // +// Args: +// tablet_type: 'replica' or 'rdonly'. func vtctlBackup(t *testing.T, tabletType string) { restoreWaitForBackup(t, tabletType) verifyInitialReplication(t) @@ -611,7 +629,7 @@ func vtctlBackup(t *testing.T, tabletType string) { _, err = primary.VttabletProcess.QueryTablet("insert into vt_insert_test (msg) values ('test2')", keyspaceName, true) require.Nil(t, err) - err = replica2.VttabletProcess.WaitForTabletStatusesForTimeout([]string{"SERVING"}, 25*time.Second) + err = replica2.VttabletProcess.WaitForTabletStatusesForTimeout([]string{"SERVING"}, timeout) require.Nil(t, err) cluster.VerifyRowsInTablet(t, replica2, keyspaceName, 2) @@ -674,7 +692,7 @@ func verifyRestoreTablet(t *testing.T, tablet *cluster.Vttablet, status string) err := tablet.VttabletProcess.Setup() require.Nil(t, err) if status != "" { - err = tablet.VttabletProcess.WaitForTabletStatusesForTimeout([]string{status}, 25*time.Second) + err = tablet.VttabletProcess.WaitForTabletStatusesForTimeout([]string{status}, timeout) require.Nil(t, err) } // We restart replication here because semi-sync will not be set correctly on tablet startup since diff --git a/go/test/endtoend/cluster/cluster_process.go b/go/test/endtoend/cluster/cluster_process.go index b17f90e57c5..a2fa8f4523f 100644 --- a/go/test/endtoend/cluster/cluster_process.go +++ b/go/test/endtoend/cluster/cluster_process.go @@ -22,7 +22,6 @@ import ( "flag" "fmt" "io" - "io/ioutil" "math/rand" "net" "os" @@ -1155,11 +1154,11 @@ func getCoveragePath(fileName string) string { // PrintMysqlctlLogFiles prints all the log files associated with the mysqlctl binary func (cluster *LocalProcessCluster) PrintMysqlctlLogFiles() { logDir := cluster.TmpDirectory - files, _ := ioutil.ReadDir(logDir) + files, _ := os.ReadDir(logDir) for _, fileInfo := range files { if !fileInfo.IsDir() && strings.Contains(fileInfo.Name(), "mysqlctl") { log.Errorf("Printing the log file - " + fileInfo.Name()) - logOut, _ := ioutil.ReadFile(path.Join(logDir, fileInfo.Name())) + logOut, _ := os.ReadFile(path.Join(logDir, fileInfo.Name())) log.Errorf(string(logOut)) } } diff --git a/go/test/endtoend/cluster/mysqlctl_process.go b/go/test/endtoend/cluster/mysqlctl_process.go index 97ccaf80f86..4bc42abcd9e 100644 --- a/go/test/endtoend/cluster/mysqlctl_process.go +++ b/go/test/endtoend/cluster/mysqlctl_process.go @@ -118,6 +118,10 @@ ssl_key={{.Dir}}/server-001-key.pem tmpProcess.Env = append(tmpProcess.Env, "EXTRA_MY_CNF="+extraMyCNF) tmpProcess.Env = append(tmpProcess.Env, "VTDATAROOT="+os.Getenv("VTDATAROOT")) + tmpProcess.Env = append(tmpProcess.Env, "VT_MYSQL_ROOT="+os.Getenv("VT_MYSQL_ROOT")) + tmpProcess.Env = append(tmpProcess.Env, "xtrabackup_root_path="+os.Getenv("xtrabackup_root_path")) + tmpProcess.Env = append(tmpProcess.Env, "MYSQL_FLAVOR="+os.Getenv("MYSQL_FLAVOR")) + tmpProcess.Env = append(tmpProcess.Env, "VT_ROOT="+os.Getenv("VT_ROOT")) } tmpProcess.Args = append(tmpProcess.Args, "init", "--", diff --git a/go/test/endtoend/cluster/vtgate_process.go b/go/test/endtoend/cluster/vtgate_process.go index fa0c1d54d91..95807b8c5ce 100644 --- a/go/test/endtoend/cluster/vtgate_process.go +++ b/go/test/endtoend/cluster/vtgate_process.go @@ -20,7 +20,6 @@ import ( "encoding/json" "fmt" "io" - "io/ioutil" "net/http" "os" "os/exec" @@ -317,7 +316,7 @@ func (vtgate *VtgateProcess) ReadVSchema() (*interface{}, error) { return nil, err } defer resp.Body.Close() - res, err := ioutil.ReadAll(resp.Body) + res, err := io.ReadAll(resp.Body) if err != nil { return nil, err } diff --git a/go/test/endtoend/clustertest/vtgate_test.go b/go/test/endtoend/clustertest/vtgate_test.go index 1d888d9abee..2f72682a391 100644 --- a/go/test/endtoend/clustertest/vtgate_test.go +++ b/go/test/endtoend/clustertest/vtgate_test.go @@ -5,7 +5,7 @@ Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 + http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, diff --git a/go/test/endtoend/recovery/pitr/shardedpitr_test.go b/go/test/endtoend/recovery/pitr/shardedpitr_test.go index 9d4c0691b6d..ee39fe67310 100644 --- a/go/test/endtoend/recovery/pitr/shardedpitr_test.go +++ b/go/test/endtoend/recovery/pitr/shardedpitr_test.go @@ -39,13 +39,15 @@ var ( var ( clusterInstance *cluster.LocalProcessCluster - - primary *cluster.Vttablet - replica *cluster.Vttablet - shard0Primary *cluster.Vttablet - shard0Replica *cluster.Vttablet - shard1Primary *cluster.Vttablet - shard1Replica *cluster.Vttablet + primary *cluster.Vttablet + replica1 *cluster.Vttablet + replica2 *cluster.Vttablet + shard0Primary *cluster.Vttablet + shard0Replica1 *cluster.Vttablet + shard0Replica2 *cluster.Vttablet + shard1Primary *cluster.Vttablet + shard1Replica1 *cluster.Vttablet + shard1Replica2 *cluster.Vttablet cell = "zone1" hostname = "localhost" @@ -127,10 +129,10 @@ func TestPITRRecovery(t *testing.T) { insertRow(t, 1, "prd-1", false) insertRow(t, 2, "prd-2", false) - cluster.VerifyRowsInTabletForTable(t, replica, keyspaceName, 2, "product") + cluster.VerifyRowsInTabletForTable(t, replica1, keyspaceName, 2, "product") // backup the replica - err = clusterInstance.VtctlclientProcess.ExecuteCommand("Backup", replica.Alias) + err = clusterInstance.VtctlclientProcess.ExecuteCommand("Backup", replica1.Alias) require.NoError(t, err) // check that the backup shows up in the listing @@ -166,14 +168,14 @@ func TestPITRRecovery(t *testing.T) { } // wait till all the shards have required data - cluster.VerifyRowsInTabletForTable(t, shard0Replica, keyspaceName, 6, "product") - cluster.VerifyRowsInTabletForTable(t, shard1Replica, keyspaceName, 4, "product") + cluster.VerifyRowsInTabletForTable(t, shard0Replica1, keyspaceName, 6, "product") + cluster.VerifyRowsInTabletForTable(t, shard1Replica1, keyspaceName, 4, "product") // take the backup (to simulate the regular backup) - err = clusterInstance.VtctlclientProcess.ExecuteCommand("Backup", shard0Replica.Alias) + err = clusterInstance.VtctlclientProcess.ExecuteCommand("Backup", shard0Replica1.Alias) require.NoError(t, err) // take the backup (to simulate the regular backup) - err = clusterInstance.VtctlclientProcess.ExecuteCommand("Backup", shard1Replica.Alias) + err = clusterInstance.VtctlclientProcess.ExecuteCommand("Backup", shard1Replica1.Alias) require.NoError(t, err) backups, err := clusterInstance.ListBackups(keyspaceName + "/-80") @@ -313,9 +315,9 @@ func performResharding(t *testing.T) { require.NoError(t, err) // remove the original tablets in the original shard - removeTablets(t, []*cluster.Vttablet{primary, replica}) + removeTablets(t, []*cluster.Vttablet{primary, replica1, replica2}) - for _, tablet := range []*cluster.Vttablet{replica} { + for _, tablet := range []*cluster.Vttablet{replica1, replica2} { err = clusterInstance.VtctlclientProcess.ExecuteCommand("DeleteTablet", tablet.Alias) require.NoError(t, err) } @@ -394,15 +396,18 @@ func initializeCluster(t *testing.T) { // Defining all the tablets primary = clusterInstance.NewVttabletInstance("replica", 0, "") - replica = clusterInstance.NewVttabletInstance("replica", 0, "") + replica1 = clusterInstance.NewVttabletInstance("replica", 0, "") + replica2 = clusterInstance.NewVttabletInstance("replica", 0, "") shard0Primary = clusterInstance.NewVttabletInstance("replica", 0, "") - shard0Replica = clusterInstance.NewVttabletInstance("replica", 0, "") + shard0Replica1 = clusterInstance.NewVttabletInstance("replica", 0, "") + shard0Replica2 = clusterInstance.NewVttabletInstance("replica", 0, "") shard1Primary = clusterInstance.NewVttabletInstance("replica", 0, "") - shard1Replica = clusterInstance.NewVttabletInstance("replica", 0, "") + shard1Replica1 = clusterInstance.NewVttabletInstance("replica", 0, "") + shard1Replica2 = clusterInstance.NewVttabletInstance("replica", 0, "") - shard.Vttablets = []*cluster.Vttablet{primary, replica} - shard0.Vttablets = []*cluster.Vttablet{shard0Primary, shard0Replica} - shard1.Vttablets = []*cluster.Vttablet{shard1Primary, shard1Replica} + shard.Vttablets = []*cluster.Vttablet{primary, replica1, replica2} + shard0.Vttablets = []*cluster.Vttablet{shard0Primary, shard0Replica1, shard0Replica2} + shard1.Vttablets = []*cluster.Vttablet{shard1Primary, shard1Replica1, shard1Replica2} clusterInstance.VtTabletExtraArgs = append(clusterInstance.VtTabletExtraArgs, commonTabletArg...) clusterInstance.VtTabletExtraArgs = append(clusterInstance.VtTabletExtraArgs, "--restore_from_backup", "--enable_semi_sync") @@ -436,14 +441,16 @@ func initializeCluster(t *testing.T) { "FLUSH PRIVILEGES;", } - for _, tablet := range []*cluster.Vttablet{primary, replica, shard0Primary, shard0Replica, shard1Primary, shard1Replica} { - for _, query := range queryCmds { - _, err = tablet.VttabletProcess.QueryTablet(query, keyspace.Name, false) + for _, shard := range clusterInstance.Keyspaces[0].Shards { + for _, tablet := range shard.Vttablets { + for _, query := range queryCmds { + _, err = tablet.VttabletProcess.QueryTablet(query, keyspace.Name, false) + require.NoError(t, err) + } + + err = tablet.VttabletProcess.Setup() require.NoError(t, err) } - - err = tablet.VttabletProcess.Setup() - require.NoError(t, err) } err = clusterInstance.VtctlclientProcess.InitShardPrimary(keyspaceName, shard.Name, cell, primary.TabletUID) diff --git a/go/test/endtoend/recovery/unshardedrecovery/recovery.go b/go/test/endtoend/recovery/unshardedrecovery/recovery.go index 0f85ad5131b..1b5a42e86f3 100644 --- a/go/test/endtoend/recovery/unshardedrecovery/recovery.go +++ b/go/test/endtoend/recovery/unshardedrecovery/recovery.go @@ -96,7 +96,7 @@ func TestMainImpl(m *testing.M) { initDb, _ := os.ReadFile(path.Join(os.Getenv("VTROOT"), "/config/init_db.sql")) sql := string(initDb) newInitDBFile = path.Join(localCluster.TmpDirectory, "init_db_with_passwords.sql") - sql = sql + initialsharding.GetPasswordUpdateSQL(localCluster) + sql = sql + initialsharding.GetPasswordUpdateSQL() // https://github.com/vitessio/vitess/issues/8315 oldAlterTableMode := ` SET GLOBAL old_alter_table = ON; diff --git a/go/test/endtoend/sharding/initialsharding/sharding_util.go b/go/test/endtoend/sharding/initialsharding/sharding_util.go index 567ffb67c99..d0094304477 100644 --- a/go/test/endtoend/sharding/initialsharding/sharding_util.go +++ b/go/test/endtoend/sharding/initialsharding/sharding_util.go @@ -646,7 +646,7 @@ func writeInitDBFile() { initDb, _ := os.ReadFile(path.Join(os.Getenv("VTROOT"), "/config/init_db.sql")) sql := string(initDb) newInitDbFile = path.Join(ClusterInstance.TmpDirectory, "init_db_with_passwords.sql") - sql = sql + GetPasswordUpdateSQL(ClusterInstance) + ` + sql = sql + GetPasswordUpdateSQL() + ` # connecting through a port requires 127.0.0.1 # --host=localhost will connect through socket CREATE USER 'vt_dba'@'127.0.0.1' IDENTIFIED BY 'VtDbaPass'; @@ -695,45 +695,17 @@ func WriteDbCredentialToTmp(tmpDir string) string { } // GetPasswordUpdateSQL returns the sql for password update -func GetPasswordUpdateSQL(localCluster *cluster.LocalProcessCluster) string { +func GetPasswordUpdateSQL() string { pwdChangeCmd := ` # Set real passwords for all users. - UPDATE mysql.user SET %s = PASSWORD('RootPass') - WHERE User = 'root' AND Host = 'localhost'; - UPDATE mysql.user SET %s = PASSWORD('VtDbaPass') - WHERE User = 'vt_dba' AND Host = 'localhost'; - UPDATE mysql.user SET %s = PASSWORD('VtAppPass') - WHERE User = 'vt_app' AND Host = 'localhost'; - UPDATE mysql.user SET %s = PASSWORD('VtAllprivsPass') - WHERE User = 'vt_allprivs' AND Host = 'localhost'; - UPDATE mysql.user SET %s = PASSWORD('VtReplPass') - WHERE User = 'vt_repl' AND Host = '%%'; - UPDATE mysql.user SET %s = PASSWORD('VtFilteredPass') - WHERE User = 'vt_filtered' AND Host = 'localhost'; + # Set real passwords for all users. + ALTER USER 'root'@'localhost' IDENTIFIED BY 'RootPass'; + ALTER USER 'vt_dba'@'localhost' IDENTIFIED BY 'VtDbaPass'; + ALTER USER 'vt_app'@'localhost' IDENTIFIED BY 'VtAppPass'; + ALTER USER 'vt_allprivs'@'localhost' IDENTIFIED BY 'VtAllprivsPass'; + ALTER USER 'vt_repl'@'%' IDENTIFIED BY 'VtReplPass'; + ALTER USER 'vt_filtered'@'localhost' IDENTIFIED BY 'VtFilteredPass'; FLUSH PRIVILEGES; ` - pwdCol, _ := getPasswordField(localCluster) - return fmt.Sprintf(pwdChangeCmd, pwdCol, pwdCol, pwdCol, pwdCol, pwdCol, pwdCol) -} - -// getPasswordField Determines which column is used for user passwords in this MySQL version. -func getPasswordField(localCluster *cluster.LocalProcessCluster) (pwdCol string, err error) { - tablet := &cluster.Vttablet{ - Type: "relpica", - TabletUID: 100, - MySQLPort: 15000, - MysqlctlProcess: *cluster.MysqlCtlProcessInstance(100, 15000, localCluster.TmpDirectory), - } - if err = tablet.MysqlctlProcess.Start(); err != nil { - return "", err - } - tablet.VttabletProcess = cluster.VttabletProcessInstance(tablet.HTTPPort, tablet.GrpcPort, tablet.TabletUID, "", "", "", 0, tablet.Type, localCluster.TopoPort, "", "", nil, false, localCluster.DefaultCharset) - result, err := tablet.VttabletProcess.QueryTablet("select password from mysql.user limit 0", "", false) - if err == nil && len(result.Rows) > 0 { - return "password", nil - } - tablet.MysqlctlProcess.Stop() - os.RemoveAll(path.Join(tablet.VttabletProcess.Directory)) - return "authentication_string", nil - + return pwdChangeCmd } diff --git a/go/test/endtoend/tabletgateway/buffer/buffer_test_helpers.go b/go/test/endtoend/tabletgateway/buffer/buffer_test_helpers.go index bfaaa2e9ac4..19c0f88ed43 100644 --- a/go/test/endtoend/tabletgateway/buffer/buffer_test_helpers.go +++ b/go/test/endtoend/tabletgateway/buffer/buffer_test_helpers.go @@ -67,7 +67,7 @@ const ( updateRowID = 2 ) -//threadParams is set of params passed into read and write threads +// threadParams is set of params passed into read and write threads type threadParams struct { quit bool rpcs int // Number of queries successfully executed. diff --git a/go/test/endtoend/tabletgateway/vtgate_test.go b/go/test/endtoend/tabletgateway/vtgate_test.go index 59f5a43791e..a3876b259f3 100644 --- a/go/test/endtoend/tabletgateway/vtgate_test.go +++ b/go/test/endtoend/tabletgateway/vtgate_test.go @@ -5,7 +5,7 @@ Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 + http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, diff --git a/go/test/endtoend/tabletmanager/commands_test.go b/go/test/endtoend/tabletmanager/commands_test.go index 276d25fc9dc..493282c0c2e 100644 --- a/go/test/endtoend/tabletmanager/commands_test.go +++ b/go/test/endtoend/tabletmanager/commands_test.go @@ -21,23 +21,26 @@ import ( "encoding/json" "fmt" "reflect" + "regexp" + "strconv" "testing" "time" - "vitess.io/vitess/go/test/endtoend/utils" - - "github.com/stretchr/testify/require" "github.com/tidwall/gjson" + "vitess.io/vitess/go/test/endtoend/utils" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "vitess.io/vitess/go/mysql" "vitess.io/vitess/go/test/endtoend/cluster" ) var ( - getSchemaT1Results = "CREATE TABLE `t1` (\n `id` bigint(20) NOT NULL,\n `value` varchar(16) DEFAULT NULL,\n PRIMARY KEY (`id`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8" - getSchemaV1Results = fmt.Sprintf("CREATE ALGORITHM=UNDEFINED DEFINER=`%s`@`%s` SQL SECURITY DEFINER VIEW {{.DatabaseName}}.`v1` AS select {{.DatabaseName}}.`t1`.`id` AS `id`,{{.DatabaseName}}.`t1`.`value` AS `value` from {{.DatabaseName}}.`t1`", username, hostname) + getSchemaT1Results57 = "CREATE TABLE `t1` (\n `id` bigint(20) NOT NULL,\n `value` varchar(16) DEFAULT NULL,\n PRIMARY KEY (`id`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8" + getSchemaT1Results80 = "CREATE TABLE `t1` (\n `id` bigint NOT NULL,\n `value` varchar(16) DEFAULT NULL,\n PRIMARY KEY (`id`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb3" + getSchemaV1Results = fmt.Sprintf("CREATE ALGORITHM=UNDEFINED DEFINER=`%s`@`%s` SQL SECURITY DEFINER VIEW {{.DatabaseName}}.`v1` AS select {{.DatabaseName}}.`t1`.`id` AS `id`,{{.DatabaseName}}.`t1`.`value` AS `value` from {{.DatabaseName}}.`t1`", username, hostname) ) // TabletCommands tests the basic tablet commands @@ -232,10 +235,45 @@ func TestGetSchema(t *testing.T) { fmt.Sprintf("%s-%d", clusterInstance.Cell, primaryTablet.TabletUID)) require.Nil(t, err) + ctx := context.Background() + conn, err := mysql.Connect(ctx, &primaryTabletParams) + require.Nil(t, err) + fmt.Printf("%s", conn.ServerVersion) + major, _, _ := parseVersionString(conn.ServerVersion) + require.LessOrEqual(t, 0, major) + t1Create := gjson.Get(res, "table_definitions.#(name==\"t1\").schema") - assert.Equal(t, getSchemaT1Results, t1Create.String()) + if major <= 5 { + assert.Equal(t, getSchemaT1Results57, t1Create.String()) + } else { + assert.Equal(t, getSchemaT1Results80, t1Create.String()) + } v1Create := gjson.Get(res, "table_definitions.#(name==\"v1\").schema") assert.Equal(t, getSchemaV1Results, v1Create.String()) + +} + +// ParseVersionString parses the output of mysqld --version into a flavor and version +func parseVersionString(version string) (int, int, int) { + versionRegex := regexp.MustCompile(`([0-9]+)\.([0-9]+)\.([0-9]+)`) + v := versionRegex.FindStringSubmatch(version) + if len(v) != 4 { + return 0, 0, 0 + } + major, err := strconv.Atoi(string(v[1])) + if err != nil { + return 0, 0, 0 + } + minor, err := strconv.Atoi(string(v[2])) + if err != nil { + return 0, 0, 0 + } + patch, err := strconv.Atoi(string(v[3])) + if err != nil { + return 0, 0, 0 + } + + return major, minor, patch } func assertNodeCount(t *testing.T, result string, want int) { diff --git a/go/test/endtoend/tabletmanager/custom_rule_topo_test.go b/go/test/endtoend/tabletmanager/custom_rule_topo_test.go index 2a7ed957ac6..0d68c7a2521 100644 --- a/go/test/endtoend/tabletmanager/custom_rule_topo_test.go +++ b/go/test/endtoend/tabletmanager/custom_rule_topo_test.go @@ -5,7 +5,7 @@ Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 + http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, diff --git a/go/test/endtoend/tabletmanager/primary/tablet_test.go b/go/test/endtoend/tabletmanager/primary/tablet_test.go index a7001393833..a87990e828b 100644 --- a/go/test/endtoend/tabletmanager/primary/tablet_test.go +++ b/go/test/endtoend/tabletmanager/primary/tablet_test.go @@ -5,7 +5,7 @@ Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 + http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, diff --git a/go/test/endtoend/tabletmanager/qps_test.go b/go/test/endtoend/tabletmanager/qps_test.go index 4dbdcffb0bd..066e56ac576 100644 --- a/go/test/endtoend/tabletmanager/qps_test.go +++ b/go/test/endtoend/tabletmanager/qps_test.go @@ -5,7 +5,7 @@ Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 + http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, diff --git a/go/test/endtoend/tabletmanager/tablet_security_policy_test.go b/go/test/endtoend/tabletmanager/tablet_security_policy_test.go index 85b1e36f8c9..a2e1e8bd987 100644 --- a/go/test/endtoend/tabletmanager/tablet_security_policy_test.go +++ b/go/test/endtoend/tabletmanager/tablet_security_policy_test.go @@ -5,7 +5,7 @@ Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 + http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, diff --git a/go/test/endtoend/tabletmanager/throttler/throttler_test.go b/go/test/endtoend/tabletmanager/throttler/throttler_test.go index d87f4d3abaa..dfd03299063 100644 --- a/go/test/endtoend/tabletmanager/throttler/throttler_test.go +++ b/go/test/endtoend/tabletmanager/throttler/throttler_test.go @@ -5,7 +5,7 @@ Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 + http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, diff --git a/go/test/endtoend/tabletmanager/throttler_custom_config/throttler_test.go b/go/test/endtoend/tabletmanager/throttler_custom_config/throttler_test.go index 3c21c4d9c5e..422309e0452 100644 --- a/go/test/endtoend/tabletmanager/throttler_custom_config/throttler_test.go +++ b/go/test/endtoend/tabletmanager/throttler_custom_config/throttler_test.go @@ -5,7 +5,7 @@ Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 + http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, diff --git a/go/test/endtoend/topoconncache/main_test.go b/go/test/endtoend/topoconncache/main_test.go index c8d71027bcf..7b02ac6fe9c 100644 --- a/go/test/endtoend/topoconncache/main_test.go +++ b/go/test/endtoend/topoconncache/main_test.go @@ -90,7 +90,9 @@ var ( shard2 cluster.Shard ) -/* This end-to-end test validates the cache fix in topo/server.go inside ConnForCell. +/* + This end-to-end test validates the cache fix in topo/server.go inside ConnForCell. + The issue was, if we delete and add back a cell with same name but at different path, the map of cells in server.go returned the connection object of the previous cell instead of newly-created one with the same name. diff --git a/go/test/endtoend/utils/mysql.go b/go/test/endtoend/utils/mysql.go index 22b6d7ddc09..0f373f929a8 100644 --- a/go/test/endtoend/utils/mysql.go +++ b/go/test/endtoend/utils/mysql.go @@ -68,7 +68,7 @@ func NewMySQL(cluster *cluster.LocalProcessCluster, dbName string, schemaSQL ... } for _, sql := range schemaSQL { - err = prepareMySQLWithSchema(err, params, sql) + err = prepareMySQLWithSchema(params, sql) if err != nil { return mysql.ConnParams{}, nil, err } @@ -119,7 +119,7 @@ func initMysqld(mysqld *mysqlctl.Mysqld, mycnf *mysqlctl.Mycnf, initSQLFile stri return nil } -func prepareMySQLWithSchema(err error, params mysql.ConnParams, sql string) error { +func prepareMySQLWithSchema(params mysql.ConnParams, sql string) error { ctx := context.Background() conn, err := mysql.Connect(ctx, ¶ms) if err != nil { diff --git a/go/test/endtoend/vreplication/config_test.go b/go/test/endtoend/vreplication/config_test.go index dd566e35f5b..95246f5d849 100644 --- a/go/test/endtoend/vreplication/config_test.go +++ b/go/test/endtoend/vreplication/config_test.go @@ -21,12 +21,14 @@ package vreplication // MySQL 5.7+ and MariaDB 10.2+ to ensure that vreplication still works everywhere and the // permissive sql_mode now used in vreplication causes no unwanted side effects. // The customer table also tests two important things: -// 1. Composite or multi-column primary keys -// 2. PKs that contain an ENUM column +// 1. Composite or multi-column primary keys +// 2. PKs that contain an ENUM column +// // The Lead and Lead-1 tables also allows us to test several things: -// 1. Mixed case identifiers -// 2. Column and table names with special characters in them, namely a dash -// 3. Identifiers using reserved words, as lead is a reserved word in MySQL 8.0+ (https://dev.mysql.com/doc/refman/8.0/en/keywords.html) +// 1. Mixed case identifiers +// 2. Column and table names with special characters in them, namely a dash +// 3. Identifiers using reserved words, as lead is a reserved word in MySQL 8.0+ (https://dev.mysql.com/doc/refman/8.0/en/keywords.html) +// // The internal table _vt_PURGE_4f9194b43b2011eb8a0104ed332e05c2_20221210194431 should be ignored by vreplication // The db_order_test table is used to ensure vreplication and vdiff work well with complex non-integer PKs, even across DB versions. var ( diff --git a/go/test/endtoend/vreplication/vreplication_test.go b/go/test/endtoend/vreplication/vreplication_test.go index bcdda81ac8a..a3a88a7fc7f 100644 --- a/go/test/endtoend/vreplication/vreplication_test.go +++ b/go/test/endtoend/vreplication/vreplication_test.go @@ -159,8 +159,8 @@ func TestVreplicationCopyThrottling(t *testing.T) { } func TestBasicVreplicationWorkflow(t *testing.T) { - sourceKsOpts["DBTypeVersion"] = "mysql-5.7" - targetKsOpts["DBTypeVersion"] = "mysql-5.7" + sourceKsOpts["DBTypeVersion"] = "mysql-8.0" + targetKsOpts["DBTypeVersion"] = "mysql-8.0" testBasicVreplicationWorkflow(t) } @@ -1178,6 +1178,7 @@ func dropSources(t *testing.T, ksWorkflow string) { // This allows us to confirm two behaviors: // 1. MoveTables blocks on starting its first copy phase until we rollback // 2. All other workflows continue to work w/o issue with this MVCC history in place (not used yet) +// // Returns a db connection used for the transaction which you can use for follow-up // work, such as rolling it back directly or using the releaseInnoDBRowHistory call. func generateInnoDBRowHistory(t *testing.T, sourceKS string, neededTrxHistory int64) *mysql.Conn { diff --git a/go/test/endtoend/vtgate/queries/aggregation/aggregation_test.go b/go/test/endtoend/vtgate/queries/aggregation/aggregation_test.go index 8467cb30abc..a4088f567f3 100644 --- a/go/test/endtoend/vtgate/queries/aggregation/aggregation_test.go +++ b/go/test/endtoend/vtgate/queries/aggregation/aggregation_test.go @@ -61,9 +61,9 @@ func TestAggregateTypes(t *testing.T) { // TODO (@frouioui): following assertions produce different results between MySQL and Vitess // their differences are ignored for now. Fix it. // `ascii(val1)` returns an `INT64` on Vitess, and `INT32` on MySQL - utils.AssertMatches(t, mcmp.VtConn, "select ascii(val1) as a, count(*) from aggr_test group by a", `[[INT64(65) INT64(1)] [INT64(69) INT64(1)] [INT64(97) INT64(1)] [INT64(98) INT64(1)] [INT64(99) INT64(2)] [INT64(100) INT64(1)] [INT64(101) INT64(1)]]`) - utils.AssertMatches(t, mcmp.VtConn, "select ascii(val1) as a, count(*) from aggr_test group by a order by a", `[[INT64(65) INT64(1)] [INT64(69) INT64(1)] [INT64(97) INT64(1)] [INT64(98) INT64(1)] [INT64(99) INT64(2)] [INT64(100) INT64(1)] [INT64(101) INT64(1)]]`) - utils.AssertMatches(t, mcmp.VtConn, "select ascii(val1) as a, count(*) from aggr_test group by a order by 2, a", `[[INT64(65) INT64(1)] [INT64(69) INT64(1)] [INT64(97) INT64(1)] [INT64(98) INT64(1)] [INT64(100) INT64(1)] [INT64(101) INT64(1)] [INT64(99) INT64(2)]]`) + utils.AssertMatches(t, mcmp.VtConn, "select ascii(val1) as a, count(*) from aggr_test group by a", `[[INT32(65) INT64(1)] [INT32(69) INT64(1)] [INT32(97) INT64(1)] [INT32(98) INT64(1)] [INT32(99) INT64(2)] [INT32(100) INT64(1)] [INT32(101) INT64(1)]]`) + utils.AssertMatches(t, mcmp.VtConn, "select ascii(val1) as a, count(*) from aggr_test group by a order by a", `[[INT32(65) INT64(1)] [INT32(69) INT64(1)] [INT32(97) INT64(1)] [INT32(98) INT64(1)] [INT32(99) INT64(2)] [INT32(100) INT64(1)] [INT32(101) INT64(1)]]`) + utils.AssertMatches(t, mcmp.VtConn, "select ascii(val1) as a, count(*) from aggr_test group by a order by 2, a", `[[INT32(65) INT64(1)] [INT32(69) INT64(1)] [INT32(97) INT64(1)] [INT32(98) INT64(1)] [INT32(100) INT64(1)] [INT32(101) INT64(1)] [INT32(99) INT64(2)]]`) mcmp.AssertMatches("select val1 as a, count(*) from aggr_test group by a", `[[VARCHAR("a") INT64(2)] [VARCHAR("b") INT64(1)] [VARCHAR("c") INT64(2)] [VARCHAR("d") INT64(1)] [VARCHAR("e") INT64(2)]]`) mcmp.AssertMatches("select val1 as a, count(*) from aggr_test group by a order by a", `[[VARCHAR("a") INT64(2)] [VARCHAR("b") INT64(1)] [VARCHAR("c") INT64(2)] [VARCHAR("d") INT64(1)] [VARCHAR("e") INT64(2)]]`) diff --git a/go/test/endtoend/vtgate/queries/aggregation/main_test.go b/go/test/endtoend/vtgate/queries/aggregation/main_test.go index 93968e861f7..084b2332791 100644 --- a/go/test/endtoend/vtgate/queries/aggregation/main_test.go +++ b/go/test/endtoend/vtgate/queries/aggregation/main_test.go @@ -63,14 +63,19 @@ func TestMain(m *testing.M) { SchemaSQL: schemaSQL, VSchema: vschema, } - clusterInstance.VtGateExtraArgs = []string{"--schema_change_signal"} - clusterInstance.VtTabletExtraArgs = []string{"--queryserver-config-schema-change-signal", "--queryserver-config-schema-change-signal-interval", "0.1"} + + clusterInstance.VtTabletExtraArgs = append(clusterInstance.VtTabletExtraArgs, + "--enable_query_plan_field_caching=false", + "--queryserver-config-schema-change-signal", + "--queryserver-config-schema-change-signal-interval", "0.1") err = clusterInstance.StartKeyspace(*keyspace, []string{"-80", "80-"}, 0, false) if err != nil { return 1 } - clusterInstance.VtGateExtraArgs = append(clusterInstance.VtGateExtraArgs, "--enable_system_settings=true") + clusterInstance.VtGateExtraArgs = append(clusterInstance.VtGateExtraArgs, + "--schema_change_signal", + "--enable_system_settings=true") // Start vtgate err = clusterInstance.StartVtgate() if err != nil { diff --git a/go/test/endtoend/vtgate/queries/informationschema/informationschema_test.go b/go/test/endtoend/vtgate/queries/informationschema/informationschema_test.go index 140cb46a0a8..a973c6e2337 100644 --- a/go/test/endtoend/vtgate/queries/informationschema/informationschema_test.go +++ b/go/test/endtoend/vtgate/queries/informationschema/informationschema_test.go @@ -106,7 +106,8 @@ func TestFKConstraintUsingInformationSchema(t *testing.T) { defer closer() query := "select fk.referenced_table_name as to_table, fk.referenced_column_name as primary_key, fk.column_name as `column`, fk.constraint_name as name, rc.update_rule as on_update, rc.delete_rule as on_delete from information_schema.referential_constraints as rc join information_schema.key_column_usage as fk on fk.constraint_schema = rc.constraint_schema and fk.constraint_name = rc.constraint_name where fk.referenced_column_name is not null and fk.table_schema = database() and fk.table_name = 't7_fk' and rc.constraint_schema = database() and rc.table_name = 't7_fk'" - mcmp.AssertMatches(query, `[[VARCHAR("t7_xxhash") VARCHAR("uid") VARCHAR("t7_uid") VARCHAR("t7_fk_ibfk_1") VARCHAR("CASCADE") VARCHAR("SET NULL")]]`) + mcmp.AssertMatches(query, + `[[VARBINARY("t7_xxhash") VARCHAR("uid") VARCHAR("t7_uid") VARCHAR("t7_fk_ibfk_1") BINARY("CASCADE") BINARY("SET NULL")]]`) } func TestConnectWithSystemSchema(t *testing.T) { diff --git a/go/test/endtoend/vtgate/unsharded/main_test.go b/go/test/endtoend/vtgate/unsharded/main_test.go index 7249f8c1c2f..f1bfeecae82 100644 --- a/go/test/endtoend/vtgate/unsharded/main_test.go +++ b/go/test/endtoend/vtgate/unsharded/main_test.go @@ -297,7 +297,7 @@ func TestDDLUnsharded(t *testing.T) { utils.AssertMatches(t, conn, "select * from v1", `[[INT64(3) INT64(0) INT64(3) VARCHAR("a")] [INT64(30) INT64(10) INT64(30) VARCHAR("ac")] [INT64(300) INT64(100) INT64(300) VARCHAR("abc")]]`) utils.Exec(t, conn, `drop view v1`) utils.Exec(t, conn, `drop table tempt1`) - utils.AssertMatches(t, conn, "show tables", `[[VARCHAR("allDefaults")] [VARCHAR("t1")]]`) + utils.AssertMatchesOneOf(t, conn, "show tables", `[[VARCHAR("allDefaults")] [VARCHAR("t1")]]`, `[[VARBINARY("allDefaults")] [VARBINARY("t1")]]`) } func TestCallProcedure(t *testing.T) { @@ -464,7 +464,7 @@ func TestFloatValueDefault(t *testing.T) { want57 := `[[VARCHAR("test_float_default") VARCHAR("pos_f") TEXT("2.1")] [VARCHAR("test_float_default") VARCHAR("neg_f") TEXT("-2.1")]]` want80 := `[[VARBINARY("test_float_default") VARCHAR("pos_f") BLOB("2.1")] [VARBINARY("test_float_default") VARCHAR("neg_f") BLOB("-2.1")]]` - query := "select table_name, column_name, column_default from information_schema.columns where table_name = 'test_float_default'" + query := "select table_name, column_name, column_default from information_schema.columns where table_name = 'test_float_default' order by column_default desc" utils.AssertMatchesOneOf(t, conn, query, want57, want80) } diff --git a/go/test/endtoend/vtorc/utils/utils.go b/go/test/endtoend/vtorc/utils/utils.go index b6020918b6a..96075aef3c0 100644 --- a/go/test/endtoend/vtorc/utils/utils.go +++ b/go/test/endtoend/vtorc/utils/utils.go @@ -19,7 +19,7 @@ package utils import ( "context" "fmt" - "io/ioutil" + "io" "net/http" "os" "os/exec" @@ -727,7 +727,7 @@ func MakeAPICall(t *testing.T, url string) (status int, response string) { res, err := http.Get(url) require.NoError(t, err) defer res.Body.Close() - bodyBytes, err := ioutil.ReadAll(res.Body) + bodyBytes, err := io.ReadAll(res.Body) require.NoError(t, err) body := string(bodyBytes) return res.StatusCode, body diff --git a/go/vt/mysqlctl/mysqld.go b/go/vt/mysqlctl/mysqld.go index 52699917df0..574c6380096 100644 --- a/go/vt/mysqlctl/mysqld.go +++ b/go/vt/mysqlctl/mysqld.go @@ -165,6 +165,7 @@ func NewMysqld(dbcfgs *dbconfigs.DBConfigs) *Mysqld { os.Getenv("VTROOT"), vtenvMysqlRoot, os.Getenv("MYSQL_FLAVOR")) + log.Errorf(message) panic(message) } } diff --git a/go/vt/mysqlctl/rice_box_test.go b/go/vt/mysqlctl/rice_box_test.go index e3126c44a35..c228c4ba5d3 100644 --- a/go/vt/mysqlctl/rice_box_test.go +++ b/go/vt/mysqlctl/rice_box_test.go @@ -18,7 +18,6 @@ package mysqlctl import ( "fmt" - "io/ioutil" "os" "path/filepath" "strings" @@ -55,7 +54,7 @@ func TestMySQLCtlRiceBox(t *testing.T) { require.NoError(t, err) require.NotNil(t, riceContents) - diskBytes, err := ioutil.ReadFile(fmt.Sprintf("%s/%s", mySQLCtlRiceBoxLocation, fileName)) + diskBytes, err := os.ReadFile(fmt.Sprintf("%s/%s", mySQLCtlRiceBoxLocation, fileName)) diskContents := string(diskBytes) require.NoError(t, err) require.Equal(t, riceContents, diskContents) diff --git a/go/vt/servenv/grpc_server.go b/go/vt/servenv/grpc_server.go index f25fff45b97..3b3d7659d61 100644 --- a/go/vt/servenv/grpc_server.go +++ b/go/vt/servenv/grpc_server.go @@ -48,11 +48,12 @@ import ( // Clients register servers, based on service map: // // servenv.RegisterGRPCFlags() -// servenv.OnRun(func() { -// if servenv.GRPCCheckServiceMap("XXX") { -// pb.RegisterXXX(servenv.GRPCServer, XXX) -// } -// } +// +// servenv.OnRun(func() { +// if servenv.GRPCCheckServiceMap("XXX") { +// pb.RegisterXXX(servenv.GRPCServer, XXX) +// } +// } // // Note servenv.GRPCServer can only be used in servenv.OnRun, // and not before, as it is initialized right before calling OnRun. diff --git a/go/vt/vtctld/vtctld_test.go b/go/vt/vtctld/vtctld_test.go index 437cc924d20..dec0e2d742d 100644 --- a/go/vt/vtctld/vtctld_test.go +++ b/go/vt/vtctld/vtctld_test.go @@ -18,7 +18,7 @@ package vtctld import ( "flag" - "io/ioutil" + "io" "net/http" "net/http/httptest" "testing" @@ -37,7 +37,7 @@ func TestWebApp(t *testing.T) { defer res.Body.Close() - data, err := ioutil.ReadAll(res.Body) + data, err := io.ReadAll(res.Body) assert.NoError(t, err) assert.Contains(t, string(data), "") } @@ -56,7 +56,7 @@ func TestWebAppDisabled(t *testing.T) { defer res.Body.Close() - data, err := ioutil.ReadAll(res.Body) + data, err := io.ReadAll(res.Body) assert.NoError(t, err) assert.Equal(t, "404 page not found\n", string(data)) } diff --git a/go/vt/workflow/parallel_runner.go b/go/vt/workflow/parallel_runner.go index 3292d2a1cac..9b8b4b0d193 100644 --- a/go/vt/workflow/parallel_runner.go +++ b/go/vt/workflow/parallel_runner.go @@ -430,7 +430,7 @@ func VerifyAllTasksDone(ctx context.Context, ts *topo.Server, uuid string) error return verifyAllTasksState(ctx, ts, uuid, workflowpb.TaskState_TaskDone) } -//verifyAllTasksState verifies that all tasks are in taskState. Only for tests purposes. +// verifyAllTasksState verifies that all tasks are in taskState. Only for tests purposes. func verifyAllTasksState(ctx context.Context, ts *topo.Server, uuid string, taskState workflowpb.TaskState) error { checkpoint, err := checkpoint(ctx, ts, uuid) if err != nil { diff --git a/test/ci_workflow_gen.go b/test/ci_workflow_gen.go index 800254b5db2..ac26cbfafcb 100644 --- a/test/ci_workflow_gen.go +++ b/test/ci_workflow_gen.go @@ -33,7 +33,7 @@ const ( mysql80 mysqlVersion = "mysql80" mariadb103 mysqlVersion = "mariadb103" - defaultMySQLVersion mysqlVersion = mysql57 + defaultMySQLVersion = mysql80 ) type mysqlVersions []mysqlVersion @@ -82,7 +82,6 @@ var ( "22", "worker_vault_heavy", "24", - "26", "vstream_failover", "vstream_stoponreshard_true", "vstream_stoponreshard_false", @@ -121,6 +120,7 @@ var ( "xb_recovery", "resharding", "resharding_bytes", + "mysql57", "mysql80", "vreplication_across_db_versions", "vreplication_multicell", @@ -155,7 +155,7 @@ type clusterTest struct { Name, Shard, Platform string FileName string MakeTools, InstallXtraBackup bool - Ubuntu20, Docker bool + Docker bool LimitResourceUsage bool } @@ -174,12 +174,8 @@ func clusterMySQLVersions(clusterName string) mysqlVersions { return allMySQLVersions case clusterName == "tabletmanager_tablegc": return allMySQLVersions - case clusterName == "mysql80": - return []mysqlVersion{mysql80} - case clusterName == "vtorc_8.0": - return []mysqlVersion{mysql80} - case clusterName == "vreplication_across_db_versions": - return []mysqlVersion{mysql80} + case clusterName == "mysql57": + return []mysqlVersion{mysql57} default: return defaultMySQLVersions } @@ -271,10 +267,15 @@ func generateSelfHostedClusterWorkflows() error { clusters := canonnizeList(clusterSelfHostedList) for _, cluster := range clusters { for _, mysqlVersion := range clusterMySQLVersions(cluster) { - directoryName := fmt.Sprintf("cluster_test_%s", cluster) + mysqlVersionIndicator := "" + if mysqlVersion != defaultMySQLVersion && len(clusterMySQLVersions(cluster)) > 1 { + mysqlVersionIndicator = "_" + string(mysqlVersion) + } + + directoryName := fmt.Sprintf("cluster_test_%s%s", cluster, mysqlVersionIndicator) test := &selfHostedTest{ - Name: fmt.Sprintf("Cluster (%s)", cluster), - ImageName: fmt.Sprintf("cluster_test_%s", cluster), + Name: fmt.Sprintf("Cluster (%s)(%s)", cluster, mysqlVersion), + ImageName: fmt.Sprintf("cluster_test_%s%s", cluster, mysqlVersionIndicator), Platform: "mysql57", directoryName: directoryName, Dockerfile: fmt.Sprintf("./.github/docker/%s/Dockerfile", directoryName), @@ -299,11 +300,6 @@ func generateSelfHostedClusterWorkflows() error { if mysqlVersion == mysql80 { test.Platform = string(mysql80) } - mysqlVersionIndicator := "" - if mysqlVersion != defaultMySQLVersion && len(clusterMySQLVersions(cluster)) > 1 { - mysqlVersionIndicator = "_" + string(mysqlVersion) - } - err := setupTestDockerFile(test) if err != nil { return err @@ -342,9 +338,8 @@ func generateClusterWorkflows(list []string, tpl string) { break } } - if mysqlVersion == mysql80 { - test.Ubuntu20 = true - test.Platform = string(mysql80) + if mysqlVersion == mysql57 { + test.Platform = string(mysql57) } if strings.HasPrefix(cluster, "vreplication") || strings.HasSuffix(cluster, "heavy") { test.LimitResourceUsage = true diff --git a/test/config.json b/test/config.json index 7dfcc44bf98..67be2465ec6 100644 --- a/test/config.json +++ b/test/config.json @@ -420,17 +420,6 @@ "site_test" ] }, - "pitrtls": { - "File": "unused.go", - "Args": ["vitess.io/vitess/go/test/endtoend/recovery/pitrtls"], - "Command": [], - "Manual": false, - "Shard": "26", - "RetryMax": 1, - "Tags": [ - "site_test" - ] - }, "recovery": { "File": "unused.go", "Args": ["vitess.io/vitess/go/test/endtoend/recovery/unshardedrecovery"], @@ -500,7 +489,7 @@ }, "sharded_recovery": { "File": "unused.go", - "Args": ["vitess.io/vitess/go/test/endtoend/recovery/shardedrecovery"], + "Args": ["vitess.io/vitess/go/test/endtoend/recovery/shardedrecovery", "-timeout", "20m"], "Command": [], "Manual": false, "Shard": "shardedrecovery_stress_verticalsplit_heavy", @@ -509,7 +498,7 @@ }, "stress": { "File": "unused.go", - "Args": ["vitess.io/vitess/go/test/endtoend/stress"], + "Args": ["vitess.io/vitess/go/test/endtoend/stress", "-timeout", "20m"], "Command": [], "Manual": false, "Shard": "shardedrecovery_stress_verticalsplit_heavy", @@ -624,7 +613,7 @@ }, "vertical_split": { "File": "unused.go", - "Args": ["vitess.io/vitess/go/test/endtoend/sharding/verticalsplit"], + "Args": ["vitess.io/vitess/go/test/endtoend/sharding/verticalsplit", "-timeout", "20m"], "Command": [], "Manual": false, "Shard": "shardedrecovery_stress_verticalsplit_heavy", diff --git a/test/templates/cluster_endtoend_test.tpl b/test/templates/cluster_endtoend_test.tpl index d2d4d6b75f1..716694df224 100644 --- a/test/templates/cluster_endtoend_test.tpl +++ b/test/templates/cluster_endtoend_test.tpl @@ -14,7 +14,7 @@ env: jobs: build: name: Run endtoend tests on {{.Name}} - {{if .Ubuntu20}}runs-on: ubuntu-20.04{{else}}runs-on: ubuntu-18.04{{end}} + runs-on: ubuntu-20.04 steps: - name: Check if workflow needs to be skipped @@ -64,6 +64,8 @@ jobs: - name: Tune the OS if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf @@ -72,10 +74,36 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + {{if .InstallXtraBackup}} + + # Setup Percona Server for MySQL 8.0 sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata - sudo service mysql stop + sudo apt-get install -y lsb-release gnupg2 curl + wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb + sudo percona-release setup ps80 + sudo apt-get update + + # Install everything else we need, and configure + sudo apt-get install -y percona-server-server percona-server-client make unzip g++ etcd git wget eatmydata xz-utils + {{else}} + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + # Setup MySQL 8.0 + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* + sudo apt-get update + # Install everything else we need, and configure + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get update + {{end}} + sudo service etcd stop + echo 'mysql version: ' + sudo mysql --version + sudo service mysql stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download @@ -84,19 +112,7 @@ jobs: go install github.com/vitessio/go-junit-report@HEAD {{if .InstallXtraBackup}} - - wget "https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb" - sudo apt-get install -y gnupg2 - sudo dpkg -i "percona-release_latest.$(lsb_release -sc)_all.deb" - sudo apt-get update - if [[ -n $XTRABACKUP_VERSION ]]; then - debfile="percona-xtrabackup-24_$XTRABACKUP_VERSION.$(lsb_release -sc)_amd64.deb" - wget "https://repo.percona.com/pxb-24/apt/pool/main/p/percona-xtrabackup-24/$debfile" - sudo apt install -y "./$debfile" - else - sudo apt-get install -y percona-xtrabackup-24 - fi - + sudo apt-get install percona-xtrabackup-80 lz4 {{end}} {{if .MakeTools}} @@ -110,7 +126,7 @@ jobs: - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. @@ -124,10 +140,11 @@ jobs: sudo sysctl -w net.ipv4.ip_local_port_range="22768 61999" # Increase our open file descriptor limit as we could hit this ulimit -n 65536 - cat <<-EOF>>./config/mycnf/mysql57.cnf + cat <<-EOF>>./config/mycnf/mysql80.cnf innodb_buffer_pool_dump_at_shutdown=OFF + innodb_buffer_pool_in_core_file=OFF innodb_buffer_pool_load_at_startup=OFF - innodb_buffer_pool_size=64M + innodb_buffer_pool_size=16M innodb_doublewrite=OFF innodb_flush_log_at_trx_commit=0 innodb_flush_method=O_DIRECT @@ -138,6 +155,7 @@ jobs: performance_schema=OFF slow-query-log=OFF EOF + make embed_config {{end}} # run the tests however you normally do, then produce a JUnit XML file diff --git a/test/templates/cluster_endtoend_test_docker.tpl b/test/templates/cluster_endtoend_test_docker.tpl index 38316965d2a..75b2c0c8c22 100644 --- a/test/templates/cluster_endtoend_test_docker.tpl +++ b/test/templates/cluster_endtoend_test_docker.tpl @@ -4,7 +4,7 @@ on: [push, pull_request] jobs: build: name: Run endtoend tests on {{.Name}} - {{if .Ubuntu20}}runs-on: ubuntu-20.04{{else}}runs-on: ubuntu-latest{{end}} + runs-on: ubuntu-20.04 steps: - name: Check if workflow needs to be skipped diff --git a/test/templates/cluster_endtoend_test_mysql80.tpl b/test/templates/cluster_endtoend_test_mysql57.tpl similarity index 75% rename from test/templates/cluster_endtoend_test_mysql80.tpl rename to test/templates/cluster_endtoend_test_mysql57.tpl index a53a2cbf82f..e225ee214b9 100644 --- a/test/templates/cluster_endtoend_test_mysql80.tpl +++ b/test/templates/cluster_endtoend_test_mysql57.tpl @@ -7,7 +7,8 @@ concurrency: jobs: build: name: Run endtoend tests on {{.Name}} - {{if .Ubuntu20}}runs-on: ubuntu-20.04{{else}}runs-on: ubuntu-18.04{{end}} + runs-on: ubuntu-20.04 + timeout-minutes: 45 steps: - name: Check if workflow needs to be skipped @@ -65,33 +66,53 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + sudo apt-get update + + # Uninstall any previously installed MySQL first + sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ + sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + + sudo systemctl stop apparmor + sudo DEBIAN_FRONTEND="noninteractive" apt-get remove -y --purge mysql-server mysql-client mysql-common + sudo apt-get -y autoremove + sudo apt-get -y autoclean + sudo deluser mysql + sudo rm -rf /var/lib/mysql + sudo rm -rf /etc/mysql + # Get key to latest MySQL repo sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 - # Setup MySQL 8.0 - wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb - echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.14-1_all.deb + # Bionic packages are still compatible for Focal since there's no MySQL 5.7 + # packages for Focal. + echo mysql-apt-config mysql-apt-config/repo-codename select bionic | sudo debconf-set-selections + echo mysql-apt-config mysql-apt-config/select-server select mysql-5.7 | sudo debconf-set-selections sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update + sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y mysql-client=5.7* mysql-community-server=5.7* mysql-server=5.7* # Install everything else we need, and configure - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get install -y make unzip g++ etcd curl git wget eatmydata sudo service mysql stop sudo service etcd stop - sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ - sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld - go mod download # install JUnit report formatter go install github.com/vitessio/go-junit-report@HEAD {{if .InstallXtraBackup}} - wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb + wget "https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb" sudo apt-get install -y gnupg2 - sudo dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb + sudo dpkg -i "percona-release_latest.$(lsb_release -sc)_all.deb" sudo apt-get update - sudo apt-get install percona-xtrabackup-24 + if [[ -n $XTRABACKUP_VERSION ]]; then + debfile="percona-xtrabackup-24_$XTRABACKUP_VERSION.$(lsb_release -sc)_amd64.deb" + wget "https://repo.percona.com/pxb-24/apt/pool/main/p/percona-xtrabackup-24/$debfile" + sudo apt install -y "./$debfile" + else + sudo apt-get install -y percona-xtrabackup-24 + fi {{end}} @@ -106,7 +127,7 @@ jobs: - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 + timeout-minutes: 45 run: | # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file # which musn't be more than 107 characters long. diff --git a/test/templates/unit_test.tpl b/test/templates/unit_test.tpl index 1d1f34b271b..9bb7757c5e6 100644 --- a/test/templates/unit_test.tpl +++ b/test/templates/unit_test.tpl @@ -6,7 +6,7 @@ concurrency: jobs: test: - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 steps: - name: Check if workflow needs to be skipped