From f606e363f41c1ccc243091198d7b1ad5f320b0d8 Mon Sep 17 00:00:00 2001 From: Tanjin Xu Date: Fri, 19 Apr 2024 17:25:16 -0700 Subject: [PATCH] fix tests --- .github/workflows/cluster_endtoend_12.yml | 7 +- .github/workflows/cluster_endtoend_13.yml | 7 +- .github/workflows/cluster_endtoend_15.yml | 7 +- .github/workflows/cluster_endtoend_21.yml | 7 +- .github/workflows/cluster_endtoend_22.yml | 7 +- .github/workflows/cluster_endtoend_26.yml | 47 ++++-- ...ter_endtoend_ers_prs_newfeatures_heavy.yml | 7 +- .../workflows/cluster_endtoend_mysql57.yml | 7 +- .../workflows/cluster_endtoend_mysql80.yml | 113 ++++++------- .../cluster_endtoend_mysql_server_vault.yml | 22 +-- ...cluster_endtoend_onlineddl_declarative.yml | 7 +- ...endtoend_onlineddl_declarative_mysql57.yml | 7 +- .../cluster_endtoend_onlineddl_ghost.yml | 7 +- ...uster_endtoend_onlineddl_ghost_mysql57.yml | 7 +- .../cluster_endtoend_onlineddl_revert.yml | 7 +- ...ster_endtoend_onlineddl_revert_mysql57.yml | 7 +- .../cluster_endtoend_onlineddl_revertible.yml | 7 +- ..._endtoend_onlineddl_revertible_mysql57.yml | 7 +- .../cluster_endtoend_onlineddl_scheduler.yml | 7 +- ...r_endtoend_onlineddl_scheduler_mysql57.yml | 7 +- .../cluster_endtoend_onlineddl_singleton.yml | 7 +- ...r_endtoend_onlineddl_singleton_mysql57.yml | 7 +- .../cluster_endtoend_onlineddl_vrepl.yml | 7 +- ...uster_endtoend_onlineddl_vrepl_mysql57.yml | 7 +- ...luster_endtoend_onlineddl_vrepl_stress.yml | 7 +- ...ndtoend_onlineddl_vrepl_stress_mysql57.yml | 7 +- ..._endtoend_onlineddl_vrepl_stress_suite.yml | 7 +- ...d_onlineddl_vrepl_stress_suite_mysql57.yml | 7 +- ...cluster_endtoend_onlineddl_vrepl_suite.yml | 7 +- ...endtoend_onlineddl_vrepl_suite_mysql57.yml | 7 +- .../cluster_endtoend_schemadiff_vrepl.yml | 7 +- ...ster_endtoend_schemadiff_vrepl_mysql57.yml | 7 +- .../cluster_endtoend_tabletmanager_consul.yml | 7 +- ...cluster_endtoend_tabletmanager_tablegc.yml | 7 +- ...endtoend_tabletmanager_tablegc_mysql57.yml | 7 +- ...uster_endtoend_tabletmanager_throttler.yml | 7 +- ..._tabletmanager_throttler_custom_config.yml | 7 +- ...cluster_endtoend_topo_connection_cache.yml | 7 +- ...dtoend_vreplication_across_db_versions.yml | 154 +++++++++--------- .../cluster_endtoend_vreplication_basic.yml | 7 +- ...luster_endtoend_vreplication_cellalias.yml | 7 +- .../cluster_endtoend_vreplication_migrate.yml | 7 +- ...luster_endtoend_vreplication_multicell.yml | 7 +- .../cluster_endtoend_vreplication_v2.yml | 7 +- .../cluster_endtoend_vstream_failover.yml | 7 +- ...r_endtoend_vstream_stoponreshard_false.yml | 7 +- ...er_endtoend_vstream_stoponreshard_true.yml | 7 +- ...dtoend_vstream_with_keyspaces_to_watch.yml | 7 +- .../cluster_endtoend_vtbackup_transform.yml | 113 ++++++------- ..._vtctlbackup_sharded_clustertest_heavy.yml | 154 +++++++++--------- .../cluster_endtoend_vtgate_concurrentdml.yml | 7 +- .../cluster_endtoend_vtgate_gen4.yml | 7 +- .../cluster_endtoend_vtgate_general_heavy.yml | 7 +- .../cluster_endtoend_vtgate_godriver.yml | 7 +- .../cluster_endtoend_vtgate_queries.yml | 7 +- ...cluster_endtoend_vtgate_readafterwrite.yml | 7 +- .../cluster_endtoend_vtgate_reservedconn.yml | 7 +- .../cluster_endtoend_vtgate_schema.yml | 7 +- ...cluster_endtoend_vtgate_schema_tracker.yml | 7 +- ...dtoend_vtgate_tablet_healthcheck_cache.yml | 7 +- .../cluster_endtoend_vtgate_topo.yml | 7 +- .../cluster_endtoend_vtgate_topo_consul.yml | 7 +- .../cluster_endtoend_vtgate_topo_etcd.yml | 7 +- .../cluster_endtoend_vtgate_transaction.yml | 7 +- .../cluster_endtoend_vtgate_unsharded.yml | 7 +- .../cluster_endtoend_vtgate_vindex_heavy.yml | 7 +- .../cluster_endtoend_vtgate_vschema.yml | 7 +- .github/workflows/cluster_endtoend_vtorc.yml | 7 +- .../workflows/cluster_endtoend_vtorc_8.0.yml | 113 ++++++------- .../workflows/cluster_endtoend_xb_backup.yml | 5 +- .../cluster_endtoend_xb_backup_mysql57.yml | 70 +++----- .../cluster_endtoend_xb_recovery.yml | 5 +- .github/workflows/unit_test_mysql57.yml | 5 +- .github/workflows/unit_test_mysql80.yml | 7 +- 74 files changed, 567 insertions(+), 675 deletions(-) diff --git a/.github/workflows/cluster_endtoend_12.yml b/.github/workflows/cluster_endtoend_12.yml index 232dd7f495..a0d08659ed 100644 --- a/.github/workflows/cluster_endtoend_12.yml +++ b/.github/workflows/cluster_endtoend_12.yml @@ -49,10 +49,9 @@ jobs: - name: Set up Go if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - uses: actions/setup-go@v4 + uses: actions/setup-go@v3 with: - go-version: '1.20' - check-latest: true + go-version: 1.19.12 - name: Set up python if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' @@ -73,7 +72,7 @@ jobs: run: | # Get key to latest MySQL repo - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 # Setup MySQL 8.0 wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.29-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections diff --git a/.github/workflows/cluster_endtoend_13.yml b/.github/workflows/cluster_endtoend_13.yml index ffe8ae9477..4be5eeb894 100644 --- a/.github/workflows/cluster_endtoend_13.yml +++ b/.github/workflows/cluster_endtoend_13.yml @@ -49,10 +49,9 @@ jobs: - name: Set up Go if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - uses: actions/setup-go@v4 + uses: actions/setup-go@v3 with: - go-version: '1.20' - check-latest: true + go-version: 1.19.12 - name: Set up python if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' @@ -73,7 +72,7 @@ jobs: run: | # Get key to latest MySQL repo - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 # Setup MySQL 8.0 wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.29-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections diff --git a/.github/workflows/cluster_endtoend_15.yml b/.github/workflows/cluster_endtoend_15.yml index cb7169b7db..2c8d354f5f 100644 --- a/.github/workflows/cluster_endtoend_15.yml +++ b/.github/workflows/cluster_endtoend_15.yml @@ -49,10 +49,9 @@ jobs: - name: Set up Go if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - uses: actions/setup-go@v4 + uses: actions/setup-go@v3 with: - go-version: '1.20' - check-latest: true + go-version: 1.19.12 - name: Set up python if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' @@ -73,7 +72,7 @@ jobs: run: | # Get key to latest MySQL repo - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 # Setup MySQL 8.0 wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.29-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections diff --git a/.github/workflows/cluster_endtoend_21.yml b/.github/workflows/cluster_endtoend_21.yml index a0a6ec48f0..41b454b3c0 100644 --- a/.github/workflows/cluster_endtoend_21.yml +++ b/.github/workflows/cluster_endtoend_21.yml @@ -49,10 +49,9 @@ jobs: - name: Set up Go if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - uses: actions/setup-go@v4 + uses: actions/setup-go@v3 with: - go-version: '1.20' - check-latest: true + go-version: 1.19.12 - name: Set up python if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' @@ -73,7 +72,7 @@ jobs: run: | # Get key to latest MySQL repo - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 # Setup MySQL 8.0 wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.29-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections diff --git a/.github/workflows/cluster_endtoend_22.yml b/.github/workflows/cluster_endtoend_22.yml index febc8a760f..8bbc175e39 100644 --- a/.github/workflows/cluster_endtoend_22.yml +++ b/.github/workflows/cluster_endtoend_22.yml @@ -49,10 +49,9 @@ jobs: - name: Set up Go if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - uses: actions/setup-go@v4 + uses: actions/setup-go@v3 with: - go-version: '1.20' - check-latest: true + go-version: 1.19.12 - name: Set up python if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' @@ -73,7 +72,7 @@ jobs: run: | # Get key to latest MySQL repo - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 # Setup MySQL 8.0 wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.29-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections diff --git a/.github/workflows/cluster_endtoend_26.yml b/.github/workflows/cluster_endtoend_26.yml index d6ae3bb249..8208972f35 100644 --- a/.github/workflows/cluster_endtoend_26.yml +++ b/.github/workflows/cluster_endtoend_26.yml @@ -17,7 +17,7 @@ jobs: id: skip-workflow run: | skip='false' - if [[ "${{github.event.pull_request}}" == "" ]] && [[ "${{github.ref}}" != "refs/heads/main" ]] && [[ ! "${{github.ref}}" =~ ^refs/heads/release-[0-9]+\.[0-9]$ ]] && [[ ! "${{github.ref}}" =~ "refs/tags/.*" ]]; then + if [[ "${{github.event.pull_request}}" == "" ]] && [[ "${{github.ref}}" != "refs/heads/main" ]] && [[ ! "${{github.ref}}" =~ ^refs/heads/slack-vitess-r[0-9]+\.[0-9]\.[0-9]+$ ]] && [[ ! "${{github.ref}}" =~ "refs/tags/.*" ]]; then skip='true' fi echo Skip ${skip} @@ -49,10 +49,9 @@ jobs: - name: Set up Go if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - uses: actions/setup-go@v4 + uses: actions/setup-go@v3 with: - go-version: '1.20' - check-latest: true + go-version: 1.19.12 - name: Set up python if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' @@ -63,7 +62,7 @@ jobs: run: | # Limit local port range to not use ports that overlap with server side # ports that we listen on. - sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" + echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf sudo sysctl -p /etc/sysctl.conf @@ -71,10 +70,22 @@ jobs: - name: Get dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + # Setup MySQL 8.0 + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.29-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata - sudo service mysql stop + # Install everything else we need, and configure + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get update + sudo service etcd stop + echo 'mysql version: ' + sudo mysql --version + sudo service mysql stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download @@ -84,14 +95,18 @@ jobs: - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 - run: | - # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file - # which musn't be more than 107 characters long. - export VTDATAROOT="/tmp/" - source build.env + uses: nick-fields/retry@v2 + with: + timeout_minutes: 45 + max_attempts: 3 + retry_on: error + command: | + # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file + # which musn't be more than 107 characters long. + export VTDATAROOT="/tmp/" + source build.env - set -x + set -x - # run the tests however you normally do, then produce a JUnit XML file - eatmydata -- go run test.go -docker=false -follow -shard 26 + # run the tests however you normally do, then produce a JUnit XML file + eatmydata -- go run test.go -docker=false -follow -shard 26 diff --git a/.github/workflows/cluster_endtoend_ers_prs_newfeatures_heavy.yml b/.github/workflows/cluster_endtoend_ers_prs_newfeatures_heavy.yml index a7944c4489..b72a8e763b 100644 --- a/.github/workflows/cluster_endtoend_ers_prs_newfeatures_heavy.yml +++ b/.github/workflows/cluster_endtoend_ers_prs_newfeatures_heavy.yml @@ -49,10 +49,9 @@ jobs: - name: Set up Go if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - uses: actions/setup-go@v4 + uses: actions/setup-go@v3 with: - go-version: '1.20' - check-latest: true + go-version: 1.19.12 - name: Set up python if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' @@ -73,7 +72,7 @@ jobs: run: | # Get key to latest MySQL repo - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 # Setup MySQL 8.0 wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.29-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections diff --git a/.github/workflows/cluster_endtoend_mysql57.yml b/.github/workflows/cluster_endtoend_mysql57.yml index 257ca54cd8..ff10577027 100644 --- a/.github/workflows/cluster_endtoend_mysql57.yml +++ b/.github/workflows/cluster_endtoend_mysql57.yml @@ -50,10 +50,9 @@ jobs: - name: Set up Go if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - uses: actions/setup-go@v4 + uses: actions/setup-go@v3 with: - go-version: '1.20' - check-latest: true + go-version: 1.19.12 - name: Set up python if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' @@ -85,7 +84,7 @@ jobs: sudo rm -rf /etc/mysql # Get key to latest MySQL repo - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.14-1_all.deb # Bionic packages are still compatible for Focal since there's no MySQL 5.7 diff --git a/.github/workflows/cluster_endtoend_mysql80.yml b/.github/workflows/cluster_endtoend_mysql80.yml index 94f8a0014c..16edae452b 100644 --- a/.github/workflows/cluster_endtoend_mysql80.yml +++ b/.github/workflows/cluster_endtoend_mysql80.yml @@ -6,23 +6,29 @@ concurrency: group: format('{0}-{1}', ${{ github.ref }}, 'Cluster (mysql80)') cancel-in-progress: true -env: - LAUNCHABLE_ORGANIZATION: "vitess" - LAUNCHABLE_WORKSPACE: "vitess-app" - GITHUB_PR_HEAD_SHA: "${{ github.event.pull_request.head.sha }}" - jobs: build: name: Run endtoend tests on Cluster (mysql80) runs-on: group: vitess-ubuntu20 - timeout-minutes: 45 steps: + - name: Check if workflow needs to be skipped + id: skip-workflow + run: | + skip='false' + if [[ "${{github.event.pull_request}}" == "" ]] && [[ "${{github.ref}}" != "refs/heads/main" ]] && [[ ! "${{github.ref}}" =~ ^refs/heads/slack-vitess-r[0-9]+\.[0-9]\.[0-9]+$ ]] && [[ ! "${{github.ref}}" =~ "refs/tags/.*" ]]; then + skip='true' + fi + echo Skip ${skip} + echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT + - name: Check out code - uses: actions/checkout@v2 + if: steps.skip-workflow.outputs.skip-workflow == 'false' + uses: actions/checkout@v3 - name: Check for changes in relevant files + if: steps.skip-workflow.outputs.skip-workflow == 'false' uses: frouioui/paths-filter@main id: changes with: @@ -33,47 +39,53 @@ jobs: - 'test.go' - 'Makefile' - 'build.env' - - 'go.[sumod]' + - 'go.sum' + - 'go.mod' - 'proto/*.proto' - 'tools/**' - 'config/**' - 'bootstrap.sh' - - '.github/workflows/**' + - '.github/workflows/cluster_endtoend_mysql80.yml' - name: Set up Go - if: steps.changes.outputs.end_to_end == 'true' - uses: actions/setup-go@v4 + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/setup-go@v3 with: - go-version: '1.20' - check-latest: true + go-version: 1.19.12 - name: Set up python - if: steps.changes.outputs.end_to_end == 'true' - uses: actions/setup-python@v2 + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/setup-python@v4 - name: Tune the OS - if: steps.changes.outputs.end_to_end == 'true' + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf sudo sysctl -p /etc/sysctl.conf - name: Get dependencies - if: steps.changes.outputs.end_to_end == 'true' + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | - # Setup Percona Server for MySQL 8.0 + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + # Setup MySQL 8.0 + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.29-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update - sudo apt-get install -y lsb-release gnupg2 curl - wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb - sudo DEBIAN_FRONTEND="noninteractive" dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb - sudo percona-release setup ps80 - sudo apt-get update - # Install everything else we need, and configure - sudo apt-get install -y percona-server-server percona-server-client make unzip g++ etcd git wget eatmydata xz-utils - sudo service mysql stop + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get update + sudo service etcd stop + echo 'mysql version: ' + sudo mysql --version + sudo service mysql stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download @@ -81,37 +93,20 @@ jobs: # install JUnit report formatter go install github.com/vitessio/go-junit-report@HEAD - - name: Setup launchable dependencies - if: steps.changes.outputs.end_to_end == 'true' - run: | - # Get Launchable CLI installed. If you can, make it a part of the builder image to speed things up - pip3 install --user launchable~=1.0 > /dev/null - - # verify that launchable setup is all correct. - launchable verify || true - - # Tell Launchable about the build you are producing and testing - launchable record build --name "$GITHUB_RUN_ID" --source . - - name: Run cluster endtoend test - if: steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 - run: | - # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file - # which musn't be more than 107 characters long. - export VTDATAROOT="/tmp/" - source build.env - - set -x - - # run the tests however you normally do, then produce a JUnit XML file - eatmydata -- go run test.go -docker=false -follow -shard mysql80 | tee -a output.txt | go-junit-report -set-exit-code > report.xml - - - name: Print test output and Record test result in launchable - if: steps.changes.outputs.end_to_end == 'true' && always() - run: | - # send recorded tests to launchable - launchable record tests --build "$GITHUB_RUN_ID" go-test . || true - - # print test output - cat output.txt + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: nick-fields/retry@v2 + with: + timeout_minutes: 45 + max_attempts: 3 + retry_on: error + command: | + # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file + # which musn't be more than 107 characters long. + export VTDATAROOT="/tmp/" + source build.env + + set -x + + # run the tests however you normally do, then produce a JUnit XML file + eatmydata -- go run test.go -docker=false -follow -shard mysql80 diff --git a/.github/workflows/cluster_endtoend_mysql_server_vault.yml b/.github/workflows/cluster_endtoend_mysql_server_vault.yml index 95dcc16c82..6038e2ea65 100644 --- a/.github/workflows/cluster_endtoend_mysql_server_vault.yml +++ b/.github/workflows/cluster_endtoend_mysql_server_vault.yml @@ -45,14 +45,13 @@ jobs: - 'tools/**' - 'config/**' - 'bootstrap.sh' - - '.github/workflows/cluster_endtoend_24.yml' + - '.github/workflows/cluster_endtoend_mysql_server_vault.yml' - name: Set up Go if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - uses: actions/setup-go@v4 + uses: actions/setup-go@v3 with: - go-version: '1.20' - check-latest: true + go-version: 1.19.12 - name: Set up python if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' @@ -73,7 +72,7 @@ jobs: run: | # Get key to latest MySQL repo - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 # Setup MySQL 8.0 wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.29-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections @@ -114,14 +113,5 @@ jobs: set -x - # run the tests however you normally do, then produce a JUnit XML file - eatmydata -- go run test.go -docker=false -follow -shard mysql_server_vault | tee -a output.txt | go-junit-report -set-exit-code > report.xml - - - name: Print test output and Record test result in launchable - if: steps.changes.outputs.end_to_end == 'true' && always() - run: | - # send recorded tests to launchable - launchable record tests --build "$GITHUB_RUN_ID" go-test . || true - - # print test output - cat output.txt + # run the tests however you normally do, then produce a JUnit XML file + eatmydata -- go run test.go -docker=false -follow -shard mysql_server_vault diff --git a/.github/workflows/cluster_endtoend_onlineddl_declarative.yml b/.github/workflows/cluster_endtoend_onlineddl_declarative.yml index de87788025..36b2ebd92d 100644 --- a/.github/workflows/cluster_endtoend_onlineddl_declarative.yml +++ b/.github/workflows/cluster_endtoend_onlineddl_declarative.yml @@ -49,10 +49,9 @@ jobs: - name: Set up Go if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - uses: actions/setup-go@v4 + uses: actions/setup-go@v3 with: - go-version: '1.20' - check-latest: true + go-version: 1.19.12 - name: Set up python if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' @@ -73,7 +72,7 @@ jobs: run: | # Get key to latest MySQL repo - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 # Setup MySQL 8.0 wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.29-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections diff --git a/.github/workflows/cluster_endtoend_onlineddl_declarative_mysql57.yml b/.github/workflows/cluster_endtoend_onlineddl_declarative_mysql57.yml index 913aff7436..0f3ae828bf 100644 --- a/.github/workflows/cluster_endtoend_onlineddl_declarative_mysql57.yml +++ b/.github/workflows/cluster_endtoend_onlineddl_declarative_mysql57.yml @@ -50,10 +50,9 @@ jobs: - name: Set up Go if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - uses: actions/setup-go@v4 + uses: actions/setup-go@v3 with: - go-version: '1.20' - check-latest: true + go-version: 1.19.12 - name: Set up python if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' @@ -85,7 +84,7 @@ jobs: sudo rm -rf /etc/mysql # Get key to latest MySQL repo - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.14-1_all.deb # Bionic packages are still compatible for Focal since there's no MySQL 5.7 diff --git a/.github/workflows/cluster_endtoend_onlineddl_ghost.yml b/.github/workflows/cluster_endtoend_onlineddl_ghost.yml index 5e608b615b..17062ebe9c 100644 --- a/.github/workflows/cluster_endtoend_onlineddl_ghost.yml +++ b/.github/workflows/cluster_endtoend_onlineddl_ghost.yml @@ -49,10 +49,9 @@ jobs: - name: Set up Go if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - uses: actions/setup-go@v4 + uses: actions/setup-go@v3 with: - go-version: '1.20' - check-latest: true + go-version: 1.19.12 - name: Set up python if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' @@ -73,7 +72,7 @@ jobs: run: | # Get key to latest MySQL repo - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 # Setup MySQL 8.0 wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.29-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections diff --git a/.github/workflows/cluster_endtoend_onlineddl_ghost_mysql57.yml b/.github/workflows/cluster_endtoend_onlineddl_ghost_mysql57.yml index ea8d1fcc1d..42a150c3a3 100644 --- a/.github/workflows/cluster_endtoend_onlineddl_ghost_mysql57.yml +++ b/.github/workflows/cluster_endtoend_onlineddl_ghost_mysql57.yml @@ -50,10 +50,9 @@ jobs: - name: Set up Go if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - uses: actions/setup-go@v4 + uses: actions/setup-go@v3 with: - go-version: '1.20' - check-latest: true + go-version: 1.19.12 - name: Set up python if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' @@ -85,7 +84,7 @@ jobs: sudo rm -rf /etc/mysql # Get key to latest MySQL repo - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.14-1_all.deb # Bionic packages are still compatible for Focal since there's no MySQL 5.7 diff --git a/.github/workflows/cluster_endtoend_onlineddl_revert.yml b/.github/workflows/cluster_endtoend_onlineddl_revert.yml index 17ec0ef036..53455b1955 100644 --- a/.github/workflows/cluster_endtoend_onlineddl_revert.yml +++ b/.github/workflows/cluster_endtoend_onlineddl_revert.yml @@ -49,10 +49,9 @@ jobs: - name: Set up Go if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - uses: actions/setup-go@v4 + uses: actions/setup-go@v3 with: - go-version: '1.20' - check-latest: true + go-version: 1.19.12 - name: Set up python if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' @@ -73,7 +72,7 @@ jobs: run: | # Get key to latest MySQL repo - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 # Setup MySQL 8.0 wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.29-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections diff --git a/.github/workflows/cluster_endtoend_onlineddl_revert_mysql57.yml b/.github/workflows/cluster_endtoend_onlineddl_revert_mysql57.yml index a9fdee1b52..df95a44f63 100644 --- a/.github/workflows/cluster_endtoend_onlineddl_revert_mysql57.yml +++ b/.github/workflows/cluster_endtoend_onlineddl_revert_mysql57.yml @@ -50,10 +50,9 @@ jobs: - name: Set up Go if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - uses: actions/setup-go@v4 + uses: actions/setup-go@v3 with: - go-version: '1.20' - check-latest: true + go-version: 1.19.12 - name: Set up python if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' @@ -85,7 +84,7 @@ jobs: sudo rm -rf /etc/mysql # Get key to latest MySQL repo - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.14-1_all.deb # Bionic packages are still compatible for Focal since there's no MySQL 5.7 diff --git a/.github/workflows/cluster_endtoend_onlineddl_revertible.yml b/.github/workflows/cluster_endtoend_onlineddl_revertible.yml index 35d7743494..777dda2a69 100644 --- a/.github/workflows/cluster_endtoend_onlineddl_revertible.yml +++ b/.github/workflows/cluster_endtoend_onlineddl_revertible.yml @@ -49,10 +49,9 @@ jobs: - name: Set up Go if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - uses: actions/setup-go@v4 + uses: actions/setup-go@v3 with: - go-version: '1.20' - check-latest: true + go-version: 1.19.12 - name: Set up python if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' @@ -73,7 +72,7 @@ jobs: run: | # Get key to latest MySQL repo - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 # Setup MySQL 8.0 wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.29-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections diff --git a/.github/workflows/cluster_endtoend_onlineddl_revertible_mysql57.yml b/.github/workflows/cluster_endtoend_onlineddl_revertible_mysql57.yml index 41b2105763..bdc00193a2 100644 --- a/.github/workflows/cluster_endtoend_onlineddl_revertible_mysql57.yml +++ b/.github/workflows/cluster_endtoend_onlineddl_revertible_mysql57.yml @@ -50,10 +50,9 @@ jobs: - name: Set up Go if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - uses: actions/setup-go@v4 + uses: actions/setup-go@v3 with: - go-version: '1.20' - check-latest: true + go-version: 1.19.12 - name: Set up python if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' @@ -85,7 +84,7 @@ jobs: sudo rm -rf /etc/mysql # Get key to latest MySQL repo - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.14-1_all.deb # Bionic packages are still compatible for Focal since there's no MySQL 5.7 diff --git a/.github/workflows/cluster_endtoend_onlineddl_scheduler.yml b/.github/workflows/cluster_endtoend_onlineddl_scheduler.yml index 4d665492d6..910819ea21 100644 --- a/.github/workflows/cluster_endtoend_onlineddl_scheduler.yml +++ b/.github/workflows/cluster_endtoend_onlineddl_scheduler.yml @@ -49,10 +49,9 @@ jobs: - name: Set up Go if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - uses: actions/setup-go@v4 + uses: actions/setup-go@v3 with: - go-version: '1.20' - check-latest: true + go-version: 1.19.12 - name: Set up python if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' @@ -73,7 +72,7 @@ jobs: run: | # Get key to latest MySQL repo - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 # Setup MySQL 8.0 wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.29-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections diff --git a/.github/workflows/cluster_endtoend_onlineddl_scheduler_mysql57.yml b/.github/workflows/cluster_endtoend_onlineddl_scheduler_mysql57.yml index 5e43369727..4327d881e7 100644 --- a/.github/workflows/cluster_endtoend_onlineddl_scheduler_mysql57.yml +++ b/.github/workflows/cluster_endtoend_onlineddl_scheduler_mysql57.yml @@ -50,10 +50,9 @@ jobs: - name: Set up Go if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - uses: actions/setup-go@v4 + uses: actions/setup-go@v3 with: - go-version: '1.20' - check-latest: true + go-version: 1.19.12 - name: Set up python if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' @@ -85,7 +84,7 @@ jobs: sudo rm -rf /etc/mysql # Get key to latest MySQL repo - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.14-1_all.deb # Bionic packages are still compatible for Focal since there's no MySQL 5.7 diff --git a/.github/workflows/cluster_endtoend_onlineddl_singleton.yml b/.github/workflows/cluster_endtoend_onlineddl_singleton.yml index 196ca283db..0e8c25ba23 100644 --- a/.github/workflows/cluster_endtoend_onlineddl_singleton.yml +++ b/.github/workflows/cluster_endtoend_onlineddl_singleton.yml @@ -49,10 +49,9 @@ jobs: - name: Set up Go if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - uses: actions/setup-go@v4 + uses: actions/setup-go@v3 with: - go-version: '1.20' - check-latest: true + go-version: 1.19.12 - name: Set up python if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' @@ -73,7 +72,7 @@ jobs: run: | # Get key to latest MySQL repo - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 # Setup MySQL 8.0 wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.29-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections diff --git a/.github/workflows/cluster_endtoend_onlineddl_singleton_mysql57.yml b/.github/workflows/cluster_endtoend_onlineddl_singleton_mysql57.yml index dd827ad0ef..eeafef5e66 100644 --- a/.github/workflows/cluster_endtoend_onlineddl_singleton_mysql57.yml +++ b/.github/workflows/cluster_endtoend_onlineddl_singleton_mysql57.yml @@ -50,10 +50,9 @@ jobs: - name: Set up Go if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - uses: actions/setup-go@v4 + uses: actions/setup-go@v3 with: - go-version: '1.20' - check-latest: true + go-version: 1.19.12 - name: Set up python if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' @@ -85,7 +84,7 @@ jobs: sudo rm -rf /etc/mysql # Get key to latest MySQL repo - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.14-1_all.deb # Bionic packages are still compatible for Focal since there's no MySQL 5.7 diff --git a/.github/workflows/cluster_endtoend_onlineddl_vrepl.yml b/.github/workflows/cluster_endtoend_onlineddl_vrepl.yml index 98d60c5a8b..51ffd3ee51 100644 --- a/.github/workflows/cluster_endtoend_onlineddl_vrepl.yml +++ b/.github/workflows/cluster_endtoend_onlineddl_vrepl.yml @@ -49,10 +49,9 @@ jobs: - name: Set up Go if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - uses: actions/setup-go@v4 + uses: actions/setup-go@v3 with: - go-version: '1.20' - check-latest: true + go-version: 1.19.12 - name: Set up python if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' @@ -73,7 +72,7 @@ jobs: run: | # Get key to latest MySQL repo - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 # Setup MySQL 8.0 wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.29-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections diff --git a/.github/workflows/cluster_endtoend_onlineddl_vrepl_mysql57.yml b/.github/workflows/cluster_endtoend_onlineddl_vrepl_mysql57.yml index cf0740c876..4e4b636fc9 100644 --- a/.github/workflows/cluster_endtoend_onlineddl_vrepl_mysql57.yml +++ b/.github/workflows/cluster_endtoend_onlineddl_vrepl_mysql57.yml @@ -50,10 +50,9 @@ jobs: - name: Set up Go if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - uses: actions/setup-go@v4 + uses: actions/setup-go@v3 with: - go-version: '1.20' - check-latest: true + go-version: 1.19.12 - name: Set up python if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' @@ -85,7 +84,7 @@ jobs: sudo rm -rf /etc/mysql # Get key to latest MySQL repo - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.14-1_all.deb # Bionic packages are still compatible for Focal since there's no MySQL 5.7 diff --git a/.github/workflows/cluster_endtoend_onlineddl_vrepl_stress.yml b/.github/workflows/cluster_endtoend_onlineddl_vrepl_stress.yml index 40645cb374..76ab54bd5b 100644 --- a/.github/workflows/cluster_endtoend_onlineddl_vrepl_stress.yml +++ b/.github/workflows/cluster_endtoend_onlineddl_vrepl_stress.yml @@ -49,10 +49,9 @@ jobs: - name: Set up Go if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - uses: actions/setup-go@v4 + uses: actions/setup-go@v3 with: - go-version: '1.20' - check-latest: true + go-version: 1.19.12 - name: Set up python if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' @@ -73,7 +72,7 @@ jobs: run: | # Get key to latest MySQL repo - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 # Setup MySQL 8.0 wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.29-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections diff --git a/.github/workflows/cluster_endtoend_onlineddl_vrepl_stress_mysql57.yml b/.github/workflows/cluster_endtoend_onlineddl_vrepl_stress_mysql57.yml index 67380671d8..5573cffdd8 100644 --- a/.github/workflows/cluster_endtoend_onlineddl_vrepl_stress_mysql57.yml +++ b/.github/workflows/cluster_endtoend_onlineddl_vrepl_stress_mysql57.yml @@ -50,10 +50,9 @@ jobs: - name: Set up Go if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - uses: actions/setup-go@v4 + uses: actions/setup-go@v3 with: - go-version: '1.20' - check-latest: true + go-version: 1.19.12 - name: Set up python if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' @@ -85,7 +84,7 @@ jobs: sudo rm -rf /etc/mysql # Get key to latest MySQL repo - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.14-1_all.deb # Bionic packages are still compatible for Focal since there's no MySQL 5.7 diff --git a/.github/workflows/cluster_endtoend_onlineddl_vrepl_stress_suite.yml b/.github/workflows/cluster_endtoend_onlineddl_vrepl_stress_suite.yml index ff458af42b..bda104d59b 100644 --- a/.github/workflows/cluster_endtoend_onlineddl_vrepl_stress_suite.yml +++ b/.github/workflows/cluster_endtoend_onlineddl_vrepl_stress_suite.yml @@ -49,10 +49,9 @@ jobs: - name: Set up Go if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - uses: actions/setup-go@v4 + uses: actions/setup-go@v3 with: - go-version: '1.20' - check-latest: true + go-version: 1.19.12 - name: Set up python if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' @@ -73,7 +72,7 @@ jobs: run: | # Get key to latest MySQL repo - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 # Setup MySQL 8.0 wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.29-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections diff --git a/.github/workflows/cluster_endtoend_onlineddl_vrepl_stress_suite_mysql57.yml b/.github/workflows/cluster_endtoend_onlineddl_vrepl_stress_suite_mysql57.yml index 1c6a6117fd..ee1a505a4f 100644 --- a/.github/workflows/cluster_endtoend_onlineddl_vrepl_stress_suite_mysql57.yml +++ b/.github/workflows/cluster_endtoend_onlineddl_vrepl_stress_suite_mysql57.yml @@ -50,10 +50,9 @@ jobs: - name: Set up Go if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - uses: actions/setup-go@v4 + uses: actions/setup-go@v3 with: - go-version: '1.20' - check-latest: true + go-version: 1.19.12 - name: Set up python if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' @@ -85,7 +84,7 @@ jobs: sudo rm -rf /etc/mysql # Get key to latest MySQL repo - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.14-1_all.deb # Bionic packages are still compatible for Focal since there's no MySQL 5.7 diff --git a/.github/workflows/cluster_endtoend_onlineddl_vrepl_suite.yml b/.github/workflows/cluster_endtoend_onlineddl_vrepl_suite.yml index c04b394cbc..b1d03bda71 100644 --- a/.github/workflows/cluster_endtoend_onlineddl_vrepl_suite.yml +++ b/.github/workflows/cluster_endtoend_onlineddl_vrepl_suite.yml @@ -49,10 +49,9 @@ jobs: - name: Set up Go if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - uses: actions/setup-go@v4 + uses: actions/setup-go@v3 with: - go-version: '1.20' - check-latest: true + go-version: 1.19.12 - name: Set up python if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' @@ -73,7 +72,7 @@ jobs: run: | # Get key to latest MySQL repo - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 # Setup MySQL 8.0 wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.29-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections diff --git a/.github/workflows/cluster_endtoend_onlineddl_vrepl_suite_mysql57.yml b/.github/workflows/cluster_endtoend_onlineddl_vrepl_suite_mysql57.yml index 77450a5e8e..7d058b47db 100644 --- a/.github/workflows/cluster_endtoend_onlineddl_vrepl_suite_mysql57.yml +++ b/.github/workflows/cluster_endtoend_onlineddl_vrepl_suite_mysql57.yml @@ -50,10 +50,9 @@ jobs: - name: Set up Go if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - uses: actions/setup-go@v4 + uses: actions/setup-go@v3 with: - go-version: '1.20' - check-latest: true + go-version: 1.19.12 - name: Set up python if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' @@ -85,7 +84,7 @@ jobs: sudo rm -rf /etc/mysql # Get key to latest MySQL repo - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.14-1_all.deb # Bionic packages are still compatible for Focal since there's no MySQL 5.7 diff --git a/.github/workflows/cluster_endtoend_schemadiff_vrepl.yml b/.github/workflows/cluster_endtoend_schemadiff_vrepl.yml index acc260001d..c81a7580d2 100644 --- a/.github/workflows/cluster_endtoend_schemadiff_vrepl.yml +++ b/.github/workflows/cluster_endtoend_schemadiff_vrepl.yml @@ -49,10 +49,9 @@ jobs: - name: Set up Go if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - uses: actions/setup-go@v4 + uses: actions/setup-go@v3 with: - go-version: '1.20' - check-latest: true + go-version: 1.19.12 - name: Set up python if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' @@ -73,7 +72,7 @@ jobs: run: | # Get key to latest MySQL repo - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 # Setup MySQL 8.0 wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.29-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections diff --git a/.github/workflows/cluster_endtoend_schemadiff_vrepl_mysql57.yml b/.github/workflows/cluster_endtoend_schemadiff_vrepl_mysql57.yml index 270ffa7fad..a2a66a422b 100644 --- a/.github/workflows/cluster_endtoend_schemadiff_vrepl_mysql57.yml +++ b/.github/workflows/cluster_endtoend_schemadiff_vrepl_mysql57.yml @@ -50,10 +50,9 @@ jobs: - name: Set up Go if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - uses: actions/setup-go@v4 + uses: actions/setup-go@v3 with: - go-version: '1.20' - check-latest: true + go-version: 1.19.12 - name: Set up python if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' @@ -85,7 +84,7 @@ jobs: sudo rm -rf /etc/mysql # Get key to latest MySQL repo - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.14-1_all.deb # Bionic packages are still compatible for Focal since there's no MySQL 5.7 diff --git a/.github/workflows/cluster_endtoend_tabletmanager_consul.yml b/.github/workflows/cluster_endtoend_tabletmanager_consul.yml index 63d1c2b14c..f1273e956d 100644 --- a/.github/workflows/cluster_endtoend_tabletmanager_consul.yml +++ b/.github/workflows/cluster_endtoend_tabletmanager_consul.yml @@ -49,10 +49,9 @@ jobs: - name: Set up Go if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - uses: actions/setup-go@v4 + uses: actions/setup-go@v3 with: - go-version: '1.20' - check-latest: true + go-version: 1.19.12 - name: Set up python if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' @@ -73,7 +72,7 @@ jobs: run: | # Get key to latest MySQL repo - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 # Setup MySQL 8.0 wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.29-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections diff --git a/.github/workflows/cluster_endtoend_tabletmanager_tablegc.yml b/.github/workflows/cluster_endtoend_tabletmanager_tablegc.yml index 22f1755e12..4c0ba5cf49 100644 --- a/.github/workflows/cluster_endtoend_tabletmanager_tablegc.yml +++ b/.github/workflows/cluster_endtoend_tabletmanager_tablegc.yml @@ -49,10 +49,9 @@ jobs: - name: Set up Go if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - uses: actions/setup-go@v4 + uses: actions/setup-go@v3 with: - go-version: '1.20' - check-latest: true + go-version: 1.19.12 - name: Set up python if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' @@ -73,7 +72,7 @@ jobs: run: | # Get key to latest MySQL repo - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 # Setup MySQL 8.0 wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.29-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections diff --git a/.github/workflows/cluster_endtoend_tabletmanager_tablegc_mysql57.yml b/.github/workflows/cluster_endtoend_tabletmanager_tablegc_mysql57.yml index f0fb34d2d3..017e10cbcc 100644 --- a/.github/workflows/cluster_endtoend_tabletmanager_tablegc_mysql57.yml +++ b/.github/workflows/cluster_endtoend_tabletmanager_tablegc_mysql57.yml @@ -50,10 +50,9 @@ jobs: - name: Set up Go if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - uses: actions/setup-go@v4 + uses: actions/setup-go@v3 with: - go-version: '1.20' - check-latest: true + go-version: 1.19.12 - name: Set up python if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' @@ -85,7 +84,7 @@ jobs: sudo rm -rf /etc/mysql # Get key to latest MySQL repo - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.14-1_all.deb # Bionic packages are still compatible for Focal since there's no MySQL 5.7 diff --git a/.github/workflows/cluster_endtoend_tabletmanager_throttler.yml b/.github/workflows/cluster_endtoend_tabletmanager_throttler.yml index 7b0187cb10..b23846cb47 100644 --- a/.github/workflows/cluster_endtoend_tabletmanager_throttler.yml +++ b/.github/workflows/cluster_endtoend_tabletmanager_throttler.yml @@ -49,10 +49,9 @@ jobs: - name: Set up Go if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - uses: actions/setup-go@v4 + uses: actions/setup-go@v3 with: - go-version: '1.20' - check-latest: true + go-version: 1.19.12 - name: Set up python if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' @@ -73,7 +72,7 @@ jobs: run: | # Get key to latest MySQL repo - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 # Setup MySQL 8.0 wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.29-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections diff --git a/.github/workflows/cluster_endtoend_tabletmanager_throttler_custom_config.yml b/.github/workflows/cluster_endtoend_tabletmanager_throttler_custom_config.yml index 040e67d33b..b3bcdcccc4 100644 --- a/.github/workflows/cluster_endtoend_tabletmanager_throttler_custom_config.yml +++ b/.github/workflows/cluster_endtoend_tabletmanager_throttler_custom_config.yml @@ -49,10 +49,9 @@ jobs: - name: Set up Go if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - uses: actions/setup-go@v4 + uses: actions/setup-go@v3 with: - go-version: '1.20' - check-latest: true + go-version: 1.19.12 - name: Set up python if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' @@ -73,7 +72,7 @@ jobs: run: | # Get key to latest MySQL repo - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 # Setup MySQL 8.0 wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.29-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections diff --git a/.github/workflows/cluster_endtoend_topo_connection_cache.yml b/.github/workflows/cluster_endtoend_topo_connection_cache.yml index dab4cd070e..9f218c80d7 100644 --- a/.github/workflows/cluster_endtoend_topo_connection_cache.yml +++ b/.github/workflows/cluster_endtoend_topo_connection_cache.yml @@ -49,10 +49,9 @@ jobs: - name: Set up Go if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - uses: actions/setup-go@v4 + uses: actions/setup-go@v3 with: - go-version: '1.20' - check-latest: true + go-version: 1.19.12 - name: Set up python if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' @@ -73,7 +72,7 @@ jobs: run: | # Get key to latest MySQL repo - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 # Setup MySQL 8.0 wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.29-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections diff --git a/.github/workflows/cluster_endtoend_vreplication_across_db_versions.yml b/.github/workflows/cluster_endtoend_vreplication_across_db_versions.yml index 8f879b7386..8609980aa9 100644 --- a/.github/workflows/cluster_endtoend_vreplication_across_db_versions.yml +++ b/.github/workflows/cluster_endtoend_vreplication_across_db_versions.yml @@ -6,23 +6,29 @@ concurrency: group: format('{0}-{1}', ${{ github.ref }}, 'Cluster (vreplication_across_db_versions)') cancel-in-progress: true -env: - LAUNCHABLE_ORGANIZATION: "vitess" - LAUNCHABLE_WORKSPACE: "vitess-app" - GITHUB_PR_HEAD_SHA: "${{ github.event.pull_request.head.sha }}" - jobs: build: name: Run endtoend tests on Cluster (vreplication_across_db_versions) runs-on: group: vitess-ubuntu20 - timeout-minutes: 45 steps: + - name: Check if workflow needs to be skipped + id: skip-workflow + run: | + skip='false' + if [[ "${{github.event.pull_request}}" == "" ]] && [[ "${{github.ref}}" != "refs/heads/main" ]] && [[ ! "${{github.ref}}" =~ ^refs/heads/slack-vitess-r[0-9]+\.[0-9]\.[0-9]+$ ]] && [[ ! "${{github.ref}}" =~ "refs/tags/.*" ]]; then + skip='true' + fi + echo Skip ${skip} + echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT + - name: Check out code - uses: actions/checkout@v2 + if: steps.skip-workflow.outputs.skip-workflow == 'false' + uses: actions/checkout@v3 - name: Check for changes in relevant files + if: steps.skip-workflow.outputs.skip-workflow == 'false' uses: frouioui/paths-filter@main id: changes with: @@ -33,47 +39,53 @@ jobs: - 'test.go' - 'Makefile' - 'build.env' - - 'go.[sumod]' + - 'go.sum' + - 'go.mod' - 'proto/*.proto' - 'tools/**' - 'config/**' - 'bootstrap.sh' - - '.github/workflows/**' + - '.github/workflows/cluster_endtoend_vreplication_across_db_versions.yml' - name: Set up Go - if: steps.changes.outputs.end_to_end == 'true' - uses: actions/setup-go@v4 + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/setup-go@v3 with: - go-version: '1.20' - check-latest: true + go-version: 1.19.12 - name: Set up python - if: steps.changes.outputs.end_to_end == 'true' - uses: actions/setup-python@v2 + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/setup-python@v4 - name: Tune the OS - if: steps.changes.outputs.end_to_end == 'true' + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf sudo sysctl -p /etc/sysctl.conf - name: Get dependencies - if: steps.changes.outputs.end_to_end == 'true' + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | - # Setup Percona Server for MySQL 8.0 - sudo apt-get update - sudo apt-get install -y lsb-release gnupg2 curl - wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb - sudo DEBIAN_FRONTEND="noninteractive" dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb - sudo percona-release setup ps80 + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + # Setup MySQL 8.0 + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.29-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update - # Install everything else we need, and configure - sudo apt-get install -y percona-server-server percona-server-client make unzip g++ etcd git wget eatmydata xz-utils - sudo service mysql stop + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get update + sudo service etcd stop + echo 'mysql version: ' + sudo mysql --version + sudo service mysql stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download @@ -81,57 +93,41 @@ jobs: # install JUnit report formatter go install github.com/vitessio/go-junit-report@HEAD - - name: Setup launchable dependencies - if: steps.changes.outputs.end_to_end == 'true' - run: | - # Get Launchable CLI installed. If you can, make it a part of the builder image to speed things up - pip3 install --user launchable~=1.0 > /dev/null - - # verify that launchable setup is all correct. - launchable verify || true - - # Tell Launchable about the build you are producing and testing - launchable record build --name "$GITHUB_RUN_ID" --source . - - name: Run cluster endtoend test - if: steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 - run: | - # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file - # which musn't be more than 107 characters long. - export VTDATAROOT="/tmp/" - source build.env - - set -x - - # Increase our local ephemeral port range as we could exhaust this - sudo sysctl -w net.ipv4.ip_local_port_range="22768 61999" - # Increase our open file descriptor limit as we could hit this - ulimit -n 65536 - cat <<-EOF>>./config/mycnf/mysql80.cnf - innodb_buffer_pool_dump_at_shutdown=OFF - innodb_buffer_pool_in_core_file=OFF - innodb_buffer_pool_load_at_startup=OFF - innodb_buffer_pool_size=64M - innodb_doublewrite=OFF - innodb_flush_log_at_trx_commit=0 - innodb_flush_method=O_DIRECT - innodb_numa_interleave=ON - innodb_adaptive_hash_index=OFF - sync_binlog=0 - sync_relay_log=0 - performance_schema=OFF - slow-query-log=OFF - EOF - - # run the tests however you normally do, then produce a JUnit XML file - eatmydata -- go run test.go -docker=false -follow -shard vreplication_across_db_versions | tee -a output.txt | go-junit-report -set-exit-code > report.xml - - - name: Print test output and Record test result in launchable - if: steps.changes.outputs.end_to_end == 'true' && always() - run: | - # send recorded tests to launchable - launchable record tests --build "$GITHUB_RUN_ID" go-test . || true - - # print test output - cat output.txt + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: nick-fields/retry@v2 + with: + timeout_minutes: 45 + max_attempts: 3 + retry_on: error + command: | + # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file + # which musn't be more than 107 characters long. + export VTDATAROOT="/tmp/" + source build.env + + set -x + + # Increase our local ephemeral port range as we could exhaust this + sudo sysctl -w net.ipv4.ip_local_port_range="22768 61999" + # Increase our open file descriptor limit as we could hit this + ulimit -n 65536 + cat <<-EOF>>./config/mycnf/mysql80.cnf + innodb_buffer_pool_dump_at_shutdown=OFF + innodb_buffer_pool_in_core_file=OFF + innodb_buffer_pool_load_at_startup=OFF + innodb_buffer_pool_size=16M + innodb_doublewrite=OFF + innodb_flush_log_at_trx_commit=0 + innodb_flush_method=O_DIRECT + innodb_numa_interleave=ON + innodb_adaptive_hash_index=OFF + sync_binlog=0 + sync_relay_log=0 + performance_schema=OFF + slow-query-log=OFF + EOF + make embed_config + + # run the tests however you normally do, then produce a JUnit XML file + eatmydata -- go run test.go -docker=false -follow -shard vreplication_across_db_versions diff --git a/.github/workflows/cluster_endtoend_vreplication_basic.yml b/.github/workflows/cluster_endtoend_vreplication_basic.yml index 313a8edc46..171165e47b 100644 --- a/.github/workflows/cluster_endtoend_vreplication_basic.yml +++ b/.github/workflows/cluster_endtoend_vreplication_basic.yml @@ -49,10 +49,9 @@ jobs: - name: Set up Go if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - uses: actions/setup-go@v4 + uses: actions/setup-go@v3 with: - go-version: '1.20' - check-latest: true + go-version: 1.19.12 - name: Set up python if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' @@ -73,7 +72,7 @@ jobs: run: | # Get key to latest MySQL repo - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 # Setup MySQL 8.0 wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.29-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections diff --git a/.github/workflows/cluster_endtoend_vreplication_cellalias.yml b/.github/workflows/cluster_endtoend_vreplication_cellalias.yml index bbfb668f61..588aec90f1 100644 --- a/.github/workflows/cluster_endtoend_vreplication_cellalias.yml +++ b/.github/workflows/cluster_endtoend_vreplication_cellalias.yml @@ -49,10 +49,9 @@ jobs: - name: Set up Go if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - uses: actions/setup-go@v4 + uses: actions/setup-go@v3 with: - go-version: '1.20' - check-latest: true + go-version: 1.19.12 - name: Set up python if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' @@ -73,7 +72,7 @@ jobs: run: | # Get key to latest MySQL repo - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 # Setup MySQL 8.0 wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.29-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections diff --git a/.github/workflows/cluster_endtoend_vreplication_migrate.yml b/.github/workflows/cluster_endtoend_vreplication_migrate.yml index a74f6ada71..ffb728e668 100644 --- a/.github/workflows/cluster_endtoend_vreplication_migrate.yml +++ b/.github/workflows/cluster_endtoend_vreplication_migrate.yml @@ -49,10 +49,9 @@ jobs: - name: Set up Go if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - uses: actions/setup-go@v4 + uses: actions/setup-go@v3 with: - go-version: '1.20' - check-latest: true + go-version: 1.19.12 - name: Set up python if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' @@ -73,7 +72,7 @@ jobs: run: | # Get key to latest MySQL repo - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 # Setup MySQL 8.0 wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.29-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections diff --git a/.github/workflows/cluster_endtoend_vreplication_multicell.yml b/.github/workflows/cluster_endtoend_vreplication_multicell.yml index df1c8f7cca..db34f75b92 100644 --- a/.github/workflows/cluster_endtoend_vreplication_multicell.yml +++ b/.github/workflows/cluster_endtoend_vreplication_multicell.yml @@ -49,10 +49,9 @@ jobs: - name: Set up Go if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - uses: actions/setup-go@v4 + uses: actions/setup-go@v3 with: - go-version: '1.20' - check-latest: true + go-version: 1.19.12 - name: Set up python if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' @@ -73,7 +72,7 @@ jobs: run: | # Get key to latest MySQL repo - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 # Setup MySQL 8.0 wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.29-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections diff --git a/.github/workflows/cluster_endtoend_vreplication_v2.yml b/.github/workflows/cluster_endtoend_vreplication_v2.yml index 4db0e93f07..8dd73c958f 100644 --- a/.github/workflows/cluster_endtoend_vreplication_v2.yml +++ b/.github/workflows/cluster_endtoend_vreplication_v2.yml @@ -49,10 +49,9 @@ jobs: - name: Set up Go if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - uses: actions/setup-go@v4 + uses: actions/setup-go@v3 with: - go-version: '1.20' - check-latest: true + go-version: 1.19.12 - name: Set up python if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' @@ -73,7 +72,7 @@ jobs: run: | # Get key to latest MySQL repo - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 # Setup MySQL 8.0 wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.29-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections diff --git a/.github/workflows/cluster_endtoend_vstream_failover.yml b/.github/workflows/cluster_endtoend_vstream_failover.yml index a7090b8939..d894db6c35 100644 --- a/.github/workflows/cluster_endtoend_vstream_failover.yml +++ b/.github/workflows/cluster_endtoend_vstream_failover.yml @@ -49,10 +49,9 @@ jobs: - name: Set up Go if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - uses: actions/setup-go@v4 + uses: actions/setup-go@v3 with: - go-version: '1.20' - check-latest: true + go-version: 1.19.12 - name: Set up python if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' @@ -73,7 +72,7 @@ jobs: run: | # Get key to latest MySQL repo - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 # Setup MySQL 8.0 wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.29-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections diff --git a/.github/workflows/cluster_endtoend_vstream_stoponreshard_false.yml b/.github/workflows/cluster_endtoend_vstream_stoponreshard_false.yml index 80d633a568..4704509206 100644 --- a/.github/workflows/cluster_endtoend_vstream_stoponreshard_false.yml +++ b/.github/workflows/cluster_endtoend_vstream_stoponreshard_false.yml @@ -49,10 +49,9 @@ jobs: - name: Set up Go if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - uses: actions/setup-go@v4 + uses: actions/setup-go@v3 with: - go-version: '1.20' - check-latest: true + go-version: 1.19.12 - name: Set up python if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' @@ -73,7 +72,7 @@ jobs: run: | # Get key to latest MySQL repo - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 # Setup MySQL 8.0 wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.29-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections diff --git a/.github/workflows/cluster_endtoend_vstream_stoponreshard_true.yml b/.github/workflows/cluster_endtoend_vstream_stoponreshard_true.yml index 39475721d1..f1fce11e2f 100644 --- a/.github/workflows/cluster_endtoend_vstream_stoponreshard_true.yml +++ b/.github/workflows/cluster_endtoend_vstream_stoponreshard_true.yml @@ -49,10 +49,9 @@ jobs: - name: Set up Go if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - uses: actions/setup-go@v4 + uses: actions/setup-go@v3 with: - go-version: '1.20' - check-latest: true + go-version: 1.19.12 - name: Set up python if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' @@ -73,7 +72,7 @@ jobs: run: | # Get key to latest MySQL repo - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 # Setup MySQL 8.0 wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.29-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections diff --git a/.github/workflows/cluster_endtoend_vstream_with_keyspaces_to_watch.yml b/.github/workflows/cluster_endtoend_vstream_with_keyspaces_to_watch.yml index 3ec4c86c6b..c825c27436 100644 --- a/.github/workflows/cluster_endtoend_vstream_with_keyspaces_to_watch.yml +++ b/.github/workflows/cluster_endtoend_vstream_with_keyspaces_to_watch.yml @@ -49,10 +49,9 @@ jobs: - name: Set up Go if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - uses: actions/setup-go@v4 + uses: actions/setup-go@v3 with: - go-version: '1.20' - check-latest: true + go-version: 1.19.12 - name: Set up python if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' @@ -73,7 +72,7 @@ jobs: run: | # Get key to latest MySQL repo - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 # Setup MySQL 8.0 wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.29-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections diff --git a/.github/workflows/cluster_endtoend_vtbackup_transform.yml b/.github/workflows/cluster_endtoend_vtbackup_transform.yml index f4067f6d79..cef5a9216f 100644 --- a/.github/workflows/cluster_endtoend_vtbackup_transform.yml +++ b/.github/workflows/cluster_endtoend_vtbackup_transform.yml @@ -6,23 +6,29 @@ concurrency: group: format('{0}-{1}', ${{ github.ref }}, 'Cluster (vtbackup_transform)') cancel-in-progress: true -env: - LAUNCHABLE_ORGANIZATION: "vitess" - LAUNCHABLE_WORKSPACE: "vitess-app" - GITHUB_PR_HEAD_SHA: "${{ github.event.pull_request.head.sha }}" - jobs: build: name: Run endtoend tests on Cluster (vtbackup_transform) runs-on: group: vitess-ubuntu20 - timeout-minutes: 45 steps: + - name: Check if workflow needs to be skipped + id: skip-workflow + run: | + skip='false' + if [[ "${{github.event.pull_request}}" == "" ]] && [[ "${{github.ref}}" != "refs/heads/main" ]] && [[ ! "${{github.ref}}" =~ ^refs/heads/slack-vitess-r[0-9]+\.[0-9]\.[0-9]+$ ]] && [[ ! "${{github.ref}}" =~ "refs/tags/.*" ]]; then + skip='true' + fi + echo Skip ${skip} + echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT + - name: Check out code - uses: actions/checkout@v2 + if: steps.skip-workflow.outputs.skip-workflow == 'false' + uses: actions/checkout@v3 - name: Check for changes in relevant files + if: steps.skip-workflow.outputs.skip-workflow == 'false' uses: frouioui/paths-filter@main id: changes with: @@ -33,47 +39,53 @@ jobs: - 'test.go' - 'Makefile' - 'build.env' - - 'go.[sumod]' + - 'go.sum' + - 'go.mod' - 'proto/*.proto' - 'tools/**' - 'config/**' - 'bootstrap.sh' - - '.github/workflows/**' + - '.github/workflows/cluster_endtoend_vtbackup_transform.yml' - name: Set up Go - if: steps.changes.outputs.end_to_end == 'true' - uses: actions/setup-go@v4 + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/setup-go@v3 with: - go-version: '1.20' - check-latest: true + go-version: 1.19.12 - name: Set up python - if: steps.changes.outputs.end_to_end == 'true' - uses: actions/setup-python@v2 + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/setup-python@v4 - name: Tune the OS - if: steps.changes.outputs.end_to_end == 'true' + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf sudo sysctl -p /etc/sysctl.conf - name: Get dependencies - if: steps.changes.outputs.end_to_end == 'true' + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | - # Setup Percona Server for MySQL 8.0 + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + # Setup MySQL 8.0 + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.29-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update - sudo apt-get install -y lsb-release gnupg2 curl - wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb - sudo DEBIAN_FRONTEND="noninteractive" dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb - sudo percona-release setup ps80 - sudo apt-get update - # Install everything else we need, and configure - sudo apt-get install -y percona-server-server percona-server-client make unzip g++ etcd git wget eatmydata xz-utils - sudo service mysql stop + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get update + sudo service etcd stop + echo 'mysql version: ' + sudo mysql --version + sudo service mysql stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download @@ -81,37 +93,20 @@ jobs: # install JUnit report formatter go install github.com/vitessio/go-junit-report@HEAD - - name: Setup launchable dependencies - if: steps.changes.outputs.end_to_end == 'true' - run: | - # Get Launchable CLI installed. If you can, make it a part of the builder image to speed things up - pip3 install --user launchable~=1.0 > /dev/null - - # verify that launchable setup is all correct. - launchable verify || true - - # Tell Launchable about the build you are producing and testing - launchable record build --name "$GITHUB_RUN_ID" --source . - - name: Run cluster endtoend test - if: steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 - run: | - # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file - # which musn't be more than 107 characters long. - export VTDATAROOT="/tmp/" - source build.env - - set -x - - # run the tests however you normally do, then produce a JUnit XML file - eatmydata -- go run test.go -docker=false -follow -shard vtbackup_transform | tee -a output.txt | go-junit-report -set-exit-code > report.xml - - - name: Print test output and Record test result in launchable - if: steps.changes.outputs.end_to_end == 'true' && always() - run: | - # send recorded tests to launchable - launchable record tests --build "$GITHUB_RUN_ID" go-test . || true - - # print test output - cat output.txt + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: nick-fields/retry@v2 + with: + timeout_minutes: 45 + max_attempts: 3 + retry_on: error + command: | + # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file + # which musn't be more than 107 characters long. + export VTDATAROOT="/tmp/" + source build.env + + set -x + + # run the tests however you normally do, then produce a JUnit XML file + eatmydata -- go run test.go -docker=false -follow -shard vtbackup_transform diff --git a/.github/workflows/cluster_endtoend_vtctlbackup_sharded_clustertest_heavy.yml b/.github/workflows/cluster_endtoend_vtctlbackup_sharded_clustertest_heavy.yml index 27276e86fb..1e3dfe2352 100644 --- a/.github/workflows/cluster_endtoend_vtctlbackup_sharded_clustertest_heavy.yml +++ b/.github/workflows/cluster_endtoend_vtctlbackup_sharded_clustertest_heavy.yml @@ -6,23 +6,29 @@ concurrency: group: format('{0}-{1}', ${{ github.ref }}, 'Cluster (vtctlbackup_sharded_clustertest_heavy)') cancel-in-progress: true -env: - LAUNCHABLE_ORGANIZATION: "vitess" - LAUNCHABLE_WORKSPACE: "vitess-app" - GITHUB_PR_HEAD_SHA: "${{ github.event.pull_request.head.sha }}" - jobs: build: name: Run endtoend tests on Cluster (vtctlbackup_sharded_clustertest_heavy) runs-on: group: vitess-ubuntu20 - timeout-minutes: 45 steps: + - name: Check if workflow needs to be skipped + id: skip-workflow + run: | + skip='false' + if [[ "${{github.event.pull_request}}" == "" ]] && [[ "${{github.ref}}" != "refs/heads/main" ]] && [[ ! "${{github.ref}}" =~ ^refs/heads/slack-vitess-r[0-9]+\.[0-9]\.[0-9]+$ ]] && [[ ! "${{github.ref}}" =~ "refs/tags/.*" ]]; then + skip='true' + fi + echo Skip ${skip} + echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT + - name: Check out code - uses: actions/checkout@v2 + if: steps.skip-workflow.outputs.skip-workflow == 'false' + uses: actions/checkout@v3 - name: Check for changes in relevant files + if: steps.skip-workflow.outputs.skip-workflow == 'false' uses: frouioui/paths-filter@main id: changes with: @@ -33,47 +39,53 @@ jobs: - 'test.go' - 'Makefile' - 'build.env' - - 'go.[sumod]' + - 'go.sum' + - 'go.mod' - 'proto/*.proto' - 'tools/**' - 'config/**' - 'bootstrap.sh' - - '.github/workflows/**' + - '.github/workflows/cluster_endtoend_vtctlbackup_sharded_clustertest_heavy.yml' - name: Set up Go - if: steps.changes.outputs.end_to_end == 'true' - uses: actions/setup-go@v4 + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/setup-go@v3 with: - go-version: '1.20' - check-latest: true + go-version: 1.19.12 - name: Set up python - if: steps.changes.outputs.end_to_end == 'true' - uses: actions/setup-python@v2 + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/setup-python@v4 - name: Tune the OS - if: steps.changes.outputs.end_to_end == 'true' + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf sudo sysctl -p /etc/sysctl.conf - name: Get dependencies - if: steps.changes.outputs.end_to_end == 'true' + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | - # Setup Percona Server for MySQL 8.0 - sudo apt-get update - sudo apt-get install -y lsb-release gnupg2 curl - wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb - sudo DEBIAN_FRONTEND="noninteractive" dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb - sudo percona-release setup ps80 + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + # Setup MySQL 8.0 + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.29-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update - # Install everything else we need, and configure - sudo apt-get install -y percona-server-server percona-server-client make unzip g++ etcd git wget eatmydata xz-utils - sudo service mysql stop + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get update + sudo service etcd stop + echo 'mysql version: ' + sudo mysql --version + sudo service mysql stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download @@ -81,57 +93,41 @@ jobs: # install JUnit report formatter go install github.com/vitessio/go-junit-report@HEAD - - name: Setup launchable dependencies - if: steps.changes.outputs.end_to_end == 'true' - run: | - # Get Launchable CLI installed. If you can, make it a part of the builder image to speed things up - pip3 install --user launchable~=1.0 > /dev/null - - # verify that launchable setup is all correct. - launchable verify || true - - # Tell Launchable about the build you are producing and testing - launchable record build --name "$GITHUB_RUN_ID" --source . - - name: Run cluster endtoend test - if: steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 - run: | - # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file - # which musn't be more than 107 characters long. - export VTDATAROOT="/tmp/" - source build.env - - set -x - - # Increase our local ephemeral port range as we could exhaust this - sudo sysctl -w net.ipv4.ip_local_port_range="22768 61999" - # Increase our open file descriptor limit as we could hit this - ulimit -n 65536 - cat <<-EOF>>./config/mycnf/mysql80.cnf - innodb_buffer_pool_dump_at_shutdown=OFF - innodb_buffer_pool_in_core_file=OFF - innodb_buffer_pool_load_at_startup=OFF - innodb_buffer_pool_size=64M - innodb_doublewrite=OFF - innodb_flush_log_at_trx_commit=0 - innodb_flush_method=O_DIRECT - innodb_numa_interleave=ON - innodb_adaptive_hash_index=OFF - sync_binlog=0 - sync_relay_log=0 - performance_schema=OFF - slow-query-log=OFF - EOF - - # run the tests however you normally do, then produce a JUnit XML file - eatmydata -- go run test.go -docker=false -follow -shard vtctlbackup_sharded_clustertest_heavy | tee -a output.txt | go-junit-report -set-exit-code > report.xml - - - name: Print test output and Record test result in launchable - if: steps.changes.outputs.end_to_end == 'true' && always() - run: | - # send recorded tests to launchable - launchable record tests --build "$GITHUB_RUN_ID" go-test . || true - - # print test output - cat output.txt + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: nick-fields/retry@v2 + with: + timeout_minutes: 45 + max_attempts: 3 + retry_on: error + command: | + # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file + # which musn't be more than 107 characters long. + export VTDATAROOT="/tmp/" + source build.env + + set -x + + # Increase our local ephemeral port range as we could exhaust this + sudo sysctl -w net.ipv4.ip_local_port_range="22768 61999" + # Increase our open file descriptor limit as we could hit this + ulimit -n 65536 + cat <<-EOF>>./config/mycnf/mysql80.cnf + innodb_buffer_pool_dump_at_shutdown=OFF + innodb_buffer_pool_in_core_file=OFF + innodb_buffer_pool_load_at_startup=OFF + innodb_buffer_pool_size=16M + innodb_doublewrite=OFF + innodb_flush_log_at_trx_commit=0 + innodb_flush_method=O_DIRECT + innodb_numa_interleave=ON + innodb_adaptive_hash_index=OFF + sync_binlog=0 + sync_relay_log=0 + performance_schema=OFF + slow-query-log=OFF + EOF + make embed_config + + # run the tests however you normally do, then produce a JUnit XML file + eatmydata -- go run test.go -docker=false -follow -shard vtctlbackup_sharded_clustertest_heavy diff --git a/.github/workflows/cluster_endtoend_vtgate_concurrentdml.yml b/.github/workflows/cluster_endtoend_vtgate_concurrentdml.yml index 19f41292b9..4943908735 100644 --- a/.github/workflows/cluster_endtoend_vtgate_concurrentdml.yml +++ b/.github/workflows/cluster_endtoend_vtgate_concurrentdml.yml @@ -49,10 +49,9 @@ jobs: - name: Set up Go if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - uses: actions/setup-go@v4 + uses: actions/setup-go@v3 with: - go-version: '1.20' - check-latest: true + go-version: 1.19.12 - name: Set up python if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' @@ -73,7 +72,7 @@ jobs: run: | # Get key to latest MySQL repo - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 # Setup MySQL 8.0 wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.29-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections diff --git a/.github/workflows/cluster_endtoend_vtgate_gen4.yml b/.github/workflows/cluster_endtoend_vtgate_gen4.yml index c3a8552d1f..eed32176e1 100644 --- a/.github/workflows/cluster_endtoend_vtgate_gen4.yml +++ b/.github/workflows/cluster_endtoend_vtgate_gen4.yml @@ -49,10 +49,9 @@ jobs: - name: Set up Go if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - uses: actions/setup-go@v4 + uses: actions/setup-go@v3 with: - go-version: '1.20' - check-latest: true + go-version: 1.19.12 - name: Set up python if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' @@ -73,7 +72,7 @@ jobs: run: | # Get key to latest MySQL repo - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 # Setup MySQL 8.0 wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.29-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections diff --git a/.github/workflows/cluster_endtoend_vtgate_general_heavy.yml b/.github/workflows/cluster_endtoend_vtgate_general_heavy.yml index 6fa87a9f8d..251ba1a17e 100644 --- a/.github/workflows/cluster_endtoend_vtgate_general_heavy.yml +++ b/.github/workflows/cluster_endtoend_vtgate_general_heavy.yml @@ -49,10 +49,9 @@ jobs: - name: Set up Go if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - uses: actions/setup-go@v4 + uses: actions/setup-go@v3 with: - go-version: '1.20' - check-latest: true + go-version: 1.19.12 - name: Set up python if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' @@ -73,7 +72,7 @@ jobs: run: | # Get key to latest MySQL repo - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 # Setup MySQL 8.0 wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.29-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections diff --git a/.github/workflows/cluster_endtoend_vtgate_godriver.yml b/.github/workflows/cluster_endtoend_vtgate_godriver.yml index 81583be459..a59fcd2e2d 100644 --- a/.github/workflows/cluster_endtoend_vtgate_godriver.yml +++ b/.github/workflows/cluster_endtoend_vtgate_godriver.yml @@ -49,10 +49,9 @@ jobs: - name: Set up Go if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - uses: actions/setup-go@v4 + uses: actions/setup-go@v3 with: - go-version: '1.20' - check-latest: true + go-version: 1.19.12 - name: Set up python if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' @@ -73,7 +72,7 @@ jobs: run: | # Get key to latest MySQL repo - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 # Setup MySQL 8.0 wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.29-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections diff --git a/.github/workflows/cluster_endtoend_vtgate_queries.yml b/.github/workflows/cluster_endtoend_vtgate_queries.yml index fcd7e3c556..8beea54a17 100644 --- a/.github/workflows/cluster_endtoend_vtgate_queries.yml +++ b/.github/workflows/cluster_endtoend_vtgate_queries.yml @@ -49,10 +49,9 @@ jobs: - name: Set up Go if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - uses: actions/setup-go@v4 + uses: actions/setup-go@v3 with: - go-version: '1.20' - check-latest: true + go-version: 1.19.12 - name: Set up python if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' @@ -73,7 +72,7 @@ jobs: run: | # Get key to latest MySQL repo - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 # Setup MySQL 8.0 wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.29-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections diff --git a/.github/workflows/cluster_endtoend_vtgate_readafterwrite.yml b/.github/workflows/cluster_endtoend_vtgate_readafterwrite.yml index 50336d4ca1..f49540f0b2 100644 --- a/.github/workflows/cluster_endtoend_vtgate_readafterwrite.yml +++ b/.github/workflows/cluster_endtoend_vtgate_readafterwrite.yml @@ -49,10 +49,9 @@ jobs: - name: Set up Go if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - uses: actions/setup-go@v4 + uses: actions/setup-go@v3 with: - go-version: '1.20' - check-latest: true + go-version: 1.19.12 - name: Set up python if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' @@ -73,7 +72,7 @@ jobs: run: | # Get key to latest MySQL repo - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 # Setup MySQL 8.0 wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.29-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections diff --git a/.github/workflows/cluster_endtoend_vtgate_reservedconn.yml b/.github/workflows/cluster_endtoend_vtgate_reservedconn.yml index 8105029466..514399104e 100644 --- a/.github/workflows/cluster_endtoend_vtgate_reservedconn.yml +++ b/.github/workflows/cluster_endtoend_vtgate_reservedconn.yml @@ -49,10 +49,9 @@ jobs: - name: Set up Go if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - uses: actions/setup-go@v4 + uses: actions/setup-go@v3 with: - go-version: '1.20' - check-latest: true + go-version: 1.19.12 - name: Set up python if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' @@ -73,7 +72,7 @@ jobs: run: | # Get key to latest MySQL repo - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 # Setup MySQL 8.0 wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.29-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections diff --git a/.github/workflows/cluster_endtoend_vtgate_schema.yml b/.github/workflows/cluster_endtoend_vtgate_schema.yml index 790891c9a5..646cb2f94a 100644 --- a/.github/workflows/cluster_endtoend_vtgate_schema.yml +++ b/.github/workflows/cluster_endtoend_vtgate_schema.yml @@ -49,10 +49,9 @@ jobs: - name: Set up Go if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - uses: actions/setup-go@v4 + uses: actions/setup-go@v3 with: - go-version: '1.20' - check-latest: true + go-version: 1.19.12 - name: Set up python if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' @@ -73,7 +72,7 @@ jobs: run: | # Get key to latest MySQL repo - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 # Setup MySQL 8.0 wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.29-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections diff --git a/.github/workflows/cluster_endtoend_vtgate_schema_tracker.yml b/.github/workflows/cluster_endtoend_vtgate_schema_tracker.yml index 8568b95491..068e5b4d4a 100644 --- a/.github/workflows/cluster_endtoend_vtgate_schema_tracker.yml +++ b/.github/workflows/cluster_endtoend_vtgate_schema_tracker.yml @@ -49,10 +49,9 @@ jobs: - name: Set up Go if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - uses: actions/setup-go@v4 + uses: actions/setup-go@v3 with: - go-version: '1.20' - check-latest: true + go-version: 1.19.12 - name: Set up python if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' @@ -73,7 +72,7 @@ jobs: run: | # Get key to latest MySQL repo - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 # Setup MySQL 8.0 wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.29-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections diff --git a/.github/workflows/cluster_endtoend_vtgate_tablet_healthcheck_cache.yml b/.github/workflows/cluster_endtoend_vtgate_tablet_healthcheck_cache.yml index 7f2776d946..45a335bf83 100644 --- a/.github/workflows/cluster_endtoend_vtgate_tablet_healthcheck_cache.yml +++ b/.github/workflows/cluster_endtoend_vtgate_tablet_healthcheck_cache.yml @@ -49,10 +49,9 @@ jobs: - name: Set up Go if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - uses: actions/setup-go@v4 + uses: actions/setup-go@v3 with: - go-version: '1.20' - check-latest: true + go-version: 1.19.12 - name: Set up python if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' @@ -73,7 +72,7 @@ jobs: run: | # Get key to latest MySQL repo - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 # Setup MySQL 8.0 wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.29-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections diff --git a/.github/workflows/cluster_endtoend_vtgate_topo.yml b/.github/workflows/cluster_endtoend_vtgate_topo.yml index a8b485706e..234ac643b3 100644 --- a/.github/workflows/cluster_endtoend_vtgate_topo.yml +++ b/.github/workflows/cluster_endtoend_vtgate_topo.yml @@ -49,10 +49,9 @@ jobs: - name: Set up Go if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - uses: actions/setup-go@v4 + uses: actions/setup-go@v3 with: - go-version: '1.20' - check-latest: true + go-version: 1.19.12 - name: Set up python if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' @@ -73,7 +72,7 @@ jobs: run: | # Get key to latest MySQL repo - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 # Setup MySQL 8.0 wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.29-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections diff --git a/.github/workflows/cluster_endtoend_vtgate_topo_consul.yml b/.github/workflows/cluster_endtoend_vtgate_topo_consul.yml index 5bf87cc33a..6651b34a1d 100644 --- a/.github/workflows/cluster_endtoend_vtgate_topo_consul.yml +++ b/.github/workflows/cluster_endtoend_vtgate_topo_consul.yml @@ -49,10 +49,9 @@ jobs: - name: Set up Go if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - uses: actions/setup-go@v4 + uses: actions/setup-go@v3 with: - go-version: '1.20' - check-latest: true + go-version: 1.19.12 - name: Set up python if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' @@ -73,7 +72,7 @@ jobs: run: | # Get key to latest MySQL repo - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 # Setup MySQL 8.0 wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.29-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections diff --git a/.github/workflows/cluster_endtoend_vtgate_topo_etcd.yml b/.github/workflows/cluster_endtoend_vtgate_topo_etcd.yml index 8a2c986582..ac58b03491 100644 --- a/.github/workflows/cluster_endtoend_vtgate_topo_etcd.yml +++ b/.github/workflows/cluster_endtoend_vtgate_topo_etcd.yml @@ -49,10 +49,9 @@ jobs: - name: Set up Go if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - uses: actions/setup-go@v4 + uses: actions/setup-go@v3 with: - go-version: '1.20' - check-latest: true + go-version: 1.19.12 - name: Set up python if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' @@ -73,7 +72,7 @@ jobs: run: | # Get key to latest MySQL repo - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 # Setup MySQL 8.0 wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.29-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections diff --git a/.github/workflows/cluster_endtoend_vtgate_transaction.yml b/.github/workflows/cluster_endtoend_vtgate_transaction.yml index cac76ab4aa..d110af2f82 100644 --- a/.github/workflows/cluster_endtoend_vtgate_transaction.yml +++ b/.github/workflows/cluster_endtoend_vtgate_transaction.yml @@ -49,10 +49,9 @@ jobs: - name: Set up Go if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - uses: actions/setup-go@v4 + uses: actions/setup-go@v3 with: - go-version: '1.20' - check-latest: true + go-version: 1.19.12 - name: Set up python if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' @@ -73,7 +72,7 @@ jobs: run: | # Get key to latest MySQL repo - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 # Setup MySQL 8.0 wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.29-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections diff --git a/.github/workflows/cluster_endtoend_vtgate_unsharded.yml b/.github/workflows/cluster_endtoend_vtgate_unsharded.yml index 413c805696..13412abe6c 100644 --- a/.github/workflows/cluster_endtoend_vtgate_unsharded.yml +++ b/.github/workflows/cluster_endtoend_vtgate_unsharded.yml @@ -49,10 +49,9 @@ jobs: - name: Set up Go if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - uses: actions/setup-go@v4 + uses: actions/setup-go@v3 with: - go-version: '1.20' - check-latest: true + go-version: 1.19.12 - name: Set up python if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' @@ -73,7 +72,7 @@ jobs: run: | # Get key to latest MySQL repo - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 # Setup MySQL 8.0 wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.29-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections diff --git a/.github/workflows/cluster_endtoend_vtgate_vindex_heavy.yml b/.github/workflows/cluster_endtoend_vtgate_vindex_heavy.yml index c083548fa6..010fed9e39 100644 --- a/.github/workflows/cluster_endtoend_vtgate_vindex_heavy.yml +++ b/.github/workflows/cluster_endtoend_vtgate_vindex_heavy.yml @@ -49,10 +49,9 @@ jobs: - name: Set up Go if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - uses: actions/setup-go@v4 + uses: actions/setup-go@v3 with: - go-version: '1.20' - check-latest: true + go-version: 1.19.12 - name: Set up python if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' @@ -73,7 +72,7 @@ jobs: run: | # Get key to latest MySQL repo - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 # Setup MySQL 8.0 wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.29-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections diff --git a/.github/workflows/cluster_endtoend_vtgate_vschema.yml b/.github/workflows/cluster_endtoend_vtgate_vschema.yml index c8752a5f6d..49645a7818 100644 --- a/.github/workflows/cluster_endtoend_vtgate_vschema.yml +++ b/.github/workflows/cluster_endtoend_vtgate_vschema.yml @@ -49,10 +49,9 @@ jobs: - name: Set up Go if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - uses: actions/setup-go@v4 + uses: actions/setup-go@v3 with: - go-version: '1.20' - check-latest: true + go-version: 1.19.12 - name: Set up python if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' @@ -73,7 +72,7 @@ jobs: run: | # Get key to latest MySQL repo - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 # Setup MySQL 8.0 wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.29-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections diff --git a/.github/workflows/cluster_endtoend_vtorc.yml b/.github/workflows/cluster_endtoend_vtorc.yml index 7e616c3568..8c57c1e68f 100644 --- a/.github/workflows/cluster_endtoend_vtorc.yml +++ b/.github/workflows/cluster_endtoend_vtorc.yml @@ -49,10 +49,9 @@ jobs: - name: Set up Go if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - uses: actions/setup-go@v4 + uses: actions/setup-go@v3 with: - go-version: '1.20' - check-latest: true + go-version: 1.19.12 - name: Set up python if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' @@ -73,7 +72,7 @@ jobs: run: | # Get key to latest MySQL repo - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 # Setup MySQL 8.0 wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.29-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections diff --git a/.github/workflows/cluster_endtoend_vtorc_8.0.yml b/.github/workflows/cluster_endtoend_vtorc_8.0.yml index 9e1673caf0..2b3bdbe0ef 100644 --- a/.github/workflows/cluster_endtoend_vtorc_8.0.yml +++ b/.github/workflows/cluster_endtoend_vtorc_8.0.yml @@ -6,23 +6,29 @@ concurrency: group: format('{0}-{1}', ${{ github.ref }}, 'Cluster (vtorc_8.0)') cancel-in-progress: true -env: - LAUNCHABLE_ORGANIZATION: "vitess" - LAUNCHABLE_WORKSPACE: "vitess-app" - GITHUB_PR_HEAD_SHA: "${{ github.event.pull_request.head.sha }}" - jobs: build: name: Run endtoend tests on Cluster (vtorc_8.0) runs-on: group: vitess-ubuntu20 - timeout-minutes: 45 steps: + - name: Check if workflow needs to be skipped + id: skip-workflow + run: | + skip='false' + if [[ "${{github.event.pull_request}}" == "" ]] && [[ "${{github.ref}}" != "refs/heads/main" ]] && [[ ! "${{github.ref}}" =~ ^refs/heads/slack-vitess-r[0-9]+\.[0-9]\.[0-9]+$ ]] && [[ ! "${{github.ref}}" =~ "refs/tags/.*" ]]; then + skip='true' + fi + echo Skip ${skip} + echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT + - name: Check out code - uses: actions/checkout@v2 + if: steps.skip-workflow.outputs.skip-workflow == 'false' + uses: actions/checkout@v3 - name: Check for changes in relevant files + if: steps.skip-workflow.outputs.skip-workflow == 'false' uses: frouioui/paths-filter@main id: changes with: @@ -33,47 +39,53 @@ jobs: - 'test.go' - 'Makefile' - 'build.env' - - 'go.[sumod]' + - 'go.sum' + - 'go.mod' - 'proto/*.proto' - 'tools/**' - 'config/**' - 'bootstrap.sh' - - '.github/workflows/**' + - '.github/workflows/cluster_endtoend_vtorc_8.0.yml' - name: Set up Go - if: steps.changes.outputs.end_to_end == 'true' - uses: actions/setup-go@v4 + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/setup-go@v3 with: - go-version: '1.20' - check-latest: true + go-version: 1.19.12 - name: Set up python - if: steps.changes.outputs.end_to_end == 'true' - uses: actions/setup-python@v2 + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/setup-python@v4 - name: Tune the OS - if: steps.changes.outputs.end_to_end == 'true' + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf sudo sysctl -p /etc/sysctl.conf - name: Get dependencies - if: steps.changes.outputs.end_to_end == 'true' + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | - # Setup Percona Server for MySQL 8.0 + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + # Setup MySQL 8.0 + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.29-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update - sudo apt-get install -y lsb-release gnupg2 curl - wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb - sudo DEBIAN_FRONTEND="noninteractive" dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb - sudo percona-release setup ps80 - sudo apt-get update - # Install everything else we need, and configure - sudo apt-get install -y percona-server-server percona-server-client make unzip g++ etcd git wget eatmydata xz-utils - sudo service mysql stop + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils + sudo apt-get update + sudo service etcd stop + echo 'mysql version: ' + sudo mysql --version + sudo service mysql stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download @@ -81,37 +93,20 @@ jobs: # install JUnit report formatter go install github.com/vitessio/go-junit-report@HEAD - - name: Setup launchable dependencies - if: steps.changes.outputs.end_to_end == 'true' - run: | - # Get Launchable CLI installed. If you can, make it a part of the builder image to speed things up - pip3 install --user launchable~=1.0 > /dev/null - - # verify that launchable setup is all correct. - launchable verify || true - - # Tell Launchable about the build you are producing and testing - launchable record build --name "$GITHUB_RUN_ID" --source . - - name: Run cluster endtoend test - if: steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 30 - run: | - # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file - # which musn't be more than 107 characters long. - export VTDATAROOT="/tmp/" - source build.env - - set -x - - # run the tests however you normally do, then produce a JUnit XML file - eatmydata -- go run test.go -docker=false -follow -shard vtorc_8.0 | tee -a output.txt | go-junit-report -set-exit-code > report.xml - - - name: Print test output and Record test result in launchable - if: steps.changes.outputs.end_to_end == 'true' && always() - run: | - # send recorded tests to launchable - launchable record tests --build "$GITHUB_RUN_ID" go-test . || true - - # print test output - cat output.txt + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: nick-fields/retry@v2 + with: + timeout_minutes: 45 + max_attempts: 3 + retry_on: error + command: | + # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file + # which musn't be more than 107 characters long. + export VTDATAROOT="/tmp/" + source build.env + + set -x + + # run the tests however you normally do, then produce a JUnit XML file + eatmydata -- go run test.go -docker=false -follow -shard vtorc_8.0 diff --git a/.github/workflows/cluster_endtoend_xb_backup.yml b/.github/workflows/cluster_endtoend_xb_backup.yml index 91f6fa0fd7..df4bf36cb9 100644 --- a/.github/workflows/cluster_endtoend_xb_backup.yml +++ b/.github/workflows/cluster_endtoend_xb_backup.yml @@ -54,10 +54,9 @@ jobs: - name: Set up Go if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - uses: actions/setup-go@v4 + uses: actions/setup-go@v3 with: - go-version: '1.20' - check-latest: true + go-version: 1.19.12 - name: Set up python if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' diff --git a/.github/workflows/cluster_endtoend_xb_backup_mysql57.yml b/.github/workflows/cluster_endtoend_xb_backup_mysql57.yml index aeb1a8dd93..a42fd7c5dc 100644 --- a/.github/workflows/cluster_endtoend_xb_backup_mysql57.yml +++ b/.github/workflows/cluster_endtoend_xb_backup_mysql57.yml @@ -6,27 +6,19 @@ concurrency: group: format('{0}-{1}', ${{ github.ref }}, 'Cluster (xb_backup) mysql57') cancel-in-progress: true -env: - LAUNCHABLE_ORGANIZATION: "vitess" - LAUNCHABLE_WORKSPACE: "vitess-app" - GITHUB_PR_HEAD_SHA: "${{ github.event.pull_request.head.sha }}" - - # This is used if we need to pin the xtrabackup version used in tests. - # If this is NOT set then the latest version available will be used. - #XTRABACKUP_VERSION: "2.4.24-1" - jobs: build: name: Run endtoend tests on Cluster (xb_backup) mysql57 runs-on: group: vitess-ubuntu20 + timeout-minutes: 45 steps: - name: Check if workflow needs to be skipped id: skip-workflow run: | skip='false' - if [[ "${{github.event.pull_request}}" == "" ]] && [[ "${{github.ref}}" != "refs/heads/main" ]] && [[ ! "${{github.ref}}" =~ ^refs/heads/release-[0-9]+\.[0-9]$ ]] && [[ ! "${{github.ref}}" =~ "refs/tags/.*" ]]; then + if [[ "${{github.event.pull_request}}" == "" ]] && [[ "${{github.ref}}" != "refs/heads/main" ]] && [[ ! "${{github.ref}}" =~ ^refs/heads/slack-vitess-r[0-9]+\.[0-9]\.[0-9]+$ ]] && [[ ! "${{github.ref}}" =~ "refs/tags/.*" ]]; then skip='true' fi echo Skip ${skip} @@ -48,7 +40,8 @@ jobs: - 'test.go' - 'Makefile' - 'build.env' - - 'go.[sumod]' + - 'go.sum' + - 'go.mod' - 'proto/*.proto' - 'tools/**' - 'config/**' @@ -57,10 +50,9 @@ jobs: - name: Set up Go if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - uses: actions/setup-go@v4 + uses: actions/setup-go@v3 with: - go-version: '1.20' - check-latest: true + go-version: 1.19.12 - name: Set up python if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' @@ -92,7 +84,7 @@ jobs: sudo rm -rf /etc/mysql # Get key to latest MySQL repo - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.14-1_all.deb # Bionic packages are still compatible for Focal since there's no MySQL 5.7 @@ -103,6 +95,7 @@ jobs: sudo apt-get update sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y mysql-client=5.7* mysql-community-server=5.7* mysql-server=5.7* + # Install everything else we need, and configure sudo apt-get install -y make unzip g++ etcd curl git wget eatmydata sudo service mysql stop sudo service etcd stop @@ -122,37 +115,20 @@ jobs: sudo apt-get install -y percona-xtrabackup-24 fi - - name: Setup launchable dependencies - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - run: | - # Get Launchable CLI installed. If you can, make it a part of the builder image to speed things up - pip3 install --user launchable~=1.0 > /dev/null - - # verify that launchable setup is all correct. - launchable verify || true - - # Tell Launchable about the build you are producing and testing - launchable record build --name "$GITHUB_RUN_ID" --source . - - name: Run cluster endtoend test if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - timeout-minutes: 45 - run: | - # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file - # which musn't be more than 107 characters long. - export VTDATAROOT="/tmp/" - source build.env - - set -x - - # run the tests however you normally do, then produce a JUnit XML file - eatmydata -- go run test.go -docker=false -follow -shard xb_backup | tee -a output.txt | go-junit-report -set-exit-code > report.xml - - - name: Print test output and Record test result in launchable - if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() - run: | - # send recorded tests to launchable - launchable record tests --build "$GITHUB_RUN_ID" go-test . || true - - # print test output - cat output.txt + uses: nick-fields/retry@v2 + with: + timeout_minutes: 45 + max_attempts: 3 + retry_on: error + command: | + # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file + # which musn't be more than 107 characters long. + export VTDATAROOT="/tmp/" + source build.env + + set -x + + # run the tests however you normally do, then produce a JUnit XML file + eatmydata -- go run test.go -docker=false -follow -shard xb_backup diff --git a/.github/workflows/cluster_endtoend_xb_recovery.yml b/.github/workflows/cluster_endtoend_xb_recovery.yml index b78243f5ba..0b8088b397 100644 --- a/.github/workflows/cluster_endtoend_xb_recovery.yml +++ b/.github/workflows/cluster_endtoend_xb_recovery.yml @@ -54,10 +54,9 @@ jobs: - name: Set up Go if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' - uses: actions/setup-go@v4 + uses: actions/setup-go@v3 with: - go-version: '1.20' - check-latest: true + go-version: 1.19.12 - name: Set up python if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' diff --git a/.github/workflows/unit_test_mysql57.yml b/.github/workflows/unit_test_mysql57.yml index 87f634b584..4ae0c38d59 100644 --- a/.github/workflows/unit_test_mysql57.yml +++ b/.github/workflows/unit_test_mysql57.yml @@ -48,10 +48,9 @@ jobs: - name: Set up Go if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.unit_tests == 'true' - uses: actions/setup-go@v4 + uses: actions/setup-go@v3 with: - go-version: '1.20' - check-latest: true + go-version: 1.19.12 - name: Tune the OS if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.unit_tests == 'true' diff --git a/.github/workflows/unit_test_mysql80.yml b/.github/workflows/unit_test_mysql80.yml index f84bbed649..1041895a0b 100644 --- a/.github/workflows/unit_test_mysql80.yml +++ b/.github/workflows/unit_test_mysql80.yml @@ -48,10 +48,9 @@ jobs: - name: Set up Go if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.unit_tests == 'true' - uses: actions/setup-go@v4 + uses: actions/setup-go@v3 with: - go-version: '1.20' - check-latest: true + go-version: 1.19.12 - name: Tune the OS if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.unit_tests == 'true' @@ -79,7 +78,7 @@ jobs: sudo rm -rf /etc/mysql # Get key to latest MySQL repo - sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 # mysql80 wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.14-1_all.deb