Skip to content

Commit

Permalink
Merge branch 'main' into vtorc-store-parsed-clustersToWatch
Browse files Browse the repository at this point in the history
Signed-off-by: Tim Vaillancourt <[email protected]>
  • Loading branch information
timvaillancourt authored Dec 16, 2024
2 parents 67c0d68 + c48d761 commit b6077f6
Show file tree
Hide file tree
Showing 417 changed files with 11,414 additions and 5,736 deletions.
7 changes: 7 additions & 0 deletions .github/workflows/cluster_endtoend_21.yml
Original file line number Diff line number Diff line change
Expand Up @@ -121,6 +121,13 @@ jobs:
# install JUnit report formatter
go install github.com/vitessio/go-junit-report@HEAD
- name: Install Minio
if: steps.skip-workflow.outputs.skip-workflow == 'false'
run: |
wget https://dl.min.io/server/minio/release/linux-amd64/minio
chmod +x minio
mv minio /usr/local/bin
- name: Setup launchable dependencies
if: steps.skip-workflow.outputs.is_draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main'
run: |
Expand Down
166 changes: 166 additions & 0 deletions .github/workflows/cluster_endtoend_vtgate_plantests.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,166 @@
# DO NOT MODIFY: THIS FILE IS GENERATED USING "make generate_ci_workflows"

name: Cluster (vtgate_plantests)
on: [push, pull_request]
concurrency:
group: format('{0}-{1}', ${{ github.ref }}, 'Cluster (vtgate_plantests)')
cancel-in-progress: true

permissions: read-all

env:
LAUNCHABLE_ORGANIZATION: "vitess"
LAUNCHABLE_WORKSPACE: "vitess-app"
GITHUB_PR_HEAD_SHA: "${{ github.event.pull_request.head.sha }}"

jobs:
build:
name: Run endtoend tests on Cluster (vtgate_plantests)
runs-on: ubuntu-24.04

steps:
- name: Skip CI
run: |
if [[ "${{contains( github.event.pull_request.labels.*.name, 'Skip CI')}}" == "true" ]]; then
echo "skipping CI due to the 'Skip CI' label"
exit 1
fi
- name: Check if workflow needs to be skipped
id: skip-workflow
run: |
skip='false'
if [[ "${{github.event.pull_request}}" == "" ]] && [[ "${{github.ref}}" != "refs/heads/main" ]] && [[ ! "${{github.ref}}" =~ ^refs/heads/release-[0-9]+\.[0-9]$ ]] && [[ ! "${{github.ref}}" =~ "refs/tags/.*" ]]; then
skip='true'
fi
echo Skip ${skip}
echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT
PR_DATA=$(curl -s\
-H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \
-H "Accept: application/vnd.github.v3+json" \
"https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}")
draft=$(echo "$PR_DATA" | jq .draft -r)
echo "is_draft=${draft}" >> $GITHUB_OUTPUT
- name: Check out code
if: steps.skip-workflow.outputs.skip-workflow == 'false'
uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7

- name: Check for changes in relevant files
if: steps.skip-workflow.outputs.skip-workflow == 'false'
uses: dorny/paths-filter@ebc4d7e9ebcb0b1eb21480bb8f43113e996ac77a # v3.0.1
id: changes
with:
token: ''
filters: |
end_to_end:
- 'go/**/*.go'
- 'go/vt/sidecardb/**/*.sql'
- 'go/test/endtoend/onlineddl/vrepl_suite/**'
- 'test.go'
- 'Makefile'
- 'build.env'
- 'go.sum'
- 'go.mod'
- 'proto/*.proto'
- 'tools/**'
- 'config/**'
- 'bootstrap.sh'
- '.github/workflows/cluster_endtoend_vtgate_plantests.yml'
- name: Set up Go
if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2
with:
go-version-file: go.mod

- name: Set up python
if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1

- name: Tune the OS
if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
run: |
# Limit local port range to not use ports that overlap with server side
# ports that we listen on.
sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535"
# Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio
echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf
sudo sysctl -p /etc/sysctl.conf
- name: Get dependencies
if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
run: |
# Get key to latest MySQL repo
sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A8D3785C
# Setup MySQL 8.0
wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.33-1_all.deb
echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections
sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config*
sudo apt-get -qq update
# We have to install this old version of libaio1 in case we end up testing with MySQL 5.7. See also:
# https://bugs.launchpad.net/ubuntu/+source/libaio/+bug/2067501
curl -L -O http://mirrors.kernel.org/ubuntu/pool/main/liba/libaio/libaio1_0.3.112-13build1_amd64.deb
sudo dpkg -i libaio1_0.3.112-13build1_amd64.deb
# libtinfo5 is also needed for older MySQL 5.7 builds.
curl -L -O http://mirrors.kernel.org/ubuntu/pool/universe/n/ncurses/libtinfo5_6.3-2ubuntu0.1_amd64.deb
sudo dpkg -i libtinfo5_6.3-2ubuntu0.1_amd64.deb
# Install everything else we need, and configure
sudo apt-get -qq install -y mysql-server mysql-shell mysql-client make unzip g++ etcd-client etcd-server curl git wget eatmydata xz-utils libncurses6
sudo service mysql stop
sudo service etcd stop
sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/
sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld
go mod download
# install JUnit report formatter
go install github.com/vitessio/go-junit-report@HEAD
- name: Setup launchable dependencies
if: steps.skip-workflow.outputs.is_draft == 'false' && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main'
run: |
# Get Launchable CLI installed. If you can, make it a part of the builder image to speed things up
pip3 install --user launchable~=1.0 > /dev/null
# verify that launchable setup is all correct.
launchable verify || true
# Tell Launchable about the build you are producing and testing
launchable record build --name "$GITHUB_RUN_ID" --no-commit-collection --source .
- name: Run cluster endtoend test
if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
timeout-minutes: 45
run: |
# We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file
# which musn't be more than 107 characters long.
export VTDATAROOT="/tmp/"
source build.env
set -exo pipefail
# run the tests however you normally do, then produce a JUnit XML file
eatmydata -- go run test.go -docker=false -follow -shard vtgate_plantests | tee -a output.txt | go-junit-report -set-exit-code > report.xml
- name: Print test output and Record test result in launchable if PR is not a draft
if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always()
run: |
if [[ "${{steps.skip-workflow.outputs.is_draft}}" == "false" ]]; then
# send recorded tests to launchable
launchable record tests --build "$GITHUB_RUN_ID" go-test . || true
fi
# print test output
cat output.txt
- name: Test Summary
if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always()
uses: test-summary/action@31493c76ec9e7aa675f1585d3ed6f1da69269a86 # v2.4
with:
paths: "report.xml"
show: "fail"
4 changes: 2 additions & 2 deletions .github/workflows/scorecards.yml
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ jobs:
# Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF
# format to the repository Actions tab.
- name: "Upload artifact"
uses: actions/upload-artifact@v3
uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
with:
name: SARIF file
path: results.sarif
Expand All @@ -62,6 +62,6 @@ jobs:
# Upload the results to GitHub's code scanning dashboard (optional).
# Commenting out will disable upload of results to your repo's Code Scanning dashboard
- name: "Upload to code-scanning"
uses: github/codeql-action/upload-sarif@v3
uses: github/codeql-action/upload-sarif@f09c1c0a94de965c15400f5634aa42fac8fb8f88 # v3.27.5
with:
sarif_file: results.sarif
2 changes: 1 addition & 1 deletion .github/workflows/update_golang_version.yml
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ jobs:
if [ ${{ matrix.branch }} == "main" ]; then
go run ./go/tools/go-upgrade/go-upgrade.go upgrade --main --allow-major-upgrade
else if [ ${{ matrix.branch }} == "release-21.0" ]; then
elif [ ${{ matrix.branch }} == "release-21.0" ]; then
go run ./go/tools/go-upgrade/go-upgrade.go upgrade
else
go run ./go/tools/go-upgrade/go-upgrade.go upgrade --workflow-update=false
Expand Down
2 changes: 1 addition & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -286,7 +286,7 @@ $(PROTO_GO_OUTS): minimaltools install_protoc-gen-go proto/*.proto
# This rule builds the bootstrap images for all flavors.
DOCKER_IMAGES_FOR_TEST = mysql80 percona80
DOCKER_IMAGES = common $(DOCKER_IMAGES_FOR_TEST)
BOOTSTRAP_VERSION=38
BOOTSTRAP_VERSION=39
ensure_bootstrap_version:
find docker/ -type f -exec sed -i "s/^\(ARG bootstrap_version\)=.*/\1=${BOOTSTRAP_VERSION}/" {} \;
sed -i 's/\(^.*flag.String(\"bootstrap-version\",\) *\"[^\"]\+\"/\1 \"${BOOTSTRAP_VERSION}\"/' test.go
Expand Down
2 changes: 1 addition & 1 deletion build.env
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
source ./tools/shell_functions.inc

go version >/dev/null 2>&1 || fail "Go is not installed or is not in \$PATH. See https://vitess.io/contributing/build-from-source for install instructions."
goversion_min 1.23.3 || echo "Go version reported: `go version`. Version 1.23.3+ recommended. See https://vitess.io/contributing/build-from-source for install instructions."
goversion_min 1.23.4 || echo "Go version reported: `go version`. Version 1.23.4+ recommended. See https://vitess.io/contributing/build-from-source for install instructions."

mkdir -p dist
mkdir -p bin
Expand Down
29 changes: 29 additions & 0 deletions changelog/19.0/19.0.8/changelog.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
# Changelog of Vitess v19.0.8

### Bug fixes
#### Topology
* [release-19.0] Close zookeeper topo connection on disconnect (#17136) [#17191](https://github.com/vitessio/vitess/pull/17191)
#### VTTablet
* [release-19.0] Fix deadlock in messager and health streamer (#17230) [#17233](https://github.com/vitessio/vitess/pull/17233)
* [release-19.0] Fix potential deadlock in health streamer (#17261) [#17268](https://github.com/vitessio/vitess/pull/17268)
### CI/Build
#### Build/CI
* [release-19.0] Specify Ubuntu 24.04 for all jobs (#17278) [#17280](https://github.com/vitessio/vitess/pull/17280)
#### Cluster management
* [release-19.0] Fix flakiness in `TestListenerShutdown` (#17024) [#17187](https://github.com/vitessio/vitess/pull/17187)
#### General
* [release-19.0] Upgrade the Golang version to `go1.22.9` [#17214](https://github.com/vitessio/vitess/pull/17214)
### Enhancement
#### Query Serving
* [release-19.0] Fix to prevent stopping buffering prematurely (#17013) [#17203](https://github.com/vitessio/vitess/pull/17203)
### Internal Cleanup
#### Build/CI
* [release-19.0] Change the name of the vitess-tester repository (#16917) [#17028](https://github.com/vitessio/vitess/pull/17028)
### Release
#### General
* [release-19.0] Bump to `v19.0.8-SNAPSHOT` after the `v19.0.7` release [#17158](https://github.com/vitessio/vitess/pull/17158)
* [release-19.0] Code Freeze for `v19.0.8` [#17310](https://github.com/vitessio/vitess/pull/17310)
### Testing
#### Build/CI
* [release-19.0] Flakes: Address flakiness in TestZkConnClosedOnDisconnect (#17194) [#17195](https://github.com/vitessio/vitess/pull/17195)

7 changes: 7 additions & 0 deletions changelog/19.0/19.0.8/release_notes.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
# Release of Vitess v19.0.8
The entire changelog for this release can be found [here](https://github.com/vitessio/vitess/blob/main/changelog/19.0/19.0.8/changelog.md).

The release includes 11 merged Pull Requests.

Thanks to all our contributors: @app/vitess-bot, @frouioui, @vitess-bot

4 changes: 4 additions & 0 deletions changelog/19.0/README.md
Original file line number Diff line number Diff line change
@@ -1,4 +1,8 @@
## v19.0
* **[19.0.8](19.0.8)**
* [Changelog](19.0.8/changelog.md)
* [Release Notes](19.0.8/release_notes.md)

* **[19.0.7](19.0.7)**
* [Changelog](19.0.7/changelog.md)
* [Release Notes](19.0.7/release_notes.md)
Expand Down
27 changes: 27 additions & 0 deletions changelog/20.0/20.0.4/changelog.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
# Changelog of Vitess v20.0.4

### Bug fixes
#### Query Serving
* [release-20.0] Use proper keyspace when updating the query graph of a reference DML (#17226) [#17257](https://github.com/vitessio/vitess/pull/17257)
#### Topology
* [release-20.0] Close zookeeper topo connection on disconnect (#17136) [#17192](https://github.com/vitessio/vitess/pull/17192)
#### VTTablet
* [release-20.0] Fix deadlock in messager and health streamer (#17230) [#17234](https://github.com/vitessio/vitess/pull/17234)
* [release-20.0] Fix potential deadlock in health streamer (#17261) [#17269](https://github.com/vitessio/vitess/pull/17269)
### CI/Build
#### Build/CI
* [release-20.0] Specify Ubuntu 24.04 for all jobs (#17278) [#17281](https://github.com/vitessio/vitess/pull/17281)
#### Cluster management
* [release-20.0] Fix flakiness in `TestListenerShutdown` (#17024) [#17188](https://github.com/vitessio/vitess/pull/17188)
#### General
* [release-20.0] Upgrade the Golang version to `go1.22.9` [#17212](https://github.com/vitessio/vitess/pull/17212)
### Enhancement
#### Query Serving
* [release-20.0] Fix to prevent stopping buffering prematurely (#17013) [#17204](https://github.com/vitessio/vitess/pull/17204)
### Internal Cleanup
#### Build/CI
* [release-20.0] Change the name of the vitess-tester repository (#16917) [#17029](https://github.com/vitessio/vitess/pull/17029)
### Testing
#### Build/CI
* [release-20.0] Flakes: Address flakiness in TestZkConnClosedOnDisconnect (#17194) [#17196](https://github.com/vitessio/vitess/pull/17196)

7 changes: 7 additions & 0 deletions changelog/20.0/20.0.4/release_notes.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
# Release of Vitess v20.0.4
The entire changelog for this release can be found [here](https://github.com/vitessio/vitess/blob/main/changelog/20.0/20.0.4/changelog.md).

The release includes 10 merged Pull Requests.

Thanks to all our contributors: @app/vitess-bot, @frouioui

4 changes: 4 additions & 0 deletions changelog/20.0/README.md
Original file line number Diff line number Diff line change
@@ -1,4 +1,8 @@
## v20.0
* **[20.0.4](20.0.4)**
* [Changelog](20.0.4/changelog.md)
* [Release Notes](20.0.4/release_notes.md)

* **[20.0.3](20.0.3)**
* [Changelog](20.0.3/changelog.md)
* [Release Notes](20.0.3/release_notes.md)
Expand Down
7 changes: 7 additions & 0 deletions changelog/21.0/21.0.0/summary.md
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
- [Deprecated VTTablet Flags](#vttablet-flags)
- [Deletion of deprecated metrics](#metric-deletion)
- [Deprecated Metrics](#deprecations-metrics)
- **[RPC Changes](#rpc-changes)**
- **[Traffic Mirroring](#traffic-mirroring)**
- **[Atomic Distributed Transaction Support](#atomic-transaction)**
- **[New VTGate Shutdown Behavior](#new-vtgate-shutdown-behavior)**
Expand Down Expand Up @@ -77,6 +78,12 @@ The following metrics are now deprecated and will be deleted in a future release
| `vttablet` | `QueryCacheHits` | `QueryEnginePlanCacheHits` |
| `vttablet` | `QueryCacheMisses` | `QueryEnginePlanCacheMisses` |

### <a id="rpc-changes"/>RPC Changes</a>

These are the RPC changes made in this release -
1. `ReadReparentJournalInfo` RPC has been added in TabletManagerClient interface, that is going to be used in EmergencyReparentShard for better errant GTID detection.
2. `PrimaryStatus` RPC in TabletManagerClient interface has been updated to also return the server UUID of the primary. This is going to be used in the vttablets so that they can do their own errant GTID detection in `SetReplicationSource`.

### <a id="traffic-mirroring"/>Traffic Mirroring</a>

Traffic mirroring is intended to help reduce some of the uncertainty inherent to `MoveTables SwitchTraffic`. When
Expand Down
57 changes: 57 additions & 0 deletions changelog/21.0/21.0.1/changelog.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
# Changelog of Vitess v21.0.1

### Bug fixes
#### Backup and Restore
* [release-21.0] Fix how we cancel the context in the builtin backup engine (#17285) [#17291](https://github.com/vitessio/vitess/pull/17291)
* [release-21.0] S3: optional endpoint resolver and correct retrier [#17307](https://github.com/vitessio/vitess/pull/17307)
#### Cluster management
* [release-21.0] Fix panic in vttablet when closing topo server twice (#17094) [#17122](https://github.com/vitessio/vitess/pull/17122)
#### Online DDL
* [release-21.0] Online DDL: fix defer function, potential connection pool exhaustion (#17207) [#17210](https://github.com/vitessio/vitess/pull/17210)
#### Query Serving
* [release-21.0] bugfix: treat EXPLAIN like SELECT (#17054) [#17058](https://github.com/vitessio/vitess/pull/17058)
* [release-21.0] Delegate Column Availability Checks to MySQL for Single-Route Queries (#17077) [#17087](https://github.com/vitessio/vitess/pull/17087)
* [release-21.0] bugfix: Handle CTEs with columns named in the CTE def (#17179) [#17181](https://github.com/vitessio/vitess/pull/17181)
* [release-21.0] Use proper keyspace when updating the query graph of a reference DML (#17226) [#17258](https://github.com/vitessio/vitess/pull/17258)
#### Topology
* [release-21.0] Close zookeeper topo connection on disconnect (#17136) [#17193](https://github.com/vitessio/vitess/pull/17193)
#### VReplication
* [release-21.0] VReplication: Qualify and SQL escape tables in created AutoIncrement VSchema definitions (#17174) [#17176](https://github.com/vitessio/vitess/pull/17176)
#### VTTablet
* [release-21.0] Fix deadlock in messager and health streamer (#17230) [#17235](https://github.com/vitessio/vitess/pull/17235)
* [release-21.0] Fix potential deadlock in health streamer (#17261) [#17270](https://github.com/vitessio/vitess/pull/17270)
### CI/Build
#### Build/CI
* [release-21.0] Specify Ubuntu 24.04 for all jobs (#17278) [#17282](https://github.com/vitessio/vitess/pull/17282)
#### Cluster management
* [release-21.0] Fix flakiness in `TestListenerShutdown` (#17024) [#17189](https://github.com/vitessio/vitess/pull/17189)
#### General
* [release-21.0] Upgrade the Golang version to `go1.23.3` [#17211](https://github.com/vitessio/vitess/pull/17211)
### Dependencies
#### Java
* [release-21.0] java package updates for grpc and protobuf and release plugins (#17100) [#17105](https://github.com/vitessio/vitess/pull/17105)
### Documentation
#### Documentation
* [Direct PR][release-21.0] Add RPC changes segment in the summary doc [#17034](https://github.com/vitessio/vitess/pull/17034)
### Enhancement
#### Online DDL
* [release-21.0] Improve Schema Engine's TablesWithSize80 query (#17066) [#17091](https://github.com/vitessio/vitess/pull/17091)
#### Query Serving
* [release-21.0] Fix to prevent stopping buffering prematurely (#17013) [#17205](https://github.com/vitessio/vitess/pull/17205)
#### VReplication
* [release-21.0] Binlog: Improve ZstdInMemoryDecompressorMaxSize management (#17220) [#17241](https://github.com/vitessio/vitess/pull/17241)
### Internal Cleanup
#### Build/CI
* [release-21.0] Change the name of the vitess-tester repository (#16917) [#17030](https://github.com/vitessio/vitess/pull/17030)
### Regression
#### Backup and Restore
* [release-21.0] Fix unreachable errors when taking a backup (#17062) [#17112](https://github.com/vitessio/vitess/pull/17112)
### Release
#### General
* [release-21.0] Bump to `v21.0.1-SNAPSHOT` after the `v21.0.0` release [#17098](https://github.com/vitessio/vitess/pull/17098)
### Testing
#### Build/CI
* [release-21.0] Flakes: Address flakiness in TestZkConnClosedOnDisconnect (#17194) [#17197](https://github.com/vitessio/vitess/pull/17197)
#### Query Serving
* [release-21.0] fix: flaky test on twopc transaction (#17068) [#17070](https://github.com/vitessio/vitess/pull/17070)

Loading

0 comments on commit b6077f6

Please sign in to comment.