Skip to content

Commit

Permalink
Merge remote-tracking branch 'upstream/main' into vtorc-config-reload…
Browse files Browse the repository at this point in the history
…able

Signed-off-by: Manan Gupta <[email protected]>
  • Loading branch information
GuptaManan100 committed Dec 3, 2024
2 parents a939288 + 9976b37 commit cb16b34
Show file tree
Hide file tree
Showing 50 changed files with 1,702 additions and 154 deletions.
1 change: 1 addition & 0 deletions .github/CODEOWNERS
Validating CODEOWNERS rules …
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ go.sum @ajm188 @deepthi @harshit-gangal @mattlord @rohit-nayak-ps @systay @froui
/go/test/endtoend/onlineddl @rohit-nayak-ps @shlomi-noach
/go/test/endtoend/messaging @mattlord @rohit-nayak-ps @derekperkins
/go/test/endtoend/schemadiff @shlomi-noach @mattlord
/go/test/endtoend/transaction @harshit-gangal @systay @frouioui @GuptaManan100
/go/test/endtoend/*throttler* @shlomi-noach @mattlord @timvaillancourt
/go/test/endtoend/vtgate @harshit-gangal @systay @frouioui
/go/test/endtoend/vtorc @deepthi @shlomi-noach @GuptaManan100 @timvaillancourt
Expand Down
97 changes: 97 additions & 0 deletions .github/workflows/vtop_example.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,97 @@
name: vtop_example
on: [push, pull_request]
concurrency:
group: format('{0}-{1}', ${{ github.ref }}, 'vtop_example')
cancel-in-progress: true

jobs:
build:
name: VTop Example
runs-on: self-hosted

steps:
- name: Skip CI
run: |
if [[ "${{contains( github.event.pull_request.labels.*.name, 'Skip CI')}}" == "true" ]]; then
echo "skipping CI due to the 'Skip CI' label"
exit 1
fi
- name: Check if workflow needs to be skipped
id: skip-workflow
run: |
skip='false'
if [[ "${{github.event.pull_request}}" == "" ]] && [[ "${{github.ref}}" != "refs/heads/main" ]] && [[ ! "${{github.ref}}" =~ ^refs/heads/release-[0-9]+\.[0-9]$ ]] && [[ ! "${{github.ref}}" =~ "refs/tags/.*" ]]; then
skip='true'
fi
echo Skip ${skip}
echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT
PR_DATA=$(curl -s\
-H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \
-H "Accept: application/vnd.github.v3+json" \
"https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}")
draft=$(echo "$PR_DATA" | jq .draft -r)
echo "is_draft=${draft}" >> $GITHUB_OUTPUT
- name: Check out code
if: steps.skip-workflow.outputs.skip-workflow == 'false'
uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7

- name: Check for changes in relevant files
if: steps.skip-workflow.outputs.skip-workflow == 'false'
uses: dorny/paths-filter@ebc4d7e9ebcb0b1eb21480bb8f43113e996ac77a # v3.0.1
id: changes
with:
token: ''
filters: |
end_to_end:
- 'go/**/*.go'
- 'go/vt/sidecardb/**/*.sql'
- 'test.go'
- 'Makefile'
- 'build.env'
- 'go.[sumod]'
- 'proto/*.proto'
- 'tools/**'
- 'config/**'
- 'bootstrap.sh'
- 'examples/**'
- 'test/**'
- '.github/workflows/vtop_example.yml'
- name: Set up Go
if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2
with:
go-version-file: go.mod

- name: Tune the OS
if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
run: |
echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range
- name: Get dependencies
if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
run: |
# Install everything we need, and configure
sudo apt-get install -y eatmydata make
go mod download
# needed for vtctldclient
- name: Build vitess
if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
run: |
make build
- name: Install kubectl & kind
if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
run: |
make install_kubectl_kind
- name: vtop_example
if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
timeout-minutes: 60
run: |
source build.env
eatmydata -- go run test.go -docker=false -skip-build -print-log -follow -retry=1 -timeout=60m vtop_example
29 changes: 29 additions & 0 deletions changelog/19.0/19.0.8/changelog.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
# Changelog of Vitess v19.0.8

### Bug fixes
#### Topology
* [release-19.0] Close zookeeper topo connection on disconnect (#17136) [#17191](https://github.com/vitessio/vitess/pull/17191)
#### VTTablet
* [release-19.0] Fix deadlock in messager and health streamer (#17230) [#17233](https://github.com/vitessio/vitess/pull/17233)
* [release-19.0] Fix potential deadlock in health streamer (#17261) [#17268](https://github.com/vitessio/vitess/pull/17268)
### CI/Build
#### Build/CI
* [release-19.0] Specify Ubuntu 24.04 for all jobs (#17278) [#17280](https://github.com/vitessio/vitess/pull/17280)
#### Cluster management
* [release-19.0] Fix flakiness in `TestListenerShutdown` (#17024) [#17187](https://github.com/vitessio/vitess/pull/17187)
#### General
* [release-19.0] Upgrade the Golang version to `go1.22.9` [#17214](https://github.com/vitessio/vitess/pull/17214)
### Enhancement
#### Query Serving
* [release-19.0] Fix to prevent stopping buffering prematurely (#17013) [#17203](https://github.com/vitessio/vitess/pull/17203)
### Internal Cleanup
#### Build/CI
* [release-19.0] Change the name of the vitess-tester repository (#16917) [#17028](https://github.com/vitessio/vitess/pull/17028)
### Release
#### General
* [release-19.0] Bump to `v19.0.8-SNAPSHOT` after the `v19.0.7` release [#17158](https://github.com/vitessio/vitess/pull/17158)
* [release-19.0] Code Freeze for `v19.0.8` [#17310](https://github.com/vitessio/vitess/pull/17310)
### Testing
#### Build/CI
* [release-19.0] Flakes: Address flakiness in TestZkConnClosedOnDisconnect (#17194) [#17195](https://github.com/vitessio/vitess/pull/17195)

7 changes: 7 additions & 0 deletions changelog/19.0/19.0.8/release_notes.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
# Release of Vitess v19.0.8
The entire changelog for this release can be found [here](https://github.com/vitessio/vitess/blob/main/changelog/19.0/19.0.8/changelog.md).

The release includes 11 merged Pull Requests.

Thanks to all our contributors: @app/vitess-bot, @frouioui, @vitess-bot

4 changes: 4 additions & 0 deletions changelog/19.0/README.md
Original file line number Diff line number Diff line change
@@ -1,4 +1,8 @@
## v19.0
* **[19.0.8](19.0.8)**
* [Changelog](19.0.8/changelog.md)
* [Release Notes](19.0.8/release_notes.md)

* **[19.0.7](19.0.7)**
* [Changelog](19.0.7/changelog.md)
* [Release Notes](19.0.7/release_notes.md)
Expand Down
27 changes: 27 additions & 0 deletions changelog/20.0/20.0.4/changelog.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
# Changelog of Vitess v20.0.4

### Bug fixes
#### Query Serving
* [release-20.0] Use proper keyspace when updating the query graph of a reference DML (#17226) [#17257](https://github.com/vitessio/vitess/pull/17257)
#### Topology
* [release-20.0] Close zookeeper topo connection on disconnect (#17136) [#17192](https://github.com/vitessio/vitess/pull/17192)
#### VTTablet
* [release-20.0] Fix deadlock in messager and health streamer (#17230) [#17234](https://github.com/vitessio/vitess/pull/17234)
* [release-20.0] Fix potential deadlock in health streamer (#17261) [#17269](https://github.com/vitessio/vitess/pull/17269)
### CI/Build
#### Build/CI
* [release-20.0] Specify Ubuntu 24.04 for all jobs (#17278) [#17281](https://github.com/vitessio/vitess/pull/17281)
#### Cluster management
* [release-20.0] Fix flakiness in `TestListenerShutdown` (#17024) [#17188](https://github.com/vitessio/vitess/pull/17188)
#### General
* [release-20.0] Upgrade the Golang version to `go1.22.9` [#17212](https://github.com/vitessio/vitess/pull/17212)
### Enhancement
#### Query Serving
* [release-20.0] Fix to prevent stopping buffering prematurely (#17013) [#17204](https://github.com/vitessio/vitess/pull/17204)
### Internal Cleanup
#### Build/CI
* [release-20.0] Change the name of the vitess-tester repository (#16917) [#17029](https://github.com/vitessio/vitess/pull/17029)
### Testing
#### Build/CI
* [release-20.0] Flakes: Address flakiness in TestZkConnClosedOnDisconnect (#17194) [#17196](https://github.com/vitessio/vitess/pull/17196)

7 changes: 7 additions & 0 deletions changelog/20.0/20.0.4/release_notes.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
# Release of Vitess v20.0.4
The entire changelog for this release can be found [here](https://github.com/vitessio/vitess/blob/main/changelog/20.0/20.0.4/changelog.md).

The release includes 10 merged Pull Requests.

Thanks to all our contributors: @app/vitess-bot, @frouioui

4 changes: 4 additions & 0 deletions changelog/20.0/README.md
Original file line number Diff line number Diff line change
@@ -1,4 +1,8 @@
## v20.0
* **[20.0.4](20.0.4)**
* [Changelog](20.0.4/changelog.md)
* [Release Notes](20.0.4/release_notes.md)

* **[20.0.3](20.0.3)**
* [Changelog](20.0.3/changelog.md)
* [Release Notes](20.0.3/release_notes.md)
Expand Down
7 changes: 7 additions & 0 deletions changelog/21.0/21.0.0/summary.md
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
- [Deprecated VTTablet Flags](#vttablet-flags)
- [Deletion of deprecated metrics](#metric-deletion)
- [Deprecated Metrics](#deprecations-metrics)
- **[RPC Changes](#rpc-changes)**
- **[Traffic Mirroring](#traffic-mirroring)**
- **[Atomic Distributed Transaction Support](#atomic-transaction)**
- **[New VTGate Shutdown Behavior](#new-vtgate-shutdown-behavior)**
Expand Down Expand Up @@ -77,6 +78,12 @@ The following metrics are now deprecated and will be deleted in a future release
| `vttablet` | `QueryCacheHits` | `QueryEnginePlanCacheHits` |
| `vttablet` | `QueryCacheMisses` | `QueryEnginePlanCacheMisses` |

### <a id="rpc-changes"/>RPC Changes</a>

These are the RPC changes made in this release -
1. `ReadReparentJournalInfo` RPC has been added in TabletManagerClient interface, that is going to be used in EmergencyReparentShard for better errant GTID detection.
2. `PrimaryStatus` RPC in TabletManagerClient interface has been updated to also return the server UUID of the primary. This is going to be used in the vttablets so that they can do their own errant GTID detection in `SetReplicationSource`.

### <a id="traffic-mirroring"/>Traffic Mirroring</a>

Traffic mirroring is intended to help reduce some of the uncertainty inherent to `MoveTables SwitchTraffic`. When
Expand Down
57 changes: 57 additions & 0 deletions changelog/21.0/21.0.1/changelog.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
# Changelog of Vitess v21.0.1

### Bug fixes
#### Backup and Restore
* [release-21.0] Fix how we cancel the context in the builtin backup engine (#17285) [#17291](https://github.com/vitessio/vitess/pull/17291)
* [release-21.0] S3: optional endpoint resolver and correct retrier [#17307](https://github.com/vitessio/vitess/pull/17307)
#### Cluster management
* [release-21.0] Fix panic in vttablet when closing topo server twice (#17094) [#17122](https://github.com/vitessio/vitess/pull/17122)
#### Online DDL
* [release-21.0] Online DDL: fix defer function, potential connection pool exhaustion (#17207) [#17210](https://github.com/vitessio/vitess/pull/17210)
#### Query Serving
* [release-21.0] bugfix: treat EXPLAIN like SELECT (#17054) [#17058](https://github.com/vitessio/vitess/pull/17058)
* [release-21.0] Delegate Column Availability Checks to MySQL for Single-Route Queries (#17077) [#17087](https://github.com/vitessio/vitess/pull/17087)
* [release-21.0] bugfix: Handle CTEs with columns named in the CTE def (#17179) [#17181](https://github.com/vitessio/vitess/pull/17181)
* [release-21.0] Use proper keyspace when updating the query graph of a reference DML (#17226) [#17258](https://github.com/vitessio/vitess/pull/17258)
#### Topology
* [release-21.0] Close zookeeper topo connection on disconnect (#17136) [#17193](https://github.com/vitessio/vitess/pull/17193)
#### VReplication
* [release-21.0] VReplication: Qualify and SQL escape tables in created AutoIncrement VSchema definitions (#17174) [#17176](https://github.com/vitessio/vitess/pull/17176)
#### VTTablet
* [release-21.0] Fix deadlock in messager and health streamer (#17230) [#17235](https://github.com/vitessio/vitess/pull/17235)
* [release-21.0] Fix potential deadlock in health streamer (#17261) [#17270](https://github.com/vitessio/vitess/pull/17270)
### CI/Build
#### Build/CI
* [release-21.0] Specify Ubuntu 24.04 for all jobs (#17278) [#17282](https://github.com/vitessio/vitess/pull/17282)
#### Cluster management
* [release-21.0] Fix flakiness in `TestListenerShutdown` (#17024) [#17189](https://github.com/vitessio/vitess/pull/17189)
#### General
* [release-21.0] Upgrade the Golang version to `go1.23.3` [#17211](https://github.com/vitessio/vitess/pull/17211)
### Dependencies
#### Java
* [release-21.0] java package updates for grpc and protobuf and release plugins (#17100) [#17105](https://github.com/vitessio/vitess/pull/17105)
### Documentation
#### Documentation
* [Direct PR][release-21.0] Add RPC changes segment in the summary doc [#17034](https://github.com/vitessio/vitess/pull/17034)
### Enhancement
#### Online DDL
* [release-21.0] Improve Schema Engine's TablesWithSize80 query (#17066) [#17091](https://github.com/vitessio/vitess/pull/17091)
#### Query Serving
* [release-21.0] Fix to prevent stopping buffering prematurely (#17013) [#17205](https://github.com/vitessio/vitess/pull/17205)
#### VReplication
* [release-21.0] Binlog: Improve ZstdInMemoryDecompressorMaxSize management (#17220) [#17241](https://github.com/vitessio/vitess/pull/17241)
### Internal Cleanup
#### Build/CI
* [release-21.0] Change the name of the vitess-tester repository (#16917) [#17030](https://github.com/vitessio/vitess/pull/17030)
### Regression
#### Backup and Restore
* [release-21.0] Fix unreachable errors when taking a backup (#17062) [#17112](https://github.com/vitessio/vitess/pull/17112)
### Release
#### General
* [release-21.0] Bump to `v21.0.1-SNAPSHOT` after the `v21.0.0` release [#17098](https://github.com/vitessio/vitess/pull/17098)
### Testing
#### Build/CI
* [release-21.0] Flakes: Address flakiness in TestZkConnClosedOnDisconnect (#17194) [#17197](https://github.com/vitessio/vitess/pull/17197)
#### Query Serving
* [release-21.0] fix: flaky test on twopc transaction (#17068) [#17070](https://github.com/vitessio/vitess/pull/17070)

7 changes: 7 additions & 0 deletions changelog/21.0/21.0.1/release_notes.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
# Release of Vitess v21.0.1
The entire changelog for this release can be found [here](https://github.com/vitessio/vitess/blob/main/changelog/21.0/21.0.1/changelog.md).

The release includes 25 merged Pull Requests.

Thanks to all our contributors: @GuptaManan100, @app/vitess-bot, @frouioui, @vitess-bot

4 changes: 4 additions & 0 deletions changelog/21.0/README.md
Original file line number Diff line number Diff line change
@@ -1,4 +1,8 @@
## v21.0
* **[21.0.1](21.0.1)**
* [Changelog](21.0.1/changelog.md)
* [Release Notes](21.0.1/release_notes.md)

* **[21.0.0](21.0.0)**
* [Changelog](21.0.0/changelog.md)
* [Release Notes](21.0.0/release_notes.md)
30 changes: 9 additions & 21 deletions examples/operator/101_initial_cluster.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ spec:
vtbackup: vitess/lite:latest
vtorc: vitess/lite:latest
mysqld:
mysql80Compatible: vitess/lite:latest
mysql80Compatible: mysql:8.0.30
mysqldExporter: prom/mysqld-exporter:v0.11.0
cells:
- name: zone1
Expand Down Expand Up @@ -155,23 +155,6 @@ stringData:
# Vitess defaults
###############################################################################
# Vitess-internal database.
CREATE DATABASE IF NOT EXISTS _vt;
# Note that definitions of local_metadata and shard_metadata should be the same
# as in production which is defined in go/vt/mysqlctl/metadata_tables.go.
CREATE TABLE IF NOT EXISTS _vt.local_metadata (
name VARCHAR(255) NOT NULL,
value VARCHAR(255) NOT NULL,
db_name VARBINARY(255) NOT NULL,
PRIMARY KEY (db_name, name)
) ENGINE=InnoDB;
CREATE TABLE IF NOT EXISTS _vt.shard_metadata (
name VARCHAR(255) NOT NULL,
value MEDIUMBLOB NOT NULL,
db_name VARBINARY(255) NOT NULL,
PRIMARY KEY (db_name, name)
) ENGINE=InnoDB;
# Admin user with all privileges.
CREATE USER 'vt_dba'@'localhost';
GRANT ALL ON *.* TO 'vt_dba'@'localhost';
Expand Down Expand Up @@ -200,19 +183,24 @@ stringData:
ON *.* TO 'vt_allprivs'@'localhost';
# User for slave replication connections.
# TODO: Should we set a password on this since it allows remote connections?
CREATE USER 'vt_repl'@'%';
GRANT REPLICATION SLAVE ON *.* TO 'vt_repl'@'%';
# User for Vitess filtered replication (binlog player).
# Same permissions as vt_app.
# User for Vitess VReplication (base vstreamers and vplayer).
CREATE USER 'vt_filtered'@'localhost';
GRANT SELECT, INSERT, UPDATE, DELETE, CREATE, DROP, RELOAD, PROCESS, FILE,
REFERENCES, INDEX, ALTER, SHOW DATABASES, CREATE TEMPORARY TABLES,
LOCK TABLES, EXECUTE, REPLICATION SLAVE, REPLICATION CLIENT, CREATE VIEW,
SHOW VIEW, CREATE ROUTINE, ALTER ROUTINE, CREATE USER, EVENT, TRIGGER
ON *.* TO 'vt_filtered'@'localhost';
# User for general MySQL monitoring.
CREATE USER 'vt_monitoring'@'localhost';
GRANT SELECT, PROCESS, SUPER, REPLICATION CLIENT, RELOAD
ON *.* TO 'vt_monitoring'@'localhost';
GRANT SELECT, UPDATE, DELETE, DROP
ON performance_schema.* TO 'vt_monitoring'@'localhost';
# custom sql is used to add custom scripts like creating users/passwords. We use it in our tests
# {{custom_sql}}
Expand Down
24 changes: 17 additions & 7 deletions go/test/endtoend/transaction/twopc/twopc_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -1395,8 +1395,6 @@ func TestReadTransactionStatus(t *testing.T) {
"insert into twopc_t1(id, col) values(6, 4)",
"insert into twopc_t1(id, col) values(9, 4)",
})
// Allow enough time for the commit to have started.
time.Sleep(1 * time.Second)

// Create a tablet manager client and use it to read the transaction state.
tmc := grpctmclient.NewClient()
Expand All @@ -1405,12 +1403,24 @@ func TestReadTransactionStatus(t *testing.T) {
defer cancel()

primaryTablet := getTablet(clusterInstance.Keyspaces[0].Shards[2].FindPrimaryTablet().GrpcPort)
// Wait for the transaction to show up in the unresolved list.
var unresTransaction *querypb.TransactionMetadata
for _, shard := range clusterInstance.Keyspaces[0].Shards {
urtRes, err := tmc.GetUnresolvedTransactions(ctx, getTablet(shard.FindPrimaryTablet().GrpcPort), 1)
require.NoError(t, err)
if len(urtRes) > 0 {
unresTransaction = urtRes[0]
timeout := time.After(10 * time.Second)
for {
for _, shard := range clusterInstance.Keyspaces[0].Shards {
urtRes, err := tmc.GetUnresolvedTransactions(ctx, getTablet(shard.FindPrimaryTablet().GrpcPort), 1)
require.NoError(t, err)
if len(urtRes) > 0 {
unresTransaction = urtRes[0]
}
}
if unresTransaction != nil {
break
}
select {
case <-timeout:
require.Fail(t, "timed out waiting for unresolved transaction")
default:
}
}
require.NotNil(t, unresTransaction)
Expand Down
Loading

0 comments on commit cb16b34

Please sign in to comment.