[cherry pick] Apply GRPC error processing change from upstream

docker/base/Dockerfile

@@ -25,6 +25,9 @@ FROM vitess/bootstrap:mysql57
 # Allows some docker builds to disable CGO
 ARG CGO_ENABLED=0
 
+# Allows docker builds to set the BUILD_NUMBER
+ARG BUILD_NUMBER
+
 # Re-copy sources from working tree
 USER root
 COPY . /vt/src/vitess.io/vitess

docker/base/Dockerfile.mariadb

@@ -1,5 +1,11 @@
 FROM vitess/bootstrap:mariadb
 
+# Allows some docker builds to disable CGO
+ARG CGO_ENABLED=0
+
+# Allows docker builds to set the BUILD_NUMBER
+ARG BUILD_NUMBER
+
 # Re-copy sources from working tree
 USER root
 COPY . /vt/src/vitess.io/vitess

docker/base/Dockerfile.mariadb103

@@ -1,5 +1,11 @@
 FROM vitess/bootstrap:mariadb103
 
+# Allows some docker builds to disable CGO
+ARG CGO_ENABLED=0
+
+# Allows docker builds to set the BUILD_NUMBER
+ARG BUILD_NUMBER
+
 # Re-copy sources from working tree
 USER root
 COPY . /vt/src/vitess.io/vitess

docker/base/Dockerfile.mysql56

@@ -1,5 +1,11 @@
 FROM vitess/bootstrap:mysql56
 
+# Allows some docker builds to disable CGO
+ARG CGO_ENABLED=0
+
+# Allows docker builds to set the BUILD_NUMBER
+ARG BUILD_NUMBER
+
 # Re-copy sources from working tree
 USER root
 COPY . /vt/src/vitess.io/vitess

docker/base/Dockerfile.mysql80

@@ -1,5 +1,11 @@
 FROM vitess/bootstrap:mysql80
 
+# Allows some docker builds to disable CGO
+ARG CGO_ENABLED=0
+
+# Allows docker builds to set the BUILD_NUMBER
+ARG BUILD_NUMBER
+
 # Re-copy sources from working tree
 USER root
 COPY . /vt/src/vitess.io/vitess

docker/base/Dockerfile.percona

@@ -1,5 +1,11 @@
 FROM vitess/bootstrap:percona
 
+# Allows some docker builds to disable CGO
+ARG CGO_ENABLED=0
+
+# Allows docker builds to set the BUILD_NUMBER
+ARG BUILD_NUMBER
+
 # Re-copy sources from working tree
 USER root
 COPY . /vt/src/vitess.io/vitess

docker/base/Dockerfile.percona57

@@ -1,5 +1,11 @@
 FROM vitess/bootstrap:percona57
 
+# Allows some docker builds to disable CGO
+ARG CGO_ENABLED=0
+
+# Allows docker builds to set the BUILD_NUMBER
+ARG BUILD_NUMBER
+
 # Re-copy sources from working tree
 USER root
 COPY . /vt/src/vitess.io/vitess

docker/base/Dockerfile.percona80

@@ -1,5 +1,11 @@
 FROM vitess/bootstrap:percona80
 
+# Allows some docker builds to disable CGO
+ARG CGO_ENABLED=0
+
+# Allows docker builds to set the BUILD_NUMBER
+ARG BUILD_NUMBER
+
 # Re-copy sources from working tree
 USER root
 COPY . /vt/src/vitess.io/vitess

docker/lite/Dockerfile.alpine

@@ -22,6 +22,9 @@ FROM vitess/bootstrap:mariadb103 AS builder
 # Allows some docker builds to disable CGO
 ARG CGO_ENABLED=0
 
+# Allows docker builds to set the BUILD_NUMBER
+ARG BUILD_NUMBER
+
 # Re-copy sources from working tree.
 COPY --chown=vitess:vitess . /vt/src/vitess.io/vitess
 

docker/lite/Dockerfile.mariadb

@@ -22,6 +22,9 @@ FROM vitess/bootstrap:mariadb AS builder
 # Allows some docker builds to disable CGO
 ARG CGO_ENABLED=0
 
+# Allows docker builds to set the BUILD_NUMBER
+ARG BUILD_NUMBER
+
 # Re-copy sources from working tree.
 COPY --chown=vitess:vitess . /vt/src/vitess.io/vitess
 

docker/lite/Dockerfile.mariadb103

@@ -22,6 +22,9 @@ FROM vitess/bootstrap:mariadb103 AS builder
 # Allows some docker builds to disable CGO
 ARG CGO_ENABLED=0
 
+# Allows docker builds to set the BUILD_NUMBER
+ARG BUILD_NUMBER
+
 # Re-copy sources from working tree.
 COPY --chown=vitess:vitess . /vt/src/vitess.io/vitess
 

docker/lite/Dockerfile.mysql56

@@ -22,6 +22,9 @@ FROM vitess/bootstrap:mysql56 AS builder
 # Allows some docker builds to disable CGO
 ARG CGO_ENABLED=0
 
+# Allows docker builds to set the BUILD_NUMBER
+ARG BUILD_NUMBER
+
 # Re-copy sources from working tree.
 COPY --chown=vitess:vitess . /vt/src/vitess.io/vitess
 

docker/lite/Dockerfile.mysql57

@@ -22,6 +22,9 @@ FROM vitess/bootstrap:mysql57 AS builder
 # Allows some docker builds to disable CGO
 ARG CGO_ENABLED=0
 
+# Allows docker builds to set the BUILD_NUMBER
+ARG BUILD_NUMBER
+
 # Re-copy sources from working tree.
 COPY --chown=vitess:vitess . /vt/src/vitess.io/vitess
 

docker/lite/Dockerfile.mysql80

@@ -22,6 +22,9 @@ FROM vitess/bootstrap:mysql80 AS builder
 # Allows some docker builds to disable CGO
 ARG CGO_ENABLED=0
 
+# Allows docker builds to set the BUILD_NUMBER
+ARG BUILD_NUMBER
+
 # Re-copy sources from working tree.
 COPY --chown=vitess:vitess . /vt/src/vitess.io/vitess
 

docker/lite/Dockerfile.percona

@@ -22,6 +22,9 @@ FROM vitess/bootstrap:percona AS builder
 # Allows some docker builds to disable CGO
 ARG CGO_ENABLED=0
 
+# Allows docker builds to set the BUILD_NUMBER
+ARG BUILD_NUMBER
+
 # Re-copy sources from working tree.
 COPY --chown=vitess:vitess . /vt/src/vitess.io/vitess
 

docker/lite/Dockerfile.percona57

@@ -22,6 +22,9 @@ FROM vitess/bootstrap:percona57 AS builder
 # Allows some docker builds to disable CGO
 ARG CGO_ENABLED=0
 
+# Allows docker builds to set the BUILD_NUMBER
+ARG BUILD_NUMBER
+
 # Re-copy sources from working tree.
 COPY --chown=vitess:vitess . /vt/src/vitess.io/vitess
 

docker/lite/Dockerfile.percona80

@@ -22,6 +22,9 @@ FROM vitess/bootstrap:percona80 AS builder
 # Allows some docker builds to disable CGO
 ARG CGO_ENABLED=0
 
+# Allows docker builds to set the BUILD_NUMBER
+ARG BUILD_NUMBER
+
 # Re-copy sources from working tree.
 COPY --chown=vitess:vitess . /vt/src/vitess.io/vitess
 

examples/compose/vtcompose/vtcompose.go

@@ -434,6 +434,7 @@ func applyKeyspaceDependentPatches(
 	if keyspaceData.shards < 2 {
 		tabAlias = tabAlias + 100
 		dockerComposeFile = applyTabletPatches(dockerComposeFile, tabAlias, shard, keyspaceData, externalDbInfoMap, opts)
+		dockerComposeFile = applyShardPatches(dockerComposeFile, tabAlias, shard, keyspaceData, opts)
 	} else {
 		// Determine shard range
 		for i := 0; i < keyspaceData.shards; i++ {
@@ -446,6 +447,7 @@ func applyKeyspaceDependentPatches(
 			}
 			tabAlias = tabAlias + 100
 			dockerComposeFile = applyTabletPatches(dockerComposeFile, tabAlias, shard, keyspaceData, externalDbInfoMap, opts)
+			dockerComposeFile = applyShardPatches(dockerComposeFile, tabAlias, shard, keyspaceData, opts)
 		}
 	}
 
@@ -475,6 +477,39 @@ func applyDockerComposePatches(
 	return dockerComposeFile
 }
 
+func applyShardPatches(
+	dockerComposeFile []byte,
+	tabAlias int,
+	shard string,
+	keyspaceData keyspaceInfo,
+	opts vtOptions,
+) []byte {
+	dockerComposeFile = applyInMemoryPatch(dockerComposeFile, generateDefaultShard(tabAlias, shard, keyspaceData, opts))
+	return dockerComposeFile
+}
+
+func generateDefaultShard(tabAlias int, shard string, keyspaceData keyspaceInfo, opts vtOptions) string {
+	aliases := []int{tabAlias + 1} // master alias, e.g. 201
+	for i := 0; i < keyspaceData.replicaTablets; i++ {
+		aliases = append(aliases, tabAlias+2+i) // replica aliases, e.g. 202, 203, ...
+	}
+	tabletDepends := make([]string, len(aliases))
+	for i, tabletId := range aliases {
+		tabletDepends[i] = fmt.Sprintf("vttablet%d: {condition : service_healthy}", tabletId)
+	}
+	// Wait on all shard tablets to be healthy
+	dependsOn := "depends_on: {" + strings.Join(tabletDepends, ", ") + "}"
+
+	return fmt.Sprintf(`
+- op: add
+  path: /services/init_shard_master%[2]d
+  value:
+    image: vitess/base
+    command: ["sh", "-c", "$$VTROOT/bin/vtctl %[5]s InitShardMaster -force %[4]s/%[3]s %[6]s-%[2]d "]
+    %[1]s
+`, dependsOn, aliases[0], shard, keyspaceData.keyspace, opts.topologyFlags, opts.cell)
+}
+
 func applyTabletPatches(
 	dockerComposeFile []byte,
 	tabAlias int,
@@ -674,7 +709,7 @@ func generatePrimaryVIndex(tableName, column, name string) string {
 	return fmt.Sprintf(`
 [{"op": "add",
 "path": "/tables/%[1]s",
-"value": 
+"value":
   {"column_vindexes": [
     {
       "column": "%[2]s",

examples/compose/vttablet-up.sh

@@ -1,13 +1,13 @@
 #!/bin/bash
 
 # Copyright 2019 The Vitess Authors.
-# 
+#
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
-# 
+#
 #     http://www.apache.org/licenses/LICENSE-2.0
-# 
+#
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -120,8 +120,6 @@ $VTROOT/bin/vtctl $TOPOLOGY_FLAGS AddCellInfo -root vitess/$CELL -server_address
 $VTROOT/bin/vtctl $TOPOLOGY_FLAGS CreateKeyspace $keyspace || true
 $VTROOT/bin/vtctl $TOPOLOGY_FLAGS CreateShard $keyspace/$shard || true
 
-$VTROOT/bin/vtctl $TOPOLOGY_FLAGS InitTablet -shard $shard -keyspace $keyspace -grpc_port $grpc_port -port $web_port -allow_master_override $alias $tablet_role
-
 #Populate external db conditional args
 if [ "$external" = "1" ]; then
     if [ $role = "master" ]; then

go/cmd/vtbackup/vtbackup.go

@@ -306,8 +306,6 @@ func takeBackup(ctx context.Context, topoServer *topo.Server, backupStorage back
 			return fmt.Errorf("no backup found; not starting up empty since -initial_backup flag was not enabled")
 		}
 		restorePos = mysql.Position{}
-	case mysqlctl.ErrExistingDB:
-		return fmt.Errorf("can't run vtbackup because data directory is not empty")
 	default:
 		return fmt.Errorf("can't restore from backup: %v", err)
 	}

go/mysql/sql_error.go

@@ -120,7 +120,7 @@ func NewSQLErrorFromError(err error) error {
 		case vtrpcpb.Code_UNAUTHENTICATED:
 			num = ERAccessDeniedError
 		case vtrpcpb.Code_RESOURCE_EXHAUSTED:
-			num = ERTooManyUserConnections
+			num = demuxResourceExhaustedErrors(err.Error())
 		case vtrpcpb.Code_FAILED_PRECONDITION:
 			num = ERUnknownError
 		case vtrpcpb.Code_ABORTED:
@@ -161,3 +161,14 @@ func NewSQLErrorFromError(err error) error {
 	}
 	return serr
 }
+
+var isGRPCOverflowRE = regexp.MustCompile(`.*grpc: received message larger than max \(\d+ vs. \d+\)`)
+
+func demuxResourceExhaustedErrors(msg string) int {
+	switch {
+	case isGRPCOverflowRE.Match([]byte(msg)):
+		return ERNetPacketTooLarge
+	default:
+		return ERTooManyUserConnections
+	}
+}

go/mysql/sql_error_test.go

@@ -0,0 +1,45 @@
+/*
+Copyright 2020 The Vitess Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package mysql
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestDumuxResourceExhaustedErrors(t *testing.T) {
+	type testCase struct {
+		msg  string
+		want int
+	}
+
+	cases := []testCase{
+		testCase{"misc", ERTooManyUserConnections},
+		testCase{"grpc: received message larger than max (99282 vs. 1234): trailer", ERNetPacketTooLarge},
+		testCase{"grpc: received message larger than max (1234 vs. 1234)", ERNetPacketTooLarge},
+		testCase{"header: grpc: received message larger than max (1234 vs. 1234)", ERNetPacketTooLarge},
+		// This should be explicitly handled by returning ERNetPacketTooLarge from the execturo directly
+		// and therefore shouldn't need to be teased out of another error.
+		testCase{"in-memory row count exceeded allowed limit of 13", ERTooManyUserConnections},
+	}
+
+	for _, c := range cases {
+		got := demuxResourceExhaustedErrors(c.msg)
+		assert.Equalf(t, c.want, got, c.msg)
+	}
+}

go/pools/numbered.go

@@ -131,13 +131,15 @@ func (nu *Numbered) Get(id int64, purpose string) (val interface{}, err error) {
 }
 
 // Put unlocks a resource for someone else to use.
-func (nu *Numbered) Put(id int64) {
+func (nu *Numbered) Put(id int64, updateTime bool) {
 	nu.mu.Lock()
 	defer nu.mu.Unlock()
 	if nw, ok := nu.resources[id]; ok {
 		nw.inUse = false
 		nw.purpose = ""
-		nw.timeUsed = time.Now()
+		if updateTime {
+			nw.timeUsed = time.Now()
+		}
 	}
 }
 
@@ -162,7 +164,7 @@ func (nu *Numbered) GetOutdated(age time.Duration, purpose string) (vals []inter
 		if nw.inUse || !nw.enforceTimeout {
 			continue
 		}
-		if nw.timeCreated.Add(age).Sub(now) <= 0 {
+		if nw.timeUsed.Add(age).Sub(now) <= 0 {
 			nw.inUse = true
 			nw.purpose = purpose
 			vals = append(vals, nw.val)