diff --git a/examples/compose/docker-compose.beginners.yml b/examples/compose/docker-compose.beginners.yml index b0a160c7b4a..b296011d418 100644 --- a/examples/compose/docker-compose.beginners.yml +++ b/examples/compose/docker-compose.beginners.yml @@ -58,7 +58,7 @@ services: - "3306" vtctld: - image: vitess/lite:v18.0.5-shopify-6 + image: vitess/lite:v18.0.5-shopify-7 ports: - "15000:$WEB_PORT" - "$GRPC_PORT" @@ -81,7 +81,7 @@ services: condition: service_healthy vtgate: - image: vitess/lite:v18.0.5-shopify-6 + image: vitess/lite:v18.0.5-shopify-7 ports: - "15099:$WEB_PORT" - "$GRPC_PORT" @@ -111,7 +111,7 @@ services: condition: service_healthy schemaload: - image: vitess/lite:v18.0.5-shopify-6 + image: vitess/lite:v18.0.5-shopify-7 command: - sh - -c @@ -144,12 +144,12 @@ services: environment: - KEYSPACES=$KEYSPACE - GRPC_PORT=15999 - image: vitess/lite:v18.0.5-shopify-6 + image: vitess/lite:v18.0.5-shopify-7 volumes: - .:/script vttablet100: - image: vitess/lite:v18.0.5-shopify-6 + image: vitess/lite:v18.0.5-shopify-7 ports: - "15100:$WEB_PORT" - "$GRPC_PORT" @@ -181,7 +181,7 @@ services: retries: 15 vttablet101: - image: vitess/lite:v18.0.5-shopify-6 + image: vitess/lite:v18.0.5-shopify-7 ports: - "15101:$WEB_PORT" - "$GRPC_PORT" @@ -213,7 +213,7 @@ services: retries: 15 vttablet102: - image: vitess/lite:v18.0.5-shopify-6 + image: vitess/lite:v18.0.5-shopify-7 ports: - "15102:$WEB_PORT" - "$GRPC_PORT" @@ -245,7 +245,7 @@ services: retries: 15 vttablet103: - image: vitess/lite:v18.0.5-shopify-6 + image: vitess/lite:v18.0.5-shopify-7 ports: - "15103:$WEB_PORT" - "$GRPC_PORT" @@ -277,7 +277,7 @@ services: retries: 15 vtorc: - image: vitess/lite:v18.0.5-shopify-6 + image: vitess/lite:v18.0.5-shopify-7 command: ["sh", "-c", "/script/vtorc-up.sh"] depends_on: - vtctld @@ -307,7 +307,7 @@ services: retries: 15 vreplication: - image: vitess/lite:v18.0.5-shopify-6 + image: vitess/lite:v18.0.5-shopify-7 volumes: - ".:/script" environment: diff --git a/examples/compose/docker-compose.yml b/examples/compose/docker-compose.yml index f83cecf25cc..0ba95bd7370 100644 --- a/examples/compose/docker-compose.yml +++ b/examples/compose/docker-compose.yml @@ -75,7 +75,7 @@ services: - SCHEMA_FILES=lookup_keyspace_schema_file.sql - POST_LOAD_FILE= - EXTERNAL_DB=0 - image: vitess/lite:v18.0.5-shopify-6 + image: vitess/lite:v18.0.5-shopify-7 volumes: - .:/script schemaload_test_keyspace: @@ -101,7 +101,7 @@ services: - SCHEMA_FILES=test_keyspace_schema_file.sql - POST_LOAD_FILE= - EXTERNAL_DB=0 - image: vitess/lite:v18.0.5-shopify-6 + image: vitess/lite:v18.0.5-shopify-7 volumes: - .:/script set_keyspace_durability_policy: @@ -115,7 +115,7 @@ services: environment: - KEYSPACES=test_keyspace lookup_keyspace - GRPC_PORT=15999 - image: vitess/lite:v18.0.5-shopify-6 + image: vitess/lite:v18.0.5-shopify-7 volumes: - .:/script vreplication: @@ -129,7 +129,7 @@ services: - TOPOLOGY_FLAGS=--topo_implementation consul --topo_global_server_address consul1:8500 --topo_global_root vitess/global - EXTERNAL_DB=0 - image: vitess/lite:v18.0.5-shopify-6 + image: vitess/lite:v18.0.5-shopify-7 volumes: - .:/script vtctld: @@ -143,7 +143,7 @@ services: depends_on: external_db_host: condition: service_healthy - image: vitess/lite:v18.0.5-shopify-6 + image: vitess/lite:v18.0.5-shopify-7 ports: - 15000:8080 - "15999" @@ -160,7 +160,7 @@ services: --normalize_queries=true ' depends_on: - vtctld - image: vitess/lite:v18.0.5-shopify-6 + image: vitess/lite:v18.0.5-shopify-7 ports: - 15099:8080 - "15999" @@ -182,7 +182,7 @@ services: - EXTERNAL_DB=0 - DB_USER= - DB_PASS= - image: vitess/lite:v18.0.5-shopify-6 + image: vitess/lite:v18.0.5-shopify-7 ports: - 13000:8080 volumes: @@ -217,7 +217,7 @@ services: - CMD-SHELL - curl -s --fail --show-error localhost:8080/debug/health timeout: 10s - image: vitess/lite:v18.0.5-shopify-6 + image: vitess/lite:v18.0.5-shopify-7 ports: - 15101:8080 - "15999" @@ -254,7 +254,7 @@ services: - CMD-SHELL - curl -s --fail --show-error localhost:8080/debug/health timeout: 10s - image: vitess/lite:v18.0.5-shopify-6 + image: vitess/lite:v18.0.5-shopify-7 ports: - 15102:8080 - "15999" @@ -291,7 +291,7 @@ services: - CMD-SHELL - curl -s --fail --show-error localhost:8080/debug/health timeout: 10s - image: vitess/lite:v18.0.5-shopify-6 + image: vitess/lite:v18.0.5-shopify-7 ports: - 15201:8080 - "15999" @@ -328,7 +328,7 @@ services: - CMD-SHELL - curl -s --fail --show-error localhost:8080/debug/health timeout: 10s - image: vitess/lite:v18.0.5-shopify-6 + image: vitess/lite:v18.0.5-shopify-7 ports: - 15202:8080 - "15999" @@ -365,7 +365,7 @@ services: - CMD-SHELL - curl -s --fail --show-error localhost:8080/debug/health timeout: 10s - image: vitess/lite:v18.0.5-shopify-6 + image: vitess/lite:v18.0.5-shopify-7 ports: - 15301:8080 - "15999" @@ -402,7 +402,7 @@ services: - CMD-SHELL - curl -s --fail --show-error localhost:8080/debug/health timeout: 10s - image: vitess/lite:v18.0.5-shopify-6 + image: vitess/lite:v18.0.5-shopify-7 ports: - 15302:8080 - "15999" diff --git a/examples/compose/vtcompose/docker-compose.test.yml b/examples/compose/vtcompose/docker-compose.test.yml index c12db16e119..7cf5b453d08 100644 --- a/examples/compose/vtcompose/docker-compose.test.yml +++ b/examples/compose/vtcompose/docker-compose.test.yml @@ -79,7 +79,7 @@ services: - SCHEMA_FILES=test_keyspace_schema_file.sql - POST_LOAD_FILE= - EXTERNAL_DB=0 - image: vitess/lite:v18.0.5-shopify-6 + image: vitess/lite:v18.0.5-shopify-7 volumes: - .:/script schemaload_unsharded_keyspace: @@ -103,7 +103,7 @@ services: - SCHEMA_FILES=unsharded_keyspace_schema_file.sql - POST_LOAD_FILE= - EXTERNAL_DB=0 - image: vitess/lite:v18.0.5-shopify-6 + image: vitess/lite:v18.0.5-shopify-7 volumes: - .:/script set_keyspace_durability_policy_test_keyspace: @@ -117,7 +117,7 @@ services: environment: - GRPC_PORT=15999 - KEYSPACES=test_keyspace - image: vitess/lite:v18.0.5-shopify-6 + image: vitess/lite:v18.0.5-shopify-7 volumes: - .:/script set_keyspace_durability_policy_unsharded_keyspace: @@ -130,7 +130,7 @@ services: environment: - GRPC_PORT=15999 - KEYSPACES=unsharded_keyspace - image: vitess/lite:v18.0.5-shopify-6 + image: vitess/lite:v18.0.5-shopify-7 volumes: - .:/script vreplication: @@ -144,7 +144,7 @@ services: - TOPOLOGY_FLAGS=--topo_implementation consul --topo_global_server_address consul1:8500 --topo_global_root vitess/global - EXTERNAL_DB=0 - image: vitess/lite:v18.0.5-shopify-6 + image: vitess/lite:v18.0.5-shopify-7 volumes: - .:/script vtctld: @@ -159,7 +159,7 @@ services: depends_on: external_db_host: condition: service_healthy - image: vitess/lite:v18.0.5-shopify-6 + image: vitess/lite:v18.0.5-shopify-7 ports: - 15000:8080 - "15999" @@ -176,7 +176,7 @@ services: ''grpc-vtgateservice'' --normalize_queries=true ' depends_on: - vtctld - image: vitess/lite:v18.0.5-shopify-6 + image: vitess/lite:v18.0.5-shopify-7 ports: - 15099:8080 - "15999" @@ -199,7 +199,7 @@ services: - EXTERNAL_DB=0 - DB_USER= - DB_PASS= - image: vitess/lite:v18.0.5-shopify-6 + image: vitess/lite:v18.0.5-shopify-7 ports: - 13000:8080 volumes: @@ -234,7 +234,7 @@ services: - CMD-SHELL - curl -s --fail --show-error localhost:8080/debug/health timeout: 10s - image: vitess/lite:v18.0.5-shopify-6 + image: vitess/lite:v18.0.5-shopify-7 ports: - 15101:8080 - "15999" @@ -271,7 +271,7 @@ services: - CMD-SHELL - curl -s --fail --show-error localhost:8080/debug/health timeout: 10s - image: vitess/lite:v18.0.5-shopify-6 + image: vitess/lite:v18.0.5-shopify-7 ports: - 15102:8080 - "15999" @@ -308,7 +308,7 @@ services: - CMD-SHELL - curl -s --fail --show-error localhost:8080/debug/health timeout: 10s - image: vitess/lite:v18.0.5-shopify-6 + image: vitess/lite:v18.0.5-shopify-7 ports: - 15201:8080 - "15999" @@ -345,7 +345,7 @@ services: - CMD-SHELL - curl -s --fail --show-error localhost:8080/debug/health timeout: 10s - image: vitess/lite:v18.0.5-shopify-6 + image: vitess/lite:v18.0.5-shopify-7 ports: - 15202:8080 - "15999" @@ -382,7 +382,7 @@ services: - CMD-SHELL - curl -s --fail --show-error localhost:8080/debug/health timeout: 10s - image: vitess/lite:v18.0.5-shopify-6 + image: vitess/lite:v18.0.5-shopify-7 ports: - 15301:8080 - "15999" diff --git a/examples/compose/vtcompose/vtcompose.go b/examples/compose/vtcompose/vtcompose.go index 40d677fec2e..865aecc2853 100644 --- a/examples/compose/vtcompose/vtcompose.go +++ b/examples/compose/vtcompose/vtcompose.go @@ -533,7 +533,7 @@ func generateDefaultShard(tabAlias int, shard string, keyspaceData keyspaceInfo, - op: add path: /services/init_shard_primary%[2]d value: - image: vitess/lite:v18.0.5-shopify-6 + image: vitess/lite:v18.0.5-shopify-7 command: ["sh", "-c", "/vt/bin/vtctldclient %[5]s InitShardPrimary --force %[4]s/%[3]s %[6]s-%[2]d "] %[1]s `, dependsOn, aliases[0], shard, keyspaceData.keyspace, opts.topologyFlags, opts.cell) @@ -565,7 +565,7 @@ func generateExternalPrimary( - op: add path: /services/vttablet%[1]d value: - image: vitess/lite:v18.0.5-shopify-6 + image: vitess/lite:v18.0.5-shopify-7 ports: - "15%[1]d:%[3]d" - "%[4]d" @@ -627,7 +627,7 @@ func generateDefaultTablet(tabAlias int, shard, role, keyspace string, dbInfo ex - op: add path: /services/vttablet%[1]d value: - image: vitess/lite:v18.0.5-shopify-6 + image: vitess/lite:v18.0.5-shopify-7 ports: - "15%[1]d:%[4]d" - "%[5]d" @@ -665,7 +665,7 @@ func generateVtctld(opts vtOptions) string { - op: add path: /services/vtctld value: - image: vitess/lite:v18.0.5-shopify-6 + image: vitess/lite:v18.0.5-shopify-7 ports: - "15000:%[1]d" - "%[2]d" @@ -696,7 +696,7 @@ func generateVtgate(opts vtOptions) string { - op: add path: /services/vtgate value: - image: vitess/lite:v18.0.5-shopify-6 + image: vitess/lite:v18.0.5-shopify-7 ports: - "15099:%[1]d" - "%[2]d" @@ -738,7 +738,7 @@ func generateVTOrc(dbInfo externalDbInfo, keyspaceInfoMap map[string]keyspaceInf - op: add path: /services/vtorc value: - image: vitess/lite:v18.0.5-shopify-6 + image: vitess/lite:v18.0.5-shopify-7 volumes: - ".:/script" environment: @@ -763,7 +763,7 @@ func generateVreplication(dbInfo externalDbInfo, opts vtOptions) string { - op: add path: /services/vreplication value: - image: vitess/lite:v18.0.5-shopify-6 + image: vitess/lite:v18.0.5-shopify-7 volumes: - ".:/script" environment: @@ -791,7 +791,7 @@ func generateSetKeyspaceDurabilityPolicy( - op: add path: /services/set_keyspace_durability_policy_%[3]s value: - image: vitess/lite:v18.0.5-shopify-6 + image: vitess/lite:v18.0.5-shopify-7 volumes: - ".:/script" environment: @@ -828,7 +828,7 @@ func generateSchemaload( - op: add path: /services/schemaload_%[7]s value: - image: vitess/lite:v18.0.5-shopify-6 + image: vitess/lite:v18.0.5-shopify-7 volumes: - ".:/script" environment: diff --git a/examples/operator/101_initial_cluster.yaml b/examples/operator/101_initial_cluster.yaml index 314ed75855c..e9c6fc637cf 100644 --- a/examples/operator/101_initial_cluster.yaml +++ b/examples/operator/101_initial_cluster.yaml @@ -8,14 +8,14 @@ metadata: name: example spec: images: - vtctld: vitess/lite:v18.0.5-shopify-6 - vtadmin: vitess/vtadmin:v18.0.5-shopify-6 - vtgate: vitess/lite:v18.0.5-shopify-6 - vttablet: vitess/lite:v18.0.5-shopify-6 - vtbackup: vitess/lite:v18.0.5-shopify-6 - vtorc: vitess/lite:v18.0.5-shopify-6 + vtctld: vitess/lite:v18.0.5-shopify-7 + vtadmin: vitess/vtadmin:v18.0.5-shopify-7 + vtgate: vitess/lite:v18.0.5-shopify-7 + vttablet: vitess/lite:v18.0.5-shopify-7 + vtbackup: vitess/lite:v18.0.5-shopify-7 + vtorc: vitess/lite:v18.0.5-shopify-7 mysqld: - mysql80Compatible: vitess/lite:v18.0.5-shopify-6 + mysql80Compatible: vitess/lite:v18.0.5-shopify-7 mysqldExporter: prom/mysqld-exporter:v0.11.0 cells: - name: zone1 diff --git a/examples/operator/201_customer_tablets.yaml b/examples/operator/201_customer_tablets.yaml index a7eadb24be7..70811eb8a71 100644 --- a/examples/operator/201_customer_tablets.yaml +++ b/examples/operator/201_customer_tablets.yaml @@ -4,14 +4,14 @@ metadata: name: example spec: images: - vtctld: vitess/lite:v18.0.5-shopify-6 - vtadmin: vitess/vtadmin:v18.0.5-shopify-6 - vtgate: vitess/lite:v18.0.5-shopify-6 - vttablet: vitess/lite:v18.0.5-shopify-6 - vtbackup: vitess/lite:v18.0.5-shopify-6 - vtorc: vitess/lite:v18.0.5-shopify-6 + vtctld: vitess/lite:v18.0.5-shopify-7 + vtadmin: vitess/vtadmin:v18.0.5-shopify-7 + vtgate: vitess/lite:v18.0.5-shopify-7 + vttablet: vitess/lite:v18.0.5-shopify-7 + vtbackup: vitess/lite:v18.0.5-shopify-7 + vtorc: vitess/lite:v18.0.5-shopify-7 mysqld: - mysql80Compatible: vitess/lite:v18.0.5-shopify-6 + mysql80Compatible: vitess/lite:v18.0.5-shopify-7 mysqldExporter: prom/mysqld-exporter:v0.11.0 cells: - name: zone1 diff --git a/examples/operator/302_new_shards.yaml b/examples/operator/302_new_shards.yaml index 8dd9aeeb6c3..daed56a6494 100644 --- a/examples/operator/302_new_shards.yaml +++ b/examples/operator/302_new_shards.yaml @@ -4,14 +4,14 @@ metadata: name: example spec: images: - vtctld: vitess/lite:v18.0.5-shopify-6 - vtadmin: vitess/vtadmin:v18.0.5-shopify-6 - vtgate: vitess/lite:v18.0.5-shopify-6 - vttablet: vitess/lite:v18.0.5-shopify-6 - vtbackup: vitess/lite:v18.0.5-shopify-6 - vtorc: vitess/lite:v18.0.5-shopify-6 + vtctld: vitess/lite:v18.0.5-shopify-7 + vtadmin: vitess/vtadmin:v18.0.5-shopify-7 + vtgate: vitess/lite:v18.0.5-shopify-7 + vttablet: vitess/lite:v18.0.5-shopify-7 + vtbackup: vitess/lite:v18.0.5-shopify-7 + vtorc: vitess/lite:v18.0.5-shopify-7 mysqld: - mysql80Compatible: vitess/lite:v18.0.5-shopify-6 + mysql80Compatible: vitess/lite:v18.0.5-shopify-7 mysqldExporter: prom/mysqld-exporter:v0.11.0 cells: - name: zone1 diff --git a/examples/operator/306_down_shard_0.yaml b/examples/operator/306_down_shard_0.yaml index 0ace66f8def..a609af0e497 100644 --- a/examples/operator/306_down_shard_0.yaml +++ b/examples/operator/306_down_shard_0.yaml @@ -4,14 +4,14 @@ metadata: name: example spec: images: - vtctld: vitess/lite:v18.0.5-shopify-6 - vtadmin: vitess/vtadmin:v18.0.5-shopify-6 - vtgate: vitess/lite:v18.0.5-shopify-6 - vttablet: vitess/lite:v18.0.5-shopify-6 - vtbackup: vitess/lite:v18.0.5-shopify-6 - vtorc: vitess/lite:v18.0.5-shopify-6 + vtctld: vitess/lite:v18.0.5-shopify-7 + vtadmin: vitess/vtadmin:v18.0.5-shopify-7 + vtgate: vitess/lite:v18.0.5-shopify-7 + vttablet: vitess/lite:v18.0.5-shopify-7 + vtbackup: vitess/lite:v18.0.5-shopify-7 + vtorc: vitess/lite:v18.0.5-shopify-7 mysqld: - mysql80Compatible: vitess/lite:v18.0.5-shopify-6 + mysql80Compatible: vitess/lite:v18.0.5-shopify-7 mysqldExporter: prom/mysqld-exporter:v0.11.0 cells: - name: zone1 diff --git a/go/vt/discovery/topology_watcher.go b/go/vt/discovery/topology_watcher.go index d1bd2d3acf8..105741aaac1 100644 --- a/go/vt/discovery/topology_watcher.go +++ b/go/vt/discovery/topology_watcher.go @@ -190,6 +190,17 @@ func (tw *TopologyWatcher) loadTablets() { topologyWatcherOperations.Add(topologyWatcherOpGetTablet, 1) <-tw.sem // Done; enable next request to run if err != nil { + if !topo.IsErrType(err, topo.NoNode) { + // We failed to get the tablet, but it may still exist. + // We don't want this tablet to be removed from the tw.tablets map or the healthcheck, + // so we fill the gap in the newTablets map using the existing tablet. + tw.mu.Lock() + aliasStr := topoproto.TabletAliasString(alias) + if val, ok := tw.tablets[aliasStr]; ok { + newTablets[aliasStr] = val + } + tw.mu.Unlock() + } topologyWatcherErrors.Add(topologyWatcherOpGetTablet, 1) select { case <-tw.ctx.Done(): diff --git a/go/vt/discovery/topology_watcher_test.go b/go/vt/discovery/topology_watcher_test.go index 3ac567acef8..254943b07bd 100644 --- a/go/vt/discovery/topology_watcher_test.go +++ b/go/vt/discovery/topology_watcher_test.go @@ -18,6 +18,7 @@ package discovery import ( "context" + "errors" "math/rand" "testing" "time" @@ -630,3 +631,145 @@ func TestFilterByKeypsaceSkipsIgnoredTablets(t *testing.T) { tw.Stop() } + +func TestGetTabletErrorDoesNotRemoveFromHealthcheck(t *testing.T) { + ctx := utils.LeakCheckContext(t) + + ts, factory := memorytopo.NewServerAndFactory(ctx, "aa") + defer ts.Close() + fhc := NewFakeHealthCheck(nil) + defer fhc.Close() + topologyWatcherOperations.ZeroAll() + counts := topologyWatcherOperations.Counts() + tw := NewCellTabletsWatcher(context.Background(), ts, fhc, nil, "aa", 10*time.Minute, true, 5) + defer tw.Stop() + + counts = checkOpCounts(t, counts, map[string]int64{}) + checkChecksum(t, tw, 0) + + // Add a tablet to the topology. + tablet1 := &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "aa", + Uid: 0, + }, + Hostname: "host1", + PortMap: map[string]int32{ + "vt": 123, + }, + Keyspace: "keyspace", + Shard: "shard", + } + require.NoError(t, ts.CreateTablet(ctx, tablet1), "CreateTablet failed for %v", tablet1.Alias) + + tw.loadTablets() + counts = checkOpCounts(t, counts, map[string]int64{"ListTablets": 1, "GetTablet": 1, "AddTablet": 1}) + checkChecksum(t, tw, 3238442862) + + // Check the tablet is returned by GetAllTablets(). + allTablets := fhc.GetAllTablets() + key1 := TabletToMapKey(tablet1) + assert.Len(t, allTablets, 1) + assert.Contains(t, allTablets, key1) + assert.True(t, proto.Equal(tablet1, allTablets[key1])) + + // Add a second tablet to the topology. + tablet2 := &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "aa", + Uid: 2, + }, + Hostname: "host2", + PortMap: map[string]int32{ + "vt": 789, + }, + Keyspace: "keyspace", + Shard: "shard", + } + require.NoError(t, ts.CreateTablet(ctx, tablet2), "CreateTablet failed for %v", tablet2.Alias) + + // Cause the Get for the first tablet to fail. + factory.AddOperationError(memorytopo.Get, "tablets/aa-0000000000/Tablet", errors.New("fake error")) + + // Ensure that a topo GetTablet error fails. If not, the rest of this test is invalid. + _, err := ts.GetTablet(ctx, tablet1.Alias) + require.ErrorContains(t, err, "fake error") + + // Now force the error during loadTablets. + tw.loadTablets() + checkOpCounts(t, counts, map[string]int64{"ListTablets": 1, "GetTablet": 2, "AddTablet": 1}) + checkChecksum(t, tw, 2762153755) + + // Ensure the first tablet is still returned by GetAllTablets() and the second tablet has been added. + allTablets = fhc.GetAllTablets() + key2 := TabletToMapKey(tablet2) + assert.Len(t, allTablets, 2) + assert.Contains(t, allTablets, key1) + assert.Contains(t, allTablets, key2) + assert.True(t, proto.Equal(tablet1, allTablets[key1])) + assert.True(t, proto.Equal(tablet2, allTablets[key2])) +} + +func TestGetTabletNoNodeErrorRemovesFromHealthcheck(t *testing.T) { + ctx := utils.LeakCheckContext(t) + + ts, factory := memorytopo.NewServerAndFactory(ctx, "aa") + defer ts.Close() + fhc := NewFakeHealthCheck(nil) + defer fhc.Close() + topologyWatcherOperations.ZeroAll() + counts := topologyWatcherOperations.Counts() + tw := NewCellTabletsWatcher(context.Background(), ts, fhc, nil, "aa", 10*time.Minute, true, 5) + defer tw.Stop() + + counts = checkOpCounts(t, counts, map[string]int64{}) + checkChecksum(t, tw, 0) + + // Add a tablet to the topology. + tablet1 := &topodatapb.Tablet{ + Alias: &topodatapb.TabletAlias{ + Cell: "aa", + Uid: 0, + }, + Hostname: "host1", + PortMap: map[string]int32{ + "vt": 123, + }, + Keyspace: "keyspace", + Shard: "shard", + } + require.NoError(t, ts.CreateTablet(ctx, tablet1), "CreateTablet failed for %v", tablet1.Alias) + + tw.loadTablets() + counts = checkOpCounts(t, counts, map[string]int64{"ListTablets": 1, "GetTablet": 1, "AddTablet": 1}) + checkChecksum(t, tw, 3238442862) + + // Check the tablet is returned by GetAllTablets(). + allTablets := fhc.GetAllTablets() + key1 := TabletToMapKey(tablet1) + assert.Len(t, allTablets, 1) + assert.Contains(t, allTablets, key1) + assert.True(t, proto.Equal(tablet1, allTablets[key1])) + + // Cause the Get for the tablet to fail with a NoNode error. This simulates a race condition where + // the tablet is removed from the topo after the ListTablets call but before the GetTablet call. + factory.AddOperationError( + memorytopo.Get, + "tablets/aa-0000000000/Tablet", + topo.NewError(topo.NoNode, "tablets/aa-0000000000/Tablet"), + ) + + // Ensure that a topo GetTablet error fails. If not, the rest of this test is invalid. + _, err := ts.GetTablet(ctx, tablet1.Alias) + require.Error(t, err) + require.True(t, topo.IsErrType(err, topo.NoNode)) + + // Now force the error during loadTablets. + tw.loadTablets() + checkOpCounts(t, counts, map[string]int64{"ListTablets": 1, "GetTablet": 1, "RemoveTablet": 1}) + checkChecksum(t, tw, 0) + + // Ensure the tablet is no longer returned by GetAllTablets() + allTablets = fhc.GetAllTablets() + assert.Len(t, allTablets, 0) +} diff --git a/go/vt/servenv/version.go b/go/vt/servenv/version.go index 9855ef7ee6c..c7d1bf49ee6 100644 --- a/go/vt/servenv/version.go +++ b/go/vt/servenv/version.go @@ -19,4 +19,4 @@ package servenv // THIS FILE IS AUTO-GENERATED DURING NEW RELEASES BY ./tools/do_releases.sh // DO NOT EDIT -const versionName = "18.0.5-shopify-6" +const versionName = "18.0.5-shopify-7" diff --git a/go/vt/topo/memorytopo/directory.go b/go/vt/topo/memorytopo/directory.go index f68c87a2166..a3e28999fb1 100644 --- a/go/vt/topo/memorytopo/directory.go +++ b/go/vt/topo/memorytopo/directory.go @@ -37,6 +37,9 @@ func (c *Conn) ListDir(ctx context.Context, dirPath string, full bool) ([]topo.D if c.factory.err != nil { return nil, c.factory.err } + if err := c.factory.getOperationError(ListDir, dirPath); err != nil { + return nil, err + } isRoot := false if dirPath == "" || dirPath == "/" { diff --git a/go/vt/topo/memorytopo/election.go b/go/vt/topo/memorytopo/election.go index ad173695099..a979dd306a5 100644 --- a/go/vt/topo/memorytopo/election.go +++ b/go/vt/topo/memorytopo/election.go @@ -33,6 +33,10 @@ func (c *Conn) NewLeaderParticipation(name, id string) (topo.LeaderParticipation c.factory.mu.Lock() defer c.factory.mu.Unlock() + if err := c.factory.getOperationError(NewLeaderParticipation, id); err != nil { + return nil, err + } + // Make sure the global path exists. electionPath := path.Join(electionsPath, name) if n := c.factory.getOrCreatePath(c.cell, electionPath); n == nil { diff --git a/go/vt/topo/memorytopo/file.go b/go/vt/topo/memorytopo/file.go index 0007203799f..f114d1baf03 100644 --- a/go/vt/topo/memorytopo/file.go +++ b/go/vt/topo/memorytopo/file.go @@ -44,6 +44,9 @@ func (c *Conn) Create(ctx context.Context, filePath string, contents []byte) (to if c.factory.err != nil { return nil, c.factory.err } + if err := c.factory.getOperationError(Create, filePath); err != nil { + return nil, err + } // Get the parent dir. dir, file := path.Split(filePath) @@ -88,6 +91,9 @@ func (c *Conn) Update(ctx context.Context, filePath string, contents []byte, ver if c.factory.err != nil { return nil, c.factory.err } + if err := c.factory.getOperationError(Update, filePath); err != nil { + return nil, err + } // Get the parent dir, we'll need it in case of creation. dir, file := path.Split(filePath) @@ -162,6 +168,9 @@ func (c *Conn) Get(ctx context.Context, filePath string) ([]byte, topo.Version, if c.factory.err != nil { return nil, nil, c.factory.err } + if err := c.factory.getOperationError(Get, filePath); err != nil { + return nil, nil, err + } // Get the node. n := c.factory.nodeByPath(c.cell, filePath) @@ -187,6 +196,9 @@ func (c *Conn) List(ctx context.Context, filePathPrefix string) ([]topo.KVInfo, if c.factory.err != nil { return nil, c.factory.err } + if err := c.factory.getOperationError(List, filePathPrefix); err != nil { + return nil, err + } dir, file := path.Split(filePathPrefix) // Get the node to list. @@ -246,6 +258,9 @@ func (c *Conn) Delete(ctx context.Context, filePath string, version topo.Version if c.factory.err != nil { return c.factory.err } + if err := c.factory.getOperationError(Delete, filePath); err != nil { + return err + } // Get the parent dir. dir, file := path.Split(filePath) diff --git a/go/vt/topo/memorytopo/lock.go b/go/vt/topo/memorytopo/lock.go index 5c2a2462495..106a158aad5 100644 --- a/go/vt/topo/memorytopo/lock.go +++ b/go/vt/topo/memorytopo/lock.go @@ -42,11 +42,25 @@ type memoryTopoLockDescriptor struct { // TryLock is part of the topo.Conn interface. Its implementation is same as Lock func (c *Conn) TryLock(ctx context.Context, dirPath, contents string) (topo.LockDescriptor, error) { + c.factory.mu.Lock() + err := c.factory.getOperationError(TryLock, dirPath) + c.factory.mu.Unlock() + if err != nil { + return nil, err + } + return c.Lock(ctx, dirPath, contents) } // Lock is part of the topo.Conn interface. func (c *Conn) Lock(ctx context.Context, dirPath, contents string) (topo.LockDescriptor, error) { + c.factory.mu.Lock() + err := c.factory.getOperationError(Lock, dirPath) + c.factory.mu.Unlock() + if err != nil { + return nil, err + } + return c.lock(ctx, dirPath, contents) } diff --git a/go/vt/topo/memorytopo/memorytopo.go b/go/vt/topo/memorytopo/memorytopo.go index b881be1b785..35c1f2fad49 100644 --- a/go/vt/topo/memorytopo/memorytopo.go +++ b/go/vt/topo/memorytopo/memorytopo.go @@ -23,6 +23,7 @@ import ( "context" "errors" "math/rand" + "regexp" "strings" "sync" "sync/atomic" @@ -49,6 +50,25 @@ const ( UnreachableServerAddr = "unreachable" ) +// Operation is one of the operations defined by topo.Conn +type Operation int + +// The following is the list of topo.Conn operations +const ( + ListDir = Operation(iota) + Create + Update + Get + List + Delete + Lock + TryLock + Watch + WatchRecursive + NewLeaderParticipation + Close +) + // Factory is a memory-based implementation of topo.Factory. It // takes a file-system like approach, with directories at each level // being an actual directory node. This is meant to be closer to @@ -71,6 +91,15 @@ type Factory struct { // err is used for testing purposes to force queries / watches // to return the given error err error + // operationErrors is used for testing purposes to fake errors from + // operations and paths matching the spec + operationErrors map[Operation][]errorSpec +} + +type errorSpec struct { + op Operation + pathPattern *regexp.Regexp + err error } // HasGlobalReadOnlyCell is part of the topo.Factory interface. @@ -236,8 +265,9 @@ func (n *node) PropagateWatchError(err error) { // in case of a problem. func NewServerAndFactory(ctx context.Context, cells ...string) (*topo.Server, *Factory) { f := &Factory{ - cells: make(map[string]*node), - generation: uint64(rand.Int63n(1 << 60)), + cells: make(map[string]*node), + generation: uint64(rand.Int63n(1 << 60)), + operationErrors: make(map[Operation][]errorSpec), } f.cells[topo.GlobalCell] = f.newDirectory(topo.GlobalCell, nil) @@ -349,3 +379,24 @@ func (f *Factory) recursiveDelete(n *node) { f.recursiveDelete(parent) } } + +func (f *Factory) AddOperationError(op Operation, pathPattern string, err error) { + f.mu.Lock() + defer f.mu.Unlock() + + f.operationErrors[op] = append(f.operationErrors[op], errorSpec{ + op: op, + pathPattern: regexp.MustCompile(pathPattern), + err: err, + }) +} + +func (f *Factory) getOperationError(op Operation, path string) error { + specs := f.operationErrors[op] + for _, spec := range specs { + if spec.pathPattern.MatchString(path) { + return spec.err + } + } + return nil +} diff --git a/go/vt/topo/memorytopo/watch.go b/go/vt/topo/memorytopo/watch.go index 0f245c95b5f..f0760a7a773 100644 --- a/go/vt/topo/memorytopo/watch.go +++ b/go/vt/topo/memorytopo/watch.go @@ -35,6 +35,9 @@ func (c *Conn) Watch(ctx context.Context, filePath string) (*topo.WatchData, <-c if c.factory.err != nil { return nil, nil, c.factory.err } + if err := c.factory.getOperationError(Watch, filePath); err != nil { + return nil, nil, err + } n := c.factory.nodeByPath(c.cell, filePath) if n == nil { @@ -85,6 +88,9 @@ func (c *Conn) WatchRecursive(ctx context.Context, dirpath string) ([]*topo.Watc if c.factory.err != nil { return nil, nil, c.factory.err } + if err := c.factory.getOperationError(WatchRecursive, dirpath); err != nil { + return nil, nil, err + } n := c.factory.getOrCreatePath(c.cell, dirpath) if n == nil { diff --git a/java/client/pom.xml b/java/client/pom.xml index f27c9ce008d..10e100aaa05 100644 --- a/java/client/pom.xml +++ b/java/client/pom.xml @@ -5,7 +5,7 @@ io.vitess vitess-parent - 18.0.5-shopify-6 + 18.0.5-shopify-7 vitess-client diff --git a/java/example/pom.xml b/java/example/pom.xml index 5db40d8b1ec..4785d91aaf8 100644 --- a/java/example/pom.xml +++ b/java/example/pom.xml @@ -5,7 +5,7 @@ io.vitess vitess-parent - 18.0.5-shopify-6 + 18.0.5-shopify-7 vitess-example diff --git a/java/grpc-client/pom.xml b/java/grpc-client/pom.xml index 3b5d129007d..99b7d1029bd 100644 --- a/java/grpc-client/pom.xml +++ b/java/grpc-client/pom.xml @@ -5,7 +5,7 @@ io.vitess vitess-parent - 18.0.5-shopify-6 + 18.0.5-shopify-7 vitess-grpc-client diff --git a/java/jdbc/pom.xml b/java/jdbc/pom.xml index 6e6db70ce51..4a23e113b93 100644 --- a/java/jdbc/pom.xml +++ b/java/jdbc/pom.xml @@ -5,7 +5,7 @@ io.vitess vitess-parent - 18.0.5-shopify-6 + 18.0.5-shopify-7 vitess-jdbc diff --git a/java/pom.xml b/java/pom.xml index d3e387df32f..b2b1504e256 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -11,7 +11,7 @@ io.vitess vitess-parent - 18.0.5-shopify-6 + 18.0.5-shopify-7 pom Vitess Java Client libraries [Parent]