From e27b6e20357cb3eaff6feceeeb423e28e25fa9ad Mon Sep 17 00:00:00 2001 From: "vitess-bot[bot]" <108069721+vitess-bot[bot]@users.noreply.github.com> Date: Thu, 21 Nov 2024 13:51:02 +0530 Subject: [PATCH] [release-21.0] Fix potential deadlock in health streamer (#17261) (#17270) Signed-off-by: Manan Gupta Co-authored-by: vitess-bot[bot] <108069721+vitess-bot[bot]@users.noreply.github.com> --- go/vt/vttablet/tabletserver/health_streamer.go | 4 +++- go/vt/vttablet/tabletserver/health_streamer_test.go | 3 ++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/go/vt/vttablet/tabletserver/health_streamer.go b/go/vt/vttablet/tabletserver/health_streamer.go index f9f65d197b2..eaeba6315e3 100644 --- a/go/vt/vttablet/tabletserver/health_streamer.go +++ b/go/vt/vttablet/tabletserver/health_streamer.go @@ -293,8 +293,10 @@ func (hs *healthStreamer) SetUnhealthyThreshold(v time.Duration) { // so it can read and write to the MySQL instance for schema-tracking. func (hs *healthStreamer) MakePrimary(serving bool) { hs.fieldsMu.Lock() - defer hs.fieldsMu.Unlock() hs.isServingPrimary = serving + // We let go of the lock here because we don't want to hold the lock when calling RegisterNotifier. + // If we keep holding the lock, there is a potential deadlock that can happen. + hs.fieldsMu.Unlock() // We register for notifications from the schema Engine only when schema tracking is enabled, // and we are going to a serving primary state. if serving && hs.signalWhenSchemaChange { diff --git a/go/vt/vttablet/tabletserver/health_streamer_test.go b/go/vt/vttablet/tabletserver/health_streamer_test.go index 09867e7a51b..d0a248223d3 100644 --- a/go/vt/vttablet/tabletserver/health_streamer_test.go +++ b/go/vt/vttablet/tabletserver/health_streamer_test.go @@ -563,13 +563,14 @@ func TestDeadlockBwCloseAndReload(t *testing.T) { wg := sync.WaitGroup{} wg.Add(2) - // Try running Close and reload in parallel multiple times. + // Try running Close & MakePrimary and reload in parallel multiple times. // This reproduces the deadlock quite readily. go func() { defer wg.Done() for i := 0; i < 100; i++ { hs.Close() hs.Open() + hs.MakePrimary(true) } }()