debug cluster tests (#2104)

Rearrange cluster shutdown code

.github/workflows/build-dev.yml

@@ -9,8 +9,6 @@ on:
 env:
   DOTNET_SYSTEM_CONSOLE_ALLOW_ANSI_COLOR_REDIRECTION: 1
   TERM: xterm
-  OPENTELEMETRY_URL: http://otel.ornell.io:30798
-  TRACEVIEW_URL: http://traceview.ornell.io
 
 jobs:
   test-slow:  # slow tests that should run in parallel

.github/workflows/pull-request.yml

@@ -5,8 +5,6 @@ on: [pull_request]
 env:
   DOTNET_SYSTEM_CONSOLE_ALLOW_ANSI_COLOR_REDIRECTION: 1
   TERM: xterm
-  OPENTELEMETRY_URL: http://otel.ornell.io:30798
-  TRACEVIEW_URL: http://traceview.ornell.io
 
 jobs:
   build:

src/Proto.Cluster/Cluster.cs

@@ -295,10 +295,10 @@ public async Task ShutdownAsync(bool graceful = true, string reason = "")
         Logger.LogInformation("Waiting for two gossip intervals to pass for {Id}", System.Id);
         // In case provider shutdown is quick, let's wait at least 2 gossip intervals.
         await Task.Delay((int)Config.GossipInterval.TotalMilliseconds * 2).ConfigureAwait(false);
-        
+
         Logger.LogInformation("Stopping cluster provider for {Id}", System.Id);
         // Deregister from configured cluster provider.
-        await Provider.ShutdownAsync(graceful).ConfigureAwait(false);
+        await Provider.ShutdownAsync(graceful);
 
         if (_clusterKindObserver != null)
         {
@@ -312,19 +312,19 @@ public async Task ShutdownAsync(bool graceful = true, string reason = "")
             _clusterMembersObserver = null;
         }
 
-        // Cancel the primary CancellationToken first which will shut down a number of concurrent systems simultaneously.
-        await System.ShutdownAsync(reason).ConfigureAwait(false);
-
         // Shut down the rest of the dependencies in reverse order that they were started.
         await Gossip.ShutdownAsync().ConfigureAwait(false);
 
         if (graceful)
         {
-            await IdentityLookup.ShutdownAsync().ConfigureAwait(false);
+         await IdentityLookup.ShutdownAsync().ConfigureAwait(false);
         }
 
         await Remote.ShutdownAsync(graceful).ConfigureAwait(false);
 
+        // Cancel the primary CancellationToken first which will shut down a number of concurrent systems simultaneously.
+        await System.ShutdownAsync(reason).ConfigureAwait(false);
+
         _shutdownCompletedTcs.TrySetResult(true);
         Logger.LogInformation("Stopped Cluster {Id}", System.Id);
     }

src/Proto.Remote/GrpcNet/GrpcNetRemote.cs

@@ -161,7 +161,7 @@ public async Task ShutdownAsync(bool graceful = true)
 
                     if (_host is not null)
                     {
-                        await _host.StopAsync().ConfigureAwait(false);
+                        await _host.StopAsync().WaitAsync(TimeSpan.FromSeconds(5)).ConfigureAwait(false);
                     }
                 }
             }

tests/Proto.Cluster.Tests/ClusterFixture.cs

@@ -182,12 +182,14 @@ public async Task RemoveNode(Cluster member, bool graceful = true)
         if (Members.Contains(member))
         {
             Members.Remove(member);
-            await member.ShutdownAsync(graceful, "Stopped by ClusterFixture");
+            var t = member.ShutdownAsync(graceful, "Stopped by ClusterFixture");
+            await t.WaitAsync(TimeSpan.FromSeconds(5));
         }
         else if (Clients.Contains(member))
         {
             Clients.Remove(member);
-            await member.ShutdownAsync(graceful, "Stopped by ClusterFixture");
+            var t =  member.ShutdownAsync(graceful, "Stopped by ClusterFixture");
+            await t.WaitAsync(TimeSpan.FromSeconds(5));
         }
         else
         {

tests/Proto.Cluster.Tests/ClusterTests.cs

@@ -565,7 +565,7 @@ private async Task PingPong(
 
             if (response == null)
             {
-                await Task.Delay(200, token);
+                await Task.Delay(200);
             }
         } while (response == null && !token.IsCancellationRequested);
 

tests/Proto.Cluster.Tests/RetryOnDeadLetterTests.cs

@@ -10,7 +10,7 @@ namespace Proto.Cluster.Tests;
 [Collection("ClusterTests")]
 public class RetryOnDeadLetterTests
 {
-    [Fact(Skip = "Flaky")]
+    [Fact]
     public async Task ShouldRetryRequestOnDeadLetterResponseRegardlessOfResponseType()
     {
         var fixture = new Fixture(1);