fix(sdk-dotnet): Call LHClient method only with one thread at a time …

…and clear all connections when the boostrap server is down
littlehorse-enterprises · Dec 19, 2024 · f616ab4 · f616ab4
1 parent 0898eb0
commit f616ab4
Show file tree

Hide file tree

Showing 2 changed files with 14 additions and 12 deletions.
diff --git a/sdk-dotnet/LittleHorse.Sdk/LHConfig.cs b/sdk-dotnet/LittleHorse.Sdk/LHConfig.cs
@@ -6,6 +6,7 @@
 using Microsoft.Extensions.Logging;
 using System.Security.Cryptography.X509Certificates;
 using System.Net.Security;
+using System.Runtime.CompilerServices;
 using static LittleHorse.Common.Proto.LittleHorse;
 
 namespace LittleHorse.Sdk {
@@ -120,7 +121,8 @@ public LittleHorseClient GetGrpcClientInstance()
         {
             return GetGrpcClientInstance(BootstrapHost, BootstrapPort);
         }
-
+
+        [MethodImpl(MethodImplOptions.Synchronized)]
         public LittleHorseClient GetGrpcClientInstance(string host, int port)
         {
             string channelKey = $"{BootstrapProtocol}://{host}:{port}";
@@ -136,7 +138,7 @@ public LittleHorseClient GetGrpcClientInstance(string host, int port)
 
             var lhClient = new LittleHorseClient(channel);
 
-            _createdChannels.TryAdd(channelKey, lhClient);
+            _createdChannels.Add(channelKey, lhClient);
 
             return lhClient;
         }

diff --git a/sdk-dotnet/LittleHorse.Sdk/Worker/Internal/LHServerConnectionManager.cs b/sdk-dotnet/LittleHorse.Sdk/Worker/Internal/LHServerConnectionManager.cs
@@ -61,12 +61,15 @@ private void RebalanceWork()
         {
             while (_running)
             {
-                DoHeartBeat();
                 try
                 {
+                    DoHeartBeat();
                     Thread.Sleep(BALANCER_SLEEP_TIME);
                 }
-                catch { }
+                catch (Exception ex)
+                {
+                    _logger?.LogError(ex, $"Something happened while doing heartbeats");
+                }
             }
         }
 
@@ -87,7 +90,7 @@ private void DoHeartBeat()
             catch (Exception ex)
             {
                 _logger?.LogError(ex, $"Failed contacting bootstrap host {_config.BootstrapHost}:{_config.BootstrapPort}");
-                CloseConnection(_config.BootstrapHost, _config.BootstrapPort);
+                CloseAllConnections();
             }
         }
 
@@ -279,16 +282,13 @@ private ReportTaskRun ExecuteTask(ScheduledTask scheduledTask, DateTime? schedul
             return _task.TaskMethod!.Invoke(_task.Executable, inputs);
         }
 
-        private void CloseConnection(string host, int port)
+        private void CloseAllConnections()
         {
-            var serverConnection = _runningConnections.FirstOrDefault(c => 
-                c.IsSame(host, port));
-
-            if (serverConnection != null)
+            _runningConnections.RemoveAll(serverConnection =>
             {
                 serverConnection.Dispose();
-                _runningConnections.Remove(serverConnection);
-            }
+                return true;
+            });
         }
     }
 }