From 3b1af2b31d801effdd37bf9dbf1d79d59a864751 Mon Sep 17 00:00:00 2001 From: Kyle Baran Date: Tue, 30 Jul 2024 15:13:31 -0700 Subject: [PATCH] Fixes for instanceserver connection bugs (#10766) A rare situation was cropping up where an instance record was getting removed from the database after an instance provision. This would cause initializeInstance to return false, and loadEngine to not run, but instanceStarted would remain true. That instance wouldn't start, but the server would still remain in the pool of Ready servers. Later clients assigned to that pod would then just endlessly wait for InstanceServerState.ready to be true, which would never happen. setting instanceStarted back to false if initializeInstance returns false should fix this. Increased the timeout on assigned but not-connected instances to 60 seconds from 30. This may be what was causing those instance records to be removed, if a client on a slow network was getting the provision but then taking more than 30 seconds to make the websocket connection. --- packages/instanceserver/src/channels.ts | 1 + .../instance-provision/instance-provision.class.ts | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/packages/instanceserver/src/channels.ts b/packages/instanceserver/src/channels.ts index fc584f61e7..38f141ed79 100755 --- a/packages/instanceserver/src/channels.ts +++ b/packages/instanceserver/src/channels.ts @@ -410,6 +410,7 @@ const updateInstance = async ({ instanceStarted = true const initialized = await initializeInstance({ app, status, headers, userId }) if (initialized) await loadEngine({ app, sceneId, headers }) + else instanceStarted = false return true } else { try { diff --git a/packages/server-core/src/networking/instance-provision/instance-provision.class.ts b/packages/server-core/src/networking/instance-provision/instance-provision.class.ts index 459cbb8e36..78c2cc89aa 100755 --- a/packages/server-core/src/networking/instance-provision/instance-provision.class.ts +++ b/packages/server-core/src/networking/instance-provision/instance-provision.class.ts @@ -87,7 +87,7 @@ export async function getFreeInstanceserver({ query: { assigned: true, assignedAt: { - $lt: toDateTimeSql(new Date(new Date().getTime() - 30000)) + $lt: toDateTimeSql(new Date(new Date().getTime() - 60000)) } }, headers @@ -510,7 +510,7 @@ export class InstanceProvisionService implements ServiceInterface