Fixes for instanceserver connection bugs (#10766)

A rare situation was cropping up where an instance record was
getting removed from the database after an instance provision.
This would cause initializeInstance to return false, and loadEngine
to not run, but instanceStarted would remain true. That instance
wouldn't start, but the server would still remain in the pool of
Ready servers. Later clients assigned to that pod would then just
endlessly wait for InstanceServerState.ready to be true, which
would never happen. setting instanceStarted back to false if
initializeInstance returns false should fix this.

Increased the timeout on assigned but not-connected instances to
60 seconds from 30. This may be what was causing those instance
records to be removed, if a client on a slow network was getting
the provision but then taking more than 30 seconds to make the
websocket connection.

packages/instanceserver/src/channels.ts

@@ -410,6 +410,7 @@ const updateInstance = async ({
     instanceStarted = true
     const initialized = await initializeInstance({ app, status, headers, userId })
     if (initialized) await loadEngine({ app, sceneId, headers })
+    else instanceStarted = false
     return true
   } else {
     try {

packages/server-core/src/networking/instance-provision/instance-provision.class.ts

@@ -87,7 +87,7 @@ export async function getFreeInstanceserver({
     query: {
       assigned: true,
       assignedAt: {
-        $lt: toDateTimeSql(new Date(new Date().getTime() - 30000))
+        $lt: toDateTimeSql(new Date(new Date().getTime() - 60000))
       }
     },
     headers
@@ -510,7 +510,7 @@ export class InstanceProvisionService implements ServiceInterface<InstanceProvis
       query: {
         assigned: true,
         assignedAt: {
-          $lt: toDateTimeSql(new Date(new Date().getTime() - 30000))
+          $lt: toDateTimeSql(new Date(new Date().getTime() - 60000))
         }
       }
     })
@@ -615,7 +615,7 @@ export class InstanceProvisionService implements ServiceInterface<InstanceProvis
       query: {
         assigned: true,
         assignedAt: {
-          $lt: toDateTimeSql(new Date(new Date().getTime() - 30000))
+          $lt: toDateTimeSql(new Date(new Date().getTime() - 60000))
         }
       }
     })