Added optional provisionConstraints on instance provisioning (#9211)

Added a new query param 'provisionConstraints' to the instance-provision
service. provisionConstraints is and object where fields on agones pod
can be specified to limit which pods will be used for that provision.
It is of the form

{
  <path.of.field>: {
    <comparator>: 'value'
  }
}

e.g.

{
  'metadata.labels.maxCCU': {
    lte: '40'
  }
}

to limit the provision to pods where metadata.labels.maxCCU is
less than or equal to 40. The sub-field selector must be passed
as a dot-separated string as the key(s) at the top level of the
object.

Use of provisionConstraints will now ignore the existing condition that
the pod's name must contain the release. This is to allow for multiple
instanceserver deployments of varying sizes; if one wanted some
instanceservers on e.g. t3a.smalls and others on t3a.xlarges, they would
need to be separate nodeGroups in AWS, and there would need to be separate
Helm deployments for the different instanceserver sizes with labels
indicating e.g. numCores or maxCCU for each deplyoment. The deployments
must have different names, which would make the name match regex not
match all deployments.

Added a script to build the full monorepo for deployment on Docker Desktop's
Kubernetes cluster. This unfortunately isn't 100% sufficient for testing/running,
as the Agones pods do not appear to be publicly addressable, so the mediasoup
UDP connections cannot be successfully made.

Removed concept of pressured instances for provisioning. getISInService now just
checks whether an existing instance is below the cap for that location or not.

packages/client-core/src/common/services/LocationInstanceConnectionService.ts

@@ -36,7 +36,6 @@ import { instanceProvisionPath } from '@etherealengine/engine/src/schemas/networ
 import { InstanceID, instancePath, InstanceType } from '@etherealengine/engine/src/schemas/networking/instance.schema'
 import { SceneID } from '@etherealengine/engine/src/schemas/projects/scene.schema'
 import { LocationID, RoomCode } from '@etherealengine/engine/src/schemas/social/location.schema'
-import { API } from '../../API'
 import { SocketWebRTCClientNetwork } from '../../transports/SocketWebRTCClientFunctions'
 import { AuthState } from '../../user/services/AuthService'
 
@@ -80,7 +79,7 @@ export const LocationInstanceConnectionService = {
     logger.info({ locationId, instanceId, sceneId }, 'Provision World Server')
     const token = getState(AuthState).authUser.accessToken
     if (instanceId != null) {
-      const instance = (await API.instance.client.service(instancePath).find({
+      const instance = (await Engine.instance.api.service(instancePath).find({
         query: {
           id: instanceId,
           ended: false
@@ -90,7 +89,7 @@ export const LocationInstanceConnectionService = {
         instanceId = null!
       }
     }
-    const provisionResult = await API.instance.client.service(instanceProvisionPath).find({
+    const provisionResult = await Engine.instance.api.service(instanceProvisionPath).find({
       query: {
         locationId,
         instanceId,
@@ -120,7 +119,7 @@ export const LocationInstanceConnectionService = {
   provisionExistingServer: async (locationId: LocationID, instanceId: InstanceID, sceneId: SceneID) => {
     logger.info({ locationId, instanceId, sceneId }, 'Provision Existing World Server')
     const token = getState(AuthState).authUser.accessToken
-    const instance = (await API.instance.client.service(instancePath).find({
+    const instance = (await Engine.instance.api.service(instancePath).find({
       query: {
         id: instanceId,
         ended: false
@@ -136,7 +135,7 @@ export const LocationInstanceConnectionService = {
       }
       return
     }
-    const provisionResult = await API.instance.client.service(instanceProvisionPath).find({
+    const provisionResult = await Engine.instance.api.service(instanceProvisionPath).find({
       query: {
         locationId,
         instanceId,
@@ -161,7 +160,7 @@ export const LocationInstanceConnectionService = {
   provisionExistingServerByRoomCode: async (locationId: LocationID, roomCode: RoomCode, sceneId: SceneID) => {
     logger.info({ locationId, roomCode, sceneId }, 'Provision Existing World Server')
     const token = getState(AuthState).authUser.accessToken
-    const instance = (await API.instance.client.service(instancePath).find({
+    const instance = (await Engine.instance.api.service(instancePath).find({
       query: {
         roomCode,
         ended: false
@@ -177,7 +176,7 @@ export const LocationInstanceConnectionService = {
       }
       return
     }
-    const provisionResult = await API.instance.client.service(instanceProvisionPath).find({
+    const provisionResult = await Engine.instance.api.service(instanceProvisionPath).find({
       query: {
         locationId,
         roomCode,

packages/client/vite.config.ts

@@ -307,7 +307,7 @@ export default defineConfig(async () => {
               ? 'dev-sw.js?dev-sw'
               : 'service-worker.js'
             : '',
-        paymentPointer: coilSetting.paymentPointer || ''
+        paymentPointer: coilSetting?.paymentPointer || ''
       }),
       viteCompression({
         filter: /\.(js|mjs|json|css)$/i,

packages/engine/src/assets/compression/ModelTransformFunctions.ts

@@ -689,7 +689,7 @@ export async function transformModel(args: ModelTransformParameters) {
         {
           name: fileName,
           byteLength: data.byteLength,
-          
+
         }
       })
     )*/

packages/server-core/src/networking/instance-provision/instance-provision.class.ts

@@ -57,7 +57,6 @@ import getLocalServerIp from '../../util/get-local-server-ip'
 const releaseRegex = /^([a-zA-Z0-9]+)-/
 
 const isNameRegex = /instanceserver-([a-zA-Z0-9]{5}-[a-zA-Z0-9]{5})/
-const pressureThresholdPercent = 0.8
 
 /**
  * Gets an instanceserver that is not in use or reserved
@@ -69,7 +68,8 @@ export async function getFreeInstanceserver({
   channelId,
   roomCode,
   userId,
-  createPrivateRoom
+  createPrivateRoom,
+  provisionConstraints
 }: {
   app: Application
   iteration: number
@@ -78,6 +78,7 @@ export async function getFreeInstanceserver({
   roomCode?: RoomCode
   userId?: UserID
   createPrivateRoom?: boolean
+  provisionConstraints?: object
 }): Promise<InstanceProvisionType> {
   await app.service(instancePath).remove(null, {
     query: {
@@ -101,15 +102,56 @@ export async function getFreeInstanceserver({
       channelId,
       roomCode,
       userId,
-      createPrivateRoom
+      createPrivateRoom,
+      provisionConstraints
     })
   }
   logger.info('Getting free instanceserver')
   const k8AgonesClient = getState(ServerState).k8AgonesClient
   const serverResult = await k8AgonesClient.listNamespacedCustomObject('agones.dev', 'v1', 'default', 'gameservers')
   const readyServers = _.filter((serverResult.body as any).items, (server: any) => {
     const releaseMatch = releaseRegex.exec(server.metadata.name)
-    return server.status.state === 'Ready' && releaseMatch != null && releaseMatch[1] === config.server.releaseName
+    let returned = server.status.state === 'Ready'
+    if (returned && !provisionConstraints)
+      returned = returned && releaseMatch != null && releaseMatch[1] === config.server.releaseName
+    if (returned && provisionConstraints) {
+      const keys = Object.keys(provisionConstraints)
+      for (let key of keys) {
+        const constraint = provisionConstraints[key]
+        const provisionFunction = Object.keys(constraint)[0]
+        const provisionValue = constraint[provisionFunction]
+        const provisionFieldSplit = key.split('.')
+        let serverField = server
+        for (const item of provisionFieldSplit) {
+          serverField = serverField[item]
+          if (!serverField) break
+        }
+        switch (provisionFunction) {
+          case 'lte':
+            returned = returned && parseFloat(serverField) <= parseFloat(provisionValue)
+            break
+          case 'lt':
+            returned = returned && parseFloat(serverField) < parseFloat(provisionValue)
+            break
+          case 'gte':
+            returned = returned && parseFloat(serverField) >= parseFloat(provisionValue)
+            break
+          case 'gt':
+            returned = returned && parseFloat(serverField) > parseFloat(provisionValue)
+            break
+          case 'eq':
+            returned =
+              returned &&
+              (parseFloat(provisionValue)
+                ? parseFloat(serverField) === parseFloat(provisionValue)
+                : serverField === provisionValue)
+            break
+          default:
+            break
+        }
+      }
+    }
+    return returned
   })
   const ipAddresses = readyServers.map((server) => `${server.status.address}:${server.status.ports[0].port}`)
   const assignedInstances: any = await app.service(instancePath).find({
@@ -147,7 +189,8 @@ export async function getFreeInstanceserver({
     roomCode,
     userId,
     createPrivateRoom,
-    podName: pod.metadata.name
+    podName: pod.metadata.name,
+    provisionConstraints
   })
 }
 
@@ -160,7 +203,8 @@ export async function checkForDuplicatedAssignments({
   roomCode,
   createPrivateRoom,
   userId,
-  podName
+  podName,
+  provisionConstraints
 }: {
   app: Application
   ipAddress: string
@@ -171,6 +215,7 @@ export async function checkForDuplicatedAssignments({
   createPrivateRoom?: boolean
   userId?: UserID
   podName?: string
+  provisionConstraints?: object
 }): Promise<InstanceProvisionType> {
   /** since in local dev we can only have one instance server of each type at a time, we must force all old instances of this type to be ended */
   if (!config.kubernetes.enabled) {
@@ -251,7 +296,15 @@ export async function checkForDuplicatedAssignments({
       await app.service(instancePath).remove(assignResult.id)
       //If this is the 10th or more attempt to get a free instanceserver, then there probably aren't any free ones,
       if (iteration < 10) {
-        return getFreeInstanceserver({ app, iteration: iteration + 1, locationId, channelId, roomCode, userId })
+        return getFreeInstanceserver({
+          app,
+          iteration: iteration + 1,
+          locationId,
+          channelId,
+          roomCode,
+          userId,
+          provisionConstraints
+        })
       } else {
         logger.info('Made 10 attempts to get free instanceserver without success, returning null')
         return {
@@ -374,7 +427,8 @@ export async function checkForDuplicatedAssignments({
       channelId,
       roomCode,
       createPrivateRoom,
-      userId
+      userId,
+      provisionConstraints
     })
   }
 
@@ -413,6 +467,7 @@ export class InstanceProvisionService implements ServiceInterface<InstanceProvis
    * @param channelId
    * @param roomCode
    * @param userId
+   * @param provisionConstraints
    * @returns id, ipAddress and port
    */
 
@@ -421,13 +476,15 @@ export class InstanceProvisionService implements ServiceInterface<InstanceProvis
     locationId,
     channelId,
     roomCode,
-    userId
+    userId,
+    provisionConstraints
   }: {
     availableLocationInstances: InstanceType[]
     locationId?: LocationID
     channelId?: ChannelID
     roomCode?: RoomCode
     userId?: UserID
+    provisionConstraints?: object
   }): Promise<InstanceProvisionType> {
     await this.app.service(instancePath).remove(null, {
       query: {
@@ -437,12 +494,21 @@ export class InstanceProvisionService implements ServiceInterface<InstanceProvis
         }
       }
     })
-    const instanceUserSort = _.orderBy(availableLocationInstances, ['currentUsers'], ['desc'])
-    const nonPressuredInstances = instanceUserSort.filter((instance) => {
-      return instance.currentUsers < pressureThresholdPercent * instance.location.maxUsersPerInstance
-    })
-    const instances = nonPressuredInstances.length > 0 ? nonPressuredInstances : instanceUserSort
-    const instance = instances[0]
+    const nonFullInstances = availableLocationInstances.filter(
+      (instance) => instance.currentUsers < instance.location.maxUsersPerInstance
+    )
+    if (nonFullInstances.length === 0)
+      return getFreeInstanceserver({
+        app: this.app,
+        iteration: 0,
+        locationId,
+        channelId,
+        roomCode,
+        userId,
+        provisionConstraints
+      })
+    const instanceUserSort = _.orderBy(nonFullInstances, ['currentUsers'], ['desc'])
+    const instance = instanceUserSort[0]
     if (!config.kubernetes.enabled) {
       logger.info('Resetting local instance to ' + instance.id)
       const localIp = await getLocalServerIp(channelId != null)
@@ -455,14 +521,24 @@ export class InstanceProvisionService implements ServiceInterface<InstanceProvis
     const isCleanup = await this.isCleanup(instance)
     if (isCleanup) {
       logger.info('IS did not exist and was cleaned up')
-      if (availableLocationInstances.length > 1)
+      if (instanceUserSort.length > 1)
         return this.getISInService({
           availableLocationInstances: availableLocationInstances.slice(1),
           locationId,
           channelId,
-          roomCode
+          roomCode,
+          provisionConstraints
+        })
+      else
+        return getFreeInstanceserver({
+          app: this.app,
+          iteration: 0,
+          locationId,
+          channelId,
+          roomCode,
+          userId,
+          provisionConstraints
         })
-      else return getFreeInstanceserver({ app: this.app, iteration: 0, locationId, channelId, roomCode, userId })
     }
     logger.info('IS existed, using it %o', instance)
     const ipAddressSplit = instance.ipAddress!.split(':')
@@ -538,6 +614,7 @@ export class InstanceProvisionService implements ServiceInterface<InstanceProvis
       const roomCode = params?.query?.roomCode as RoomCode
       const createPrivateRoom = params?.query?.createPrivateRoom
       const token = params?.query?.token
+      const provisionConstraints = params?.query?.provisionConstraints
       logger.info('instance-provision find %s %s %s %s', locationId, instanceId, channelId, roomCode)
       if (!token) throw new NotAuthenticated('No token provided')
       // Check if JWT resolves to a user
@@ -563,11 +640,26 @@ export class InstanceProvisionService implements ServiceInterface<InstanceProvis
           }
         })) as Paginated<InstanceType>
         if (channelInstance == null || channelInstance.data.length === 0)
-          return getFreeInstanceserver({ app: this.app, iteration: 0, channelId, roomCode, userId })
+          return getFreeInstanceserver({
+            app: this.app,
+            iteration: 0,
+            channelId,
+            roomCode,
+            userId,
+            provisionConstraints
+          })
         else {
           if (config.kubernetes.enabled) {
             const isCleanup = await this.isCleanup(channelInstance.data[0])
-            if (isCleanup) return getFreeInstanceserver({ app: this.app, iteration: 0, channelId, roomCode, userId })
+            if (isCleanup)
+              return getFreeInstanceserver({
+                app: this.app,
+                iteration: 0,
+                channelId,
+                roomCode,
+                userId,
+                provisionConstraints
+              })
           }
           const actualInstance = channelInstance.data[0]
           const ipAddressSplit = actualInstance.ipAddress!.split(':')
@@ -599,7 +691,15 @@ export class InstanceProvisionService implements ServiceInterface<InstanceProvis
         }
 
         if ((roomCode && (instance == null || instance.ended)) || createPrivateRoom)
-          return getFreeInstanceserver({ app: this.app, iteration: 0, locationId, roomCode, userId, createPrivateRoom })
+          return getFreeInstanceserver({
+            app: this.app,
+            iteration: 0,
+            locationId,
+            roomCode,
+            userId,
+            createPrivateRoom,
+            provisionConstraints
+          })
 
         let isCleanup
 
@@ -735,14 +835,22 @@ export class InstanceProvisionService implements ServiceInterface<InstanceProvis
             )
         )
         if (allowedLocationInstances.length === 0)
-          return getFreeInstanceserver({ app: this.app, iteration: 0, locationId, roomCode, userId })
+          return getFreeInstanceserver({
+            app: this.app,
+            iteration: 0,
+            locationId,
+            roomCode,
+            userId,
+            provisionConstraints
+          })
         else
           return this.getISInService({
             availableLocationInstances: allowedLocationInstances,
             locationId,
             channelId,
             roomCode,
-            userId
+            userId,
+            provisionConstraints
           })
       }
     } catch (err) {