Remove performance logs (twentyhq#6709)

We have found the root cause of the issue: - when using a datasource (including the cached ones), we are fetching ObjectMetadataCollection from cache (700kB). Datasource usage is happening any time we are using twentyORM, which is everywhere in the jobs and in some resolvers (including the GetCurrentUser one). This is leading to a high load on redis and leading to the performance issues we are seeing. - we actually don't need to fetch this objectMetadataCollection while using a cached datasource, only when we instantiate a new one
ehconitin · Aug 21, 2024 · 612a875 · 612a875
1 parent aa4ae53
commit 612a875
Show file tree

Hide file tree

Showing 5 changed files with 25 additions and 68 deletions.
diff --git a/packages/twenty-server/src/engine/core-modules/user/services/user.service.ts b/packages/twenty-server/src/engine/core-modules/user/services/user.service.ts
@@ -40,24 +40,18 @@ export class UserService extends TypeOrmQueryService<User> {
       return null;
     }
 
-    console.time('loadWorkspaceMember repo');
     const workspaceMemberRepository =
       await this.twentyORMGlobalManager.getRepositoryForWorkspace<WorkspaceMemberWorkspaceEntity>(
         user.defaultWorkspaceId,
         'workspaceMember',
       );
 
-    console.timeEnd('loadWorkspaceMember repo');
-
-    console.time('loadWorkspaceMember find');
     const workspaceMember = await workspaceMemberRepository.findOne({
       where: {
         userId: user.id,
       },
     });
 
-    console.timeEnd('loadWorkspaceMember find');
-
     return workspaceMember;
   }
 
@@ -66,21 +60,14 @@ export class UserService extends TypeOrmQueryService<User> {
       return [];
     }
 
-    console.time('loadWorkspaceMembers repo');
     const workspaceMemberRepository =
       await this.twentyORMGlobalManager.getRepositoryForWorkspace<WorkspaceMemberWorkspaceEntity>(
         workspace.id,
         'workspaceMember',
       );
 
-    console.timeEnd('loadWorkspaceMembers repo');
-
-    console.time('loadWorkspaceMembers find');
-
     const workspaceMembers = workspaceMemberRepository.find();
 
-    console.timeEnd('loadWorkspaceMembers find');
-
     return workspaceMembers;
   }
 

diff --git a/packages/twenty-server/src/engine/core-modules/user/user.resolver.ts b/packages/twenty-server/src/engine/core-modules/user/user.resolver.ts
@@ -98,7 +98,6 @@ export class UserResolver {
     nullable: true,
   })
   async workspaceMember(@Parent() user: User): Promise<WorkspaceMember | null> {
-    console.time('resolver workspaceMember');
     const workspaceMember = await this.userService.loadWorkspaceMember(user);
 
     if (workspaceMember && workspaceMember.avatarUrl) {
@@ -109,7 +108,6 @@ export class UserResolver {
 
       workspaceMember.avatarUrl = `${workspaceMember.avatarUrl}?token=${avatarUrlToken}`;
     }
-    console.timeEnd('resolver workspaceMember');
 
     // TODO: Fix typing disrepency between Entity and DTO
     return workspaceMember as WorkspaceMember | null;
@@ -119,7 +117,6 @@ export class UserResolver {
     nullable: true,
   })
   async workspaceMembers(@Parent() user: User): Promise<WorkspaceMember[]> {
-    console.time('resolver workspaceMembers');
     const workspaceMembers = await this.userService.loadWorkspaceMembers(
       user.defaultWorkspace,
     );
@@ -135,8 +132,6 @@ export class UserResolver {
       }
     }
 
-    console.timeEnd('resolver workspaceMembers');
-
     // TODO: Fix typing disrepency between Entity and DTO
     return workspaceMembers as WorkspaceMember[];
   }

diff --git a/packages/twenty-server/src/engine/twenty-orm/factories/workspace-datasource.factory.ts b/packages/twenty-server/src/engine/twenty-orm/factories/workspace-datasource.factory.ts
@@ -2,7 +2,6 @@ import { Injectable } from '@nestjs/common';
 import { InjectRepository } from '@nestjs/typeorm';
 
 import { EntitySchema, Repository } from 'typeorm';
-import { v4 } from 'uuid';
 
 import { EnvironmentService } from 'src/engine/integrations/environment/environment.service';
 import { DataSourceService } from 'src/engine/metadata-modules/data-source/data-source.service';
@@ -29,17 +28,11 @@ export class WorkspaceDatasourceFactory {
     workspaceId: string,
     workspaceMetadataVersion: string | null,
   ): Promise<WorkspaceDataSource> {
-    const logId = v4();
-
-    console.time(`fetch in datasource factory ${logId}`);
-
     const latestWorkspaceMetadataVersion =
       await this.workspaceMetadataVersionService.getMetadataVersion(
         workspaceId,
       );
 
-    console.timeEnd(`fetch in datasource factory ${logId}`);
-
     const desiredWorkspaceMetadataVersion =
       workspaceMetadataVersion ?? latestWorkspaceMetadataVersion;
 
@@ -55,38 +48,35 @@ export class WorkspaceDatasourceFactory {
       );
     }
 
-    let cachedObjectMetadataCollection =
-      await this.workspaceCacheStorageService.getObjectMetadataCollection(
-        workspaceId,
-      );
-
-    if (!cachedObjectMetadataCollection) {
-      const freshObjectMetadataCollection =
-        await this.objectMetadataRepository.find({
-          where: { workspaceId },
-          relations: [
-            'fields.object',
-            'fields',
-            'fields.fromRelationMetadata',
-            'fields.toRelationMetadata',
-            'fields.fromRelationMetadata.toObjectMetadata',
-          ],
-        });
-
-      await this.workspaceCacheStorageService.setObjectMetadataCollection(
-        workspaceId,
-        freshObjectMetadataCollection,
-      );
-
-      cachedObjectMetadataCollection = freshObjectMetadataCollection;
-    }
-
     const workspaceDataSource = await workspaceDataSourceCacheInstance.execute(
       `${workspaceId}-${latestWorkspaceMetadataVersion}`,
       async () => {
-        const logId = v4();
+        let cachedObjectMetadataCollection =
+          await this.workspaceCacheStorageService.getObjectMetadataCollection(
+            workspaceId,
+          );
+
+        if (!cachedObjectMetadataCollection) {
+          const freshObjectMetadataCollection =
+            await this.objectMetadataRepository.find({
+              where: { workspaceId },
+              relations: [
+                'fields.object',
+                'fields',
+                'fields.fromRelationMetadata',
+                'fields.toRelationMetadata',
+                'fields.fromRelationMetadata.toObjectMetadata',
+              ],
+            });
+
+          await this.workspaceCacheStorageService.setObjectMetadataCollection(
+            workspaceId,
+            freshObjectMetadataCollection,
+          );
+
+          cachedObjectMetadataCollection = freshObjectMetadataCollection;
+        }
 
-        console.log('Creating workspace fresh data source...' + logId);
         const dataSourceMetadata =
           await this.dataSourceService.getLastDataSourceMetadataFromWorkspaceId(
             workspaceId,
@@ -104,7 +94,6 @@ export class WorkspaceDatasourceFactory {
           );
         }
 
-        console.time('create entity schema' + logId);
         const cachedEntitySchemaOptions =
           await this.workspaceCacheStorageService.getORMEntitySchema(
             workspaceId,
@@ -130,9 +119,7 @@ export class WorkspaceDatasourceFactory {
 
           cachedEntitySchemas = entitySchemas;
         }
-        console.timeEnd('create entity schema' + logId);
 
-        console.time('create workspace data source' + logId);
         const workspaceDataSource = new WorkspaceDataSource(
           {
             workspaceId,
@@ -156,11 +143,7 @@ export class WorkspaceDatasourceFactory {
           },
         );
 
-        console.timeEnd('create workspace data source' + logId);
-
-        console.time('initialize workspace data source' + logId);
         await workspaceDataSource.initialize();
-        console.timeEnd('initialize workspace data source' + logId);
 
         return workspaceDataSource;
       },

diff --git a/packages/twenty-server/src/engine/twenty-orm/storage/cache-manager.storage.ts b/packages/twenty-server/src/engine/twenty-orm/storage/cache-manager.storage.ts
@@ -16,15 +16,13 @@ export class CacheManager<T> {
       return this.cache.get(cacheKey)!;
     }
 
-    // Remove old entries with the same workspaceId
     for (const key of this.cache.keys()) {
       if (key.startsWith(`${workspaceId}-`)) {
         await onDelete?.(this.cache.get(key)!);
         this.cache.delete(key);
       }
     }
 
-    // Create a new value using the factory callback
     const value = await factory();
 
     if (!value) {

diff --git a/packages/twenty-server/src/engine/twenty-orm/twenty-orm-global.manager.ts b/packages/twenty-server/src/engine/twenty-orm/twenty-orm-global.manager.ts
@@ -1,7 +1,6 @@
 import { Injectable, Type } from '@nestjs/common';
 
 import { ObjectLiteral } from 'typeorm';
-import { v4 } from 'uuid';
 
 import { WorkspaceDatasourceFactory } from 'src/engine/twenty-orm/factories/workspace-datasource.factory';
 import { WorkspaceRepository } from 'src/engine/twenty-orm/repository/workspace.repository';
@@ -37,16 +36,11 @@ export class TwentyORMGlobalManager {
       );
     }
 
-    const logId = v4();
-
-    console.time(`createDataSource in orm ${logId}`);
     const workspaceDataSource = await this.workspaceDataSourceFactory.create(
       workspaceId,
       null,
     );
 
-    console.timeEnd(`createDataSource in orm ${logId}`);
-
     const repository = workspaceDataSource.getRepository<T>(objectMetadataName);
 
     return repository;