diff --git a/packages/api/scripts/commonObject.sh b/packages/api/scripts/commonObject.sh index 28dc4b7ad..9a28c93d9 100755 --- a/packages/api/scripts/commonObject.sh +++ b/packages/api/scripts/commonObject.sh @@ -137,7 +137,7 @@ import { Unified${ObjectCap}Output } from '../types/model.unified'; import { I${ObjectCap}Service } from '../types'; @Injectable() -export class SyncService implements OnModuleInit { +export class SyncService implements OnModuleInit, IBaseSync { constructor( private prisma: PrismaService, private logger: LoggerService, diff --git a/packages/api/src/@core/@core-services/registries/core-sync.registry.ts b/packages/api/src/@core/@core-services/registries/core-sync.registry.ts index 2dd1677b1..107674e39 100644 --- a/packages/api/src/@core/@core-services/registries/core-sync.registry.ts +++ b/packages/api/src/@core/@core-services/registries/core-sync.registry.ts @@ -1,18 +1,19 @@ +import { IBaseSync } from '@@core/utils/types/interface'; import { Injectable } from '@nestjs/common'; @Injectable() export class CoreSyncRegistry { - private serviceMap: Map; + private serviceMap: Map; constructor() { - this.serviceMap = new Map(); + this.serviceMap = new Map(); } // Register a service with a composite key registerService( category_vertical: string, common_object: string, - service: any, + service: IBaseSync, ) { const compositeKey = this.createCompositeKey( category_vertical, @@ -22,7 +23,7 @@ export class CoreSyncRegistry { } // Retrieve a service using the composite key - getService(category_vertical: string, common_object: string): any { + getService(category_vertical: string, common_object: string): IBaseSync { const compositeKey = this.createCompositeKey( category_vertical, common_object, diff --git a/packages/api/src/@core/@core-services/unification/ingest-data.service.ts b/packages/api/src/@core/@core-services/unification/ingest-data.service.ts new file mode 100644 index 000000000..aefde70d9 --- /dev/null +++ b/packages/api/src/@core/@core-services/unification/ingest-data.service.ts @@ -0,0 +1,77 @@ +import { Injectable } from '@nestjs/common'; +import { CoreSyncRegistry } from '../registries/core-sync.registry'; +import { CoreUnification } from './core-unification.service'; +import { v4 as uuidv4 } from 'uuid'; +import { PrismaService } from '../prisma/prisma.service'; +import { TargetObject } from '@@core/utils/types'; +import { UnifySourceType } from '@@core/utils/types/unify.output'; +import { WebhookService } from '../webhooks/panora-webhooks/webhook.service'; +import { ConnectionUtils } from '@@core/connections/@utils'; + +@Injectable() +export class IngestDataService { + constructor( + private syncRegistry: CoreSyncRegistry, + private coreUnification: CoreUnification, + private webhook: WebhookService, + private prisma: PrismaService, + private connectionUtils: ConnectionUtils, + ) {} + + async ingestData( + sourceObject: U[], + integrationId: string, + connectionId: string, + vertical: string, + commonObject: string, + customFieldMappings?: { + slug: string; + remote_id: string; + }[], + ): Promise { + const unifiedObject = (await this.coreUnification.unify({ + sourceObject, + targetType: commonObject as TargetObject, + providerName: integrationId, + vertical: vertical, + connectionId: connectionId, + customFieldMappings, + })) as T[]; + + const { linkedUserId, projectId } = + await this.connectionUtils.getConnectionMetadataFromConnectionId( + connectionId, + ); + + // insert the data in the DB with the fieldMappings (value table) + const data = await this.syncRegistry + .getService(vertical, commonObject) + .saveToDb( + connectionId, + linkedUserId, + unifiedObject, + integrationId, + sourceObject, + ); + const event = await this.prisma.events.create({ + data: { + id_event: uuidv4(), + status: 'success', + type: `${vertical}.${commonObject}.synced`, + method: 'SYNC', + url: '/sync', + provider: integrationId, + direction: '0', + timestamp: new Date(), + id_linked_user: linkedUserId, + }, + }); + await this.webhook.dispatchWebhook( + data, + `${vertical}.${commonObject}.pulled`, + projectId, + event.id_event, + ); + return data; + } +} diff --git a/packages/api/src/@core/@core-services/webhooks/panora-webhooks/webhook.processor.ts b/packages/api/src/@core/@core-services/webhooks/panora-webhooks/webhook.processor.ts index d9fd9213f..a6c5f2f40 100644 --- a/packages/api/src/@core/@core-services/webhooks/panora-webhooks/webhook.processor.ts +++ b/packages/api/src/@core/@core-services/webhooks/panora-webhooks/webhook.processor.ts @@ -92,7 +92,7 @@ export class WebhookProcessor { }); //re-insert the webhook in the queue - await this.webhookService.handleFailedWebhook(id_webhook_delivery); + await this.webhookService.dispatchFailedWebhook(id_webhook_delivery); this.logger.log( 'Webhook delivery failed. Job reinserted in the queue for retry.', diff --git a/packages/api/src/@core/@core-services/webhooks/panora-webhooks/webhook.service.ts b/packages/api/src/@core/@core-services/webhooks/panora-webhooks/webhook.service.ts index da35a4634..66c7c67aa 100644 --- a/packages/api/src/@core/@core-services/webhooks/panora-webhooks/webhook.service.ts +++ b/packages/api/src/@core/@core-services/webhooks/panora-webhooks/webhook.service.ts @@ -83,7 +83,7 @@ export class WebhookService { } } - async handleWebhook( + async dispatchWebhook( data: any, eventType: string, projectId: string, @@ -144,7 +144,7 @@ export class WebhookService { } } - async handlePriorityWebhook( + async deliverWebhook( data: any, eventType: string, projectId: string, @@ -258,7 +258,7 @@ export class WebhookService { }); //re-insert the webhook in the queue - await this.handleFailedWebhook( + await this.dispatchFailedWebhook( w_delivery.id_webhook_delivery_attempt, ); } @@ -268,7 +268,7 @@ export class WebhookService { } } - async handleFailedWebhook(failed_id_delivery_webhook: string) { + async dispatchFailedWebhook(failed_id_delivery_webhook: string) { try { await this.queues.getPanoraWebhookSender().add( { diff --git a/packages/api/src/@core/connections/@utils/index.ts b/packages/api/src/@core/connections/@utils/index.ts index 6684f3ad2..e0aaf1075 100644 --- a/packages/api/src/@core/connections/@utils/index.ts +++ b/packages/api/src/@core/connections/@utils/index.ts @@ -33,6 +33,20 @@ export class ConnectionUtils { } } + async getConnectionMetadataFromConnectionId(uuid: string) { + try { + const conn = await this.prisma.connections.findUnique({ + where: { + id_connection: uuid, + }, + }); + return { + linkedUserId: conn.id_linked_user, + projectId: conn.id_project, + }; + } catch (error) {} + } + async getLinkedUserId( projectId: string, linkedUserId: string, diff --git a/packages/api/src/@core/connections/accounting/services/accounting.connection.service.ts b/packages/api/src/@core/connections/accounting/services/accounting.connection.service.ts index 7adf2a783..813a6087e 100644 --- a/packages/api/src/@core/connections/accounting/services/accounting.connection.service.ts +++ b/packages/api/src/@core/connections/accounting/services/accounting.connection.service.ts @@ -66,7 +66,7 @@ export class AccountingConnectionsService implements IConnectionCategory { }, }); //directly send the webhook - await this.webhook.handlePriorityWebhook( + await this.webhook.deliverWebhook( data, 'connection.created', callbackOpts.projectId, diff --git a/packages/api/src/@core/connections/ats/services/ats.connection.service.ts b/packages/api/src/@core/connections/ats/services/ats.connection.service.ts index 3097379b4..444e43038 100644 --- a/packages/api/src/@core/connections/ats/services/ats.connection.service.ts +++ b/packages/api/src/@core/connections/ats/services/ats.connection.service.ts @@ -66,7 +66,7 @@ export class AtsConnectionsService implements IConnectionCategory { }, }); //directly send the webhook - await this.webhook.handlePriorityWebhook( + await this.webhook.deliverWebhook( data, 'connection.created', callbackOpts.projectId, diff --git a/packages/api/src/@core/connections/crm/services/crm.connection.service.ts b/packages/api/src/@core/connections/crm/services/crm.connection.service.ts index f60cb0c11..24da89f42 100644 --- a/packages/api/src/@core/connections/crm/services/crm.connection.service.ts +++ b/packages/api/src/@core/connections/crm/services/crm.connection.service.ts @@ -53,7 +53,7 @@ export class CrmConnectionsService implements IConnectionCategory { }, }); //directly send the webhook - await this.webhook.handlePriorityWebhook( + await this.webhook.deliverWebhook( data, 'connection.created', callbackOpts.projectId, diff --git a/packages/api/src/@core/connections/filestorage/services/filestorage.connection.service.ts b/packages/api/src/@core/connections/filestorage/services/filestorage.connection.service.ts index 4f44b64e9..d88445993 100644 --- a/packages/api/src/@core/connections/filestorage/services/filestorage.connection.service.ts +++ b/packages/api/src/@core/connections/filestorage/services/filestorage.connection.service.ts @@ -67,7 +67,7 @@ export class FilestorageConnectionsService implements IConnectionCategory { }, }); //directly send the webhook - await this.webhook.handlePriorityWebhook( + await this.webhook.deliverWebhook( data, 'connection.created', callbackOpts.projectId, diff --git a/packages/api/src/@core/connections/hris/services/hris.connection.service.ts b/packages/api/src/@core/connections/hris/services/hris.connection.service.ts index 6b1cdeb67..8c66a6405 100644 --- a/packages/api/src/@core/connections/hris/services/hris.connection.service.ts +++ b/packages/api/src/@core/connections/hris/services/hris.connection.service.ts @@ -67,7 +67,7 @@ export class HrisConnectionsService implements IConnectionCategory { }, }); //directly send the webhook - await this.webhook.handlePriorityWebhook( + await this.webhook.deliverWebhook( data, 'connection.created', callbackOpts.projectId, diff --git a/packages/api/src/@core/connections/management/services/management.connection.service.ts b/packages/api/src/@core/connections/management/services/management.connection.service.ts index a7d0a39d2..0daef8bce 100644 --- a/packages/api/src/@core/connections/management/services/management.connection.service.ts +++ b/packages/api/src/@core/connections/management/services/management.connection.service.ts @@ -66,7 +66,7 @@ export class ManagementConnectionsService implements IConnectionCategory { }, }); //directly send the webhook - await this.webhook.handlePriorityWebhook( + await this.webhook.deliverWebhook( data, 'connection.created', callbackOpts.projectId, diff --git a/packages/api/src/@core/connections/marketingautomation/services/marketingautomation.connection.service.ts b/packages/api/src/@core/connections/marketingautomation/services/marketingautomation.connection.service.ts index fb2822aac..92220414c 100644 --- a/packages/api/src/@core/connections/marketingautomation/services/marketingautomation.connection.service.ts +++ b/packages/api/src/@core/connections/marketingautomation/services/marketingautomation.connection.service.ts @@ -71,7 +71,7 @@ export class MarketingAutomationConnectionsService }, }); //directly send the webhook - await this.webhook.handlePriorityWebhook( + await this.webhook.deliverWebhook( data, 'connection.created', callbackOpts.projectId, diff --git a/packages/api/src/@core/connections/ticketing/services/ticketing.connection.service.ts b/packages/api/src/@core/connections/ticketing/services/ticketing.connection.service.ts index 658a2447d..80b264adf 100644 --- a/packages/api/src/@core/connections/ticketing/services/ticketing.connection.service.ts +++ b/packages/api/src/@core/connections/ticketing/services/ticketing.connection.service.ts @@ -66,7 +66,7 @@ export class TicketingConnectionsService implements IConnectionCategory { }, }); //directly send the webhook - await this.webhook.handlePriorityWebhook( + await this.webhook.deliverWebhook( data, 'connection.created', callbackOpts.projectId, diff --git a/packages/api/src/@core/core.module.ts b/packages/api/src/@core/core.module.ts index 7b9b0d884..feedf7e21 100644 --- a/packages/api/src/@core/core.module.ts +++ b/packages/api/src/@core/core.module.ts @@ -22,6 +22,7 @@ import { BullQueueModule } from './@core-services/queues/queue.module'; import { WebhookModule } from './@core-services/webhooks/panora-webhooks/webhook.module'; import { ManagedWebhooksModule } from './@core-services/webhooks/third-parties-webhooks/managed-webhooks.module'; import { CategoryConnectionRegistry } from './@core-services/registries/connections-categories.registry'; +import { IngestDataService } from './@core-services/unification/ingest-data.service'; @Module({ imports: [ @@ -71,6 +72,7 @@ import { CategoryConnectionRegistry } from './@core-services/registries/connecti UnificationRegistry, CoreUnification, CoreSyncRegistry, + IngestDataService, CategoryConnectionRegistry, ], }) diff --git a/packages/api/src/@core/utils/types/interface.ts b/packages/api/src/@core/utils/types/interface.ts index 6113ede9c..2e4b71316 100644 --- a/packages/api/src/@core/utils/types/interface.ts +++ b/packages/api/src/@core/utils/types/interface.ts @@ -35,3 +35,13 @@ export interface IUnification { }[]; }): Promise; } + +export interface IBaseSync { + saveToDb( + connection_id: string, + linkedUserId: string, + data: any[], + originSource: string, + remote_data: Record[], + ): Promise; +} diff --git a/packages/api/src/accounting/account/account.module.ts b/packages/api/src/accounting/account/account.module.ts index cf6b4f9cc..e8378159a 100644 --- a/packages/api/src/accounting/account/account.module.ts +++ b/packages/api/src/accounting/account/account.module.ts @@ -8,6 +8,7 @@ import { AccountController } from './account.controller'; import { AccountService } from './services/account.service'; import { ServiceRegistry } from './services/registry.service'; import { SyncService } from './sync/sync.service'; +import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; @Module({ controllers: [AccountController], @@ -20,6 +21,7 @@ import { SyncService } from './sync/sync.service'; FieldMappingService, ServiceRegistry, ConnectionUtils, + IngestDataService, /* PROVIDERS SERVICES */ ], exports: [SyncService], diff --git a/packages/api/src/accounting/account/sync/sync.service.ts b/packages/api/src/accounting/account/sync/sync.service.ts index 7fbfd3b01..f877c410d 100644 --- a/packages/api/src/accounting/account/sync/sync.service.ts +++ b/packages/api/src/accounting/account/sync/sync.service.ts @@ -6,7 +6,7 @@ import { Injectable, OnModuleInit } from '@nestjs/common'; import { ServiceRegistry } from '../services/registry.service'; @Injectable() -export class SyncService implements OnModuleInit { +export class SyncService implements OnModuleInit, IBaseSync { constructor( private prisma: PrismaService, private logger: LoggerService, diff --git a/packages/api/src/accounting/address/address.module.ts b/packages/api/src/accounting/address/address.module.ts index d1f4195ac..6d468460e 100644 --- a/packages/api/src/accounting/address/address.module.ts +++ b/packages/api/src/accounting/address/address.module.ts @@ -8,6 +8,7 @@ import { AddressController } from './address.controller'; import { AddressService } from './services/address.service'; import { ServiceRegistry } from './services/registry.service'; import { SyncService } from './sync/sync.service'; +import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; @Module({ controllers: [AddressController], @@ -20,6 +21,7 @@ import { SyncService } from './sync/sync.service'; FieldMappingService, ServiceRegistry, ConnectionUtils, + IngestDataService, /* PROVIDERS SERVICES */ ], exports: [SyncService], diff --git a/packages/api/src/accounting/address/sync/sync.service.ts b/packages/api/src/accounting/address/sync/sync.service.ts index 2dbadbcd8..12ec8cfee 100644 --- a/packages/api/src/accounting/address/sync/sync.service.ts +++ b/packages/api/src/accounting/address/sync/sync.service.ts @@ -11,7 +11,7 @@ import { UnifiedAddressOutput } from '../types/model.unified'; import { IAddressService } from '../types'; @Injectable() -export class SyncService implements OnModuleInit { +export class SyncService implements OnModuleInit, IBaseSync { constructor( private prisma: PrismaService, private logger: LoggerService, diff --git a/packages/api/src/accounting/attachment/attachment.module.ts b/packages/api/src/accounting/attachment/attachment.module.ts index c839c3059..2ae28a2c6 100644 --- a/packages/api/src/accounting/attachment/attachment.module.ts +++ b/packages/api/src/accounting/attachment/attachment.module.ts @@ -8,6 +8,7 @@ import { AttachmentController } from './attachment.controller'; import { AttachmentService } from './services/attachment.service'; import { ServiceRegistry } from './services/registry.service'; import { SyncService } from './sync/sync.service'; +import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; @Module({ controllers: [AttachmentController], @@ -20,6 +21,7 @@ import { SyncService } from './sync/sync.service'; FieldMappingService, ServiceRegistry, ConnectionUtils, + IngestDataService, /* PROVIDERS SERVICES */ ], exports: [SyncService], diff --git a/packages/api/src/accounting/attachment/sync/sync.service.ts b/packages/api/src/accounting/attachment/sync/sync.service.ts index 7b55ee5a1..0cb270838 100644 --- a/packages/api/src/accounting/attachment/sync/sync.service.ts +++ b/packages/api/src/accounting/attachment/sync/sync.service.ts @@ -11,7 +11,7 @@ import { UnifiedAttachmentOutput } from '../types/model.unified'; import { IAttachmentService } from '../types'; @Injectable() -export class SyncService implements OnModuleInit { +export class SyncService implements OnModuleInit, IBaseSync { constructor( private prisma: PrismaService, private logger: LoggerService, diff --git a/packages/api/src/accounting/balancesheet/balancesheet.module.ts b/packages/api/src/accounting/balancesheet/balancesheet.module.ts index 0667f5cd5..fa62468b8 100644 --- a/packages/api/src/accounting/balancesheet/balancesheet.module.ts +++ b/packages/api/src/accounting/balancesheet/balancesheet.module.ts @@ -8,6 +8,7 @@ import { BalanceSheetController } from './balancesheet.controller'; import { BalanceSheetService } from './services/balancesheet.service'; import { ServiceRegistry } from './services/registry.service'; import { SyncService } from './sync/sync.service'; +import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; @Module({ controllers: [BalanceSheetController], @@ -20,6 +21,7 @@ import { SyncService } from './sync/sync.service'; FieldMappingService, ServiceRegistry, ConnectionUtils, + IngestDataService, /* PROVIDERS SERVICES */ ], exports: [SyncService], diff --git a/packages/api/src/accounting/balancesheet/sync/sync.service.ts b/packages/api/src/accounting/balancesheet/sync/sync.service.ts index 1abfd7d21..cfb77fa10 100644 --- a/packages/api/src/accounting/balancesheet/sync/sync.service.ts +++ b/packages/api/src/accounting/balancesheet/sync/sync.service.ts @@ -11,7 +11,7 @@ import { UnifiedBalanceSheetOutput } from '../types/model.unified'; import { IBalanceSheetService } from '../types'; @Injectable() -export class SyncService implements OnModuleInit { +export class SyncService implements OnModuleInit, IBaseSync { constructor( private prisma: PrismaService, private logger: LoggerService, diff --git a/packages/api/src/accounting/cashflowstatement/cashflowstatement.module.ts b/packages/api/src/accounting/cashflowstatement/cashflowstatement.module.ts index 85a82d58d..a05f81053 100644 --- a/packages/api/src/accounting/cashflowstatement/cashflowstatement.module.ts +++ b/packages/api/src/accounting/cashflowstatement/cashflowstatement.module.ts @@ -8,6 +8,7 @@ import { CashflowStatementController } from './cashflowstatement.controller'; import { CashflowStatementService } from './services/cashflowstatement.service'; import { ServiceRegistry } from './services/registry.service'; import { SyncService } from './sync/sync.service'; +import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; @Module({ controllers: [CashflowStatementController], @@ -20,6 +21,7 @@ import { SyncService } from './sync/sync.service'; FieldMappingService, ServiceRegistry, ConnectionUtils, + IngestDataService, /* PROVIDERS SERVICES */ ], exports: [SyncService], diff --git a/packages/api/src/accounting/cashflowstatement/sync/sync.service.ts b/packages/api/src/accounting/cashflowstatement/sync/sync.service.ts index 33891e4db..9737e0a50 100644 --- a/packages/api/src/accounting/cashflowstatement/sync/sync.service.ts +++ b/packages/api/src/accounting/cashflowstatement/sync/sync.service.ts @@ -11,7 +11,7 @@ import { UnifiedCashflowStatementOutput } from '../types/model.unified'; import { ICashflowStatementService } from '../types'; @Injectable() -export class SyncService implements OnModuleInit { +export class SyncService implements OnModuleInit, IBaseSync { constructor( private prisma: PrismaService, private logger: LoggerService, diff --git a/packages/api/src/accounting/companyinfo/companyinfo.module.ts b/packages/api/src/accounting/companyinfo/companyinfo.module.ts index e252f50d1..ef48495a9 100644 --- a/packages/api/src/accounting/companyinfo/companyinfo.module.ts +++ b/packages/api/src/accounting/companyinfo/companyinfo.module.ts @@ -8,6 +8,7 @@ import { CompanyInfoController } from './companyinfo.controller'; import { CompanyInfoService } from './services/companyinfo.service'; import { ServiceRegistry } from './services/registry.service'; import { SyncService } from './sync/sync.service'; +import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; @Module({ controllers: [CompanyInfoController], @@ -20,6 +21,7 @@ import { SyncService } from './sync/sync.service'; FieldMappingService, ServiceRegistry, ConnectionUtils, + IngestDataService, /* PROVIDERS SERVICES */ ], exports: [SyncService], diff --git a/packages/api/src/accounting/companyinfo/sync/sync.service.ts b/packages/api/src/accounting/companyinfo/sync/sync.service.ts index 9ce45199c..405f060f4 100644 --- a/packages/api/src/accounting/companyinfo/sync/sync.service.ts +++ b/packages/api/src/accounting/companyinfo/sync/sync.service.ts @@ -12,7 +12,7 @@ import { UnifiedCompanyInfoOutput } from '../types/model.unified'; import { ICompanyInfoService } from '../types'; @Injectable() -export class SyncService implements OnModuleInit { +export class SyncService implements OnModuleInit, IBaseSync { constructor( private prisma: PrismaService, private logger: LoggerService, diff --git a/packages/api/src/accounting/contact/contact.module.ts b/packages/api/src/accounting/contact/contact.module.ts index 28b1ae71b..76e432ad7 100644 --- a/packages/api/src/accounting/contact/contact.module.ts +++ b/packages/api/src/accounting/contact/contact.module.ts @@ -8,6 +8,7 @@ import { ContactController } from './contact.controller'; import { ContactService } from './services/contact.service'; import { ServiceRegistry } from './services/registry.service'; import { SyncService } from './sync/sync.service'; +import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; @Module({ controllers: [ContactController], @@ -20,6 +21,7 @@ import { SyncService } from './sync/sync.service'; FieldMappingService, ServiceRegistry, ConnectionUtils, + IngestDataService, /* PROVIDERS SERVICES */ ], exports: [SyncService], diff --git a/packages/api/src/accounting/contact/sync/sync.service.ts b/packages/api/src/accounting/contact/sync/sync.service.ts index 733884a8c..e4711bdc9 100644 --- a/packages/api/src/accounting/contact/sync/sync.service.ts +++ b/packages/api/src/accounting/contact/sync/sync.service.ts @@ -12,7 +12,7 @@ import { UnifiedContactOutput } from '../types/model.unified'; import { IContactService } from '../types'; @Injectable() -export class SyncService implements OnModuleInit { +export class SyncService implements OnModuleInit, IBaseSync { constructor( private prisma: PrismaService, private logger: LoggerService, diff --git a/packages/api/src/accounting/creditnote/creditnote.module.ts b/packages/api/src/accounting/creditnote/creditnote.module.ts index 4b984a315..ca0e138e7 100644 --- a/packages/api/src/accounting/creditnote/creditnote.module.ts +++ b/packages/api/src/accounting/creditnote/creditnote.module.ts @@ -8,6 +8,7 @@ import { CreditNoteController } from './creditnote.controller'; import { CreditNoteService } from './services/creditnote.service'; import { ServiceRegistry } from './services/registry.service'; import { SyncService } from './sync/sync.service'; +import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; @Module({ controllers: [CreditNoteController], @@ -20,6 +21,7 @@ import { SyncService } from './sync/sync.service'; FieldMappingService, ServiceRegistry, ConnectionUtils, + IngestDataService, /* PROVIDERS SERVICES */ ], exports: [SyncService], diff --git a/packages/api/src/accounting/creditnote/sync/sync.service.ts b/packages/api/src/accounting/creditnote/sync/sync.service.ts index 37aeca927..776194e10 100644 --- a/packages/api/src/accounting/creditnote/sync/sync.service.ts +++ b/packages/api/src/accounting/creditnote/sync/sync.service.ts @@ -12,7 +12,7 @@ import { UnifiedCreditNoteOutput } from '../types/model.unified'; import { ICreditNoteService } from '../types'; @Injectable() -export class SyncService implements OnModuleInit { +export class SyncService implements OnModuleInit, IBaseSync { constructor( private prisma: PrismaService, private logger: LoggerService, diff --git a/packages/api/src/accounting/expense/expense.module.ts b/packages/api/src/accounting/expense/expense.module.ts index fdaac0853..ef1d40ac3 100644 --- a/packages/api/src/accounting/expense/expense.module.ts +++ b/packages/api/src/accounting/expense/expense.module.ts @@ -8,6 +8,7 @@ import { ExpenseController } from './expense.controller'; import { ExpenseService } from './services/expense.service'; import { ServiceRegistry } from './services/registry.service'; import { SyncService } from './sync/sync.service'; +import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; @Module({ controllers: [ExpenseController], @@ -20,6 +21,7 @@ import { SyncService } from './sync/sync.service'; FieldMappingService, ServiceRegistry, ConnectionUtils, + IngestDataService, /* PROVIDERS SERVICES */ ], exports: [SyncService], diff --git a/packages/api/src/accounting/expense/sync/sync.service.ts b/packages/api/src/accounting/expense/sync/sync.service.ts index 1cc72f79f..b3ca4489e 100644 --- a/packages/api/src/accounting/expense/sync/sync.service.ts +++ b/packages/api/src/accounting/expense/sync/sync.service.ts @@ -12,7 +12,7 @@ import { UnifiedExpenseOutput } from '../types/model.unified'; import { IExpenseService } from '../types'; @Injectable() -export class SyncService implements OnModuleInit { +export class SyncService implements OnModuleInit, IBaseSync { constructor( private prisma: PrismaService, private logger: LoggerService, diff --git a/packages/api/src/accounting/incomestatement/incomestatement.module.ts b/packages/api/src/accounting/incomestatement/incomestatement.module.ts index 08931362e..3c5fe5fee 100644 --- a/packages/api/src/accounting/incomestatement/incomestatement.module.ts +++ b/packages/api/src/accounting/incomestatement/incomestatement.module.ts @@ -8,6 +8,7 @@ import { IncomeStatementController } from './incomestatement.controller'; import { IncomeStatementService } from './services/incomestatement.service'; import { ServiceRegistry } from './services/registry.service'; import { SyncService } from './sync/sync.service'; +import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; @Module({ controllers: [IncomeStatementController], @@ -20,6 +21,7 @@ import { SyncService } from './sync/sync.service'; FieldMappingService, ServiceRegistry, ConnectionUtils, + IngestDataService, /* PROVIDERS SERVICES */ ], exports: [SyncService], diff --git a/packages/api/src/accounting/incomestatement/sync/sync.service.ts b/packages/api/src/accounting/incomestatement/sync/sync.service.ts index f8f24026c..237a8b8bf 100644 --- a/packages/api/src/accounting/incomestatement/sync/sync.service.ts +++ b/packages/api/src/accounting/incomestatement/sync/sync.service.ts @@ -12,7 +12,7 @@ import { UnifiedIncomeStatementOutput } from '../types/model.unified'; import { IIncomeStatementService } from '../types'; @Injectable() -export class SyncService implements OnModuleInit { +export class SyncService implements OnModuleInit, IBaseSync { constructor( private prisma: PrismaService, private logger: LoggerService, diff --git a/packages/api/src/accounting/invoice/invoice.module.ts b/packages/api/src/accounting/invoice/invoice.module.ts index 75aaa82ca..f0ff07bc2 100644 --- a/packages/api/src/accounting/invoice/invoice.module.ts +++ b/packages/api/src/accounting/invoice/invoice.module.ts @@ -8,6 +8,7 @@ import { InvoiceController } from './invoice.controller'; import { InvoiceService } from './services/invoice.service'; import { ServiceRegistry } from './services/registry.service'; import { SyncService } from './sync/sync.service'; +import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; @Module({ controllers: [InvoiceController], @@ -20,6 +21,7 @@ import { SyncService } from './sync/sync.service'; FieldMappingService, ServiceRegistry, ConnectionUtils, + IngestDataService, /* PROVIDERS SERVICES */ ], exports: [SyncService], diff --git a/packages/api/src/accounting/invoice/sync/sync.service.ts b/packages/api/src/accounting/invoice/sync/sync.service.ts index 45c24dd98..ec9199127 100644 --- a/packages/api/src/accounting/invoice/sync/sync.service.ts +++ b/packages/api/src/accounting/invoice/sync/sync.service.ts @@ -12,7 +12,7 @@ import { UnifiedInvoiceOutput } from '../types/model.unified'; import { IInvoiceService } from '../types'; @Injectable() -export class SyncService implements OnModuleInit { +export class SyncService implements OnModuleInit, IBaseSync { constructor( private prisma: PrismaService, private logger: LoggerService, diff --git a/packages/api/src/accounting/item/item.module.ts b/packages/api/src/accounting/item/item.module.ts index f04a9a021..e8ba7f1d4 100644 --- a/packages/api/src/accounting/item/item.module.ts +++ b/packages/api/src/accounting/item/item.module.ts @@ -8,6 +8,7 @@ import { ItemController } from './item.controller'; import { ItemService } from './services/item.service'; import { ServiceRegistry } from './services/registry.service'; import { SyncService } from './sync/sync.service'; +import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; @Module({ controllers: [ItemController], @@ -20,6 +21,7 @@ import { SyncService } from './sync/sync.service'; FieldMappingService, ServiceRegistry, ConnectionUtils, + IngestDataService, /* PROVIDERS SERVICES */ ], exports: [SyncService], diff --git a/packages/api/src/accounting/item/sync/sync.service.ts b/packages/api/src/accounting/item/sync/sync.service.ts index efee84c37..a01553c30 100644 --- a/packages/api/src/accounting/item/sync/sync.service.ts +++ b/packages/api/src/accounting/item/sync/sync.service.ts @@ -12,7 +12,7 @@ import { UnifiedItemOutput } from '../types/model.unified'; import { IItemService } from '../types'; @Injectable() -export class SyncService implements OnModuleInit { +export class SyncService implements OnModuleInit, IBaseSync { constructor( private prisma: PrismaService, private logger: LoggerService, diff --git a/packages/api/src/accounting/journalentry/journalentry.module.ts b/packages/api/src/accounting/journalentry/journalentry.module.ts index 3068df33f..4ed4b2789 100644 --- a/packages/api/src/accounting/journalentry/journalentry.module.ts +++ b/packages/api/src/accounting/journalentry/journalentry.module.ts @@ -8,6 +8,7 @@ import { JournalEntryController } from './journalentry.controller'; import { JournalEntryService } from './services/journalentry.service'; import { ServiceRegistry } from './services/registry.service'; import { SyncService } from './sync/sync.service'; +import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; @Module({ controllers: [JournalEntryController], @@ -20,6 +21,7 @@ import { SyncService } from './sync/sync.service'; FieldMappingService, ServiceRegistry, ConnectionUtils, + IngestDataService, /* PROVIDERS SERVICES */ ], exports: [SyncService], diff --git a/packages/api/src/accounting/journalentry/sync/sync.service.ts b/packages/api/src/accounting/journalentry/sync/sync.service.ts index e3768b786..3571e1526 100644 --- a/packages/api/src/accounting/journalentry/sync/sync.service.ts +++ b/packages/api/src/accounting/journalentry/sync/sync.service.ts @@ -12,7 +12,7 @@ import { UnifiedJournalEntryOutput } from '../types/model.unified'; import { IJournalEntryService } from '../types'; @Injectable() -export class SyncService implements OnModuleInit { +export class SyncService implements OnModuleInit, IBaseSync { constructor( private prisma: PrismaService, private logger: LoggerService, diff --git a/packages/api/src/accounting/payment/payment.module.ts b/packages/api/src/accounting/payment/payment.module.ts index 262bf0be4..352339c84 100644 --- a/packages/api/src/accounting/payment/payment.module.ts +++ b/packages/api/src/accounting/payment/payment.module.ts @@ -8,6 +8,7 @@ import { PaymentController } from './payment.controller'; import { PaymentService } from './services/payment.service'; import { ServiceRegistry } from './services/registry.service'; import { SyncService } from './sync/sync.service'; +import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; @Module({ controllers: [PaymentController], @@ -20,6 +21,7 @@ import { SyncService } from './sync/sync.service'; FieldMappingService, ServiceRegistry, ConnectionUtils, + IngestDataService, /* PROVIDERS SERVICES */ ], exports: [SyncService], diff --git a/packages/api/src/accounting/payment/sync/sync.service.ts b/packages/api/src/accounting/payment/sync/sync.service.ts index 5610ecf25..b9a2b93d3 100644 --- a/packages/api/src/accounting/payment/sync/sync.service.ts +++ b/packages/api/src/accounting/payment/sync/sync.service.ts @@ -12,7 +12,7 @@ import { UnifiedPaymentOutput } from '../types/model.unified'; import { IPaymentService } from '../types'; @Injectable() -export class SyncService implements OnModuleInit { +export class SyncService implements OnModuleInit, IBaseSync { constructor( private prisma: PrismaService, private logger: LoggerService, diff --git a/packages/api/src/accounting/phonenumber/phonenumber.module.ts b/packages/api/src/accounting/phonenumber/phonenumber.module.ts index 5c17b3be8..a8d67722c 100644 --- a/packages/api/src/accounting/phonenumber/phonenumber.module.ts +++ b/packages/api/src/accounting/phonenumber/phonenumber.module.ts @@ -8,6 +8,7 @@ import { PhoneNumberController } from './phonenumber.controller'; import { PhoneNumberService } from './services/phonenumber.service'; import { ServiceRegistry } from './services/registry.service'; import { SyncService } from './sync/sync.service'; +import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; @Module({ controllers: [PhoneNumberController], @@ -20,6 +21,7 @@ import { SyncService } from './sync/sync.service'; FieldMappingService, ServiceRegistry, ConnectionUtils, + IngestDataService, /* PROVIDERS SERVICES */ ], exports: [SyncService], diff --git a/packages/api/src/accounting/phonenumber/sync/sync.service.ts b/packages/api/src/accounting/phonenumber/sync/sync.service.ts index 7a70c8b1f..b0cb356bc 100644 --- a/packages/api/src/accounting/phonenumber/sync/sync.service.ts +++ b/packages/api/src/accounting/phonenumber/sync/sync.service.ts @@ -12,7 +12,7 @@ import { UnifiedPhoneNumberOutput } from '../types/model.unified'; import { IPhoneNumberService } from '../types'; @Injectable() -export class SyncService implements OnModuleInit { +export class SyncService implements OnModuleInit, IBaseSync { constructor( private prisma: PrismaService, private logger: LoggerService, diff --git a/packages/api/src/accounting/purchaseorder/purchaseorder.module.ts b/packages/api/src/accounting/purchaseorder/purchaseorder.module.ts index 6655b4bc7..80b737474 100644 --- a/packages/api/src/accounting/purchaseorder/purchaseorder.module.ts +++ b/packages/api/src/accounting/purchaseorder/purchaseorder.module.ts @@ -8,6 +8,7 @@ import { PurchaseOrderController } from './purchaseorder.controller'; import { PurchaseOrderService } from './services/purchaseorder.service'; import { ServiceRegistry } from './services/registry.service'; import { SyncService } from './sync/sync.service'; +import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; @Module({ controllers: [PurchaseOrderController], @@ -20,6 +21,7 @@ import { SyncService } from './sync/sync.service'; FieldMappingService, ServiceRegistry, ConnectionUtils, + IngestDataService, /* PROVIDERS SERVICES */ ], exports: [SyncService], diff --git a/packages/api/src/accounting/purchaseorder/sync/sync.service.ts b/packages/api/src/accounting/purchaseorder/sync/sync.service.ts index 3f60f7df4..b644a56eb 100644 --- a/packages/api/src/accounting/purchaseorder/sync/sync.service.ts +++ b/packages/api/src/accounting/purchaseorder/sync/sync.service.ts @@ -12,7 +12,7 @@ import { UnifiedPurchaseOrderOutput } from '../types/model.unified'; import { IPurchaseOrderService } from '../types'; @Injectable() -export class SyncService implements OnModuleInit { +export class SyncService implements OnModuleInit, IBaseSync { constructor( private prisma: PrismaService, private logger: LoggerService, diff --git a/packages/api/src/accounting/taxrate/sync/sync.service.ts b/packages/api/src/accounting/taxrate/sync/sync.service.ts index 1b716f995..e60b0cd13 100644 --- a/packages/api/src/accounting/taxrate/sync/sync.service.ts +++ b/packages/api/src/accounting/taxrate/sync/sync.service.ts @@ -12,7 +12,7 @@ import { UnifiedTaxRateOutput } from '../types/model.unified'; import { ITaxRateService } from '../types'; @Injectable() -export class SyncService implements OnModuleInit { +export class SyncService implements OnModuleInit, IBaseSync { constructor( private prisma: PrismaService, private logger: LoggerService, diff --git a/packages/api/src/accounting/taxrate/taxrate.module.ts b/packages/api/src/accounting/taxrate/taxrate.module.ts index b6ac22751..b30a23bf9 100644 --- a/packages/api/src/accounting/taxrate/taxrate.module.ts +++ b/packages/api/src/accounting/taxrate/taxrate.module.ts @@ -8,6 +8,7 @@ import { ServiceRegistry } from './services/registry.service'; import { TaxRateService } from './services/taxrate.service'; import { SyncService } from './sync/sync.service'; import { TaxRateController } from './taxrate.controller'; +import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; @Module({ controllers: [TaxRateController], @@ -20,6 +21,7 @@ import { TaxRateController } from './taxrate.controller'; FieldMappingService, ServiceRegistry, ConnectionUtils, + IngestDataService, /* PROVIDERS SERVICES */ ], exports: [SyncService], diff --git a/packages/api/src/accounting/trackingcategory/sync/sync.service.ts b/packages/api/src/accounting/trackingcategory/sync/sync.service.ts index af785ae91..310bbd8ab 100644 --- a/packages/api/src/accounting/trackingcategory/sync/sync.service.ts +++ b/packages/api/src/accounting/trackingcategory/sync/sync.service.ts @@ -12,7 +12,7 @@ import { UnifiedTrackingCategoryOutput } from '../types/model.unified'; import { ITrackingCategoryService } from '../types'; @Injectable() -export class SyncService implements OnModuleInit { +export class SyncService implements OnModuleInit, IBaseSync { constructor( private prisma: PrismaService, private logger: LoggerService, diff --git a/packages/api/src/accounting/trackingcategory/trackingcategory.module.ts b/packages/api/src/accounting/trackingcategory/trackingcategory.module.ts index bd02eb636..18632db51 100644 --- a/packages/api/src/accounting/trackingcategory/trackingcategory.module.ts +++ b/packages/api/src/accounting/trackingcategory/trackingcategory.module.ts @@ -8,6 +8,7 @@ import { ServiceRegistry } from './services/registry.service'; import { TrackingCategoryService } from './services/trackingcategory.service'; import { SyncService } from './sync/sync.service'; import { TrackingCategoryController } from './trackingcategory.controller'; +import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; @Module({ controllers: [TrackingCategoryController], @@ -20,6 +21,7 @@ import { TrackingCategoryController } from './trackingcategory.controller'; FieldMappingService, ServiceRegistry, ConnectionUtils, + IngestDataService, /* PROVIDERS SERVICES */ ], exports: [SyncService], diff --git a/packages/api/src/accounting/transaction/sync/sync.service.ts b/packages/api/src/accounting/transaction/sync/sync.service.ts index 3831ae446..eb8991140 100644 --- a/packages/api/src/accounting/transaction/sync/sync.service.ts +++ b/packages/api/src/accounting/transaction/sync/sync.service.ts @@ -12,7 +12,7 @@ import { UnifiedTransactionOutput } from '../types/model.unified'; import { ITransactionService } from '../types'; @Injectable() -export class SyncService implements OnModuleInit { +export class SyncService implements OnModuleInit, IBaseSync { constructor( private prisma: PrismaService, private logger: LoggerService, diff --git a/packages/api/src/accounting/transaction/transaction.module.ts b/packages/api/src/accounting/transaction/transaction.module.ts index 0bf4161d8..f33c4e486 100644 --- a/packages/api/src/accounting/transaction/transaction.module.ts +++ b/packages/api/src/accounting/transaction/transaction.module.ts @@ -8,6 +8,7 @@ import { ServiceRegistry } from './services/registry.service'; import { TransactionService } from './services/transaction.service'; import { SyncService } from './sync/sync.service'; import { TransactionController } from './transaction.controller'; +import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; @Module({ controllers: [TransactionController], @@ -20,6 +21,7 @@ import { TransactionController } from './transaction.controller'; FieldMappingService, ServiceRegistry, ConnectionUtils, + IngestDataService, /* PROVIDERS SERVICES */ ], exports: [SyncService], diff --git a/packages/api/src/accounting/vendorcredit/sync/sync.service.ts b/packages/api/src/accounting/vendorcredit/sync/sync.service.ts index f5910749b..0a3cb248c 100644 --- a/packages/api/src/accounting/vendorcredit/sync/sync.service.ts +++ b/packages/api/src/accounting/vendorcredit/sync/sync.service.ts @@ -12,7 +12,7 @@ import { UnifiedVendorCreditOutput } from '../types/model.unified'; import { IVendorCreditService } from '../types'; @Injectable() -export class SyncService implements OnModuleInit { +export class SyncService implements OnModuleInit, IBaseSync { constructor( private prisma: PrismaService, private logger: LoggerService, diff --git a/packages/api/src/accounting/vendorcredit/vendorcredit.module.ts b/packages/api/src/accounting/vendorcredit/vendorcredit.module.ts index ad1f6c3bc..0f5ebfae5 100644 --- a/packages/api/src/accounting/vendorcredit/vendorcredit.module.ts +++ b/packages/api/src/accounting/vendorcredit/vendorcredit.module.ts @@ -8,6 +8,7 @@ import { ServiceRegistry } from './services/registry.service'; import { VendorCreditService } from './services/vendorcredit.service'; import { SyncService } from './sync/sync.service'; import { VendorCreditController } from './vendorcredit.controller'; +import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; @Module({ controllers: [VendorCreditController], @@ -20,6 +21,7 @@ import { VendorCreditController } from './vendorcredit.controller'; FieldMappingService, ServiceRegistry, ConnectionUtils, + IngestDataService, /* PROVIDERS SERVICES */ ], exports: [SyncService], diff --git a/packages/api/src/ats/activity/activity.module.ts b/packages/api/src/ats/activity/activity.module.ts index 4436d9016..8bd03b884 100644 --- a/packages/api/src/ats/activity/activity.module.ts +++ b/packages/api/src/ats/activity/activity.module.ts @@ -8,6 +8,7 @@ import { EncryptionService } from '@@core/@core-services/encryption/encryption.s import { FieldMappingService } from '@@core/field-mapping/field-mapping.service'; import { WebhookService } from '@@core/@core-services/webhooks/panora-webhooks/webhook.service'; import { ConnectionUtils } from '@@core/connections/@utils'; +import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; @Module({ controllers: [ActivityController], @@ -20,6 +21,7 @@ import { ConnectionUtils } from '@@core/connections/@utils'; FieldMappingService, ServiceRegistry, ConnectionUtils, + IngestDataService, /* PROVIDERS SERVICES */ ], exports: [SyncService], diff --git a/packages/api/src/ats/activity/services/activity.service.ts b/packages/api/src/ats/activity/services/activity.service.ts index e2aa39b01..c4c952e47 100644 --- a/packages/api/src/ats/activity/services/activity.service.ts +++ b/packages/api/src/ats/activity/services/activity.service.ts @@ -207,7 +207,7 @@ export class ActivityService { id_linked_user: linkedUserId, }, }); - await this.webhook.handleWebhook( + await this.webhook.dispatchWebhook( result_activity, 'ats.activity.created', linkedUser.id_project, diff --git a/packages/api/src/ats/activity/sync/sync.service.ts b/packages/api/src/ats/activity/sync/sync.service.ts index 7a3a8a050..5a9ce493b 100644 --- a/packages/api/src/ats/activity/sync/sync.service.ts +++ b/packages/api/src/ats/activity/sync/sync.service.ts @@ -1,6 +1,8 @@ import { LoggerService } from '@@core/@core-services/logger/logger.service'; import { PrismaService } from '@@core/@core-services/prisma/prisma.service'; import { BullQueueService } from '@@core/@core-services/queues/shared.service'; +import { IBaseSync } from '@@core/utils/types/interface'; +import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; import { CoreSyncRegistry } from '@@core/@core-services/registries/core-sync.registry'; import { CoreUnification } from '@@core/@core-services/unification/core-unification.service'; import { FieldMappingService } from '@@core/field-mapping/field-mapping.service'; @@ -18,7 +20,7 @@ import { UnifiedActivityOutput } from '../types/model.unified'; import { WebhookService } from '@@core/@core-services/webhooks/panora-webhooks/webhook.service'; @Injectable() -export class SyncService implements OnModuleInit { +export class SyncService implements OnModuleInit, IBaseSync { constructor( private prisma: PrismaService, private logger: LoggerService, @@ -28,6 +30,7 @@ export class SyncService implements OnModuleInit { private coreUnification: CoreUnification, private registry: CoreSyncRegistry, private bullQueueService: BullQueueService, + private ingestService: IngestDataService, ) { this.logger.setContext(SyncService.name); this.registry.registerService('ats', 'activity', this); @@ -82,7 +85,6 @@ export class SyncService implements OnModuleInit { await this.syncActivitiesForLinkedUser( provider, linkedUser.id_linked_user, - id_project, ); } catch (error) { throw error; @@ -104,7 +106,6 @@ export class SyncService implements OnModuleInit { async syncActivitiesForLinkedUser( integrationId: string, linkedUserId: string, - id_project: string, ) { try { this.logger.log( @@ -142,6 +143,17 @@ export class SyncService implements OnModuleInit { const sourceObject: OriginalActivityOutput[] = resp.data; + await this.ingestService.ingestData< + UnifiedCompanyOutput, + OriginalCompanyOutput + >( + sourceObject, + integrationId, + connection.id_connection, + 'crm', + 'company', + customFieldMappings, + ); //unify the data according to the target obj wanted const unifiedObject = (await this.coreUnification.unify< OriginalActivityOutput[] @@ -174,7 +186,7 @@ export class SyncService implements OnModuleInit { id_linked_user: linkedUserId, }, }); - await this.webhook.handleWebhook( + await this.webhook.dispatchWebhook( activities_data, 'ats.activity.pulled', id_project, diff --git a/packages/api/src/ats/application/application.module.ts b/packages/api/src/ats/application/application.module.ts index e6a6906a8..cff186180 100644 --- a/packages/api/src/ats/application/application.module.ts +++ b/packages/api/src/ats/application/application.module.ts @@ -8,6 +8,7 @@ import { EncryptionService } from '@@core/@core-services/encryption/encryption.s import { FieldMappingService } from '@@core/field-mapping/field-mapping.service'; import { WebhookService } from '@@core/@core-services/webhooks/panora-webhooks/webhook.service'; import { ConnectionUtils } from '@@core/connections/@utils'; +import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; @Module({ controllers: [ApplicationController], @@ -20,6 +21,7 @@ import { ConnectionUtils } from '@@core/connections/@utils'; FieldMappingService, ServiceRegistry, ConnectionUtils, + IngestDataService, /* PROVIDERS SERVICES */ ], exports: [SyncService], diff --git a/packages/api/src/ats/application/services/application.service.ts b/packages/api/src/ats/application/services/application.service.ts index 1222c0d0b..f527f258e 100644 --- a/packages/api/src/ats/application/services/application.service.ts +++ b/packages/api/src/ats/application/services/application.service.ts @@ -223,7 +223,7 @@ export class ApplicationService { id_linked_user: linkedUserId, }, }); - await this.webhook.handleWebhook( + await this.webhook.dispatchWebhook( result_application, 'ats.application.created', linkedUser.id_project, diff --git a/packages/api/src/ats/application/sync/sync.service.ts b/packages/api/src/ats/application/sync/sync.service.ts index 09737c5f8..de738314b 100644 --- a/packages/api/src/ats/application/sync/sync.service.ts +++ b/packages/api/src/ats/application/sync/sync.service.ts @@ -1,6 +1,8 @@ import { LoggerService } from '@@core/@core-services/logger/logger.service'; import { PrismaService } from '@@core/@core-services/prisma/prisma.service'; import { BullQueueService } from '@@core/@core-services/queues/shared.service'; +import { IBaseSync } from '@@core/utils/types/interface'; +import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; import { CoreSyncRegistry } from '@@core/@core-services/registries/core-sync.registry'; import { CoreUnification } from '@@core/@core-services/unification/core-unification.service'; import { WebhookService } from '@@core/@core-services/webhooks/panora-webhooks/webhook.service'; @@ -18,7 +20,7 @@ import { IApplicationService } from '../types'; import { UnifiedApplicationOutput } from '../types/model.unified'; @Injectable() -export class SyncService implements OnModuleInit { +export class SyncService implements OnModuleInit, IBaseSync { constructor( private prisma: PrismaService, private logger: LoggerService, @@ -28,6 +30,7 @@ export class SyncService implements OnModuleInit { private coreUnification: CoreUnification, private registry: CoreSyncRegistry, private bullQueueService: BullQueueService, + private ingestService: IngestDataService, ) { this.logger.setContext(SyncService.name); this.registry.registerService('ats', 'application', this); @@ -79,7 +82,6 @@ export class SyncService implements OnModuleInit { await this.syncApplicationsForLinkedUser( provider, linkedUser.id_linked_user, - id_project, ); } catch (error) { throw error; @@ -100,7 +102,6 @@ export class SyncService implements OnModuleInit { async syncApplicationsForLinkedUser( integrationId: string, linkedUserId: string, - id_project: string, ) { try { this.logger.log( @@ -139,6 +140,17 @@ export class SyncService implements OnModuleInit { const sourceObject: OriginalApplicationOutput[] = resp.data; + await this.ingestService.ingestData< + UnifiedCompanyOutput, + OriginalCompanyOutput + >( + sourceObject, + integrationId, + connection.id_connection, + 'crm', + 'company', + customFieldMappings, + ); // unify the data according to the target obj wanted const unifiedObject = (await this.coreUnification.unify< OriginalApplicationOutput[] @@ -171,7 +183,7 @@ export class SyncService implements OnModuleInit { id_linked_user: linkedUserId, }, }); - await this.webhook.handleWebhook( + await this.webhook.dispatchWebhook( applications_data, 'ats.application.pulled', id_project, diff --git a/packages/api/src/ats/attachment/attachment.module.ts b/packages/api/src/ats/attachment/attachment.module.ts index b7ae57f0f..e3f6e64f5 100644 --- a/packages/api/src/ats/attachment/attachment.module.ts +++ b/packages/api/src/ats/attachment/attachment.module.ts @@ -8,6 +8,7 @@ import { EncryptionService } from '@@core/@core-services/encryption/encryption.s import { FieldMappingService } from '@@core/field-mapping/field-mapping.service'; import { WebhookService } from '@@core/@core-services/webhooks/panora-webhooks/webhook.service'; import { ConnectionUtils } from '@@core/connections/@utils'; +import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; @Module({ controllers: [AttachmentController], @@ -20,6 +21,7 @@ import { ConnectionUtils } from '@@core/connections/@utils'; FieldMappingService, ServiceRegistry, ConnectionUtils, + IngestDataService, /* PROVIDERS SERVICES */ ], exports: [SyncService], diff --git a/packages/api/src/ats/attachment/services/attachment.service.ts b/packages/api/src/ats/attachment/services/attachment.service.ts index 1789bc3d4..2b0426e51 100644 --- a/packages/api/src/ats/attachment/services/attachment.service.ts +++ b/packages/api/src/ats/attachment/services/attachment.service.ts @@ -217,7 +217,7 @@ export class AttachmentService { id_linked_user: linkedUserId, }, }); - await this.webhook.handleWebhook( + await this.webhook.dispatchWebhook( result_attachment, 'ats.attachment.created', linkedUser.id_project, diff --git a/packages/api/src/ats/attachment/sync/sync.service.ts b/packages/api/src/ats/attachment/sync/sync.service.ts index ef897f7b9..ec3b67bf1 100644 --- a/packages/api/src/ats/attachment/sync/sync.service.ts +++ b/packages/api/src/ats/attachment/sync/sync.service.ts @@ -15,10 +15,12 @@ import { ats_attachments as AtsAttachment } from '@prisma/client'; import { ATS_PROVIDERS } from '@panora/shared'; import { AtsObject } from '@ats/@lib/@types'; import { BullQueueService } from '@@core/@core-services/queues/shared.service'; +import { IBaseSync } from '@@core/utils/types/interface'; +import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; import { CoreUnification } from '@@core/@core-services/unification/core-unification.service'; @Injectable() -export class SyncService implements OnModuleInit { +export class SyncService implements OnModuleInit, IBaseSync { constructor( private prisma: PrismaService, private logger: LoggerService, @@ -28,6 +30,7 @@ export class SyncService implements OnModuleInit { private coreUnification: CoreUnification, private registry: CoreSyncRegistry, private bullQueueService: BullQueueService, + private ingestService: IngestDataService, ) { this.logger.setContext(SyncService.name); this.registry.registerService('ats', 'attachment', this); @@ -79,7 +82,6 @@ export class SyncService implements OnModuleInit { await this.syncAttachmentsForLinkedUser( provider, linkedUser.id_linked_user, - id_project, ); } catch (error) { throw error; @@ -100,7 +102,6 @@ export class SyncService implements OnModuleInit { async syncAttachmentsForLinkedUser( integrationId: string, linkedUserId: string, - id_project: string, ) { try { this.logger.log( @@ -139,6 +140,17 @@ export class SyncService implements OnModuleInit { const sourceObject: OriginalAttachmentOutput[] = resp.data; + await this.ingestService.ingestData< + UnifiedCompanyOutput, + OriginalCompanyOutput + >( + sourceObject, + integrationId, + connection.id_connection, + 'crm', + 'company', + customFieldMappings, + ); // unify the data according to the target obj wanted const unifiedObject = (await this.coreUnification.unify< OriginalAttachmentOutput[] @@ -171,7 +183,7 @@ export class SyncService implements OnModuleInit { id_linked_user: linkedUserId, }, }); - await this.webhook.handleWebhook( + await this.webhook.dispatchWebhook( attachments_data, 'ats.attachment.pulled', id_project, diff --git a/packages/api/src/ats/candidate/candidate.module.ts b/packages/api/src/ats/candidate/candidate.module.ts index a3cdb8fa0..5d22f00f1 100644 --- a/packages/api/src/ats/candidate/candidate.module.ts +++ b/packages/api/src/ats/candidate/candidate.module.ts @@ -8,6 +8,7 @@ import { CandidateController } from './candidate.controller'; import { CandidateService } from './services/candidate.service'; import { ServiceRegistry } from './services/registry.service'; import { SyncService } from './sync/sync.service'; +import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; @Module({ controllers: [CandidateController], @@ -20,6 +21,7 @@ import { SyncService } from './sync/sync.service'; FieldMappingService, ServiceRegistry, ConnectionUtils, + IngestDataService, /* PROVIDERS SERVICES */ ], exports: [SyncService], diff --git a/packages/api/src/ats/candidate/services/candidate.service.ts b/packages/api/src/ats/candidate/services/candidate.service.ts index fa81403e8..80b1bc39c 100644 --- a/packages/api/src/ats/candidate/services/candidate.service.ts +++ b/packages/api/src/ats/candidate/services/candidate.service.ts @@ -231,7 +231,7 @@ export class CandidateService { id_linked_user: linkedUserId, }, }); - await this.webhook.handleWebhook( + await this.webhook.dispatchWebhook( result_candidate, 'ats.candidate.created', linkedUser.id_project, diff --git a/packages/api/src/ats/candidate/sync/sync.service.ts b/packages/api/src/ats/candidate/sync/sync.service.ts index d3d6866f5..d7648d6c1 100644 --- a/packages/api/src/ats/candidate/sync/sync.service.ts +++ b/packages/api/src/ats/candidate/sync/sync.service.ts @@ -14,10 +14,12 @@ import { ats_candidates as AtsCandidate } from '@prisma/client'; import { ATS_PROVIDERS } from '@panora/shared'; import { AtsObject } from '@ats/@lib/@types'; import { BullQueueService } from '@@core/@core-services/queues/shared.service'; +import { IBaseSync } from '@@core/utils/types/interface'; +import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; import { CoreUnification } from '@@core/@core-services/unification/core-unification.service'; @Injectable() -export class SyncService implements OnModuleInit { +export class SyncService implements OnModuleInit, IBaseSync { constructor( private prisma: PrismaService, private logger: LoggerService, @@ -27,6 +29,7 @@ export class SyncService implements OnModuleInit { private coreUnification: CoreUnification, private registry: CoreSyncRegistry, private bullQueueService: BullQueueService, + private ingestService: IngestDataService, ) { this.logger.setContext(SyncService.name); this.registry.registerService('ats', 'candidate', this); @@ -78,7 +81,6 @@ export class SyncService implements OnModuleInit { await this.syncCandidatesForLinkedUser( provider, linkedUser.id_linked_user, - id_project, ); } catch (error) { throw error; @@ -99,7 +101,6 @@ export class SyncService implements OnModuleInit { async syncCandidatesForLinkedUser( integrationId: string, linkedUserId: string, - id_project: string, ) { try { this.logger.log( @@ -138,6 +139,17 @@ export class SyncService implements OnModuleInit { const sourceObject: OriginalCandidateOutput[] = resp.data; + await this.ingestService.ingestData< + UnifiedCompanyOutput, + OriginalCompanyOutput + >( + sourceObject, + integrationId, + connection.id_connection, + 'crm', + 'company', + customFieldMappings, + ); // unify the data according to the target obj wanted const unifiedObject = (await this.coreUnification.unify< OriginalCandidateOutput[] @@ -170,7 +182,7 @@ export class SyncService implements OnModuleInit { id_linked_user: linkedUserId, }, }); - await this.webhook.handleWebhook( + await this.webhook.dispatchWebhook( candidates_data, 'ats.candidate.pulled', id_project, diff --git a/packages/api/src/ats/department/department.module.ts b/packages/api/src/ats/department/department.module.ts index 8f4ca06f6..a2da5de70 100644 --- a/packages/api/src/ats/department/department.module.ts +++ b/packages/api/src/ats/department/department.module.ts @@ -8,6 +8,7 @@ import { DepartmentController } from './department.controller'; import { DepartmentService } from './services/department.service'; import { ServiceRegistry } from './services/registry.service'; import { SyncService } from './sync/sync.service'; +import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; @Module({ controllers: [DepartmentController], @@ -20,6 +21,7 @@ import { SyncService } from './sync/sync.service'; FieldMappingService, ServiceRegistry, ConnectionUtils, + IngestDataService, /* PROVIDERS SERVICES */ ], exports: [SyncService], diff --git a/packages/api/src/ats/department/sync/sync.service.ts b/packages/api/src/ats/department/sync/sync.service.ts index 4aa4f64b6..de937db3c 100644 --- a/packages/api/src/ats/department/sync/sync.service.ts +++ b/packages/api/src/ats/department/sync/sync.service.ts @@ -14,10 +14,12 @@ import { ats_departments as AtsDepartment } from '@prisma/client'; import { ATS_PROVIDERS } from '@panora/shared'; import { AtsObject } from '@ats/@lib/@types'; import { BullQueueService } from '@@core/@core-services/queues/shared.service'; +import { IBaseSync } from '@@core/utils/types/interface'; +import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; import { CoreUnification } from '@@core/@core-services/unification/core-unification.service'; @Injectable() -export class SyncService implements OnModuleInit { +export class SyncService implements OnModuleInit, IBaseSync { constructor( private prisma: PrismaService, private logger: LoggerService, @@ -27,6 +29,7 @@ export class SyncService implements OnModuleInit { private coreUnification: CoreUnification, private registry: CoreSyncRegistry, private bullQueueService: BullQueueService, + private ingestService: IngestDataService, ) { this.logger.setContext(SyncService.name); this.registry.registerService('ats', 'department', this); @@ -78,7 +81,6 @@ export class SyncService implements OnModuleInit { await this.syncDepartmentsForLinkedUser( provider, linkedUser.id_linked_user, - id_project, ); } catch (error) { throw error; @@ -99,7 +101,6 @@ export class SyncService implements OnModuleInit { async syncDepartmentsForLinkedUser( integrationId: string, linkedUserId: string, - id_project: string, ) { try { this.logger.log( @@ -138,6 +139,17 @@ export class SyncService implements OnModuleInit { const sourceObject: OriginalDepartmentOutput[] = resp.data; + await this.ingestService.ingestData< + UnifiedCompanyOutput, + OriginalCompanyOutput + >( + sourceObject, + integrationId, + connection.id_connection, + 'crm', + 'company', + customFieldMappings, + ); // unify the data according to the target obj wanted const unifiedObject = (await this.coreUnification.unify< OriginalDepartmentOutput[] @@ -170,7 +182,7 @@ export class SyncService implements OnModuleInit { id_linked_user: linkedUserId, }, }); - await this.webhook.handleWebhook( + await this.webhook.dispatchWebhook( departments_data, 'ats.department.pulled', id_project, diff --git a/packages/api/src/ats/eeocs/eeocs.module.ts b/packages/api/src/ats/eeocs/eeocs.module.ts index 65f11217b..102461f04 100644 --- a/packages/api/src/ats/eeocs/eeocs.module.ts +++ b/packages/api/src/ats/eeocs/eeocs.module.ts @@ -8,6 +8,7 @@ import { EeocsController } from './eeocs.controller'; import { EeocsService } from './services/eeocs.service'; import { ServiceRegistry } from './services/registry.service'; import { SyncService } from './sync/sync.service'; +import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; @Module({ controllers: [EeocsController], @@ -20,6 +21,7 @@ import { SyncService } from './sync/sync.service'; FieldMappingService, ServiceRegistry, ConnectionUtils, + IngestDataService, /* PROVIDERS SERVICES */ ], exports: [SyncService], diff --git a/packages/api/src/ats/eeocs/sync/sync.service.ts b/packages/api/src/ats/eeocs/sync/sync.service.ts index 234dd22ec..7c1ac44b5 100644 --- a/packages/api/src/ats/eeocs/sync/sync.service.ts +++ b/packages/api/src/ats/eeocs/sync/sync.service.ts @@ -4,7 +4,8 @@ import { PrismaService } from '@@core/@core-services/prisma/prisma.service'; import { Cron } from '@nestjs/schedule'; import { v4 as uuidv4 } from 'uuid'; import { FieldMappingService } from '@@core/field-mapping/field-mapping.service'; -import { ServiceRegistry } from '../services/registry.service';import { WebhookService } from '@@core/@core-services/webhooks/panora-webhooks/webhook.service'; +import { ServiceRegistry } from '../services/registry.service'; +import { WebhookService } from '@@core/@core-services/webhooks/panora-webhooks/webhook.service'; import { CoreSyncRegistry } from '@@core/@core-services/registries/core-sync.registry'; import { ApiResponse } from '@@core/utils/types'; import { IEeocsService } from '../types'; @@ -14,10 +15,12 @@ import { ats_eeocs as AtsEeocs } from '@prisma/client'; import { ATS_PROVIDERS } from '@panora/shared'; import { AtsObject } from '@ats/@lib/@types'; import { BullQueueService } from '@@core/@core-services/queues/shared.service'; +import { IBaseSync } from '@@core/utils/types/interface'; +import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; import { CoreUnification } from '@@core/@core-services/unification/core-unification.service'; @Injectable() -export class SyncService implements OnModuleInit { +export class SyncService implements OnModuleInit, IBaseSync { constructor( private prisma: PrismaService, private logger: LoggerService, @@ -27,9 +30,13 @@ export class SyncService implements OnModuleInit { private coreUnification: CoreUnification, private registry: CoreSyncRegistry, private bullQueueService: BullQueueService, + private ingestService: IngestDataService, ) { this.logger.setContext(SyncService.name); this.registry.registerService('ats', 'eeocs', this); + saveToDb(connection_id: string, linkedUserId: string, data: any[], originSource: string, remote_data: Record[]): Promise { + throw new Error('Method not implemented.'); + } } async onModuleInit() { @@ -75,7 +82,6 @@ export class SyncService implements OnModuleInit { await this.syncEeocsForLinkedUser( provider, linkedUser.id_linked_user, - id_project, ); } catch (error) { throw error; @@ -93,11 +99,7 @@ export class SyncService implements OnModuleInit { } } - async syncEeocsForLinkedUser( - integrationId: string, - linkedUserId: string, - id_project: string, - ) { + async syncEeocsForLinkedUser(integrationId: string, linkedUserId: string) { try { this.logger.log( `Syncing ${integrationId} EEOCs for linkedUser ${linkedUserId}`, @@ -137,6 +139,17 @@ export class SyncService implements OnModuleInit { const sourceObject: OriginalEeocsOutput[] = resp.data; + await this.ingestService.ingestData< + UnifiedCompanyOutput, + OriginalCompanyOutput + >( + sourceObject, + integrationId, + connection.id_connection, + 'crm', + 'company', + customFieldMappings, + ); // unify the data according to the target obj wanted const unifiedObject = (await this.coreUnification.unify< OriginalEeocsOutput[] @@ -169,7 +182,7 @@ export class SyncService implements OnModuleInit { id_linked_user: linkedUserId, }, }); - await this.webhook.handleWebhook( + await this.webhook.dispatchWebhook( eeocs_data, 'ats.eeocs.pulled', id_project, diff --git a/packages/api/src/ats/interview/interview.module.ts b/packages/api/src/ats/interview/interview.module.ts index 63c894d96..69335de55 100644 --- a/packages/api/src/ats/interview/interview.module.ts +++ b/packages/api/src/ats/interview/interview.module.ts @@ -8,6 +8,7 @@ import { InterviewController } from './interview.controller'; import { InterviewService } from './services/interview.service'; import { ServiceRegistry } from './services/registry.service'; import { SyncService } from './sync/sync.service'; +import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; @Module({ controllers: [InterviewController], @@ -20,6 +21,7 @@ import { SyncService } from './sync/sync.service'; FieldMappingService, ServiceRegistry, ConnectionUtils, + IngestDataService, /* PROVIDERS SERVICES */ ], exports: [SyncService], diff --git a/packages/api/src/ats/interview/services/interview.service.ts b/packages/api/src/ats/interview/services/interview.service.ts index 320ab79a1..85c1e97ca 100644 --- a/packages/api/src/ats/interview/services/interview.service.ts +++ b/packages/api/src/ats/interview/services/interview.service.ts @@ -222,7 +222,7 @@ export class InterviewService { id_linked_user: linkedUserId, }, }); - await this.webhook.handleWebhook( + await this.webhook.dispatchWebhook( result_interview, 'ats.interview.created', linkedUser.id_project, diff --git a/packages/api/src/ats/interview/sync/sync.service.ts b/packages/api/src/ats/interview/sync/sync.service.ts index 1a0cafe29..6c6e501e8 100644 --- a/packages/api/src/ats/interview/sync/sync.service.ts +++ b/packages/api/src/ats/interview/sync/sync.service.ts @@ -14,10 +14,12 @@ import { ats_interviews as AtsInterview } from '@prisma/client'; import { ATS_PROVIDERS } from '@panora/shared'; import { AtsObject } from '@ats/@lib/@types'; import { BullQueueService } from '@@core/@core-services/queues/shared.service'; +import { IBaseSync } from '@@core/utils/types/interface'; +import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; import { CoreUnification } from '@@core/@core-services/unification/core-unification.service'; @Injectable() -export class SyncService implements OnModuleInit { +export class SyncService implements OnModuleInit, IBaseSync { constructor( private prisma: PrismaService, private logger: LoggerService, @@ -27,6 +29,7 @@ export class SyncService implements OnModuleInit { private coreUnification: CoreUnification, private registry: CoreSyncRegistry, private bullQueueService: BullQueueService, + private ingestService: IngestDataService, ) { this.logger.setContext(SyncService.name); this.registry.registerService('ats', 'interview', this); @@ -78,7 +81,6 @@ export class SyncService implements OnModuleInit { await this.syncInterviewsForLinkedUser( provider, linkedUser.id_linked_user, - id_project, ); } catch (error) { throw error; @@ -99,7 +101,6 @@ export class SyncService implements OnModuleInit { async syncInterviewsForLinkedUser( integrationId: string, linkedUserId: string, - id_project: string, ) { try { this.logger.log( @@ -138,6 +139,17 @@ export class SyncService implements OnModuleInit { const sourceObject: OriginalInterviewOutput[] = resp.data; + await this.ingestService.ingestData< + UnifiedCompanyOutput, + OriginalCompanyOutput + >( + sourceObject, + integrationId, + connection.id_connection, + 'crm', + 'company', + customFieldMappings, + ); // unify the data according to the target obj wanted const unifiedObject = (await this.coreUnification.unify< OriginalInterviewOutput[] @@ -170,7 +182,7 @@ export class SyncService implements OnModuleInit { id_linked_user: linkedUserId, }, }); - await this.webhook.handleWebhook( + await this.webhook.dispatchWebhook( interviews_data, 'ats.interview.pulled', id_project, diff --git a/packages/api/src/ats/job/job.module.ts b/packages/api/src/ats/job/job.module.ts index 3d198f8cb..ac4a5ca42 100644 --- a/packages/api/src/ats/job/job.module.ts +++ b/packages/api/src/ats/job/job.module.ts @@ -8,6 +8,7 @@ import { JobController } from './job.controller'; import { JobService } from './services/job.service'; import { ServiceRegistry } from './services/registry.service'; import { SyncService } from './sync/sync.service'; +import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; @Module({ controllers: [JobController], @@ -20,6 +21,7 @@ import { SyncService } from './sync/sync.service'; FieldMappingService, ServiceRegistry, ConnectionUtils, + IngestDataService, /* PROVIDERS SERVICES */ ], exports: [SyncService], diff --git a/packages/api/src/ats/job/sync/sync.service.ts b/packages/api/src/ats/job/sync/sync.service.ts index 0ac784aa6..11c2a1e17 100644 --- a/packages/api/src/ats/job/sync/sync.service.ts +++ b/packages/api/src/ats/job/sync/sync.service.ts @@ -15,10 +15,12 @@ import { ats_jobs as AtsJob } from '@prisma/client'; import { ATS_PROVIDERS } from '@panora/shared'; import { AtsObject } from '@ats/@lib/@types'; import { BullQueueService } from '@@core/@core-services/queues/shared.service'; +import { IBaseSync } from '@@core/utils/types/interface'; +import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; import { CoreUnification } from '@@core/@core-services/unification/core-unification.service'; @Injectable() -export class SyncService implements OnModuleInit { +export class SyncService implements OnModuleInit, IBaseSync { constructor( private prisma: PrismaService, private logger: LoggerService, @@ -28,6 +30,7 @@ export class SyncService implements OnModuleInit { private coreUnification: CoreUnification, private registry: CoreSyncRegistry, private bullQueueService: BullQueueService, + private ingestService: IngestDataService, ) { this.logger.setContext(SyncService.name); this.registry.registerService('ats', 'job', this); @@ -76,7 +79,6 @@ export class SyncService implements OnModuleInit { await this.syncJobsForLinkedUser( provider, linkedUser.id_linked_user, - id_project, ); } catch (error) { throw error; @@ -97,7 +99,6 @@ export class SyncService implements OnModuleInit { async syncJobsForLinkedUser( integrationId: string, linkedUserId: string, - id_project: string, ) { try { this.logger.log( @@ -138,6 +139,17 @@ export class SyncService implements OnModuleInit { const sourceObject: OriginalJobOutput[] = resp.data; + await this.ingestService.ingestData< + UnifiedCompanyOutput, + OriginalCompanyOutput + >( + sourceObject, + integrationId, + connection.id_connection, + 'crm', + 'company', + customFieldMappings, + ); // unify the data according to the target obj wanted const unifiedObject = (await this.coreUnification.unify< OriginalJobOutput[] @@ -170,7 +182,7 @@ export class SyncService implements OnModuleInit { id_linked_user: linkedUserId, }, }); - await this.webhook.handleWebhook( + await this.webhook.dispatchWebhook( jobs_data, 'ats.job.pulled', id_project, diff --git a/packages/api/src/ats/jobinterviewstage/jobinterviewstage.module.ts b/packages/api/src/ats/jobinterviewstage/jobinterviewstage.module.ts index b61191759..874925d23 100644 --- a/packages/api/src/ats/jobinterviewstage/jobinterviewstage.module.ts +++ b/packages/api/src/ats/jobinterviewstage/jobinterviewstage.module.ts @@ -8,6 +8,7 @@ import { JobInterviewStageController } from './jobinterviewstage.controller'; import { JobInterviewStageService } from './services/jobinterviewstage.service'; import { ServiceRegistry } from './services/registry.service'; import { SyncService } from './sync/sync.service'; +import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; @Module({ controllers: [JobInterviewStageController], @@ -20,6 +21,7 @@ import { SyncService } from './sync/sync.service'; FieldMappingService, ServiceRegistry, ConnectionUtils, + IngestDataService, /* PROVIDERS SERVICES */ ], exports: [SyncService], diff --git a/packages/api/src/ats/jobinterviewstage/sync/sync.service.ts b/packages/api/src/ats/jobinterviewstage/sync/sync.service.ts index f9a50f5df..5abb30ecc 100644 --- a/packages/api/src/ats/jobinterviewstage/sync/sync.service.ts +++ b/packages/api/src/ats/jobinterviewstage/sync/sync.service.ts @@ -14,10 +14,12 @@ import { ats_job_interview_stages as AtsJobInterviewStage } from '@prisma/client import { ATS_PROVIDERS } from '@panora/shared'; import { AtsObject } from '@ats/@lib/@types'; import { BullQueueService } from '@@core/@core-services/queues/shared.service'; +import { IBaseSync } from '@@core/utils/types/interface'; +import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; import { CoreUnification } from '@@core/@core-services/unification/core-unification.service'; @Injectable() -export class SyncService implements OnModuleInit { +export class SyncService implements OnModuleInit, IBaseSync { constructor( private prisma: PrismaService, private logger: LoggerService, @@ -27,6 +29,7 @@ export class SyncService implements OnModuleInit { private coreUnification: CoreUnification, private registry: CoreSyncRegistry, private bullQueueService: BullQueueService, + private ingestService: IngestDataService, ) { this.logger.setContext(SyncService.name); this.registry.registerService('ats', 'jobinterviewstage', this); @@ -78,7 +81,6 @@ export class SyncService implements OnModuleInit { await this.syncJobInterviewStagesForLinkedUser( provider, linkedUser.id_linked_user, - id_project, ); } catch (error) { throw error; @@ -99,7 +101,6 @@ export class SyncService implements OnModuleInit { async syncJobInterviewStagesForLinkedUser( integrationId: string, linkedUserId: string, - id_project: string, ) { try { this.logger.log( @@ -138,6 +139,17 @@ export class SyncService implements OnModuleInit { const sourceObject: OriginalJobInterviewStageOutput[] = resp.data; + await this.ingestService.ingestData< + UnifiedCompanyOutput, + OriginalCompanyOutput + >( + sourceObject, + integrationId, + connection.id_connection, + 'crm', + 'company', + customFieldMappings, + ); // unify the data according to the target obj wanted const unifiedObject = (await this.coreUnification.unify< OriginalJobInterviewStageOutput[] @@ -170,7 +182,7 @@ export class SyncService implements OnModuleInit { id_linked_user: linkedUserId, }, }); - await this.webhook.handleWebhook( + await this.webhook.dispatchWebhook( jobInterviewStages_data, 'ats.job_interview_stage.pulled', id_project, diff --git a/packages/api/src/ats/offer/offer.module.ts b/packages/api/src/ats/offer/offer.module.ts index d0a01223a..0790856ae 100644 --- a/packages/api/src/ats/offer/offer.module.ts +++ b/packages/api/src/ats/offer/offer.module.ts @@ -8,6 +8,7 @@ import { OfferController } from './offer.controller'; import { OfferService } from './services/offer.service'; import { ServiceRegistry } from './services/registry.service'; import { SyncService } from './sync/sync.service'; +import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; @Module({ controllers: [OfferController], @@ -20,6 +21,7 @@ import { SyncService } from './sync/sync.service'; FieldMappingService, ServiceRegistry, ConnectionUtils, + IngestDataService, /* PROVIDERS SERVICES */ ], exports: [SyncService], diff --git a/packages/api/src/ats/offer/sync/sync.service.ts b/packages/api/src/ats/offer/sync/sync.service.ts index d392a35f8..f9033cd11 100644 --- a/packages/api/src/ats/offer/sync/sync.service.ts +++ b/packages/api/src/ats/offer/sync/sync.service.ts @@ -14,10 +14,12 @@ import { ats_offers as AtsOffer } from '@prisma/client'; import { ATS_PROVIDERS } from '@panora/shared'; import { AtsObject } from '@ats/@lib/@types'; import { BullQueueService } from '@@core/@core-services/queues/shared.service'; +import { IBaseSync } from '@@core/utils/types/interface'; +import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; import { CoreUnification } from '@@core/@core-services/unification/core-unification.service'; @Injectable() -export class SyncService implements OnModuleInit { +export class SyncService implements OnModuleInit, IBaseSync { constructor( private prisma: PrismaService, private logger: LoggerService, @@ -27,6 +29,7 @@ export class SyncService implements OnModuleInit { private coreUnification: CoreUnification, private registry: CoreSyncRegistry, private bullQueueService: BullQueueService, + private ingestService: IngestDataService, ) { this.logger.setContext(SyncService.name); this.registry.registerService('ats', 'offer', this); @@ -75,7 +78,6 @@ export class SyncService implements OnModuleInit { await this.syncOffersForLinkedUser( provider, linkedUser.id_linked_user, - id_project, ); } catch (error) { throw error; @@ -96,7 +98,6 @@ export class SyncService implements OnModuleInit { async syncOffersForLinkedUser( integrationId: string, linkedUserId: string, - id_project: string, ) { try { this.logger.log( @@ -137,6 +138,17 @@ export class SyncService implements OnModuleInit { const sourceObject: OriginalOfferOutput[] = resp.data; + await this.ingestService.ingestData< + UnifiedCompanyOutput, + OriginalCompanyOutput + >( + sourceObject, + integrationId, + connection.id_connection, + 'crm', + 'company', + customFieldMappings, + ); // unify the data according to the target obj wanted const unifiedObject = (await this.coreUnification.unify< OriginalOfferOutput[] @@ -169,7 +181,7 @@ export class SyncService implements OnModuleInit { id_linked_user: linkedUserId, }, }); - await this.webhook.handleWebhook( + await this.webhook.dispatchWebhook( offers_data, 'ats.offer.pulled', id_project, diff --git a/packages/api/src/ats/office/office.module.ts b/packages/api/src/ats/office/office.module.ts index 30fba9be3..1ce5bc17f 100644 --- a/packages/api/src/ats/office/office.module.ts +++ b/packages/api/src/ats/office/office.module.ts @@ -8,6 +8,7 @@ import { OfficeController } from './office.controller'; import { OfficeService } from './services/office.service'; import { ServiceRegistry } from './services/registry.service'; import { SyncService } from './sync/sync.service'; +import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; @Module({ controllers: [OfficeController], @@ -20,6 +21,7 @@ import { SyncService } from './sync/sync.service'; FieldMappingService, ServiceRegistry, ConnectionUtils, + IngestDataService, /* PROVIDERS SERVICES */ ], exports: [SyncService], diff --git a/packages/api/src/ats/office/sync/sync.service.ts b/packages/api/src/ats/office/sync/sync.service.ts index 73e6e9a21..99c46136c 100644 --- a/packages/api/src/ats/office/sync/sync.service.ts +++ b/packages/api/src/ats/office/sync/sync.service.ts @@ -15,10 +15,12 @@ import { ats_offices as AtsOffice } from '@prisma/client'; import { ATS_PROVIDERS } from '@panora/shared'; import { AtsObject } from '@ats/@lib/@types'; import { BullQueueService } from '@@core/@core-services/queues/shared.service'; +import { IBaseSync } from '@@core/utils/types/interface'; +import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; import { CoreUnification } from '@@core/@core-services/unification/core-unification.service'; @Injectable() -export class SyncService implements OnModuleInit { +export class SyncService implements OnModuleInit, IBaseSync { constructor( private prisma: PrismaService, private logger: LoggerService, @@ -28,6 +30,7 @@ export class SyncService implements OnModuleInit { private coreUnification: CoreUnification, private registry: CoreSyncRegistry, private bullQueueService: BullQueueService, + private ingestService: IngestDataService, ) { this.logger.setContext(SyncService.name); this.registry.registerService('ats', 'office', this); @@ -76,7 +79,6 @@ export class SyncService implements OnModuleInit { await this.syncOfficesForLinkedUser( provider, linkedUser.id_linked_user, - id_project, ); } catch (error) { throw error; @@ -97,7 +99,6 @@ export class SyncService implements OnModuleInit { async syncOfficesForLinkedUser( integrationId: string, linkedUserId: string, - id_project: string, ) { try { this.logger.log( @@ -136,6 +137,17 @@ export class SyncService implements OnModuleInit { const sourceObject: OriginalOfficeOutput[] = resp.data; + await this.ingestService.ingestData< + UnifiedCompanyOutput, + OriginalCompanyOutput + >( + sourceObject, + integrationId, + connection.id_connection, + 'crm', + 'company', + customFieldMappings, + ); // unify the data according to the target obj wanted const unifiedObject = (await this.coreUnification.unify< OriginalOfficeOutput[] @@ -168,7 +180,7 @@ export class SyncService implements OnModuleInit { id_linked_user: linkedUserId, }, }); - await this.webhook.handleWebhook( + await this.webhook.dispatchWebhook( offices_data, 'ats.office.pulled', id_project, diff --git a/packages/api/src/ats/rejectreason/rejectreason.module.ts b/packages/api/src/ats/rejectreason/rejectreason.module.ts index c10ad7319..2501e2ced 100644 --- a/packages/api/src/ats/rejectreason/rejectreason.module.ts +++ b/packages/api/src/ats/rejectreason/rejectreason.module.ts @@ -8,6 +8,7 @@ import { RejectReasonController } from './rejectreason.controller'; import { ServiceRegistry } from './services/registry.service'; import { RejectReasonService } from './services/rejectreason.service'; import { SyncService } from './sync/sync.service'; +import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; @Module({ controllers: [RejectReasonController], @@ -20,6 +21,7 @@ import { SyncService } from './sync/sync.service'; FieldMappingService, ServiceRegistry, ConnectionUtils, + IngestDataService, /* PROVIDERS SERVICES */ ], exports: [SyncService], diff --git a/packages/api/src/ats/rejectreason/sync/sync.service.ts b/packages/api/src/ats/rejectreason/sync/sync.service.ts index 82fc54ced..3cd5aad22 100644 --- a/packages/api/src/ats/rejectreason/sync/sync.service.ts +++ b/packages/api/src/ats/rejectreason/sync/sync.service.ts @@ -4,7 +4,8 @@ import { PrismaService } from '@@core/@core-services/prisma/prisma.service'; import { Cron } from '@nestjs/schedule'; import { v4 as uuidv4 } from 'uuid'; import { FieldMappingService } from '@@core/field-mapping/field-mapping.service'; -import { ServiceRegistry } from '../services/registry.service';import { WebhookService } from '@@core/@core-services/webhooks/panora-webhooks/webhook.service'; +import { ServiceRegistry } from '../services/registry.service'; +import { WebhookService } from '@@core/@core-services/webhooks/panora-webhooks/webhook.service'; import { CoreSyncRegistry } from '@@core/@core-services/registries/core-sync.registry'; import { ApiResponse } from '@@core/utils/types'; import { IRejectReasonService } from '../types'; @@ -14,10 +15,12 @@ import { ats_reject_reasons as AtsRejectReason } from '@prisma/client'; import { ATS_PROVIDERS } from '@panora/shared'; import { AtsObject } from '@ats/@lib/@types'; import { BullQueueService } from '@@core/@core-services/queues/shared.service'; +import { IBaseSync } from '@@core/utils/types/interface'; +import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; import { CoreUnification } from '@@core/@core-services/unification/core-unification.service'; @Injectable() -export class SyncService implements OnModuleInit { +export class SyncService implements OnModuleInit, IBaseSync { constructor( private prisma: PrismaService, private logger: LoggerService, @@ -27,9 +30,13 @@ export class SyncService implements OnModuleInit { private coreUnification: CoreUnification, private registry: CoreSyncRegistry, private bullQueueService: BullQueueService, + private ingestService: IngestDataService, ) { this.logger.setContext(SyncService.name); this.registry.registerService('ats', 'rejectreason', this); + saveToDb(connection_id: string, linkedUserId: string, data: any[], originSource: string, remote_data: Record[]): Promise { + throw new Error('Method not implemented.'); + } } async onModuleInit() { @@ -78,7 +85,6 @@ export class SyncService implements OnModuleInit { await this.syncRejectReasonsForLinkedUser( provider, linkedUser.id_linked_user, - id_project, ); } catch (error) { throw error; @@ -99,7 +105,6 @@ export class SyncService implements OnModuleInit { async syncRejectReasonsForLinkedUser( integrationId: string, linkedUserId: string, - id_project: string, ) { try { this.logger.log( @@ -138,6 +143,17 @@ export class SyncService implements OnModuleInit { const sourceObject: OriginalRejectReasonOutput[] = resp.data; + await this.ingestService.ingestData< + UnifiedCompanyOutput, + OriginalCompanyOutput + >( + sourceObject, + integrationId, + connection.id_connection, + 'crm', + 'company', + customFieldMappings, + ); // unify the data according to the target obj wanted const unifiedObject = (await this.coreUnification.unify< OriginalRejectReasonOutput[] @@ -170,7 +186,7 @@ export class SyncService implements OnModuleInit { id_linked_user: linkedUserId, }, }); - await this.webhook.handleWebhook( + await this.webhook.dispatchWebhook( rejectReasons_data, 'ats.reject_reason.pulled', id_project, diff --git a/packages/api/src/ats/scorecard/scorecard.module.ts b/packages/api/src/ats/scorecard/scorecard.module.ts index 2f8a3c92f..359fdb70b 100644 --- a/packages/api/src/ats/scorecard/scorecard.module.ts +++ b/packages/api/src/ats/scorecard/scorecard.module.ts @@ -8,6 +8,7 @@ import { ScoreCardController } from './scorecard.controller'; import { ServiceRegistry } from './services/registry.service'; import { ScoreCardService } from './services/scorecard.service'; import { SyncService } from './sync/sync.service'; +import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; @Module({ controllers: [ScoreCardController], @@ -20,6 +21,7 @@ import { SyncService } from './sync/sync.service'; FieldMappingService, ServiceRegistry, ConnectionUtils, + IngestDataService, /* PROVIDERS SERVICES */ ], exports: [SyncService], diff --git a/packages/api/src/ats/scorecard/sync/sync.service.ts b/packages/api/src/ats/scorecard/sync/sync.service.ts index 67d17dd8e..d89295007 100644 --- a/packages/api/src/ats/scorecard/sync/sync.service.ts +++ b/packages/api/src/ats/scorecard/sync/sync.service.ts @@ -4,7 +4,8 @@ import { PrismaService } from '@@core/@core-services/prisma/prisma.service'; import { Cron } from '@nestjs/schedule'; import { v4 as uuidv4 } from 'uuid'; import { FieldMappingService } from '@@core/field-mapping/field-mapping.service'; -import { ServiceRegistry } from '../services/registry.service';import { WebhookService } from '@@core/@core-services/webhooks/panora-webhooks/webhook.service'; +import { ServiceRegistry } from '../services/registry.service'; +import { WebhookService } from '@@core/@core-services/webhooks/panora-webhooks/webhook.service'; import { CoreSyncRegistry } from '@@core/@core-services/registries/core-sync.registry'; import { ApiResponse } from '@@core/utils/types'; import { IScoreCardService } from '../types'; @@ -14,10 +15,12 @@ import { ats_scorecards as AtsScoreCard } from '@prisma/client'; import { ATS_PROVIDERS } from '@panora/shared'; import { AtsObject } from '@ats/@lib/@types'; import { BullQueueService } from '@@core/@core-services/queues/shared.service'; +import { IBaseSync } from '@@core/utils/types/interface'; +import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; import { CoreUnification } from '@@core/@core-services/unification/core-unification.service'; @Injectable() -export class SyncService implements OnModuleInit { +export class SyncService implements OnModuleInit, IBaseSync { constructor( private prisma: PrismaService, private logger: LoggerService, @@ -27,9 +30,13 @@ export class SyncService implements OnModuleInit { private coreUnification: CoreUnification, private registry: CoreSyncRegistry, private bullQueueService: BullQueueService, + private ingestService: IngestDataService, ) { this.logger.setContext(SyncService.name); this.registry.registerService('ats', 'scorecard', this); + saveToDb(connection_id: string, linkedUserId: string, data: any[], originSource: string, remote_data: Record[]): Promise { + throw new Error('Method not implemented.'); + } } async onModuleInit() { @@ -78,7 +85,6 @@ export class SyncService implements OnModuleInit { await this.syncScoreCardsForLinkedUser( provider, linkedUser.id_linked_user, - id_project, ); } catch (error) { throw error; @@ -99,7 +105,6 @@ export class SyncService implements OnModuleInit { async syncScoreCardsForLinkedUser( integrationId: string, linkedUserId: string, - id_project: string, ) { try { this.logger.log( @@ -138,6 +143,17 @@ export class SyncService implements OnModuleInit { const sourceObject: OriginalScoreCardOutput[] = resp.data; + await this.ingestService.ingestData< + UnifiedCompanyOutput, + OriginalCompanyOutput + >( + sourceObject, + integrationId, + connection.id_connection, + 'crm', + 'company', + customFieldMappings, + ); // unify the data according to the target obj wanted const unifiedObject = (await this.coreUnification.unify< OriginalScoreCardOutput[] @@ -170,7 +186,7 @@ export class SyncService implements OnModuleInit { id_linked_user: linkedUserId, }, }); - await this.webhook.handleWebhook( + await this.webhook.dispatchWebhook( scoreCards_data, 'ats.score_card.pulled', id_project, diff --git a/packages/api/src/ats/screeningquestion/screeningquestion.module.ts b/packages/api/src/ats/screeningquestion/screeningquestion.module.ts index e6cb92a38..7c6184808 100644 --- a/packages/api/src/ats/screeningquestion/screeningquestion.module.ts +++ b/packages/api/src/ats/screeningquestion/screeningquestion.module.ts @@ -8,6 +8,7 @@ import { ScreeningQuestionController } from './screeningquestion.controller'; import { ServiceRegistry } from './services/registry.service'; import { ScreeningQuestionService } from './services/screeningquestion.service'; import { SyncService } from './sync/sync.service'; +import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; @Module({ controllers: [ScreeningQuestionController], @@ -20,6 +21,7 @@ import { SyncService } from './sync/sync.service'; FieldMappingService, ServiceRegistry, ConnectionUtils, + IngestDataService, /* PROVIDERS SERVICES */ ], exports: [SyncService], diff --git a/packages/api/src/ats/screeningquestion/sync/sync.service.ts b/packages/api/src/ats/screeningquestion/sync/sync.service.ts index 0a1d0dd37..a507fe03a 100644 --- a/packages/api/src/ats/screeningquestion/sync/sync.service.ts +++ b/packages/api/src/ats/screeningquestion/sync/sync.service.ts @@ -12,7 +12,7 @@ import { UnifiedScreeningQuestionOutput } from '../types/model.unified'; import { IScreeningQuestionService } from '../types'; @Injectable() -export class SyncService implements OnModuleInit { +export class SyncService implements OnModuleInit, IBaseSync { constructor( private prisma: PrismaService, private logger: LoggerService, diff --git a/packages/api/src/ats/tag/sync/sync.service.ts b/packages/api/src/ats/tag/sync/sync.service.ts index d07425865..9a844ddad 100644 --- a/packages/api/src/ats/tag/sync/sync.service.ts +++ b/packages/api/src/ats/tag/sync/sync.service.ts @@ -13,11 +13,13 @@ import { ats_candidate_tags as AtsTag } from '@prisma/client'; import { FILESTORAGE_PROVIDERS } from '@panora/shared'; import { AtsObject } from '@ats/@lib/@types'; import { BullQueueService } from '@@core/@core-services/queues/shared.service'; +import { IBaseSync } from '@@core/utils/types/interface'; +import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; import { CoreUnification } from '@@core/@core-services/unification/core-unification.service'; import { OriginalTagOutput } from '@@core/utils/types/original/original.ats'; @Injectable() -export class SyncService implements OnModuleInit { +export class SyncService implements OnModuleInit, IBaseSync { constructor( private prisma: PrismaService, private logger: LoggerService, @@ -27,6 +29,7 @@ export class SyncService implements OnModuleInit { private coreUnification: CoreUnification, private registry: CoreSyncRegistry, private bullQueueService: BullQueueService, + private ingestService: IngestDataService, ) { this.logger.setContext(SyncService.name); this.registry.registerService('ats', 'tag', this); @@ -75,7 +78,6 @@ export class SyncService implements OnModuleInit { await this.syncTagsForLinkedUser( provider, linkedUser.id_linked_user, - id_project, ); } catch (error) { throw error; @@ -96,7 +98,6 @@ export class SyncService implements OnModuleInit { async syncTagsForLinkedUser( integrationId: string, linkedUserId: string, - id_project: string, ) { try { this.logger.log( @@ -137,6 +138,17 @@ export class SyncService implements OnModuleInit { const sourceObject: OriginalTagOutput[] = resp.data; + await this.ingestService.ingestData< + UnifiedCompanyOutput, + OriginalCompanyOutput + >( + sourceObject, + integrationId, + connection.id_connection, + 'crm', + 'company', + customFieldMappings, + ); // unify the data according to the target obj wanted const unifiedObject = (await this.coreUnification.unify< OriginalTagOutput[] @@ -169,7 +181,7 @@ export class SyncService implements OnModuleInit { id_linked_user: linkedUserId, }, }); - await this.webhook.handleWebhook( + await this.webhook.dispatchWebhook( tags_data, 'ats.tag.pulled', id_project, diff --git a/packages/api/src/ats/tag/tag.module.ts b/packages/api/src/ats/tag/tag.module.ts index 1d8570d21..613becd67 100644 --- a/packages/api/src/ats/tag/tag.module.ts +++ b/packages/api/src/ats/tag/tag.module.ts @@ -8,6 +8,7 @@ import { ServiceRegistry } from './services/registry.service'; import { TagService } from './services/tag.service'; import { SyncService } from './sync/sync.service'; import { TagController } from './tag.controller'; +import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; @Module({ controllers: [TagController], @@ -20,6 +21,7 @@ import { TagController } from './tag.controller'; FieldMappingService, ServiceRegistry, ConnectionUtils, + IngestDataService, /* PROVIDERS SERVICES */ ], exports: [SyncService], diff --git a/packages/api/src/ats/user/sync/sync.service.ts b/packages/api/src/ats/user/sync/sync.service.ts index facaadeab..64039d03e 100644 --- a/packages/api/src/ats/user/sync/sync.service.ts +++ b/packages/api/src/ats/user/sync/sync.service.ts @@ -4,7 +4,8 @@ import { PrismaService } from '@@core/@core-services/prisma/prisma.service'; import { Cron } from '@nestjs/schedule'; import { v4 as uuidv4 } from 'uuid'; import { FieldMappingService } from '@@core/field-mapping/field-mapping.service'; -import { ServiceRegistry } from '../services/registry.service';import { WebhookService } from '@@core/@core-services/webhooks/panora-webhooks/webhook.service'; +import { ServiceRegistry } from '../services/registry.service'; +import { WebhookService } from '@@core/@core-services/webhooks/panora-webhooks/webhook.service'; import { CoreSyncRegistry } from '@@core/@core-services/registries/core-sync.registry'; import { ApiResponse } from '@@core/utils/types'; import { IUserService } from '../types'; @@ -14,10 +15,12 @@ import { ats_users as AtsUser } from '@prisma/client'; import { ATS_PROVIDERS } from '@panora/shared'; import { AtsObject } from '@ats/@lib/@types'; import { BullQueueService } from '@@core/@core-services/queues/shared.service'; +import { IBaseSync } from '@@core/utils/types/interface'; +import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; import { CoreUnification } from '@@core/@core-services/unification/core-unification.service'; @Injectable() -export class SyncService implements OnModuleInit { +export class SyncService implements OnModuleInit, IBaseSync { constructor( private prisma: PrismaService, private logger: LoggerService, @@ -27,9 +30,13 @@ export class SyncService implements OnModuleInit { private coreUnification: CoreUnification, private registry: CoreSyncRegistry, private bullQueueService: BullQueueService, + private ingestService: IngestDataService, ) { this.logger.setContext(SyncService.name); this.registry.registerService('ats', 'user', this); + saveToDb(connection_id: string, linkedUserId: string, data: any[], originSource: string, remote_data: Record[]): Promise { + throw new Error('Method not implemented.'); + } } async onModuleInit() { @@ -75,7 +82,6 @@ export class SyncService implements OnModuleInit { await this.syncUsersForLinkedUser( provider, linkedUser.id_linked_user, - id_project, ); } catch (error) { throw error; @@ -93,11 +99,7 @@ export class SyncService implements OnModuleInit { } } - async syncUsersForLinkedUser( - integrationId: string, - linkedUserId: string, - id_project: string, - ) { + async syncUsersForLinkedUser(integrationId: string, linkedUserId: string) { try { this.logger.log( `Syncing ${integrationId} users for linkedUser ${linkedUserId}`, @@ -137,6 +139,17 @@ export class SyncService implements OnModuleInit { const sourceObject: OriginalUserOutput[] = resp.data; + await this.ingestService.ingestData< + UnifiedCompanyOutput, + OriginalCompanyOutput + >( + sourceObject, + integrationId, + connection.id_connection, + 'crm', + 'company', + customFieldMappings, + ); // unify the data according to the target obj wanted const unifiedObject = (await this.coreUnification.unify< OriginalUserOutput[] @@ -169,7 +182,7 @@ export class SyncService implements OnModuleInit { id_linked_user: linkedUserId, }, }); - await this.webhook.handleWebhook( + await this.webhook.dispatchWebhook( users_data, 'ats.user.pulled', id_project, diff --git a/packages/api/src/ats/user/user.module.ts b/packages/api/src/ats/user/user.module.ts index fe7267e16..b65b2ef45 100644 --- a/packages/api/src/ats/user/user.module.ts +++ b/packages/api/src/ats/user/user.module.ts @@ -8,6 +8,7 @@ import { ServiceRegistry } from './services/registry.service'; import { UserService } from './services/user.service'; import { SyncService } from './sync/sync.service'; import { UserController } from './user.controller'; +import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; @Module({ controllers: [UserController], @@ -20,6 +21,7 @@ import { UserController } from './user.controller'; FieldMappingService, ServiceRegistry, ConnectionUtils, + IngestDataService, /* PROVIDERS SERVICES */ ], exports: [SyncService], diff --git a/packages/api/src/crm/@webhook/handler.module.ts b/packages/api/src/crm/@webhook/handler.module.ts index 15b7f4afb..edbb5fd31 100644 --- a/packages/api/src/crm/@webhook/handler.module.ts +++ b/packages/api/src/crm/@webhook/handler.module.ts @@ -11,6 +11,7 @@ import { CrmWebhookHandlerService } from './handler.service'; EncryptionService, EnvironmentService, CrmWebhookHandlerService, + IngestDataService, /* PROVIDERS SERVICES */ ], exports: [LoggerService, CrmWebhookHandlerService], diff --git a/packages/api/src/crm/company/company.module.ts b/packages/api/src/crm/company/company.module.ts index c7cc0a6b5..7b51a0363 100644 --- a/packages/api/src/crm/company/company.module.ts +++ b/packages/api/src/crm/company/company.module.ts @@ -19,6 +19,7 @@ import { ZendeskService } from './services/zendesk'; import { ZohoService } from './services/zoho'; import { SyncService } from './sync/sync.service'; import { CoreUnification } from '@@core/@core-services/unification/core-unification.service'; +import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; @Module({ imports: [ @@ -41,6 +42,7 @@ import { CoreUnification } from '@@core/@core-services/unification/core-unificat UnificationRegistry, MappersRegistry, Utils, + IngestDataService, /* PROVIDERS SERVICES */ ZendeskService, ZohoService, diff --git a/packages/api/src/crm/company/services/company.service.ts b/packages/api/src/crm/company/services/company.service.ts index acd22fb83..dc895c914 100644 --- a/packages/api/src/crm/company/services/company.service.ts +++ b/packages/api/src/crm/company/services/company.service.ts @@ -336,7 +336,7 @@ export class CompanyService { }, }); - await this.webhook.handleWebhook( + await this.webhook.dispatchWebhook( result_company, 'crm.company.created', linkedUser.id_project, diff --git a/packages/api/src/crm/company/sync/sync.service.ts b/packages/api/src/crm/company/sync/sync.service.ts index 795a29a47..eed7a174e 100644 --- a/packages/api/src/crm/company/sync/sync.service.ts +++ b/packages/api/src/crm/company/sync/sync.service.ts @@ -6,8 +6,6 @@ import { ApiResponse } from '@@core/utils/types'; import { v4 as uuidv4 } from 'uuid'; import { FieldMappingService } from '@@core/field-mapping/field-mapping.service'; import { ServiceRegistry } from '../services/registry.service'; -import { CrmObject } from '@crm/@lib/@types'; -import { WebhookService } from '@@core/@core-services/webhooks/panora-webhooks/webhook.service'; import { UnifiedCompanyOutput } from '../types/model.unified'; import { ICompanyService } from '../types'; import { OriginalCompanyOutput } from '@@core/utils/types/original/original.crm'; @@ -16,20 +14,20 @@ import { CRM_PROVIDERS } from '@panora/shared'; import { Utils } from '@crm/@lib/@utils'; import { CoreSyncRegistry } from '@@core/@core-services/registries/core-sync.registry'; import { BullQueueService } from '@@core/@core-services/queues/shared.service'; -import { CoreUnification } from '@@core/@core-services/unification/core-unification.service'; +import { IBaseSync } from '@@core/utils/types/interface'; +import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; @Injectable() -export class SyncService implements OnModuleInit { +export class SyncService implements OnModuleInit, IBaseSync { constructor( private prisma: PrismaService, private logger: LoggerService, - private webhook: WebhookService, private fieldMappingService: FieldMappingService, private serviceRegistry: ServiceRegistry, private utils: Utils, - private coreUnification: CoreUnification, private registry: CoreSyncRegistry, private bullQueueService: BullQueueService, + private ingestService: IngestDataService, ) { this.logger.setContext(SyncService.name); this.registry.registerService('crm', 'company', this); @@ -86,7 +84,6 @@ export class SyncService implements OnModuleInit { await this.syncCompaniesForLinkedUser( provider, linkedUser.id_linked_user, - id_project, ); } catch (error) { throw error; @@ -108,7 +105,6 @@ export class SyncService implements OnModuleInit { async syncCompaniesForLinkedUser( integrationId: string, linkedUserId: string, - id_project: string, ) { try { this.logger.log( @@ -145,62 +141,34 @@ export class SyncService implements OnModuleInit { await service.syncCompanies(linkedUserId, remoteProperties); const sourceObject: OriginalCompanyOutput[] = resp.data; - //this.logger.log('SOURCE OBJECT DATA = ' + JSON.stringify(sourceObject)); - //unify the data according to the target obj wanted - const unifiedObject = (await this.coreUnification.unify< - OriginalCompanyOutput[] - >({ - sourceObject, - targetType: CrmObject.company, - providerName: integrationId, - vertical: 'crm', - connectionId: connection.id_connection, - customFieldMappings, - })) as UnifiedCompanyOutput[]; - //insert the data in the DB with the fieldMappings (value table) - const companies_data = await this.saveCompanysInDb( - connection.id_connection, - linkedUserId, - unifiedObject, - integrationId, + await this.ingestService.ingestData< + UnifiedCompanyOutput, + OriginalCompanyOutput + >( sourceObject, - ); - const event = await this.prisma.events.create({ - data: { - id_event: uuidv4(), - status: 'success', - type: 'crm.company.synced', - method: 'SYNC', - url: '/sync', - provider: integrationId, - direction: '0', - timestamp: new Date(), - id_linked_user: linkedUserId, - }, - }); - await this.webhook.handleWebhook( - companies_data, - 'crm.company.pulled', - id_project, - event.id_event, + integrationId, + connection.id_connection, + 'crm', + 'company', + customFieldMappings, ); } catch (error) { throw error; } } - async saveCompanysInDb( + async saveToDb( connection_id: string, linkedUserId: string, - companies: UnifiedCompanyOutput[], + data: UnifiedCompanyOutput[], originSource: string, remote_data: Record[], ): Promise { try { let companies_results: CrmCompany[] = []; - for (let i = 0; i < companies.length; i++) { - const company = companies[i]; + for (let i = 0; i < data.length; i++) { + const company = data[i]; const originId = company.remote_id; if (!originId || originId == '') { diff --git a/packages/api/src/crm/contact/contact.module.ts b/packages/api/src/crm/contact/contact.module.ts index 2b165d4c1..d7d462e1f 100644 --- a/packages/api/src/crm/contact/contact.module.ts +++ b/packages/api/src/crm/contact/contact.module.ts @@ -18,6 +18,7 @@ import { ZendeskService } from './services/zendesk'; import { ZohoService } from './services/zoho'; import { SyncService } from './sync/sync.service'; import { CoreUnification } from '@@core/@core-services/unification/core-unification.service'; +import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; @Module({ imports: [], @@ -35,6 +36,7 @@ import { CoreUnification } from '@@core/@core-services/unification/core-unificat MappersRegistry, Utils, ConnectionUtils, + IngestDataService, /* PROVIDERS SERVICES */ AttioService, ZendeskService, diff --git a/packages/api/src/crm/contact/services/contact.service.ts b/packages/api/src/crm/contact/services/contact.service.ts index d7cabe4ac..96049e5f8 100644 --- a/packages/api/src/crm/contact/services/contact.service.ts +++ b/packages/api/src/crm/contact/services/contact.service.ts @@ -368,7 +368,7 @@ export class ContactService { id_linked_user: linkedUserId, }, }); - await this.webhook.handleWebhook( + await this.webhook.dispatchWebhook( result_contact, 'crm.contact.created', linkedUser.id_project, diff --git a/packages/api/src/crm/contact/sync/sync.service.ts b/packages/api/src/crm/contact/sync/sync.service.ts index 46e0d6390..7fe9d4e14 100644 --- a/packages/api/src/crm/contact/sync/sync.service.ts +++ b/packages/api/src/crm/contact/sync/sync.service.ts @@ -17,9 +17,11 @@ import { Utils } from '@crm/@lib/@utils'; import { CoreSyncRegistry } from '@@core/@core-services/registries/core-sync.registry'; import { CoreUnification } from '@@core/@core-services/unification/core-unification.service'; import { BullQueueService } from '@@core/@core-services/queues/shared.service'; +import { IBaseSync } from '@@core/utils/types/interface'; +import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; @Injectable() -export class SyncService implements OnModuleInit { +export class SyncService implements OnModuleInit, IBaseSync { constructor( private prisma: PrismaService, private logger: LoggerService, @@ -30,6 +32,7 @@ export class SyncService implements OnModuleInit { private coreUnification: CoreUnification, private registry: CoreSyncRegistry, private bullQueueService: BullQueueService, + private ingestService: IngestDataService, ) { this.logger.setContext(SyncService.name); this.registry.registerService('crm', 'contact', this); @@ -86,7 +89,6 @@ export class SyncService implements OnModuleInit { await this.syncContactsForLinkedUser( provider, linkedUser.id_linked_user, - id_project, ); } catch (error) { throw error; @@ -105,11 +107,7 @@ export class SyncService implements OnModuleInit { } //todo: HANDLE DATA REMOVED FROM PROVIDER - async syncContactsForLinkedUser( - integrationId: string, - linkedUserId: string, - id_project: string, - ) { + async syncContactsForLinkedUser(integrationId: string, linkedUserId: string) { try { this.logger.log( `Syncing ${integrationId} contacts for linkedUser ${linkedUserId}`, @@ -145,62 +143,34 @@ export class SyncService implements OnModuleInit { await service.syncContacts(linkedUserId, remoteProperties); const sourceObject: OriginalContactOutput[] = resp.data; - //this.logger.log('SOURCE OBJECT DATA = ' + JSON.stringify(sourceObject)); - //unify the data according to the target obj wanted - const unifiedObject = (await this.coreUnification.unify< - OriginalContactOutput[] - >({ - sourceObject, - targetType: CrmObject.contact, - providerName: integrationId, - vertical: 'crm', - connectionId: connection.id_connection, - customFieldMappings, - })) as UnifiedContactOutput[]; - //insert the data in the DB with the fieldMappings (value table) - const contacts_data = await this.saveContactsInDb( - connection.id_connection, - linkedUserId, - unifiedObject, - integrationId, + await this.ingestService.ingestData< + OriginalContactOutput, + OriginalContactOutput + >( sourceObject, - ); - const event = await this.prisma.events.create({ - data: { - id_event: uuidv4(), - status: 'success', - type: 'crm.contact.synced', - method: 'SYNC', - url: '/sync', - provider: integrationId, - direction: '0', - timestamp: new Date(), - id_linked_user: linkedUserId, - }, - }); - await this.webhook.handleWebhook( - contacts_data, - 'crm.contact.pulled', - id_project, - event.id_event, + integrationId, + connection.id_connection, + 'crm', + 'contact', + customFieldMappings, ); } catch (error) { throw error; } } - async saveContactsInDb( + async saveToDb( connection_id: string, linkedUserId: string, - contacts: UnifiedContactOutput[], + data: UnifiedContactOutput[], originSource: string, remote_data: Record[], ): Promise { try { let contacts_results: CrmContact[] = []; - for (let i = 0; i < contacts.length; i++) { - const contact = contacts[i]; + for (let i = 0; i < data.length; i++) { + const contact = data[i]; const originId = contact.remote_id; if (!originId || originId == '') { diff --git a/packages/api/src/crm/deal/deal.module.ts b/packages/api/src/crm/deal/deal.module.ts index 1d207b534..60d52008e 100644 --- a/packages/api/src/crm/deal/deal.module.ts +++ b/packages/api/src/crm/deal/deal.module.ts @@ -17,6 +17,7 @@ import { ZendeskService } from './services/zendesk'; import { ZohoService } from './services/zoho'; import { SyncService } from './sync/sync.service'; import { CoreUnification } from '@@core/@core-services/unification/core-unification.service'; +import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; @Module({ imports: [], @@ -33,6 +34,7 @@ import { CoreUnification } from '@@core/@core-services/unification/core-unificat CoreUnification, UnificationRegistry, MappersRegistry, + IngestDataService, Utils, /* PROVIDERS SERVICES */ ZendeskService, diff --git a/packages/api/src/crm/deal/services/deal.service.ts b/packages/api/src/crm/deal/services/deal.service.ts index 45b61c6e7..641e2c880 100644 --- a/packages/api/src/crm/deal/services/deal.service.ts +++ b/packages/api/src/crm/deal/services/deal.service.ts @@ -221,7 +221,7 @@ export class DealService { id_linked_user: linkedUserId, }, }); - await this.webhook.handleWebhook( + await this.webhook.dispatchWebhook( result_deal, 'crm.deal.created', linkedUser.id_project, diff --git a/packages/api/src/crm/deal/sync/sync.service.ts b/packages/api/src/crm/deal/sync/sync.service.ts index 0c4b67468..ea60487f5 100644 --- a/packages/api/src/crm/deal/sync/sync.service.ts +++ b/packages/api/src/crm/deal/sync/sync.service.ts @@ -6,8 +6,6 @@ import { ApiResponse } from '@@core/utils/types'; import { v4 as uuidv4 } from 'uuid'; import { FieldMappingService } from '@@core/field-mapping/field-mapping.service'; import { ServiceRegistry } from '../services/registry.service'; -import { CrmObject } from '@crm/@lib/@types'; -import { WebhookService } from '@@core/@core-services/webhooks/panora-webhooks/webhook.service'; import { UnifiedDealOutput } from '../types/model.unified'; import { IDealService } from '../types'; import { OriginalDealOutput } from '@@core/utils/types/original/original.crm'; @@ -15,19 +13,19 @@ import { crm_deals as CrmDeal } from '@prisma/client'; import { CRM_PROVIDERS } from '@panora/shared'; import { CoreSyncRegistry } from '@@core/@core-services/registries/core-sync.registry'; import { BullQueueService } from '@@core/@core-services/queues/shared.service'; -import { CoreUnification } from '@@core/@core-services/unification/core-unification.service'; +import { IBaseSync } from '@@core/utils/types/interface'; +import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; @Injectable() -export class SyncService implements OnModuleInit { +export class SyncService implements OnModuleInit, IBaseSync { constructor( private prisma: PrismaService, private logger: LoggerService, - private webhook: WebhookService, private fieldMappingService: FieldMappingService, private serviceRegistry: ServiceRegistry, - private coreUnification: CoreUnification, private registry: CoreSyncRegistry, private bullQueueService: BullQueueService, + private ingestService: IngestDataService, ) { this.logger.setContext(SyncService.name); this.registry.registerService('crm', 'deal', this); @@ -81,7 +79,6 @@ export class SyncService implements OnModuleInit { await this.syncDealsForLinkedUser( provider, linkedUser.id_linked_user, - id_project, ); } catch (error) { throw error; @@ -100,11 +97,7 @@ export class SyncService implements OnModuleInit { } //todo: HANDLE DATA REMOVED FROM PROVIDER - async syncDealsForLinkedUser( - integrationId: string, - linkedUserId: string, - id_project: string, - ) { + async syncDealsForLinkedUser(integrationId: string, linkedUserId: string) { try { this.logger.log( `Syncing ${integrationId} deals for linkedUser ${linkedUserId}`, @@ -142,62 +135,34 @@ export class SyncService implements OnModuleInit { ); const sourceObject: OriginalDealOutput[] = resp.data; - // this.logger.log('SOURCE OBJECT DATA = ' + JSON.stringify(sourceObject)); - //unify the data according to the target obj wanted - const unifiedObject = (await this.coreUnification.unify< - OriginalDealOutput[] - >({ - sourceObject, - targetType: CrmObject.deal, - providerName: integrationId, - vertical: 'crm', - connectionId: connection.id_connection, - customFieldMappings, - })) as UnifiedDealOutput[]; - //insert the data in the DB with the fieldMappings (value table) - const deals_data = await this.saveDealsInDb( - connection.id_connection, - linkedUserId, - unifiedObject, - integrationId, + await this.ingestService.ingestData< + UnifiedDealOutput, + OriginalDealOutput + >( sourceObject, - ); - const event = await this.prisma.events.create({ - data: { - id_event: uuidv4(), - status: 'success', - type: 'crm.deal.synced', - method: 'SYNC', - url: '/sync', - provider: integrationId, - direction: '0', - timestamp: new Date(), - id_linked_user: linkedUserId, - }, - }); - await this.webhook.handleWebhook( - deals_data, - 'crm.deal.pulled', - id_project, - event.id_event, + integrationId, + connection.id_connection, + 'crm', + 'deal', + customFieldMappings, ); } catch (error) { throw error; } } - async saveDealsInDb( + async saveToDb( connection_id: string, linkedUserId: string, - deals: UnifiedDealOutput[], + data: UnifiedDealOutput[], originSource: string, remote_data: Record[], ): Promise { try { let deals_results: CrmDeal[] = []; - for (let i = 0; i < deals.length; i++) { - const deal = deals[i]; + for (let i = 0; i < data.length; i++) { + const deal = data[i]; const originId = deal.remote_id; if (!originId || originId == '') { diff --git a/packages/api/src/crm/engagement/engagement.module.ts b/packages/api/src/crm/engagement/engagement.module.ts index 6a5fce210..d80eaf24a 100644 --- a/packages/api/src/crm/engagement/engagement.module.ts +++ b/packages/api/src/crm/engagement/engagement.module.ts @@ -17,6 +17,7 @@ import { ZendeskService } from './services/zendesk'; import { ZohoService } from './services/zoho'; import { SyncService } from './sync/sync.service'; import { CoreUnification } from '@@core/@core-services/unification/core-unification.service'; +import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; @Module({ imports: [], @@ -34,6 +35,7 @@ import { CoreUnification } from '@@core/@core-services/unification/core-unificat UnificationRegistry, MappersRegistry, Utils, + IngestDataService, /* PROVIDERS SERVICES */ ZendeskService, ZohoService, diff --git a/packages/api/src/crm/engagement/services/engagement.service.ts b/packages/api/src/crm/engagement/services/engagement.service.ts index 82d0f52a7..302ab1043 100644 --- a/packages/api/src/crm/engagement/services/engagement.service.ts +++ b/packages/api/src/crm/engagement/services/engagement.service.ts @@ -260,7 +260,7 @@ export class EngagementService { id_linked_user: linkedUserId, }, }); - await this.webhook.handleWebhook( + await this.webhook.dispatchWebhook( result_engagement, 'crm.engagement.created', linkedUser.id_project, diff --git a/packages/api/src/crm/engagement/sync/sync.service.ts b/packages/api/src/crm/engagement/sync/sync.service.ts index 034069872..87f5fd106 100644 --- a/packages/api/src/crm/engagement/sync/sync.service.ts +++ b/packages/api/src/crm/engagement/sync/sync.service.ts @@ -15,10 +15,12 @@ import { OriginalEngagementOutput } from '@@core/utils/types/original/original.c import { CRM_PROVIDERS } from '@panora/shared'; import { CoreSyncRegistry } from '@@core/@core-services/registries/core-sync.registry'; import { BullQueueService } from '@@core/@core-services/queues/shared.service'; +import { IBaseSync } from '@@core/utils/types/interface'; +import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; import { CoreUnification } from '@@core/@core-services/unification/core-unification.service'; @Injectable() -export class SyncService implements OnModuleInit { +export class SyncService implements OnModuleInit, IBaseSync { constructor( private prisma: PrismaService, private logger: LoggerService, @@ -28,6 +30,7 @@ export class SyncService implements OnModuleInit { private coreUnification: CoreUnification, private registry: CoreSyncRegistry, private bullQueueService: BullQueueService, + private ingestService: IngestDataService, ) { this.logger.setContext(SyncService.name); this.registry.registerService('crm', 'engagement', this); @@ -85,7 +88,6 @@ export class SyncService implements OnModuleInit { await this.syncEngagementsForLinkedUser( provider, linkedUser.id_linked_user, - id_project, type, ); } @@ -109,7 +111,6 @@ export class SyncService implements OnModuleInit { async syncEngagementsForLinkedUser( integrationId: string, linkedUserId: string, - id_project: string, engagement_type: string, ) { try { @@ -151,64 +152,34 @@ export class SyncService implements OnModuleInit { ); const sourceObject: OriginalEngagementOutput[] = resp.data; - //this.logger.log('SOURCE OBJECT DATA = ' + JSON.stringify(sourceObject)); - //unify the data according to the target obj wanted - const unifiedObject = (await this.coreUnification.unify< - OriginalEngagementOutput[] - >({ - sourceObject, - targetType: `engagement${ - engagement_type ? `_${engagement_type}` : '' - }` as CrmObject, - providerName: integrationId, - vertical: 'crm', - connectionId: connection.id_connection, - customFieldMappings, - })) as UnifiedEngagementOutput[]; - //insert the data in the DB with the fieldMappings (value table) - const engagements_data = await this.saveEngagementsInDb( - connection.id_connection, - linkedUserId, - unifiedObject, - integrationId, + await this.ingestService.ingestData< + UnifiedEngagementOutput, + OriginalEngagementOutput + >( sourceObject, - ); - const event = await this.prisma.events.create({ - data: { - id_event: uuidv4(), - status: 'success', - type: 'crm.engagement.synced', - method: 'SYNC', - url: '/sync', - provider: integrationId, - direction: '0', - timestamp: new Date(), - id_linked_user: linkedUserId, - }, - }); - await this.webhook.handleWebhook( - engagements_data, - 'crm.engagement.pulled', - id_project, - event.id_event, + integrationId, + connection.id_connection, + 'crm', + 'engagement', + customFieldMappings, ); } catch (error) { throw error; } } - async saveEngagementsInDb( + async saveToDb( connection_id: string, linkedUserId: string, - engagements: UnifiedEngagementOutput[], + data: UnifiedEngagementOutput[], originSource: string, remote_data: Record[], ): Promise { try { let engagements_results: CrmEngagement[] = []; - for (let i = 0; i < engagements.length; i++) { - const engagement = engagements[i]; + for (let i = 0; i < data.length; i++) { + const engagement = data[i]; const originId = engagement.remote_id; if (!originId || originId == '') { diff --git a/packages/api/src/crm/note/note.module.ts b/packages/api/src/crm/note/note.module.ts index a1c59a958..7ee5c7f32 100644 --- a/packages/api/src/crm/note/note.module.ts +++ b/packages/api/src/crm/note/note.module.ts @@ -17,6 +17,7 @@ import { ZendeskService } from './services/zendesk'; import { ZohoService } from './services/zoho'; import { SyncService } from './sync/sync.service'; import { CoreUnification } from '@@core/@core-services/unification/core-unification.service'; +import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; @Module({ imports: [], @@ -34,6 +35,7 @@ import { CoreUnification } from '@@core/@core-services/unification/core-unificat UnificationRegistry, MappersRegistry, Utils, + IngestDataService, /* PROVIDERS SERVICES */ ZendeskService, ZohoService, diff --git a/packages/api/src/crm/note/services/note.service.ts b/packages/api/src/crm/note/services/note.service.ts index 77da3373d..2c2147c73 100644 --- a/packages/api/src/crm/note/services/note.service.ts +++ b/packages/api/src/crm/note/services/note.service.ts @@ -240,7 +240,7 @@ export class NoteService { id_linked_user: linkedUserId, }, }); - await this.webhook.handleWebhook( + await this.webhook.dispatchWebhook( result_note, 'crm.note.created', linkedUser.id_project, diff --git a/packages/api/src/crm/note/sync/sync.service.ts b/packages/api/src/crm/note/sync/sync.service.ts index ef95c45fd..6d3b0e4bc 100644 --- a/packages/api/src/crm/note/sync/sync.service.ts +++ b/packages/api/src/crm/note/sync/sync.service.ts @@ -15,10 +15,12 @@ import { OriginalNoteOutput } from '@@core/utils/types/original/original.crm'; import { CRM_PROVIDERS } from '@panora/shared'; import { CoreSyncRegistry } from '@@core/@core-services/registries/core-sync.registry'; import { BullQueueService } from '@@core/@core-services/queues/shared.service'; +import { IBaseSync } from '@@core/utils/types/interface'; +import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; import { CoreUnification } from '@@core/@core-services/unification/core-unification.service'; @Injectable() -export class SyncService implements OnModuleInit { +export class SyncService implements OnModuleInit, IBaseSync { constructor( private prisma: PrismaService, private logger: LoggerService, @@ -28,6 +30,7 @@ export class SyncService implements OnModuleInit { private coreUnification: CoreUnification, private registry: CoreSyncRegistry, private bullQueueService: BullQueueService, + private ingestService: IngestDataService, ) { this.logger.setContext(SyncService.name); this.registry.registerService('crm', 'note', this); @@ -81,7 +84,6 @@ export class SyncService implements OnModuleInit { await this.syncNotesForLinkedUser( provider, linkedUser.id_linked_user, - id_project, ); } catch (error) { throw error; @@ -100,11 +102,7 @@ export class SyncService implements OnModuleInit { } //todo: HANDLE DATA REMOVED FROM PROVIDER - async syncNotesForLinkedUser( - integrationId: string, - linkedUserId: string, - id_project: string, - ) { + async syncNotesForLinkedUser(integrationId: string, linkedUserId: string) { try { this.logger.log( `Syncing ${integrationId} notes for linkedUser ${linkedUserId}`, @@ -142,62 +140,34 @@ export class SyncService implements OnModuleInit { ); const sourceObject: OriginalNoteOutput[] = resp.data; - //this.logger.log('SOURCE OBJECT DATA = ' + JSON.stringify(sourceObject)); - //unify the data according to the target obj wanted - const unifiedObject = (await this.coreUnification.unify< - OriginalNoteOutput[] - >({ - sourceObject, - targetType: CrmObject.note, - providerName: integrationId, - vertical: 'crm', - connectionId: connection.id_connection, - customFieldMappings, - })) as UnifiedNoteOutput[]; - //insert the data in the DB with the fieldMappings (value table) - const notes_data = await this.saveNotesInDb( - connection.id_connection, - linkedUserId, - unifiedObject, - integrationId, + await this.ingestService.ingestData< + UnifiedNoteOutput, + OriginalNoteOutput + >( sourceObject, - ); - const event = await this.prisma.events.create({ - data: { - id_event: uuidv4(), - status: 'success', - type: 'crm.note.synced', - method: 'SYNC', - url: '/sync', - provider: integrationId, - direction: '0', - timestamp: new Date(), - id_linked_user: linkedUserId, - }, - }); - await this.webhook.handleWebhook( - notes_data, - 'crm.note.pulled', - id_project, - event.id_event, + integrationId, + connection.id_connection, + 'crm', + 'note', + customFieldMappings, ); } catch (error) { throw error; } } - async saveNotesInDb( + async saveToDb( connection_id: string, linkedUserId: string, - notes: UnifiedNoteOutput[], + data: UnifiedNoteOutput[], originSource: string, remote_data: Record[], ): Promise { try { let notes_results: CrmNote[] = []; - for (let i = 0; i < notes.length; i++) { - const note = notes[i]; + for (let i = 0; i < data.length; i++) { + const note = data[i]; const originId = note.remote_id; if (!originId || originId == '') { diff --git a/packages/api/src/crm/stage/stage.module.ts b/packages/api/src/crm/stage/stage.module.ts index 79bedd85c..1f2f88356 100644 --- a/packages/api/src/crm/stage/stage.module.ts +++ b/packages/api/src/crm/stage/stage.module.ts @@ -17,6 +17,7 @@ import { ZohoService } from './services/zoho'; import { StageController } from './stage.controller'; import { SyncService } from './sync/sync.service'; import { CoreUnification } from '@@core/@core-services/unification/core-unification.service'; +import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; @Module({ imports: [], @@ -34,6 +35,7 @@ import { CoreUnification } from '@@core/@core-services/unification/core-unificat UnificationRegistry, MappersRegistry, Utils, + IngestDataService, /* PROVIDERS SERVICES */ ZendeskService, ZohoService, diff --git a/packages/api/src/crm/stage/sync/sync.service.ts b/packages/api/src/crm/stage/sync/sync.service.ts index 62db5e9ce..bd65080d5 100644 --- a/packages/api/src/crm/stage/sync/sync.service.ts +++ b/packages/api/src/crm/stage/sync/sync.service.ts @@ -15,10 +15,12 @@ import { OriginalStageOutput } from '@@core/utils/types/original/original.crm'; import { CRM_PROVIDERS } from '@panora/shared'; import { CoreSyncRegistry } from '@@core/@core-services/registries/core-sync.registry'; import { BullQueueService } from '@@core/@core-services/queues/shared.service'; +import { IBaseSync } from '@@core/utils/types/interface'; +import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; import { CoreUnification } from '@@core/@core-services/unification/core-unification.service'; @Injectable() -export class SyncService implements OnModuleInit { +export class SyncService implements OnModuleInit, IBaseSync { constructor( private prisma: PrismaService, private logger: LoggerService, @@ -28,6 +30,7 @@ export class SyncService implements OnModuleInit { private coreUnification: CoreUnification, private registry: CoreSyncRegistry, private bullQueueService: BullQueueService, + private ingestService: IngestDataService, ) { this.logger.setContext(SyncService.name); this.registry.registerService('crm', 'stage', this); @@ -96,7 +99,6 @@ export class SyncService implements OnModuleInit { await this.syncStagesForLinkedUser( provider, linkedUser.id_linked_user, - id_project, deal.id_crm_deal, ); } @@ -123,7 +125,6 @@ export class SyncService implements OnModuleInit { async syncStagesForLinkedUser( integrationId: string, linkedUserId: string, - id_project: string, deal_id: string, ) { try { @@ -164,64 +165,35 @@ export class SyncService implements OnModuleInit { ); const sourceObject: OriginalStageOutput[] = resp.data; - //this.logger.log('SOURCE OBJECT DATA = ' + JSON.stringify(sourceObject)); - //unify the data according to the target obj wanted - const unifiedObject = (await this.coreUnification.unify< - OriginalStageOutput[] - >({ - sourceObject, - targetType: CrmObject.stage, - providerName: integrationId, - vertical: 'crm', - connectionId: connection.id_connection, - customFieldMappings, - })) as UnifiedStageOutput[]; - //insert the data in the DB with the fieldMappings (value table) - const stages_data = await this.saveStagesInDb( - connection.id_connection, - linkedUserId, - unifiedObject, - integrationId, - deal_id, + await this.ingestService.ingestData< + UnifiedStageOutput, + OriginalStageOutput + >( sourceObject, - ); - const event = await this.prisma.events.create({ - data: { - id_event: uuidv4(), - status: 'success', - type: 'crm.stage.synced', - method: 'SYNC', - url: '/sync', - provider: integrationId, - direction: '0', - timestamp: new Date(), - id_linked_user: linkedUserId, - }, - }); - await this.webhook.handleWebhook( - stages_data, - 'crm.stage.pulled', - id_project, - event.id_event, + integrationId, + connection.id_connection, + 'crm', + 'stage', + customFieldMappings, ); } catch (error) { throw error; } } - async saveStagesInDb( + async saveToDb( connection_id: string, linkedUserId: string, - stages: UnifiedStageOutput[], + data: UnifiedStageOutput[], originSource: string, deal_id: string, remote_data: Record[], ): Promise { try { let stages_results: CrmStage[] = []; - for (let i = 0; i < stages.length; i++) { - const stage = stages[i]; + for (let i = 0; i < data.length; i++) { + const stage = data[i]; const originId = stage.remote_id; if (!originId || originId == '') { diff --git a/packages/api/src/crm/task/services/task.service.ts b/packages/api/src/crm/task/services/task.service.ts index 6b7cd7e53..86c943784 100644 --- a/packages/api/src/crm/task/services/task.service.ts +++ b/packages/api/src/crm/task/services/task.service.ts @@ -244,7 +244,7 @@ export class TaskService { id_linked_user: linkedUserId, }, }); - await this.webhook.handleWebhook( + await this.webhook.dispatchWebhook( result_task, 'crm.task.created', linkedUser.id_project, diff --git a/packages/api/src/crm/task/sync/sync.service.ts b/packages/api/src/crm/task/sync/sync.service.ts index f064a960d..b87aa7984 100644 --- a/packages/api/src/crm/task/sync/sync.service.ts +++ b/packages/api/src/crm/task/sync/sync.service.ts @@ -16,10 +16,12 @@ import { CRM_PROVIDERS } from '@panora/shared'; import { throwTypedError, SyncError } from '@@core/utils/errors'; import { CoreSyncRegistry } from '@@core/@core-services/registries/core-sync.registry'; import { BullQueueService } from '@@core/@core-services/queues/shared.service'; +import { IBaseSync } from '@@core/utils/types/interface'; +import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; import { CoreUnification } from '@@core/@core-services/unification/core-unification.service'; @Injectable() -export class SyncService implements OnModuleInit { +export class SyncService implements OnModuleInit, IBaseSync { constructor( private prisma: PrismaService, private logger: LoggerService, @@ -29,6 +31,7 @@ export class SyncService implements OnModuleInit { private coreUnification: CoreUnification, private registry: CoreSyncRegistry, private bullQueueService: BullQueueService, + private ingestService: IngestDataService, ) { this.logger.setContext(SyncService.name); this.registry.registerService('crm', 'task', this); @@ -82,7 +85,6 @@ export class SyncService implements OnModuleInit { await this.syncTasksForLinkedUser( provider, linkedUser.id_linked_user, - id_project, ); } catch (error) { throw error; @@ -101,11 +103,7 @@ export class SyncService implements OnModuleInit { } //todo: HANDLE DATA REMOVED FROM PROVIDER - async syncTasksForLinkedUser( - integrationId: string, - linkedUserId: string, - id_project: string, - ) { + async syncTasksForLinkedUser(integrationId: string, linkedUserId: string) { try { this.logger.log( `Syncing ${integrationId} tasks for linkedUser ${linkedUserId}`, @@ -143,62 +141,34 @@ export class SyncService implements OnModuleInit { ); const sourceObject: OriginalTaskOutput[] = resp.data; - //this.logger.log('SOURCE OBJECT DATA = ' + JSON.stringify(sourceObject)); - //unify the data according to the target obj wanted - const unifiedObject = (await this.coreUnification.unify< - OriginalTaskOutput[] - >({ - sourceObject, - targetType: CrmObject.task, - providerName: integrationId, - vertical: 'crm', - connectionId: connection.id_connection, - customFieldMappings, - })) as UnifiedTaskOutput[]; - //insert the data in the DB with the fieldMappings (value table) - const tasks_data = await this.saveTasksInDb( - connection.id_connection, - linkedUserId, - unifiedObject, - integrationId, + await this.ingestService.ingestData< + UnifiedTaskOutput, + OriginalTaskOutput + >( sourceObject, - ); - const event = await this.prisma.events.create({ - data: { - id_event: uuidv4(), - status: 'success', - type: 'crm.task.synced', - method: 'SYNC', - url: '/sync', - provider: integrationId, - direction: '0', - timestamp: new Date(), - id_linked_user: linkedUserId, - }, - }); - await this.webhook.handleWebhook( - tasks_data, - 'crm.task.pulled', - id_project, - event.id_event, + integrationId, + connection.id_connection, + 'crm', + 'task', + customFieldMappings, ); } catch (error) { throw error; } } - async saveTasksInDb( + async saveToDb( connection_id: string, linkedUserId: string, - tasks: UnifiedTaskOutput[], + data: UnifiedTaskOutput[], originSource: string, remote_data: Record[], ): Promise { try { let tasks_results: CrmTask[] = []; - for (let i = 0; i < tasks.length; i++) { - const task = tasks[i]; + for (let i = 0; i < data.length; i++) { + const task = data[i]; const originId = task.remote_id; if (!originId || originId == '') { diff --git a/packages/api/src/crm/task/task.module.ts b/packages/api/src/crm/task/task.module.ts index 0826b990e..1c310d433 100644 --- a/packages/api/src/crm/task/task.module.ts +++ b/packages/api/src/crm/task/task.module.ts @@ -17,6 +17,7 @@ import { ZohoService } from './services/zoho'; import { SyncService } from './sync/sync.service'; import { TaskController } from './task.controller'; import { CoreUnification } from '@@core/@core-services/unification/core-unification.service'; +import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; @Module({ imports: [], @@ -34,6 +35,7 @@ import { CoreUnification } from '@@core/@core-services/unification/core-unificat UnificationRegistry, MappersRegistry, Utils, + IngestDataService, /* PROVIDERS SERVICES */ ZendeskService, ZohoService, diff --git a/packages/api/src/crm/user/sync/sync.service.ts b/packages/api/src/crm/user/sync/sync.service.ts index dbccf833d..6ef2c9dab 100644 --- a/packages/api/src/crm/user/sync/sync.service.ts +++ b/packages/api/src/crm/user/sync/sync.service.ts @@ -15,10 +15,12 @@ import { OriginalUserOutput } from '@@core/utils/types/original/original.crm'; import { CRM_PROVIDERS } from '@panora/shared'; import { CoreSyncRegistry } from '@@core/@core-services/registries/core-sync.registry'; import { BullQueueService } from '@@core/@core-services/queues/shared.service'; +import { IBaseSync } from '@@core/utils/types/interface'; +import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; import { CoreUnification } from '@@core/@core-services/unification/core-unification.service'; @Injectable() -export class SyncService implements OnModuleInit { +export class SyncService implements OnModuleInit, IBaseSync { constructor( private prisma: PrismaService, private logger: LoggerService, @@ -28,6 +30,7 @@ export class SyncService implements OnModuleInit { private coreUnification: CoreUnification, private registry: CoreSyncRegistry, private bullQueueService: BullQueueService, + private ingestService: IngestDataService, ) { this.logger.setContext(SyncService.name); this.registry.registerService('crm', 'user', this); @@ -81,7 +84,6 @@ export class SyncService implements OnModuleInit { await this.syncUsersForLinkedUser( provider, linkedUser.id_linked_user, - id_project, ); } catch (error) { throw error; @@ -100,11 +102,7 @@ export class SyncService implements OnModuleInit { } //todo: HANDLE DATA REMOVED FROM PROVIDER - async syncUsersForLinkedUser( - integrationId: string, - linkedUserId: string, - id_project: string, - ) { + async syncUsersForLinkedUser(integrationId: string, linkedUserId: string) { try { this.logger.log( `Syncing ${integrationId} users for linkedUser ${linkedUserId}`, @@ -142,62 +140,34 @@ export class SyncService implements OnModuleInit { ); const sourceObject: OriginalUserOutput[] = resp.data; - //this.logger.log('SOURCE OBJECT DATA = ' + JSON.stringify(sourceObject)); - //unify the data according to the target obj wanted - const unifiedObject = (await this.coreUnification.unify< - OriginalUserOutput[] - >({ - sourceObject, - targetType: CrmObject.user, - providerName: integrationId, - vertical: 'crm', - connectionId: connection.id_connection, - customFieldMappings, - })) as UnifiedUserOutput[]; - //insert the data in the DB with the fieldMappings (value table) - const users_data = await this.saveUsersInDb( - connection.id_connection, - linkedUserId, - unifiedObject, - integrationId, + await this.ingestService.ingestData< + UnifiedUserOutput, + OriginalUserOutput + >( sourceObject, - ); - const event = await this.prisma.events.create({ - data: { - id_event: uuidv4(), - status: 'success', - type: 'crm.user.synced', - method: 'SYNC', - url: '/sync', - provider: integrationId, - direction: '0', - timestamp: new Date(), - id_linked_user: linkedUserId, - }, - }); - await this.webhook.handleWebhook( - users_data, - 'crm.user.pulled', - id_project, - event.id_event, + integrationId, + connection.id_connection, + 'crm', + 'user', + customFieldMappings, ); } catch (error) { throw error; } } - async saveUsersInDb( + async saveToDb( connection_id: string, linkedUserId: string, - users: UnifiedUserOutput[], + data: UnifiedUserOutput[], originSource: string, remote_data: Record[], ): Promise { try { let users_results: CrmUser[] = []; - for (let i = 0; i < users.length; i++) { - const user = users[i]; + for (let i = 0; i < data.length; i++) { + const user = data[i]; const originId = user.remote_id; if (!originId || originId == '') { diff --git a/packages/api/src/crm/user/user.module.ts b/packages/api/src/crm/user/user.module.ts index 0ad6d8b46..7daf1b472 100644 --- a/packages/api/src/crm/user/user.module.ts +++ b/packages/api/src/crm/user/user.module.ts @@ -17,6 +17,7 @@ import { ZohoService } from './services/zoho'; import { SyncService } from './sync/sync.service'; import { UserController } from './user.controller'; import { CoreUnification } from '@@core/@core-services/unification/core-unification.service'; +import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; @Module({ imports: [], @@ -34,6 +35,7 @@ import { CoreUnification } from '@@core/@core-services/unification/core-unificat UnificationRegistry, MappersRegistry, Utils, + IngestDataService, /* PROVIDERS SERVICES */ ZendeskService, ZohoService, diff --git a/packages/api/src/filestorage/drive/drive.module.ts b/packages/api/src/filestorage/drive/drive.module.ts index 4585b8d0e..d1acb3489 100644 --- a/packages/api/src/filestorage/drive/drive.module.ts +++ b/packages/api/src/filestorage/drive/drive.module.ts @@ -8,6 +8,7 @@ import { DriveController } from './drive.controller'; import { DriveService } from './services/drive.service'; import { ServiceRegistry } from './services/registry.service'; import { SyncService } from './sync/sync.service'; +import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; @Module({ controllers: [DriveController], @@ -20,6 +21,7 @@ import { SyncService } from './sync/sync.service'; FieldMappingService, ServiceRegistry, ConnectionUtils, + IngestDataService, /* PROVIDERS SERVICES */ ], exports: [SyncService], diff --git a/packages/api/src/filestorage/drive/sync/sync.service.ts b/packages/api/src/filestorage/drive/sync/sync.service.ts index 0a8771c1e..a472789df 100644 --- a/packages/api/src/filestorage/drive/sync/sync.service.ts +++ b/packages/api/src/filestorage/drive/sync/sync.service.ts @@ -15,9 +15,11 @@ import { fs_drives as FileStorageDrive } from '@prisma/client'; import { FILESTORAGE_PROVIDERS } from '@panora/shared'; import { FileStorageObject } from '@filestorage/@lib/@types'; import { BullQueueService } from '@@core/@core-services/queues/shared.service'; +import { IBaseSync } from '@@core/utils/types/interface'; +import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; import { CoreUnification } from '@@core/@core-services/unification/core-unification.service'; @Injectable() -export class SyncService implements OnModuleInit { +export class SyncService implements OnModuleInit, IBaseSync { constructor( private prisma: PrismaService, private logger: LoggerService, @@ -27,6 +29,7 @@ export class SyncService implements OnModuleInit { private coreUnification: CoreUnification, private registry: CoreSyncRegistry, private bullQueueService: BullQueueService, + private ingestService: IngestDataService, ) { this.logger.setContext(SyncService.name); this.registry.registerService('filestorage', 'drive', this); @@ -78,7 +81,6 @@ export class SyncService implements OnModuleInit { await this.syncDrivesForLinkedUser( provider, linkedUser.id_linked_user, - id_project, ); } catch (error) { throw error; @@ -96,11 +98,7 @@ export class SyncService implements OnModuleInit { } } - async syncDrivesForLinkedUser( - integrationId: string, - linkedUserId: string, - id_project: string, - ) { + async syncDrivesForLinkedUser(integrationId: string, linkedUserId: string) { try { this.logger.log( `Syncing ${integrationId} drives for linkedUser ${linkedUserId}`, @@ -140,51 +138,23 @@ export class SyncService implements OnModuleInit { const sourceObject: OriginalDriveOutput[] = resp.data; - // unify the data according to the target obj wanted - const unifiedObject = (await this.coreUnification.unify< - OriginalDriveOutput[] - >({ + await this.ingestService.ingestData< + UnifiedDriveOutput, + OriginalDriveOutput + >( sourceObject, - targetType: FileStorageObject.drive, - providerName: integrationId, - vertical: 'filestorage', - connectionId: connection.id_connection, - customFieldMappings, - })) as UnifiedDriveOutput[]; - - // insert the data in the DB with the fieldMappings (value table) - const drives_data = await this.saveDrivesInDb( - connection.id_connection, - linkedUserId, - unifiedObject, integrationId, - sourceObject, - ); - const event = await this.prisma.events.create({ - data: { - id_event: uuidv4(), - status: 'success', - type: 'filestorage.drive.synced', - method: 'SYNC', - url: '/sync', - provider: integrationId, - direction: '0', - timestamp: new Date(), - id_linked_user: linkedUserId, - }, - }); - await this.webhook.handleWebhook( - drives_data, - 'filestorage.drive.pulled', - id_project, - event.id_event, + connection.id_connection, + 'filestorage', + 'drive', + customFieldMappings, ); } catch (error) { throw error; } } - async saveDrivesInDb( + async saveToDb( connection_id: string, linkedUserId: string, drives: UnifiedDriveOutput[], diff --git a/packages/api/src/filestorage/file/file.module.ts b/packages/api/src/filestorage/file/file.module.ts index b5b278186..331addc83 100644 --- a/packages/api/src/filestorage/file/file.module.ts +++ b/packages/api/src/filestorage/file/file.module.ts @@ -9,6 +9,7 @@ import { BoxService } from './services/box'; import { FileService } from './services/file.service'; import { ServiceRegistry } from './services/registry.service'; import { SyncService } from './sync/sync.service'; +import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; @Module({ controllers: [FileController], @@ -21,6 +22,7 @@ import { SyncService } from './sync/sync.service'; FieldMappingService, ServiceRegistry, ConnectionUtils, + IngestDataService, /* PROVIDERS SERVICES */ BoxService, ], diff --git a/packages/api/src/filestorage/file/services/box/mappers.ts b/packages/api/src/filestorage/file/services/box/mappers.ts index 5ce0cc0a0..d1798feff 100644 --- a/packages/api/src/filestorage/file/services/box/mappers.ts +++ b/packages/api/src/filestorage/file/services/box/mappers.ts @@ -78,7 +78,6 @@ export class BoxFileMapper implements IFileMapper { field_mappings, //remote_created_at: file.created_at || null, //remote_modified_at: file.modified_at || null, - remote_data: file, }; } } diff --git a/packages/api/src/filestorage/file/services/file.service.ts b/packages/api/src/filestorage/file/services/file.service.ts index 9cacfef72..d81f9fbe6 100644 --- a/packages/api/src/filestorage/file/services/file.service.ts +++ b/packages/api/src/filestorage/file/services/file.service.ts @@ -211,7 +211,7 @@ export class FileService { id_linked_user: linkedUserId, }, }); - await this.webhook.handleWebhook( + await this.webhook.dispatchWebhook( result_file, 'filestorage.file.created', linkedUser.id_project, diff --git a/packages/api/src/filestorage/file/sync/sync.service.ts b/packages/api/src/filestorage/file/sync/sync.service.ts index 61473f4dd..ddda92ab6 100644 --- a/packages/api/src/filestorage/file/sync/sync.service.ts +++ b/packages/api/src/filestorage/file/sync/sync.service.ts @@ -15,10 +15,12 @@ import { fs_files as FileStorageFile } from '@prisma/client'; import { FILESTORAGE_PROVIDERS } from '@panora/shared'; import { FileStorageObject } from '@filestorage/@lib/@types'; import { BullQueueService } from '@@core/@core-services/queues/shared.service'; +import { IBaseSync } from '@@core/utils/types/interface'; +import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; import { CoreUnification } from '@@core/@core-services/unification/core-unification.service'; @Injectable() -export class SyncService implements OnModuleInit { +export class SyncService implements OnModuleInit, IBaseSync { constructor( private prisma: PrismaService, private logger: LoggerService, @@ -28,6 +30,7 @@ export class SyncService implements OnModuleInit { private coreUnification: CoreUnification, private registry: CoreSyncRegistry, private bullQueueService: BullQueueService, + private ingestService: IngestDataService, ) { this.logger.setContext(SyncService.name); this.registry.registerService('filestorage', 'file', this); @@ -92,7 +95,6 @@ export class SyncService implements OnModuleInit { await this.syncFilesForLinkedUser( provider, linkedUser.id_linked_user, - id_project, folder.id_fs_folder, ); } @@ -100,7 +102,6 @@ export class SyncService implements OnModuleInit { await this.syncFilesForLinkedUser( provider, linkedUser.id_linked_user, - id_project, ); } catch (error) { throw error; @@ -121,7 +122,6 @@ export class SyncService implements OnModuleInit { async syncFilesForLinkedUser( integrationId: string, linkedUserId: string, - id_project: string, folder_id?: string, ) { try { @@ -165,51 +165,23 @@ export class SyncService implements OnModuleInit { const sourceObject: OriginalFileOutput[] = resp.data; - // unify the data according to the target obj wanted - const unifiedObject = (await this.coreUnification.unify< - OriginalFileOutput[] - >({ + await this.ingestService.ingestData< + UnifiedFileOutput, + OriginalFileOutput + >( sourceObject, - targetType: FileStorageObject.file, - providerName: integrationId, - vertical: 'filestorage', - connectionId: connection.id_connection, - customFieldMappings, - })) as UnifiedFileOutput[]; - - // insert the data in the DB with the fieldMappings (value table) - const files_data = await this.saveFilesInDb( - connection.id_connection, - linkedUserId, - unifiedObject, integrationId, - sourceObject, - ); - const event = await this.prisma.events.create({ - data: { - id_event: uuidv4(), - status: 'success', - type: 'filestorage.file.synced', - method: 'SYNC', - url: '/sync', - provider: integrationId, - direction: '0', - timestamp: new Date(), - id_linked_user: linkedUserId, - }, - }); - await this.webhook.handleWebhook( - files_data, - 'filestorage.file.pulled', - id_project, - event.id_event, + connection.id_connection, + 'filestorage', + 'file', + customFieldMappings, ); } catch (error) { throw error; } } - async saveFilesInDb( + async saveToDb( connection_id: string, linkedUserId: string, files: UnifiedFileOutput[], diff --git a/packages/api/src/filestorage/folder/folder.module.ts b/packages/api/src/filestorage/folder/folder.module.ts index ea5c2361c..7675c324b 100644 --- a/packages/api/src/filestorage/folder/folder.module.ts +++ b/packages/api/src/filestorage/folder/folder.module.ts @@ -9,6 +9,7 @@ import { BoxService } from './services/box'; import { FolderService } from './services/folder.service'; import { ServiceRegistry } from './services/registry.service'; import { SyncService } from './sync/sync.service'; +import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; @Module({ controllers: [FolderController], @@ -21,6 +22,7 @@ import { SyncService } from './sync/sync.service'; FieldMappingService, ServiceRegistry, ConnectionUtils, + IngestDataService, /* PROVIDERS SERVICES */ BoxService, ], diff --git a/packages/api/src/filestorage/folder/services/folder.service.ts b/packages/api/src/filestorage/folder/services/folder.service.ts index 5dfb3730d..efe86ccca 100644 --- a/packages/api/src/filestorage/folder/services/folder.service.ts +++ b/packages/api/src/filestorage/folder/services/folder.service.ts @@ -217,7 +217,7 @@ export class FolderService { id_linked_user: linkedUserId, }, }); - await this.webhook.handleWebhook( + await this.webhook.dispatchWebhook( result_folder, 'filestorage.folder.created', linkedUser.id_project, diff --git a/packages/api/src/filestorage/folder/sync/sync.service.ts b/packages/api/src/filestorage/folder/sync/sync.service.ts index 81d0d1001..e7dd392f1 100644 --- a/packages/api/src/filestorage/folder/sync/sync.service.ts +++ b/packages/api/src/filestorage/folder/sync/sync.service.ts @@ -1,33 +1,31 @@ -import { Injectable, OnModuleInit } from '@nestjs/common'; import { LoggerService } from '@@core/@core-services/logger/logger.service'; import { PrismaService } from '@@core/@core-services/prisma/prisma.service'; +import { BullQueueService } from '@@core/@core-services/queues/shared.service'; +import { CoreSyncRegistry } from '@@core/@core-services/registries/core-sync.registry'; +import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; +import { FieldMappingService } from '@@core/field-mapping/field-mapping.service'; +import { ApiResponse } from '@@core/utils/types'; +import { IBaseSync } from '@@core/utils/types/interface'; +import { OriginalFolderOutput } from '@@core/utils/types/original/original.file-storage'; +import { Injectable, OnModuleInit } from '@nestjs/common'; import { Cron } from '@nestjs/schedule'; +import { FILESTORAGE_PROVIDERS } from '@panora/shared'; +import { fs_folders as FileStorageFolder } from '@prisma/client'; import { v4 as uuidv4 } from 'uuid'; -import { FieldMappingService } from '@@core/field-mapping/field-mapping.service'; import { ServiceRegistry } from '../services/registry.service'; -import { WebhookService } from '@@core/@core-services/webhooks/panora-webhooks/webhook.service'; -import { CoreSyncRegistry } from '@@core/@core-services/registries/core-sync.registry'; -import { ApiResponse } from '@@core/utils/types'; import { IFolderService } from '../types'; -import { OriginalFolderOutput } from '@@core/utils/types/original/original.file-storage'; import { UnifiedFolderOutput } from '../types/model.unified'; -import { fs_folders as FileStorageFolder } from '@prisma/client'; -import { FILESTORAGE_PROVIDERS } from '@panora/shared'; -import { FileStorageObject } from '@filestorage/@lib/@types'; -import { BullQueueService } from '@@core/@core-services/queues/shared.service'; -import { CoreUnification } from '@@core/@core-services/unification/core-unification.service'; @Injectable() -export class SyncService implements OnModuleInit { +export class SyncService implements OnModuleInit, IBaseSync { constructor( private prisma: PrismaService, private logger: LoggerService, - private webhook: WebhookService, private fieldMappingService: FieldMappingService, private serviceRegistry: ServiceRegistry, - private coreUnification: CoreUnification, private registry: CoreSyncRegistry, private bullQueueService: BullQueueService, + private ingestService: IngestDataService, ) { this.logger.setContext(SyncService.name); this.registry.registerService('filestorage', 'folder', this); @@ -79,7 +77,6 @@ export class SyncService implements OnModuleInit { await this.syncFoldersForLinkedUser( provider, linkedUser.id_linked_user, - id_project, ); } catch (error) { throw error; @@ -97,11 +94,7 @@ export class SyncService implements OnModuleInit { } } - async syncFoldersForLinkedUser( - integrationId: string, - linkedUserId: string, - id_project: string, - ) { + async syncFoldersForLinkedUser(integrationId: string, linkedUserId: string) { try { this.logger.log( `Syncing ${integrationId} folders for linkedUser ${linkedUserId}`, @@ -140,61 +133,30 @@ export class SyncService implements OnModuleInit { const sourceObject: OriginalFolderOutput[] = resp.data; - // unify the data according to the target obj wanted - const unifiedObject = (await this.coreUnification.unify< - OriginalFolderOutput[] - >({ + await this.ingestService.ingestData( sourceObject, - targetType: FileStorageObject.folder, - providerName: integrationId, - vertical: 'filestorage', - connectionId: connection.id_connection, - customFieldMappings, - })) as UnifiedFolderOutput[]; - - // insert the data in the DB with the fieldMappings (value table) - const folders_data = await this.saveFoldersInDb( - connection.id_connection, - linkedUserId, - unifiedObject, integrationId, - sourceObject, - ); - const event = await this.prisma.events.create({ - data: { - id_event: uuidv4(), - status: 'success', - type: 'filestorage.folder.synced', - method: 'SYNC', - url: '/sync', - provider: integrationId, - direction: '0', - timestamp: new Date(), - id_linked_user: linkedUserId, - }, - }); - await this.webhook.handleWebhook( - folders_data, - 'filestorage.folder.pulled', - id_project, - event.id_event, + connection.id_connection, + 'filestorage', + 'folder', + customFieldMappings, ); } catch (error) { throw error; } } - async saveFoldersInDb( + async saveToDb( connection_id: string, linkedUserId: string, - folders: UnifiedFolderOutput[], + data: UnifiedFolderOutput[], originSource: string, remote_data: Record[], ): Promise { try { let folders_results: FileStorageFolder[] = []; - for (let i = 0; i < folders.length; i++) { - const folder = folders[i]; + for (let i = 0; i < data.length; i++) { + const folder = data[i]; const originId = folder.remote_id; if (!originId || originId === '') { diff --git a/packages/api/src/filestorage/group/group.module.ts b/packages/api/src/filestorage/group/group.module.ts index 733135944..6db0aeb29 100644 --- a/packages/api/src/filestorage/group/group.module.ts +++ b/packages/api/src/filestorage/group/group.module.ts @@ -9,6 +9,7 @@ import { BoxService } from './services/box'; import { GroupService } from './services/group.service'; import { ServiceRegistry } from './services/registry.service'; import { SyncService } from './sync/sync.service'; +import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; @Module({ controllers: [GroupController], @@ -21,6 +22,7 @@ import { SyncService } from './sync/sync.service'; FieldMappingService, ServiceRegistry, ConnectionUtils, + IngestDataService, /* PROVIDERS SERVICES */ BoxService, ], diff --git a/packages/api/src/filestorage/group/sync/sync.service.ts b/packages/api/src/filestorage/group/sync/sync.service.ts index c5d385110..cb19adb05 100644 --- a/packages/api/src/filestorage/group/sync/sync.service.ts +++ b/packages/api/src/filestorage/group/sync/sync.service.ts @@ -14,12 +14,14 @@ import { fs_groups as FileStorageGroup } from '@prisma/client'; import { FILESTORAGE_PROVIDERS } from '@panora/shared'; import { FileStorageObject } from '@filestorage/@lib/@types'; import { BullQueueService } from '@@core/@core-services/queues/shared.service'; +import { IBaseSync } from '@@core/utils/types/interface'; +import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; import { CoreUnification } from '@@core/@core-services/unification/core-unification.service'; import { OriginalGroupOutput } from '@@core/utils/types/original/original.file-storage'; @Injectable() -export class SyncService implements OnModuleInit { +export class SyncService implements OnModuleInit, IBaseSync { constructor( private prisma: PrismaService, private logger: LoggerService, @@ -29,6 +31,7 @@ export class SyncService implements OnModuleInit { private coreUnification: CoreUnification, private registry: CoreSyncRegistry, private bullQueueService: BullQueueService, + private ingestService: IngestDataService, ) { this.logger.setContext(SyncService.name); this.registry.registerService('filestorage', 'group', this); @@ -80,7 +83,6 @@ export class SyncService implements OnModuleInit { await this.syncGroupsForLinkedUser( provider, linkedUser.id_linked_user, - id_project, ); } catch (error) { throw error; @@ -98,11 +100,7 @@ export class SyncService implements OnModuleInit { } } - async syncGroupsForLinkedUser( - integrationId: string, - linkedUserId: string, - id_project: string, - ) { + async syncGroupsForLinkedUser(integrationId: string, linkedUserId: string) { try { this.logger.log( `Syncing ${integrationId} groups for linkedUser ${linkedUserId}`, @@ -142,51 +140,23 @@ export class SyncService implements OnModuleInit { const sourceObject: OriginalGroupOutput[] = resp.data; - // unify the data according to the target obj wanted - const unifiedObject = (await this.coreUnification.unify< - OriginalGroupOutput[] - >({ + await this.ingestService.ingestData< + UnifiedGroupOutput, + OriginalGroupOutput + >( sourceObject, - targetType: FileStorageObject.group, - providerName: integrationId, - vertical: 'filestorage', - connectionId: connection.id_connection, - customFieldMappings, - })) as UnifiedGroupOutput[]; - - // insert the data in the DB with the fieldMappings (value table) - const groups_data = await this.saveGroupsInDb( - connection.id_connection, - linkedUserId, - unifiedObject, integrationId, - sourceObject, - ); - const event = await this.prisma.events.create({ - data: { - id_event: uuidv4(), - status: 'success', - type: 'filestorage.group.synced', - method: 'SYNC', - url: '/sync', - provider: integrationId, - direction: '0', - timestamp: new Date(), - id_linked_user: linkedUserId, - }, - }); - await this.webhook.handleWebhook( - groups_data, - 'filestorage.group.pulled', - id_project, - event.id_event, + connection.id_connection, + 'filestorage', + 'group', + customFieldMappings, ); } catch (error) { throw error; } } - async saveGroupsInDb( + async saveToDb( connection_id: string, linkedUserId: string, groups: UnifiedGroupOutput[], diff --git a/packages/api/src/filestorage/permission/permission.module.ts b/packages/api/src/filestorage/permission/permission.module.ts index 6a0a79e6f..3977829d0 100644 --- a/packages/api/src/filestorage/permission/permission.module.ts +++ b/packages/api/src/filestorage/permission/permission.module.ts @@ -8,6 +8,7 @@ import { PermissionController } from './permission.controller'; import { PermissionService } from './services/permission.service'; import { ServiceRegistry } from './services/registry.service'; import { SyncService } from './sync/sync.service'; +import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; @Module({ controllers: [PermissionController], @@ -20,6 +21,7 @@ import { SyncService } from './sync/sync.service'; FieldMappingService, ServiceRegistry, ConnectionUtils, + IngestDataService, /* PROVIDERS SERVICES */ ], exports: [SyncService], diff --git a/packages/api/src/filestorage/permission/services/permission.service.ts b/packages/api/src/filestorage/permission/services/permission.service.ts index 64ee0136b..772181eec 100644 --- a/packages/api/src/filestorage/permission/services/permission.service.ts +++ b/packages/api/src/filestorage/permission/services/permission.service.ts @@ -209,7 +209,7 @@ export class PermissionService { id_linked_user: linkedUserId, }, }); - await this.webhook.handleWebhook( + await this.webhook.dispatchWebhook( result_permission, 'filestorage.permission.created', linkedUser.id_project, diff --git a/packages/api/src/filestorage/permission/sync/sync.service.ts b/packages/api/src/filestorage/permission/sync/sync.service.ts index b051acd3d..aaf6fc5cb 100644 --- a/packages/api/src/filestorage/permission/sync/sync.service.ts +++ b/packages/api/src/filestorage/permission/sync/sync.service.ts @@ -15,10 +15,12 @@ import { fs_permissions as FileStoragePermission } from '@prisma/client'; import { FILESTORAGE_PROVIDERS } from '@panora/shared'; import { FileStorageObject } from '@filestorage/@lib/@types'; import { BullQueueService } from '@@core/@core-services/queues/shared.service'; +import { IBaseSync } from '@@core/utils/types/interface'; +import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; import { CoreUnification } from '@@core/@core-services/unification/core-unification.service'; @Injectable() -export class SyncService implements OnModuleInit { +export class SyncService implements OnModuleInit, IBaseSync { constructor( private prisma: PrismaService, private logger: LoggerService, @@ -28,6 +30,7 @@ export class SyncService implements OnModuleInit { private coreUnification: CoreUnification, private registry: CoreSyncRegistry, private bullQueueService: BullQueueService, + private ingestService: IngestDataService, ) { this.logger.setContext(SyncService.name); this.registry.registerService('filestorage', 'permission', this); @@ -79,7 +82,6 @@ export class SyncService implements OnModuleInit { await this.syncPermissionsForLinkedUser( provider, linkedUser.id_linked_user, - id_project, ); } catch (error) { throw error; @@ -100,7 +102,6 @@ export class SyncService implements OnModuleInit { async syncPermissionsForLinkedUser( integrationId: string, linkedUserId: string, - id_project: string, ) { try { this.logger.log( @@ -139,51 +140,23 @@ export class SyncService implements OnModuleInit { const sourceObject: OriginalPermissionOutput[] = resp.data; - // unify the data according to the target obj wanted - const unifiedObject = (await this.coreUnification.unify< - OriginalPermissionOutput[] - >({ + await this.ingestService.ingestData< + UnifiedPermissionOutput, + OriginalPermissionOutput + >( sourceObject, - targetType: FileStorageObject.permission, - providerName: integrationId, - vertical: 'filestorage', - connectionId: connection.id_connection, - customFieldMappings, - })) as UnifiedPermissionOutput[]; - - // insert the data in the DB with the fieldMappings (value table) - const permissions_data = await this.savePermissionsInDb( - connection.id_connection, - linkedUserId, - unifiedObject, integrationId, - sourceObject, - ); - const event = await this.prisma.events.create({ - data: { - id_event: uuidv4(), - status: 'success', - type: 'filestorage.permission.synced', - method: 'SYNC', - url: '/sync', - provider: integrationId, - direction: '0', - timestamp: new Date(), - id_linked_user: linkedUserId, - }, - }); - await this.webhook.handleWebhook( - permissions_data, - 'filestorage.permission.pulled', - id_project, - event.id_event, + connection.id_connection, + 'filestorage', + 'permission', + customFieldMappings, ); } catch (error) { throw error; } } - async savePermissionsInDb( + async saveToDb( connection_id: string, linkedUserId: string, permissions: UnifiedPermissionOutput[], diff --git a/packages/api/src/filestorage/sharedlink/services/sharedlink.service.ts b/packages/api/src/filestorage/sharedlink/services/sharedlink.service.ts index d8d4b3337..72b3e213b 100644 --- a/packages/api/src/filestorage/sharedlink/services/sharedlink.service.ts +++ b/packages/api/src/filestorage/sharedlink/services/sharedlink.service.ts @@ -215,7 +215,7 @@ export class SharedLinkService { id_linked_user: linkedUserId, }, }); - await this.webhook.handleWebhook( + await this.webhook.dispatchWebhook( result_sharedlink, 'filestorage.sharedlink.created', linkedUser.id_project, diff --git a/packages/api/src/filestorage/sharedlink/sharedlink.module.ts b/packages/api/src/filestorage/sharedlink/sharedlink.module.ts index 193d46b60..0ce3a40ae 100644 --- a/packages/api/src/filestorage/sharedlink/sharedlink.module.ts +++ b/packages/api/src/filestorage/sharedlink/sharedlink.module.ts @@ -8,6 +8,7 @@ import { ServiceRegistry } from './services/registry.service'; import { SharedLinkService } from './services/sharedlink.service'; import { SharedlinkController } from './sharedlink.controller'; import { SyncService } from './sync/sync.service'; +import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; @Module({ controllers: [SharedlinkController], @@ -20,6 +21,7 @@ import { SyncService } from './sync/sync.service'; FieldMappingService, ServiceRegistry, ConnectionUtils, + IngestDataService, /* PROVIDERS SERVICES */ ], exports: [SyncService], diff --git a/packages/api/src/filestorage/sharedlink/sync/sync.service.ts b/packages/api/src/filestorage/sharedlink/sync/sync.service.ts index 7cab1a584..8a1be8804 100644 --- a/packages/api/src/filestorage/sharedlink/sync/sync.service.ts +++ b/packages/api/src/filestorage/sharedlink/sync/sync.service.ts @@ -15,10 +15,12 @@ import { fs_shared_links as FileStorageSharedLink } from '@prisma/client'; import { FILESTORAGE_PROVIDERS } from '@panora/shared'; import { FileStorageObject } from '@filestorage/@lib/@types'; import { BullQueueService } from '@@core/@core-services/queues/shared.service'; +import { IBaseSync } from '@@core/utils/types/interface'; +import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; import { CoreUnification } from '@@core/@core-services/unification/core-unification.service'; @Injectable() -export class SyncService implements OnModuleInit { +export class SyncService implements OnModuleInit, IBaseSync { constructor( private prisma: PrismaService, private logger: LoggerService, @@ -28,6 +30,7 @@ export class SyncService implements OnModuleInit { private coreUnification: CoreUnification, private registry: CoreSyncRegistry, private bullQueueService: BullQueueService, + private ingestService: IngestDataService, ) { this.logger.setContext(SyncService.name); this.registry.registerService('filestorage', 'sharedlink', this); @@ -79,7 +82,6 @@ export class SyncService implements OnModuleInit { await this.syncSharedLinksForLinkedUser( provider, linkedUser.id_linked_user, - id_project, ); } catch (error) { throw error; @@ -100,7 +102,6 @@ export class SyncService implements OnModuleInit { async syncSharedLinksForLinkedUser( integrationId: string, linkedUserId: string, - id_project: string, ) { try { this.logger.log( @@ -139,51 +140,23 @@ export class SyncService implements OnModuleInit { const sourceObject: OriginalSharedLinkOutput[] = resp.data; - // unify the data according to the target obj wanted - const unifiedObject = (await this.coreUnification.unify< - OriginalSharedLinkOutput[] - >({ + await this.ingestService.ingestData< + UnifiedSharedLinkOutput, + OriginalSharedLinkOutput + >( sourceObject, - targetType: FileStorageObject.sharedlink, - providerName: integrationId, - vertical: 'filestorage', - connectionId: connection.id_connection, - customFieldMappings, - })) as UnifiedSharedLinkOutput[]; - - // insert the data in the DB with the fieldMappings (value table) - const shared_links_data = await this.saveSharedLinksInDb( - connection.id_connection, - linkedUserId, - unifiedObject, integrationId, - sourceObject, - ); - const event = await this.prisma.events.create({ - data: { - id_event: uuidv4(), - status: 'success', - type: 'filestorage.sharedlink.synced', - method: 'SYNC', - url: '/sync', - provider: integrationId, - direction: '0', - timestamp: new Date(), - id_linked_user: linkedUserId, - }, - }); - await this.webhook.handleWebhook( - shared_links_data, - 'filestorage.shared_link.pulled', - id_project, - event.id_event, + connection.id_connection, + 'filestorage', + 'sharedlink', + customFieldMappings, ); } catch (error) { throw error; } } - async saveSharedLinksInDb( + async saveToDb( connection_id: string, linkedUserId: string, sharedLinks: UnifiedSharedLinkOutput[], diff --git a/packages/api/src/filestorage/user/sync/sync.service.ts b/packages/api/src/filestorage/user/sync/sync.service.ts index 08abeac5e..4bd2eb89b 100644 --- a/packages/api/src/filestorage/user/sync/sync.service.ts +++ b/packages/api/src/filestorage/user/sync/sync.service.ts @@ -15,10 +15,12 @@ import { fs_users as FileStorageUser } from '@prisma/client'; import { FILESTORAGE_PROVIDERS } from '@panora/shared'; import { FileStorageObject } from '@filestorage/@lib/@types'; import { BullQueueService } from '@@core/@core-services/queues/shared.service'; +import { IBaseSync } from '@@core/utils/types/interface'; +import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; import { CoreUnification } from '@@core/@core-services/unification/core-unification.service'; @Injectable() -export class SyncService implements OnModuleInit { +export class SyncService implements OnModuleInit, IBaseSync { constructor( private prisma: PrismaService, private logger: LoggerService, @@ -28,6 +30,7 @@ export class SyncService implements OnModuleInit { private coreUnification: CoreUnification, private registry: CoreSyncRegistry, private bullQueueService: BullQueueService, + private ingestService: IngestDataService, ) { this.logger.setContext(SyncService.name); this.registry.registerService('filestorage', 'user', this); @@ -91,14 +94,12 @@ export class SyncService implements OnModuleInit { await this.syncUsersForLinkedUser( provider, linkedUser.id_linked_user, - id_project, group.id_fs_group, ); } await this.syncUsersForLinkedUser( provider, linkedUser.id_linked_user, - id_project, ); } catch (error) { throw error; @@ -119,7 +120,6 @@ export class SyncService implements OnModuleInit { async syncUsersForLinkedUser( integrationId: string, linkedUserId: string, - id_project: string, group_id?: string, ) { try { @@ -162,51 +162,23 @@ export class SyncService implements OnModuleInit { if (!resp) return; const sourceObject: OriginalUserOutput[] = resp.data; - // unify the data according to the target obj wanted - const unifiedObject = (await this.coreUnification.unify< - OriginalUserOutput[] - >({ + await this.ingestService.ingestData< + UnifiedUserOutput, + OriginalUserOutput + >( sourceObject, - targetType: FileStorageObject.user, - providerName: integrationId, - vertical: 'filestorage', - connectionId: connection.id_connection, - customFieldMappings, - })) as UnifiedUserOutput[]; - - // insert the data in the DB with the fieldMappings (value table) - const users_data = await this.saveUsersInDb( - connection.id_connection, - linkedUserId, - unifiedObject, integrationId, - sourceObject, - ); - const event = await this.prisma.events.create({ - data: { - id_event: uuidv4(), - status: 'success', - type: 'filestorage.user.synced', - method: 'SYNC', - url: '/sync', - provider: integrationId, - direction: '0', - timestamp: new Date(), - id_linked_user: linkedUserId, - }, - }); - await this.webhook.handleWebhook( - users_data, - 'filestorage.user.pulled', - id_project, - event.id_event, + connection.id_connection, + 'filestorage', + 'user', + customFieldMappings, ); } catch (error) { throw error; } } - async saveUsersInDb( + async saveToDb( connection_id: string, linkedUserId: string, users: UnifiedUserOutput[], diff --git a/packages/api/src/filestorage/user/user.module.ts b/packages/api/src/filestorage/user/user.module.ts index 28aa9556c..2860058bb 100644 --- a/packages/api/src/filestorage/user/user.module.ts +++ b/packages/api/src/filestorage/user/user.module.ts @@ -9,6 +9,7 @@ import { ServiceRegistry } from './services/registry.service'; import { UserService } from './services/user.service'; import { SyncService } from './sync/sync.service'; import { UserController } from './user.controller'; +import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; @Module({ controllers: [UserController], @@ -21,6 +22,7 @@ import { UserController } from './user.controller'; FieldMappingService, ServiceRegistry, ConnectionUtils, + IngestDataService, /* PROVIDERS SERVICES */ BoxService, ], diff --git a/packages/api/src/hris/bankinfo/bankinfo.module.ts b/packages/api/src/hris/bankinfo/bankinfo.module.ts index 522d7481b..60e0a1d62 100644 --- a/packages/api/src/hris/bankinfo/bankinfo.module.ts +++ b/packages/api/src/hris/bankinfo/bankinfo.module.ts @@ -11,6 +11,7 @@ import { WebhookService } from '@@core/@core-services/webhooks/panora-webhooks/w import { BullModule } from '@nestjs/bull'; import { ConnectionUtils } from '@@core/connections/@utils'; import { ApiKeyAuthGuard } from '@@core/auth/guards/api-key.guard'; +import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; @Module({ @@ -23,6 +24,7 @@ import { ApiKeyAuthGuard } from '@@core/auth/guards/api-key.guard'; FieldMappingService, ServiceRegistry, ConnectionUtils, + IngestDataService, /* PROVIDERS SERVICES */ ], exports: [SyncService], diff --git a/packages/api/src/hris/bankinfo/sync/sync.service.ts b/packages/api/src/hris/bankinfo/sync/sync.service.ts index c6d0fb41d..526b050f9 100644 --- a/packages/api/src/hris/bankinfo/sync/sync.service.ts +++ b/packages/api/src/hris/bankinfo/sync/sync.service.ts @@ -9,7 +9,7 @@ import { ServiceRegistry } from '../services/registry.service'; import { WebhookService } from '@@core/@core-services/webhooks/panora-webhooks/webhook.service'; @Injectable() -export class SyncService implements OnModuleInit { +export class SyncService implements OnModuleInit, IBaseSync { constructor( private prisma: PrismaService, private logger: LoggerService, diff --git a/packages/api/src/hris/benefit/benefit.module.ts b/packages/api/src/hris/benefit/benefit.module.ts index f67a39b8f..79a98510e 100644 --- a/packages/api/src/hris/benefit/benefit.module.ts +++ b/packages/api/src/hris/benefit/benefit.module.ts @@ -11,6 +11,7 @@ import { WebhookService } from '@@core/@core-services/webhooks/panora-webhooks/w import { BullModule } from '@nestjs/bull'; import { ConnectionUtils } from '@@core/connections/@utils'; import { ApiKeyAuthGuard } from '@@core/auth/guards/api-key.guard'; +import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; @Module({ @@ -23,6 +24,7 @@ import { ApiKeyAuthGuard } from '@@core/auth/guards/api-key.guard'; FieldMappingService, ServiceRegistry, ConnectionUtils, + IngestDataService, /* PROVIDERS SERVICES */ ], exports: [SyncService], diff --git a/packages/api/src/hris/benefit/sync/sync.service.ts b/packages/api/src/hris/benefit/sync/sync.service.ts index 7c8eb4abd..4e85b2a1e 100644 --- a/packages/api/src/hris/benefit/sync/sync.service.ts +++ b/packages/api/src/hris/benefit/sync/sync.service.ts @@ -11,7 +11,7 @@ import { UnifiedBenefitOutput } from '../types/model.unified'; import { IBenefitService } from '../types'; @Injectable() -export class SyncService implements OnModuleInit { +export class SyncService implements OnModuleInit, IBaseSync { constructor( private prisma: PrismaService, private logger: LoggerService, diff --git a/packages/api/src/hris/company/company.module.ts b/packages/api/src/hris/company/company.module.ts index 65d895281..f72db97e0 100644 --- a/packages/api/src/hris/company/company.module.ts +++ b/packages/api/src/hris/company/company.module.ts @@ -11,6 +11,7 @@ import { WebhookService } from '@@core/@core-services/webhooks/panora-webhooks/w import { BullModule } from '@nestjs/bull'; import { ConnectionUtils } from '@@core/connections/@utils'; import { ApiKeyAuthGuard } from '@@core/auth/guards/api-key.guard'; +import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; @Module({ @@ -23,6 +24,7 @@ import { ApiKeyAuthGuard } from '@@core/auth/guards/api-key.guard'; FieldMappingService, ServiceRegistry, ConnectionUtils, + IngestDataService, /* PROVIDERS SERVICES */ ], exports: [SyncService], diff --git a/packages/api/src/hris/company/sync/sync.service.ts b/packages/api/src/hris/company/sync/sync.service.ts index 0372b8b28..39c223539 100644 --- a/packages/api/src/hris/company/sync/sync.service.ts +++ b/packages/api/src/hris/company/sync/sync.service.ts @@ -11,7 +11,7 @@ import { UnifiedCompanyOutput } from '../types/model.unified'; import { ICompanyService } from '../types'; @Injectable() -export class SyncService implements OnModuleInit { +export class SyncService implements OnModuleInit, IBaseSync { constructor( private prisma: PrismaService, private logger: LoggerService, diff --git a/packages/api/src/hris/dependent/dependent.module.ts b/packages/api/src/hris/dependent/dependent.module.ts index 9f751c26f..83adba408 100644 --- a/packages/api/src/hris/dependent/dependent.module.ts +++ b/packages/api/src/hris/dependent/dependent.module.ts @@ -11,6 +11,7 @@ import { WebhookService } from '@@core/@core-services/webhooks/panora-webhooks/w import { BullModule } from '@nestjs/bull'; import { ConnectionUtils } from '@@core/connections/@utils'; import { ApiKeyAuthGuard } from '@@core/auth/guards/api-key.guard'; +import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; @Module({ @@ -23,6 +24,7 @@ import { ApiKeyAuthGuard } from '@@core/auth/guards/api-key.guard'; FieldMappingService, ServiceRegistry, ConnectionUtils, + IngestDataService, /* PROVIDERS SERVICES */ ], exports: [SyncService], diff --git a/packages/api/src/hris/dependent/sync/sync.service.ts b/packages/api/src/hris/dependent/sync/sync.service.ts index 154f34442..6dd0bbd13 100644 --- a/packages/api/src/hris/dependent/sync/sync.service.ts +++ b/packages/api/src/hris/dependent/sync/sync.service.ts @@ -11,7 +11,7 @@ import { UnifiedDependentOutput } from '../types/model.unified'; import { IDependentService } from '../types'; @Injectable() -export class SyncService implements OnModuleInit { +export class SyncService implements OnModuleInit, IBaseSync { constructor( private prisma: PrismaService, private logger: LoggerService, diff --git a/packages/api/src/hris/employee/employee.module.ts b/packages/api/src/hris/employee/employee.module.ts index ebd44bb9c..e72c0233e 100644 --- a/packages/api/src/hris/employee/employee.module.ts +++ b/packages/api/src/hris/employee/employee.module.ts @@ -11,6 +11,7 @@ import { WebhookService } from '@@core/@core-services/webhooks/panora-webhooks/w import { BullModule } from '@nestjs/bull'; import { ConnectionUtils } from '@@core/connections/@utils'; import { ApiKeyAuthGuard } from '@@core/auth/guards/api-key.guard'; +import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; @Module({ @@ -23,6 +24,7 @@ import { ApiKeyAuthGuard } from '@@core/auth/guards/api-key.guard'; FieldMappingService, ServiceRegistry, ConnectionUtils, + IngestDataService, /* PROVIDERS SERVICES */ ], exports: [SyncService], diff --git a/packages/api/src/hris/employee/sync/sync.service.ts b/packages/api/src/hris/employee/sync/sync.service.ts index 7694b25c0..ef3e77e6f 100644 --- a/packages/api/src/hris/employee/sync/sync.service.ts +++ b/packages/api/src/hris/employee/sync/sync.service.ts @@ -11,7 +11,7 @@ import { UnifiedEmployeeOutput } from '../types/model.unified'; import { IEmployeeService } from '../types'; @Injectable() -export class SyncService implements OnModuleInit { +export class SyncService implements OnModuleInit, IBaseSync { constructor( private prisma: PrismaService, private logger: LoggerService, diff --git a/packages/api/src/hris/employeepayrollrun/employeepayrollrun.module.ts b/packages/api/src/hris/employeepayrollrun/employeepayrollrun.module.ts index a40d1d9e4..b2dcb4c8b 100644 --- a/packages/api/src/hris/employeepayrollrun/employeepayrollrun.module.ts +++ b/packages/api/src/hris/employeepayrollrun/employeepayrollrun.module.ts @@ -11,6 +11,7 @@ import { WebhookService } from '@@core/@core-services/webhooks/panora-webhooks/w import { BullModule } from '@nestjs/bull'; import { ConnectionUtils } from '@@core/connections/@utils'; import { ApiKeyAuthGuard } from '@@core/auth/guards/api-key.guard'; +import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; @Module({ @@ -23,6 +24,7 @@ import { ApiKeyAuthGuard } from '@@core/auth/guards/api-key.guard'; FieldMappingService, ServiceRegistry, ConnectionUtils, + IngestDataService, /* PROVIDERS SERVICES */ ], exports: [SyncService], diff --git a/packages/api/src/hris/employeepayrollrun/sync/sync.service.ts b/packages/api/src/hris/employeepayrollrun/sync/sync.service.ts index 142a3a2fb..7853893c4 100644 --- a/packages/api/src/hris/employeepayrollrun/sync/sync.service.ts +++ b/packages/api/src/hris/employeepayrollrun/sync/sync.service.ts @@ -11,7 +11,7 @@ import { UnifiedEmployeePayrollRunOutput } from '../types/model.unified'; import { IEmployeePayrollRunService } from '../types'; @Injectable() -export class SyncService implements OnModuleInit { +export class SyncService implements OnModuleInit, IBaseSync { constructor( private prisma: PrismaService, private logger: LoggerService, diff --git a/packages/api/src/hris/employerbenefit/employerbenefit.module.ts b/packages/api/src/hris/employerbenefit/employerbenefit.module.ts index d85f89c9a..89246870d 100644 --- a/packages/api/src/hris/employerbenefit/employerbenefit.module.ts +++ b/packages/api/src/hris/employerbenefit/employerbenefit.module.ts @@ -11,6 +11,7 @@ import { WebhookService } from '@@core/@core-services/webhooks/panora-webhooks/w import { BullModule } from '@nestjs/bull'; import { ConnectionUtils } from '@@core/connections/@utils'; import { ApiKeyAuthGuard } from '@@core/auth/guards/api-key.guard'; +import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; @Module({ @@ -23,6 +24,7 @@ import { ApiKeyAuthGuard } from '@@core/auth/guards/api-key.guard'; FieldMappingService, ServiceRegistry, ConnectionUtils, + IngestDataService, /* PROVIDERS SERVICES */ ], exports: [SyncService], diff --git a/packages/api/src/hris/employerbenefit/sync/sync.service.ts b/packages/api/src/hris/employerbenefit/sync/sync.service.ts index e1d4e9509..92377308b 100644 --- a/packages/api/src/hris/employerbenefit/sync/sync.service.ts +++ b/packages/api/src/hris/employerbenefit/sync/sync.service.ts @@ -11,7 +11,7 @@ import { UnifiedEmployerBenefitOutput } from '../types/model.unified'; import { IEmployerBenefitService } from '../types'; @Injectable() -export class SyncService implements OnModuleInit { +export class SyncService implements OnModuleInit, IBaseSync { constructor( private prisma: PrismaService, private logger: LoggerService, diff --git a/packages/api/src/hris/employment/employment.module.ts b/packages/api/src/hris/employment/employment.module.ts index f3b401e2c..7f4c6e438 100644 --- a/packages/api/src/hris/employment/employment.module.ts +++ b/packages/api/src/hris/employment/employment.module.ts @@ -11,6 +11,7 @@ import { WebhookService } from '@@core/@core-services/webhooks/panora-webhooks/w import { BullModule } from '@nestjs/bull'; import { ConnectionUtils } from '@@core/connections/@utils'; import { ApiKeyAuthGuard } from '@@core/auth/guards/api-key.guard'; +import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; @Module({ @@ -23,6 +24,7 @@ import { ApiKeyAuthGuard } from '@@core/auth/guards/api-key.guard'; FieldMappingService, ServiceRegistry, ConnectionUtils, + IngestDataService, /* PROVIDERS SERVICES */ ], exports: [SyncService], diff --git a/packages/api/src/hris/employment/sync/sync.service.ts b/packages/api/src/hris/employment/sync/sync.service.ts index e291530ee..e32c55c23 100644 --- a/packages/api/src/hris/employment/sync/sync.service.ts +++ b/packages/api/src/hris/employment/sync/sync.service.ts @@ -11,7 +11,7 @@ import { UnifiedEmploymentOutput } from '../types/model.unified'; import { IEmploymentService } from '../types'; @Injectable() -export class SyncService implements OnModuleInit { +export class SyncService implements OnModuleInit, IBaseSync { constructor( private prisma: PrismaService, private logger: LoggerService, diff --git a/packages/api/src/hris/group/group.module.ts b/packages/api/src/hris/group/group.module.ts index b7d623572..bcdd0ee2a 100644 --- a/packages/api/src/hris/group/group.module.ts +++ b/packages/api/src/hris/group/group.module.ts @@ -11,6 +11,7 @@ import { WebhookService } from '@@core/@core-services/webhooks/panora-webhooks/w import { BullModule } from '@nestjs/bull'; import { ConnectionUtils } from '@@core/connections/@utils'; import { ApiKeyAuthGuard } from '@@core/auth/guards/api-key.guard'; +import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; @Module({ @@ -23,6 +24,7 @@ import { ApiKeyAuthGuard } from '@@core/auth/guards/api-key.guard'; FieldMappingService, ServiceRegistry, ConnectionUtils, + IngestDataService, /* PROVIDERS SERVICES */ ], exports: [SyncService], diff --git a/packages/api/src/hris/group/sync/sync.service.ts b/packages/api/src/hris/group/sync/sync.service.ts index cb8bfa04c..7c4fc79e5 100644 --- a/packages/api/src/hris/group/sync/sync.service.ts +++ b/packages/api/src/hris/group/sync/sync.service.ts @@ -11,7 +11,7 @@ import { UnifiedGroupOutput } from '../types/model.unified'; import { IGroupService } from '../types'; @Injectable() -export class SyncService implements OnModuleInit { +export class SyncService implements OnModuleInit, IBaseSync { constructor( private prisma: PrismaService, private logger: LoggerService, diff --git a/packages/api/src/hris/location/location.module.ts b/packages/api/src/hris/location/location.module.ts index 51861ba26..bb2908e0e 100644 --- a/packages/api/src/hris/location/location.module.ts +++ b/packages/api/src/hris/location/location.module.ts @@ -11,6 +11,7 @@ import { WebhookService } from '@@core/@core-services/webhooks/panora-webhooks/w import { BullModule } from '@nestjs/bull'; import { ConnectionUtils } from '@@core/connections/@utils'; import { ApiKeyAuthGuard } from '@@core/auth/guards/api-key.guard'; +import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; @Module({ @@ -23,6 +24,7 @@ import { ApiKeyAuthGuard } from '@@core/auth/guards/api-key.guard'; FieldMappingService, ServiceRegistry, ConnectionUtils, + IngestDataService, /* PROVIDERS SERVICES */ ], exports: [SyncService], diff --git a/packages/api/src/hris/location/sync/sync.service.ts b/packages/api/src/hris/location/sync/sync.service.ts index 134eb63f8..937637b32 100644 --- a/packages/api/src/hris/location/sync/sync.service.ts +++ b/packages/api/src/hris/location/sync/sync.service.ts @@ -11,7 +11,7 @@ import { UnifiedLocationOutput } from '../types/model.unified'; import { ILocationService } from '../types'; @Injectable() -export class SyncService implements OnModuleInit { +export class SyncService implements OnModuleInit, IBaseSync { constructor( private prisma: PrismaService, private logger: LoggerService, diff --git a/packages/api/src/hris/paygroup/paygroup.module.ts b/packages/api/src/hris/paygroup/paygroup.module.ts index 777855ccf..ac3e67a85 100644 --- a/packages/api/src/hris/paygroup/paygroup.module.ts +++ b/packages/api/src/hris/paygroup/paygroup.module.ts @@ -11,6 +11,7 @@ import { WebhookService } from '@@core/@core-services/webhooks/panora-webhooks/w import { BullModule } from '@nestjs/bull'; import { ConnectionUtils } from '@@core/connections/@utils'; import { ApiKeyAuthGuard } from '@@core/auth/guards/api-key.guard'; +import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; @Module({ @@ -23,6 +24,7 @@ import { ApiKeyAuthGuard } from '@@core/auth/guards/api-key.guard'; FieldMappingService, ServiceRegistry, ConnectionUtils, + IngestDataService, /* PROVIDERS SERVICES */ ], exports: [SyncService], diff --git a/packages/api/src/hris/paygroup/sync/sync.service.ts b/packages/api/src/hris/paygroup/sync/sync.service.ts index 8ec44a0b9..413165712 100644 --- a/packages/api/src/hris/paygroup/sync/sync.service.ts +++ b/packages/api/src/hris/paygroup/sync/sync.service.ts @@ -11,7 +11,7 @@ import { UnifiedPayGroupOutput } from '../types/model.unified'; import { IPayGroupService } from '../types'; @Injectable() -export class SyncService implements OnModuleInit { +export class SyncService implements OnModuleInit, IBaseSync { constructor( private prisma: PrismaService, private logger: LoggerService, diff --git a/packages/api/src/hris/payrollrun/payrollrun.module.ts b/packages/api/src/hris/payrollrun/payrollrun.module.ts index b693b0b7f..81b2a278f 100644 --- a/packages/api/src/hris/payrollrun/payrollrun.module.ts +++ b/packages/api/src/hris/payrollrun/payrollrun.module.ts @@ -11,6 +11,7 @@ import { WebhookService } from '@@core/@core-services/webhooks/panora-webhooks/w import { BullModule } from '@nestjs/bull'; import { ConnectionUtils } from '@@core/connections/@utils'; import { ApiKeyAuthGuard } from '@@core/auth/guards/api-key.guard'; +import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; @Module({ @@ -23,6 +24,7 @@ import { ApiKeyAuthGuard } from '@@core/auth/guards/api-key.guard'; FieldMappingService, ServiceRegistry, ConnectionUtils, + IngestDataService, /* PROVIDERS SERVICES */ ], exports: [SyncService], diff --git a/packages/api/src/hris/payrollrun/sync/sync.service.ts b/packages/api/src/hris/payrollrun/sync/sync.service.ts index e1463bc4f..48e54fdb4 100644 --- a/packages/api/src/hris/payrollrun/sync/sync.service.ts +++ b/packages/api/src/hris/payrollrun/sync/sync.service.ts @@ -11,7 +11,7 @@ import { UnifiedPayrollRunOutput } from '../types/model.unified'; import { IPayrollRunService } from '../types'; @Injectable() -export class SyncService implements OnModuleInit { +export class SyncService implements OnModuleInit, IBaseSync { constructor( private prisma: PrismaService, private logger: LoggerService, diff --git a/packages/api/src/hris/timeoff/sync/sync.service.ts b/packages/api/src/hris/timeoff/sync/sync.service.ts index 00f29ff93..2d6cdccc1 100644 --- a/packages/api/src/hris/timeoff/sync/sync.service.ts +++ b/packages/api/src/hris/timeoff/sync/sync.service.ts @@ -11,7 +11,7 @@ import { UnifiedTimeoffOutput } from '../types/model.unified'; import { ITimeoffService } from '../types'; @Injectable() -export class SyncService implements OnModuleInit { +export class SyncService implements OnModuleInit, IBaseSync { constructor( private prisma: PrismaService, private logger: LoggerService, diff --git a/packages/api/src/hris/timeoff/timeoff.module.ts b/packages/api/src/hris/timeoff/timeoff.module.ts index ffb6c7b0c..3e06059db 100644 --- a/packages/api/src/hris/timeoff/timeoff.module.ts +++ b/packages/api/src/hris/timeoff/timeoff.module.ts @@ -11,6 +11,7 @@ import { WebhookService } from '@@core/@core-services/webhooks/panora-webhooks/w import { BullModule } from '@nestjs/bull'; import { ConnectionUtils } from '@@core/connections/@utils'; import { ApiKeyAuthGuard } from '@@core/auth/guards/api-key.guard'; +import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; @Module({ @@ -23,6 +24,7 @@ import { ApiKeyAuthGuard } from '@@core/auth/guards/api-key.guard'; FieldMappingService, ServiceRegistry, ConnectionUtils, + IngestDataService, /* PROVIDERS SERVICES */ ], exports: [SyncService], diff --git a/packages/api/src/hris/timeoffbalance/sync/sync.service.ts b/packages/api/src/hris/timeoffbalance/sync/sync.service.ts index 2b2257dfa..0dde3c495 100644 --- a/packages/api/src/hris/timeoffbalance/sync/sync.service.ts +++ b/packages/api/src/hris/timeoffbalance/sync/sync.service.ts @@ -12,7 +12,7 @@ import { UnifiedTimeoffBalanceOutput } from '../types/model.unified'; import { ITimeoffBalanceService } from '../types'; @Injectable() -export class SyncService implements OnModuleInit { +export class SyncService implements OnModuleInit, IBaseSync { constructor( private prisma: PrismaService, private logger: LoggerService, diff --git a/packages/api/src/hris/timeoffbalance/timeoffbalance.module.ts b/packages/api/src/hris/timeoffbalance/timeoffbalance.module.ts index 495f9b95d..9f66b8621 100644 --- a/packages/api/src/hris/timeoffbalance/timeoffbalance.module.ts +++ b/packages/api/src/hris/timeoffbalance/timeoffbalance.module.ts @@ -11,6 +11,7 @@ import { WebhookService } from '@@core/@core-services/webhooks/panora-webhooks/w import { BullModule } from '@nestjs/bull'; import { ConnectionUtils } from '@@core/connections/@utils'; import { ApiKeyAuthGuard } from '@@core/auth/guards/api-key.guard'; +import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; @Module({ @@ -23,6 +24,7 @@ import { ApiKeyAuthGuard } from '@@core/auth/guards/api-key.guard'; FieldMappingService, ServiceRegistry, ConnectionUtils, + IngestDataService, /* PROVIDERS SERVICES */ ], exports: [SyncService], diff --git a/packages/api/src/marketingautomation/action/action.module.ts b/packages/api/src/marketingautomation/action/action.module.ts index 68b15242a..df82e9f81 100644 --- a/packages/api/src/marketingautomation/action/action.module.ts +++ b/packages/api/src/marketingautomation/action/action.module.ts @@ -9,6 +9,7 @@ import { ActionController } from './action.controller'; import { ActionService } from './services/action.service'; import { ServiceRegistry } from './services/registry.service'; import { SyncService } from './sync/sync.service'; +import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; @Module({ controllers: [ActionController], @@ -21,6 +22,7 @@ import { SyncService } from './sync/sync.service'; FieldMappingService, ServiceRegistry, ConnectionUtils, + IngestDataService, /* PROVIDERS SERVICES */ ], exports: [SyncService, CoreModule], diff --git a/packages/api/src/marketingautomation/action/sync/sync.service.ts b/packages/api/src/marketingautomation/action/sync/sync.service.ts index 69aed2893..beccb6a09 100644 --- a/packages/api/src/marketingautomation/action/sync/sync.service.ts +++ b/packages/api/src/marketingautomation/action/sync/sync.service.ts @@ -11,7 +11,7 @@ import { UnifiedActionOutput } from '../types/model.unified'; import { IActionService } from '../types'; @Injectable() -export class SyncService implements OnModuleInit { +export class SyncService implements OnModuleInit, IBaseSync { constructor( private prisma: PrismaService, private logger: LoggerService, diff --git a/packages/api/src/marketingautomation/automation/automation.module.ts b/packages/api/src/marketingautomation/automation/automation.module.ts index 646b36d87..0f09a6b14 100644 --- a/packages/api/src/marketingautomation/automation/automation.module.ts +++ b/packages/api/src/marketingautomation/automation/automation.module.ts @@ -9,6 +9,7 @@ import { AutomationController } from './automation.controller'; import { AutomationService } from './services/automation.service'; import { ServiceRegistry } from './services/registry.service'; import { SyncService } from './sync/sync.service'; +import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; @Module({ controllers: [AutomationController], @@ -21,6 +22,7 @@ import { SyncService } from './sync/sync.service'; FieldMappingService, ServiceRegistry, ConnectionUtils, + IngestDataService, /* PROVIDERS SERVICES */ ], exports: [SyncService, CoreModule], diff --git a/packages/api/src/marketingautomation/automation/sync/sync.service.ts b/packages/api/src/marketingautomation/automation/sync/sync.service.ts index 789cf3a09..1bc7edd14 100644 --- a/packages/api/src/marketingautomation/automation/sync/sync.service.ts +++ b/packages/api/src/marketingautomation/automation/sync/sync.service.ts @@ -11,7 +11,7 @@ import { UnifiedAutomationOutput } from '../types/model.unified'; import { IAutomationService } from '../types'; @Injectable() -export class SyncService implements OnModuleInit { +export class SyncService implements OnModuleInit, IBaseSync { constructor( private prisma: PrismaService, private logger: LoggerService, diff --git a/packages/api/src/marketingautomation/campaign/campaign.module.ts b/packages/api/src/marketingautomation/campaign/campaign.module.ts index 4389e668b..2ecc29d5a 100644 --- a/packages/api/src/marketingautomation/campaign/campaign.module.ts +++ b/packages/api/src/marketingautomation/campaign/campaign.module.ts @@ -9,6 +9,7 @@ import { CampaignController } from './campaign.controller'; import { CampaignService } from './services/campaign.service'; import { ServiceRegistry } from './services/registry.service'; import { SyncService } from './sync/sync.service'; +import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; @Module({ controllers: [CampaignController], @@ -21,6 +22,7 @@ import { SyncService } from './sync/sync.service'; FieldMappingService, ServiceRegistry, ConnectionUtils, + IngestDataService, /* PROVIDERS SERVICES */ ], exports: [SyncService, CoreModule], diff --git a/packages/api/src/marketingautomation/campaign/sync/sync.service.ts b/packages/api/src/marketingautomation/campaign/sync/sync.service.ts index 7b7d7b224..2ee1bff09 100644 --- a/packages/api/src/marketingautomation/campaign/sync/sync.service.ts +++ b/packages/api/src/marketingautomation/campaign/sync/sync.service.ts @@ -11,7 +11,7 @@ import { UnifiedCampaignOutput } from '../types/model.unified'; import { ICampaignService } from '../types'; @Injectable() -export class SyncService implements OnModuleInit { +export class SyncService implements OnModuleInit, IBaseSync { constructor( private prisma: PrismaService, private logger: LoggerService, diff --git a/packages/api/src/marketingautomation/contact/contact.module.ts b/packages/api/src/marketingautomation/contact/contact.module.ts index 0df9577fe..0f82d9896 100644 --- a/packages/api/src/marketingautomation/contact/contact.module.ts +++ b/packages/api/src/marketingautomation/contact/contact.module.ts @@ -9,6 +9,7 @@ import { ContactController } from './contact.controller'; import { ContactService } from './services/contact.service'; import { ServiceRegistry } from './services/registry.service'; import { SyncService } from './sync/sync.service'; +import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; @Module({ controllers: [ContactController], @@ -21,6 +22,7 @@ import { SyncService } from './sync/sync.service'; FieldMappingService, ServiceRegistry, ConnectionUtils, + IngestDataService, /* PROVIDERS SERVICES */ ], exports: [SyncService, CoreModule], diff --git a/packages/api/src/marketingautomation/contact/sync/sync.service.ts b/packages/api/src/marketingautomation/contact/sync/sync.service.ts index 8f71c1975..63448e54a 100644 --- a/packages/api/src/marketingautomation/contact/sync/sync.service.ts +++ b/packages/api/src/marketingautomation/contact/sync/sync.service.ts @@ -11,7 +11,7 @@ import { UnifiedContactOutput } from '../types/model.unified'; import { IContactService } from '../types'; @Injectable() -export class SyncService implements OnModuleInit { +export class SyncService implements OnModuleInit, IBaseSync { constructor( private prisma: PrismaService, private logger: LoggerService, diff --git a/packages/api/src/marketingautomation/email/email.module.ts b/packages/api/src/marketingautomation/email/email.module.ts index 9d19c0d26..b8a111116 100644 --- a/packages/api/src/marketingautomation/email/email.module.ts +++ b/packages/api/src/marketingautomation/email/email.module.ts @@ -10,6 +10,7 @@ import { EmailController } from './email.controller'; import { EmailService } from './services/email.service'; import { ServiceRegistry } from './services/registry.service'; import { SyncService } from './sync/sync.service'; +import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; @Module({ imports: [ @@ -28,6 +29,7 @@ import { SyncService } from './sync/sync.service'; FieldMappingService, ServiceRegistry, ConnectionUtils, + IngestDataService, /* PROVIDERS SERVICES */ ], exports: [SyncService, CoreModule], diff --git a/packages/api/src/marketingautomation/email/sync/sync.service.ts b/packages/api/src/marketingautomation/email/sync/sync.service.ts index 2e7577f30..c7ff7374c 100644 --- a/packages/api/src/marketingautomation/email/sync/sync.service.ts +++ b/packages/api/src/marketingautomation/email/sync/sync.service.ts @@ -11,7 +11,7 @@ import { UnifiedEmailOutput } from '../types/model.unified'; import { IEmailService } from '../types'; @Injectable() -export class SyncService implements OnModuleInit { +export class SyncService implements OnModuleInit, IBaseSync { constructor( private prisma: PrismaService, private logger: LoggerService, diff --git a/packages/api/src/marketingautomation/event/event.module.ts b/packages/api/src/marketingautomation/event/event.module.ts index d0729c9f1..9c64a8237 100644 --- a/packages/api/src/marketingautomation/event/event.module.ts +++ b/packages/api/src/marketingautomation/event/event.module.ts @@ -10,6 +10,7 @@ import { EventController } from './event.controller'; import { EventService } from './services/event.service'; import { ServiceRegistry } from './services/registry.service'; import { SyncService } from './sync/sync.service'; +import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; @Module({ imports: [ @@ -28,6 +29,7 @@ import { SyncService } from './sync/sync.service'; FieldMappingService, ServiceRegistry, ConnectionUtils, + IngestDataService, /* PROVIDERS SERVICES */ ], exports: [SyncService, CoreModule], diff --git a/packages/api/src/marketingautomation/event/sync/sync.service.ts b/packages/api/src/marketingautomation/event/sync/sync.service.ts index 5ddf5e0bd..4ff562b57 100644 --- a/packages/api/src/marketingautomation/event/sync/sync.service.ts +++ b/packages/api/src/marketingautomation/event/sync/sync.service.ts @@ -11,7 +11,7 @@ import { UnifiedEventOutput } from '../types/model.unified'; import { IEventService } from '../types'; @Injectable() -export class SyncService implements OnModuleInit { +export class SyncService implements OnModuleInit, IBaseSync { constructor( private prisma: PrismaService, private logger: LoggerService, diff --git a/packages/api/src/marketingautomation/list/list.module.ts b/packages/api/src/marketingautomation/list/list.module.ts index f910caf05..48a05845b 100644 --- a/packages/api/src/marketingautomation/list/list.module.ts +++ b/packages/api/src/marketingautomation/list/list.module.ts @@ -10,6 +10,7 @@ import { ListController } from './list.controller'; import { ListService } from './services/list.service'; import { ServiceRegistry } from './services/registry.service'; import { SyncService } from './sync/sync.service'; +import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; @Module({ imports: [ @@ -28,6 +29,7 @@ import { SyncService } from './sync/sync.service'; FieldMappingService, ServiceRegistry, ConnectionUtils, + IngestDataService, /* PROVIDERS SERVICES */ ], exports: [SyncService, CoreModule], diff --git a/packages/api/src/marketingautomation/list/sync/sync.service.ts b/packages/api/src/marketingautomation/list/sync/sync.service.ts index 997b0b20f..8a711ed2a 100644 --- a/packages/api/src/marketingautomation/list/sync/sync.service.ts +++ b/packages/api/src/marketingautomation/list/sync/sync.service.ts @@ -11,7 +11,7 @@ import { UnifiedListOutput } from '../types/model.unified'; import { IListService } from '../types'; @Injectable() -export class SyncService implements OnModuleInit { +export class SyncService implements OnModuleInit, IBaseSync { constructor( private prisma: PrismaService, private logger: LoggerService, diff --git a/packages/api/src/marketingautomation/message/message.module.ts b/packages/api/src/marketingautomation/message/message.module.ts index 962acb26d..4bfed6efa 100644 --- a/packages/api/src/marketingautomation/message/message.module.ts +++ b/packages/api/src/marketingautomation/message/message.module.ts @@ -10,6 +10,7 @@ import { MessageController } from './message.controller'; import { MessageService } from './services/message.service'; import { ServiceRegistry } from './services/registry.service'; import { SyncService } from './sync/sync.service'; +import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; @Module({ imports: [ @@ -28,6 +29,7 @@ import { SyncService } from './sync/sync.service'; FieldMappingService, ServiceRegistry, ConnectionUtils, + IngestDataService, /* PROVIDERS SERVICES */ ], exports: [SyncService, CoreModule], diff --git a/packages/api/src/marketingautomation/message/sync/sync.service.ts b/packages/api/src/marketingautomation/message/sync/sync.service.ts index c6f498041..e35f44843 100644 --- a/packages/api/src/marketingautomation/message/sync/sync.service.ts +++ b/packages/api/src/marketingautomation/message/sync/sync.service.ts @@ -11,7 +11,7 @@ import { UnifiedMessageOutput } from '../types/model.unified'; import { IMessageService } from '../types'; @Injectable() -export class SyncService implements OnModuleInit { +export class SyncService implements OnModuleInit, IBaseSync { constructor( private prisma: PrismaService, private logger: LoggerService, diff --git a/packages/api/src/marketingautomation/template/sync/sync.service.ts b/packages/api/src/marketingautomation/template/sync/sync.service.ts index 3a0c22a18..6ee7e9ef6 100644 --- a/packages/api/src/marketingautomation/template/sync/sync.service.ts +++ b/packages/api/src/marketingautomation/template/sync/sync.service.ts @@ -11,7 +11,7 @@ import { UnifiedTemplateOutput } from '../types/model.unified'; import { ITemplateService } from '../types'; @Injectable() -export class SyncService implements OnModuleInit { +export class SyncService implements OnModuleInit, IBaseSync { constructor( private prisma: PrismaService, private logger: LoggerService, diff --git a/packages/api/src/marketingautomation/template/template.module.ts b/packages/api/src/marketingautomation/template/template.module.ts index 68d4c9aef..cbb2280a1 100644 --- a/packages/api/src/marketingautomation/template/template.module.ts +++ b/packages/api/src/marketingautomation/template/template.module.ts @@ -10,6 +10,7 @@ import { ServiceRegistry } from './services/registry.service'; import { TemplateService } from './services/template.service'; import { SyncService } from './sync/sync.service'; import { TemplateController } from './template.controller'; +import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; @Module({ imports: [ @@ -28,6 +29,7 @@ import { TemplateController } from './template.controller'; FieldMappingService, ServiceRegistry, ConnectionUtils, + IngestDataService, /* PROVIDERS SERVICES */ ], exports: [SyncService, CoreModule], diff --git a/packages/api/src/marketingautomation/user/sync/sync.service.ts b/packages/api/src/marketingautomation/user/sync/sync.service.ts index 5ef4720f9..2cfdfe0e6 100644 --- a/packages/api/src/marketingautomation/user/sync/sync.service.ts +++ b/packages/api/src/marketingautomation/user/sync/sync.service.ts @@ -11,7 +11,7 @@ import { UnifiedUserOutput } from '../types/model.unified'; import { IUserService } from '../types'; @Injectable() -export class SyncService implements OnModuleInit { +export class SyncService implements OnModuleInit, IBaseSync { constructor( private prisma: PrismaService, private logger: LoggerService, diff --git a/packages/api/src/marketingautomation/user/user.module.ts b/packages/api/src/marketingautomation/user/user.module.ts index 2afd2224a..e09a3caac 100644 --- a/packages/api/src/marketingautomation/user/user.module.ts +++ b/packages/api/src/marketingautomation/user/user.module.ts @@ -10,6 +10,7 @@ import { ServiceRegistry } from './services/registry.service'; import { UserService } from './services/user.service'; import { SyncService } from './sync/sync.service'; import { UserController } from './user.controller'; +import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; @Module({ imports: [ @@ -28,6 +29,7 @@ import { UserController } from './user.controller'; FieldMappingService, ServiceRegistry, ConnectionUtils, + IngestDataService, /* PROVIDERS SERVICES */ ], exports: [SyncService, CoreModule], diff --git a/packages/api/src/ticketing/@webhook/handler.module.ts b/packages/api/src/ticketing/@webhook/handler.module.ts index e0ad6d224..5daec7d9a 100644 --- a/packages/api/src/ticketing/@webhook/handler.module.ts +++ b/packages/api/src/ticketing/@webhook/handler.module.ts @@ -16,6 +16,7 @@ import { ZendeskHandlerService } from './zendesk/handler'; EncryptionService, EnvironmentService, TicketingWebhookHandlerService, + IngestDataService, /* PROVIDERS SERVICES */ ZendeskHandlerService, ], diff --git a/packages/api/src/ticketing/account/account.module.ts b/packages/api/src/ticketing/account/account.module.ts index bf2316ee8..f1d97090c 100644 --- a/packages/api/src/ticketing/account/account.module.ts +++ b/packages/api/src/ticketing/account/account.module.ts @@ -13,6 +13,7 @@ import { ServiceRegistry } from './services/registry.service'; import { ZendeskService } from './services/zendesk'; import { SyncService } from './sync/sync.service'; import { CoreUnification } from '@@core/@core-services/unification/core-unification.service'; +import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; @Module({ imports: [], @@ -29,6 +30,7 @@ import { CoreUnification } from '@@core/@core-services/unification/core-unificat CoreUnification, UnificationRegistry, MappersRegistry, + IngestDataService, /* PROVIDERS SERVICES */ ZendeskService, FrontService, diff --git a/packages/api/src/ticketing/account/sync/sync.service.ts b/packages/api/src/ticketing/account/sync/sync.service.ts index 2dcaa5e7d..9ce8b9801 100644 --- a/packages/api/src/ticketing/account/sync/sync.service.ts +++ b/packages/api/src/ticketing/account/sync/sync.service.ts @@ -16,9 +16,11 @@ import { TICKETING_PROVIDERS } from '@panora/shared'; import { CoreSyncRegistry } from '@@core/@core-services/registries/core-sync.registry'; import { CoreUnification } from '@@core/@core-services/unification/core-unification.service'; import { BullQueueService } from '@@core/@core-services/queues/shared.service'; +import { IBaseSync } from '@@core/utils/types/interface'; +import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; @Injectable() -export class SyncService implements OnModuleInit { +export class SyncService implements OnModuleInit, IBaseSync { constructor( private prisma: PrismaService, private logger: LoggerService, @@ -28,6 +30,7 @@ export class SyncService implements OnModuleInit { private coreUnification: CoreUnification, private registry: CoreSyncRegistry, private bullQueueService: BullQueueService, + private ingestService: IngestDataService, ) { this.logger.setContext(SyncService.name); this.registry.registerService('ticketing', 'account', this); @@ -82,7 +85,6 @@ export class SyncService implements OnModuleInit { await this.syncAccountsForLinkedUser( provider, linkedUser.id_linked_user, - id_project, ); } catch (error) { throw error; @@ -104,7 +106,6 @@ export class SyncService implements OnModuleInit { async syncAccountsForLinkedUser( integrationId: string, linkedUserId: string, - id_project: string, wh_real_time_trigger?: { action: 'UPDATE' | 'DELETE'; data: { @@ -170,63 +171,34 @@ export class SyncService implements OnModuleInit { } const sourceObject: OriginalAccountOutput[] = resp.data; - // this.logger.log('resp is ' + sourceObject); - //unify the data according to the target obj wanted - const unifiedObject = (await this.coreUnification.unify< - OriginalAccountOutput[] - >({ + await this.ingestService.ingestData< + UnifiedAccountOutput, + OriginalAccountOutput + >( sourceObject, - targetType: TicketingObject.account, - providerName: integrationId, - vertical: 'ticketing', - connectionId: connection.id_connection, - customFieldMappings, - })) as UnifiedAccountOutput[]; - - //insert the data in the DB with the fieldMappings (value table) - const account_data = await this.saveAccountsInDb( - connection.id_connection, - linkedUserId, - unifiedObject, integrationId, - sourceObject, - ); - const event = await this.prisma.events.create({ - data: { - id_event: uuidv4(), - status: 'success', - type: 'ticketing.account.synced', - method: 'SYNC', - url: '/sync', - provider: integrationId, - direction: '0', - timestamp: new Date(), - id_linked_user: linkedUserId, - }, - }); - await this.webhook.handleWebhook( - account_data, - 'ticketing.account.sync', - id_project, - event.id_event, + connection.id_connection, + 'ticketing', + 'account', + customFieldMappings, ); } catch (error) { throw error; } } - async saveAccountsInDb( + async saveToDb( connection_id: string, linkedUserId: string, - accounts: UnifiedAccountOutput[], + data: UnifiedAccountOutput[], originSource: string, remote_data: Record[], ): Promise { try { let accounts_results: TicketingAccount[] = []; - for (let i = 0; i < accounts.length; i++) { - const account = accounts[i]; + for (let i = 0; i < data.length; i++) { + const account = data[i]; const originId = account.remote_id; if (!originId || originId == '') { diff --git a/packages/api/src/ticketing/attachment/attachment.module.ts b/packages/api/src/ticketing/attachment/attachment.module.ts index fa149dcd1..c133928e3 100644 --- a/packages/api/src/ticketing/attachment/attachment.module.ts +++ b/packages/api/src/ticketing/attachment/attachment.module.ts @@ -10,6 +10,7 @@ import { AttachmentController } from './attachment.controller'; import { AttachmentService } from './services/attachment.service'; import { ServiceRegistry } from './services/registry.service'; import { CoreUnification } from '@@core/@core-services/unification/core-unification.service'; +import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; @Module({ imports: [], @@ -25,6 +26,7 @@ import { CoreUnification } from '@@core/@core-services/unification/core-unificat CoreUnification, UnificationRegistry, MappersRegistry, + IngestDataService, /* PROVIDERS SERVICES */ ], }) diff --git a/packages/api/src/ticketing/attachment/services/attachment.service.ts b/packages/api/src/ticketing/attachment/services/attachment.service.ts index b94758634..0f061b5a2 100644 --- a/packages/api/src/ticketing/attachment/services/attachment.service.ts +++ b/packages/api/src/ticketing/attachment/services/attachment.service.ts @@ -99,7 +99,7 @@ export class AttachmentService { }, }); - await this.webhook.handleWebhook( + await this.webhook.dispatchWebhook( result_attachment, 'ticketing.attachment.created', linkedUser.id_project, diff --git a/packages/api/src/ticketing/collection/collection.module.ts b/packages/api/src/ticketing/collection/collection.module.ts index 8ab3ccd17..f50c48c85 100644 --- a/packages/api/src/ticketing/collection/collection.module.ts +++ b/packages/api/src/ticketing/collection/collection.module.ts @@ -13,6 +13,7 @@ import { JiraService } from './services/jira'; import { UnificationRegistry } from '@@core/@core-services/registries/unification.registry'; import { CoreUnification } from '@@core/@core-services/unification/core-unification.service'; import { MappersRegistry } from '@@core/@core-services/registries/mappers.registry'; +import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; @Module({ imports: [], @@ -29,6 +30,7 @@ import { MappersRegistry } from '@@core/@core-services/registries/mappers.regist CoreUnification, UnificationRegistry, MappersRegistry, + IngestDataService, /* PROVIDERS SERVICES */ JiraService, GitlabService, diff --git a/packages/api/src/ticketing/collection/sync/sync.service.ts b/packages/api/src/ticketing/collection/sync/sync.service.ts index fa8b881e1..fff70d3aa 100644 --- a/packages/api/src/ticketing/collection/sync/sync.service.ts +++ b/packages/api/src/ticketing/collection/sync/sync.service.ts @@ -15,10 +15,12 @@ import { tcg_collections as TicketingCollection } from '@prisma/client'; import { TICKETING_PROVIDERS } from '@panora/shared'; import { CoreSyncRegistry } from '@@core/@core-services/registries/core-sync.registry'; import { BullQueueService } from '@@core/@core-services/queues/shared.service'; +import { IBaseSync } from '@@core/utils/types/interface'; +import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; import { CoreUnification } from '@@core/@core-services/unification/core-unification.service'; @Injectable() -export class SyncService implements OnModuleInit { +export class SyncService implements OnModuleInit, IBaseSync { constructor( private prisma: PrismaService, private logger: LoggerService, @@ -27,6 +29,7 @@ export class SyncService implements OnModuleInit { private coreUnification: CoreUnification, private registry: CoreSyncRegistry, private bullQueueService: BullQueueService, + private ingestService: IngestDataService, ) { this.logger.setContext(SyncService.name); this.registry.registerService('ticketing', 'collection', this); @@ -81,7 +84,6 @@ export class SyncService implements OnModuleInit { await this.syncCollectionsForLinkedUser( provider, linkedUser.id_linked_user, - id_project, ); } catch (error) { throw error; @@ -103,7 +105,6 @@ export class SyncService implements OnModuleInit { async syncCollectionsForLinkedUser( integrationId: string, linkedUserId: string, - id_project: string, ) { try { this.logger.log( @@ -130,65 +131,34 @@ export class SyncService implements OnModuleInit { await service.syncCollections(linkedUserId); const sourceObject: OriginalCollectionOutput[] = resp.data; - //this.logger.log('SOURCE OBJECT DATA = ' + JSON.stringify(sourceObject)); - //unify the data according to the target obj wanted - const unifiedObject = (await this.coreUnification.unify< - OriginalCollectionOutput[] - >({ - sourceObject, - targetType: TicketingObject.collection, - providerName: integrationId, - vertical: 'ticketing', - connectionId: connection.id_connection, - customFieldMappings: [], - })) as UnifiedCollectionOutput[]; - //insert the data in the DB with the fieldMappings (value table) - const collection_data = await this.saveCollectionsInDb( - connection.id_connection, - linkedUserId, - unifiedObject, - integrationId, + await this.ingestService.ingestData< + UnifiedCollectionOutput, + OriginalCollectionOutput + >( sourceObject, - ); - - const event = await this.prisma.events.create({ - data: { - id_event: uuidv4(), - status: 'success', - type: 'ticketing.collection.synced', - method: 'SYNC', - url: '/sync', - provider: integrationId, - direction: '0', - timestamp: new Date(), - id_linked_user: linkedUserId, - }, - }); - - await this.webhook.handleWebhook( - collection_data, - 'ticketing.collection.pulled', - id_project, - event.id_event, + integrationId, + connection.id_connection, + 'ticketing', + 'collection', + [], ); } catch (error) { throw error; } } - async saveCollectionsInDb( + async saveToDb( connection_id: string, linkedUserId: string, - collections: UnifiedCollectionOutput[], + data: UnifiedCollectionOutput[], originSource: string, remote_data: Record[], ): Promise { try { let collections_results: TicketingCollection[] = []; - console.log(collections); - for (let i = 0; i < collections.length; i++) { - const collection = collections[i]; + for (let i = 0; i < data.length; i++) { + const collection = data[i]; const originId = collection.remote_id; if (!originId || originId == '') { diff --git a/packages/api/src/ticketing/comment/comment.module.ts b/packages/api/src/ticketing/comment/comment.module.ts index f9f7494af..c3e40c89a 100644 --- a/packages/api/src/ticketing/comment/comment.module.ts +++ b/packages/api/src/ticketing/comment/comment.module.ts @@ -17,6 +17,8 @@ import { ServiceRegistry } from './services/registry.service'; import { ZendeskService } from './services/zendesk'; import { SyncService } from './sync/sync.service'; import { CoreUnification } from '@@core/@core-services/unification/core-unification.service'; +import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; +import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; @Module({ imports: [], @@ -33,7 +35,9 @@ import { CoreUnification } from '@@core/@core-services/unification/core-unificat CoreUnification, UnificationRegistry, MappersRegistry, + IngestDataService, Utils, + IngestDataService, /* PROVIDERS SERVICES */ ZendeskService, FrontService, diff --git a/packages/api/src/ticketing/comment/services/comment.service.ts b/packages/api/src/ticketing/comment/services/comment.service.ts index d7cd9b8c7..81f797f38 100644 --- a/packages/api/src/ticketing/comment/services/comment.service.ts +++ b/packages/api/src/ticketing/comment/services/comment.service.ts @@ -264,7 +264,7 @@ export class CommentService { id_linked_user: linkedUserId, }, }); - await this.webhook.handleWebhook( + await this.webhook.dispatchWebhook( result_comment, 'ticketing.comment.created', linkedUser.id_project, diff --git a/packages/api/src/ticketing/comment/services/zendesk/mappers.ts b/packages/api/src/ticketing/comment/services/zendesk/mappers.ts index 17997cd1c..0aad67970 100644 --- a/packages/api/src/ticketing/comment/services/zendesk/mappers.ts +++ b/packages/api/src/ticketing/comment/services/zendesk/mappers.ts @@ -5,20 +5,17 @@ import { UnifiedCommentOutput, } from '@ticketing/comment/types/model.unified'; import { UnifiedAttachmentOutput } from '@ticketing/attachment/types/model.unified'; - -import { TicketingObject } from '@ticketing/@lib/@types'; import { OriginalAttachmentOutput } from '@@core/utils/types/original/original.ticketing'; import { Utils } from '@ticketing/@lib/@utils'; - import { MappersRegistry } from '@@core/@core-services/registries/mappers.registry'; import { Injectable } from '@nestjs/common'; -import { CoreUnification } from '@@core/@core-services/unification/core-unification.service'; +import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; @Injectable() export class ZendeskCommentMapper implements ICommentMapper { constructor( private mappersRegistry: MappersRegistry, private utils: Utils, - private coreUnification: CoreUnification, + private ingestService: IngestDataService, ) { this.mappersRegistry.registerService( 'ticketing', @@ -100,18 +97,20 @@ export class ZendeskCommentMapper implements ICommentMapper { let opts; if (comment.attachments && comment.attachments.length > 0) { - const unifiedObject = (await this.coreUnification.unify< - OriginalAttachmentOutput[] - >({ - sourceObject: comment.attachments, - targetType: TicketingObject.attachment, - providerName: 'zendesk', - vertical: 'ticketing', - connectionId: connectionId, - customFieldMappings: [], - })) as UnifiedAttachmentOutput[]; + const results = await this.ingestService.ingestData< + UnifiedAttachmentOutput, + OriginalAttachmentOutput + >( + comment.attachments, + 'zendesk', + connectionId, + 'ticketing', + 'attachment', + [], + ); + const attachment_ids: string[] = results.map((res) => res.id); - opts = { ...opts, attachments: unifiedObject }; + opts = { ...opts, attachments: attachment_ids }; } if (comment.author_id) { diff --git a/packages/api/src/ticketing/comment/sync/sync.service.ts b/packages/api/src/ticketing/comment/sync/sync.service.ts index 4d5089223..da02e6996 100644 --- a/packages/api/src/ticketing/comment/sync/sync.service.ts +++ b/packages/api/src/ticketing/comment/sync/sync.service.ts @@ -15,10 +15,12 @@ import { ServiceRegistry } from '../services/registry.service'; import { TICKETING_PROVIDERS } from '@panora/shared'; import { CoreSyncRegistry } from '@@core/@core-services/registries/core-sync.registry'; import { BullQueueService } from '@@core/@core-services/queues/shared.service'; +import { IBaseSync } from '@@core/utils/types/interface'; +import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; import { CoreUnification } from '@@core/@core-services/unification/core-unification.service'; @Injectable() -export class SyncService implements OnModuleInit { +export class SyncService implements OnModuleInit, IBaseSync { constructor( private prisma: PrismaService, private logger: LoggerService, @@ -28,6 +30,7 @@ export class SyncService implements OnModuleInit { private coreUnification: CoreUnification, private registry: CoreSyncRegistry, private bullQueueService: BullQueueService, + private ingestService: IngestDataService, ) { this.logger.setContext(SyncService.name); this.registry.registerService('ticketing', 'comment', this); @@ -95,7 +98,6 @@ export class SyncService implements OnModuleInit { await this.syncCommentsForLinkedUser( provider, linkedUser.id_linked_user, - id_project, ticket.id_tcg_ticket, ); } @@ -119,7 +121,6 @@ export class SyncService implements OnModuleInit { async syncCommentsForLinkedUser( integrationId: string, linkedUserId: string, - id_project: string, id_ticket: string, ) { try { @@ -158,52 +159,23 @@ export class SyncService implements OnModuleInit { const sourceObject: OriginalCommentOutput[] = resp.data; - //unify the data according to the target obj wanted - const unifiedObject = (await this.coreUnification.unify< - OriginalCommentOutput[] - >({ + await this.ingestService.ingestData< + UnifiedCommentOutput, + OriginalCommentOutput + >( sourceObject, - targetType: TicketingObject.comment, - providerName: integrationId, - vertical: 'ticketing', - connectionId: connection.id_connection, - customFieldMappings, - })) as UnifiedCommentOutput[]; - - //insert the data in the DB with the fieldMappings (value table) - const comments_data = await this.saveCommentsInDb( + integrationId, connection.id_connection, - linkedUserId, - unifiedObject, - id_ticket, - sourceObject, - ); - - const event = await this.prisma.events.create({ - data: { - id_event: uuidv4(), - status: 'success', - type: 'ticketing.comment.synced', - method: 'SYNC', - url: '/sync', - provider: integrationId, - direction: '0', - timestamp: new Date(), - id_linked_user: linkedUserId, - }, - }); - await this.webhook.handleWebhook( - comments_data, - 'ticketing.comment.pulled', - id_project, - event.id_event, + 'tikceting', + 'comment', + customFieldMappings, ); } catch (error) { throw error; } } - async saveCommentsInDb( + async saveToDb( connection_id: string, linkedUserId: string, comments: UnifiedCommentOutput[], diff --git a/packages/api/src/ticketing/comment/types/model.unified.ts b/packages/api/src/ticketing/comment/types/model.unified.ts index 5bfdf00d0..7349a1106 100644 --- a/packages/api/src/ticketing/comment/types/model.unified.ts +++ b/packages/api/src/ticketing/comment/types/model.unified.ts @@ -92,7 +92,7 @@ export class UnifiedCommentOutput extends UnifiedCommentInput { @ApiPropertyOptional({ type: [UnifiedAttachmentOutput], - description: 'The attachemnets tied to the comment', + description: 'The attachments tied to the comment', }) @IsOptional() attachments?: UnifiedAttachmentOutput[]; // Attachments objects diff --git a/packages/api/src/ticketing/contact/contact.module.ts b/packages/api/src/ticketing/contact/contact.module.ts index f173465d1..506ce81d7 100644 --- a/packages/api/src/ticketing/contact/contact.module.ts +++ b/packages/api/src/ticketing/contact/contact.module.ts @@ -15,6 +15,7 @@ import { ServiceRegistry } from './services/registry.service'; import { ZendeskService } from './services/zendesk'; import { SyncService } from './sync/sync.service'; import { CoreUnification } from '@@core/@core-services/unification/core-unification.service'; +import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; @Module({ imports: [], @@ -32,6 +33,7 @@ import { CoreUnification } from '@@core/@core-services/unification/core-unificat UnificationRegistry, MappersRegistry, Utils, + IngestDataService, /* PROVIDERS SERVICES */ ZendeskService, FrontService, diff --git a/packages/api/src/ticketing/contact/sync/sync.service.ts b/packages/api/src/ticketing/contact/sync/sync.service.ts index 235c2e214..39caab5aa 100644 --- a/packages/api/src/ticketing/contact/sync/sync.service.ts +++ b/packages/api/src/ticketing/contact/sync/sync.service.ts @@ -17,10 +17,12 @@ import { OriginalContactOutput } from '@@core/utils/types/original/original.tick import { TICKETING_PROVIDERS } from '@panora/shared'; import { CoreSyncRegistry } from '@@core/@core-services/registries/core-sync.registry'; import { BullQueueService } from '@@core/@core-services/queues/shared.service'; +import { IBaseSync } from '@@core/utils/types/interface'; +import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; import { CoreUnification } from '@@core/@core-services/unification/core-unification.service'; @Injectable() -export class SyncService implements OnModuleInit { +export class SyncService implements OnModuleInit, IBaseSync { constructor( private prisma: PrismaService, private logger: LoggerService, @@ -30,10 +32,20 @@ export class SyncService implements OnModuleInit { private coreUnification: CoreUnification, private registry: CoreSyncRegistry, private bullQueueService: BullQueueService, + private ingestService: IngestDataService, ) { this.logger.setContext(SyncService.name); this.registry.registerService('ticketing', 'contact', this); } + saveToDb( + connection_id: string, + linkedUserId: string, + data: any[], + originSource: string, + remote_data: Record[], + ): Promise { + throw new Error('Method not implemented.'); + } async onModuleInit() { try { @@ -87,7 +99,6 @@ export class SyncService implements OnModuleInit { await this.syncContactsForLinkedUser( provider, linkedUser.id_linked_user, - id_project, acc.remote_id, ); } @@ -118,7 +129,6 @@ export class SyncService implements OnModuleInit { async syncContactsForLinkedUser( integrationId: string, linkedUserId: string, - id_project: string, remote_account_id?: string, wh_real_time_trigger?: { action: 'UPDATE' | 'DELETE'; @@ -163,8 +173,7 @@ export class SyncService implements OnModuleInit { switch (wh_real_time_trigger.action) { case 'DELETE': return await this.removeContactInDb( - linkedUserId, - integrationId, + connection.id_connection, wh_real_time_trigger.data.remote_id, ); default: @@ -183,45 +192,17 @@ export class SyncService implements OnModuleInit { ); } const sourceObject: OriginalContactOutput[] = resp.data; - //unify the data according to the target obj wanted - const unifiedObject = (await this.coreUnification.unify< - OriginalContactOutput[] - >({ - sourceObject, - targetType: TicketingObject.contact, - providerName: integrationId, - vertical: 'ticketing', - connectionId: connection.id_connection, - customFieldMappings, - })) as UnifiedContactOutput[]; - //insert the data in the DB with the fieldMappings (value table) - const contact_data = await this.saveContactsInDb( - connection.id_connection, - linkedUserId, - unifiedObject, - integrationId, + await this.ingestService.ingestData< + UnifiedContactOutput, + OriginalContactOutput + >( sourceObject, - remote_account_id, - ); - const event = await this.prisma.events.create({ - data: { - id_event: uuidv4(), - status: 'success', - type: 'ticketing.contact.synced', - method: 'SYNC', - url: '/sync', - provider: integrationId, - direction: '0', - timestamp: new Date(), - id_linked_user: linkedUserId, - }, - }); - await this.webhook.handleWebhook( - contact_data, - 'ticketing.contact.pulled', - id_project, - event.id_event, + integrationId, + connection.id_connection, + 'ticketing', + 'contact', + customFieldMappings, ); } catch (error) { throw error; @@ -376,11 +357,7 @@ export class SyncService implements OnModuleInit { throw error; } } - async removeContactInDb( - linkedUserId: string, - originSource: string, - remote_id: string, - ) { + async removeContactInDb(connection_id: string, remote_id: string) { const existingContact = await this.prisma.tcg_contacts.findFirst({ where: { remote_id: remote_id, diff --git a/packages/api/src/ticketing/tag/sync/sync.service.ts b/packages/api/src/ticketing/tag/sync/sync.service.ts index aa7b91ede..f9d6361cd 100644 --- a/packages/api/src/ticketing/tag/sync/sync.service.ts +++ b/packages/api/src/ticketing/tag/sync/sync.service.ts @@ -16,10 +16,12 @@ import { tcg_tags as TicketingTag } from '@prisma/client'; import { TICKETING_PROVIDERS } from '@panora/shared'; import { CoreSyncRegistry } from '@@core/@core-services/registries/core-sync.registry'; import { BullQueueService } from '@@core/@core-services/queues/shared.service'; +import { IBaseSync } from '@@core/utils/types/interface'; +import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; import { CoreUnification } from '@@core/@core-services/unification/core-unification.service'; @Injectable() -export class SyncService implements OnModuleInit { +export class SyncService implements OnModuleInit, IBaseSync { constructor( private prisma: PrismaService, private logger: LoggerService, @@ -29,6 +31,7 @@ export class SyncService implements OnModuleInit { private coreUnification: CoreUnification, private registry: CoreSyncRegistry, private bullQueueService: BullQueueService, + private ingestService: IngestDataService, ) { this.logger.setContext(SyncService.name); this.registry.registerService('ticketing', 'tag', this); @@ -96,7 +99,6 @@ export class SyncService implements OnModuleInit { await this.syncTagsForLinkedUser( provider, linkedUser.id_linked_user, - id_project, ticket.id_tcg_ticket, ); } @@ -120,7 +122,6 @@ export class SyncService implements OnModuleInit { async syncTagsForLinkedUser( integrationId: string, linkedUserId: string, - id_project: string, id_ticket: string, ) { try { @@ -158,60 +159,25 @@ export class SyncService implements OnModuleInit { const sourceObject: OriginalTagOutput[] = resp.data; + await this.ingestService.ingestData( + sourceObject, + integrationId, + connection.id_connection, + 'ticketing', + 'tag', + customFieldMappings, + ); //TODO; do it in every file if (!sourceObject || sourceObject.length == 0) { this.logger.warn('Source object is empty, returning :) ....'); return; } - - //unify the data according to the target obj wanted - const unifiedObject = (await this.coreUnification.unify< - OriginalTagOutput[] - >({ - sourceObject, - targetType: TicketingObject.tag, - providerName: integrationId, - vertical: 'ticketing', - connectionId: connection.id_connection, - customFieldMappings, - })) as UnifiedTagOutput[]; - - //TODO: exceptionally we use the unifiedObject as we might need to get the fake remote ids from Zendesk store in id field - - //insert the data in the DB with the fieldMappings (value table) - const tag_data = await this.saveTagsInDb( - connection.id_connection, - linkedUserId, - unifiedObject, - integrationId, - id_ticket, - sourceObject, - ); - const event = await this.prisma.events.create({ - data: { - id_event: uuidv4(), - status: 'success', - type: 'ticketing.tag.synced', - method: 'SYNC', - url: '/sync', - provider: integrationId, - direction: '0', - timestamp: new Date(), - id_linked_user: linkedUserId, - }, - }); - await this.webhook.handleWebhook( - tag_data, - 'ticketing.tag.pulled', - id_project, - event.id_event, - ); } catch (error) { throw error; } } - async saveTagsInDb( + async saveToDb( connection_id: string, linkedUserId: string, tags: UnifiedTagOutput[], diff --git a/packages/api/src/ticketing/tag/tag.module.ts b/packages/api/src/ticketing/tag/tag.module.ts index 623ecd90c..c83e2b6bb 100644 --- a/packages/api/src/ticketing/tag/tag.module.ts +++ b/packages/api/src/ticketing/tag/tag.module.ts @@ -16,6 +16,7 @@ import { ZendeskService } from './services/zendesk'; import { SyncService } from './sync/sync.service'; import { TagController } from './tag.controller'; import { CoreUnification } from '@@core/@core-services/unification/core-unification.service'; +import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; @Module({ imports: [], @@ -33,6 +34,7 @@ import { CoreUnification } from '@@core/@core-services/unification/core-unificat UnificationRegistry, MappersRegistry, Utils, + IngestDataService, /* PROVIDERS SERVICES */ ZendeskService, FrontService, diff --git a/packages/api/src/ticketing/team/sync/sync.service.ts b/packages/api/src/ticketing/team/sync/sync.service.ts index 793f901cb..fa6301541 100644 --- a/packages/api/src/ticketing/team/sync/sync.service.ts +++ b/packages/api/src/ticketing/team/sync/sync.service.ts @@ -1,6 +1,8 @@ import { LoggerService } from '@@core/@core-services/logger/logger.service'; import { PrismaService } from '@@core/@core-services/prisma/prisma.service'; import { BullQueueService } from '@@core/@core-services/queues/shared.service'; +import { IBaseSync } from '@@core/utils/types/interface'; +import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; import { CoreSyncRegistry } from '@@core/@core-services/registries/core-sync.registry'; import { CoreUnification } from '@@core/@core-services/unification/core-unification.service'; import { WebhookService } from '@@core/@core-services/webhooks/panora-webhooks/webhook.service'; @@ -18,7 +20,7 @@ import { ITeamService } from '../types'; import { UnifiedTeamOutput } from '../types/model.unified'; @Injectable() -export class SyncService implements OnModuleInit { +export class SyncService implements OnModuleInit, IBaseSync { constructor( private prisma: PrismaService, private logger: LoggerService, @@ -28,6 +30,7 @@ export class SyncService implements OnModuleInit { private coreUnification: CoreUnification, private registry: CoreSyncRegistry, private bullQueueService: BullQueueService, + private ingestService: IngestDataService, ) { this.logger.setContext(SyncService.name); this.registry.registerService('ticketing', 'team', this); @@ -82,7 +85,6 @@ export class SyncService implements OnModuleInit { await this.syncTeamsForLinkedUser( provider, linkedUser.id_linked_user, - id_project, ); } catch (error) { throw error; @@ -101,11 +103,7 @@ export class SyncService implements OnModuleInit { } //todo: HANDLE DATA REMOVED FROM PROVIDER - async syncTeamsForLinkedUser( - integrationId: string, - linkedUserId: string, - id_project: string, - ) { + async syncTeamsForLinkedUser(integrationId: string, linkedUserId: string) { try { this.logger.log( `Syncing ${integrationId} teams for linkedTeam ${linkedUserId}`, @@ -143,54 +141,24 @@ export class SyncService implements OnModuleInit { ); const sourceObject: OriginalTeamOutput[] = resp.data; - //this.logger.log('SOURCE OBJECT DATA = ' + JSON.stringify(sourceObject)); - //unify the data according to the target obj wanted - const unifiedObject = (await this.coreUnification.unify< - OriginalTeamOutput[] - >({ - sourceObject, - targetType: TicketingObject.team, - providerName: integrationId, - vertical: 'ticketing', - connectionId: connection.id_connection, - customFieldMappings, - })) as UnifiedTeamOutput[]; - //insert the data in the DB with the fieldMappings (value table) - const team_data = await this.saveTeamsInDb( - connection.id_connection, - linkedUserId, - unifiedObject, - integrationId, + await this.ingestService.ingestData< + UnifiedTeamOutput, + OriginalTeamOutput + >( sourceObject, - ); - - const event = await this.prisma.events.create({ - data: { - id_event: uuidv4(), - status: 'success', - type: 'ticketing.team..synced', - method: 'SYNC', - url: '/sync', - provider: integrationId, - direction: '0', - timestamp: new Date(), - id_linked_user: linkedUserId, - }, - }); - - await this.webhook.handleWebhook( - team_data, - 'ticketing.team.pulled', - id_project, - event.id_event, + integrationId, + connection.id_connection, + 'ticketing', + 'team', + customFieldMappings, ); } catch (error) { throw error; } } - async saveTeamsInDb( + async saveToDb( connection_id: string, linkedUserId: string, teams: UnifiedTeamOutput[], diff --git a/packages/api/src/ticketing/team/team.module.ts b/packages/api/src/ticketing/team/team.module.ts index bb66b0e42..32708f237 100644 --- a/packages/api/src/ticketing/team/team.module.ts +++ b/packages/api/src/ticketing/team/team.module.ts @@ -16,6 +16,7 @@ import { ZendeskService } from './services/zendesk'; import { SyncService } from './sync/sync.service'; import { TeamController } from './team.controller'; import { CoreUnification } from '@@core/@core-services/unification/core-unification.service'; +import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; @Module({ imports: [], @@ -33,6 +34,7 @@ import { CoreUnification } from '@@core/@core-services/unification/core-unificat UnificationRegistry, MappersRegistry, Utils, + IngestDataService, /* PROVIDERS SERVICES */ ZendeskService, FrontService, diff --git a/packages/api/src/ticketing/ticket/services/ticket.service.ts b/packages/api/src/ticketing/ticket/services/ticket.service.ts index 4fbe5a378..5384a30a5 100644 --- a/packages/api/src/ticketing/ticket/services/ticket.service.ts +++ b/packages/api/src/ticketing/ticket/services/ticket.service.ts @@ -308,7 +308,7 @@ export class TicketService { id_linked_user: linkedUserId, }, }); - await this.webhook.handleWebhook( + await this.webhook.dispatchWebhook( result_ticket, 'ticketing.ticket.created', linkedUser.id_project, diff --git a/packages/api/src/ticketing/ticket/sync/sync.service.ts b/packages/api/src/ticketing/ticket/sync/sync.service.ts index 04aa128c2..769adb7e9 100644 --- a/packages/api/src/ticketing/ticket/sync/sync.service.ts +++ b/packages/api/src/ticketing/ticket/sync/sync.service.ts @@ -15,10 +15,12 @@ import { ServiceRegistry } from '../services/registry.service'; import { TICKETING_PROVIDERS } from '@panora/shared'; import { CoreSyncRegistry } from '@@core/@core-services/registries/core-sync.registry'; import { BullQueueService } from '@@core/@core-services/queues/shared.service'; +import { IBaseSync } from '@@core/utils/types/interface'; +import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; import { CoreUnification } from '@@core/@core-services/unification/core-unification.service'; @Injectable() -export class SyncService implements OnModuleInit { +export class SyncService implements OnModuleInit, IBaseSync { constructor( private prisma: PrismaService, private logger: LoggerService, @@ -28,6 +30,7 @@ export class SyncService implements OnModuleInit { private coreUnification: CoreUnification, private registry: CoreSyncRegistry, private bullQueueService: BullQueueService, + private ingestService: IngestDataService, ) { this.logger.setContext(SyncService.name); this.registry.registerService('ticketing', 'ticket', this); @@ -82,7 +85,6 @@ export class SyncService implements OnModuleInit { await this.syncTicketsForLinkedUser( provider, linkedUser.id_linked_user, - id_project, ); } catch (error) { throw error; @@ -104,7 +106,6 @@ export class SyncService implements OnModuleInit { async syncTicketsForLinkedUser( integrationId: string, linkedUserId: string, - id_project: string, wh_real_time_trigger?: { action: 'UPDATE' | 'DELETE'; data: { @@ -149,8 +150,7 @@ export class SyncService implements OnModuleInit { switch (wh_real_time_trigger.action) { case 'DELETE': return await this.removeTicketInDb( - linkedUserId, - integrationId, + connection.id_connection, wh_real_time_trigger.data.remote_id, ); default: @@ -170,54 +170,24 @@ export class SyncService implements OnModuleInit { } const sourceObject: OriginalTicketOutput[] = resp.data; - //this.logger.log('SOURCE OBJECT DATA = ' + JSON.stringify(sourceObject)); - //unify the data according to the target obj wanted - const unifiedObject = (await this.coreUnification.unify< - OriginalTicketOutput[] - >({ - sourceObject, - targetType: TicketingObject.ticket, - providerName: integrationId, - vertical: 'ticketing', - connectionId: connection.id_connection, - customFieldMappings, - })) as UnifiedTicketOutput[]; - //insert the data in the DB with the fieldMappings (value table) - const tickets_data = await this.saveTicketsInDb( - connection.id_connection, - linkedUserId, - unifiedObject, - integrationId, + await this.ingestService.ingestData< + UnifiedTicketOutput, + OriginalTicketOutput + >( sourceObject, - ); - - const event = await this.prisma.events.create({ - data: { - id_event: uuidv4(), - status: 'success', - type: 'ticketing.ticket.synced', - method: 'SYNC', - url: '/sync', - provider: integrationId, - direction: '0', - timestamp: new Date(), - id_linked_user: linkedUserId, - }, - }); - - await this.webhook.handleWebhook( - tickets_data, - 'ticketing.ticket.pulled', - id_project, - event.id_event, + integrationId, + connection.id_connection, + 'ticketing', + 'ticket', + customFieldMappings, ); } catch (error) { throw error; } } - async saveTicketsInDb( + async saveToDb( connection_id: string, linkedUserId: string, tickets: UnifiedTicketOutput[], @@ -394,11 +364,7 @@ export class SyncService implements OnModuleInit { throw error; } } - async removeTicketInDb( - linkedUserId: string, - originSource: string, - remote_id: string, - ) { + async removeTicketInDb(connection_id: string, remote_id: string) { const existingTicket = await this.prisma.tcg_tickets.findFirst({ where: { remote_id: remote_id, diff --git a/packages/api/src/ticketing/ticket/ticket.module.ts b/packages/api/src/ticketing/ticket/ticket.module.ts index 548faad9f..8f872da8a 100644 --- a/packages/api/src/ticketing/ticket/ticket.module.ts +++ b/packages/api/src/ticketing/ticket/ticket.module.ts @@ -19,6 +19,7 @@ import { MappersRegistry } from '@@core/@core-services/registries/mappers.regist import { Utils } from '@ticketing/@lib/@utils'; import { ConnectionUtils } from '@@core/connections/@utils'; import { CoreUnification } from '@@core/@core-services/unification/core-unification.service'; +import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; @Module({ imports: [], @@ -36,6 +37,7 @@ import { CoreUnification } from '@@core/@core-services/unification/core-unificat UnificationRegistry, MappersRegistry, Utils, + IngestDataService, /* PROVIDERS SERVICES */ ZendeskService, HubspotService, diff --git a/packages/api/src/ticketing/user/sync/sync.service.ts b/packages/api/src/ticketing/user/sync/sync.service.ts index 8597e27f9..e8e860162 100644 --- a/packages/api/src/ticketing/user/sync/sync.service.ts +++ b/packages/api/src/ticketing/user/sync/sync.service.ts @@ -1,33 +1,31 @@ -import { Injectable, OnModuleInit } from '@nestjs/common'; import { LoggerService } from '@@core/@core-services/logger/logger.service'; import { PrismaService } from '@@core/@core-services/prisma/prisma.service'; -import { Cron } from '@nestjs/schedule'; +import { BullQueueService } from '@@core/@core-services/queues/shared.service'; +import { CoreSyncRegistry } from '@@core/@core-services/registries/core-sync.registry'; +import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; +import { FieldMappingService } from '@@core/field-mapping/field-mapping.service'; import { ApiResponse } from '@@core/utils/types'; +import { IBaseSync } from '@@core/utils/types/interface'; +import { OriginalUserOutput } from '@@core/utils/types/original/original.ticketing'; +import { Injectable, OnModuleInit } from '@nestjs/common'; +import { Cron } from '@nestjs/schedule'; +import { TICKETING_PROVIDERS } from '@panora/shared'; +import { tcg_users as TicketingUser } from '@prisma/client'; import { v4 as uuidv4 } from 'uuid'; -import { FieldMappingService } from '@@core/field-mapping/field-mapping.service'; import { ServiceRegistry } from '../services/registry.service'; -import { TicketingObject } from '@ticketing/@lib/@types'; -import { WebhookService } from '@@core/@core-services/webhooks/panora-webhooks/webhook.service'; import { IUserService } from '../types'; -import { OriginalUserOutput } from '@@core/utils/types/original/original.ticketing'; -import { tcg_users as TicketingUser } from '@prisma/client'; import { UnifiedUserOutput } from '../types/model.unified'; -import { TICKETING_PROVIDERS } from '@panora/shared'; -import { CoreSyncRegistry } from '@@core/@core-services/registries/core-sync.registry'; -import { BullQueueService } from '@@core/@core-services/queues/shared.service'; -import { CoreUnification } from '@@core/@core-services/unification/core-unification.service'; @Injectable() -export class SyncService implements OnModuleInit { +export class SyncService implements OnModuleInit, IBaseSync { constructor( private prisma: PrismaService, private logger: LoggerService, - private webhook: WebhookService, private fieldMappingService: FieldMappingService, private serviceRegistry: ServiceRegistry, - private coreUnification: CoreUnification, private registry: CoreSyncRegistry, private bullQueueService: BullQueueService, + private ingestService: IngestDataService, ) { this.logger.setContext(SyncService.name); this.registry.registerService('ticketing', 'user', this); @@ -82,7 +80,6 @@ export class SyncService implements OnModuleInit { await this.syncUsersForLinkedUser( provider, linkedUser.id_linked_user, - id_project, ); } catch (error) { throw error; @@ -104,7 +101,6 @@ export class SyncService implements OnModuleInit { async syncUsersForLinkedUser( integrationId: string, linkedUserId: string, - id_project: string, wh_real_time_trigger?: { action: 'UPDATE' | 'DELETE'; data: { @@ -169,63 +165,34 @@ export class SyncService implements OnModuleInit { } const sourceObject: OriginalUserOutput[] = resp.data; - // this.logger.log('SOURCE OBJECT DATA = ' + JSON.stringify(sourceObject)); - // console.log("Source Data ", sourceObject) - //unify the data according to the target obj wanted - const unifiedObject = (await this.coreUnification.unify< - OriginalUserOutput[] - >({ - sourceObject, - targetType: TicketingObject.user, - providerName: integrationId, - vertical: 'ticketing', - connectionId: connection.id_connection, - customFieldMappings, - })) as UnifiedUserOutput[]; - //insert the data in the DB with the fieldMappings (value table) - const user_data = await this.saveUsersInDb( - connection.id_connection, - linkedUserId, - unifiedObject, - integrationId, + await this.ingestService.ingestData< + UnifiedUserOutput, + OriginalUserOutput + >( sourceObject, - ); - const event = await this.prisma.events.create({ - data: { - id_event: uuidv4(), - status: 'success', - type: 'ticketing.user.synced', - method: 'SYNC', - url: '/sync', - provider: integrationId, - direction: '0', - timestamp: new Date(), - id_linked_user: linkedUserId, - }, - }); - await this.webhook.handleWebhook( - user_data, - 'ticketing.user.pulled', - id_project, - event.id_event, + integrationId, + connection.id_connection, + 'ticketing', + 'user', + customFieldMappings, ); } catch (error) { throw error; } } - async saveUsersInDb( + async saveToDb( connection_id: string, linkedUserId: string, - users: UnifiedUserOutput[], + data: UnifiedUserOutput[], originSource: string, remote_data: Record[], ): Promise { try { let users_results: TicketingUser[] = []; - for (let i = 0; i < users.length; i++) { - const user = users[i]; + for (let i = 0; i < data.length; i++) { + const user = data[i]; const originId = user.remote_id; if (!originId || originId == '') { diff --git a/packages/api/src/ticketing/user/user.module.ts b/packages/api/src/ticketing/user/user.module.ts index 9c123b474..c644450fd 100644 --- a/packages/api/src/ticketing/user/user.module.ts +++ b/packages/api/src/ticketing/user/user.module.ts @@ -17,6 +17,7 @@ import { MappersRegistry } from '@@core/@core-services/registries/mappers.regist import { UnificationRegistry } from '@@core/@core-services/registries/unification.registry'; import { Utils } from '@ticketing/@lib/@utils'; import { CoreUnification } from '@@core/@core-services/unification/core-unification.service'; +import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; @Module({ imports: [], @@ -34,6 +35,7 @@ import { CoreUnification } from '@@core/@core-services/unification/core-unificat UnificationRegistry, MappersRegistry, Utils, + IngestDataService, /* PROVIDERS SERVICES */ ZendeskService, FrontService,