Automated Import (#720)

apps/api/package.json

@@ -29,6 +29,7 @@
     "@nestjs/platform-express": "^9.1.2",
     "@nestjs/platform-socket.io": "9.4.3",
     "@nestjs/platform-ws": "9.4.3",
+    "@nestjs/schedule": "^4.1.0",
     "@nestjs/swagger": "^6.1.2",
     "@nestjs/terminus": "^9.1.3",
     "@nestjs/websockets": "9.4.3",
@@ -44,6 +45,7 @@
     "class-validator": "^0.14.0",
     "compression": "^1.7.4",
     "cookie-parser": "^1.4.6",
+    "cron": "^3.1.7",
     "date-fns": "^2.30.0",
     "dayjs": "^1.11.11",
     "dotenv": "^16.0.2",
@@ -63,7 +65,8 @@
     "source-map-support": "^0.5.21",
     "uuid": "^9.0.0",
     "xlsx": "https://cdn.sheetjs.com/xlsx-0.20.1/xlsx-0.20.1.tgz",
-    "xlsx-populate": "^1.21.0"
+    "xlsx-populate": "^1.21.0",
+    "xml2js": "^0.6.2"
   },
   "devDependencies": {
     "@nestjs/cli": "^9.1.5",
@@ -100,4 +103,4 @@
   "main": ".eslintrc.js",
   "keywords": [],
   "description": ""
-}
+}

apps/api/src/app.module.ts

@@ -14,6 +14,7 @@ import { AuthModule } from './app/auth/auth.module';
 import { EnvironmentModule } from './app/environment/environment.module';
 import { ActivityModule } from './app/activity/activity.module';
 import { UserModule } from './app/user/user.module';
+import { ImportJobsModule } from 'app/import-jobs/import-jobs.module';
 
 const modules: Array<Type | DynamicModule | Promise<DynamicModule> | ForwardReference> = [
   ProjectModule,
@@ -29,6 +30,7 @@ const modules: Array<Type | DynamicModule | Promise<DynamicModule> | ForwardRefe
   UserModule,
   EnvironmentModule,
   ActivityModule,
+  ImportJobsModule,
 ];
 
 const providers = [Logger];

apps/api/src/app/common/common.controller.ts

@@ -57,12 +57,15 @@ export class CommonController {
   @ApiOperation({
     summary: 'Get import config',
   })
-  async getImportConfigRoute(@Query('projectId') projectId: string): Promise<ImportConfigResponseDto> {
+  async getImportConfigRoute(
+    @Query('projectId') projectId: string,
+    @Query('templateId') templateId: string
+  ): Promise<ImportConfigResponseDto> {
     if (!projectId) {
       throw new BadRequestException();
     }
 
-    return this.getImportConfig.execute(projectId);
+    return this.getImportConfig.execute(projectId, templateId);
   }
 
   @Post('/sheet-names')

apps/api/src/app/common/dtos/import-config-response.dto.ts

@@ -1,5 +1,5 @@
 import { ApiPropertyOptional } from '@nestjs/swagger';
-import { IsBoolean, IsDefined } from 'class-validator';
+import { IsBoolean, IsDefined, IsString } from 'class-validator';
 
 export class ImportConfigResponseDto {
   @ApiPropertyOptional({
@@ -8,4 +8,11 @@ export class ImportConfigResponseDto {
   @IsDefined()
   @IsBoolean()
   showBranding: boolean;
+
+  @ApiPropertyOptional({
+    description: 'Whether the current Import is Manual is Atomatic',
+  })
+  @IsDefined()
+  @IsString()
+  mode?: string;
 }

apps/api/src/app/common/usecases/get-import-config/get-import-config.usecase.ts

@@ -1,20 +1,34 @@
-import { Injectable } from '@nestjs/common';
-import { UserRepository } from '@impler/dal';
+import { BadRequestException, Injectable } from '@nestjs/common';
+import { UserRepository, TemplateRepository, TemplateEntity } from '@impler/dal';
 import { IImportConfig } from '@impler/shared';
 import { PaymentAPIService } from '@impler/services';
+import { APIMessages } from '@shared/constants';
 
 @Injectable()
 export class GetImportConfig {
   constructor(
     private userRepository: UserRepository,
-    private paymentAPIService: PaymentAPIService
+    private paymentAPIService: PaymentAPIService,
+    private templateRepository: TemplateRepository
   ) {}
 
-  async execute(projectId: string): Promise<IImportConfig> {
+  async execute(projectId: string, templateId?: string): Promise<IImportConfig> {
     const userEmail = await this.userRepository.findUserEmailFromProjectId(projectId);
 
     const removeBrandingAvailable = await this.paymentAPIService.checkEvent(userEmail, 'REMOVE_BRANDING');
 
-    return { showBranding: !removeBrandingAvailable };
+    let template: TemplateEntity;
+    if (templateId) {
+      template = await this.templateRepository.findOne({
+        _projectId: projectId,
+        _id: templateId,
+      });
+
+      if (!template) {
+        throw new BadRequestException(APIMessages.TEMPLATE_NOT_FOUND);
+      }
+    }
+
+    return { showBranding: !removeBrandingAvailable, mode: template?.mode, title: template?.name };
   }
 }

apps/api/src/app/import-jobs/dtos/create-userjob.dto.ts

@@ -0,0 +1,15 @@
+import { IsNotEmpty, IsOptional, IsString } from 'class-validator';
+
+export class CreateUserJobDto {
+  @IsString()
+  @IsNotEmpty()
+  url: string;
+
+  @IsString()
+  @IsOptional()
+  externalUserId?: string;
+
+  @IsString()
+  @IsOptional()
+  extra?: string;
+}

apps/api/src/app/import-jobs/dtos/index.ts

@@ -0,0 +1,3 @@
+export * from './create-userjob.dto';
+export * from './update-userjob.dto';
+export * from './update-jobmapping.dto';

apps/api/src/app/import-jobs/dtos/update-jobmapping.dto.ts

@@ -0,0 +1,18 @@
+import { IsBoolean, IsString } from 'class-validator';
+
+export class UpdateJobMappingDto {
+  @IsString()
+  key: string;
+
+  @IsString()
+  name: string;
+
+  @IsBoolean()
+  isRequired: boolean;
+
+  @IsString()
+  path: string;
+
+  @IsString()
+  _jobId: string;
+}

apps/api/src/app/import-jobs/dtos/update-userjob.dto.ts

@@ -0,0 +1,27 @@
+import { IsArray, IsOptional, IsString } from 'class-validator';
+
+export class UpdateJobDto {
+  @IsString()
+  @IsOptional()
+  url: string;
+
+  @IsString()
+  @IsOptional()
+  _templateId: string;
+
+  @IsString()
+  @IsOptional()
+  cron: string;
+
+  @IsArray()
+  @IsOptional()
+  headings: string[];
+
+  @IsString()
+  @IsOptional()
+  status: string;
+
+  @IsString()
+  @IsOptional()
+  externalUserId: string;
+}

apps/api/src/app/import-jobs/import-jobs.controller.ts

@@ -0,0 +1,101 @@
+import { ApiTags, ApiSecurity, ApiOperation } from '@nestjs/swagger';
+import { Body, Controller, Delete, Get, Param, ParseArrayPipe, Post, Put, UseGuards } from '@nestjs/common';
+import {
+  CreateUserJob,
+  GetColumnSchemaMapping,
+  CreateJobMapping,
+  UpdateUserJob,
+  GetUserJob,
+  UserJobPause,
+  UserJobResume,
+  UserJobTerminate,
+  UserJobDelete,
+} from './usecase';
+import { ACCESS_KEY_NAME } from '@impler/shared';
+import { JwtAuthGuard } from '@shared/framework/auth.gaurd';
+import { UpdateJobDto, CreateUserJobDto, UpdateJobMappingDto } from './dtos';
+
+@ApiTags('Import-Jobs')
+@Controller('/import-jobs')
+@UseGuards(JwtAuthGuard)
+@ApiSecurity(ACCESS_KEY_NAME)
+export class ImportJobsController {
+  constructor(
+    private createUserJob: CreateUserJob,
+    private updateJobMapping: CreateJobMapping,
+    private getColumnSchemaMapping: GetColumnSchemaMapping,
+    private getUserJob: GetUserJob,
+    private updateUserJob: UpdateUserJob,
+    private userJobPause: UserJobPause,
+    private userJobResume: UserJobResume,
+    private userJobDelete: UserJobDelete,
+    private userJobTerminate: UserJobTerminate
+  ) {}
+
+  @Post(':templateId')
+  @ApiOperation({ summary: 'Create User-Job' })
+  @ApiSecurity(ACCESS_KEY_NAME)
+  async createUserJobRoute(@Param('templateId') templateId: string, @Body() createUserJobData: CreateUserJobDto) {
+    return this.createUserJob.execute({
+      _templateId: templateId,
+      url: createUserJobData.url,
+    });
+  }
+
+  @Get(':jobId/mappings')
+  @ApiOperation({ summary: 'Fetch the User-Job Mapping Information based on jobId' })
+  @UseGuards(JwtAuthGuard)
+  @ApiSecurity(ACCESS_KEY_NAME)
+  async getImportJobInfoRoute(@Param('jobId') _jobId: string) {
+    return this.getColumnSchemaMapping.execute(_jobId);
+  }
+
+  @Put(':jobId/mappings')
+  @ApiOperation({ summary: 'Update User-Job Mappings' })
+  @UseGuards(JwtAuthGuard)
+  async updateJobMappingRoute(@Body(new ParseArrayPipe({ items: UpdateJobMappingDto })) body: UpdateJobMappingDto[]) {
+    return this.updateJobMapping.execute(body);
+  }
+
+  @Put(':jobId')
+  @ApiOperation({ summary: 'Update User-Job Fields' })
+  @UseGuards(JwtAuthGuard)
+  async updateUserJobRoute(@Param('jobId') _jobId: string, @Body() userJobData: UpdateJobDto) {
+    return this.updateUserJob.execute(_jobId, userJobData);
+  }
+
+  @Get('/user/:externalUserId')
+  @ApiOperation({ summary: 'Get User Jobs' })
+  @UseGuards(JwtAuthGuard)
+  async getUserJobs(@Param('externalUserId') externalUserId: string) {
+    return this.getUserJob.execute(externalUserId);
+  }
+
+  @Put('/user/:jobId/pause')
+  @ApiOperation({ summary: 'Pause User-Job from Running' })
+  @UseGuards(JwtAuthGuard)
+  async pauseCronJob(@Param('jobId') jobId: string) {
+    return await this.userJobPause.execute(jobId);
+  }
+
+  @Put('/user/:jobId/resume')
+  @ApiOperation({ summary: 'Resume stopped User-Job' })
+  @UseGuards(JwtAuthGuard)
+  async resumeUserJobRoute(@Param('jobId') _jobId: string) {
+    return await this.userJobResume.execute(_jobId);
+  }
+
+  @Delete('/user/:externalUserId/:jobId')
+  @ApiOperation({ summary: 'Delete User-Job' })
+  @UseGuards(JwtAuthGuard)
+  async deleteJob(@Param('externalUserId') externalUserId: string, @Param('jobId') _jobId: string) {
+    return await this.userJobDelete.execute({ externalUserId, _jobId });
+  }
+
+  @Delete(':jobId')
+  @ApiOperation({ summary: 'Delete User-Job and Update the status of UserJob to TERMINATED' })
+  @UseGuards(JwtAuthGuard)
+  async deleteJobRoute(@Param('jobId') _jobId: string) {
+    return await this.userJobTerminate.execute(_jobId);
+  }
+}

apps/api/src/app/import-jobs/import-jobs.module.ts

@@ -0,0 +1,11 @@
+import { Module } from '@nestjs/common';
+import { SharedModule } from '@shared/shared.module';
+import { USECASES } from './usecase';
+import { ImportJobsController } from './import-jobs.controller';
+
+@Module({
+  imports: [SharedModule],
+  providers: [...USECASES],
+  controllers: [ImportJobsController],
+})
+export class ImportJobsModule {}

apps/api/src/app/import-jobs/usecase/create-jobmapping/create-jobmapping.command.ts

@@ -0,0 +1,11 @@
+export class CreateJobMappingCommand {
+  key: string;
+
+  name: string;
+
+  isRequired: boolean;
+
+  path: string;
+
+  _jobId: string;
+}

apps/api/src/app/import-jobs/usecase/create-jobmapping/create-jobmapping.usecase.ts

@@ -0,0 +1,20 @@
+import { JobMappingRepository } from '@impler/dal';
+import { BadRequestException, Injectable } from '@nestjs/common';
+import { CreateJobMappingCommand } from './create-jobmapping.command';
+
+@Injectable()
+export class CreateJobMapping {
+  constructor(private readonly jobMappingRepository: JobMappingRepository) {}
+
+  async execute(jobMappingCommand: CreateJobMappingCommand[]) {
+    jobMappingCommand.filter((command) => !!command.key).map((command) => command.key);
+
+    for (const mappingCommand of jobMappingCommand) {
+      if (mappingCommand.isRequired && !mappingCommand.path) {
+        throw new BadRequestException(`${mappingCommand.name} is required`);
+      }
+    }
+
+    return this.jobMappingRepository.createMany(jobMappingCommand);
+  }
+}

apps/api/src/app/import-jobs/usecase/create-userjob/create-userjob.command.ts

@@ -0,0 +1,6 @@
+export class CreateUserJobCommand {
+  url: string;
+  _templateId: string;
+  externalUserId?: string;
+  extra?: string;
+}

apps/api/src/app/import-jobs/usecase/create-userjob/create-userjob.usecase.ts

@@ -0,0 +1,35 @@
+import { BadRequestException, Injectable } from '@nestjs/common';
+import { FileMimeTypesEnum } from '@impler/shared';
+import { APIMessages } from '@shared/constants';
+import { UserJobEntity, UserJobRepository } from '@impler/dal';
+import { RSSService } from '@shared/services';
+import { CreateUserJobCommand } from './create-userjob.command';
+
+@Injectable()
+export class CreateUserJob {
+  constructor(
+    private readonly rssService: RSSService,
+    private readonly userJobRepository: UserJobRepository
+  ) {}
+
+  async execute({ _templateId, url, externalUserId, extra }: CreateUserJobCommand): Promise<UserJobEntity> {
+    const mimeType = await this.rssService.getMimeType(url);
+    if (mimeType === FileMimeTypesEnum.XML || mimeType === FileMimeTypesEnum.TEXTXML) {
+      const { rssKeyHeading } = await this.rssService.parseRssFeed(url);
+      let formattedExtra = extra || '{}';
+      try {
+        formattedExtra = JSON.parse(extra);
+      } catch (_) {}
+
+      return await this.userJobRepository.create({
+        url,
+        headings: rssKeyHeading,
+        _templateId: _templateId,
+        extra,
+        externalUserId: externalUserId || (formattedExtra as unknown as Record<string, any>)?.externalUserId,
+      });
+    } else {
+      throw new BadRequestException(APIMessages.INVALID_RSS_URL);
+    }
+  }
+}