From ca496bed640237514ba7051941d186664232c392 Mon Sep 17 00:00:00 2001 From: Mayur Date: Mon, 8 Jul 2024 14:39:16 +0530 Subject: [PATCH 001/299] feat: Dowpdown created for manual and automatic import modes --- apps/web/pages/imports/[id].tsx | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/apps/web/pages/imports/[id].tsx b/apps/web/pages/imports/[id].tsx index c02e80244..2e6a65079 100644 --- a/apps/web/pages/imports/[id].tsx +++ b/apps/web/pages/imports/[id].tsx @@ -2,7 +2,7 @@ import Link from 'next/link'; import { useState } from 'react'; import dynamic from 'next/dynamic'; import { useRouter } from 'next/router'; -import { ActionIcon, Flex, Group, LoadingOverlay, Title, useMantineTheme } from '@mantine/core'; +import { ActionIcon, Flex, Group, LoadingOverlay, Title, useMantineTheme, Select } from '@mantine/core'; import { track } from '@libs/amplitude'; import { ROUTES, colors } from '@config'; @@ -34,6 +34,7 @@ const Validator = dynamic(() => import('@components/imports/validator').then((mo export default function ImportDetails({}) { const router = useRouter(); + // const [importMode, setImportMode] = useState<'manual' | 'automatic'>('manual'); const [activeTab, setActiveTab] = useState<'schema' | 'destination' | 'snippet' | 'validator' | 'output'>(); const { colorScheme } = useMantineTheme(); const { @@ -44,6 +45,7 @@ export default function ImportDetails({}) { onDeleteClick, isTemplateDataLoading, onSpreadsheetImported, + updateImport, } = useImportDetails({ templateId: router.query.id as string, }); @@ -78,6 +80,21 @@ export default function ImportDetails({}) { + { - updateImport({ - mode: importMode, - }); - }} + defaultValue={ImportModeEnum.MANUAL} + value={templateData?.mode} + onChange={(mode) => updateImport({ mode: mode || undefined })} /> + ), + [PhasesEnum.MAPCOLUMNS]: ( + <> + + + ), + [PhasesEnum.SCHEDULE]: ( + <> + + + ), + [PhasesEnum.CONFORM]: ( + <> + + + ), + }; + + return ( + + {importConfig && importConfig.showBranding === true ? ( + + + Powered by + + + ) : ( +
+ )} + {FooterActions[active]} + + ); +} From b662ac54b1a657610438aad1dda4f9078b3e6dc8 Mon Sep 17 00:00:00 2001 From: Mayur Date: Wed, 17 Jul 2024 12:45:02 +0530 Subject: [PATCH 157/299] feat: Added BUTTONTEXT in AUTOIMPORTPHASES --- apps/widget/src/config/texts.config.ts | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/apps/widget/src/config/texts.config.ts b/apps/widget/src/config/texts.config.ts index 11a3c9fb4..ce35b75ff 100644 --- a/apps/widget/src/config/texts.config.ts +++ b/apps/widget/src/config/texts.config.ts @@ -49,6 +49,15 @@ export const TEXTS = { PHASE4: { CLOSE: 'Close', }, + + AUTOIMPORTPHASES: { + BUTTONTEXT: { + MAPCOLUMN: 'Map Column', + SCHEDULE: 'Schedule', + CONFORM: 'Conform', + CLOSE: 'Close', + }, + }, AUTOIMPORTPHASE2: { NAME_IN_SCHEMA_TITLE: 'Column in schema', KEY_IN_FEED_TITLE: 'Key in your RSS feed ', From 059143fd615942dde961c38d16a6e213d8163742 Mon Sep 17 00:00:00 2001 From: Mayur Date: Wed, 17 Jul 2024 12:54:49 +0530 Subject: [PATCH 158/299] feat: Files deleted --- .../app/import-jobs/dto/create-userjob.dto.ts | 22 ------------------- .../get-mapping/get-mapping.usecase.ts | 17 -------------- .../AutoImportPhase0/index.ts | 1 - .../UserJobInformation.interface.ts | 13 ----------- .../src/entities/UserJobInformation/index.ts | 1 - 5 files changed, 54 deletions(-) delete mode 100644 apps/api/src/app/import-jobs/dto/create-userjob.dto.ts delete mode 100644 apps/api/src/app/import-jobs/usecase/get-mapping/get-mapping.usecase.ts delete mode 100644 apps/widget/src/components/widget/Phases/AutoImportPhases/AutoImportPhase0/index.ts delete mode 100644 libs/shared/src/entities/UserJobInformation/UserJobInformation.interface.ts delete mode 100644 libs/shared/src/entities/UserJobInformation/index.ts diff --git a/apps/api/src/app/import-jobs/dto/create-userjob.dto.ts b/apps/api/src/app/import-jobs/dto/create-userjob.dto.ts deleted file mode 100644 index d2311ae55..000000000 --- a/apps/api/src/app/import-jobs/dto/create-userjob.dto.ts +++ /dev/null @@ -1,22 +0,0 @@ -import { ApiProperty } from '@nestjs/swagger'; -import { IsArray, IsDefined, IsString, IsUrl } from 'class-validator'; - -export class CreateUserJobDto { - @ApiProperty({ - description: 'RSS URL from where the data should be imported', - }) - @IsUrl() - @IsDefined() - url: string; - - @ApiProperty({ - description: 'Headings array is to be gotten when the RSS URL is valid and properly parsed', - }) - @IsArray() - @IsDefined() - headings: string[]; - - @IsDefined() - @IsString() - _templateId: string; -} diff --git a/apps/api/src/app/import-jobs/usecase/get-mapping/get-mapping.usecase.ts b/apps/api/src/app/import-jobs/usecase/get-mapping/get-mapping.usecase.ts deleted file mode 100644 index 44e1a2153..000000000 --- a/apps/api/src/app/import-jobs/usecase/get-mapping/get-mapping.usecase.ts +++ /dev/null @@ -1,17 +0,0 @@ -import { Injectable } from '@nestjs/common'; -import { ColumnRepository } from '@impler/dal'; - -@Injectable() -export class GetImportMappingInfo { - constructor(private readonly columnRepository: ColumnRepository) {} - - async execute(templateId: string): Promise { - const column = await this.columnRepository.find({ _templateId: templateId }); - - if (!column) { - throw new Error('Column not found'); - } - - return column; - } -} diff --git a/apps/widget/src/components/widget/Phases/AutoImportPhases/AutoImportPhase0/index.ts b/apps/widget/src/components/widget/Phases/AutoImportPhases/AutoImportPhase0/index.ts deleted file mode 100644 index 05cfc33d5..000000000 --- a/apps/widget/src/components/widget/Phases/AutoImportPhases/AutoImportPhase0/index.ts +++ /dev/null @@ -1 +0,0 @@ -export * from './AutoImportPhase0'; diff --git a/libs/shared/src/entities/UserJobInformation/UserJobInformation.interface.ts b/libs/shared/src/entities/UserJobInformation/UserJobInformation.interface.ts deleted file mode 100644 index 3e0823847..000000000 --- a/libs/shared/src/entities/UserJobInformation/UserJobInformation.interface.ts +++ /dev/null @@ -1,13 +0,0 @@ -export interface IUserJobInformation { - _id?: string; - - url: string; - - _templateId: string; - - cron: string; - - headings: string[]; - - createdOn: Date; -} diff --git a/libs/shared/src/entities/UserJobInformation/index.ts b/libs/shared/src/entities/UserJobInformation/index.ts deleted file mode 100644 index 31c88494e..000000000 --- a/libs/shared/src/entities/UserJobInformation/index.ts +++ /dev/null @@ -1 +0,0 @@ -export * from './UserJobInformation.interface'; From 1e6459d11aded76589e84820b81c9b8c57a9ee14 Mon Sep 17 00:00:00 2001 From: Mayur Date: Wed, 17 Jul 2024 14:28:04 +0530 Subject: [PATCH 159/299] feat: Handled invalid MIME type which are other than application/rss+xml --- .../create-userjob/create-userjob.usecase.ts | 24 ++++++++++++------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/apps/api/src/app/import-jobs/usecase/create-userjob/create-userjob.usecase.ts b/apps/api/src/app/import-jobs/usecase/create-userjob/create-userjob.usecase.ts index 1383add68..37a280d9e 100644 --- a/apps/api/src/app/import-jobs/usecase/create-userjob/create-userjob.usecase.ts +++ b/apps/api/src/app/import-jobs/usecase/create-userjob/create-userjob.usecase.ts @@ -1,5 +1,7 @@ -import { Injectable } from '@nestjs/common'; -import { parseRssFeed } from '@shared/services/parse-xml'; +import { BadRequestException, Injectable } from '@nestjs/common'; +import { FileMimeTypesEnum } from '@impler/shared'; +import { APIMessages } from '@shared/constants'; +import { parseRssFeed, getMimeType } from '@shared/services/parse-xml'; import { UserJobEntity, UserJobRepository } from '@impler/dal'; @Injectable() @@ -7,12 +9,18 @@ export class CreateUserJob { constructor(private readonly userJobRepository: UserJobRepository) {} async execute({ _templateId, url }: { url: string; _templateId: string }): Promise { - const { rssKeyHeading } = await parseRssFeed(url); + const mimeType = await getMimeType(url); - return await this.userJobRepository.create({ - url, - headings: rssKeyHeading, - _templateId: _templateId, - }); + if (mimeType !== FileMimeTypesEnum.XML) { + throw new BadRequestException(APIMessages.INVALID_RSS_URL); + } else { + const { rssKeyHeading } = await parseRssFeed(url); + + return await this.userJobRepository.create({ + url, + headings: rssKeyHeading, + _templateId: _templateId, + }); + } } } From c3243b9cc8d2a39e237084ce69eccae4f85f07d9 Mon Sep 17 00:00:00 2001 From: Mayur Date: Wed, 17 Jul 2024 14:42:53 +0530 Subject: [PATCH 160/299] feat: Given proper information on focusing corresponding cron expression text box --- .../AutoImportPhase3/AutoImportPhase3.tsx | 65 ++++++++++++++++--- 1 file changed, 56 insertions(+), 9 deletions(-) diff --git a/apps/widget/src/components/widget/Phases/AutoImportPhases/AutoImportPhase3/AutoImportPhase3.tsx b/apps/widget/src/components/widget/Phases/AutoImportPhases/AutoImportPhase3/AutoImportPhase3.tsx index c5bc2b2e8..67ffe6cb0 100644 --- a/apps/widget/src/components/widget/Phases/AutoImportPhases/AutoImportPhase3/AutoImportPhase3.tsx +++ b/apps/widget/src/components/widget/Phases/AutoImportPhases/AutoImportPhase3/AutoImportPhase3.tsx @@ -1,6 +1,5 @@ -import React, { useEffect, useState } from 'react'; +import { useEffect, useState } from 'react'; import { useForm } from 'react-hook-form'; -import { useDisclosure } from '@mantine/hooks'; import { Group, Text, Stack, Flex, Container } from '@mantine/core'; const parseCronExpression = require('util/helpers/cronstrue'); import { colors, cronExampleBadges, cronExamples, ScheduleFormValues, defaultCronValues, TEXTS } from '@config'; @@ -16,12 +15,13 @@ interface IAutoImportPhase3Props { } export function AutoImportPhase3({ onNextClick }: IAutoImportPhase3Props) { - const [opened, { toggle }] = useDisclosure(false); + const [tableOpened, setTableOpened] = useState(false); const { control, watch, setValue } = useForm({ defaultValues: defaultCronValues, }); const scheduleData = watch(); const [cronDescription, setCronDescription] = useState({ description: '', isError: false }); + const [focusedField, setFocusedField] = useState(null); const { updateUserJob } = useAutoImportPhase3({ goNext: onNextClick }); @@ -66,16 +66,58 @@ export function AutoImportPhase3({ onNextClick }: IAutoImportPhase3Props) { }); }; + const handleFocus = (fieldName: keyof ScheduleFormValues) => { + setFocusedField(fieldName); + setTableOpened(true); + }; + + const handleBlur = () => { + setFocusedField(null); + setTableOpened(false); + }; + + const toggleTable = () => { + if (focusedField) { + setFocusedField(null); + } + setTableOpened(!tableOpened); + }; + return ( <> - - - - - + handleFocus('Minute')} + onBlur={handleBlur} + /> + handleFocus('Hour')} + onBlur={handleBlur} + /> + handleFocus('Day')} + onBlur={handleBlur} + /> + handleFocus('Month')} + onBlur={handleBlur} + /> + handleFocus('Days')} + onBlur={handleBlur} + /> {cronDescription.description} @@ -90,7 +132,12 @@ export function AutoImportPhase3({ onNextClick }: IAutoImportPhase3Props) { /> ))} - + {}} onNextClick={handleNextClick} /> From e2a93f05c7bd63c2f993f0ef7da42456d689053a Mon Sep 17 00:00:00 2001 From: Mayur Date: Wed, 17 Jul 2024 14:43:53 +0530 Subject: [PATCH 161/299] feat: Added an error message for invalid xml rss link --- apps/api/src/app/shared/constants.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/api/src/app/shared/constants.ts b/apps/api/src/app/shared/constants.ts index 30f170019..d4e333dc0 100644 --- a/apps/api/src/app/shared/constants.ts +++ b/apps/api/src/app/shared/constants.ts @@ -27,6 +27,7 @@ export const APIMessages = { EMAIL_ALREADY_EXISTS: 'Email already exists', INCORRECT_KEYS_FOUND: 'Invalid keys found! Please check and correct them from web', INVALID_AUTH_TOKEN: 'Invalid authentication token', + INVALID_RSS_URL: 'The Specified URL doesn`t contain any RSS XML Feed, Please enter a Valid RSS XML URL', }; export const CONSTANTS = { From 413aee3ed94ac57d21add32e73c746cc1d780e2c Mon Sep 17 00:00:00 2001 From: Mayur Date: Wed, 17 Jul 2024 14:46:03 +0530 Subject: [PATCH 162/299] feat: Added a function to check the mimeType of the provided url --- .../shared/services/parse-xml/parse-xml.service.ts | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/apps/api/src/app/shared/services/parse-xml/parse-xml.service.ts b/apps/api/src/app/shared/services/parse-xml/parse-xml.service.ts index fe5f01f76..933a1b062 100644 --- a/apps/api/src/app/shared/services/parse-xml/parse-xml.service.ts +++ b/apps/api/src/app/shared/services/parse-xml/parse-xml.service.ts @@ -3,6 +3,17 @@ import axios from 'axios'; import { keysToOmit } from './keysToOmit'; import { IMappedResult, IMapping, IXmlObject } from '@shared/types/parse-xml.types'; +export async function getMimeType(url: string): Promise { + try { + const response = await axios.head(url); + const mimeType = response.headers['content-type'] || null; + + return mimeType?.split(';')[0] || null; + } catch (error) { + return null; + } +} + const parseXmlFromUrl = async (url: string): Promise => { try { const parser = new xml2js.Parser({ strict: false }); From 347e9fc92f6ccbce180beaf6212528e708d1c925 Mon Sep 17 00:00:00 2001 From: Mayur Date: Wed, 17 Jul 2024 14:49:51 +0530 Subject: [PATCH 163/299] feat: Added information on clicking on the particular cron expression text box --- .../CollapsibleExplanationTable.tsx | 25 +++++++++++++++---- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/apps/widget/src/components/widget/Phases/AutoImportPhases/AutoImportPhase3/CollapsibleExplanationTable.tsx b/apps/widget/src/components/widget/Phases/AutoImportPhases/AutoImportPhase3/CollapsibleExplanationTable.tsx index de209cd74..866ca0d20 100644 --- a/apps/widget/src/components/widget/Phases/AutoImportPhases/AutoImportPhase3/CollapsibleExplanationTable.tsx +++ b/apps/widget/src/components/widget/Phases/AutoImportPhases/AutoImportPhase3/CollapsibleExplanationTable.tsx @@ -1,4 +1,5 @@ import { Flex, Text, Collapse, Table, UnstyledButton, Stack } from '@mantine/core'; +import { fieldAllowedValues } from '@config'; import { ChevronDown } from '@icons'; import { colors } from '@config'; @@ -6,15 +7,29 @@ interface CollapsibleExplanationTableProps { opened: boolean; toggle: () => void; cronExamples: { expression: string; schedule: string }[]; + focusedField: string | null; } -export function CollapsibleExplanationTable({ cronExamples, opened, toggle }: CollapsibleExplanationTableProps) { +export function CollapsibleExplanationTable({ + cronExamples, + opened, + toggle, + focusedField, +}: CollapsibleExplanationTableProps) { + const getAllowedValues = () => { + if (!focusedField) return []; + + return [{ expression: fieldAllowedValues[focusedField], schedule: `Allowed values for ${focusedField}` }]; + }; + + const displayExamples = focusedField ? getAllowedValues() : cronExamples; + return ( - Valid Values + {focusedField ? `Allowed Values for ${focusedField}` : 'Valid Values'} - Cron Expression - Schedule + Allowed Values + Description - {cronExamples.map((example, index) => ( + {displayExamples.map((example, index) => ( {example.expression} {example.schedule} From b9df753ee1c18ff6a31f62231244185387559d7d Mon Sep 17 00:00:00 2001 From: Mayur Date: Wed, 17 Jul 2024 14:50:36 +0530 Subject: [PATCH 164/299] feat: Removed custom styling and added onFocus and onBlur --- .../CronScheduleInputTextBox.tsx | 40 ++++++------------- 1 file changed, 12 insertions(+), 28 deletions(-) diff --git a/apps/widget/src/components/widget/Phases/AutoImportPhases/AutoImportPhase3/CronScheduleInputTextBox.tsx b/apps/widget/src/components/widget/Phases/AutoImportPhases/AutoImportPhase3/CronScheduleInputTextBox.tsx index 5843af98d..881e507d7 100644 --- a/apps/widget/src/components/widget/Phases/AutoImportPhases/AutoImportPhase3/CronScheduleInputTextBox.tsx +++ b/apps/widget/src/components/widget/Phases/AutoImportPhases/AutoImportPhase3/CronScheduleInputTextBox.tsx @@ -1,46 +1,30 @@ import { TextInput } from '@mantine/core'; -import { isValidCronCharacter } from '@util'; import { Control, Controller } from 'react-hook-form'; import { ScheduleFormValues } from '@config'; interface CronScheduleInputProps { name: keyof ScheduleFormValues; - control: Control; + control: Control; + onFocus?: () => void; + onBlur?: () => void; } -export function CronScheduleInputTextBox({ name, control }: CronScheduleInputProps) { +export function CronScheduleInputTextBox({ name, control, onFocus, onBlur }: CronScheduleInputProps) { return ( ( { - const value = e.target.value.trim(); - field.onChange(value || '*'); - field.onBlur(); - }} - onChange={(e) => { - const value = e.target.value; - field.onChange(value); + {...field} + label={name} + onFocus={(e) => { + field.onChange(e); + if (onFocus) onFocus(); }} - styles={{ - label: { - textAlign: 'center', - display: 'block', - }, - input: { - textAlign: 'center', - }, - wrapper: { - textAlign: 'center', - }, + onBlur={(e) => { + field.onChange(e); + if (onBlur) onBlur(); }} /> )} From 101e6ffbf2feec5382733222e00654c547db8ae8 Mon Sep 17 00:00:00 2001 From: Mayur Date: Wed, 17 Jul 2024 14:51:24 +0530 Subject: [PATCH 165/299] feat: Added notification upon error --- .../hooks/AutoImportPhase1/useAutoImportPhase1.tsx | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/apps/widget/src/components/widget/Phases/AutoImportPhases/hooks/AutoImportPhase1/useAutoImportPhase1.tsx b/apps/widget/src/components/widget/Phases/AutoImportPhases/hooks/AutoImportPhase1/useAutoImportPhase1.tsx index 7346866d0..1ffce3a01 100644 --- a/apps/widget/src/components/widget/Phases/AutoImportPhases/hooks/AutoImportPhase1/useAutoImportPhase1.tsx +++ b/apps/widget/src/components/widget/Phases/AutoImportPhases/hooks/AutoImportPhase1/useAutoImportPhase1.tsx @@ -4,6 +4,7 @@ import { IErrorObject } from '@impler/shared'; import { IUserJob } from '@impler/shared'; import { useImplerState } from '@store/impler.context'; import { useJobsInfo } from '@store/jobinfo.context'; +import { notifier } from '@util'; interface IUseAutoImportPhase1Props { goNext: () => void; @@ -23,6 +24,9 @@ export function useAutoImportPhase1({ goNext }: IUseAutoImportPhase1Props) { setJobsInfo(data); goNext(); }, + onError(error) { + notifier.showError({ message: error.message, title: error.error }); + }, } ); From 2e28bd8625d989e28ce595542316a6d0d3026b9a Mon Sep 17 00:00:00 2001 From: Mayur Date: Wed, 17 Jul 2024 14:51:58 +0530 Subject: [PATCH 166/299] feat: Added fieldAllowedValues --- apps/widget/src/config/variable.config.ts | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/apps/widget/src/config/variable.config.ts b/apps/widget/src/config/variable.config.ts index 39211cd11..b5c2fcb3d 100644 --- a/apps/widget/src/config/variable.config.ts +++ b/apps/widget/src/config/variable.config.ts @@ -66,3 +66,11 @@ export const defaultCronValues: ScheduleFormValues = { Month: '*', Days: '0', }; + +export const fieldAllowedValues = { + Minute: '0-59, *, /, -', + Hour: '0-23, *, /, -', + Day: '1-31, *, /, -, ?', + Month: '1-12, JAN-DEC, *, /, -', + Days: '0-7, SUN-SAT, *, /, -, ?', +}; From 057ce1a6a2951c5a5fc2afd14adea56953c16043 Mon Sep 17 00:00:00 2001 From: Mayur Date: Wed, 17 Jul 2024 14:52:56 +0530 Subject: [PATCH 167/299] feat: Added application/rss+xml in FileMimeTypesEnum --- libs/shared/src/types/upload/upload.types.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/libs/shared/src/types/upload/upload.types.ts b/libs/shared/src/types/upload/upload.types.ts index dd2588269..b5d6add54 100644 --- a/libs/shared/src/types/upload/upload.types.ts +++ b/libs/shared/src/types/upload/upload.types.ts @@ -25,6 +25,7 @@ export enum FileMimeTypesEnum { 'EXCELX' = 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', 'EXCELM' = 'application/vnd.ms-excel.sheet.macroenabled.12', 'JSON' = 'application/json', + 'XML' = 'application/rss+xml', } export enum FileEncodingsEnum { From dbae76c89afb844d0531c11d6e9d44e40d872b89 Mon Sep 17 00:00:00 2001 From: Mayur Date: Wed, 17 Jul 2024 15:00:56 +0530 Subject: [PATCH 168/299] feat: Added styling for centering the label and the text in the textbox --- .../AutoImportPhase3/CronScheduleInputTextBox.tsx | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/apps/widget/src/components/widget/Phases/AutoImportPhases/AutoImportPhase3/CronScheduleInputTextBox.tsx b/apps/widget/src/components/widget/Phases/AutoImportPhases/AutoImportPhase3/CronScheduleInputTextBox.tsx index 881e507d7..f306162f3 100644 --- a/apps/widget/src/components/widget/Phases/AutoImportPhases/AutoImportPhase3/CronScheduleInputTextBox.tsx +++ b/apps/widget/src/components/widget/Phases/AutoImportPhases/AutoImportPhase3/CronScheduleInputTextBox.tsx @@ -16,6 +16,18 @@ export function CronScheduleInputTextBox({ name, control, onFocus, onBlur }: Cro control={control} render={({ field }) => ( { From 79f2e471888662b48022b00374a67d065838d389 Mon Sep 17 00:00:00 2001 From: Mayur Date: Wed, 17 Jul 2024 17:34:46 +0530 Subject: [PATCH 169/299] feat: Added dynamic cron expression explanation upon focusing on particular textbox --- .../CollapsibleExplanationTable.tsx | 39 +++++++++++-------- 1 file changed, 23 insertions(+), 16 deletions(-) diff --git a/apps/widget/src/components/widget/Phases/AutoImportPhases/AutoImportPhase3/CollapsibleExplanationTable.tsx b/apps/widget/src/components/widget/Phases/AutoImportPhases/AutoImportPhase3/CollapsibleExplanationTable.tsx index 866ca0d20..5d45acb00 100644 --- a/apps/widget/src/components/widget/Phases/AutoImportPhases/AutoImportPhase3/CollapsibleExplanationTable.tsx +++ b/apps/widget/src/components/widget/Phases/AutoImportPhases/AutoImportPhase3/CollapsibleExplanationTable.tsx @@ -1,7 +1,6 @@ -import { Flex, Text, Collapse, Table, UnstyledButton, Stack } from '@mantine/core'; -import { fieldAllowedValues } from '@config'; +import { Flex, Text, Collapse, Table, UnstyledButton, Stack, Group } from '@mantine/core'; import { ChevronDown } from '@icons'; -import { colors } from '@config'; +import { colors, cronFieldDefinitions } from '@config'; interface CollapsibleExplanationTableProps { opened: boolean; @@ -16,18 +15,19 @@ export function CollapsibleExplanationTable({ toggle, focusedField, }: CollapsibleExplanationTableProps) { - const getAllowedValues = () => { - if (!focusedField) return []; + const getFieldData = () => { + if (!focusedField) return null; - return [{ expression: fieldAllowedValues[focusedField], schedule: `Allowed values for ${focusedField}` }]; + return cronFieldDefinitions.find((fieldObj) => fieldObj[focusedField]); }; - const displayExamples = focusedField ? getAllowedValues() : cronExamples; + const fieldData = getFieldData(); return ( - + + - + {focusedField ? `Allowed Values for ${focusedField}` : 'Valid Values'} @@ -44,17 +44,24 @@ export function CollapsibleExplanationTable({ - + - {displayExamples.map((example, index) => ( - - - - - ))} + {!focusedField || !fieldData + ? cronExamples.map((example, index) => ( + + + + + )) + : fieldData[focusedField].values.map((value, index) => ( + + + + + ))}
Allowed Values{focusedField ? 'Allowed Values' : 'Expression'} Description
{example.expression}{example.schedule}
{example.expression}{example.schedule}
{value}{fieldData[focusedField].description[index]}
From 7ce2edff8e9326305340d96f9e696a43bddb74eb Mon Sep 17 00:00:00 2001 From: Mayur Date: Wed, 17 Jul 2024 17:35:18 +0530 Subject: [PATCH 170/299] feat: Removed setTableOpened --- .../AutoImportPhases/AutoImportPhase3/AutoImportPhase3.tsx | 1 - 1 file changed, 1 deletion(-) diff --git a/apps/widget/src/components/widget/Phases/AutoImportPhases/AutoImportPhase3/AutoImportPhase3.tsx b/apps/widget/src/components/widget/Phases/AutoImportPhases/AutoImportPhase3/AutoImportPhase3.tsx index 67ffe6cb0..8f3f76f46 100644 --- a/apps/widget/src/components/widget/Phases/AutoImportPhases/AutoImportPhase3/AutoImportPhase3.tsx +++ b/apps/widget/src/components/widget/Phases/AutoImportPhases/AutoImportPhase3/AutoImportPhase3.tsx @@ -73,7 +73,6 @@ export function AutoImportPhase3({ onNextClick }: IAutoImportPhase3Props) { const handleBlur = () => { setFocusedField(null); - setTableOpened(false); }; const toggleTable = () => { From 9e5947aecf6c81fc2536eb92f85b13ce903a3b07 Mon Sep 17 00:00:00 2001 From: Mayur Date: Wed, 17 Jul 2024 17:36:14 +0530 Subject: [PATCH 171/299] feat: Added consiced cronFieldDefinitions and the corresponding values and description into it --- apps/widget/src/config/variable.config.ts | 52 ++++++++++++++++++++--- 1 file changed, 45 insertions(+), 7 deletions(-) diff --git a/apps/widget/src/config/variable.config.ts b/apps/widget/src/config/variable.config.ts index b5c2fcb3d..fc783cce3 100644 --- a/apps/widget/src/config/variable.config.ts +++ b/apps/widget/src/config/variable.config.ts @@ -67,10 +67,48 @@ export const defaultCronValues: ScheduleFormValues = { Days: '0', }; -export const fieldAllowedValues = { - Minute: '0-59, *, /, -', - Hour: '0-23, *, /, -', - Day: '1-31, *, /, -, ?', - Month: '1-12, JAN-DEC, *, /, -', - Days: '0-7, SUN-SAT, *, /, -, ?', -}; +export const cronFieldDefinitions = [ + { + Minute: { + values: ['*', '/', '-', '0-59'], + description: ['Any value', 'Step values', 'Range of values', 'Range of allowed values'], + }, + }, + { + Hour: { + values: ['*', '/', '-', '0-23'], + description: ['Any value', 'Step values', 'Range of values', 'Range of allowed values'], + }, + }, + { + Day: { + values: ['*', '/', '-', '?', '1-31'], + description: ['Any value', 'Step values', 'Range of values', 'No specific value', 'Range of allowed values'], + }, + }, + { + Month: { + values: ['*', '/', '-', '1-12', 'JAN-DEC'], + description: [ + 'Any value', + 'Step values', + 'Range of values', + 'Range of allowed values', + 'Alternative single values', + ], + }, + }, + { + Days: { + values: ['*', '/', '-', '?', '0-7', 'SUN-SAT'], + description: [ + 'Any value', + 'Step values', + 'Range of values', + 'No specific value', + 'Range of allowed values (0-7)', + 'Alternative single values', + ], + }, + }, +]; From 3b254436c737c19fd67e15c1074711847eafdf3f Mon Sep 17 00:00:00 2001 From: Mayur Date: Thu, 18 Jul 2024 16:29:33 +0530 Subject: [PATCH 172/299] feat: Added more keys that were necessary to omit --- .../src/app/shared/services/parse-xml/keysToOmit.ts | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/apps/api/src/app/shared/services/parse-xml/keysToOmit.ts b/apps/api/src/app/shared/services/parse-xml/keysToOmit.ts index 6c7c4b293..9e0b633a9 100644 --- a/apps/api/src/app/shared/services/parse-xml/keysToOmit.ts +++ b/apps/api/src/app/shared/services/parse-xml/keysToOmit.ts @@ -18,4 +18,15 @@ export const keysToOmit: Set = new Set([ 'rss > $ > xmlns:slash', 'feed > $ > xmlns', 'feed > $ > xmlns:media', + 'rss', + 'rss > $', + 'rss > channel', + 'rss > channel[]', + 'rss > channel[] > item', + 'rss > channel[] > item[]', + 'rss > channel[] > item[] > guid', + 'rss > channel[] > item[] > guid[]', + 'rss > channel[] > item[] > guid[] > _', + 'rss > channel[] > item[] > guid[] > $', + 'rss > channel[] > item[] > guid[] > $ > isPermaLink', ]); From 413522311a6b80d0ec04f3ca044a807a9234353c Mon Sep 17 00:00:00 2001 From: Mayur Date: Thu, 18 Jul 2024 16:30:10 +0530 Subject: [PATCH 173/299] feat: Revised Xml-parser --- .../services/parse-xml/parse-xml.service.ts | 181 ++++++++++++++---- 1 file changed, 143 insertions(+), 38 deletions(-) diff --git a/apps/api/src/app/shared/services/parse-xml/parse-xml.service.ts b/apps/api/src/app/shared/services/parse-xml/parse-xml.service.ts index 933a1b062..75eb11831 100644 --- a/apps/api/src/app/shared/services/parse-xml/parse-xml.service.ts +++ b/apps/api/src/app/shared/services/parse-xml/parse-xml.service.ts @@ -1,4 +1,4 @@ -import * as xml2js from 'xml2js'; +import { parseStringPromise } from 'xml2js'; import axios from 'axios'; import { keysToOmit } from './keysToOmit'; import { IMappedResult, IMapping, IXmlObject } from '@shared/types/parse-xml.types'; @@ -14,14 +14,73 @@ export async function getMimeType(url: string): Promise { } } +class XMLParserService { + xmlData: any; + + constructor(xmlContent: string) { + this.parseXML(xmlContent); + } + + async parseXML(xmlContent: string): Promise { + this.xmlData = await parseStringPromise(xmlContent); + } + + public getHeadings(): string[] { + const headings: string[] = []; + this.extractHeadings(this.xmlData, '', headings); + + return headings; + } + + extractHeadings(node: any, path: string, headings: string[]): void { + if (typeof node === 'object') { + for (const key in node) { + const newPath = path ? `${path} > ${key}` : `$ > ${key}`; + headings.push(newPath); + if (Array.isArray(node[key])) { + node[key].forEach((item: any) => this.extractHeadings(item, newPath, headings)); + } else { + this.extractHeadings(node[key], newPath, headings); + } + } + } + } + + public getDataByHeading(heading: string): any[] { + const pathArray = heading.split(' > ').map((part) => part.replace('[]', '')); + + return this.extractDataByPath(this.xmlData, pathArray); + } + + private extractDataByPath(node: any, pathArray: string[]): any[] { + if (pathArray.length === 0) return [node]; + + const [current, ...rest] = pathArray; + const result: any[] = []; + + if (node[current]) { + if (Array.isArray(node[current])) { + node[current].forEach((item: any) => { + result.push(...this.extractDataByPath(item, rest)); + }); + } else { + result.push(...this.extractDataByPath(node[current], rest)); + } + } + + return result; + } +} + const parseXmlFromUrl = async (url: string): Promise => { try { - const parser = new xml2js.Parser({ strict: false }); const response = await axios.get(url); const xmlData = response.data; - const parsedData = await parser.parseStringPromise(xmlData); + const parserService = new XMLParserService(xmlData); - return JSON.parse(JSON.stringify(parsedData).toLocaleLowerCase()); + await parserService.parseXML(xmlData); + + return parserService.xmlData; } catch (error) { throw error; } @@ -30,20 +89,15 @@ const parseXmlFromUrl = async (url: string): Promise => { function printKeys(obj: IXmlObject, prefix = '', result: Set = new Set()): string[] { if (Array.isArray(obj)) { obj.forEach((item) => { - const newPrefix = `${prefix}`; + const newPrefix = `${prefix}[]`; printKeys(item, newPrefix, result); }); } else if (typeof obj === 'object' && obj !== null) { Object.keys(obj).forEach((key) => { const newPrefix = prefix ? `${prefix} > ${key}` : key; - if (typeof obj[key] === 'object' && obj[key] !== null) { - printKeys(obj[key], newPrefix, result); - } else if (typeof obj[key] === 'string' || typeof obj[key] === 'number') { - result.add(newPrefix); - } + result.add(newPrefix); + printKeys(obj[key], newPrefix, result); }); - } else { - result.add(prefix); } return Array.from(new Set([...result].filter((key) => !keysToOmit.has(key)))); @@ -52,51 +106,102 @@ function printKeys(obj: IXmlObject, prefix = '', result: Set = new Set() function getValueByPath(obj: IXmlObject, path: string): any { const keys = path.split('>').map((key) => key.trim()); let current: any = obj; - try { - for (const key of keys) { - if (current === undefined) { - return undefined; - } - if (Array.isArray(current)) { - current = current.flatMap((item) => item[key]); - } else { - current = current[key]; - } + + for (const key of keys) { + if (current === undefined) return undefined; + + const isArray = key.endsWith('[]'); + const actualKey = isArray ? key.slice(0, -2) : key; + + if (Array.isArray(current)) { + current = current.flatMap((item) => (item ? item[actualKey] : undefined)).filter(Boolean); + } else { + current = current ? current[actualKey] : undefined; } - if (Array.isArray(current) && current.length === 1) { - return current[0]; + + if (isArray && !Array.isArray(current)) { + current = current ? [current] : []; } + } + // Apply conditions to filter the result + if (Array.isArray(current)) { + return current.filter( + (item) => + typeof item === 'string' || + typeof item === 'number' || + (Array.isArray(item) && item.every((subItem) => typeof subItem === 'string' || typeof subItem === 'number')) + ); + } else if (typeof current === 'string' || typeof current === 'number') { return current; - } catch (error) {} + } + + return undefined; +} +function mapXmlToObject(xmlData: IXmlObject): IMappedResult { + const mappings: IMapping[] = [ + { key: 'Title', mapping: 'rss > channel > title' }, + { key: 'Description', mapping: 'rss > channel > description' }, + { key: 'Link', mapping: 'rss > channel > link' }, + + { key: 'title', mapping: 'rss > channel[] > item[] > title' }, + { key: 'link', mapping: 'rss > channel[] > item[] > link' }, + { key: 'pubdate', mapping: 'rss > channel[] > item[] > pubDate' }, + ]; + + const returnedValues = mappings.map(({ mapping }) => getValueByPath(xmlData, mapping)); + const mappedObject = mappingFunction(mappings, returnedValues); + + return mappedObject; } -function mappingFunction(mappings: IMapping[], values: any[]): IMappedResult[] { +function mappingFunction(mappings: IMapping[], values: any[]): IMappedResult { const mappedObject: IMappedResult = {}; + const arrayFields: string[] = []; + mappings.forEach((mapping, index) => { mappedObject[mapping.key] = values[index]; + if (Array.isArray(values[index])) { + arrayFields.push(mapping.key); + } }); - return [mappedObject]; + if (arrayFields.length > 0) { + const maxLength = Math.max(...arrayFields.map((field) => mappedObject[field]?.length ?? 0)); + + const combinedArray = Array.from({ length: maxLength }, (_, index) => { + const item: any = {}; + + arrayFields.forEach((field) => { + if (mappedObject[field]?.[index] !== undefined) { + item[field.replace('s', '')] = mappedObject[field][index]; + } + }); + + mappings + .filter((mapping) => !arrayFields.includes(mapping.key)) + .forEach((mapping) => { + item[mapping.key] = mappedObject[mapping.key]; + }); + + return item; + }); + + mappedObject.items = combinedArray; + arrayFields.forEach((field) => delete mappedObject[field]); + } + + return mappedObject; } export async function parseRssFeed(url: string): Promise<{ rssKeyHeading: string[]; - mappedObject: IMappedResult[]; + mappedObject: IMappedResult; }> { try { const outputFile = await parseXmlFromUrl(url); const rssKeyHeading = printKeys(outputFile); - - const mappings: IMapping[] = [ - { key: 'title', mapping: 'rss > channel > title' }, - { key: 'description', mapping: 'rss > channel > description' }, - { key: 'lastBuildDate', mapping: 'rss > channel > lastbuilddate' }, - { key: 'latestItemTitle', mapping: 'rss > channel > item > title' }, - ]; - - const returnedValues = mappings.map(({ mapping }) => getValueByPath(outputFile, mapping)); - const mappedObject = mappingFunction(mappings, returnedValues); + const mappedObject = mapXmlToObject(outputFile); return { rssKeyHeading, mappedObject }; } catch (error) { From 350edfcee23b910351498e36ec36a007a1aee1e7 Mon Sep 17 00:00:00 2001 From: Mayur Date: Thu, 18 Jul 2024 16:30:41 +0530 Subject: [PATCH 174/299] feat: Added any --- apps/api/src/app/shared/types/parse-xml.types.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/api/src/app/shared/types/parse-xml.types.ts b/apps/api/src/app/shared/types/parse-xml.types.ts index c97ce12f4..32cd8f286 100644 --- a/apps/api/src/app/shared/types/parse-xml.types.ts +++ b/apps/api/src/app/shared/types/parse-xml.types.ts @@ -8,5 +8,5 @@ export interface IMapping { } export interface IMappedResult { - [key: string]: string | undefined; + [key: string]: string | undefined | any; } From 99cc45d28fef25ce5020a8ea0e711380aaa28a8c Mon Sep 17 00:00:00 2001 From: Mayur Date: Thu, 18 Jul 2024 16:33:06 +0530 Subject: [PATCH 175/299] feat: Streamlined AutoImportPhase3 --- .../AutoImportPhases/AutoImportPhase2/AutoImportPhase2.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/widget/src/components/widget/Phases/AutoImportPhases/AutoImportPhase2/AutoImportPhase2.tsx b/apps/widget/src/components/widget/Phases/AutoImportPhases/AutoImportPhase2/AutoImportPhase2.tsx index a8b17b98b..7f712e96b 100644 --- a/apps/widget/src/components/widget/Phases/AutoImportPhases/AutoImportPhase2/AutoImportPhase2.tsx +++ b/apps/widget/src/components/widget/Phases/AutoImportPhases/AutoImportPhase2/AutoImportPhase2.tsx @@ -46,7 +46,7 @@ export function AutoImportPhase2({ onNextClick }: IAutoImportPhase2Props) { {Array.isArray(mappings) && mappings.map((column, index) => ( ( From 3287b47f4bcc4c58f27214d2b38ead74dd4602f3 Mon Sep 17 00:00:00 2001 From: Mayur Date: Sat, 20 Jul 2024 11:08:17 +0530 Subject: [PATCH 176/299] feat: Streamlined import-jobs controller --- .../api/src/app/import-jobs/import-jobs.controller.ts | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/apps/api/src/app/import-jobs/import-jobs.controller.ts b/apps/api/src/app/import-jobs/import-jobs.controller.ts index fd2febdff..2639fd372 100644 --- a/apps/api/src/app/import-jobs/import-jobs.controller.ts +++ b/apps/api/src/app/import-jobs/import-jobs.controller.ts @@ -32,24 +32,23 @@ export class ImportJobsController { @ApiOperation({ summary: 'Fetch the Import Job Information based on jobId' }) @UseGuards(JwtAuthGuard) @ApiSecurity(ACCESS_KEY_NAME) - async getImportJobInfoRoute(@Param('jobId') jobId: string) { - return this.getJobMapping.execute(jobId); + async getImportJobInfoRoute(@Param('jobId') _jobId: string) { + return this.getJobMapping.execute(_jobId); } @Put(':jobId/mappings') @ApiOperation({ summary: 'Update Mappings Route' }) @UseGuards(JwtAuthGuard) async updateMappingRoute( - @Param('jobId') jobId: string, @Body(new ParseArrayPipe({ items: UpdateJobMappingDto, optional: true })) body: UpdateJobMappingDto[] ) { return this.updateJobMapping.execute(body); } @Put(':jobId') - @ApiOperation({ summary: 'Update Fields' }) + @ApiOperation({ summary: 'Update UserJob Fields' }) @UseGuards(JwtAuthGuard) - async updateUserJobRoute(@Param('jobId') jobId: string, @Body() userJobData: UpdateJobInfoDto) { - return this.updateUserJob.execute(jobId, userJobData); + async updateUserJobRoute(@Param('jobId') _jobId: string, @Body() userJobData: UpdateJobInfoDto) { + return this.updateUserJob.execute(_jobId, userJobData); } } From c78c4abacb9d18915eb9c7996c69658adcb5cd65 Mon Sep 17 00:00:00 2001 From: Mayur Date: Sat, 20 Jul 2024 11:09:45 +0530 Subject: [PATCH 177/299] feat: Removed @IsOptional fron _jobId field --- apps/api/src/app/import-jobs/dtos/update-jobmapping.dto.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/apps/api/src/app/import-jobs/dtos/update-jobmapping.dto.ts b/apps/api/src/app/import-jobs/dtos/update-jobmapping.dto.ts index e87e8d61e..519e0cb85 100644 --- a/apps/api/src/app/import-jobs/dtos/update-jobmapping.dto.ts +++ b/apps/api/src/app/import-jobs/dtos/update-jobmapping.dto.ts @@ -17,6 +17,5 @@ export class UpdateJobMappingDto { path: string; @IsString() - @IsOptional() - jobId: string; + _jobId: string; } From 87417562f0c7f271e90b0f1517df833c799a5b28 Mon Sep 17 00:00:00 2001 From: Mayur Date: Sat, 20 Jul 2024 11:11:16 +0530 Subject: [PATCH 178/299] feat: Renamed jobId to _jobId --- .../usecase/create-jobmapping/create-jobmapping.command.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/api/src/app/import-jobs/usecase/create-jobmapping/create-jobmapping.command.ts b/apps/api/src/app/import-jobs/usecase/create-jobmapping/create-jobmapping.command.ts index 2f40df9bf..d6510ce02 100644 --- a/apps/api/src/app/import-jobs/usecase/create-jobmapping/create-jobmapping.command.ts +++ b/apps/api/src/app/import-jobs/usecase/create-jobmapping/create-jobmapping.command.ts @@ -7,5 +7,5 @@ export class CreateJobMappingCommand { path: string; - jobId: string; + _jobId: string; } From ac3aff0e0b700c2ea2a436d6853298a544f2054f Mon Sep 17 00:00:00 2001 From: Mayur Date: Sat, 20 Jul 2024 11:12:39 +0530 Subject: [PATCH 179/299] feat: Streamlined CreateJobMapping --- .../create-jobmapping/create-jobmapping.usecase.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/apps/api/src/app/import-jobs/usecase/create-jobmapping/create-jobmapping.usecase.ts b/apps/api/src/app/import-jobs/usecase/create-jobmapping/create-jobmapping.usecase.ts index d7ef9ba7d..3c48e6e3a 100644 --- a/apps/api/src/app/import-jobs/usecase/create-jobmapping/create-jobmapping.usecase.ts +++ b/apps/api/src/app/import-jobs/usecase/create-jobmapping/create-jobmapping.usecase.ts @@ -6,15 +6,15 @@ import { CreateJobMappingCommand } from './create-jobmapping.command'; export class CreateJobMapping { constructor(private readonly jobMappingRepository: JobMappingRepository) {} - async execute(mappingCommands: CreateJobMappingCommand[]) { - mappingCommands.filter((command) => !!command.key).map((command) => command.key); + async execute(jobMappingCommand: CreateJobMappingCommand[]) { + jobMappingCommand.filter((command) => !!command.key).map((command) => command.key); - for (const mappingCommand of mappingCommands) { + for (const mappingCommand of jobMappingCommand) { if (mappingCommand.isRequired && !mappingCommand.path) { throw new BadRequestException(`${mappingCommand.name} is required`); } } - return this.jobMappingRepository.createMany(mappingCommands); + return this.jobMappingRepository.createMany(jobMappingCommand); } } From 366413370a2b6af44147b33ab73636e0c90419b4 Mon Sep 17 00:00:00 2001 From: Mayur Date: Sat, 20 Jul 2024 11:17:36 +0530 Subject: [PATCH 180/299] feat: Streamlined CreateUserJob --- .../create-userjob/create-userjob.usecase.ts | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/apps/api/src/app/import-jobs/usecase/create-userjob/create-userjob.usecase.ts b/apps/api/src/app/import-jobs/usecase/create-userjob/create-userjob.usecase.ts index 37a280d9e..5837ae10b 100644 --- a/apps/api/src/app/import-jobs/usecase/create-userjob/create-userjob.usecase.ts +++ b/apps/api/src/app/import-jobs/usecase/create-userjob/create-userjob.usecase.ts @@ -1,20 +1,21 @@ import { BadRequestException, Injectable } from '@nestjs/common'; import { FileMimeTypesEnum } from '@impler/shared'; import { APIMessages } from '@shared/constants'; -import { parseRssFeed, getMimeType } from '@shared/services/parse-xml'; import { UserJobEntity, UserJobRepository } from '@impler/dal'; +import { RSSService } from '@shared/services'; @Injectable() export class CreateUserJob { - constructor(private readonly userJobRepository: UserJobRepository) {} + constructor( + private readonly userJobRepository: UserJobRepository, + private readonly rssService: RSSService + ) {} async execute({ _templateId, url }: { url: string; _templateId: string }): Promise { - const mimeType = await getMimeType(url); - - if (mimeType !== FileMimeTypesEnum.XML) { + if ((await this.rssService.getMimeType(url)) !== FileMimeTypesEnum.XML) { throw new BadRequestException(APIMessages.INVALID_RSS_URL); } else { - const { rssKeyHeading } = await parseRssFeed(url); + const { rssKeyHeading } = await this.rssService.parseRssFeed(url); return await this.userJobRepository.create({ url, From 7bf56537c4a969d8f74419f4303a0dcc35d58e7f Mon Sep 17 00:00:00 2001 From: Mayur Date: Sat, 20 Jul 2024 11:18:11 +0530 Subject: [PATCH 181/299] feat: Renamed jobId to _jobId --- .../usecase/get-jobmapping/get-jobmapping.usecase.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/api/src/app/import-jobs/usecase/get-jobmapping/get-jobmapping.usecase.ts b/apps/api/src/app/import-jobs/usecase/get-jobmapping/get-jobmapping.usecase.ts index 62c693e75..3c2c553f7 100644 --- a/apps/api/src/app/import-jobs/usecase/get-jobmapping/get-jobmapping.usecase.ts +++ b/apps/api/src/app/import-jobs/usecase/get-jobmapping/get-jobmapping.usecase.ts @@ -8,8 +8,8 @@ export class GetJobMapping { private readonly columnRepository: ColumnRepository ) {} - async execute(jobId: string): Promise { - const userJob = await this.userJobRepository.findOne({ _id: jobId }); + async execute(_jobId: string): Promise { + const userJob = await this.userJobRepository.findOne({ _id: _jobId }); if (!userJob) { throw new Error('User job not found'); @@ -21,7 +21,7 @@ export class GetJobMapping { name: column.name, isRequired: column.isRequired, path: undefined, - jobId, + _jobId, })); return jobsMapping; From 03eed7f6f25fd56e59628cbfad3587284d14ab45 Mon Sep 17 00:00:00 2001 From: Mayur Date: Sat, 20 Jul 2024 11:19:18 +0530 Subject: [PATCH 182/299] feat: Added QueueService and RSSService --- apps/api/src/app/shared/shared.module.ts | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/apps/api/src/app/shared/shared.module.ts b/apps/api/src/app/shared/shared.module.ts index 08153dd41..64ce08437 100644 --- a/apps/api/src/app/shared/shared.module.ts +++ b/apps/api/src/app/shared/shared.module.ts @@ -19,9 +19,12 @@ import { JobMappingRepository, } from '@impler/dal'; import { FileNameService } from '@impler/shared'; +import { SchedulerRegistry } from '@nestjs/schedule'; import { S3StorageService, StorageService } from '@impler/shared/dist/services/storage'; import { CSVFileService2, ExcelFileService } from './services/file/file.service'; import { EmailService, SESEmailService } from './services/email.service'; +import { QueueService } from './services/queue.service'; +import { RSSService } from './services/rss.service'; const DAL_MODELS = [ ProjectRepository, @@ -39,6 +42,8 @@ const DAL_MODELS = [ BubbleDestinationRepository, UserJobRepository, JobMappingRepository, + UserJobRepository, + SchedulerRegistry, ]; const FILE_SERVICES = [CSVFileService2, FileNameService, ExcelFileService]; @@ -52,6 +57,14 @@ function getEmailServiceClass() { return SESEmailService; } +function getQueueServiceClass() { + return QueueService; +} + +function getRSSServiceClass() { + return RSSService; +} + const PROVIDERS = [ { provide: DalService, @@ -70,6 +83,14 @@ const PROVIDERS = [ provide: EmailService, useClass: getEmailServiceClass(), }, + { + provide: QueueService, + useClass: getQueueServiceClass(), + }, + { + provide: RSSService, + useClass: getRSSServiceClass(), + }, ...FILE_SERVICES, JwtService, ]; From 448d5ff789e88813f809902e71504c1742b0bb1a Mon Sep 17 00:00:00 2001 From: Mayur Date: Sat, 20 Jul 2024 11:22:48 +0530 Subject: [PATCH 183/299] feat: Exported member rss.service --- apps/api/src/app/shared/services/index.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/api/src/app/shared/services/index.ts b/apps/api/src/app/shared/services/index.ts index 706b0d228..b70598312 100644 --- a/apps/api/src/app/shared/services/index.ts +++ b/apps/api/src/app/shared/services/index.ts @@ -1 +1,2 @@ export * from './file'; +export * from './rss.service'; From fc729e7be8e842010c0145110a62730489c75ffc Mon Sep 17 00:00:00 2001 From: Mayur Date: Sat, 20 Jul 2024 11:28:19 +0530 Subject: [PATCH 184/299] feat: Added publishToQueue overload for SEND_RSS_XML_DATA --- apps/api/src/app/shared/services/queue.service.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/apps/api/src/app/shared/services/queue.service.ts b/apps/api/src/app/shared/services/queue.service.ts index 0f6de3767..4ecfb8c79 100644 --- a/apps/api/src/app/shared/services/queue.service.ts +++ b/apps/api/src/app/shared/services/queue.service.ts @@ -1,6 +1,6 @@ import { Injectable, Logger } from '@nestjs/common'; import amqp from 'amqp-connection-manager'; -import { EndImportData, SendWebhookData, PublishToQueueData, QueuesEnum } from '@impler/shared'; +import { EndImportData, SendWebhookData, PublishToQueueData, QueuesEnum, SendRSSXMLData } from '@impler/shared'; @Injectable() export class QueueService { @@ -30,7 +30,9 @@ export class QueueService { publishToQueue(queueName: QueuesEnum.END_IMPORT, data: EndImportData): void; publishToQueue(queueName: QueuesEnum.SEND_WEBHOOK_DATA, data: SendWebhookData): void; - async publishToQueue(queueName: QueuesEnum, data: PublishToQueueData) { + publishToQueue(queueName: QueuesEnum.SEND_RSS_XML_DATA, data: SendRSSXMLData): void; + + async publishToQueue(queueName: QueuesEnum, data: PublishToQueueData | SendRSSXMLData) { if (this.connection.isConnected()) { await this.chanelWrapper.sendToQueue(queueName, data, { durable: false }); } else { From 6482fd5da3083f2089e817821773cfbda18ceb0c Mon Sep 17 00:00:00 2001 From: Mayur Date: Sat, 20 Jul 2024 11:30:40 +0530 Subject: [PATCH 185/299] feat: Added channel.assertQueue for SEND_RSS_XML_DATA --- apps/queue-manager/src/bootstrap.ts | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/apps/queue-manager/src/bootstrap.ts b/apps/queue-manager/src/bootstrap.ts index e3f03d73e..102fe1202 100644 --- a/apps/queue-manager/src/bootstrap.ts +++ b/apps/queue-manager/src/bootstrap.ts @@ -6,7 +6,7 @@ import { IAmqpConnectionManager } from 'amqp-connection-manager/dist/esm/AmqpCon import { QueuesEnum } from '@impler/shared'; import { DalService } from '@impler/dal'; -import { SendWebhookDataConsumer, EndImportConsumer, SendBubbleDataConsumer } from './consumers'; +import { SendWebhookDataConsumer, EndImportConsumer, SendBubbleDataConsumer, ImportJobDataConsumer } from './consumers'; let connection: IAmqpConnectionManager, chanelWrapper: ChannelWrapper; @@ -39,6 +39,7 @@ export async function bootstrap() { const endImportConsumer = new EndImportConsumer(); const sendBubbleDataConsumer = new SendBubbleDataConsumer(); const sendWebhookdataConsumer = new SendWebhookDataConsumer(); + const importJobbDataConsumer = new ImportJobDataConsumer(); // add queues to channel chanelWrapper.addSetup((channel) => { @@ -59,6 +60,12 @@ export async function bootstrap() { channel.consume(QueuesEnum.SEND_BUBBLE_DATA, sendBubbleDataConsumer.message.bind(sendBubbleDataConsumer), { noAck: true, }), + channel.assertQueue(QueuesEnum.SEND_RSS_XML_DATA, { + durable: false, + }), + channel.consume(QueuesEnum.SEND_RSS_XML_DATA, importJobbDataConsumer.message.bind(importJobbDataConsumer), { + noAck: true, + }), ]); }); } From 6d013a4b318b26002265ba2f1204b08a96f2404c Mon Sep 17 00:00:00 2001 From: Mayur Date: Sat, 20 Jul 2024 11:31:14 +0530 Subject: [PATCH 186/299] feat: Added SEND_RSS_XML_DATA in QueuesEnum --- libs/shared/src/types/upload/upload.types.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/libs/shared/src/types/upload/upload.types.ts b/libs/shared/src/types/upload/upload.types.ts index b5d6add54..6a35e1443 100644 --- a/libs/shared/src/types/upload/upload.types.ts +++ b/libs/shared/src/types/upload/upload.types.ts @@ -39,6 +39,7 @@ export enum QueuesEnum { 'SEND_WEBHOOK_DATA' = 'SEND_WEBHOOK_DATA', 'SEND_BUBBLE_DATA' = 'SEND_BUBBLE_DATA', 'END_IMPORT' = 'END_IMPORT', + 'SEND_RSS_XML_DATA' = 'SEND_RSS_XML_DATA', } export type SendWebhookCachedData = { @@ -78,6 +79,10 @@ export type SendWebhookData = { uploadId: string; cache?: SendWebhookCachedData; }; + +export type SendRSSXMLData = { + _jobId: string; +}; export type PublishToQueueData = SendWebhookData; export type EndImportData = { From 1c45dbaf91db7fe9cc0a7f92a7e7d83ea8602b49 Mon Sep 17 00:00:00 2001 From: Mayur Date: Sat, 20 Jul 2024 11:31:39 +0530 Subject: [PATCH 187/299] feat: Renamed _jobId to jobId --- .../src/entities/UserJobMapping/UserJobMapping.interface.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/shared/src/entities/UserJobMapping/UserJobMapping.interface.ts b/libs/shared/src/entities/UserJobMapping/UserJobMapping.interface.ts index c1c7dd985..777ecbd1a 100644 --- a/libs/shared/src/entities/UserJobMapping/UserJobMapping.interface.ts +++ b/libs/shared/src/entities/UserJobMapping/UserJobMapping.interface.ts @@ -4,5 +4,5 @@ export interface IUserJobMapping { name: string; isRequired: boolean; path: string; - jobId: string; + _jobId: string; } From 414c89ff3807784fbf7a4defd57beecd0eea303c Mon Sep 17 00:00:00 2001 From: Mayur Date: Sat, 20 Jul 2024 11:32:06 +0530 Subject: [PATCH 188/299] feat: Changed _userJobId to _jobId --- libs/dal/src/repositories/jobmapping/jobmapping.schema.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/dal/src/repositories/jobmapping/jobmapping.schema.ts b/libs/dal/src/repositories/jobmapping/jobmapping.schema.ts index 8da714287..8fae4082d 100644 --- a/libs/dal/src/repositories/jobmapping/jobmapping.schema.ts +++ b/libs/dal/src/repositories/jobmapping/jobmapping.schema.ts @@ -16,7 +16,7 @@ const jobMappingSchema = new Schema( path: { type: Schema.Types.String, }, - _userJobId: { + _jobId: { type: Schema.Types.ObjectId, ref: 'UserJob', }, From 3119f881d458f2d03bfb3ad6eb085b34a8107134 Mon Sep 17 00:00:00 2001 From: Mayur Date: Sat, 20 Jul 2024 11:32:53 +0530 Subject: [PATCH 189/299] feat: Exported member import-job-data.consumer --- apps/queue-manager/src/consumers/index.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/queue-manager/src/consumers/index.ts b/apps/queue-manager/src/consumers/index.ts index 64a068753..abd4f8579 100644 --- a/apps/queue-manager/src/consumers/index.ts +++ b/apps/queue-manager/src/consumers/index.ts @@ -1,3 +1,4 @@ export * from './send-webhook-data.consumer'; export * from './end-import.consumer'; export * from './send-bubble-data.consumer'; +export * from './import-job-data.consumer'; From 505cc5ac5f0f6132ce58cdc5eff56217eede9d5d Mon Sep 17 00:00:00 2001 From: Mayur Date: Sat, 20 Jul 2024 11:33:37 +0530 Subject: [PATCH 190/299] feat: Renamed jobId to _jobId --- libs/dal/src/repositories/jobmapping/jobmapping.entity.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/dal/src/repositories/jobmapping/jobmapping.entity.ts b/libs/dal/src/repositories/jobmapping/jobmapping.entity.ts index ff89b9102..f707e9024 100644 --- a/libs/dal/src/repositories/jobmapping/jobmapping.entity.ts +++ b/libs/dal/src/repositories/jobmapping/jobmapping.entity.ts @@ -9,5 +9,5 @@ export class JobMappingEntity { path: string; - jobId: string; + _jobId: string; } From e69968d86cd06210cf990b8665fe7fbc5639c5ed Mon Sep 17 00:00:00 2001 From: Mayur Date: Sat, 20 Jul 2024 11:38:22 +0530 Subject: [PATCH 191/299] feat: Added keysToOmit --- apps/widget/src/config/variable.config.ts | 33 +++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/apps/widget/src/config/variable.config.ts b/apps/widget/src/config/variable.config.ts index fc783cce3..d0a7a67b8 100644 --- a/apps/widget/src/config/variable.config.ts +++ b/apps/widget/src/config/variable.config.ts @@ -112,3 +112,36 @@ export const cronFieldDefinitions = [ }, }, ]; + +export const keysToOmit = new Set([ + 'rss > channel > atom:id', + 'rss > $ > xmlns:opensearch', + 'rss > $ > xmlns:blogger', + 'rss > $ > xmlns:georss', + 'rss > $ > xmlns:gd', + 'rss > $ > xmlns:thr', + 'rss > channel > atom:link > $ > type', + 'rss > channel > atom:link > $ > rel', + 'rss > channel > atom:link > $ > href', + 'rss > $ > xmlns:media', + 'rss > $ > xmlns:dc', + 'rss > $ > xmlns:content', + 'rss > $ > xmlns:atom', + 'rss > $ > version', + 'rss > $ > xmlns:wfw', + 'rss > $ > xmlns:sy', + 'rss > $ > xmlns:slash', + 'feed > $ > xmlns', + 'feed > $ > xmlns:media', + 'rss', + 'rss > $', + 'rss > channel', + 'rss > channel[]', + 'rss > channel[] > item', + 'rss > channel[] > item[]', + 'rss > channel[] > item[] > guid', + 'rss > channel[] > item[] > guid[]', + 'rss > channel[] > item[] > guid[] > _', + 'rss > channel[] > item[] > guid[] > $', + 'rss > channel[] > item[] > guid[] > $ > isPermaLink', +]); From 1ca53bc77338c4eff949aee1e39f5d0fad9bcb54 Mon Sep 17 00:00:00 2001 From: Mayur Date: Sat, 20 Jul 2024 11:39:12 +0530 Subject: [PATCH 192/299] feat: Imported and used keysToOmit --- .../AutoImportPhase2/useAutoImportPhase2.tsx | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/apps/widget/src/components/widget/Phases/AutoImportPhases/hooks/AutoImportPhase2/useAutoImportPhase2.tsx b/apps/widget/src/components/widget/Phases/AutoImportPhases/hooks/AutoImportPhase2/useAutoImportPhase2.tsx index 511d99a00..7154fe83d 100644 --- a/apps/widget/src/components/widget/Phases/AutoImportPhases/hooks/AutoImportPhase2/useAutoImportPhase2.tsx +++ b/apps/widget/src/components/widget/Phases/AutoImportPhases/hooks/AutoImportPhase2/useAutoImportPhase2.tsx @@ -6,6 +6,7 @@ import { IErrorObject, IUserJobMapping } from '@impler/shared'; import { useAPIState } from '@store/api.context'; import { useJobsInfo } from '@store/jobinfo.context'; import { notifier } from '@util'; +import { keysToOmit } from '@config'; interface IUseAutoImportPhase2Props { goNext: () => void; @@ -25,20 +26,18 @@ export function useAutoImportPhase2({ goNext }: IUseAutoImportPhase2Props) { string[] >(['getUserJobMappings'], () => api.getUserJobMappings(jobsInfo?._id), { onSuccess(mappingsResponse) { - setHeadings( - jobsInfo.headings.map((heading) => ({ - value: heading, - label: heading, - disabled: mappingsResponse.some((mapping) => mapping.path === heading), - })) - ); + const allHeadings = jobsInfo.headings.map((heading) => ({ value: heading, label: heading, disabled: false })); + const filteredHeadings = allHeadings.filter((headingItem) => !keysToOmit.has(headingItem.value)); + + setHeadings(filteredHeadings); + reset({ mappings: mappingsResponse.map((mapping) => ({ key: mapping.key, name: mapping.name, path: mapping.path, isRequired: mapping.isRequired, - jobId: mapping.jobId, + _jobId: mapping._jobId, })), }); }, From 04cf225e1f019dc2aa4214cff2d009998fa40032 Mon Sep 17 00:00:00 2001 From: Mayur Date: Sat, 20 Jul 2024 11:42:25 +0530 Subject: [PATCH 193/299] feat: Unused files deleted --- .../app/shared/services/parse-xml/index.ts | 2 - .../shared/services/parse-xml/keysToOmit.ts | 32 --- .../services/parse-xml/parse-xml.service.ts | 210 ------------------ .../src/app/shared/types/parse-xml.types.ts | 12 - 4 files changed, 256 deletions(-) delete mode 100644 apps/api/src/app/shared/services/parse-xml/index.ts delete mode 100644 apps/api/src/app/shared/services/parse-xml/keysToOmit.ts delete mode 100644 apps/api/src/app/shared/services/parse-xml/parse-xml.service.ts delete mode 100644 apps/api/src/app/shared/types/parse-xml.types.ts diff --git a/apps/api/src/app/shared/services/parse-xml/index.ts b/apps/api/src/app/shared/services/parse-xml/index.ts deleted file mode 100644 index b65bc6014..000000000 --- a/apps/api/src/app/shared/services/parse-xml/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -export * from './parse-xml.service'; -export * from './keysToOmit'; diff --git a/apps/api/src/app/shared/services/parse-xml/keysToOmit.ts b/apps/api/src/app/shared/services/parse-xml/keysToOmit.ts deleted file mode 100644 index 9e0b633a9..000000000 --- a/apps/api/src/app/shared/services/parse-xml/keysToOmit.ts +++ /dev/null @@ -1,32 +0,0 @@ -export const keysToOmit: Set = new Set([ - 'rss > channel > atom:id', - 'rss > $ > xmlns:opensearch', - 'rss > $ > xmlns:blogger', - 'rss > $ > xmlns:georss', - 'rss > $ > xmlns:gd', - 'rss > $ > xmlns:thr', - 'rss > channel > atom:link > $ > type', - 'rss > channel > atom:link > $ > rel', - 'rss > channel > atom:link > $ > href', - 'rss > $ > xmlns:media', - 'rss > $ > xmlns:dc', - 'rss > $ > xmlns:content', - 'rss > $ > xmlns:atom', - 'rss > $ > version', - 'rss > $ > xmlns:wfw', - 'rss > $ > xmlns:sy', - 'rss > $ > xmlns:slash', - 'feed > $ > xmlns', - 'feed > $ > xmlns:media', - 'rss', - 'rss > $', - 'rss > channel', - 'rss > channel[]', - 'rss > channel[] > item', - 'rss > channel[] > item[]', - 'rss > channel[] > item[] > guid', - 'rss > channel[] > item[] > guid[]', - 'rss > channel[] > item[] > guid[] > _', - 'rss > channel[] > item[] > guid[] > $', - 'rss > channel[] > item[] > guid[] > $ > isPermaLink', -]); diff --git a/apps/api/src/app/shared/services/parse-xml/parse-xml.service.ts b/apps/api/src/app/shared/services/parse-xml/parse-xml.service.ts deleted file mode 100644 index 75eb11831..000000000 --- a/apps/api/src/app/shared/services/parse-xml/parse-xml.service.ts +++ /dev/null @@ -1,210 +0,0 @@ -import { parseStringPromise } from 'xml2js'; -import axios from 'axios'; -import { keysToOmit } from './keysToOmit'; -import { IMappedResult, IMapping, IXmlObject } from '@shared/types/parse-xml.types'; - -export async function getMimeType(url: string): Promise { - try { - const response = await axios.head(url); - const mimeType = response.headers['content-type'] || null; - - return mimeType?.split(';')[0] || null; - } catch (error) { - return null; - } -} - -class XMLParserService { - xmlData: any; - - constructor(xmlContent: string) { - this.parseXML(xmlContent); - } - - async parseXML(xmlContent: string): Promise { - this.xmlData = await parseStringPromise(xmlContent); - } - - public getHeadings(): string[] { - const headings: string[] = []; - this.extractHeadings(this.xmlData, '', headings); - - return headings; - } - - extractHeadings(node: any, path: string, headings: string[]): void { - if (typeof node === 'object') { - for (const key in node) { - const newPath = path ? `${path} > ${key}` : `$ > ${key}`; - headings.push(newPath); - if (Array.isArray(node[key])) { - node[key].forEach((item: any) => this.extractHeadings(item, newPath, headings)); - } else { - this.extractHeadings(node[key], newPath, headings); - } - } - } - } - - public getDataByHeading(heading: string): any[] { - const pathArray = heading.split(' > ').map((part) => part.replace('[]', '')); - - return this.extractDataByPath(this.xmlData, pathArray); - } - - private extractDataByPath(node: any, pathArray: string[]): any[] { - if (pathArray.length === 0) return [node]; - - const [current, ...rest] = pathArray; - const result: any[] = []; - - if (node[current]) { - if (Array.isArray(node[current])) { - node[current].forEach((item: any) => { - result.push(...this.extractDataByPath(item, rest)); - }); - } else { - result.push(...this.extractDataByPath(node[current], rest)); - } - } - - return result; - } -} - -const parseXmlFromUrl = async (url: string): Promise => { - try { - const response = await axios.get(url); - const xmlData = response.data; - const parserService = new XMLParserService(xmlData); - - await parserService.parseXML(xmlData); - - return parserService.xmlData; - } catch (error) { - throw error; - } -}; - -function printKeys(obj: IXmlObject, prefix = '', result: Set = new Set()): string[] { - if (Array.isArray(obj)) { - obj.forEach((item) => { - const newPrefix = `${prefix}[]`; - printKeys(item, newPrefix, result); - }); - } else if (typeof obj === 'object' && obj !== null) { - Object.keys(obj).forEach((key) => { - const newPrefix = prefix ? `${prefix} > ${key}` : key; - result.add(newPrefix); - printKeys(obj[key], newPrefix, result); - }); - } - - return Array.from(new Set([...result].filter((key) => !keysToOmit.has(key)))); -} - -function getValueByPath(obj: IXmlObject, path: string): any { - const keys = path.split('>').map((key) => key.trim()); - let current: any = obj; - - for (const key of keys) { - if (current === undefined) return undefined; - - const isArray = key.endsWith('[]'); - const actualKey = isArray ? key.slice(0, -2) : key; - - if (Array.isArray(current)) { - current = current.flatMap((item) => (item ? item[actualKey] : undefined)).filter(Boolean); - } else { - current = current ? current[actualKey] : undefined; - } - - if (isArray && !Array.isArray(current)) { - current = current ? [current] : []; - } - } - - // Apply conditions to filter the result - if (Array.isArray(current)) { - return current.filter( - (item) => - typeof item === 'string' || - typeof item === 'number' || - (Array.isArray(item) && item.every((subItem) => typeof subItem === 'string' || typeof subItem === 'number')) - ); - } else if (typeof current === 'string' || typeof current === 'number') { - return current; - } - - return undefined; -} -function mapXmlToObject(xmlData: IXmlObject): IMappedResult { - const mappings: IMapping[] = [ - { key: 'Title', mapping: 'rss > channel > title' }, - { key: 'Description', mapping: 'rss > channel > description' }, - { key: 'Link', mapping: 'rss > channel > link' }, - - { key: 'title', mapping: 'rss > channel[] > item[] > title' }, - { key: 'link', mapping: 'rss > channel[] > item[] > link' }, - { key: 'pubdate', mapping: 'rss > channel[] > item[] > pubDate' }, - ]; - - const returnedValues = mappings.map(({ mapping }) => getValueByPath(xmlData, mapping)); - const mappedObject = mappingFunction(mappings, returnedValues); - - return mappedObject; -} - -function mappingFunction(mappings: IMapping[], values: any[]): IMappedResult { - const mappedObject: IMappedResult = {}; - const arrayFields: string[] = []; - - mappings.forEach((mapping, index) => { - mappedObject[mapping.key] = values[index]; - if (Array.isArray(values[index])) { - arrayFields.push(mapping.key); - } - }); - - if (arrayFields.length > 0) { - const maxLength = Math.max(...arrayFields.map((field) => mappedObject[field]?.length ?? 0)); - - const combinedArray = Array.from({ length: maxLength }, (_, index) => { - const item: any = {}; - - arrayFields.forEach((field) => { - if (mappedObject[field]?.[index] !== undefined) { - item[field.replace('s', '')] = mappedObject[field][index]; - } - }); - - mappings - .filter((mapping) => !arrayFields.includes(mapping.key)) - .forEach((mapping) => { - item[mapping.key] = mappedObject[mapping.key]; - }); - - return item; - }); - - mappedObject.items = combinedArray; - arrayFields.forEach((field) => delete mappedObject[field]); - } - - return mappedObject; -} - -export async function parseRssFeed(url: string): Promise<{ - rssKeyHeading: string[]; - mappedObject: IMappedResult; -}> { - try { - const outputFile = await parseXmlFromUrl(url); - const rssKeyHeading = printKeys(outputFile); - const mappedObject = mapXmlToObject(outputFile); - - return { rssKeyHeading, mappedObject }; - } catch (error) { - throw error; - } -} diff --git a/apps/api/src/app/shared/types/parse-xml.types.ts b/apps/api/src/app/shared/types/parse-xml.types.ts deleted file mode 100644 index 32cd8f286..000000000 --- a/apps/api/src/app/shared/types/parse-xml.types.ts +++ /dev/null @@ -1,12 +0,0 @@ -export interface IXmlObject { - [key: string]: any; -} - -export interface IMapping { - key: string; - mapping: string; -} - -export interface IMappedResult { - [key: string]: string | undefined | any; -} From 9ebc7a1c1f197966af7fcbb92c44f7bb66ba992b Mon Sep 17 00:00:00 2001 From: Mayur Date: Sat, 20 Jul 2024 14:52:58 +0530 Subject: [PATCH 194/299] feat: Added TEXTXML to identify MIME type text/xml --- libs/shared/src/types/upload/upload.types.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/libs/shared/src/types/upload/upload.types.ts b/libs/shared/src/types/upload/upload.types.ts index 6a35e1443..5434528b9 100644 --- a/libs/shared/src/types/upload/upload.types.ts +++ b/libs/shared/src/types/upload/upload.types.ts @@ -26,6 +26,7 @@ export enum FileMimeTypesEnum { 'EXCELM' = 'application/vnd.ms-excel.sheet.macroenabled.12', 'JSON' = 'application/json', 'XML' = 'application/rss+xml', + 'TEXTXML' = 'text/xml', } export enum FileEncodingsEnum { From 8692abbc3d8afd3777cf4cf1eb029f9c049008b9 Mon Sep 17 00:00:00 2001 From: Mayur Date: Sat, 20 Jul 2024 15:03:34 +0530 Subject: [PATCH 195/299] feat: Refactored CreateUserJob usecase to improve MIME type validation and parsing logic --- .../usecase/create-userjob/create-userjob.usecase.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/apps/api/src/app/import-jobs/usecase/create-userjob/create-userjob.usecase.ts b/apps/api/src/app/import-jobs/usecase/create-userjob/create-userjob.usecase.ts index 5837ae10b..6cd38aa72 100644 --- a/apps/api/src/app/import-jobs/usecase/create-userjob/create-userjob.usecase.ts +++ b/apps/api/src/app/import-jobs/usecase/create-userjob/create-userjob.usecase.ts @@ -12,9 +12,8 @@ export class CreateUserJob { ) {} async execute({ _templateId, url }: { url: string; _templateId: string }): Promise { - if ((await this.rssService.getMimeType(url)) !== FileMimeTypesEnum.XML) { - throw new BadRequestException(APIMessages.INVALID_RSS_URL); - } else { + const mimeType = await this.rssService.getMimeType(url); + if (mimeType === FileMimeTypesEnum.XML || mimeType === FileMimeTypesEnum.TEXTXML) { const { rssKeyHeading } = await this.rssService.parseRssFeed(url); return await this.userJobRepository.create({ @@ -22,6 +21,8 @@ export class CreateUserJob { headings: rssKeyHeading, _templateId: _templateId, }); + } else { + throw new BadRequestException(APIMessages.INVALID_RSS_URL); } } } From 1227d2af10cb0a45d984c4a53eeebfb1ebe31f6f Mon Sep 17 00:00:00 2001 From: Mayur Date: Sat, 20 Jul 2024 16:04:29 +0530 Subject: [PATCH 196/299] feat: Exported member cronjob.service --- apps/api/src/app/shared/services/index.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/api/src/app/shared/services/index.ts b/apps/api/src/app/shared/services/index.ts index b70598312..ff15ce10c 100644 --- a/apps/api/src/app/shared/services/index.ts +++ b/apps/api/src/app/shared/services/index.ts @@ -1,2 +1,3 @@ export * from './file'; export * from './rss.service'; +export * from './cronjob.service'; From ab742ecb3f5bf8d0f653680284fe3c20e5871024 Mon Sep 17 00:00:00 2001 From: Mayur Date: Sat, 20 Jul 2024 18:41:20 +0530 Subject: [PATCH 197/299] feat: New Routes forgetting, pausing, starting, deleting user jobs added --- .../app/import-jobs/import-jobs.controller.ts | 34 +++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/apps/api/src/app/import-jobs/import-jobs.controller.ts b/apps/api/src/app/import-jobs/import-jobs.controller.ts index 2639fd372..29051700e 100644 --- a/apps/api/src/app/import-jobs/import-jobs.controller.ts +++ b/apps/api/src/app/import-jobs/import-jobs.controller.ts @@ -1,6 +1,7 @@ import { ApiTags, ApiSecurity, ApiOperation } from '@nestjs/swagger'; -import { Body, Controller, Get, Param, ParseArrayPipe, Post, Put, UseGuards } from '@nestjs/common'; +import { Body, Controller, Delete, Get, Param, ParseArrayPipe, Post, Put, UseGuards } from '@nestjs/common'; import { CreateUserJob, GetJobMapping, CreateJobMapping, UpdateUserJob } from './usecase'; +import { CronJobService } from '../shared/services/cronjob.service'; import { UpdateJobMappingDto } from './dtos/update-jobmapping.dto'; import { UpdateJobInfoDto } from './dtos/update-jobinfo.dto'; import { JwtAuthGuard } from '@shared/framework/auth.gaurd'; @@ -15,7 +16,8 @@ export class ImportJobsController { private createUserJob: CreateUserJob, private getJobMapping: GetJobMapping, private updateJobMapping: CreateJobMapping, - private updateUserJob: UpdateUserJob + private updateUserJob: UpdateUserJob, + private cronJobService: CronJobService ) {} @Post(':templateId') @ApiOperation({ summary: 'Create User Job' }) @@ -51,4 +53,32 @@ export class ImportJobsController { async updateUserJobRoute(@Param('jobId') _jobId: string, @Body() userJobData: UpdateJobInfoDto) { return this.updateUserJob.execute(_jobId, userJobData); } + + @Get('/user/:id') + @ApiOperation({ summary: 'Get Import Jobs for a User' }) + @UseGuards(JwtAuthGuard) + async getUserJobs(@Param('id') userId: string) { + return this.getJobMapping.getUserJobs(userId); + } + + @Put('/user/:id/:jobId/pause') + @ApiOperation({ summary: 'Pause a Cron Job' }) + @UseGuards(JwtAuthGuard) + async pauseCronJob(@Param('jobId') jobId: string) { + return this.cronJobService.pauseJob(jobId); + } + + @Put('/user/:id/:jobId/start') + @ApiOperation({ summary: 'Start a Cron Job' }) + @UseGuards(JwtAuthGuard) + async startCronJob(@Param('id') userId: string, @Param('jobId') jobId: string) { + return this.cronJobService.startJob(userId, jobId); + } + + @Delete('/user/:id/:jobId/delete') + @ApiOperation({ summary: 'Delete a Job' }) + @UseGuards(JwtAuthGuard) + async deleteJob(@Param('id') userId: string, @Param('jobId') jobId: string) { + return this.cronJobService.deleteJob(userId, jobId); + } } From 3274c144b132ce2a04a3703a5e244bdd02e6a370 Mon Sep 17 00:00:00 2001 From: Mayur Date: Sat, 20 Jul 2024 18:47:51 +0530 Subject: [PATCH 198/299] feat: CronJobService added --- apps/api/src/app/shared/shared.module.ts | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/apps/api/src/app/shared/shared.module.ts b/apps/api/src/app/shared/shared.module.ts index 64ce08437..bafc95a7c 100644 --- a/apps/api/src/app/shared/shared.module.ts +++ b/apps/api/src/app/shared/shared.module.ts @@ -23,6 +23,7 @@ import { SchedulerRegistry } from '@nestjs/schedule'; import { S3StorageService, StorageService } from '@impler/shared/dist/services/storage'; import { CSVFileService2, ExcelFileService } from './services/file/file.service'; import { EmailService, SESEmailService } from './services/email.service'; +import { CronJobService } from '@shared/services/cronjob.service'; import { QueueService } from './services/queue.service'; import { RSSService } from './services/rss.service'; @@ -64,6 +65,9 @@ function getQueueServiceClass() { function getRSSServiceClass() { return RSSService; } +function getCronJobService() { + return CronJobService; +} const PROVIDERS = [ { @@ -91,6 +95,10 @@ const PROVIDERS = [ provide: RSSService, useClass: getRSSServiceClass(), }, + { + provide: CronJobService, + useClass: getCronJobService(), + }, ...FILE_SERVICES, JwtService, ]; From fa3f1bc27ca1bdc8b8ddbe445e433cd412571dd3 Mon Sep 17 00:00:00 2001 From: Mayur Date: Sat, 20 Jul 2024 18:50:08 +0530 Subject: [PATCH 199/299] feat: New keys added in keysToOmit --- apps/widget/src/config/variable.config.ts | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/apps/widget/src/config/variable.config.ts b/apps/widget/src/config/variable.config.ts index d0a7a67b8..33ffd779f 100644 --- a/apps/widget/src/config/variable.config.ts +++ b/apps/widget/src/config/variable.config.ts @@ -120,11 +120,21 @@ export const keysToOmit = new Set([ 'rss > $ > xmlns:georss', 'rss > $ > xmlns:gd', 'rss > $ > xmlns:thr', + 'rss', + 'rss > $', + 'rss > $ > xmlns:content', + 'rss > $ > xmlns:rdf', + 'rss > $ > xmlns:itunes', + 'rss > $ > xmlns:media', + 'rss > $ > xmlns:dc', + 'rss > $ > xmlns:gml', + 'rss > $ > xmlns:taxo', + 'rss > $ > xmlns:geo', + 'rss > $ > version', + 'rss > $ > xmlns:atom', 'rss > channel > atom:link > $ > type', 'rss > channel > atom:link > $ > rel', 'rss > channel > atom:link > $ > href', - 'rss > $ > xmlns:media', - 'rss > $ > xmlns:dc', 'rss > $ > xmlns:content', 'rss > $ > xmlns:atom', 'rss > $ > version', @@ -144,4 +154,9 @@ export const keysToOmit = new Set([ 'rss > channel[] > item[] > guid[] > _', 'rss > channel[] > item[] > guid[] > $', 'rss > channel[] > item[] > guid[] > $ > isPermaLink', + 'rss > channel[] > atom:link', + 'rss > channel[] > atom:link[] > $', + 'rss > channel[] > atom:link[] > $ > rel', + 'rss > channel[] > atom:link[] > $ > type', + 'rss > channel[] > atom:link[] > $ > href', ]); From ddde7c25bcb36c078ed85021c71f7e25a75c68ef Mon Sep 17 00:00:00 2001 From: Mayur Date: Sat, 20 Jul 2024 18:50:55 +0530 Subject: [PATCH 200/299] feat: New fields added in UserJobEntity --- libs/dal/src/repositories/userjob/userjob.entity.ts | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/libs/dal/src/repositories/userjob/userjob.entity.ts b/libs/dal/src/repositories/userjob/userjob.entity.ts index 15928704f..4d3553988 100644 --- a/libs/dal/src/repositories/userjob/userjob.entity.ts +++ b/libs/dal/src/repositories/userjob/userjob.entity.ts @@ -8,4 +8,14 @@ export class UserJobEntity { cron: string; headings: string[]; + + extra: string; + + userId: string; + + customRecordFormat: string; + + customChunkFormat: string; + + customSchema: string; } From c4a8a7989dbd109008fa0ca7b177be8c3f323a5e Mon Sep 17 00:00:00 2001 From: Mayur Date: Sat, 20 Jul 2024 18:51:49 +0530 Subject: [PATCH 201/299] feat: New fields added in userJobSchema --- libs/dal/src/repositories/userjob/userjob.schema.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/libs/dal/src/repositories/userjob/userjob.schema.ts b/libs/dal/src/repositories/userjob/userjob.schema.ts index 8cc363af7..005001240 100644 --- a/libs/dal/src/repositories/userjob/userjob.schema.ts +++ b/libs/dal/src/repositories/userjob/userjob.schema.ts @@ -17,6 +17,11 @@ const userJobSchema = new Schema( headings: { type: [Schema.Types.String], }, + extra: String, + userId: String, + customRecordFormat: String, + customChunkFormat: String, + customSchema: String, }, { ...schemaOptions } ); From 6f67881eaf1e4d1f5cf7bb80ff055d1576a418b0 Mon Sep 17 00:00:00 2001 From: Mayur Date: Sat, 20 Jul 2024 18:52:53 +0530 Subject: [PATCH 202/299] feat: Streamlined usecase GetJobMapping --- .../usecase/get-jobmapping/get-jobmapping.usecase.ts | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/apps/api/src/app/import-jobs/usecase/get-jobmapping/get-jobmapping.usecase.ts b/apps/api/src/app/import-jobs/usecase/get-jobmapping/get-jobmapping.usecase.ts index 3c2c553f7..28c0fb125 100644 --- a/apps/api/src/app/import-jobs/usecase/get-jobmapping/get-jobmapping.usecase.ts +++ b/apps/api/src/app/import-jobs/usecase/get-jobmapping/get-jobmapping.usecase.ts @@ -1,13 +1,18 @@ import { Injectable } from '@nestjs/common'; -import { ColumnRepository, UserJobRepository, JobMappingEntity } from '@impler/dal'; +import { ColumnRepository, UserJobRepository, JobMappingEntity, JobMappingRepository } from '@impler/dal'; @Injectable() export class GetJobMapping { constructor( private readonly userJobRepository: UserJobRepository, - private readonly columnRepository: ColumnRepository + private readonly columnRepository: ColumnRepository, + private readonly jobMappingRepository: JobMappingRepository ) {} + async getUserJobs(userId: string) { + return this.userJobRepository.find({ userId }); + } + async execute(_jobId: string): Promise { const userJob = await this.userJobRepository.findOne({ _id: _jobId }); @@ -15,6 +20,7 @@ export class GetJobMapping { throw new Error('User job not found'); } + await this.jobMappingRepository.find({ _id: _jobId }); const columns = await this.columnRepository.find({ _templateId: userJob._templateId }); const jobsMapping = columns.map((column) => ({ key: column.key, From 77d4c2bffe5a78ad5e9c41093eb27d610d220aa0 Mon Sep 17 00:00:00 2001 From: Mayur Date: Sat, 20 Jul 2024 18:56:50 +0530 Subject: [PATCH 203/299] feat: Added files package.json and pnpm-lock.yaml --- apps/api/package.json | 2 ++ apps/queue-manager/package.json | 3 ++- package.json | 1 - pnpm-lock.yaml | 38 ++++++++++++++++++++++++++++++++- 4 files changed, 41 insertions(+), 3 deletions(-) diff --git a/apps/api/package.json b/apps/api/package.json index 6ae8fa598..0a1553a43 100644 --- a/apps/api/package.json +++ b/apps/api/package.json @@ -30,6 +30,7 @@ "@nestjs/platform-express": "^9.1.2", "@nestjs/platform-socket.io": "9.4.3", "@nestjs/platform-ws": "9.4.3", + "@nestjs/schedule": "^4.1.0", "@nestjs/swagger": "^6.1.2", "@nestjs/terminus": "^9.1.3", "@nestjs/websockets": "9.4.3", @@ -45,6 +46,7 @@ "class-validator": "^0.14.0", "compression": "^1.7.4", "cookie-parser": "^1.4.6", + "cron": "^3.1.7", "date-fns": "^2.30.0", "dayjs": "^1.11.11", "dotenv": "^16.0.2", diff --git a/apps/queue-manager/package.json b/apps/queue-manager/package.json index 0052a98c6..5129a12b4 100644 --- a/apps/queue-manager/package.json +++ b/apps/queue-manager/package.json @@ -21,7 +21,8 @@ "@sentry/node": "^7.112.2", "axios": "1.6.2", "dotenv": "^16.0.2", - "envalid": "^7.3.1" + "envalid": "^7.3.1", + "xml2js": "^0.6.2" }, "devDependencies": { "@types/node": "^18.7.18", diff --git a/package.json b/package.json index 5851640f1..bd0178fa8 100644 --- a/package.json +++ b/package.json @@ -46,7 +46,6 @@ "release:preminor": "lerna version preminor --no-push", "release:prerelease": "lerna version prerelease --no-push", "publish": "pnpm build && pnpm -r --if-present publish", - "postinstall": "pnpm build", "prepare": "husky install" }, "keywords": [ diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 80bad5391..a1e458fc0 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -123,6 +123,9 @@ importers: '@nestjs/platform-ws': specifier: 9.4.3 version: 9.4.3(@nestjs/common@9.4.3)(@nestjs/websockets@9.4.3)(rxjs@7.8.1) + '@nestjs/schedule': + specifier: ^4.1.0 + version: 4.1.0(@nestjs/common@9.4.3)(@nestjs/core@9.4.3) '@nestjs/swagger': specifier: ^6.1.2 version: 6.3.0(@nestjs/common@9.4.3)(@nestjs/core@9.4.3)(class-transformer@0.5.1)(class-validator@0.14.1)(reflect-metadata@0.1.14) @@ -168,6 +171,9 @@ importers: cookie-parser: specifier: ^1.4.6 version: 1.4.6 + cron: + specifier: ^3.1.7 + version: 3.1.7 date-fns: specifier: ^2.30.0 version: 2.30.0 @@ -392,6 +398,9 @@ importers: envalid: specifier: ^7.3.1 version: 7.3.1 + xml2js: + specifier: ^0.6.2 + version: 0.6.2 devDependencies: '@types/node': specifier: ^18.7.18 @@ -4818,6 +4827,18 @@ packages: - utf-8-validate dev: false + /@nestjs/schedule@4.1.0(@nestjs/common@9.4.3)(@nestjs/core@9.4.3): + resolution: {integrity: sha512-WEc96WTXZW+VI/Ng+uBpiBUwm6TWtAbQ4RKWkfbmzKvmbRGzA/9k/UyAWDS9k0pp+ZcbC+MaZQtt7TjQHrwX6g==} + peerDependencies: + '@nestjs/common': ^8.0.0 || ^9.0.0 || ^10.0.0 + '@nestjs/core': ^8.0.0 || ^9.0.0 || ^10.0.0 + dependencies: + '@nestjs/common': 9.4.3(class-transformer@0.5.1)(class-validator@0.14.1)(reflect-metadata@0.1.14)(rxjs@7.8.1) + '@nestjs/core': 9.4.3(@nestjs/common@9.4.3)(@nestjs/platform-express@9.4.3)(@nestjs/websockets@9.4.3)(reflect-metadata@0.1.14)(rxjs@7.8.1) + cron: 3.1.7 + uuid: 10.0.0 + dev: false + /@nestjs/schematics@9.2.0(chokidar@3.5.3)(typescript@4.9.5): resolution: {integrity: sha512-wHpNJDPzM6XtZUOB3gW0J6mkFCSJilzCM3XrHI1o0C8vZmFE1snbmkIXNyoi1eV0Nxh1BMymcgz5vIMJgQtTqw==} peerDependencies: @@ -8764,6 +8785,10 @@ packages: /@types/lodash@4.17.6: resolution: {integrity: sha512-OpXEVoCKSS3lQqjx9GGGOapBeuW5eUboYHRlHP9urXPX25IKZ6AnP5ZRxtVf63iieUbsHxLn8NQ5Nlftc6yzAA==} + /@types/luxon@3.4.2: + resolution: {integrity: sha512-TifLZlFudklWlMBfhubvgqTXRzLDI5pCbGa4P8a3wPyUQSW+1xQ5eDsreP9DWHX3tjq1ke96uYG/nwundroWcA==} + dev: false + /@types/mdast@3.0.15: resolution: {integrity: sha512-LnwD+mUEfxWMa1QpDraczIn6k0Ee3SMicuYSSzS6ZYl2gKS09EClnJYGd8Du6rfc5r/GZEk5o1mRb8TaTj03sQ==} dependencies: @@ -11966,6 +11991,13 @@ packages: /create-require@1.1.1: resolution: {integrity: sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==} + /cron@3.1.7: + resolution: {integrity: sha512-tlBg7ARsAMQLzgwqVxy8AZl/qlTc5nibqYwtNGoCrd+cV+ugI+tvZC1oT/8dFH8W455YrywGykx/KMmAqOr7Jw==} + dependencies: + '@types/luxon': 3.4.2 + luxon: 3.4.4 + dev: false + /cross-env@7.0.3: resolution: {integrity: sha512-+/HKd6EgcQCJGh2PSjZuUitQBQynKor4wrFbRg4DtAgS1aWO+gU52xpH7M9ScGgXSYmAVS9bIJ8EzuaGw0oNAw==} engines: {node: '>=10.14', npm: '>=6', yarn: '>=1'} @@ -17708,6 +17740,11 @@ packages: resolution: {integrity: sha512-zTU3DaZaF3Rt9rhN3uBMGQD3dD2/vFQqnvZCDv4dl5iOzq2IZQqTxu90r4E5J+nP70J3ilqVCrbho2eWaeW8Ow==} dev: true + /luxon@3.4.4: + resolution: {integrity: sha512-zobTr7akeGHnv7eBOXcRgMeCP6+uyYsczwmeRCauvpvaAltgNyTbLH/+VaEAPUeWBT+1GuNmz4wC/6jtQzbbVA==} + engines: {node: '>=12'} + dev: false + /macos-release@2.5.1: resolution: {integrity: sha512-DXqXhEM7gW59OjZO8NIjBCz9AQ1BEMrfiOAl4AYByHCtVHRF4KoGNO8mqQeM8lRCtQe/UnJ4imO/d2HdkKsd+A==} engines: {node: '>=6'} @@ -24543,7 +24580,6 @@ packages: /uuid@10.0.0: resolution: {integrity: sha512-8XkAphELsDnEGrDxUOHB3RGvXz6TeuYSGEZBOjtTtPm2lwhGBjLgOzLHB63IUWfBpNucQjND6d3AOudO+H3RWQ==} hasBin: true - dev: true /uuid@3.4.0: resolution: {integrity: sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==} From 4a28df742f937d647c26fb88d0a98128f69684ad Mon Sep 17 00:00:00 2001 From: Mayur Date: Mon, 22 Jul 2024 11:00:10 +0530 Subject: [PATCH 204/299] feat: Added CronJobService to pause, start and delete cron job --- .../app/shared/services/cronjob.service.ts | 52 +++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 apps/api/src/app/shared/services/cronjob.service.ts diff --git a/apps/api/src/app/shared/services/cronjob.service.ts b/apps/api/src/app/shared/services/cronjob.service.ts new file mode 100644 index 000000000..ea788f8dd --- /dev/null +++ b/apps/api/src/app/shared/services/cronjob.service.ts @@ -0,0 +1,52 @@ +import { Injectable } from '@nestjs/common'; +import { SchedulerRegistry } from '@nestjs/schedule'; +import { CronJob } from 'cron'; +import { QueueService } from '@shared/services/queue.service'; +import { UserJobRepository } from '@impler/dal'; +import { QueuesEnum } from '@impler/shared'; + +@Injectable() +export class CronJobService { + constructor( + private readonly schedulerRegistry: SchedulerRegistry, + private readonly userJobRepository: UserJobRepository, + private queueService: QueueService + ) {} + + async pauseJob(jobId: string) { + const job = this.schedulerRegistry.getCronJob(`${jobId}_rss_import`); + job.stop(); + + return { message: `Job ${jobId} paused successfully` }; + } + + async startJob(userId: string, jobId: string) { + const userJob = await this.userJobRepository.findOne({ _id: jobId, userId }); + if (!userJob) { + throw new Error(`Job ${jobId} not found for user ${userId}`); + } + + const cronExpression = userJob.cron; + const job = new CronJob(cronExpression, () => this.triggerImportJob(jobId)); + this.schedulerRegistry.addCronJob(`${jobId}_rss_import`, job); + job.start(); + + return { message: `Job ${jobId} started successfully with schedule: ${cronExpression}` }; + } + + async deleteJob(userId: string, jobId: string) { + await this.userJobRepository.delete({ _id: jobId, userId }); + this.schedulerRegistry.deleteCronJob(`${jobId}_rss_import`); + + return { message: `Job ${jobId} deleted successfully` }; + } + + async triggerImportJob(_jobId: string) { + try { + this.queueService.publishToQueue(QueuesEnum.SEND_RSS_XML_DATA, { _jobId }); + console.log(`Import job triggered for job ${_jobId}`); + } catch (error) { + console.error(`Error triggering import job for ${_jobId}:`, error); + } + } +} From b0e53a7ea68fd844a5740f8304618a40453db276 Mon Sep 17 00:00:00 2001 From: Mayur Date: Mon, 22 Jul 2024 16:53:05 +0530 Subject: [PATCH 205/299] feat: Exported member import-job-history --- libs/dal/src/index.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/libs/dal/src/index.ts b/libs/dal/src/index.ts index 73713e444..4e1ae4180 100644 --- a/libs/dal/src/index.ts +++ b/libs/dal/src/index.ts @@ -16,3 +16,4 @@ export * from './repositories/webhook-destination'; export * from './repositories/bubble-destination'; export * from './repositories/jobmapping'; export * from './repositories/userjob'; +export * from './repositories/import-job-history'; From 8db543d6ff87ba6c409722f3872ae20cad43fcbe Mon Sep 17 00:00:00 2001 From: Mayur Date: Mon, 22 Jul 2024 16:53:58 +0530 Subject: [PATCH 206/299] feat: Streamlined and refactored the RSSService class --- .../src/app/shared/services/rss.service.ts | 103 ++++++++++++++++++ 1 file changed, 103 insertions(+) create mode 100644 apps/api/src/app/shared/services/rss.service.ts diff --git a/apps/api/src/app/shared/services/rss.service.ts b/apps/api/src/app/shared/services/rss.service.ts new file mode 100644 index 000000000..2b83932bf --- /dev/null +++ b/apps/api/src/app/shared/services/rss.service.ts @@ -0,0 +1,103 @@ +import axios from 'axios'; +import { parseStringPromise } from 'xml2js'; + +export class RSSService { + public getHeadings(xmlData: string): string[] { + const headings: string[] = []; + this.extractHeadings(xmlData, '', headings); + + return headings; + } + async parseXML(xmlContent: string) { + return await parseStringPromise(xmlContent); + } + + extractHeadings(node: any, path: string, headings: string[]): void { + if (typeof node === 'object') { + for (const key in node) { + const newPath = path ? `${path} > ${key}` : `$ > ${key}`; + headings.push(newPath); + if (Array.isArray(node[key])) { + node[key].forEach((item: any) => this.extractHeadings(item, newPath, headings)); + } else { + this.extractHeadings(node[key], newPath, headings); + } + } + } + } + + public getDataByHeading(xmlData: string, heading: string): any[] { + const pathArray = heading.split(' > ').map((part) => part.replace('[]', '')); + + return this.extractDataByPath(xmlData, pathArray); + } + + private extractDataByPath(node: any, pathArray: string[]): any[] { + if (pathArray.length === 0) return [node]; + + const [current, ...rest] = pathArray; + const result: any[] = []; + + if (node[current]) { + if (Array.isArray(node[current])) { + node[current].forEach((item: any) => { + result.push(...this.extractDataByPath(item, rest)); + }); + } else { + result.push(...this.extractDataByPath(node[current], rest)); + } + } + + return result; + } + + async parseRssFeed(url: string): Promise<{ + rssKeyHeading: string[]; + xmlData: Record; + }> { + try { + const xmlData = await this.parseXmlFromUrl(url); + const rssKeyHeading = this.printKeys(xmlData); + + return { rssKeyHeading, xmlData }; + } catch (error) { + throw error; + } + } + printKeys(obj: Record, prefix = '', result: Set = new Set()): string[] { + if (Array.isArray(obj)) { + obj.forEach((item) => { + const newPrefix = `${prefix}[]`; + this.printKeys(item, newPrefix, result); + }); + } else if (typeof obj === 'object' && obj !== null) { + Object.keys(obj).forEach((key) => { + const newPrefix = prefix ? `${prefix} > ${key}` : key; + result.add(newPrefix); + this.printKeys(obj[key], newPrefix, result); + }); + } + + return Array.from(result); + } + async parseXmlFromUrl(url: string): Promise> { + try { + const response = await axios.get(url); + const xmlData = response.data; + + return this.parseXML(xmlData); + } catch (error) { + throw error; + } + } + async getMimeType(url: string): Promise { + try { + const response = await axios.head(url); + const mimeType = response.headers['content-type'] || null; + + return mimeType?.split(';')[0] || null; + } catch (error) { + return null; + } + } +} From a49519d7d5c6a1b504030c4faf6649617a460244 Mon Sep 17 00:00:00 2001 From: Mayur Date: Mon, 22 Jul 2024 16:54:48 +0530 Subject: [PATCH 207/299] feat: Entity ImportJobHistory created --- .../import-job-history.entity.ts | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 libs/dal/src/repositories/import-job-history/import-job-history.entity.ts diff --git a/libs/dal/src/repositories/import-job-history/import-job-history.entity.ts b/libs/dal/src/repositories/import-job-history/import-job-history.entity.ts new file mode 100644 index 000000000..44181f213 --- /dev/null +++ b/libs/dal/src/repositories/import-job-history/import-job-history.entity.ts @@ -0,0 +1,17 @@ +export class ImportJobHistoryEntity { + _id: string; + + _jobId: string; + + validFileId: string; + + invalidFileId: string; + + allDataFileId: string; + + fetchStatus: string; + + status: string; + + runOn: Date; +} From a021759c9618164b1b028c1f92c50cf77700b8f7 Mon Sep 17 00:00:00 2001 From: Mayur Date: Mon, 22 Jul 2024 16:55:27 +0530 Subject: [PATCH 208/299] feat: Repository for ImportJobHistoryRepository added --- .../import-job-history/import-job-history.repository.ts | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 libs/dal/src/repositories/import-job-history/import-job-history.repository.ts diff --git a/libs/dal/src/repositories/import-job-history/import-job-history.repository.ts b/libs/dal/src/repositories/import-job-history/import-job-history.repository.ts new file mode 100644 index 000000000..aa93b6751 --- /dev/null +++ b/libs/dal/src/repositories/import-job-history/import-job-history.repository.ts @@ -0,0 +1,9 @@ +import { BaseRepository } from '../base-repository'; +import { ImportJobHistoryEntity } from './import-job-history.entity'; +import { ImportJobHistory } from './import-job-history.schema'; + +export class ImportJobHistoryRepository extends BaseRepository { + constructor() { + super(ImportJobHistory, ImportJobHistoryEntity); + } +} From 8b47b9c8d3b114a81541e8b54d7add98e23de8d7 Mon Sep 17 00:00:00 2001 From: Mayur Date: Mon, 22 Jul 2024 16:56:40 +0530 Subject: [PATCH 209/299] feat: Added importJobHistorySchema --- .../import-job-history.schema.ts | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 libs/dal/src/repositories/import-job-history/import-job-history.schema.ts diff --git a/libs/dal/src/repositories/import-job-history/import-job-history.schema.ts b/libs/dal/src/repositories/import-job-history/import-job-history.schema.ts new file mode 100644 index 000000000..29caa1cb4 --- /dev/null +++ b/libs/dal/src/repositories/import-job-history/import-job-history.schema.ts @@ -0,0 +1,42 @@ +import { model, models, Schema } from 'mongoose'; +import { schemaOptions } from '../schema-default.options'; +import { ImportJobHistoryEntity } from './import-job-history.entity'; + +const importJobHistorySchema = new Schema( + { + _fileId: { + type: Schema.Types.ObjectId, + ref: 'File', + }, + validFileId: { + type: Schema.Types.String, + }, + invalidFileId: { + type: Schema.Types.String, + }, + allDataFileId: { + type: Schema.Types.String, + }, + fetchStatus: { + type: Schema.Types.String, + }, + status: { + type: Schema.Types.String, + }, + runOn: { + type: Schema.Types.Date, + }, + _jobId: { + type: Schema.Types.ObjectId, + ref: 'UserJob', + }, + }, + { ...schemaOptions } +); + +interface IImportJobHistory extends ImportJobHistoryEntity, Document { + _id: never; +} + +export const ImportJobHistory = + models.ImportJobHistory || model('ImportJobHistory', importJobHistorySchema); From 4cf4b5655236ee3da5e828cafb454c63d8563429 Mon Sep 17 00:00:00 2001 From: Mayur Date: Mon, 22 Jul 2024 16:58:00 +0530 Subject: [PATCH 210/299] feat: Exported members entity, repository and schema of import-job --- libs/dal/src/repositories/import-job-history/index.ts | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 libs/dal/src/repositories/import-job-history/index.ts diff --git a/libs/dal/src/repositories/import-job-history/index.ts b/libs/dal/src/repositories/import-job-history/index.ts new file mode 100644 index 000000000..83c9c44c4 --- /dev/null +++ b/libs/dal/src/repositories/import-job-history/index.ts @@ -0,0 +1,3 @@ +export * from './import-job-history.entity'; +export * from './import-job-history.repository'; +export * from './import-job-history.schema'; From 11881f955163c3adc73638aaba43403abed73894 Mon Sep 17 00:00:00 2001 From: Mayur Date: Mon, 22 Jul 2024 19:16:27 +0530 Subject: [PATCH 211/299] feat: Exported member send-import-job-data.consumer --- apps/queue-manager/src/consumers/index.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/queue-manager/src/consumers/index.ts b/apps/queue-manager/src/consumers/index.ts index abd4f8579..054f1cd12 100644 --- a/apps/queue-manager/src/consumers/index.ts +++ b/apps/queue-manager/src/consumers/index.ts @@ -2,3 +2,4 @@ export * from './send-webhook-data.consumer'; export * from './end-import.consumer'; export * from './send-bubble-data.consumer'; export * from './import-job-data.consumer'; +export * from './send-import-job-data.consumer'; From 11aa563800f93db280b1fb12afc27dbc54574b9c Mon Sep 17 00:00:00 2001 From: Mayur Date: Mon, 22 Jul 2024 19:19:56 +0530 Subject: [PATCH 212/299] feat: Exported members import-job.types and user-job.types from import-job and user-job respectively --- libs/shared/src/types/index.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/libs/shared/src/types/index.ts b/libs/shared/src/types/index.ts index 3f0954c99..26ef459a3 100644 --- a/libs/shared/src/types/index.ts +++ b/libs/shared/src/types/index.ts @@ -10,3 +10,5 @@ export * from './destination/destination.types'; export * from './review'; export * from './subscription'; export * from './template'; +export * from './import-job/import-job.types'; +export * from './user-job/user-job.types'; From ab88047bc36f9e87ce6a34fa44d6fa6a3174bd24 Mon Sep 17 00:00:00 2001 From: Mayur Date: Mon, 22 Jul 2024 19:25:57 +0530 Subject: [PATCH 213/299] feat: Added SendImportJobDataConsumer --- apps/queue-manager/src/bootstrap.ts | 31 ++++++++++++++++++++++++----- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/apps/queue-manager/src/bootstrap.ts b/apps/queue-manager/src/bootstrap.ts index 102fe1202..eccfeafc0 100644 --- a/apps/queue-manager/src/bootstrap.ts +++ b/apps/queue-manager/src/bootstrap.ts @@ -6,7 +6,13 @@ import { IAmqpConnectionManager } from 'amqp-connection-manager/dist/esm/AmqpCon import { QueuesEnum } from '@impler/shared'; import { DalService } from '@impler/dal'; -import { SendWebhookDataConsumer, EndImportConsumer, SendBubbleDataConsumer, ImportJobDataConsumer } from './consumers'; +import { + SendWebhookDataConsumer, + EndImportConsumer, + SendBubbleDataConsumer, + GetImportJobDataConsumer, + SendImportJobDataConsumer, +} from './consumers'; let connection: IAmqpConnectionManager, chanelWrapper: ChannelWrapper; @@ -39,7 +45,8 @@ export async function bootstrap() { const endImportConsumer = new EndImportConsumer(); const sendBubbleDataConsumer = new SendBubbleDataConsumer(); const sendWebhookdataConsumer = new SendWebhookDataConsumer(); - const importJobbDataConsumer = new ImportJobDataConsumer(); + const getImportJobbDataConsumer = new GetImportJobDataConsumer(); + const sendImportJobDataConsumer = new SendImportJobDataConsumer(); // add queues to channel chanelWrapper.addSetup((channel) => { @@ -60,12 +67,26 @@ export async function bootstrap() { channel.consume(QueuesEnum.SEND_BUBBLE_DATA, sendBubbleDataConsumer.message.bind(sendBubbleDataConsumer), { noAck: true, }), - channel.assertQueue(QueuesEnum.SEND_RSS_XML_DATA, { + channel.assertQueue(QueuesEnum.GET_IMPORT_JOB_DATA, { durable: false, }), - channel.consume(QueuesEnum.SEND_RSS_XML_DATA, importJobbDataConsumer.message.bind(importJobbDataConsumer), { - noAck: true, + channel.consume( + QueuesEnum.GET_IMPORT_JOB_DATA, + getImportJobbDataConsumer.message.bind(getImportJobbDataConsumer), + { + noAck: true, + } + ), + channel.assertQueue(QueuesEnum.SEND_IMPORT_JOB_DATA, { + durable: false, }), + channel.consume( + QueuesEnum.SEND_IMPORT_JOB_DATA, + sendImportJobDataConsumer.message.bind(sendImportJobDataConsumer), + { + noAck: true, + } + ), ]); }); } From 72ebd371a838b407e7da1d187ea8cd177610f47b Mon Sep 17 00:00:00 2001 From: Mayur Date: Mon, 22 Jul 2024 19:30:49 +0530 Subject: [PATCH 214/299] feat: Added types SendWebhookCachedData,SendImportJobData and renamed SendWebhookCachedData --- libs/shared/src/types/upload/upload.types.ts | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/libs/shared/src/types/upload/upload.types.ts b/libs/shared/src/types/upload/upload.types.ts index 5434528b9..83cbede80 100644 --- a/libs/shared/src/types/upload/upload.types.ts +++ b/libs/shared/src/types/upload/upload.types.ts @@ -40,10 +40,11 @@ export enum QueuesEnum { 'SEND_WEBHOOK_DATA' = 'SEND_WEBHOOK_DATA', 'SEND_BUBBLE_DATA' = 'SEND_BUBBLE_DATA', 'END_IMPORT' = 'END_IMPORT', - 'SEND_RSS_XML_DATA' = 'SEND_RSS_XML_DATA', + 'GET_IMPORT_JOB_DATA' = 'GET_IMPORT_JOB_DATA', + SEND_IMPORT_JOB_DATA = 'SEND_IMPORT_JOB_DATA', } -export type SendWebhookCachedData = { +export type CommonCachedData = { page: number; callbackUrl: string; chunkSize: number; @@ -53,13 +54,18 @@ export type SendWebhookCachedData = { authHeaderValue: string; _templateId: string; allDataFilePath?: string; - fileName: string; recordFormat?: string; chunkFormat?: string; defaultValues: string; multiSelectHeadings?: string[]; }; +export type SendWebhookCachedData = { + fileName: string; +} & CommonCachedData; + +export type SendImportJobCachedData = CommonCachedData; + export type SendBubbleCachedData = { page: number; chunkSize: number; @@ -81,6 +87,11 @@ export type SendWebhookData = { cache?: SendWebhookCachedData; }; +export type SendImportJobData = { + importJobHistoryId: string; + cache?: SendImportJobCachedData; +}; + export type SendRSSXMLData = { _jobId: string; }; From 982e45d4a18abd58b29a4a77236fd9570b108060 Mon Sep 17 00:00:00 2001 From: Mayur Date: Mon, 22 Jul 2024 19:31:25 +0530 Subject: [PATCH 215/299] feat: Added enum UserJobImportStatusEnum --- libs/shared/src/types/user-job/user-job.types.ts | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 libs/shared/src/types/user-job/user-job.types.ts diff --git a/libs/shared/src/types/user-job/user-job.types.ts b/libs/shared/src/types/user-job/user-job.types.ts new file mode 100644 index 000000000..ded759ba0 --- /dev/null +++ b/libs/shared/src/types/user-job/user-job.types.ts @@ -0,0 +1,4 @@ +export enum UserJobImportStatusEnum { + 'PAUSED' = 'PAUSED', + 'RUNNING' = 'RUNNING', +} From becfc355cd63360621a7e1a370cf6f7176ca2b82 Mon Sep 17 00:00:00 2001 From: Mayur Date: Mon, 22 Jul 2024 19:31:55 +0530 Subject: [PATCH 216/299] feat: Exported enum ImportJobHistoryStatusEnum --- libs/shared/src/types/import-job/import-job.types.ts | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 libs/shared/src/types/import-job/import-job.types.ts diff --git a/libs/shared/src/types/import-job/import-job.types.ts b/libs/shared/src/types/import-job/import-job.types.ts new file mode 100644 index 000000000..b326a46c0 --- /dev/null +++ b/libs/shared/src/types/import-job/import-job.types.ts @@ -0,0 +1,4 @@ +export enum ImportJobHistoryStatusEnum { + 'PROCESSING' = 'Processing', + 'COMPLETED' = 'Completed', +} From 404b5ead7220c330c8550427700f039be00ec643 Mon Sep 17 00:00:00 2001 From: Mayur Date: Mon, 22 Jul 2024 19:32:30 +0530 Subject: [PATCH 217/299] feat: Added status in UserJobEntity --- libs/dal/src/repositories/userjob/userjob.entity.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/libs/dal/src/repositories/userjob/userjob.entity.ts b/libs/dal/src/repositories/userjob/userjob.entity.ts index 4d3553988..13e4750b8 100644 --- a/libs/dal/src/repositories/userjob/userjob.entity.ts +++ b/libs/dal/src/repositories/userjob/userjob.entity.ts @@ -13,6 +13,8 @@ export class UserJobEntity { userId: string; + status: string; + customRecordFormat: string; customChunkFormat: string; From 76e529adf36d6d2222fc17494d6a5fc5e478b845 Mon Sep 17 00:00:00 2001 From: Mayur Date: Mon, 22 Jul 2024 19:35:34 +0530 Subject: [PATCH 218/299] feat: Removed validFileId,invalidFileId and renamed _fileId to and allDataFileId --- .../import-job-history.schema.ts | 24 +++++++------------ 1 file changed, 8 insertions(+), 16 deletions(-) diff --git a/libs/dal/src/repositories/import-job-history/import-job-history.schema.ts b/libs/dal/src/repositories/import-job-history/import-job-history.schema.ts index 29caa1cb4..37b172230 100644 --- a/libs/dal/src/repositories/import-job-history/import-job-history.schema.ts +++ b/libs/dal/src/repositories/import-job-history/import-job-history.schema.ts @@ -1,20 +1,14 @@ -import { model, models, Schema } from 'mongoose'; +import { model, models, Schema, Model } from 'mongoose'; import { schemaOptions } from '../schema-default.options'; import { ImportJobHistoryEntity } from './import-job-history.entity'; const importJobHistorySchema = new Schema( { - _fileId: { + _jobId: { type: Schema.Types.ObjectId, - ref: 'File', - }, - validFileId: { - type: Schema.Types.String, - }, - invalidFileId: { - type: Schema.Types.String, + ref: 'UserJob', }, - allDataFileId: { + allDataFilePath: { type: Schema.Types.String, }, fetchStatus: { @@ -25,18 +19,16 @@ const importJobHistorySchema = new Schema( }, runOn: { type: Schema.Types.Date, - }, - _jobId: { - type: Schema.Types.ObjectId, - ref: 'UserJob', + default: Date.now, }, }, { ...schemaOptions } ); -interface IImportJobHistory extends ImportJobHistoryEntity, Document { +interface IImportJobHistoryDocument extends ImportJobHistoryEntity, Document { _id: never; } export const ImportJobHistory = - models.ImportJobHistory || model('ImportJobHistory', importJobHistorySchema); + (models.ImportJobHistory as Model) || + model('ImportJobHistory', importJobHistorySchema); From 33826cdfe0bb8dec1dabf6dc5689e14a9c2aa14c Mon Sep 17 00:00:00 2001 From: Mayur Date: Mon, 22 Jul 2024 19:36:20 +0530 Subject: [PATCH 219/299] feat: Added method getHistoryWithJob --- .../import-job-history/import-job-history.repository.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/libs/dal/src/repositories/import-job-history/import-job-history.repository.ts b/libs/dal/src/repositories/import-job-history/import-job-history.repository.ts index aa93b6751..ba96d6beb 100644 --- a/libs/dal/src/repositories/import-job-history/import-job-history.repository.ts +++ b/libs/dal/src/repositories/import-job-history/import-job-history.repository.ts @@ -6,4 +6,7 @@ export class ImportJobHistoryRepository extends BaseRepository { + return await ImportJobHistory.findById(importJobHistoryId).populate('_jobId', fields.join(' ')); + } } From d7a2c15560ab847777908288757f31180b320803 Mon Sep 17 00:00:00 2001 From: Mayur Date: Mon, 22 Jul 2024 19:38:52 +0530 Subject: [PATCH 220/299] feat: Removed validFileId,invalidFileId and renamed allDataFileId to allDataFilePath --- .../import-job-history/import-job-history.entity.ts | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/libs/dal/src/repositories/import-job-history/import-job-history.entity.ts b/libs/dal/src/repositories/import-job-history/import-job-history.entity.ts index 44181f213..faf5759ae 100644 --- a/libs/dal/src/repositories/import-job-history/import-job-history.entity.ts +++ b/libs/dal/src/repositories/import-job-history/import-job-history.entity.ts @@ -3,11 +3,7 @@ export class ImportJobHistoryEntity { _jobId: string; - validFileId: string; - - invalidFileId: string; - - allDataFileId: string; + allDataFilePath: string; fetchStatus: string; From 6f433e9a0d000702ccc975dedf2e4f16f8dd704b Mon Sep 17 00:00:00 2001 From: Mayur Date: Mon, 22 Jul 2024 19:40:35 +0530 Subject: [PATCH 221/299] feat: Renamed SEND_RSS_XML_DATA to GET_IMPORT_JOB_DATA --- apps/api/src/app/shared/services/queue.service.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/api/src/app/shared/services/queue.service.ts b/apps/api/src/app/shared/services/queue.service.ts index 4ecfb8c79..9d6e6110e 100644 --- a/apps/api/src/app/shared/services/queue.service.ts +++ b/apps/api/src/app/shared/services/queue.service.ts @@ -30,7 +30,7 @@ export class QueueService { publishToQueue(queueName: QueuesEnum.END_IMPORT, data: EndImportData): void; publishToQueue(queueName: QueuesEnum.SEND_WEBHOOK_DATA, data: SendWebhookData): void; - publishToQueue(queueName: QueuesEnum.SEND_RSS_XML_DATA, data: SendRSSXMLData): void; + publishToQueue(queueName: QueuesEnum.GET_IMPORT_JOB_DATA, data: SendRSSXMLData): void; async publishToQueue(queueName: QueuesEnum, data: PublishToQueueData | SendRSSXMLData) { if (this.connection.isConnected()) { From e4c0dfef8d4e4317d15567b5e209e0c684affd81 Mon Sep 17 00:00:00 2001 From: Mayur Date: Tue, 23 Jul 2024 13:39:36 +0530 Subject: [PATCH 222/299] feat: Renamed userID to externalUserId --- libs/dal/src/repositories/userjob/userjob.entity.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/dal/src/repositories/userjob/userjob.entity.ts b/libs/dal/src/repositories/userjob/userjob.entity.ts index 13e4750b8..99e98969b 100644 --- a/libs/dal/src/repositories/userjob/userjob.entity.ts +++ b/libs/dal/src/repositories/userjob/userjob.entity.ts @@ -11,7 +11,7 @@ export class UserJobEntity { extra: string; - userId: string; + externalUserId: string; status: string; From 9f07bf5441acb0fbed0b5beb8c68d447e9bc8392 Mon Sep 17 00:00:00 2001 From: Mayur Date: Tue, 23 Jul 2024 13:41:05 +0530 Subject: [PATCH 223/299] feat: Added statuses MAPPING and SCHEDULING --- libs/shared/src/types/user-job/user-job.types.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/libs/shared/src/types/user-job/user-job.types.ts b/libs/shared/src/types/user-job/user-job.types.ts index ded759ba0..75117adac 100644 --- a/libs/shared/src/types/user-job/user-job.types.ts +++ b/libs/shared/src/types/user-job/user-job.types.ts @@ -1,4 +1,6 @@ export enum UserJobImportStatusEnum { 'PAUSED' = 'PAUSED', 'RUNNING' = 'RUNNING', + 'MAPPING' = 'MAPPING', + 'SCHEDULING' = 'SCHEDULING', } From 23a4432705eac2b594a9ae91461987144fbb9974 Mon Sep 17 00:00:00 2001 From: Mayur Date: Tue, 23 Jul 2024 17:00:38 +0530 Subject: [PATCH 224/299] feat: Routes for getting, pausing, starting and deleting userjob created --- .../app/import-jobs/import-jobs.controller.ts | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/apps/api/src/app/import-jobs/import-jobs.controller.ts b/apps/api/src/app/import-jobs/import-jobs.controller.ts index 29051700e..40a21f6d5 100644 --- a/apps/api/src/app/import-jobs/import-jobs.controller.ts +++ b/apps/api/src/app/import-jobs/import-jobs.controller.ts @@ -54,31 +54,31 @@ export class ImportJobsController { return this.updateUserJob.execute(_jobId, userJobData); } - @Get('/user/:id') + @Get('/user/:externalUserId') @ApiOperation({ summary: 'Get Import Jobs for a User' }) @UseGuards(JwtAuthGuard) - async getUserJobs(@Param('id') userId: string) { - return this.getJobMapping.getUserJobs(userId); + async getUserJobs(@Param('externalUserId') externalUserId: string) { + return this.getJobMapping.getUserJobs(externalUserId); } - @Put('/user/:id/:jobId/pause') + @Put('/user/:jobId/pause') @ApiOperation({ summary: 'Pause a Cron Job' }) @UseGuards(JwtAuthGuard) async pauseCronJob(@Param('jobId') jobId: string) { return this.cronJobService.pauseJob(jobId); } - @Put('/user/:id/:jobId/start') + @Put('/user/:externalUserId/:jobId/start') @ApiOperation({ summary: 'Start a Cron Job' }) @UseGuards(JwtAuthGuard) - async startCronJob(@Param('id') userId: string, @Param('jobId') jobId: string) { - return this.cronJobService.startJob(userId, jobId); + async startCronJob(@Param('externalUserId') externalUserId: string, @Param('jobId') jobId: string) { + return this.cronJobService.startJob(externalUserId, jobId); } - @Delete('/user/:id/:jobId/delete') + @Delete('/user/:externalUserId/:jobId/delete') @ApiOperation({ summary: 'Delete a Job' }) @UseGuards(JwtAuthGuard) - async deleteJob(@Param('id') userId: string, @Param('jobId') jobId: string) { - return this.cronJobService.deleteJob(userId, jobId); + async deleteJob(@Param('externalUserId') externalUserId: string, @Param('jobId') jobId: string) { + return this.cronJobService.deleteJob(externalUserId, jobId); } } From aef479e9a9d8ab489df1700527d9e2cb6f2bfb98 Mon Sep 17 00:00:00 2001 From: Mayur Date: Tue, 23 Jul 2024 17:45:24 +0530 Subject: [PATCH 225/299] feat: Streamlined and Refactored webhook data sending logic --- .../src/consumers/import-job-data.consumer.ts | 130 ++++++++++++++++++ 1 file changed, 130 insertions(+) create mode 100644 apps/queue-manager/src/consumers/import-job-data.consumer.ts diff --git a/apps/queue-manager/src/consumers/import-job-data.consumer.ts b/apps/queue-manager/src/consumers/import-job-data.consumer.ts new file mode 100644 index 000000000..fafb0410e --- /dev/null +++ b/apps/queue-manager/src/consumers/import-job-data.consumer.ts @@ -0,0 +1,130 @@ +import axios from 'axios'; +import { Injectable } from '@nestjs/common'; +import { parseStringPromise } from 'xml2js'; + +import { FileMimeTypesEnum, FileNameService, ImportJobHistoryStatusEnum, QueuesEnum } from '@impler/shared'; +import { UserJobRepository, JobMappingRepository, ImportJobHistoryRepository, CommonRepository } from '@impler/dal'; +import { BaseConsumer } from './base.consumer'; +import { publishToQueue } from '../bootstrap'; +import { StorageService } from 'libs/shared/dist/services/storage'; +import { getStorageServiceClass } from '../helpers/storage.helper'; + +@Injectable() +export class GetImportJobDataConsumer extends BaseConsumer { + private commonRepository: CommonRepository = new CommonRepository(); + private userJobRepository: UserJobRepository = new UserJobRepository(); + private importJobHistoryRepository: ImportJobHistoryRepository = new ImportJobHistoryRepository(); + private jobMappingRepository: JobMappingRepository = new JobMappingRepository(); + + private storageService: StorageService = getStorageServiceClass(); + private fileNameService: FileNameService = new FileNameService(); + + async message(message: { content: string }) { + const data = JSON.parse(message.content) as { _jobId: string }; + const importJobHistoryId = this.commonRepository.generateMongoId().toString(); + const importedData = await this.getJobImportedData(data._jobId); + await this.convertRecordsToJsonFile(importJobHistoryId, importedData); + await this.importJobHistoryRepository.create({ + _id: importJobHistoryId, + _jobId: data._jobId, + allDataFilePath: this.fileNameService.getAllJsonDataFilePath(importJobHistoryId), + status: ImportJobHistoryStatusEnum.PROCESSING, + }); + publishToQueue(QueuesEnum.SEND_IMPORT_JOB_DATA, { importJobHistoryId }); + } + + getXMLJsonValueByPath(obj: Record, path: string | undefined): string | number | undefined | any { + if (!path) { + return undefined; + } + + const keys = path.split('>').map((key) => key.trim()); + let current: any = obj; + + for (const key of keys) { + if (current === undefined) return undefined; + + const isArray = key.endsWith('[]'); + const actualKey = isArray ? key.slice(0, -2) : key; + + if (Array.isArray(current)) { + current = current.flatMap((item) => (item ? item[actualKey] : undefined)).filter(Boolean); + } else { + current = current ? current[actualKey] : undefined; + } + + if (isArray && !Array.isArray(current)) { + current = current ? [current] : []; + } + } + + if (Array.isArray(current)) { + return current.filter( + (item) => + typeof item === 'string' || + typeof item === 'number' || + (Array.isArray(item) && (typeof item[0] === 'string' || typeof item[0] === 'number')) + ); + } else if (typeof current === 'string' || typeof current === 'number') { + return current; + } + + return undefined; + } + + async parseXmlFromUrl(url: string): Promise> { + try { + const response = await axios.get(url); + const xmlData = response.data; + + return await parseStringPromise(xmlData); + } catch (error) { + throw error; + } + } + + mappingFunction(mappingsData: any, values: any[]): any[] { + const result = []; + const maxLength = Math.max(...values.map((value) => (Array.isArray(value) ? value.length : 1))); + + for (let i = 0; i < maxLength; i++) { + const item: any = {}; + mappingsData.forEach((mapping, index) => { + const value = values[index]; + item[mapping.key] = Array.isArray(value) ? value[i] : value; + }); + result.push(item); + } + + return result; + } + + async getJobImportedData(_jobId: string) { + try { + const userJob = await this.userJobRepository.findOne({ _id: _jobId }); + if (!userJob) { + throw new Error(`Job not found for _jobId: ${_jobId}`); + } + + const jobMappings = await this.jobMappingRepository.find({ _jobId }); + const mappings: { key: string; mapping: string }[] = jobMappings.map((jobMapping) => ({ + key: jobMapping.key, + mapping: jobMapping.path, + })); + + const xmlJsonData = await this.parseXmlFromUrl(userJob.url); + + const returnedValues = mappings.map(({ mapping }) => this.getXMLJsonValueByPath(xmlJsonData, mapping)); + const mappedData = this.mappingFunction(mappings, returnedValues); + + return mappedData; + } catch (error) { + throw error; + } + } + + private async convertRecordsToJsonFile(importJobId: string, importRecords: Record[]) { + const allJsonDataFilePath = this.fileNameService.getAllJsonDataFilePath(importJobId); + await this.storageService.uploadFile(allJsonDataFilePath, JSON.stringify(importRecords), FileMimeTypesEnum.JSON); + } +} From 114401a9cf9b622e04d1583b410e53dded42e163 Mon Sep 17 00:00:00 2001 From: Mayur Date: Tue, 23 Jul 2024 17:48:29 +0530 Subject: [PATCH 226/299] feat: Removed = condition to get corrected current page number --- apps/queue-manager/src/consumers/base.consumer.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/queue-manager/src/consumers/base.consumer.ts b/apps/queue-manager/src/consumers/base.consumer.ts index 2766fc3d3..941fcabf6 100644 --- a/apps/queue-manager/src/consumers/base.consumer.ts +++ b/apps/queue-manager/src/consumers/base.consumer.ts @@ -16,7 +16,7 @@ export abstract class BaseConsumer { totalRecords: number; chunkSize; }): number | null { - if (totalRecords >= currentPage * chunkSize) { + if (totalRecords > currentPage * chunkSize) { return currentPage + this.DEFAULT_PAGE; } From 08da552ba8fba1aa80c09e002adcb540d4aa4780 Mon Sep 17 00:00:00 2001 From: Mayur Date: Tue, 23 Jul 2024 17:54:54 +0530 Subject: [PATCH 227/299] feat: Given types to fields from Schema.Types --- .../repositories/userjob/userjob.schema.ts | 20 ++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/libs/dal/src/repositories/userjob/userjob.schema.ts b/libs/dal/src/repositories/userjob/userjob.schema.ts index 005001240..af2e92b50 100644 --- a/libs/dal/src/repositories/userjob/userjob.schema.ts +++ b/libs/dal/src/repositories/userjob/userjob.schema.ts @@ -17,11 +17,21 @@ const userJobSchema = new Schema( headings: { type: [Schema.Types.String], }, - extra: String, - userId: String, - customRecordFormat: String, - customChunkFormat: String, - customSchema: String, + extra: { + type: Schema.Types.String, + }, + externalUserId: { + type: Schema.Types.String, + }, + customRecordFormat: { + type: Schema.Types.String, + }, + customChunkFormat: { + type: Schema.Types.String, + }, + customSchema: { + type: Schema.Types.String, + }, }, { ...schemaOptions } ); From 087fef48653edb3a0386e52e6c2cf7cd3b293406 Mon Sep 17 00:00:00 2001 From: Mayur Date: Tue, 23 Jul 2024 17:57:07 +0530 Subject: [PATCH 228/299] feat: Streamlined and Refactored SendImportJobData to send the data in the specified webhook url --- .../send-import-job-data.consumer.ts | 194 ++++++++++++++++++ 1 file changed, 194 insertions(+) create mode 100644 apps/queue-manager/src/consumers/send-import-job-data.consumer.ts diff --git a/apps/queue-manager/src/consumers/send-import-job-data.consumer.ts b/apps/queue-manager/src/consumers/send-import-job-data.consumer.ts new file mode 100644 index 000000000..588bf888b --- /dev/null +++ b/apps/queue-manager/src/consumers/send-import-job-data.consumer.ts @@ -0,0 +1,194 @@ +import { + TemplateRepository, + WebhookDestinationRepository, + ColumnRepository, + ImportJobHistoryRepository, + UserJobEntity, + WebhookLogEntity, + UploadRepository, + WebhookLogRepository, +} from '@impler/dal'; +import { StorageService } from '@impler/shared/dist/services/storage'; +import { + ColumnTypesEnum, + SendImportJobData, + SendImportJobCachedData, + replaceVariablesInObject, + FileEncodingsEnum, + QueuesEnum, + UploadStatusEnum, +} from '@impler/shared'; + +import { BaseConsumer } from './base.consumer'; +import { getStorageServiceClass } from '../helpers/storage.helper'; +import { publishToQueue } from '../bootstrap'; + +const MIN_LIMIT = 0; +const DEFAULT_PAGE = 1; + +export class SendImportJobDataConsumer extends BaseConsumer { + private columnRepository: ColumnRepository = new ColumnRepository(); + private uploadRepository: UploadRepository = new UploadRepository(); + private webhookLogRepository: WebhookLogRepository = new WebhookLogRepository(); + private templateRepository: TemplateRepository = new TemplateRepository(); + private importJobHistoryRepository: ImportJobHistoryRepository = new ImportJobHistoryRepository(); + private storageService: StorageService = getStorageServiceClass(); + private webhookDestinationRepository: WebhookDestinationRepository = new WebhookDestinationRepository(); + + async message(message: { content: string }) { + const data = JSON.parse(message.content) as SendImportJobData; + const cachedData = data.cache || (await this.getInitialCachedData(data.importJobHistoryId)); + let allDataJson: null | any[] = null; + + if (cachedData && cachedData.callbackUrl) { + if (cachedData.allDataFilePath) { + const allDataContent = await this.storageService.getFileContent( + cachedData.allDataFilePath, + FileEncodingsEnum.JSON + ); + allDataJson = JSON.parse(allDataContent); + } + const { sendData, page } = this.buildSendData({ + data: allDataJson, + uploadId: data.importJobHistoryId, + chunkSize: cachedData.chunkSize, + recordFormat: cachedData.recordFormat, + chunkFormat: cachedData.chunkFormat, + ...cachedData, + }); + + const headers = + cachedData.authHeaderName && cachedData.authHeaderValue + ? { [cachedData.authHeaderName]: cachedData.authHeaderValue } + : null; + + const response = await this.makeApiCall({ + data: sendData, + uploadId: data.importJobHistoryId, + page, + method: 'POST', + url: cachedData.callbackUrl, + headers, + }); + + this.makeResponseEntry(response); + + const nextPageNumber = this.getNextPageNumber({ + totalRecords: allDataJson.length, + currentPage: page, + chunkSize: cachedData.chunkSize, + }); + + if (nextPageNumber) { + // Make next call + publishToQueue(QueuesEnum.SEND_IMPORT_JOB_DATA, { + importJobHistoryId: data.importJobHistoryId, + cache: { + ...cachedData, + page: nextPageNumber, + }, + }); + } else { + // Processing is done + this.finalizeUpload(data.importJobHistoryId); + } + } + } + + private buildSendData({ + data, + defaultValues, + page = DEFAULT_PAGE, + chunkSize, + uploadId, + chunkFormat, + recordFormat, + extra = '', + multiSelectHeadings, + }: SendImportJobCachedData & { data: any[]; uploadId: string }): { sendData: Record; page: number } { + const defaultValuesObj = JSON.parse(defaultValues); + let slicedData = data.slice( + Math.max((page - DEFAULT_PAGE) * chunkSize, MIN_LIMIT), + Math.min(page * chunkSize, data.length) + ); + + if (Array.isArray(multiSelectHeadings) && multiSelectHeadings.length > 0) { + slicedData = slicedData.map((obj) => { + multiSelectHeadings.forEach((heading) => { + obj[heading] = obj[heading] ? (Array.isArray(obj[heading]) ? obj[heading] : obj[heading].split(',')) : []; + }); + + return obj; + }); + } + + if (recordFormat) + slicedData = slicedData.map((obj) => replaceVariablesInObject(JSON.parse(recordFormat), obj, defaultValuesObj)); + + const sendData = { + page, + uploadId, + data: slicedData, + totalRecords: data.length, + chunkSize: slicedData.length, + extra: extra ? JSON.parse(extra) : '', + totalPages: this.getTotalPages(data.length, chunkSize), + }; + + return { + sendData: chunkFormat ? replaceVariablesInObject(JSON.parse(chunkFormat), sendData as any) : sendData, + page, + }; + } + + private async getInitialCachedData(_importJobHistoryId: string): Promise { + const userJob = await this.importJobHistoryRepository.getHistoryWithJob(_importJobHistoryId, ['_templateId']); + const columns = await this.columnRepository.find({ + _templateId: (userJob._jobId as unknown as UserJobEntity)._templateId, + }); + const templateData = await this.templateRepository.findById( + (userJob._jobId as unknown as UserJobEntity)._templateId, + 'name _projectId code' + ); + const webhookDestination = await this.webhookDestinationRepository.findOne({ + _templateId: (userJob._jobId as unknown as UserJobEntity)._templateId, + }); + + const defaultValueObj = {}; + const multiSelectHeadings = []; + if (Array.isArray(columns)) { + columns.forEach((item) => { + if (item.defaultValue) defaultValueObj[item.key] = item.defaultValue; + if (item.type === ColumnTypesEnum.SELECT && item.allowMultiSelect) multiSelectHeadings.push(item.key); + }); + } + + this.importJobHistoryRepository.create({ + _jobId: userJob._id, + }); + + return { + _templateId: (userJob._jobId as unknown as UserJobEntity)._templateId, + callbackUrl: webhookDestination?.callbackUrl, + chunkSize: webhookDestination?.chunkSize, + name: templateData.name, + page: 1, + extra: (userJob._jobId as unknown as UserJobEntity).extra, + authHeaderName: webhookDestination.authHeaderName, + authHeaderValue: '', + allDataFilePath: userJob.allDataFilePath, + defaultValues: JSON.stringify(defaultValueObj), + recordFormat: (userJob._jobId as unknown as UserJobEntity).customRecordFormat, + chunkFormat: (userJob._jobId as unknown as UserJobEntity).customChunkFormat, + multiSelectHeadings, + }; + } + + private async makeResponseEntry(data: Partial) { + return await this.webhookLogRepository.create(data); + } + + private async finalizeUpload(importJobHistoryId: string) { + return await this.uploadRepository.update({ _id: importJobHistoryId }, { status: UploadStatusEnum.COMPLETED }); + } +} From b6187e591f1422d800c8d95b29dbdc3e98a38eb2 Mon Sep 17 00:00:00 2001 From: Mayur Date: Tue, 23 Jul 2024 17:59:55 +0530 Subject: [PATCH 229/299] feat: Removed scheduleData from dependency array --- .../AutoImportPhases/AutoImportPhase3/AutoImportPhase3.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/widget/src/components/widget/Phases/AutoImportPhases/AutoImportPhase3/AutoImportPhase3.tsx b/apps/widget/src/components/widget/Phases/AutoImportPhases/AutoImportPhase3/AutoImportPhase3.tsx index 8f3f76f46..840f2cd57 100644 --- a/apps/widget/src/components/widget/Phases/AutoImportPhases/AutoImportPhase3/AutoImportPhase3.tsx +++ b/apps/widget/src/components/widget/Phases/AutoImportPhases/AutoImportPhase3/AutoImportPhase3.tsx @@ -29,7 +29,7 @@ export function AutoImportPhase3({ onNextClick }: IAutoImportPhase3Props) { const cronExpression = `${scheduleData.Minute} ${scheduleData.Hour} ${scheduleData.Day} ${scheduleData.Month} ${scheduleData.Days}`; const result = getCronDescriptionfromCronExpression(cronExpression); setCronDescription(result); - }, [scheduleData]); + }, []); const handleBadgeClick = (cronExpression: string) => { const [minute, hour, day, month, days] = cronExpression.split(' '); From 84032f375c01848d5e7005edbec951564b363735 Mon Sep 17 00:00:00 2001 From: Mayur Date: Tue, 23 Jul 2024 18:01:08 +0530 Subject: [PATCH 230/299] feat: Streamlined SEND_IMPORT_JOB_DATA --- libs/shared/src/types/upload/upload.types.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/shared/src/types/upload/upload.types.ts b/libs/shared/src/types/upload/upload.types.ts index 83cbede80..16df6fcc8 100644 --- a/libs/shared/src/types/upload/upload.types.ts +++ b/libs/shared/src/types/upload/upload.types.ts @@ -41,7 +41,7 @@ export enum QueuesEnum { 'SEND_BUBBLE_DATA' = 'SEND_BUBBLE_DATA', 'END_IMPORT' = 'END_IMPORT', 'GET_IMPORT_JOB_DATA' = 'GET_IMPORT_JOB_DATA', - SEND_IMPORT_JOB_DATA = 'SEND_IMPORT_JOB_DATA', + 'SEND_IMPORT_JOB_DATA' = 'SEND_IMPORT_JOB_DATA', } export type CommonCachedData = { From be6330d8732c154cc7cd6c1b23f82690b5bdcdfa Mon Sep 17 00:00:00 2001 From: Mayur Date: Tue, 23 Jul 2024 20:37:12 +0530 Subject: [PATCH 231/299] feat: Made Dto for body parameters for creating user job route --- apps/api/src/app/import-jobs/import-jobs.controller.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/api/src/app/import-jobs/import-jobs.controller.ts b/apps/api/src/app/import-jobs/import-jobs.controller.ts index 40a21f6d5..c67f8bf12 100644 --- a/apps/api/src/app/import-jobs/import-jobs.controller.ts +++ b/apps/api/src/app/import-jobs/import-jobs.controller.ts @@ -6,6 +6,7 @@ import { UpdateJobMappingDto } from './dtos/update-jobmapping.dto'; import { UpdateJobInfoDto } from './dtos/update-jobinfo.dto'; import { JwtAuthGuard } from '@shared/framework/auth.gaurd'; import { ACCESS_KEY_NAME } from '@impler/shared'; +import { CreateUserJobDto } from './dtos/create-userjob.dto'; @ApiTags('Import-Jobs') @Controller('/import-jobs') @@ -21,12 +22,11 @@ export class ImportJobsController { ) {} @Post(':templateId') @ApiOperation({ summary: 'Create User Job' }) - @UseGuards(JwtAuthGuard) @ApiSecurity(ACCESS_KEY_NAME) - async createUserJobRoute(@Param('templateId') templateId: string, @Body() body: { url: string }) { + async createUserJobRoute(@Param('templateId') templateId: string, @Body() createUserJobData: CreateUserJobDto) { return this.createUserJob.execute({ _templateId: templateId, - url: body.url, + url: createUserJobData.url, }); } From 96cf59a0c9ceb83ca0d79b3c26ba3a4125463ccb Mon Sep 17 00:00:00 2001 From: Mayur Date: Tue, 23 Jul 2024 20:39:44 +0530 Subject: [PATCH 232/299] feat: Created UserJobDto --- .../import-jobs/dtos/create-userjob.dto.ts | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 apps/api/src/app/import-jobs/dtos/create-userjob.dto.ts diff --git a/apps/api/src/app/import-jobs/dtos/create-userjob.dto.ts b/apps/api/src/app/import-jobs/dtos/create-userjob.dto.ts new file mode 100644 index 000000000..4016ded1a --- /dev/null +++ b/apps/api/src/app/import-jobs/dtos/create-userjob.dto.ts @@ -0,0 +1,19 @@ +import { IsNotEmpty, IsOptional, IsString } from 'class-validator'; + +export class CreateUserJobDto { + @IsString() + @IsNotEmpty() + url: string; + + @IsString() + @IsNotEmpty() + _templateId: string; + + @IsString() + @IsOptional() + externalUserId?: string; + + @IsString() + @IsOptional() + extra?: string; +} From 6e6c156c15045bcaac4b13af137533770ef651a8 Mon Sep 17 00:00:00 2001 From: Mayur Date: Tue, 23 Jul 2024 20:40:21 +0530 Subject: [PATCH 233/299] feat: Added status and externalUserId fields --- apps/api/src/app/import-jobs/dtos/update-jobinfo.dto.ts | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/apps/api/src/app/import-jobs/dtos/update-jobinfo.dto.ts b/apps/api/src/app/import-jobs/dtos/update-jobinfo.dto.ts index a00be2d1d..e8a235804 100644 --- a/apps/api/src/app/import-jobs/dtos/update-jobinfo.dto.ts +++ b/apps/api/src/app/import-jobs/dtos/update-jobinfo.dto.ts @@ -16,4 +16,12 @@ export class UpdateJobInfoDto { @IsArray() @IsOptional() headings: string[]; + + @IsString() + @IsOptional() + status: string; + + @IsString() + @IsOptional() + externalUserId: string; } From b543ef55fe88c4ad3d00177654c08a84671516b2 Mon Sep 17 00:00:00 2001 From: Mayur Date: Tue, 23 Jul 2024 20:41:44 +0530 Subject: [PATCH 234/299] feat: Added CreateUserJobCommand to be used in the CreateUserJob execute method --- .../usecase/create-userjob/create-userjob.command.ts | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 apps/api/src/app/import-jobs/usecase/create-userjob/create-userjob.command.ts diff --git a/apps/api/src/app/import-jobs/usecase/create-userjob/create-userjob.command.ts b/apps/api/src/app/import-jobs/usecase/create-userjob/create-userjob.command.ts new file mode 100644 index 000000000..65af3a32a --- /dev/null +++ b/apps/api/src/app/import-jobs/usecase/create-userjob/create-userjob.command.ts @@ -0,0 +1,6 @@ +export class CreateUserJobCommand { + url: string; + _templateId: string; + externalUserId?: string; + extra?: string; +} From b2c4327747f85719e0f9be78a81e3e5f74f82254 Mon Sep 17 00:00:00 2001 From: Mayur Date: Tue, 23 Jul 2024 20:42:31 +0530 Subject: [PATCH 235/299] feat: Used CreateUserJobCommand in execute method --- .../usecase/create-userjob/create-userjob.usecase.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/apps/api/src/app/import-jobs/usecase/create-userjob/create-userjob.usecase.ts b/apps/api/src/app/import-jobs/usecase/create-userjob/create-userjob.usecase.ts index 6cd38aa72..f9a0148d8 100644 --- a/apps/api/src/app/import-jobs/usecase/create-userjob/create-userjob.usecase.ts +++ b/apps/api/src/app/import-jobs/usecase/create-userjob/create-userjob.usecase.ts @@ -3,6 +3,7 @@ import { FileMimeTypesEnum } from '@impler/shared'; import { APIMessages } from '@shared/constants'; import { UserJobEntity, UserJobRepository } from '@impler/dal'; import { RSSService } from '@shared/services'; +import { CreateUserJobCommand } from './create-userjob.command'; @Injectable() export class CreateUserJob { @@ -11,7 +12,7 @@ export class CreateUserJob { private readonly rssService: RSSService ) {} - async execute({ _templateId, url }: { url: string; _templateId: string }): Promise { + async execute({ _templateId, url }: CreateUserJobCommand): Promise { const mimeType = await this.rssService.getMimeType(url); if (mimeType === FileMimeTypesEnum.XML || mimeType === FileMimeTypesEnum.TEXTXML) { const { rssKeyHeading } = await this.rssService.parseRssFeed(url); From f4a7bbe076eae2015c87de53c516193733075e7e Mon Sep 17 00:00:00 2001 From: Mayur Date: Tue, 23 Jul 2024 20:47:12 +0530 Subject: [PATCH 236/299] feat: Added status in UpdateUserJobCommand --- .../usecase/update-userjob/update-userjob.command.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/apps/api/src/app/import-jobs/usecase/update-userjob/update-userjob.command.ts b/apps/api/src/app/import-jobs/usecase/update-userjob/update-userjob.command.ts index 7fba92858..7918a9c0b 100644 --- a/apps/api/src/app/import-jobs/usecase/update-userjob/update-userjob.command.ts +++ b/apps/api/src/app/import-jobs/usecase/update-userjob/update-userjob.command.ts @@ -6,4 +6,6 @@ export class UpdateUserJobCommand { cron: string; headings: string[]; + + status: string; } From 45fe0591d06959f71e03bd12576f09c96217929e Mon Sep 17 00:00:00 2001 From: Mayur Date: Tue, 23 Jul 2024 20:52:54 +0530 Subject: [PATCH 237/299] feat: Removed _templateId --- apps/api/src/app/import-jobs/dtos/create-userjob.dto.ts | 4 ---- 1 file changed, 4 deletions(-) diff --git a/apps/api/src/app/import-jobs/dtos/create-userjob.dto.ts b/apps/api/src/app/import-jobs/dtos/create-userjob.dto.ts index 4016ded1a..b95b25af7 100644 --- a/apps/api/src/app/import-jobs/dtos/create-userjob.dto.ts +++ b/apps/api/src/app/import-jobs/dtos/create-userjob.dto.ts @@ -5,10 +5,6 @@ export class CreateUserJobDto { @IsNotEmpty() url: string; - @IsString() - @IsNotEmpty() - _templateId: string; - @IsString() @IsOptional() externalUserId?: string; From b51a7f2f6c3418ab8bd9f1c9e2bb021a65135a17 Mon Sep 17 00:00:00 2001 From: Mayur Date: Tue, 23 Jul 2024 20:55:51 +0530 Subject: [PATCH 238/299] feat: Updated status to MAPPING --- .../usecase/get-jobmapping/get-jobmapping.usecase.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/apps/api/src/app/import-jobs/usecase/get-jobmapping/get-jobmapping.usecase.ts b/apps/api/src/app/import-jobs/usecase/get-jobmapping/get-jobmapping.usecase.ts index 28c0fb125..24bdd138d 100644 --- a/apps/api/src/app/import-jobs/usecase/get-jobmapping/get-jobmapping.usecase.ts +++ b/apps/api/src/app/import-jobs/usecase/get-jobmapping/get-jobmapping.usecase.ts @@ -1,5 +1,6 @@ import { Injectable } from '@nestjs/common'; import { ColumnRepository, UserJobRepository, JobMappingEntity, JobMappingRepository } from '@impler/dal'; +import { UserJobImportStatusEnum } from '@impler/shared'; @Injectable() export class GetJobMapping { @@ -20,6 +21,8 @@ export class GetJobMapping { throw new Error('User job not found'); } + await this.userJobRepository.update({ _id: _jobId }, { $set: { status: UserJobImportStatusEnum.MAPPING } }); + await this.jobMappingRepository.find({ _id: _jobId }); const columns = await this.columnRepository.find({ _templateId: userJob._templateId }); const jobsMapping = columns.map((column) => ({ From f1c35a46bf543cb35b2153d1c1e00ea917bb2dc1 Mon Sep 17 00:00:00 2001 From: Mayur Date: Wed, 24 Jul 2024 12:45:08 +0530 Subject: [PATCH 239/299] feat: Added status in the userjob schema --- libs/dal/src/repositories/userjob/userjob.schema.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/libs/dal/src/repositories/userjob/userjob.schema.ts b/libs/dal/src/repositories/userjob/userjob.schema.ts index af2e92b50..1cc1a21c4 100644 --- a/libs/dal/src/repositories/userjob/userjob.schema.ts +++ b/libs/dal/src/repositories/userjob/userjob.schema.ts @@ -23,6 +23,9 @@ const userJobSchema = new Schema( externalUserId: { type: Schema.Types.String, }, + status: { + type: Schema.Types.String, + }, customRecordFormat: { type: Schema.Types.String, }, From 486f40b945c8e67e65e3046d54561676b3cfd37f Mon Sep 17 00:00:00 2001 From: Mayur Date: Wed, 24 Jul 2024 16:20:16 +0530 Subject: [PATCH 240/299] feat: Added templeteId in getImportConfig and extra fields in getRssXmlMappingHeading --- packages/client/src/api/api.service.ts | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/packages/client/src/api/api.service.ts b/packages/client/src/api/api.service.ts index 5ae98ca11..b3dfc0787 100644 --- a/packages/client/src/api/api.service.ts +++ b/packages/client/src/api/api.service.ts @@ -47,10 +47,11 @@ export class ApiService { }); } - async getImportConfig(projectId: string) { - return this.httpClient.get( - `/common/import-config?projectId=${projectId}`, - ) as Promise; + async getImportConfig(projectId: string, templateId?: string) { + return this.httpClient.get(`/common/import-config`, { + projectId, + templateId, + }) as Promise; } async getExcelSheetNames(data: { file: File }) { @@ -194,9 +195,20 @@ export class ApiService { ); } - async getRssXmlMappingHeading(_templateId: string, url: string) { - return this.httpClient.post(`/import-jobs/${_templateId}`, { - url, + async getRssXmlMappingHeading(data: { + templateId: string; + url: string; + authHeaderValue?: string; + extra?: string; + schema?: string; + output?: string; + }) { + return this.httpClient.post(`/import-jobs/${data.templateId}`, { + url: data.url, + authHeaderValue: data.authHeaderValue, + extra: data.extra, + schema: data.schema, + output: data.output, }); } From 5aaffcfc7862df89e13ddf8dea294ccdc40a9499 Mon Sep 17 00:00:00 2001 From: Mayur Date: Wed, 24 Jul 2024 16:21:07 +0530 Subject: [PATCH 241/299] feat: Added templateId in getImportConfigRoute --- apps/api/src/app/common/common.controller.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/apps/api/src/app/common/common.controller.ts b/apps/api/src/app/common/common.controller.ts index 84e1eb29c..4b3caaf5f 100644 --- a/apps/api/src/app/common/common.controller.ts +++ b/apps/api/src/app/common/common.controller.ts @@ -57,12 +57,15 @@ export class CommonController { @ApiOperation({ summary: 'Get import config', }) - async getImportConfigRoute(@Query('projectId') projectId: string): Promise { + async getImportConfigRoute( + @Query('projectId') projectId: string, + @Query('templateId') templateId: string + ): Promise { if (!projectId) { throw new BadRequestException(); } - return await this.getImportConfig.execute(projectId); + return await this.getImportConfig.execute(projectId, templateId); } @Post('/sheet-names') From eae5635bd6aacaea5214b65780dbb1824b0b0276 Mon Sep 17 00:00:00 2001 From: Mayur Date: Wed, 24 Jul 2024 16:23:58 +0530 Subject: [PATCH 242/299] feat: Maintained proper statuses while pause and start cron job --- .../app/shared/services/cronjob.service.ts | 38 +++++++++++-------- 1 file changed, 22 insertions(+), 16 deletions(-) diff --git a/apps/api/src/app/shared/services/cronjob.service.ts b/apps/api/src/app/shared/services/cronjob.service.ts index ea788f8dd..f310729e4 100644 --- a/apps/api/src/app/shared/services/cronjob.service.ts +++ b/apps/api/src/app/shared/services/cronjob.service.ts @@ -3,7 +3,7 @@ import { SchedulerRegistry } from '@nestjs/schedule'; import { CronJob } from 'cron'; import { QueueService } from '@shared/services/queue.service'; import { UserJobRepository } from '@impler/dal'; -import { QueuesEnum } from '@impler/shared'; +import { QueuesEnum, UserJobImportStatusEnum } from '@impler/shared'; @Injectable() export class CronJobService { @@ -13,29 +13,36 @@ export class CronJobService { private queueService: QueueService ) {} - async pauseJob(jobId: string) { - const job = this.schedulerRegistry.getCronJob(`${jobId}_rss_import`); - job.stop(); + async pauseJob(_jobId: string) { + const userJob = this.schedulerRegistry.getCronJob(`${_jobId}_rss_import`); + if (userJob) { + userJob.stop(); + } else { + throw new Error(`Job ${_jobId}_rss_import not found`); + } + + await this.userJobRepository.update({ _id: _jobId }, { $set: { status: UserJobImportStatusEnum.PAUSED } }); - return { message: `Job ${jobId} paused successfully` }; + return { message: `Job ${_jobId} paused successfully` }; } - async startJob(userId: string, jobId: string) { - const userJob = await this.userJobRepository.findOne({ _id: jobId, userId }); + async startJob(externalUserId: string, _jobId: string) { + const userJob = await this.userJobRepository.findOne({ _id: _jobId, externalUserId }); if (!userJob) { - throw new Error(`Job ${jobId} not found for user ${userId}`); + throw new Error(`Job ${_jobId} not found for user ${externalUserId}`); } const cronExpression = userJob.cron; - const job = new CronJob(cronExpression, () => this.triggerImportJob(jobId)); - this.schedulerRegistry.addCronJob(`${jobId}_rss_import`, job); + await this.userJobRepository.update({ _id: _jobId }, { $set: { status: UserJobImportStatusEnum.RUNNING } }); + const job = new CronJob(cronExpression, () => this.triggerImportJob(_jobId)); + this.schedulerRegistry.addCronJob(`${_jobId}_rss_import`, job); job.start(); - return { message: `Job ${jobId} started successfully with schedule: ${cronExpression}` }; + return { message: `Job ${_jobId} started successfully with schedule: ${cronExpression}` }; } - async deleteJob(userId: string, jobId: string) { - await this.userJobRepository.delete({ _id: jobId, userId }); + async deleteJob(externalUserId: string, jobId: string) { + await this.userJobRepository.delete({ _id: jobId, externalUserId }); this.schedulerRegistry.deleteCronJob(`${jobId}_rss_import`); return { message: `Job ${jobId} deleted successfully` }; @@ -43,10 +50,9 @@ export class CronJobService { async triggerImportJob(_jobId: string) { try { - this.queueService.publishToQueue(QueuesEnum.SEND_RSS_XML_DATA, { _jobId }); - console.log(`Import job triggered for job ${_jobId}`); + this.queueService.publishToQueue(QueuesEnum.GET_IMPORT_JOB_DATA, { _jobId }); } catch (error) { - console.error(`Error triggering import job for ${_jobId}:`, error); + throw error; } } } From 10e3f9756284ef0cba6bfe6ad51ac3ae7be4aaf3 Mon Sep 17 00:00:00 2001 From: Mayur Date: Wed, 24 Jul 2024 16:27:59 +0530 Subject: [PATCH 243/299] feat: Taking templeteId as parameter in execute method --- .../get-import-config/get-import-config.usecase.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/apps/api/src/app/common/usecases/get-import-config/get-import-config.usecase.ts b/apps/api/src/app/common/usecases/get-import-config/get-import-config.usecase.ts index 9757eaae3..b9fe8e297 100644 --- a/apps/api/src/app/common/usecases/get-import-config/get-import-config.usecase.ts +++ b/apps/api/src/app/common/usecases/get-import-config/get-import-config.usecase.ts @@ -1,6 +1,7 @@ -import { Injectable } from '@nestjs/common'; +import { BadRequestException, Injectable } from '@nestjs/common'; import { UserRepository, TemplateRepository } from '@impler/dal'; import { IImportConfig, PaymentAPIService } from '@impler/shared'; +import { APIMessages } from '@shared/constants'; @Injectable() export class GetImportConfig { @@ -10,21 +11,20 @@ export class GetImportConfig { private templateRepository: TemplateRepository ) {} - async execute(projectId: string): Promise { + async execute(projectId: string, templateId: string): Promise { const userEmail = await this.userRepository.findUserEmailFromProjectId(projectId); const removeBrandingAvailable = await this.paymentAPIService.checkEvent(userEmail, 'REMOVE_BRANDING'); const template = await this.templateRepository.findOne({ _projectId: projectId, + ...(templateId ? { _id: templateId } : {}), }); if (!template) { - throw new Error('Template not found'); + throw new BadRequestException(APIMessages.TEMPLATE_NOT_FOUND); } - const importMode = template.mode; - - return { showBranding: !removeBrandingAvailable, mode: importMode }; + return { showBranding: !removeBrandingAvailable, mode: template.mode, title: template.name }; } } From 3a331b0cc992338475c53b758aaa4eb25ef469ab Mon Sep 17 00:00:00 2001 From: Mayur Date: Wed, 24 Jul 2024 16:29:21 +0530 Subject: [PATCH 244/299] feat: Saved extra and externalUserId in database while creating userJob --- .../usecase/create-userjob/create-userjob.usecase.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/apps/api/src/app/import-jobs/usecase/create-userjob/create-userjob.usecase.ts b/apps/api/src/app/import-jobs/usecase/create-userjob/create-userjob.usecase.ts index f9a0148d8..f86400eaa 100644 --- a/apps/api/src/app/import-jobs/usecase/create-userjob/create-userjob.usecase.ts +++ b/apps/api/src/app/import-jobs/usecase/create-userjob/create-userjob.usecase.ts @@ -12,15 +12,21 @@ export class CreateUserJob { private readonly rssService: RSSService ) {} - async execute({ _templateId, url }: CreateUserJobCommand): Promise { + async execute({ _templateId, url, externalUserId, extra }: CreateUserJobCommand): Promise { const mimeType = await this.rssService.getMimeType(url); if (mimeType === FileMimeTypesEnum.XML || mimeType === FileMimeTypesEnum.TEXTXML) { const { rssKeyHeading } = await this.rssService.parseRssFeed(url); + let formattedExtra = extra || '{}'; + try { + formattedExtra = JSON.parse(extra); + } catch (_) {} return await this.userJobRepository.create({ url, headings: rssKeyHeading, _templateId: _templateId, + extra, + externalUserId: externalUserId || (formattedExtra as unknown as Record)?.externalUserId, }); } else { throw new BadRequestException(APIMessages.INVALID_RSS_URL); From 1789b8e4afbc470aad898f76d927973c13fdfb0e Mon Sep 17 00:00:00 2001 From: Mayur Date: Wed, 24 Jul 2024 16:31:45 +0530 Subject: [PATCH 245/299] feat: Refactored conform to confirm and resolved spelling issue --- apps/widget/src/config/texts.config.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/widget/src/config/texts.config.ts b/apps/widget/src/config/texts.config.ts index ce35b75ff..a7e1a1a37 100644 --- a/apps/widget/src/config/texts.config.ts +++ b/apps/widget/src/config/texts.config.ts @@ -15,7 +15,7 @@ export const TEXTS = { CONFIGURE: 'Configure', MAPCOLUMNS: 'Map Columns', SCHEDULE: 'Schedule', - CONFORM: 'Conform', + CONFIRM: 'Confirm', }, DROPZONE: { @@ -54,7 +54,7 @@ export const TEXTS = { BUTTONTEXT: { MAPCOLUMN: 'Map Column', SCHEDULE: 'Schedule', - CONFORM: 'Conform', + CONFIRM: 'Confirm', CLOSE: 'Close', }, }, From 4bac63f5a5109d65cb184dd53c4a8c182f6041b3 Mon Sep 17 00:00:00 2001 From: Mayur Date: Wed, 24 Jul 2024 16:33:29 +0530 Subject: [PATCH 246/299] feat: Implemented scheduleRssImportJob that will trigger the start cron job --- .../update-userjob/update-userjob.usecase.ts | 31 ++++++++++++++++--- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/apps/api/src/app/import-jobs/usecase/update-userjob/update-userjob.usecase.ts b/apps/api/src/app/import-jobs/usecase/update-userjob/update-userjob.usecase.ts index 80938c09d..3521c7a0e 100644 --- a/apps/api/src/app/import-jobs/usecase/update-userjob/update-userjob.usecase.ts +++ b/apps/api/src/app/import-jobs/usecase/update-userjob/update-userjob.usecase.ts @@ -1,12 +1,35 @@ -import { Injectable } from '@nestjs/common'; +import { CronJob } from 'cron'; +import { Injectable, Logger } from '@nestjs/common'; +import { SchedulerRegistry } from '@nestjs/schedule'; + +import { UserJobImportStatusEnum } from '@impler/shared'; import { UserJobEntity, UserJobRepository } from '@impler/dal'; import { UpdateUserJobCommand } from './update-userjob.command'; +import { CronJobService } from '@shared/services/cronjob.service'; @Injectable() export class UpdateUserJob { - constructor(private readonly userJobRepository: UserJobRepository) {} + constructor( + private readonly cronJobService: CronJobService, + private readonly schedulerRegistry: SchedulerRegistry, + private readonly userJobRepository: UserJobRepository + ) {} + + async execute(jobId: string, data: UpdateUserJobCommand): Promise { + data.status = UserJobImportStatusEnum.SCHEDULING; + const userJob = await this.userJobRepository.findOneAndUpdate({ _id: jobId }, data); + + this.scheduleRssImportJob(jobId, data.cron); + + return userJob; + } + + private scheduleRssImportJob(jobId: string, cronExpression: string) { + const job = new CronJob(cronExpression, () => this.cronJobService.triggerImportJob(jobId)); + this.schedulerRegistry.addCronJob(`${jobId}_rss_import`, job); + + job.start(); - async execute(_id: string, data: UpdateUserJobCommand): Promise { - return await this.userJobRepository.findOneAndUpdate({ _id }, data); + Logger.log(`Cron job for RSS import started with schedule: ${cronExpression}`); } } From 8d44d8f175015fa4639aae26275c67f853c7fa0b Mon Sep 17 00:00:00 2001 From: Mayur Date: Wed, 24 Jul 2024 16:34:30 +0530 Subject: [PATCH 247/299] feat: Refactored and removed code block of the getImportConfig api call --- apps/widget/src/hooks/Phase1/usePhase1.ts | 28 ++++++++++------------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/apps/widget/src/hooks/Phase1/usePhase1.ts b/apps/widget/src/hooks/Phase1/usePhase1.ts index 9bd8ce687..389e52ea3 100644 --- a/apps/widget/src/hooks/Phase1/usePhase1.ts +++ b/apps/widget/src/hooks/Phase1/usePhase1.ts @@ -8,9 +8,17 @@ import { useAPIState } from '@store/api.context'; import { useAppState } from '@store/app.context'; import { IFormvalues, IUploadValues } from '@types'; import { useImplerState } from '@store/impler.context'; -import { ColumnTypesEnum, IErrorObject, IOption, ISchemaItem, ITemplate, IUpload } from '@impler/shared'; -import { FileMimeTypesEnum, IImportConfig, downloadFile } from '@impler/shared'; import { downloadFileFromURL, getFileNameFromUrl, notifier, ParentWindow } from '@util'; +import { + ColumnTypesEnum, + IErrorObject, + IOption, + ISchemaItem, + ITemplate, + IUpload, + FileMimeTypesEnum, + downloadFile, +} from '@impler/shared'; interface IUsePhase1Props { goNext: () => void; @@ -22,20 +30,8 @@ export function usePhase1({ goNext }: IUsePhase1Props) { const [excelSheetNames, setExcelSheetNames] = useState([]); const { projectId, templateId, authHeaderValue, extra } = useImplerState(); const [isDownloadInProgress, setIsDownloadInProgress] = useState(false); - const { setUploadInfo, setTemplateInfo, setImportConfig, output, schema, data } = useAppState(); + const { setUploadInfo, setTemplateInfo, output, schema, data } = useAppState(); - const { isFetched: isImportConfigLoaded, isLoading: isImportConfigLoading } = useQuery< - IImportConfig, - IErrorObject, - IImportConfig - >(['importConfig'], () => api.getImportConfig(projectId), { - onSuccess(importConfigResponse) { - setImportConfig(importConfigResponse); - }, - onError(error: IErrorObject) { - notifier.showError({ message: error.message, title: error.error }); - }, - }); const { data: dataTemplates, isFetched, @@ -225,6 +221,6 @@ export function usePhase1({ goNext }: IUsePhase1Props) { onSelectSheetModalReset, showSelectTemplate: !templateId, onSelectExcelSheet: handleSubmit(uploadFile), - isInitialDataLoaded: isFetched && !isLoading && isImportConfigLoaded && !isImportConfigLoading, + isInitialDataLoaded: isFetched && !isLoading, }; } From a70979ab806bb6172d22e08d44d39ab2d98e0d22 Mon Sep 17 00:00:00 2001 From: Mayur Date: Wed, 24 Jul 2024 16:35:04 +0530 Subject: [PATCH 248/299] feat: Refactored conform to confirm and resolved spelling issue --- apps/widget/src/components/Common/Footer/AutoImportFooter.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/widget/src/components/Common/Footer/AutoImportFooter.tsx b/apps/widget/src/components/Common/Footer/AutoImportFooter.tsx index d4b93a22b..1d5ad01bb 100644 --- a/apps/widget/src/components/Common/Footer/AutoImportFooter.tsx +++ b/apps/widget/src/components/Common/Footer/AutoImportFooter.tsx @@ -37,11 +37,11 @@ export function AutoImportFooter({ active, onNextClick, primaryButtonLoading, pr [PhasesEnum.SCHEDULE]: ( <> ), - [PhasesEnum.CONFORM]: ( + [PhasesEnum.CONFIRM]: ( <>