From 40e5d4d0a0c62f95f517d2364a38f1e99efaf828 Mon Sep 17 00:00:00 2001 From: Pierre Lehnen <55164754+pierre-lehnen-rc@users.noreply.github.com> Date: Wed, 6 Dec 2023 09:13:37 -0300 Subject: [PATCH] chore: Convert Importer classes to typescript (#29714) Co-authored-by: Tasso Evangelista --- apps/meteor/app/api/server/v1/import.ts | 28 +- apps/meteor/app/importer-csv/client/adder.js | 4 - apps/meteor/app/importer-csv/client/index.ts | 1 - apps/meteor/app/importer-csv/lib/info.js | 12 - .../server/{importer.js => CsvImporter.ts} | 49 ++-- apps/meteor/app/importer-csv/server/index.ts | 9 +- .../client/adder.ts | 4 - .../client/index.ts | 1 - .../importer-hipchat-enterprise/lib/info.ts | 12 - ...porter.js => HipChatEnterpriseImporter.js} | 5 +- .../server/index.ts | 9 +- .../{importer.js => PendingAvatarImporter.ts} | 21 +- .../importer-pending-avatars/server/index.ts | 10 +- .../importer-pending-avatars/server/info.js | 7 - .../{importer.js => PendingFileImporter.ts} | 116 +++++---- .../importer-pending-files/server/index.ts | 10 +- .../app/importer-pending-files/server/info.js | 7 - .../app/importer-slack-users/client/adder.js | 4 - .../app/importer-slack-users/client/index.ts | 1 - .../app/importer-slack-users/lib/info.js | 12 - .../{importer.js => SlackUsersImporter.ts} | 25 +- .../app/importer-slack-users/server/index.ts | 9 +- .../meteor/app/importer-slack/client/adder.js | 4 - .../meteor/app/importer-slack/client/index.ts | 1 - apps/meteor/app/importer-slack/lib/info.js | 7 - .../server/{importer.js => SlackImporter.ts} | 242 +++++++++++++----- .../meteor/app/importer-slack/server/index.ts | 9 +- apps/meteor/app/importer/client/index.ts | 1 - apps/meteor/app/importer/lib/ImporterInfo.js | 19 -- .../app/importer/lib/ImporterProgressStep.ts | 14 +- apps/meteor/app/importer/lib/Importers.js | 47 ---- .../server/classes/ImportDataConverter.ts | 21 -- .../classes/{ImporterBase.js => Importer.ts} | 170 ++++++------ .../server/classes/ImporterProgress.js | 16 -- .../server/classes/ImporterProgress.ts | 30 +++ .../server/classes/ImporterSelection.js | 16 -- .../server/classes/ImporterSelection.ts | 26 ++ .../classes/ImporterSelectionChannel.js | 22 -- .../classes/ImporterSelectionChannel.ts | 34 +++ .../server/classes/ImporterSelectionUser.js | 22 -- .../server/classes/ImporterSelectionUser.ts | 46 ++++ ...orterWebsocket.js => ImporterWebsocket.ts} | 7 +- .../server/classes/ImportersContainer.ts | 27 ++ .../server/definitions/ImporterInfo.ts | 10 + apps/meteor/app/importer/server/index.ts | 19 +- .../methods/downloadPublicImportFile.ts | 20 +- .../server/methods/getImportFileData.ts | 36 ++- .../server/methods/getImportProgress.ts | 4 +- .../importer/server/methods/startImport.ts | 14 +- .../server/methods/uploadImportFile.ts | 21 +- apps/meteor/client/importPackages.ts | 5 - .../views/admin/import/ChannelDescriptor.tsx | 6 + .../views/admin/import/ImportHistoryPage.tsx | 4 +- .../import/ImportOperationSummary.stories.tsx | 5 +- ...nSummary.js => ImportOperationSummary.tsx} | 28 +- ....js => ImportOperationSummarySkeleton.tsx} | 6 +- .../views/admin/import/ImportProgressPage.tsx | 1 + .../{ImportRoute.js => ImportRoute.tsx} | 6 +- .../{NewImportPage.js => NewImportPage.tsx} | 85 ++++-- .../views/admin/import/PrepareChannels.tsx | 8 +- ...areImportPage.js => PrepareImportPage.tsx} | 55 ++-- .../views/admin/import/PrepareUsers.tsx | 9 +- .../views/admin/import/UserDescriptor.tsx | 7 + .../views/admin/import/useErrorHandler.js | 32 --- .../views/admin/import/useErrorHandler.ts | 12 + .../rocketchat-i18n/i18n/en.i18n.json | 7 + apps/meteor/server/models/raw/Imports.ts | 6 +- .../notifications/notifications.module.ts | 4 +- apps/meteor/server/services/import/service.ts | 10 +- .../core-typings/src/import/IImporterInfo.ts | 4 + packages/core-typings/src/import/index.ts | 1 + .../model-typings/src/models/IImportsModel.ts | 2 +- .../src/v1/import/ImportersListParamsGET.ts | 16 ++ .../src/v1/import/StartImportParamsPOST.ts | 40 ++- packages/rest-typings/src/v1/import/import.ts | 5 +- packages/rest-typings/src/v1/import/index.ts | 1 + 76 files changed, 932 insertions(+), 694 deletions(-) delete mode 100644 apps/meteor/app/importer-csv/client/adder.js delete mode 100644 apps/meteor/app/importer-csv/client/index.ts delete mode 100644 apps/meteor/app/importer-csv/lib/info.js rename apps/meteor/app/importer-csv/server/{importer.js => CsvImporter.ts} (82%) delete mode 100644 apps/meteor/app/importer-hipchat-enterprise/client/adder.ts delete mode 100644 apps/meteor/app/importer-hipchat-enterprise/client/index.ts delete mode 100644 apps/meteor/app/importer-hipchat-enterprise/lib/info.ts rename apps/meteor/app/importer-hipchat-enterprise/server/{importer.js => HipChatEnterpriseImporter.js} (98%) rename apps/meteor/app/importer-pending-avatars/server/{importer.js => PendingAvatarImporter.ts} (70%) delete mode 100644 apps/meteor/app/importer-pending-avatars/server/info.js rename apps/meteor/app/importer-pending-files/server/{importer.js => PendingFileImporter.ts} (60%) delete mode 100644 apps/meteor/app/importer-pending-files/server/info.js delete mode 100644 apps/meteor/app/importer-slack-users/client/adder.js delete mode 100644 apps/meteor/app/importer-slack-users/client/index.ts delete mode 100644 apps/meteor/app/importer-slack-users/lib/info.js rename apps/meteor/app/importer-slack-users/server/{importer.js => SlackUsersImporter.ts} (67%) delete mode 100644 apps/meteor/app/importer-slack/client/adder.js delete mode 100644 apps/meteor/app/importer-slack/client/index.ts delete mode 100644 apps/meteor/app/importer-slack/lib/info.js rename apps/meteor/app/importer-slack/server/{importer.js => SlackImporter.ts} (74%) delete mode 100644 apps/meteor/app/importer/client/index.ts delete mode 100644 apps/meteor/app/importer/lib/ImporterInfo.js delete mode 100644 apps/meteor/app/importer/lib/Importers.js rename apps/meteor/app/importer/server/classes/{ImporterBase.js => Importer.ts} (67%) delete mode 100644 apps/meteor/app/importer/server/classes/ImporterProgress.js create mode 100644 apps/meteor/app/importer/server/classes/ImporterProgress.ts delete mode 100644 apps/meteor/app/importer/server/classes/ImporterSelection.js create mode 100644 apps/meteor/app/importer/server/classes/ImporterSelection.ts delete mode 100644 apps/meteor/app/importer/server/classes/ImporterSelectionChannel.js create mode 100644 apps/meteor/app/importer/server/classes/ImporterSelectionChannel.ts delete mode 100644 apps/meteor/app/importer/server/classes/ImporterSelectionUser.js create mode 100644 apps/meteor/app/importer/server/classes/ImporterSelectionUser.ts rename apps/meteor/app/importer/server/classes/{ImporterWebsocket.js => ImporterWebsocket.ts} (62%) create mode 100644 apps/meteor/app/importer/server/classes/ImportersContainer.ts create mode 100644 apps/meteor/app/importer/server/definitions/ImporterInfo.ts create mode 100644 apps/meteor/client/views/admin/import/ChannelDescriptor.tsx rename apps/meteor/client/views/admin/import/{ImportOperationSummary.js => ImportOperationSummary.tsx} (79%) rename apps/meteor/client/views/admin/import/{ImportOperationSummarySkeleton.js => ImportOperationSummarySkeleton.tsx} (79%) rename apps/meteor/client/views/admin/import/{ImportRoute.js => ImportRoute.tsx} (83%) rename apps/meteor/client/views/admin/import/{NewImportPage.js => NewImportPage.tsx} (76%) rename apps/meteor/client/views/admin/import/{PrepareImportPage.js => PrepareImportPage.tsx} (76%) create mode 100644 apps/meteor/client/views/admin/import/UserDescriptor.tsx delete mode 100644 apps/meteor/client/views/admin/import/useErrorHandler.js create mode 100644 apps/meteor/client/views/admin/import/useErrorHandler.ts create mode 100644 packages/core-typings/src/import/IImporterInfo.ts create mode 100644 packages/rest-typings/src/v1/import/ImportersListParamsGET.ts diff --git a/apps/meteor/app/api/server/v1/import.ts b/apps/meteor/app/api/server/v1/import.ts index d0fc8643e07c..54dbce4d82d1 100644 --- a/apps/meteor/app/api/server/v1/import.ts +++ b/apps/meteor/app/api/server/v1/import.ts @@ -10,10 +10,13 @@ import { isDownloadPendingFilesParamsPOST, isDownloadPendingAvatarsParamsPOST, isGetCurrentImportOperationParamsGET, + isImportersListParamsGET, isImportAddUsersParamsPOST, } from '@rocket.chat/rest-typings'; import { Meteor } from 'meteor/meteor'; +import { PendingAvatarImporter } from '../../../importer-pending-avatars/server/PendingAvatarImporter'; +import { PendingFileImporter } from '../../../importer-pending-files/server/PendingFileImporter'; import { Importers } from '../../../importer/server'; import { executeUploadImportFile, @@ -136,9 +139,8 @@ API.v1.addRoute( } const operation = await Import.newOperation(this.userId, importer.name, importer.key); - - importer.instance = new importer.importer(importer, operation); // eslint-disable-line new-cap - const count = await importer.instance.prepareFileCount(); + const instance = new PendingFileImporter(importer, operation); + const count = await instance.prepareFileCount(); return API.v1.success({ count, @@ -162,8 +164,8 @@ API.v1.addRoute( } const operation = await Import.newOperation(this.userId, importer.name, importer.key); - importer.instance = new importer.importer(importer, operation); // eslint-disable-line new-cap - const count = await importer.instance.prepareFileCount(); + const instance = new PendingAvatarImporter(importer, operation); + const count = await instance.prepareFileCount(); return API.v1.success({ count, @@ -189,6 +191,22 @@ API.v1.addRoute( }, ); +API.v1.addRoute( + 'importers.list', + { + authRequired: true, + validateParams: isImportersListParamsGET, + permissionsRequired: ['run-import'], + }, + { + async get() { + const importers = Importers.getAllVisible().map(({ key, name }) => ({ key, name })); + + return API.v1.success(importers); + }, + }, +); + API.v1.addRoute( 'import.clear', { diff --git a/apps/meteor/app/importer-csv/client/adder.js b/apps/meteor/app/importer-csv/client/adder.js deleted file mode 100644 index 929a84640be9..000000000000 --- a/apps/meteor/app/importer-csv/client/adder.js +++ /dev/null @@ -1,4 +0,0 @@ -import { Importers } from '../../importer/client'; -import { CsvImporterInfo } from '../lib/info'; - -Importers.add(new CsvImporterInfo()); diff --git a/apps/meteor/app/importer-csv/client/index.ts b/apps/meteor/app/importer-csv/client/index.ts deleted file mode 100644 index 44a1b3bab84c..000000000000 --- a/apps/meteor/app/importer-csv/client/index.ts +++ /dev/null @@ -1 +0,0 @@ -import './adder'; diff --git a/apps/meteor/app/importer-csv/lib/info.js b/apps/meteor/app/importer-csv/lib/info.js deleted file mode 100644 index c4fccdf74991..000000000000 --- a/apps/meteor/app/importer-csv/lib/info.js +++ /dev/null @@ -1,12 +0,0 @@ -import { ImporterInfo } from '../../importer/lib/ImporterInfo'; - -export class CsvImporterInfo extends ImporterInfo { - constructor() { - super('csv', 'CSV', 'application/zip', [ - { - text: 'Importer_CSV_Information', - href: 'https://rocket.chat/docs/administrator-guides/import/csv/', - }, - ]); - } -} diff --git a/apps/meteor/app/importer-csv/server/importer.js b/apps/meteor/app/importer-csv/server/CsvImporter.ts similarity index 82% rename from apps/meteor/app/importer-csv/server/importer.js rename to apps/meteor/app/importer-csv/server/CsvImporter.ts index c2f20f75615d..302aeb882ac5 100644 --- a/apps/meteor/app/importer-csv/server/importer.js +++ b/apps/meteor/app/importer-csv/server/CsvImporter.ts @@ -1,18 +1,23 @@ +import type { IImport } from '@rocket.chat/core-typings'; import { Settings, Users } from '@rocket.chat/models'; import { Random } from '@rocket.chat/random'; +import { parse } from 'csv-parse/lib/sync'; -import { Base, ProgressStep, ImporterWebsocket } from '../../importer/server'; +import { Importer, ProgressStep, ImporterWebsocket } from '../../importer/server'; +import type { IConverterOptions } from '../../importer/server/classes/ImportDataConverter'; +import type { ImporterProgress } from '../../importer/server/classes/ImporterProgress'; +import type { ImporterInfo } from '../../importer/server/definitions/ImporterInfo'; -export class CsvImporter extends Base { - constructor(info, importRecord, converterOptions = {}) { - super(info, importRecord, converterOptions); +export class CsvImporter extends Importer { + private csvParser: (csv: string) => string[]; - const { parse } = require('csv-parse/lib/sync'); + constructor(info: ImporterInfo, importRecord: IImport, converterOptions: IConverterOptions = {}) { + super(info, importRecord, converterOptions); this.csvParser = parse; } - async prepareUsingLocalFile(fullFilePath) { + async prepareUsingLocalFile(fullFilePath: string): Promise { this.logger.debug('start preparing import operation'); await this.converter.clearImportData(); @@ -40,17 +45,21 @@ export class CsvImporter extends Base { let messagesCount = 0; let usersCount = 0; let channelsCount = 0; - const dmRooms = new Map(); - const roomIds = new Map(); - const usedUsernames = new Set(); - const availableUsernames = new Set(); - - const getRoomId = (roomName) => { - if (!roomIds.has(roomName)) { - roomIds.set(roomName, Random.id()); + const dmRooms = new Set(); + const roomIds = new Map(); + const usedUsernames = new Set(); + const availableUsernames = new Set(); + + const getRoomId = (roomName: string) => { + const roomId = roomIds.get(roomName); + + if (roomId === undefined) { + const fallbackRoomId = Random.id(); + roomIds.set(roomName, fallbackRoomId); + return fallbackRoomId; } - return roomIds.get(roomName); + return roomId; }; for await (const entry of zip.getEntries()) { @@ -149,7 +158,7 @@ export class CsvImporter extends Base { continue; } - let data; + let data: { username: string; ts: string; text: string; otherUsername?: string; isDirect?: true }[]; const msgGroupData = item[1].split('.')[0]; // messages let isDirect = false; @@ -173,6 +182,10 @@ export class CsvImporter extends Base { if (isDirect) { for await (const msg of data) { + if (!msg.otherUsername) { + continue; + } + const sourceId = [msg.username, msg.otherUsername].sort().join('/'); if (!dmRooms.has(sourceId)) { @@ -182,7 +195,7 @@ export class CsvImporter extends Base { t: 'd', }); - dmRooms.set(sourceId, true); + dmRooms.add(sourceId); } const newMessage = { @@ -217,8 +230,6 @@ export class CsvImporter extends Base { } await super.updateRecord({ 'count.messages': messagesCount, 'messagesstatus': null }); - increaseProgressCount(); - continue; } increaseProgressCount(); diff --git a/apps/meteor/app/importer-csv/server/index.ts b/apps/meteor/app/importer-csv/server/index.ts index 8ab31878fb7b..2d913f740955 100644 --- a/apps/meteor/app/importer-csv/server/index.ts +++ b/apps/meteor/app/importer-csv/server/index.ts @@ -1,5 +1,8 @@ import { Importers } from '../../importer/server'; -import { CsvImporterInfo } from '../lib/info'; -import { CsvImporter } from './importer'; +import { CsvImporter } from './CsvImporter'; -Importers.add(new CsvImporterInfo(), CsvImporter); +Importers.add({ + key: 'csv', + name: 'CSV', + importer: CsvImporter, +}); diff --git a/apps/meteor/app/importer-hipchat-enterprise/client/adder.ts b/apps/meteor/app/importer-hipchat-enterprise/client/adder.ts deleted file mode 100644 index 2c03c872c7eb..000000000000 --- a/apps/meteor/app/importer-hipchat-enterprise/client/adder.ts +++ /dev/null @@ -1,4 +0,0 @@ -import { Importers } from '../../importer/client'; -import { HipChatEnterpriseImporterInfo } from '../lib/info'; - -Importers.add(new HipChatEnterpriseImporterInfo()); diff --git a/apps/meteor/app/importer-hipchat-enterprise/client/index.ts b/apps/meteor/app/importer-hipchat-enterprise/client/index.ts deleted file mode 100644 index 44a1b3bab84c..000000000000 --- a/apps/meteor/app/importer-hipchat-enterprise/client/index.ts +++ /dev/null @@ -1 +0,0 @@ -import './adder'; diff --git a/apps/meteor/app/importer-hipchat-enterprise/lib/info.ts b/apps/meteor/app/importer-hipchat-enterprise/lib/info.ts deleted file mode 100644 index 71c1f17a72d7..000000000000 --- a/apps/meteor/app/importer-hipchat-enterprise/lib/info.ts +++ /dev/null @@ -1,12 +0,0 @@ -import { ImporterInfo } from '../../importer/lib/ImporterInfo'; - -export class HipChatEnterpriseImporterInfo extends ImporterInfo { - constructor() { - super('hipchatenterprise', 'HipChat (tar.gz)', 'application/gzip', [ - { - text: 'Importer_HipChatEnterprise_Information', - href: 'https://rocket.chat/docs/administrator-guides/import/hipchat/enterprise/', - }, - ]); - } -} diff --git a/apps/meteor/app/importer-hipchat-enterprise/server/importer.js b/apps/meteor/app/importer-hipchat-enterprise/server/HipChatEnterpriseImporter.js similarity index 98% rename from apps/meteor/app/importer-hipchat-enterprise/server/importer.js rename to apps/meteor/app/importer-hipchat-enterprise/server/HipChatEnterpriseImporter.js index 18fefea074bf..ac3d278d82ab 100644 --- a/apps/meteor/app/importer-hipchat-enterprise/server/importer.js +++ b/apps/meteor/app/importer-hipchat-enterprise/server/HipChatEnterpriseImporter.js @@ -5,9 +5,10 @@ import { Readable } from 'stream'; import { Settings } from '@rocket.chat/models'; import { Meteor } from 'meteor/meteor'; -import { Base, ProgressStep } from '../../importer/server'; +import { Importer, ProgressStep } from '../../importer/server'; -export class HipChatEnterpriseImporter extends Base { +/** @deprecated HipChat was discontinued at 2019-02-15 */ +export class HipChatEnterpriseImporter extends Importer { constructor(info, importRecord, converterOptions = {}) { super(info, importRecord, converterOptions); diff --git a/apps/meteor/app/importer-hipchat-enterprise/server/index.ts b/apps/meteor/app/importer-hipchat-enterprise/server/index.ts index 097a5071306d..e50a9b9c4bd3 100644 --- a/apps/meteor/app/importer-hipchat-enterprise/server/index.ts +++ b/apps/meteor/app/importer-hipchat-enterprise/server/index.ts @@ -1,5 +1,8 @@ import { Importers } from '../../importer/server'; -import { HipChatEnterpriseImporterInfo } from '../lib/info'; -import { HipChatEnterpriseImporter } from './importer'; +import { HipChatEnterpriseImporter } from './HipChatEnterpriseImporter'; -Importers.add(new HipChatEnterpriseImporterInfo(), HipChatEnterpriseImporter); +Importers.add({ + key: 'hipchatenterprise', + name: 'HipChat (tar.gz)', + importer: HipChatEnterpriseImporter, +}); diff --git a/apps/meteor/app/importer-pending-avatars/server/importer.js b/apps/meteor/app/importer-pending-avatars/server/PendingAvatarImporter.ts similarity index 70% rename from apps/meteor/app/importer-pending-avatars/server/importer.js rename to apps/meteor/app/importer-pending-avatars/server/PendingAvatarImporter.ts index a7517285762b..0f6c8c7d41df 100644 --- a/apps/meteor/app/importer-pending-avatars/server/importer.js +++ b/apps/meteor/app/importer-pending-avatars/server/PendingAvatarImporter.ts @@ -1,14 +1,15 @@ import { Users } from '@rocket.chat/models'; -import { Base, ProgressStep, Selection } from '../../importer/server'; +import { Importer, ProgressStep, Selection } from '../../importer/server'; +import type { ImporterProgress } from '../../importer/server/classes/ImporterProgress'; import { setAvatarFromServiceWithValidation } from '../../lib/server/functions/setUserAvatar'; -export class PendingAvatarImporter extends Base { +export class PendingAvatarImporter extends Importer { async prepareFileCount() { this.logger.debug('start preparing import operation'); await super.updateProgress(ProgressStep.PREPARING_STARTED); - const users = await Users.findAllUsersWithPendingAvatar(); + const users = Users.findAllUsersWithPendingAvatar(); const fileCount = await users.count(); if (fileCount === 0) { @@ -19,26 +20,26 @@ export class PendingAvatarImporter extends Base { await this.updateRecord({ 'count.messages': fileCount, 'messagesstatus': null }); await this.addCountToTotal(fileCount); - const fileData = new Selection(this.name, [], [], fileCount); + const fileData = new Selection(this.info.name, [], [], fileCount); await this.updateRecord({ fileData }); await super.updateProgress(ProgressStep.IMPORTING_FILES); setImmediate(() => { - this.startImport(fileData); + void this.startImport(fileData); }); return fileCount; } - async startImport() { - const pendingFileUserList = await Users.findAllUsersWithPendingAvatar(); + async startImport(importSelection: Selection): Promise { + const pendingFileUserList = Users.findAllUsersWithPendingAvatar(); try { for await (const user of pendingFileUserList) { try { const { _pendingAvatarUrl: url, name, _id } = user; try { - if (!url || !url.startsWith('http')) { + if (!url?.startsWith('http')) { continue; } @@ -57,9 +58,9 @@ export class PendingAvatarImporter extends Base { } } catch (error) { // If the cursor expired, restart the method - if (error && error.codeName === 'CursorNotFound') { + if (this.isCursorNotFoundError(error)) { this.logger.info('CursorNotFound'); - return this.startImport(); + return this.startImport(importSelection); } await super.updateProgress(ProgressStep.ERROR); diff --git a/apps/meteor/app/importer-pending-avatars/server/index.ts b/apps/meteor/app/importer-pending-avatars/server/index.ts index 9c6bc278abee..b69c6de8e745 100644 --- a/apps/meteor/app/importer-pending-avatars/server/index.ts +++ b/apps/meteor/app/importer-pending-avatars/server/index.ts @@ -1,5 +1,9 @@ import { Importers } from '../../importer/server'; -import { PendingAvatarImporter } from './importer'; -import { PendingAvatarImporterInfo } from './info'; +import { PendingAvatarImporter } from './PendingAvatarImporter'; -Importers.add(new PendingAvatarImporterInfo(), PendingAvatarImporter); +Importers.add({ + key: 'pending-avatars', + name: 'Pending Avatars', + visible: false, + importer: PendingAvatarImporter, +}); diff --git a/apps/meteor/app/importer-pending-avatars/server/info.js b/apps/meteor/app/importer-pending-avatars/server/info.js deleted file mode 100644 index 32e8886cb538..000000000000 --- a/apps/meteor/app/importer-pending-avatars/server/info.js +++ /dev/null @@ -1,7 +0,0 @@ -import { ImporterInfo } from '../../importer/lib/ImporterInfo'; - -export class PendingAvatarImporterInfo extends ImporterInfo { - constructor() { - super('pending-avatars', 'Pending Avatars', ''); - } -} diff --git a/apps/meteor/app/importer-pending-files/server/importer.js b/apps/meteor/app/importer-pending-files/server/PendingFileImporter.ts similarity index 60% rename from apps/meteor/app/importer-pending-files/server/importer.js rename to apps/meteor/app/importer-pending-files/server/PendingFileImporter.ts index 3d8d083c495b..657a64002a5a 100644 --- a/apps/meteor/app/importer-pending-files/server/importer.js +++ b/apps/meteor/app/importer-pending-files/server/PendingFileImporter.ts @@ -1,17 +1,19 @@ import http from 'http'; import https from 'https'; +import type { IImport, MessageAttachment, IUpload } from '@rocket.chat/core-typings'; import { Messages } from '@rocket.chat/models'; import { Random } from '@rocket.chat/random'; import { FileUpload } from '../../file-upload/server'; -import { Base, ProgressStep, Selection } from '../../importer/server'; - -export class PendingFileImporter extends Base { - constructor(info, importRecord) { - super(info, importRecord); - this.userTags = []; - this.bots = {}; +import { Importer, ProgressStep, Selection } from '../../importer/server'; +import type { IConverterOptions } from '../../importer/server/classes/ImportDataConverter'; +import type { ImporterProgress } from '../../importer/server/classes/ImporterProgress'; +import type { ImporterInfo } from '../../importer/server/definitions/ImporterInfo'; + +export class PendingFileImporter extends Importer { + constructor(info: ImporterInfo, importRecord: IImport, converterOptions: IConverterOptions = {}) { + super(info, importRecord, converterOptions); } async prepareFileCount() { @@ -27,19 +29,19 @@ export class PendingFileImporter extends Base { await this.updateRecord({ 'count.messages': fileCount, 'messagesstatus': null }); await this.addCountToTotal(fileCount); - const fileData = new Selection(this.name, [], [], fileCount); + const fileData = new Selection(this.info.name, [], [], fileCount); await this.updateRecord({ fileData }); await super.updateProgress(ProgressStep.IMPORTING_FILES); setImmediate(() => { - this.startImport(fileData); + void this.startImport(fileData); }); return fileCount; } - async startImport() { - const downloadedFileIds = []; + async startImport(importSelection: Selection): Promise { + const downloadedFileIds: string[] = []; const maxFileCount = 10; const maxFileSize = 1024 * 1024 * 500; @@ -52,7 +54,7 @@ export class PendingFileImporter extends Base { return; } - return new Promise((resolve) => { + return new Promise((resolve) => { const handler = setInterval(() => { if (count + 1 >= maxFileCount) { return; @@ -68,15 +70,13 @@ export class PendingFileImporter extends Base { }); }; - const completeFile = async (details) => { + const completeFile = async (details: { size: number }) => { await this.addCountCompleted(1); count--; currentSize -= details.size; }; - const logError = (error) => { - this.logger.error(error); - }; + const logError = this.logger.error.bind(this.logger); try { const pendingFileMessageList = Messages.findAllImportedMessagesWithFilesToDownload(); @@ -86,16 +86,16 @@ export class PendingFileImporter extends Base { if (!_importFile || _importFile.downloaded || downloadedFileIds.includes(_importFile.id)) { await this.addCountCompleted(1); - return; + continue; } const url = _importFile.downloadUrl; - if (!url || !url.startsWith('http')) { + if (!url?.startsWith('http')) { await this.addCountCompleted(1); - return; + continue; } - const details = { + const details: { message_id: string; name: string; size: number; userId: string; rid: string; type?: string } = { message_id: `${message._id}-file-${_importFile.id}`, name: _importFile.name || Random.id(), size: _importFile.size || 0, @@ -105,7 +105,6 @@ export class PendingFileImporter extends Base { const requestModule = /https/i.test(url) ? https : http; const fileStore = FileUpload.getStore('Uploads'); - const reportProgress = this.reportProgress.bind(this); nextSize = details.size; await waitForFiles(); @@ -119,12 +118,12 @@ export class PendingFileImporter extends Base { details.type = contentType; } - const rawData = []; + const rawData: Uint8Array[] = []; res.on('data', (chunk) => { rawData.push(chunk); // Update progress more often on large files - reportProgress(); + this.reportProgress(); }); res.on('error', async (error) => { await completeFile(details); @@ -136,30 +135,8 @@ export class PendingFileImporter extends Base { // Bypass the fileStore filters const file = await fileStore._doInsert(details, Buffer.concat(rawData)); - const url = FileUpload.getPath(`${file._id}/${encodeURI(file.name)}`); - const attachment = { - title: file.name, - title_link: url, - }; - - if (/^image\/.+/.test(file.type)) { - attachment.image_url = url; - attachment.image_type = file.type; - attachment.image_size = file.size; - attachment.image_dimensions = file.identify != null ? file.identify.size : undefined; - } - - if (/^audio\/.+/.test(file.type)) { - attachment.audio_url = url; - attachment.audio_type = file.type; - attachment.audio_size = file.size; - } - - if (/^video\/.+/.test(file.type)) { - attachment.video_url = url; - attachment.video_type = file.type; - attachment.video_size = file.size; - } + const url = FileUpload.getPath(`${file._id}/${encodeURI(file.name || '')}`); + const attachment = this.getMessageAttachment(file, url); await Messages.setImportFileRocketChatAttachment(_importFile.id, url, attachment); await completeFile(details); @@ -175,8 +152,9 @@ export class PendingFileImporter extends Base { } } catch (error) { // If the cursor expired, restart the method - if (error && error.codeName === 'CursorNotFound') { - return this.startImport(); + if (this.isCursorNotFoundError(error)) { + this.logger.info('CursorNotFound'); + return this.startImport(importSelection); } await super.updateProgress(ProgressStep.ERROR); @@ -186,4 +164,44 @@ export class PendingFileImporter extends Base { await super.updateProgress(ProgressStep.DONE); return this.getProgress(); } + + getMessageAttachment(file: IUpload, url: string): MessageAttachment { + if (file.type) { + if (/^image\/.+/.test(file.type)) { + return { + title: file.name, + title_link: url, + image_url: url, + image_type: file.type, + image_size: file.size, + image_dimensions: file.identify ? file.identify.size : undefined, + }; + } + + if (/^audio\/.+/.test(file.type)) { + return { + title: file.name, + title_link: url, + audio_url: url, + audio_type: file.type, + audio_size: file.size, + }; + } + + if (/^video\/.+/.test(file.type)) { + return { + title: file.name, + title_link: url, + video_url: url, + video_type: file.type, + video_size: file.size, + }; + } + } + + return { + title: file.name, + title_link: url, + }; + } } diff --git a/apps/meteor/app/importer-pending-files/server/index.ts b/apps/meteor/app/importer-pending-files/server/index.ts index 2486e6b0f0d9..24961551cdb5 100644 --- a/apps/meteor/app/importer-pending-files/server/index.ts +++ b/apps/meteor/app/importer-pending-files/server/index.ts @@ -1,5 +1,9 @@ import { Importers } from '../../importer/server'; -import { PendingFileImporter } from './importer'; -import { PendingFileImporterInfo } from './info'; +import { PendingFileImporter } from './PendingFileImporter'; -Importers.add(new PendingFileImporterInfo(), PendingFileImporter); +Importers.add({ + key: 'pending-files', + name: 'Pending Files', + visible: false, + importer: PendingFileImporter, +}); diff --git a/apps/meteor/app/importer-pending-files/server/info.js b/apps/meteor/app/importer-pending-files/server/info.js deleted file mode 100644 index f12d01a52611..000000000000 --- a/apps/meteor/app/importer-pending-files/server/info.js +++ /dev/null @@ -1,7 +0,0 @@ -import { ImporterInfo } from '../../importer/lib/ImporterInfo'; - -export class PendingFileImporterInfo extends ImporterInfo { - constructor() { - super('pending-files', 'Pending Files', ''); - } -} diff --git a/apps/meteor/app/importer-slack-users/client/adder.js b/apps/meteor/app/importer-slack-users/client/adder.js deleted file mode 100644 index 8c6b0276bfdd..000000000000 --- a/apps/meteor/app/importer-slack-users/client/adder.js +++ /dev/null @@ -1,4 +0,0 @@ -import { Importers } from '../../importer/client'; -import { SlackUsersImporterInfo } from '../lib/info'; - -Importers.add(new SlackUsersImporterInfo()); diff --git a/apps/meteor/app/importer-slack-users/client/index.ts b/apps/meteor/app/importer-slack-users/client/index.ts deleted file mode 100644 index 44a1b3bab84c..000000000000 --- a/apps/meteor/app/importer-slack-users/client/index.ts +++ /dev/null @@ -1 +0,0 @@ -import './adder'; diff --git a/apps/meteor/app/importer-slack-users/lib/info.js b/apps/meteor/app/importer-slack-users/lib/info.js deleted file mode 100644 index 999c5e76cfc9..000000000000 --- a/apps/meteor/app/importer-slack-users/lib/info.js +++ /dev/null @@ -1,12 +0,0 @@ -import { ImporterInfo } from '../../importer/lib/ImporterInfo'; - -export class SlackUsersImporterInfo extends ImporterInfo { - constructor() { - super('slack-users', 'Slack_Users', 'text/csv', [ - { - text: 'Importer_Slack_Users_CSV_Information', - href: 'https://rocket.chat/docs/administrator-guides/import/slack/users', - }, - ]); - } -} diff --git a/apps/meteor/app/importer-slack-users/server/importer.js b/apps/meteor/app/importer-slack-users/server/SlackUsersImporter.ts similarity index 67% rename from apps/meteor/app/importer-slack-users/server/importer.js rename to apps/meteor/app/importer-slack-users/server/SlackUsersImporter.ts index 177885b7833a..2c26531bd5c4 100644 --- a/apps/meteor/app/importer-slack-users/server/importer.js +++ b/apps/meteor/app/importer-slack-users/server/SlackUsersImporter.ts @@ -1,20 +1,25 @@ import fs from 'fs'; +import type { IImport, IImportUser } from '@rocket.chat/core-typings'; import { Settings } from '@rocket.chat/models'; +import { parse } from 'csv-parse/lib/sync'; import { RocketChatFile } from '../../file/server'; -import { Base, ProgressStep } from '../../importer/server'; +import { Importer, ProgressStep } from '../../importer/server'; +import type { IConverterOptions } from '../../importer/server/classes/ImportDataConverter'; +import type { ImporterProgress } from '../../importer/server/classes/ImporterProgress'; +import type { ImporterInfo } from '../../importer/server/definitions/ImporterInfo'; -export class SlackUsersImporter extends Base { - constructor(info, importRecord, converterOptions = {}) { - super(info, importRecord, converterOptions); +export class SlackUsersImporter extends Importer { + private csvParser: (csv: string) => string[]; - const { parse } = require('csv-parse/lib/sync'); + constructor(info: ImporterInfo, importRecord: IImport, converterOptions: IConverterOptions = {}) { + super(info, importRecord, converterOptions); this.csvParser = parse; } - async prepareUsingLocalFile(fullFilePath) { + async prepareUsingLocalFile(fullFilePath: string): Promise { this.logger.debug('start preparing import operation'); await this.converter.clearImportData(); @@ -27,6 +32,12 @@ export class SlackUsersImporter extends Base { const data = buffer.toString('base64'); const dataURI = `data:${contentType};base64,${data}`; + return this.prepare(dataURI, fileName || ''); + } + + async prepare(dataURI: string, fileName: string): Promise { + this.logger.debug('start preparing import operation'); + await this.converter.clearImportData(); await this.updateRecord({ file: fileName }); await super.updateProgress(ProgressStep.PREPARING_USERS); @@ -49,7 +60,7 @@ export class SlackUsersImporter extends Base { const name = user[7] || user[8] || username; - const newUser = { + const newUser: IImportUser = { emails: [email], importIds: [email], username, diff --git a/apps/meteor/app/importer-slack-users/server/index.ts b/apps/meteor/app/importer-slack-users/server/index.ts index 6d23f6939b13..ab99ede8f912 100644 --- a/apps/meteor/app/importer-slack-users/server/index.ts +++ b/apps/meteor/app/importer-slack-users/server/index.ts @@ -1,5 +1,8 @@ import { Importers } from '../../importer/server'; -import { SlackUsersImporterInfo } from '../lib/info'; -import { SlackUsersImporter } from './importer'; +import { SlackUsersImporter } from './SlackUsersImporter'; -Importers.add(new SlackUsersImporterInfo(), SlackUsersImporter); +Importers.add({ + key: 'slack-users', + name: 'Slack_Users', + importer: SlackUsersImporter, +}); diff --git a/apps/meteor/app/importer-slack/client/adder.js b/apps/meteor/app/importer-slack/client/adder.js deleted file mode 100644 index 06e1941f38df..000000000000 --- a/apps/meteor/app/importer-slack/client/adder.js +++ /dev/null @@ -1,4 +0,0 @@ -import { Importers } from '../../importer/client'; -import { SlackImporterInfo } from '../lib/info'; - -Importers.add(new SlackImporterInfo()); diff --git a/apps/meteor/app/importer-slack/client/index.ts b/apps/meteor/app/importer-slack/client/index.ts deleted file mode 100644 index 44a1b3bab84c..000000000000 --- a/apps/meteor/app/importer-slack/client/index.ts +++ /dev/null @@ -1 +0,0 @@ -import './adder'; diff --git a/apps/meteor/app/importer-slack/lib/info.js b/apps/meteor/app/importer-slack/lib/info.js deleted file mode 100644 index bf431e8f7c58..000000000000 --- a/apps/meteor/app/importer-slack/lib/info.js +++ /dev/null @@ -1,7 +0,0 @@ -import { ImporterInfo } from '../../importer/lib/ImporterInfo'; - -export class SlackImporterInfo extends ImporterInfo { - constructor() { - super('slack', 'Slack', 'application/zip'); - } -} diff --git a/apps/meteor/app/importer-slack/server/importer.js b/apps/meteor/app/importer-slack/server/SlackImporter.ts similarity index 74% rename from apps/meteor/app/importer-slack/server/importer.js rename to apps/meteor/app/importer-slack/server/SlackImporter.ts index a5b0da6e549d..7d56c7784b76 100644 --- a/apps/meteor/app/importer-slack/server/importer.js +++ b/apps/meteor/app/importer-slack/server/SlackImporter.ts @@ -1,26 +1,128 @@ +import type { IImportUser, IImportMessage, IImportPendingFile } from '@rocket.chat/core-typings'; import { Messages, Settings, ImportData } from '@rocket.chat/models'; -import _ from 'underscore'; +import type { IZipEntry } from 'adm-zip'; -import { Base, ProgressStep, ImporterWebsocket } from '../../importer/server'; +import { Importer, ProgressStep, ImporterWebsocket } from '../../importer/server'; +import type { ImporterProgress } from '../../importer/server/classes/ImporterProgress'; import { MentionsParser } from '../../mentions/lib/MentionsParser'; import { settings } from '../../settings/server'; import { getUserAvatarURL } from '../../utils/server/getUserAvatarURL'; -export class SlackImporter extends Base { - parseData(data) { - const dataString = data.toString(); - try { - this.logger.debug('parsing file contents'); - return JSON.parse(dataString); - } catch (e) { - this.logger.error(e); - return false; - } - } - - async prepareChannelsFile(entry) { +type SlackChannel = { + id: string; + name: string; + topic?: { + value: string; + creator: string; + last_set: number; + }; + members: string[]; + purpose?: { + value: string; + creator: string; + last_set: number; + }; + created: number; + creator: string | null; + is_general: boolean; + is_archived: boolean; +}; + +type SlackUser = { + id: string; + name: string; + profile: { + real_name: string; + email: string; + image_512: string; + image_original: string; + status_text: string; + title: string; + }; + tz_offset: number; + deleted: boolean; + is_bot: boolean; +}; + +type SlackFile = { + id: string; + url_private_download: string; + size: number; + name: string; + is_external: boolean; +}; + +type SlackMessage = { + id: string; + ts: string; + user: string; + reactions?: { + name: string; + users: string[]; + }[]; + type: 'message'; + subtype?: string; + files?: SlackFile[]; + text: string; + edited?: { + ts: string; + user: string; + }; + thread_ts?: string; + reply_users?: string[]; + reply_count?: number; + replies?: { + user: string; + }[]; + latest_reply: string; + icons?: { + emoji: string; + }; + attachments?: SlackAttachment[]; +} & ( + | { + subtype: 'channel_purpose' | 'group_purpose'; + purpose: string; + } + | { + subtype: 'channel_join' | 'group_join' | 'channel_leave' | 'group_leave'; + } + | { + subtype: 'channel_topic' | 'group_topic'; + topic: string; + } + | { + subtype: 'channel_name' | 'group_name'; + name: string; + } + | { + subtype: 'pinned_item'; + attachments: SlackAttachment[]; + } + | { + subtype: 'file_share'; + file: SlackFile; + } + | { + subtype: 'me_message'; + } +); + +type SlackAttachment = { + text: string; + title: string; + fallback: string; + author_subname: string; +}; + +export class SlackImporter extends Importer { + private _useUpsert = false; + + async prepareChannelsFile(entry: IZipEntry): Promise { await super.updateProgress(ProgressStep.PREPARING_CHANNELS); - const data = JSON.parse(entry.getData().toString()).filter((channel) => channel.creator != null); + const data = (JSON.parse(entry.getData().toString()) as SlackChannel[]).filter( + (channel): channel is SlackChannel & { creator: string } => 'creator' in channel && channel.creator != null, + ); this.logger.debug(`loaded ${data.length} channels.`); @@ -46,9 +148,11 @@ export class SlackImporter extends Base { return data.length; } - async prepareGroupsFile(entry) { + async prepareGroupsFile(entry: IZipEntry): Promise { await super.updateProgress(ProgressStep.PREPARING_CHANNELS); - const data = JSON.parse(entry.getData().toString()).filter((channel) => channel.creator != null); + const data = (JSON.parse(entry.getData().toString()) as SlackChannel[]).filter( + (channel): channel is SlackChannel & { creator: string } => 'creator' in channel && channel.creator != null, + ); this.logger.debug(`loaded ${data.length} groups.`); @@ -73,15 +177,17 @@ export class SlackImporter extends Base { return data.length; } - async prepareMpimpsFile(entry) { + async prepareMpimpsFile(entry: IZipEntry): Promise { await super.updateProgress(ProgressStep.PREPARING_CHANNELS); - const data = JSON.parse(entry.getData().toString()).filter((channel) => channel.creator != null); + const data = (JSON.parse(entry.getData().toString()) as SlackChannel[]).filter( + (channel): channel is SlackChannel & { creator: string } => 'creator' in channel && channel.creator != null, + ); this.logger.debug(`loaded ${data.length} mpims.`); await this.addCountToTotal(data.length); - const maxUsers = settings.get('DirectMesssage_maxUsers') || 1; + const maxUsers = settings.get('DirectMesssage_maxUsers') || 1; for await (const channel of data) { await this.converter.addChannel({ @@ -102,9 +208,9 @@ export class SlackImporter extends Base { return data.length; } - async prepareDMsFile(entry) { + async prepareDMsFile(entry: IZipEntry): Promise { await super.updateProgress(ProgressStep.PREPARING_CHANNELS); - const data = JSON.parse(entry.getData().toString()); + const data = JSON.parse(entry.getData().toString()) as SlackChannel[]; this.logger.debug(`loaded ${data.length} dms.`); @@ -121,9 +227,9 @@ export class SlackImporter extends Base { return data.length; } - async prepareUsersFile(entry) { + async prepareUsersFile(entry: IZipEntry): Promise { await super.updateProgress(ProgressStep.PREPARING_USERS); - const data = JSON.parse(entry.getData().toString()); + const data = JSON.parse(entry.getData().toString()) as SlackUser[]; this.logger.debug(`loaded ${data.length} users.`); @@ -132,7 +238,7 @@ export class SlackImporter extends Base { await this.addCountToTotal(data.length); for await (const user of data) { - const newUser = { + const newUser: IImportUser = { emails: [], importIds: [user.id], username: user.name, @@ -160,7 +266,7 @@ export class SlackImporter extends Base { return data.length; } - async prepareUsingLocalFile(fullFilePath) { + async prepareUsingLocalFile(fullFilePath: string): Promise { this.logger.debug('start preparing import operation'); await this.converter.clearImportData(); @@ -234,7 +340,7 @@ export class SlackImporter extends Base { await Settings.incrementValueById('Slack_Importer_Count', userCount); } - const missedTypes = {}; + const missedTypes: Record = {}; // If we have no slack message yet, then we can insert them instead of upserting this._useUpsert = !(await Messages.findOne({ _id: /slack\-.*/ })); @@ -262,7 +368,7 @@ export class SlackImporter extends Base { await super.updateProgress(ProgressStep.PREPARING_MESSAGES); } - const tempMessages = JSON.parse(entry.getData().toString()); + const tempMessages = JSON.parse(entry.getData().toString()) as SlackMessage[]; messagesCount += tempMessages.length; await this.updateRecord({ messagesstatus: `${channel}/${date}` }); await this.addCountToTotal(tempMessages.length); @@ -285,7 +391,7 @@ export class SlackImporter extends Base { increaseProgress(); } - if (!_.isEmpty(missedTypes)) { + if (Object.keys(missedTypes).length > 0) { this.logger.info('Missed import types:', missedTypes); } } catch (e) { @@ -299,7 +405,7 @@ export class SlackImporter extends Base { return this.progress; } - parseMentions(newMessage) { + parseMentions(newMessage: IImportMessage): void { const mentionsParser = new MentionsParser({ pattern: () => '[0-9a-zA-Z]+', useRealName: () => settings.get('UI_Use_Real_Name'), @@ -308,8 +414,8 @@ export class SlackImporter extends Base { const users = mentionsParser .getUserMentions(newMessage.msg) - .filter((u) => u) - .map((uid) => this._replaceSlackUserId(uid.slice(1, uid.length))); + .filter((u: string) => u) + .map((uid: string) => this._replaceSlackUserId(uid.slice(1, uid.length))); if (users.length) { if (!newMessage.mentions) { newMessage.mentions = []; @@ -319,8 +425,8 @@ export class SlackImporter extends Base { const channels = mentionsParser .getChannelMentions(newMessage.msg) - .filter((c) => c) - .map((name) => name.slice(1, name.length)); + .filter((c: string) => c) + .map((name: string) => name.slice(1, name.length)); if (channels.length) { if (!newMessage.channels) { newMessage.channels = []; @@ -329,8 +435,13 @@ export class SlackImporter extends Base { } } - async processMessageSubType(message, slackChannelId, newMessage, missedTypes) { - const ignoreTypes = { bot_add: true, file_comment: true, file_mention: true }; + async processMessageSubType( + message: SlackMessage, + slackChannelId: string, + newMessage: IImportMessage, + missedTypes: Record, + ): Promise { + const ignoreTypes: Record = { bot_add: true, file_comment: true, file_mention: true }; switch (message.subtype) { case 'channel_join': @@ -377,7 +488,7 @@ export class SlackImporter extends Base { case 'file_share': if (message.file?.url_private_download) { const fileId = this.makeSlackMessageId(slackChannelId, message.ts, 'share'); - const fileMessage = { + const fileMessage: IImportMessage = { _id: fileId, rid: newMessage.rid, ts: newMessage.ts, @@ -406,7 +517,7 @@ export class SlackImporter extends Base { return false; } - makeSlackMessageId(channelId, ts, fileIndex = undefined) { + makeSlackMessageId(channelId: string, ts: string, fileIndex?: string): string { const base = `slack-${channelId}-${ts.replace(/\./g, '-')}`; if (fileIndex) { @@ -416,12 +527,13 @@ export class SlackImporter extends Base { return base; } - async prepareMessageObject(message, missedTypes, slackChannelId) { + async prepareMessageObject(message: SlackMessage, missedTypes: Record, slackChannelId: string): Promise { const id = this.makeSlackMessageId(slackChannelId, message.ts); - const newMessage = { + const newMessage: IImportMessage = { _id: id, rid: slackChannelId, ts: new Date(parseInt(message.ts.split('.')[0]) * 1000), + msg: '', u: { _id: this._replaceSlackUserId(message.user), }, @@ -429,17 +541,13 @@ export class SlackImporter extends Base { // Process the reactions if (message.reactions && message.reactions.length > 0) { - newMessage.reactions = new Map(); - - message.reactions.forEach((reaction) => { + newMessage.reactions = message.reactions.reduce((newReactions, reaction) => { const name = `:${reaction.name}:`; - if (reaction.users && reaction.users.length) { - newMessage.reactions.set(name, { - name, - users: this._replaceSlackUserIds(reaction.users), - }); - } - }); + return { + ...newReactions, + ...(reaction.users?.length ? { name: { name, users: this._replaceSlackUserIds(reaction.users) } } : {}), + }; + }, {} as Required['reactions']); } if (message.type === 'message') { @@ -448,8 +556,8 @@ export class SlackImporter extends Base { const promises = message.files.map(async (file) => { fileIndex++; - const fileId = this.makeSlackMessageId(slackChannelId, message.ts, fileIndex); - const fileMessage = { + const fileId = this.makeSlackMessageId(slackChannelId, message.ts, String(fileIndex)); + const fileMessage: IImportMessage = { _id: fileId, rid: slackChannelId, ts: newMessage.ts, @@ -493,21 +601,21 @@ export class SlackImporter extends Base { if (message.thread_ts) { if (message.thread_ts === message.ts) { if (message.reply_users) { - const replies = new Set(); - message.reply_users.forEach((item) => { + const replies = new Set(); + message.reply_users.forEach((item: string) => { replies.add(this._replaceSlackUserId(item)); }); - if (replies.length) { + if (replies.size) { newMessage.replies = Array.from(replies); } } else if (message.replies) { - const replies = new Set(); - message.repĺies.forEach((item) => { + const replies = new Set(); + message.replies.forEach((item: { user: string }) => { replies.add(this._replaceSlackUserId(item.user)); }); - if (replies.length) { + if (replies.size) { newMessage.replies = Array.from(replies); } } else { @@ -532,7 +640,7 @@ export class SlackImporter extends Base { newMessage.attachments = this.convertMessageAttachments(message.attachments); } - if (message.icons && message.icons.emoji) { + if (message.icons?.emoji) { newMessage.emoji = message.icons.emoji; } @@ -542,7 +650,7 @@ export class SlackImporter extends Base { } } - _replaceSlackUserId(userId) { + _replaceSlackUserId(userId: string): string { if (userId === 'USLACKBOT') { return 'rocket.cat'; } @@ -550,14 +658,14 @@ export class SlackImporter extends Base { return userId; } - _replaceSlackUserIds(members) { + _replaceSlackUserIds(members: string[]) { if (!members?.length) { return []; } return members.map((userId) => this._replaceSlackUserId(userId)); } - convertSlackMessageToRocketChat(message) { + convertSlackMessageToRocketChat(message: string): string { if (message) { message = message.replace(//g, '@all'); message = message.replace(//g, '@all'); @@ -581,7 +689,7 @@ export class SlackImporter extends Base { return message; } - convertSlackFileToPendingFile(file) { + convertSlackFileToPendingFile(file: SlackFile): IImportPendingFile { return { downloadUrl: file.url_private_download, id: file.id, @@ -595,9 +703,9 @@ export class SlackImporter extends Base { }; } - convertMessageAttachments(attachments) { - if (!attachments || !attachments.length) { - return attachments; + convertMessageAttachments(attachments: SlackAttachment[]): IImportMessage['attachments'] { + if (!attachments?.length) { + return undefined; } return attachments.map((attachment) => ({ diff --git a/apps/meteor/app/importer-slack/server/index.ts b/apps/meteor/app/importer-slack/server/index.ts index 6442d31dfa42..b8040d77538a 100644 --- a/apps/meteor/app/importer-slack/server/index.ts +++ b/apps/meteor/app/importer-slack/server/index.ts @@ -1,5 +1,8 @@ import { Importers } from '../../importer/server'; -import { SlackImporterInfo } from '../lib/info'; -import { SlackImporter } from './importer'; +import { SlackImporter } from './SlackImporter'; -Importers.add(new SlackImporterInfo(), SlackImporter); +Importers.add({ + key: 'slack', + name: 'Slack', + importer: SlackImporter, +}); diff --git a/apps/meteor/app/importer/client/index.ts b/apps/meteor/app/importer/client/index.ts deleted file mode 100644 index 7cd9fa3c3440..000000000000 --- a/apps/meteor/app/importer/client/index.ts +++ /dev/null @@ -1 +0,0 @@ -export { Importers } from '../lib/Importers'; diff --git a/apps/meteor/app/importer/lib/ImporterInfo.js b/apps/meteor/app/importer/lib/ImporterInfo.js deleted file mode 100644 index 627c1d786a28..000000000000 --- a/apps/meteor/app/importer/lib/ImporterInfo.js +++ /dev/null @@ -1,19 +0,0 @@ -export class ImporterInfo { - /** - * Creates a new class which contains information about the importer. - * - * @param {string} key The unique key of this importer. - * @param {string} name The i18n name. - * @param {string} mimeType The type of file it expects. - * @param {{ href: string, text: string }[]} warnings An array of warning objects. `{ href, text }` - */ - constructor(key, name = '', mimeType = '', warnings = []) { - this.key = key; - this.name = name; - this.mimeType = mimeType; - this.warnings = warnings; - - this.importer = undefined; - this.instance = undefined; - } -} diff --git a/apps/meteor/app/importer/lib/ImporterProgressStep.ts b/apps/meteor/app/importer/lib/ImporterProgressStep.ts index da81c050967c..1b5ffe53c93f 100644 --- a/apps/meteor/app/importer/lib/ImporterProgressStep.ts +++ b/apps/meteor/app/importer/lib/ImporterProgressStep.ts @@ -1,3 +1,5 @@ +import type { IImportProgress } from '@rocket.chat/core-typings'; + /** The progress step that an importer is at. */ export const ProgressStep = Object.freeze({ NEW: 'importer_new', @@ -22,20 +24,20 @@ export const ProgressStep = Object.freeze({ DONE: 'importer_done', ERROR: 'importer_import_failed', CANCELLED: 'importer_import_cancelled', -}); +} satisfies Record); -export const ImportWaitingStates = [ProgressStep.NEW, ProgressStep.UPLOADING, ProgressStep.DOWNLOADING_FILE]; +export const ImportWaitingStates: IImportProgress['step'][] = [ProgressStep.NEW, ProgressStep.UPLOADING, ProgressStep.DOWNLOADING_FILE]; -export const ImportFileReadyStates = [ProgressStep.FILE_LOADED]; +export const ImportFileReadyStates: IImportProgress['step'][] = [ProgressStep.FILE_LOADED]; -export const ImportPreparingStartedStates = [ +export const ImportPreparingStartedStates: IImportProgress['step'][] = [ ProgressStep.PREPARING_STARTED, ProgressStep.PREPARING_USERS, ProgressStep.PREPARING_CHANNELS, ProgressStep.PREPARING_MESSAGES, ]; -export const ImportingStartedStates = [ +export const ImportingStartedStates: IImportProgress['step'][] = [ ProgressStep.IMPORTING_STARTED, ProgressStep.IMPORTING_USERS, ProgressStep.IMPORTING_CHANNELS, @@ -44,4 +46,4 @@ export const ImportingStartedStates = [ ProgressStep.FINISHING, ]; -export const ImportingErrorStates = [ProgressStep.ERROR, ProgressStep.CANCELLED]; +export const ImportingErrorStates: IImportProgress['step'][] = [ProgressStep.ERROR, ProgressStep.CANCELLED]; diff --git a/apps/meteor/app/importer/lib/Importers.js b/apps/meteor/app/importer/lib/Importers.js deleted file mode 100644 index 13acf2b616d8..000000000000 --- a/apps/meteor/app/importer/lib/Importers.js +++ /dev/null @@ -1,47 +0,0 @@ -import { ImporterInfo } from './ImporterInfo'; - -/** Container class which holds all of the importer details. */ -class ImportersContainer { - constructor() { - this.importers = new Map(); - } - - /** - * Adds an importer to the import collection. Adding it more than once will - * overwrite the previous one. - * - * @param {ImporterInfo} info The information related to the importer. - * @param {*} [importer] The class for the importer, will be undefined on the client. - */ - add(info, importer) { - if (!(info instanceof ImporterInfo)) { - throw new Error('The importer must be a valid ImporterInfo instance.'); - } - - info.importer = importer; - - this.importers.set(info.key, info); - - return this.importers.get(info.key); - } - - /** - * Gets the importer information that is stored. - * - * @param {string} key The key of the importer. - */ - get(key) { - return this.importers.get(key); - } - - /** - * Gets all of the importers in array format. - * - * @returns {ImporterInfo[]} The array of importer information. - */ - getAll() { - return Array.from(this.importers.values()); - } -} - -export const Importers = new ImportersContainer(); diff --git a/apps/meteor/app/importer/server/classes/ImportDataConverter.ts b/apps/meteor/app/importer/server/classes/ImportDataConverter.ts index 1b596d625d9b..fff8e8c9efd4 100644 --- a/apps/meteor/app/importer/server/classes/ImportDataConverter.ts +++ b/apps/meteor/app/importer/server/classes/ImportDataConverter.ts @@ -60,7 +60,6 @@ export type IConverterOptions = { flagEmailsAsVerified?: boolean; skipExistingUsers?: boolean; skipNewUsers?: boolean; - bindSkippedUsers?: boolean; skipUserCallbacks?: boolean; skipDefaultChannels?: boolean; @@ -100,7 +99,6 @@ export class ImportDataConverter { flagEmailsAsVerified: false, skipExistingUsers: false, skipNewUsers: false, - bindSkippedUsers: false, }; this._userCache = new Map(); this._userDisplayNameCache = new Map(); @@ -396,18 +394,6 @@ export class ImportDataConverter { } async findExistingUser(data: IImportUser): Promise { - // If we're gonna force-bind importIds, we search for them first to ensure they are unique - if (this._options.bindSkippedUsers) { - // #TODO: Use a single operation for multiple IDs - // (Currently there's no existing use case with multiple IDs being passed to this function) - for await (const importId of data.importIds) { - const importedUser = await Users.findOneByImportId(importId, {}); - if (importedUser) { - return importedUser; - } - } - } - if (data.emails.length) { const emailUser = await Users.findOneByEmailAddress(data.emails[0], {}); @@ -488,13 +474,6 @@ export class ImportDataConverter { const existingUser = await this.findExistingUser(data); if (existingUser && this._options.skipExistingUsers) { - if (this._options.bindSkippedUsers) { - const newImportIds = data.importIds.filter((importId) => !(existingUser as IUser).importIds?.includes(importId)); - if (newImportIds.length) { - await Users.addImportIds(existingUser._id, newImportIds); - } - } - await this.skipRecord(_id); skippedCount++; continue; diff --git a/apps/meteor/app/importer/server/classes/ImporterBase.js b/apps/meteor/app/importer/server/classes/Importer.ts similarity index 67% rename from apps/meteor/app/importer/server/classes/ImporterBase.js rename to apps/meteor/app/importer/server/classes/Importer.ts index 061644130a66..92f506b379ad 100644 --- a/apps/meteor/app/importer/server/classes/ImporterBase.js +++ b/apps/meteor/app/importer/server/classes/Importer.ts @@ -1,71 +1,87 @@ +import type { IImport, IImportRecord, IImportChannel, IImportUser, IImportProgress } from '@rocket.chat/core-typings'; import { Logger } from '@rocket.chat/logger'; import { Settings, ImportData, Imports } from '@rocket.chat/models'; import AdmZip from 'adm-zip'; +import type { MatchKeysAndValues, MongoServerError } from 'mongodb'; import { Selection, SelectionChannel, SelectionUser } from '..'; import { callbacks } from '../../../../lib/callbacks'; import { t } from '../../../utils/lib/i18n'; -import { ImporterInfo } from '../../lib/ImporterInfo'; import { ProgressStep, ImportPreparingStartedStates } from '../../lib/ImporterProgressStep'; +import type { ImporterInfo } from '../definitions/ImporterInfo'; import { ImportDataConverter } from './ImportDataConverter'; -import { Progress } from './ImporterProgress'; +import type { IConverterOptions } from './ImportDataConverter'; +import { ImporterProgress } from './ImporterProgress'; import { ImporterWebsocket } from './ImporterWebsocket'; +type OldSettings = { + allowedDomainList?: string | null; + allowUsernameChange?: boolean | null; + maxFileSize?: number | null; + mediaTypeWhiteList?: string | null; + mediaTypeBlackList?: string | null; +}; + /** * Base class for all of the importers. */ -export class Base { - constructor(info, importRecord, converterOptions = {}) { - if (!(info instanceof ImporterInfo)) { +export class Importer { + private _reportProgressHandler: ReturnType | undefined; + + protected AdmZip = AdmZip; + + protected converter: ImportDataConverter; + + protected info: ImporterInfo; + + protected logger: Logger; + + protected oldSettings: OldSettings; + + protected _lastProgressReportTotal = 0; + + public importRecord: IImport; + + public progress: ImporterProgress; + + constructor(info: ImporterInfo, importRecord: IImport, converterOptions: IConverterOptions = {}) { + if (!info.key || !info.importer) { throw new Error('Information passed in must be a valid ImporterInfo instance.'); } - this.AdmZip = AdmZip; this.converter = new ImportDataConverter(converterOptions); - this.startImport = this.startImport.bind(this); - this.getProgress = this.getProgress.bind(this); - this.updateProgress = this.updateProgress.bind(this); - this.addCountToTotal = this.addCountToTotal.bind(this); - this.addCountCompleted = this.addCountCompleted.bind(this); - this.updateRecord = this.updateRecord.bind(this); - this.info = info; this.logger = new Logger(`${this.info.name} Importer`); this.converter.setLogger(this.logger); this.importRecord = importRecord; - this.progress = new Progress(this.info.key, this.info.name); - this.users = {}; - this.channels = {}; - this.messages = {}; + this.progress = new ImporterProgress(this.info.key, this.info.name); this.oldSettings = {}; this.progress.step = this.importRecord.status; this._lastProgressReportTotal = 0; this.reloadCount(); + + this.logger.debug(`Constructed a new ${this.info.name} Importer.`); } /** * Registers the file name and content type on the import operation - * - * @param {string} fileName The name of the uploaded file. - * @param {string} contentType The sent file type. - * @returns {Progress} The progress record of the import. */ - async startFileUpload(fileName, contentType) { + async startFileUpload(fileName: string, contentType?: string): Promise { await this.updateProgress(ProgressStep.UPLOADING); - return this.updateRecord({ file: fileName, contentType }); + return this.updateRecord({ file: fileName, ...(contentType ? { contentType } : {}) }); } /** * Takes the uploaded file and extracts the users, channels, and messages from it. * - * @param {string} fullFilePath the full path of the uploaded file - * @returns {Progress} The progress record of the import. + * @param {string} _fullFilePath the full path of the uploaded file + * @returns {ImporterProgress} The progress record of the import. */ - async prepareUsingLocalFile() { + async prepareUsingLocalFile(_fullFilePath: string): Promise { return this.updateProgress(ProgressStep.PREPARING_STARTED); } @@ -76,9 +92,9 @@ export class Base { * The returned object should be the progress. * * @param {Selection} importSelection The selection data. - * @returns {Progress} The progress record of the import. + * @returns {ImporterProgress} The progress record of the import. */ - async startImport(importSelection, startedByUserId) { + async startImport(importSelection: Selection, startedByUserId: string): Promise { if (!(importSelection instanceof Selection)) { throw new Error(`Invalid Selection data provided to the ${this.info.name} importer.`); } else if (importSelection.users === undefined) { @@ -88,6 +104,9 @@ export class Base { `Channels in the selected data wasn't found, it must but at least an empty array for the ${this.info.name} importer.`, ); } + if (!startedByUserId) { + throw new Error('You must be logged in to do this.'); + } if (!startedByUserId) { throw new Error('You must be logged in to do this.'); @@ -97,7 +116,7 @@ export class Base { this.reloadCount(); const started = Date.now(); - const beforeImportFn = async ({ data, dataType: type }) => { + const beforeImportFn = async ({ data, dataType: type }: IImportRecord) => { if (this.importRecord.valid === false) { this.converter.aborted = true; throw new Error('The import operation is no longer valid.'); @@ -105,7 +124,13 @@ export class Base { switch (type) { case 'channel': { - const id = data.t === 'd' ? '__directMessages__' : data.importIds[0]; + if (!importSelection.channels) { + return true; + } + + const channelData = data as IImportChannel; + + const id = channelData.t === 'd' ? '__directMessages__' : channelData.importIds[0]; for (const channel of importSelection.channels) { if (channel.channel_id === id) { return channel.do_import; @@ -115,12 +140,14 @@ export class Base { return false; } case 'user': { - // #TODO: Replace this workaround in the final version of the API importer + // #TODO: Replace this workaround if (importSelection.users.length === 0 && this.info.key === 'api') { return true; } - const id = data.importIds[0]; + const userData = data as IImportUser; + + const id = userData.importIds[0]; for (const user of importSelection.users) { if (user.user_id === id) { return user.do_import; @@ -143,7 +170,7 @@ export class Base { } }; - const afterBatchFn = async (successCount, errorCount) => { + const afterBatchFn = async (successCount: number, errorCount: number) => { if (successCount) { await this.addCountCompleted(successCount); } @@ -200,10 +227,10 @@ export class Base { } async backupSettingValues() { - const allowUsernameChange = await Settings.findOneById('Accounts_AllowUsernameChange').value; - const maxFileSize = await Settings.findOneById('FileUpload_MaxFileSize').value; - const mediaTypeWhiteList = await Settings.findOneById('FileUpload_MediaTypeWhiteList').value; - const mediaTypeBlackList = await Settings.findOneById('FileUpload_MediaTypeBlackList').value; + const allowUsernameChange = (await Settings.findOneById('Accounts_AllowUsernameChange'))?.value as boolean | null; + const maxFileSize = (await Settings.findOneById('FileUpload_MaxFileSize'))?.value as number | null; + const mediaTypeWhiteList = (await Settings.findOneById('FileUpload_MediaTypeWhiteList'))?.value as string | null; + const mediaTypeBlackList = (await Settings.findOneById('FileUpload_MediaTypeBlackList'))?.value as string | null; this.oldSettings = { allowUsernameChange, @@ -213,19 +240,14 @@ export class Base { }; } - async applySettingValues(settingValues) { + async applySettingValues(settingValues: OldSettings) { await Settings.updateValueById('Accounts_AllowUsernameChange', settingValues.allowUsernameChange ?? true); await Settings.updateValueById('FileUpload_MaxFileSize', settingValues.maxFileSize ?? -1); await Settings.updateValueById('FileUpload_MediaTypeWhiteList', settingValues.mediaTypeWhiteList ?? '*'); await Settings.updateValueById('FileUpload_MediaTypeBlackList', settingValues.mediaTypeBlackList ?? ''); } - /** - * Gets the progress of this import. - * - * @returns {Progress} The progress record of the import. - */ - getProgress() { + getProgress(): ImporterProgress { return this.progress; } @@ -235,9 +257,9 @@ export class Base { * This way the importer can adjust user/room information at will. * * @param {ProgressStep} step The progress step which this import is currently at. - * @returns {Progress} The progress record of the import. + * @returns {ImporterProgress} The progress record of the import. */ - async updateProgress(step) { + async updateProgress(step: IImportProgress['step']): Promise { this.progress.step = step; this.logger.debug(`${this.info.name} is now at ${step}.`); @@ -252,12 +274,6 @@ export class Base { } reloadCount() { - if (!this.importRecord.count) { - this.progress.count.total = 0; - this.progress.count.completed = 0; - this.progress.count.error = 0; - } - this.progress.count.total = this.importRecord.count?.total || 0; this.progress.count.completed = this.importRecord.count?.completed || 0; this.progress.count.error = this.importRecord.count?.error || 0; @@ -267,9 +283,9 @@ export class Base { * Adds the passed in value to the total amount of items needed to complete. * * @param {number} count The amount to add to the total count of items. - * @returns {Progress} The progress record of the import. + * @returns {ImporterProgress} The progress record of the import. */ - async addCountToTotal(count) { + async addCountToTotal(count: number): Promise { this.progress.count.total += count; await this.updateRecord({ 'count.total': this.progress.count.total }); @@ -280,15 +296,15 @@ export class Base { * Adds the passed in value to the total amount of items completed. * * @param {number} count The amount to add to the total count of finished items. - * @returns {Progress} The progress record of the import. + * @returns {ImporterProgress} The progress record of the import. */ - async addCountCompleted(count) { + async addCountCompleted(count: number): Promise { this.progress.count.completed += count; return this.maybeUpdateRecord(); } - async addCountError(count) { + async addCountError(count: number): Promise { this.progress.count.error += count; return this.maybeUpdateRecord(); @@ -298,7 +314,11 @@ export class Base { // Only update the database every 500 messages (or 50 for users/channels) // Or the completed is greater than or equal to the total amount const count = this.progress.count.completed + this.progress.count.error; - const range = [ProgressStep.IMPORTING_USERS, ProgressStep.IMPORTING_CHANNELS].includes(this.progress.step) ? 50 : 500; + const range = ([ProgressStep.IMPORTING_USERS, ProgressStep.IMPORTING_CHANNELS] as IImportProgress['step'][]).includes( + this.progress.step, + ) + ? 50 + : 500; if (count % range === 0 || count >= this.progress.count.total || count - this._lastProgressReportTotal > range) { this._lastProgressReportTotal = this.progress.count.completed + this.progress.count.error; @@ -328,18 +348,20 @@ export class Base { /** * Updates the import record with the given fields being `set`. - * - * @param {any} fields The fields to set, it should be an object with key/values. - * @returns {Imports} The import record. */ - async updateRecord(fields) { + async updateRecord(fields: MatchKeysAndValues): Promise { + if (!this.importRecord) { + return this.importRecord; + } + await Imports.update({ _id: this.importRecord._id }, { $set: fields }); - this.importRecord = await Imports.findOne(this.importRecord._id); + // #TODO: Remove need for the typecast + this.importRecord = (await Imports.findOne(this.importRecord._id)) as IImport; return this.importRecord; } - async buildSelection() { + async buildSelection(): Promise { await this.updateProgress(ProgressStep.USER_SELECTION); const users = await ImportData.getAllUsersForSelection(); @@ -351,25 +373,23 @@ export class Base { new SelectionUser(u.data.importIds[0], u.data.username, u.data.emails[0], Boolean(u.data.deleted), u.data.type === 'bot', true), ); const selectionChannels = channels.map( - (c) => - new SelectionChannel( - c.data.importIds[0], - c.data.name, - Boolean(c.data.archived), - true, - c.data.t === 'p', - undefined, - c.data.t === 'd', - ), + (c) => new SelectionChannel(c.data.importIds[0], c.data.name, Boolean(c.data.archived), true, c.data.t === 'p', c.data.t === 'd'), ); const selectionMessages = await ImportData.countMessages(); if (hasDM) { - selectionChannels.push(new SelectionChannel('__directMessages__', t('Direct_Messages'), false, true, true, undefined, true)); + selectionChannels.push(new SelectionChannel('__directMessages__', t('Direct_Messages'), false, true, true, true)); } const results = new Selection(this.info.name, selectionUsers, selectionChannels, selectionMessages); return results; } + + /** + * Utility method to check if the passed in error is a `MongoServerError` with the `codeName` of `'CursorNotFound'`. + */ + protected isCursorNotFoundError(error: unknown): error is MongoServerError & { codeName: 'CursorNotFound' } { + return typeof error === 'object' && error !== null && 'codeName' in error && error.codeName === 'CursorNotFound'; + } } diff --git a/apps/meteor/app/importer/server/classes/ImporterProgress.js b/apps/meteor/app/importer/server/classes/ImporterProgress.js deleted file mode 100644 index 7bb49c50c4c2..000000000000 --- a/apps/meteor/app/importer/server/classes/ImporterProgress.js +++ /dev/null @@ -1,16 +0,0 @@ -import { ProgressStep } from '../../lib/ImporterProgressStep'; - -export class Progress { - /** - * Creates a new progress container for the importer. - * - * @param {string} key The unique key of the importer. - * @param {string} name The name of the importer. - */ - constructor(key, name) { - this.key = key; - this.name = name; - this.step = ProgressStep.NEW; - this.count = { completed: 0, total: 0 }; - } -} diff --git a/apps/meteor/app/importer/server/classes/ImporterProgress.ts b/apps/meteor/app/importer/server/classes/ImporterProgress.ts new file mode 100644 index 000000000000..ff59a5eb20b1 --- /dev/null +++ b/apps/meteor/app/importer/server/classes/ImporterProgress.ts @@ -0,0 +1,30 @@ +import type { IImportProgress } from '@rocket.chat/core-typings'; + +import { ProgressStep } from '../../lib/ImporterProgressStep'; + +export class ImporterProgress implements IImportProgress { + public key: string; + + public name: string; + + public step: IImportProgress['step']; + + public count: { + completed: number; + total: number; + error: number; + }; + + /** + * Creates a new progress container for the importer. + * + * @param {string} key The unique key of the importer. + * @param {string} name The name of the importer. + */ + constructor(key: string, name: string) { + this.key = key; + this.name = name; + this.step = ProgressStep.NEW; + this.count = { completed: 0, total: 0, error: 0 }; + } +} diff --git a/apps/meteor/app/importer/server/classes/ImporterSelection.js b/apps/meteor/app/importer/server/classes/ImporterSelection.js deleted file mode 100644 index f3e055e9fb75..000000000000 --- a/apps/meteor/app/importer/server/classes/ImporterSelection.js +++ /dev/null @@ -1,16 +0,0 @@ -export class Selection { - /** - * Constructs a new importer selection object. - * - * @param {string} name the name of the importer - * @param {SelectionUser[]} users the users which can be selected - * @param {SelectionChannel[]} channels the channels which can be selected - * @param {number} message_count the number of messages - */ - constructor(name, users, channels, message_count) { - this.name = name; - this.users = users; - this.channels = channels; - this.message_count = message_count; - } -} diff --git a/apps/meteor/app/importer/server/classes/ImporterSelection.ts b/apps/meteor/app/importer/server/classes/ImporterSelection.ts new file mode 100644 index 000000000000..107dbbf9c824 --- /dev/null +++ b/apps/meteor/app/importer/server/classes/ImporterSelection.ts @@ -0,0 +1,26 @@ +import type { IImporterSelection, IImporterSelectionChannel, IImporterSelectionUser } from '@rocket.chat/core-typings'; + +export class ImporterSelection implements IImporterSelection { + public name: string; + + public users: IImporterSelectionUser[]; + + public channels: IImporterSelectionChannel[]; + + public message_count: number; + + /** + * Constructs a new importer selection object. + * + * @param name the name of the importer + * @param users the users which can be selected + * @param channels the channels which can be selected + * @param messageCount the number of messages + */ + constructor(name: string, users: IImporterSelectionUser[], channels: IImporterSelectionChannel[], messageCount: number) { + this.name = name; + this.users = users; + this.channels = channels; + this.message_count = messageCount; + } +} diff --git a/apps/meteor/app/importer/server/classes/ImporterSelectionChannel.js b/apps/meteor/app/importer/server/classes/ImporterSelectionChannel.js deleted file mode 100644 index deb23849f108..000000000000 --- a/apps/meteor/app/importer/server/classes/ImporterSelectionChannel.js +++ /dev/null @@ -1,22 +0,0 @@ -export class SelectionChannel { - /** - * Constructs a new selection channel. - * - * @param {string} channel_id the unique identifier of the channel - * @param {string} name the name of the channel - * @param {boolean} is_archived whether the channel was archived or not - * @param {boolean} do_import whether we will be importing the channel or not - * @param {boolean} is_private whether the channel is private or public - * @param {int} creator the id of the channel owner - * @param {boolean} is_direct whether the channel represents direct messages - */ - constructor(channel_id, name, is_archived, do_import, is_private, creator, is_direct) { - this.channel_id = channel_id; - this.name = name; - this.is_archived = is_archived; - this.do_import = do_import; - this.is_private = is_private; - this.creator = creator; - this.is_direct = is_direct; - } -} diff --git a/apps/meteor/app/importer/server/classes/ImporterSelectionChannel.ts b/apps/meteor/app/importer/server/classes/ImporterSelectionChannel.ts new file mode 100644 index 000000000000..359a18de5a6c --- /dev/null +++ b/apps/meteor/app/importer/server/classes/ImporterSelectionChannel.ts @@ -0,0 +1,34 @@ +import type { IImporterSelectionChannel } from '@rocket.chat/core-typings'; + +export class SelectionChannel implements IImporterSelectionChannel { + public channel_id: string; + + public name: string | undefined; + + public is_archived: boolean; + + public do_import: boolean; + + public is_private: boolean; + + public is_direct: boolean; + + public creator: undefined; + + /** + * Constructs a new selection channel. + * + * @param channelId the unique identifier of the channel + * @param name the name of the channel + * @param isArchived whether the channel was archived or not + * @param doImport whether we will be importing the channel or not + */ + constructor(channelId: string, name: string | undefined, isArchived: boolean, doImport: boolean, isPrivate: boolean, isDirect: boolean) { + this.channel_id = channelId; + this.name = name; + this.is_archived = isArchived; + this.do_import = doImport; + this.is_private = isPrivate; + this.is_direct = isDirect; + } +} diff --git a/apps/meteor/app/importer/server/classes/ImporterSelectionUser.js b/apps/meteor/app/importer/server/classes/ImporterSelectionUser.js deleted file mode 100644 index 7d243fde3dde..000000000000 --- a/apps/meteor/app/importer/server/classes/ImporterSelectionUser.js +++ /dev/null @@ -1,22 +0,0 @@ -export class SelectionUser { - /** - * Constructs a new selection user. - * - * @param {string} user_id the unique user identifier - * @param {string} username the user's username - * @param {string} email the user's email - * @param {boolean} is_deleted whether the user was deleted or not - * @param {boolean} is_bot whether the user is a bot or not - * @param {boolean} do_import whether we are going to import this user or not - * @param {boolean} is_email_taken whether there's an existing user with the same email - */ - constructor(user_id, username, email, is_deleted, is_bot, do_import, is_email_taken = false) { - this.user_id = user_id; - this.username = username; - this.email = email; - this.is_deleted = is_deleted; - this.is_bot = is_bot; - this.do_import = do_import; - this.is_email_taken = is_email_taken; - } -} diff --git a/apps/meteor/app/importer/server/classes/ImporterSelectionUser.ts b/apps/meteor/app/importer/server/classes/ImporterSelectionUser.ts new file mode 100644 index 000000000000..5dc9cb91b4c4 --- /dev/null +++ b/apps/meteor/app/importer/server/classes/ImporterSelectionUser.ts @@ -0,0 +1,46 @@ +import type { IImporterSelectionUser } from '@rocket.chat/core-typings'; + +export class SelectionUser implements IImporterSelectionUser { + public user_id: string; + + public username: string | undefined; + + public email: string; + + public is_deleted: boolean; + + public is_bot: boolean; + + public do_import: boolean; + + public is_email_taken: boolean; + + /** + * Constructs a new selection user. + * + * @param userId the unique user identifier + * @param username the user's username + * @param email the user's email + * @param isDeleted whether the user was deleted or not + * @param isBot whether the user is a bot or not + * @param doImport whether we are going to import this user or not + * @param isEmailTaken whether there's an existing user with the same email + */ + constructor( + userId: string, + username: string | undefined, + email: string, + isDeleted: boolean, + isBot: boolean, + doImport: boolean, + isEmailTaken = false, + ) { + this.user_id = userId; + this.username = username; + this.email = email; + this.is_deleted = isDeleted; + this.is_bot = isBot; + this.do_import = doImport; + this.is_email_taken = isEmailTaken; + } +} diff --git a/apps/meteor/app/importer/server/classes/ImporterWebsocket.js b/apps/meteor/app/importer/server/classes/ImporterWebsocket.ts similarity index 62% rename from apps/meteor/app/importer/server/classes/ImporterWebsocket.js rename to apps/meteor/app/importer/server/classes/ImporterWebsocket.ts index ba067fda3867..a08e62f8435c 100644 --- a/apps/meteor/app/importer/server/classes/ImporterWebsocket.js +++ b/apps/meteor/app/importer/server/classes/ImporterWebsocket.ts @@ -1,6 +1,11 @@ +import type { IImportProgress } from '@rocket.chat/core-typings'; +import type { IStreamer } from 'meteor/rocketchat:streamer'; + import notifications from '../../../notifications/server/lib/Notifications'; class ImporterWebsocketDef { + private streamer: IStreamer<'importers'>; + constructor() { this.streamer = notifications.streamImporters; } @@ -10,7 +15,7 @@ class ImporterWebsocketDef { * * @param {Progress} progress The progress of the import. */ - progressUpdated(progress) { + progressUpdated(progress: { rate: number } | IImportProgress) { this.streamer.emit('progress', progress); } } diff --git a/apps/meteor/app/importer/server/classes/ImportersContainer.ts b/apps/meteor/app/importer/server/classes/ImportersContainer.ts new file mode 100644 index 000000000000..97c98701e49e --- /dev/null +++ b/apps/meteor/app/importer/server/classes/ImportersContainer.ts @@ -0,0 +1,27 @@ +import type { ImporterInfo } from '../definitions/ImporterInfo'; + +/** Container class which holds all of the importer details. */ +export class ImportersContainer { + private importers: Map; + + constructor() { + this.importers = new Map(); + } + + add({ key, name, importer, visible }: Omit & { visible?: boolean }) { + this.importers.set(key, { + key, + name, + visible: visible !== false, + importer, + }); + } + + get(key: string): ImporterInfo | undefined { + return this.importers.get(key); + } + + getAllVisible(): ImporterInfo[] { + return Array.from(this.importers.values()).filter(({ visible }) => visible); + } +} diff --git a/apps/meteor/app/importer/server/definitions/ImporterInfo.ts b/apps/meteor/app/importer/server/definitions/ImporterInfo.ts new file mode 100644 index 000000000000..4741b3b70f2d --- /dev/null +++ b/apps/meteor/app/importer/server/definitions/ImporterInfo.ts @@ -0,0 +1,10 @@ +import type { RocketchatI18nKeys } from '@rocket.chat/i18n'; + +import type { Importer } from '../classes/Importer'; + +export type ImporterInfo = { + key: string; + name: RocketchatI18nKeys; + visible: boolean; // Determines if this importer can be selected by the user in the New Import screen. + importer: typeof Importer; +}; diff --git a/apps/meteor/app/importer/server/index.ts b/apps/meteor/app/importer/server/index.ts index 8d7d872455c0..02949733174c 100644 --- a/apps/meteor/app/importer/server/index.ts +++ b/apps/meteor/app/importer/server/index.ts @@ -1,16 +1,21 @@ -import { ImporterInfo } from '../lib/ImporterInfo'; import { ProgressStep } from '../lib/ImporterProgressStep'; -import { Importers } from '../lib/Importers'; -import { Base } from './classes/ImporterBase'; -import { Selection } from './classes/ImporterSelection'; +import { Importer } from './classes/Importer'; +import { ImporterSelection } from './classes/ImporterSelection'; import { SelectionChannel } from './classes/ImporterSelectionChannel'; import { SelectionUser } from './classes/ImporterSelectionUser'; import { ImporterWebsocket } from './classes/ImporterWebsocket'; +import { ImportersContainer } from './classes/ImportersContainer'; import './methods'; import './startup/setImportsToInvalid'; import './startup/store'; -// Adding a link to the base class using the 'api' key. This won't be needed in the new importer structure implemented on the parallel PR -Importers.add(new ImporterInfo('api', 'API', ''), Base); +export { Importer, ImporterWebsocket, ProgressStep, ImporterSelection as Selection, SelectionChannel, SelectionUser }; -export { Base, Importers, ImporterWebsocket, ProgressStep, Selection, SelectionChannel, SelectionUser }; +export const Importers = new ImportersContainer(); + +Importers.add({ + key: 'api', + name: 'API', + visible: false, + importer: Importer, +}); diff --git a/apps/meteor/app/importer/server/methods/downloadPublicImportFile.ts b/apps/meteor/app/importer/server/methods/downloadPublicImportFile.ts index dc92901435ea..81e06ec8eb0f 100644 --- a/apps/meteor/app/importer/server/methods/downloadPublicImportFile.ts +++ b/apps/meteor/app/importer/server/methods/downloadPublicImportFile.ts @@ -35,14 +35,12 @@ export const executeDownloadPublicImportFile = async (userId: IUser['_id'], file ); } // Check if it's a valid url or path before creating a new import record - if (!isUrl) { - if (!fs.existsSync(fileUrl)) { - throw new Meteor.Error('error-import-file-missing', fileUrl, 'downloadPublicImportFile'); - } + if (!isUrl && !fs.existsSync(fileUrl)) { + throw new Meteor.Error('error-import-file-missing', fileUrl, 'downloadPublicImportFile'); } const operation = await Import.newOperation(userId, importer.name, importer.key); - importer.instance = new importer.importer(importer, operation); // eslint-disable-line new-cap + const instance = new importer.importer(importer, operation); // eslint-disable-line new-cap const oldFileName = fileUrl.substring(fileUrl.lastIndexOf('/') + 1).split('?')[0]; const date = new Date(); @@ -50,17 +48,17 @@ export const executeDownloadPublicImportFile = async (userId: IUser['_id'], file const newFileName = `${dateStr}_${userId}_${oldFileName}`; // Store the file name on the imports collection - await importer.instance.startFileUpload(newFileName); - await importer.instance.updateProgress(ProgressStep.DOWNLOADING_FILE); + await instance.startFileUpload(newFileName); + await instance.updateProgress(ProgressStep.DOWNLOADING_FILE); const writeStream = RocketChatImportFileInstance.createWriteStream(newFileName); writeStream.on('error', () => { - void importer.instance.updateProgress(ProgressStep.ERROR); + void instance.updateProgress(ProgressStep.ERROR); }); writeStream.on('end', () => { - void importer.instance.updateProgress(ProgressStep.FILE_LOADED); + void instance.updateProgress(ProgressStep.FILE_LOADED); }); if (isUrl) { @@ -68,8 +66,8 @@ export const executeDownloadPublicImportFile = async (userId: IUser['_id'], file } else { // If the url is actually a folder path on the current machine, skip moving it to the file store if (fs.statSync(fileUrl).isDirectory()) { - await importer.instance.updateRecord({ file: fileUrl }); - await importer.instance.updateProgress(ProgressStep.FILE_LOADED); + await instance.updateRecord({ file: fileUrl }); + await instance.updateProgress(ProgressStep.FILE_LOADED); return; } diff --git a/apps/meteor/app/importer/server/methods/getImportFileData.ts b/apps/meteor/app/importer/server/methods/getImportFileData.ts index cf81e3074da5..03f9a53abe6c 100644 --- a/apps/meteor/app/importer/server/methods/getImportFileData.ts +++ b/apps/meteor/app/importer/server/methods/getImportFileData.ts @@ -1,7 +1,7 @@ import fs from 'fs'; import path from 'path'; -import type { IImportFileData } from '@rocket.chat/core-typings'; +import type { IImportProgress, IImporterSelection } from '@rocket.chat/core-typings'; import { Imports } from '@rocket.chat/models'; import type { ServerMethods } from '@rocket.chat/ui-contexts'; import { Meteor } from 'meteor/meteor'; @@ -11,7 +11,7 @@ import { hasPermissionAsync } from '../../../authorization/server/functions/hasP import { ProgressStep } from '../../lib/ImporterProgressStep'; import { RocketChatImportFileInstance } from '../startup/store'; -export const executeGetImportFileData = async (): Promise => { +export const executeGetImportFileData = async (): Promise => { const operation = await Imports.findLastImport(); if (!operation) { throw new Meteor.Error('error-operation-not-found', 'Import Operation Not Found', 'getImportFileData'); @@ -24,9 +24,9 @@ export const executeGetImportFileData = async (): Promise= 0) { - if (importer.instance.importRecord?.valid) { + if (waitingSteps.indexOf(instance.progress.step) >= 0) { + if (instance.importRecord?.valid) { return { waiting: true }; } throw new Meteor.Error('error-import-operation-invalid', 'Invalid Import Operation', 'getImportFileData'); } - const readySteps = [ProgressStep.USER_SELECTION, ProgressStep.DONE, ProgressStep.CANCELLED, ProgressStep.ERROR]; + const readySteps: IImportProgress['step'][] = [ + ProgressStep.USER_SELECTION, + ProgressStep.DONE, + ProgressStep.CANCELLED, + ProgressStep.ERROR, + ]; + + if (readySteps.indexOf(instance.progress.step) >= 0) { + return instance.buildSelection(); + } - if (readySteps.indexOf(importer.instance.progress.step) >= 0) { - return importer.instance.buildSelection(); + const fileName = instance.importRecord.file; + if (fileName) { + const fullFilePath = fs.existsSync(fileName) ? fileName : path.join(RocketChatImportFileInstance.absolutePath, fileName); + await instance.prepareUsingLocalFile(fullFilePath); } - const fileName = importer.instance.importRecord.file; - const fullFilePath = fs.existsSync(fileName) ? fileName : path.join(RocketChatImportFileInstance.absolutePath, fileName); - await importer.instance.prepareUsingLocalFile(fullFilePath); - return importer.instance.buildSelection(); + return instance.buildSelection(); }; declare module '@rocket.chat/ui-contexts' { // eslint-disable-next-line @typescript-eslint/naming-convention interface ServerMethods { - getImportFileData(): IImportFileData | { waiting: true }; + getImportFileData(): IImporterSelection | { waiting: true }; } } diff --git a/apps/meteor/app/importer/server/methods/getImportProgress.ts b/apps/meteor/app/importer/server/methods/getImportProgress.ts index cdce1da395d7..c03bc44b056d 100644 --- a/apps/meteor/app/importer/server/methods/getImportProgress.ts +++ b/apps/meteor/app/importer/server/methods/getImportProgress.ts @@ -18,9 +18,9 @@ export const executeGetImportProgress = async (): Promise => { throw new Meteor.Error('error-importer-not-defined', `The importer (${importerKey}) has no import class defined.`, 'getImportProgress'); } - importer.instance = new importer.importer(importer, operation); // eslint-disable-line new-cap + const instance = new importer.importer(importer, operation); // eslint-disable-line new-cap - return importer.instance.getProgress(); + return instance.getProgress(); }; declare module '@rocket.chat/ui-contexts' { diff --git a/apps/meteor/app/importer/server/methods/startImport.ts b/apps/meteor/app/importer/server/methods/startImport.ts index fda135b4fba7..af91295ede29 100644 --- a/apps/meteor/app/importer/server/methods/startImport.ts +++ b/apps/meteor/app/importer/server/methods/startImport.ts @@ -19,25 +19,17 @@ export const executeStartImport = async ({ input }: StartImportParamsPOST, start throw new Meteor.Error('error-importer-not-defined', `The importer (${importerKey}) has no import class defined.`, 'startImport'); } - importer.instance = new importer.importer(importer, operation); // eslint-disable-line new-cap + const instance = new importer.importer(importer, operation); // eslint-disable-line new-cap const usersSelection = input.users.map( (user) => new SelectionUser(user.user_id, user.username, user.email, user.is_deleted, user.is_bot, user.do_import), ); const channelsSelection = input.channels.map( (channel) => - new SelectionChannel( - channel.channel_id, - channel.name, - channel.is_archived, - channel.do_import, - channel.is_private, - channel.creator, - channel.is_direct, - ), + new SelectionChannel(channel.channel_id, channel.name, channel.is_archived, channel.do_import, channel.is_private, channel.is_direct), ); const selection = new Selection(importer.name, usersSelection, channelsSelection, 0); - return importer.instance.startImport(selection, startedByUserId); + await instance.startImport(selection, startedByUserId); }; declare module '@rocket.chat/ui-contexts' { diff --git a/apps/meteor/app/importer/server/methods/uploadImportFile.ts b/apps/meteor/app/importer/server/methods/uploadImportFile.ts index 68955a102b94..d6ded455793b 100644 --- a/apps/meteor/app/importer/server/methods/uploadImportFile.ts +++ b/apps/meteor/app/importer/server/methods/uploadImportFile.ts @@ -23,25 +23,36 @@ export const executeUploadImportFile = async ( const operation = await Import.newOperation(userId, importer.name, importer.key); - importer.instance = new importer.importer(importer, operation); // eslint-disable-line new-cap + const instance = new importer.importer(importer, operation); // eslint-disable-line new-cap const date = new Date(); const dateStr = `${date.getUTCFullYear()}${date.getUTCMonth()}${date.getUTCDate()}${date.getUTCHours()}${date.getUTCMinutes()}${date.getUTCSeconds()}`; const newFileName = `${dateStr}_${userId}_${fileName}`; // Store the file name and content type on the imports collection - await importer.instance.startFileUpload(newFileName, contentType); + await instance.startFileUpload(newFileName, contentType); // Save the file on the File Store const file = Buffer.from(binaryContent, 'base64'); const readStream = RocketChatFile.bufferToStream(file); const writeStream = RocketChatImportFileInstance.createWriteStream(newFileName, contentType); - writeStream.on('end', () => { - importer.instance.updateProgress(ProgressStep.FILE_LOADED); + await new Promise((resolve, reject) => { + try { + writeStream.on('end', () => { + resolve(); + }); + writeStream.on('error', (e: Error) => { + reject(e); + }); + + readStream.pipe(writeStream); + } catch (error) { + reject(error); + } }); - readStream.pipe(writeStream); + await instance.updateProgress(ProgressStep.FILE_LOADED); }; declare module '@rocket.chat/ui-contexts' { diff --git a/apps/meteor/client/importPackages.ts b/apps/meteor/client/importPackages.ts index ec10c6dd014c..fc9ce52e9993 100644 --- a/apps/meteor/client/importPackages.ts +++ b/apps/meteor/client/importPackages.ts @@ -16,11 +16,6 @@ import '../app/file-upload/client'; import '../app/github-enterprise/client'; import '../app/gitlab/client'; import '../app/iframe-login/client'; -import '../app/importer/client'; -import '../app/importer-csv/client'; -import '../app/importer-hipchat-enterprise/client'; -import '../app/importer-slack/client'; -import '../app/importer-slack-users/client'; import '../app/lib/client'; import '../app/message-mark-as-unread/client'; import '../app/nextcloud/client'; diff --git a/apps/meteor/client/views/admin/import/ChannelDescriptor.tsx b/apps/meteor/client/views/admin/import/ChannelDescriptor.tsx new file mode 100644 index 000000000000..fdcfc0c7989e --- /dev/null +++ b/apps/meteor/client/views/admin/import/ChannelDescriptor.tsx @@ -0,0 +1,6 @@ +export type ChannelDescriptor = { + channel_id: string; + name: string; + is_archived: boolean; + do_import: boolean; +}; diff --git a/apps/meteor/client/views/admin/import/ImportHistoryPage.tsx b/apps/meteor/client/views/admin/import/ImportHistoryPage.tsx index daa3f3e392d8..e8ce0b3cef7b 100644 --- a/apps/meteor/client/views/admin/import/ImportHistoryPage.tsx +++ b/apps/meteor/client/views/admin/import/ImportHistoryPage.tsx @@ -7,7 +7,9 @@ import React, { useMemo } from 'react'; import { ProgressStep } from '../../../../app/importer/lib/ImporterProgressStep'; import Page from '../../../components/Page'; import ImportOperationSummary from './ImportOperationSummary'; +import ImportOperationSummarySkeleton from './ImportOperationSummarySkeleton'; +// TODO: review inner logic function ImportHistoryPage() { const queryClient = useQueryClient(); const t = useTranslation(); @@ -165,7 +167,7 @@ function ImportHistoryPage() { {isLoading && ( <> {Array.from({ length: 20 }, (_, i) => ( - + ))} )} diff --git a/apps/meteor/client/views/admin/import/ImportOperationSummary.stories.tsx b/apps/meteor/client/views/admin/import/ImportOperationSummary.stories.tsx index 39972c78fa2d..1eec51aab3aa 100644 --- a/apps/meteor/client/views/admin/import/ImportOperationSummary.stories.tsx +++ b/apps/meteor/client/views/admin/import/ImportOperationSummary.stories.tsx @@ -3,12 +3,13 @@ import type { ComponentMeta, ComponentStory } from '@storybook/react'; import React from 'react'; import ImportOperationSummary from './ImportOperationSummary'; +import ImportOperationSummarySkeleton from './ImportOperationSummarySkeleton'; export default { title: 'Admin/Import/ImportOperationSummary', component: ImportOperationSummary, subcomponents: { - 'ImportOperationSummary.Skeleton': ImportOperationSummary.Skeleton, + ImportOperationSummarySkeleton, }, parameters: { layout: 'centered', @@ -24,4 +25,4 @@ export default { export const Default: ComponentStory = (args) => ; -export const Skeleton: ComponentStory = (args) => ; +export const Skeleton: ComponentStory = (args) => ; diff --git a/apps/meteor/client/views/admin/import/ImportOperationSummary.js b/apps/meteor/client/views/admin/import/ImportOperationSummary.tsx similarity index 79% rename from apps/meteor/client/views/admin/import/ImportOperationSummary.js rename to apps/meteor/client/views/admin/import/ImportOperationSummary.tsx index ddb21c01b07e..dc8d6bfe0ec5 100644 --- a/apps/meteor/client/views/admin/import/ImportOperationSummary.js +++ b/apps/meteor/client/views/admin/import/ImportOperationSummary.tsx @@ -1,4 +1,6 @@ +import type { Serialized } from '@rocket.chat/core-typings'; import { TableRow, TableCell } from '@rocket.chat/fuselage'; +import type { TranslationKey } from '@rocket.chat/ui-contexts'; import { useRouter, useTranslation } from '@rocket.chat/ui-contexts'; import React, { useMemo } from 'react'; @@ -10,8 +12,24 @@ import { ProgressStep, } from '../../../../app/importer/lib/ImporterProgressStep'; import { useFormatDateAndTime } from '../../../hooks/useFormatDateAndTime'; -import ImportOperationSummarySkeleton from './ImportOperationSummarySkeleton'; +type ImportOperationSummaryProps = { + type: string; + _updatedAt: Serialized; + status: (typeof ProgressStep)[keyof typeof ProgressStep]; + file?: string; + user: string; + small?: boolean; + count?: { + users?: number; + channels?: number; + messages?: number; + total?: number; + }; + valid?: boolean; +}; + +// TODO: review inner logic function ImportOperationSummary({ type, _updatedAt, @@ -21,7 +39,7 @@ function ImportOperationSummary({ small, count: { users = 0, channels = 0, messages = 0, total = 0 } = {}, valid, -}) { +}: ImportOperationSummaryProps) { const t = useTranslation(); const formatDateAndTime = useFormatDateAndTime(); @@ -80,7 +98,7 @@ function ImportOperationSummary({ {formatDateAndTime(_updatedAt)} {!small && ( <> - {status && t(status.replace('importer_', 'importer_status_'))} + {status && t(status.replace('importer_', 'importer_status_') as TranslationKey)} {fileName} {users} {channels} @@ -92,6 +110,4 @@ function ImportOperationSummary({ ); } -export default Object.assign(ImportOperationSummary, { - Skeleton: ImportOperationSummarySkeleton, -}); +export default ImportOperationSummary; diff --git a/apps/meteor/client/views/admin/import/ImportOperationSummarySkeleton.js b/apps/meteor/client/views/admin/import/ImportOperationSummarySkeleton.tsx similarity index 79% rename from apps/meteor/client/views/admin/import/ImportOperationSummarySkeleton.js rename to apps/meteor/client/views/admin/import/ImportOperationSummarySkeleton.tsx index 81509516d234..8c2a465cb58b 100644 --- a/apps/meteor/client/views/admin/import/ImportOperationSummarySkeleton.js +++ b/apps/meteor/client/views/admin/import/ImportOperationSummarySkeleton.tsx @@ -1,7 +1,11 @@ import { Skeleton, TableRow, TableCell } from '@rocket.chat/fuselage'; import React from 'react'; -function ImportOperationSummarySkeleton({ small }) { +type ImportOperationSummarySkeletonProps = { + small?: boolean; +}; + +function ImportOperationSummarySkeleton({ small = false }: ImportOperationSummarySkeletonProps) { return ( diff --git a/apps/meteor/client/views/admin/import/ImportProgressPage.tsx b/apps/meteor/client/views/admin/import/ImportProgressPage.tsx index 2710721e2bcb..5e5a5de715ab 100644 --- a/apps/meteor/client/views/admin/import/ImportProgressPage.tsx +++ b/apps/meteor/client/views/admin/import/ImportProgressPage.tsx @@ -10,6 +10,7 @@ import { numberFormat } from '../../../../lib/utils/stringUtils'; import Page from '../../../components/Page'; import { useErrorHandler } from './useErrorHandler'; +// TODO: review inner logic const ImportProgressPage = function ImportProgressPage() { const queryClient = useQueryClient(); const streamer = useStream('importers'); diff --git a/apps/meteor/client/views/admin/import/ImportRoute.js b/apps/meteor/client/views/admin/import/ImportRoute.tsx similarity index 83% rename from apps/meteor/client/views/admin/import/ImportRoute.js rename to apps/meteor/client/views/admin/import/ImportRoute.tsx index a9904a178a3b..96b5179b9ae0 100644 --- a/apps/meteor/client/views/admin/import/ImportRoute.js +++ b/apps/meteor/client/views/admin/import/ImportRoute.tsx @@ -7,7 +7,11 @@ import ImportProgressPage from './ImportProgressPage'; import NewImportPage from './NewImportPage'; import PrepareImportPage from './PrepareImportPage'; -function ImportHistoryRoute({ page }) { +type ImportHistoryRouteProps = { + page: 'history' | 'new' | 'prepare' | 'progress'; +}; + +function ImportHistoryRoute({ page }: ImportHistoryRouteProps) { const canRunImport = usePermission('run-import'); if (!canRunImport) { diff --git a/apps/meteor/client/views/admin/import/NewImportPage.js b/apps/meteor/client/views/admin/import/NewImportPage.tsx similarity index 76% rename from apps/meteor/client/views/admin/import/NewImportPage.js rename to apps/meteor/client/views/admin/import/NewImportPage.tsx index eefc39b677d7..e60c34b387c6 100644 --- a/apps/meteor/client/views/admin/import/NewImportPage.js +++ b/apps/meteor/client/views/admin/import/NewImportPage.tsx @@ -1,13 +1,16 @@ import { Box, Button, ButtonGroup, Callout, Chip, Field, Margins, Select, InputBox, TextInput, UrlInput } from '@rocket.chat/fuselage'; import { useUniqueId, useSafely } from '@rocket.chat/fuselage-hooks'; +import type { TranslationKey } from '@rocket.chat/ui-contexts'; import { useToastMessageDispatch, useRouter, useRouteParameter, useSetting, useEndpoint, useTranslation } from '@rocket.chat/ui-contexts'; +import { useQuery } from '@tanstack/react-query'; +import type { ChangeEvent, DragEvent, FormEvent, Key, SyntheticEvent } from 'react'; import React, { useState, useMemo, useEffect } from 'react'; -import { Importers } from '../../../../app/importer/client/index'; import Page from '../../../components/Page'; import { useFormatMemorySize } from '../../../hooks/useFormatMemorySize'; import { useErrorHandler } from './useErrorHandler'; +// TODO: review inner logic function NewImportPage() { const t = useTranslation(); const dispatchToastMessage = useToastMessageDispatch(); @@ -15,10 +18,18 @@ function NewImportPage() { const [isLoading, setLoading] = useSafely(useState(false)); const [fileType, setFileType] = useSafely(useState('upload')); + + const listImportersEndpoint = useEndpoint('GET', '/v1/importers.list'); + const { data: importers, isLoading: isLoadingImporters } = useQuery(['importers'], async () => listImportersEndpoint(), { + refetchOnWindowFocus: false, + }); + + const options = useMemo(() => importers?.map(({ key, name }) => [key, t(name as TranslationKey)] as const) || [], [importers, t]); + const importerKey = useRouteParameter('importerKey'); - const importer = useMemo(() => Importers.get(importerKey), [importerKey]); + const importer = useMemo(() => (importers || []).find(({ key }) => key === importerKey), [importerKey, importers]); - const maxFileSize = useSetting('FileUpload_MaxFileSize'); + const maxFileSize = useSetting('FileUpload_MaxFileSize') ?? 0; const router = useRouter(); @@ -26,10 +37,10 @@ function NewImportPage() { const downloadPublicImportFile = useEndpoint('POST', '/v1/downloadPublicImportFile'); useEffect(() => { - if (importerKey && !importer) { + if (importerKey && !importer && !isLoadingImporters) { router.navigate('/admin/import/new', { replace: true }); } - }, [importer, importerKey, router]); + }, [importer, importerKey, router, isLoadingImporters]); const formatMemorySize = useFormatMemorySize(); @@ -37,37 +48,53 @@ function NewImportPage() { router.navigate('/admin/import'); }; - const handleImporterKeyChange = (importerKey) => { + const handleImporterKeyChange = (importerKey: Key) => { + if (typeof importerKey !== 'string') { + return; + } + router.navigate( - router.buildRoutePath({ - pattern: '/admin/import/new/:importerKey', + { + pattern: '/admin/import/new/:importerKey?', params: { importerKey }, - }), + }, { replace: true }, ); }; - const handleFileTypeChange = (fileType) => { + const handleFileTypeChange = (fileType: Key) => { + if (typeof fileType !== 'string') { + return; + } + setFileType(fileType); }; - const [files, setFiles] = useState([]); + const [files, setFiles] = useState([]); - const handleImportFileChange = async (event) => { - event = event.originalEvent || event; + const isDataTransferEvent = (event: T): event is T & DragEvent => + Boolean('dataTransfer' in event && (event as any).dataTransfer.files); + + const handleImportFileChange = async (event: ChangeEvent) => { let { files } = event.target; if (!files || files.length === 0) { - files = (event.dataTransfer != null ? event.dataTransfer.files : undefined) || []; + if (isDataTransferEvent(event)) { + files = event.dataTransfer.files; + } } - setFiles(Array.from(files)); + setFiles(Array.from(files ?? [])); }; - const handleFileUploadChipClick = (file) => () => { + const handleFileUploadChipClick = (file: File) => () => { setFiles((files) => files.filter((_file) => _file !== file)); }; const handleFileUploadImportButtonClick = async () => { + if (!importerKey) { + return; + } + setLoading(true); try { @@ -75,13 +102,14 @@ function NewImportPage() { Array.from( files, (file) => - new Promise((resolve) => { + new Promise((resolve) => { const reader = new FileReader(); reader.readAsDataURL(file); reader.onloadend = async () => { + const result = reader.result as string; try { await uploadImportFile({ - binaryContent: reader.result.split(';base64,')[1], + binaryContent: result.split(';base64,')[1], contentType: file.type, fileName: file.name, importerKey, @@ -104,11 +132,15 @@ function NewImportPage() { const [fileUrl, setFileUrl] = useSafely(useState('')); - const handleFileUrlChange = (event) => { + const handleFileUrlChange = (event: FormEvent) => { setFileUrl(event.currentTarget.value); }; const handleFileUrlImportButtonClick = async () => { + if (!importerKey) { + return; + } + setLoading(true); try { @@ -124,11 +156,15 @@ function NewImportPage() { const [filePath, setFilePath] = useSafely(useState('')); - const handleFilePathChange = (event) => { + const handleFilePathChange = (event: FormEvent) => { setFilePath(event.currentTarget.value); }; const handleFilePathImportButtonClick = async () => { + if (!importerKey) { + return; + } + setLoading(true); try { @@ -148,7 +184,8 @@ function NewImportPage() { const handleImportButtonClick = (fileType === 'upload' && handleFileUploadImportButtonClick) || (fileType === 'url' && handleFileUrlImportButtonClick) || - (fileType === 'path' && handleFilePathImportButtonClick); + (fileType === 'path' && handleFilePathImportButtonClick) || + undefined; return ( @@ -178,14 +215,14 @@ function NewImportPage() { disabled={isLoading} placeholder={t('Select_an_option')} onChange={handleImporterKeyChange} - options={Importers.getAll().map(({ key, name }) => [key, t(name)])} + options={options} /> {importer && ( - {importer.name === 'CSV' + {importer.key === 'csv' ? t('Importer_From_Description_CSV') - : t('Importer_From_Description', { from: t(importer.name) })} + : t('Importer_From_Description', { from: t(importer.name as TranslationKey) })} )} diff --git a/apps/meteor/client/views/admin/import/PrepareChannels.tsx b/apps/meteor/client/views/admin/import/PrepareChannels.tsx index e7b5f4a6a6a3..14d7cdc0614f 100644 --- a/apps/meteor/client/views/admin/import/PrepareChannels.tsx +++ b/apps/meteor/client/views/admin/import/PrepareChannels.tsx @@ -3,12 +3,7 @@ import { useTranslation } from '@rocket.chat/ui-contexts'; import type { FC, Dispatch, SetStateAction, ChangeEvent } from 'react'; import React, { useState, useCallback } from 'react'; -type ChannelDescriptor = { - channel_id: string; - name: string; - is_archived: boolean; - do_import: boolean; -}; +import type { ChannelDescriptor } from './ChannelDescriptor'; type PrepareChannelsProps = { channelsCount: number; @@ -16,6 +11,7 @@ type PrepareChannelsProps = { setChannels: Dispatch>; }; +// TODO: review inner logic const PrepareChannels: FC = ({ channels, channelsCount, setChannels }) => { const t = useTranslation(); const [current, setCurrent] = useState(0); diff --git a/apps/meteor/client/views/admin/import/PrepareImportPage.js b/apps/meteor/client/views/admin/import/PrepareImportPage.tsx similarity index 76% rename from apps/meteor/client/views/admin/import/PrepareImportPage.js rename to apps/meteor/client/views/admin/import/PrepareImportPage.tsx index 3b86b3c06bf7..cb1689a644e0 100644 --- a/apps/meteor/client/views/admin/import/PrepareImportPage.js +++ b/apps/meteor/client/views/admin/import/PrepareImportPage.tsx @@ -1,5 +1,7 @@ +import type { IImport, IImporterSelection, Serialized } from '@rocket.chat/core-typings'; import { Badge, Box, Button, ButtonGroup, Margins, ProgressBar, Throbber, Tabs } from '@rocket.chat/fuselage'; import { useDebouncedValue, useSafely } from '@rocket.chat/fuselage-hooks'; +import type { TranslationKey } from '@rocket.chat/ui-contexts'; import { useEndpoint, useTranslation, useStream, useRouter } from '@rocket.chat/ui-contexts'; import React, { useEffect, useState, useMemo } from 'react'; @@ -13,12 +15,14 @@ import { } from '../../../../app/importer/lib/ImporterProgressStep'; import { numberFormat } from '../../../../lib/utils/stringUtils'; import Page from '../../../components/Page'; +import type { ChannelDescriptor } from './ChannelDescriptor'; import PrepareChannels from './PrepareChannels'; import PrepareUsers from './PrepareUsers'; +import type { UserDescriptor } from './UserDescriptor'; import { useErrorHandler } from './useErrorHandler'; -const waitFor = (fn, predicate) => - new Promise((resolve, reject) => { +const waitFor = (fn: () => Promise, predicate: (arg: T) => arg is U) => + new Promise((resolve, reject) => { const callPromise = () => { fn().then((result) => { if (predicate(result)) { @@ -33,16 +37,17 @@ const waitFor = (fn, predicate) => callPromise(); }); +// TODO: review inner logic function PrepareImportPage() { const t = useTranslation(); const handleError = useErrorHandler(); const [isPreparing, setPreparing] = useSafely(useState(true)); - const [progressRate, setProgressRate] = useSafely(useState(null)); - const [status, setStatus] = useSafely(useState(null)); + const [progressRate, setProgressRate] = useSafely(useState(null)); + const [status, setStatus] = useSafely(useState(null)); const [messageCount, setMessageCount] = useSafely(useState(0)); - const [users, setUsers] = useState([]); - const [channels, setChannels] = useState([]); + const [users, setUsers] = useState([]); + const [channels, setChannels] = useState([]); const [isImporting, setImporting] = useSafely(useState(false)); const usersCount = useMemo(() => users.filter(({ do_import }) => do_import).length, [users]); @@ -58,8 +63,11 @@ function PrepareImportPage() { useEffect( () => - streamer('progress', ({ rate }) => { - setProgressRate(rate); + streamer('progress', (progress) => { + // Ignore any update without the rate since we're not showing any other info anyway + if ('rate' in progress) { + setProgressRate(progress.rate); + } }), [streamer, setProgressRate], ); @@ -67,7 +75,10 @@ function PrepareImportPage() { useEffect(() => { const loadImportFileData = async () => { try { - const data = await waitFor(getImportFileData, (data) => data && !data.waiting); + const data = await waitFor( + getImportFileData, + (data): data is IImporterSelection => data && (!('waiting' in data) || !data.waiting), + ); if (!data) { handleError(t('Importer_not_setup')); @@ -75,15 +86,9 @@ function PrepareImportPage() { return; } - if (data.step) { - handleError(t('Failed_To_Load_Import_Data')); - router.navigate('/admin/import'); - return; - } - setMessageCount(data.message_count); - setUsers(data.users.map((user) => ({ ...user, do_import: true }))); - setChannels(data.channels.map((channel) => ({ ...channel, do_import: true }))); + setUsers(data.users.map((user) => ({ ...user, username: user.username ?? '', do_import: true }))); + setChannels(data.channels.map((channel) => ({ ...channel, name: channel.name ?? '', do_import: true }))); setPreparing(false); setProgressRate(null); } catch (error) { @@ -96,7 +101,8 @@ function PrepareImportPage() { try { const { operation } = await waitFor( getCurrentImportOperation, - ({ operation }) => operation.valid && !ImportWaitingStates.includes(operation.status), + (data): data is Serialized<{ operation: IImport }> => + data.operation.valid && !ImportWaitingStates.includes(data.operation.status), ); if (!operation.valid) { @@ -149,7 +155,12 @@ function PrepareImportPage() { setImporting(true); try { - await startImport({ input: { users, channels } }); + await startImport({ + input: { + users: users.map((user) => ({ is_bot: false, is_email_taken: false, ...user })), + channels: channels.map((channel) => ({ is_private: false, is_direct: false, ...channel })), + }, + }); router.navigate('/admin/import/progress'); } catch (error) { handleError(error, t('Failed_To_Start_Import')); @@ -158,7 +169,7 @@ function PrepareImportPage() { }; const [tab, setTab] = useState('users'); - const handleTabClick = useMemo(() => (tab) => () => setTab(tab), []); + const handleTabClick = useMemo(() => (tab: string) => () => setTab(tab), []); const statusDebounced = useDebouncedValue(status, 100); @@ -183,7 +194,7 @@ function PrepareImportPage() { - {statusDebounced && t(statusDebounced.replace('importer_', 'importer_status_'))} + {statusDebounced && t(statusDebounced.replace('importer_', 'importer_status_') as TranslationKey)} {!isPreparing && ( @@ -204,7 +215,7 @@ function PrepareImportPage() { <> {progressRate ? ( - + {numberFormat(progressRate, 0)}% diff --git a/apps/meteor/client/views/admin/import/PrepareUsers.tsx b/apps/meteor/client/views/admin/import/PrepareUsers.tsx index 837b32c50e00..c3000230a16f 100644 --- a/apps/meteor/client/views/admin/import/PrepareUsers.tsx +++ b/apps/meteor/client/views/admin/import/PrepareUsers.tsx @@ -3,13 +3,7 @@ import { useTranslation } from '@rocket.chat/ui-contexts'; import type { FC, Dispatch, SetStateAction, ChangeEvent } from 'react'; import React, { useState, useCallback } from 'react'; -type UserDescriptor = { - user_id: string; - username: string; - email: string; - is_deleted: boolean; - do_import: boolean; -}; +import type { UserDescriptor } from './UserDescriptor'; type PrepareUsersProps = { usersCount: number; @@ -17,6 +11,7 @@ type PrepareUsersProps = { setUsers: Dispatch>; }; +// TODO: review inner logic const PrepareUsers: FC = ({ usersCount, users, setUsers }) => { const t = useTranslation(); const [current, setCurrent] = useState(0); diff --git a/apps/meteor/client/views/admin/import/UserDescriptor.tsx b/apps/meteor/client/views/admin/import/UserDescriptor.tsx new file mode 100644 index 000000000000..b7a611fbd213 --- /dev/null +++ b/apps/meteor/client/views/admin/import/UserDescriptor.tsx @@ -0,0 +1,7 @@ +export type UserDescriptor = { + user_id: string; + username: string; + email: string; + is_deleted: boolean; + do_import: boolean; +}; diff --git a/apps/meteor/client/views/admin/import/useErrorHandler.js b/apps/meteor/client/views/admin/import/useErrorHandler.js deleted file mode 100644 index 0afa59f2cfbb..000000000000 --- a/apps/meteor/client/views/admin/import/useErrorHandler.js +++ /dev/null @@ -1,32 +0,0 @@ -import { useMutableCallback } from '@rocket.chat/fuselage-hooks'; -import { useToastMessageDispatch, useTranslation } from '@rocket.chat/ui-contexts'; - -export const useErrorHandler = () => { - const t = useTranslation(); - const dispatchToastMessage = useToastMessageDispatch(); - - return useMutableCallback((error, defaultMessage) => { - console.error(error); - - if (typeof error === 'string') { - dispatchToastMessage({ type: 'error', message: error }); - return; - } - - const errorType = error?.xhr?.responseJSON?.errorType; - - if (typeof errorType === 'string' && t.has(errorType)) { - dispatchToastMessage({ type: 'error', message: t(errorType) }); - return; - } - - if (typeof errorType?.error === 'string' && t.has(errorType.error)) { - dispatchToastMessage({ type: 'error', message: t(errorType?.error) }); - return; - } - - if (defaultMessage) { - dispatchToastMessage({ type: 'error', message: defaultMessage }); - } - }); -}; diff --git a/apps/meteor/client/views/admin/import/useErrorHandler.ts b/apps/meteor/client/views/admin/import/useErrorHandler.ts new file mode 100644 index 000000000000..124b7a6e195a --- /dev/null +++ b/apps/meteor/client/views/admin/import/useErrorHandler.ts @@ -0,0 +1,12 @@ +import { useMutableCallback } from '@rocket.chat/fuselage-hooks'; +import { useToastMessageDispatch } from '@rocket.chat/ui-contexts'; + +export const useErrorHandler = () => { + const dispatchToastMessage = useToastMessageDispatch(); + + return useMutableCallback((error: unknown, defaultMessage?: unknown) => { + console.error(error); + + dispatchToastMessage({ type: 'error', message: error ?? defaultMessage }); + }); +}; diff --git a/apps/meteor/packages/rocketchat-i18n/i18n/en.i18n.json b/apps/meteor/packages/rocketchat-i18n/i18n/en.i18n.json index b9fc55199d35..dc4a3387c6bd 100644 --- a/apps/meteor/packages/rocketchat-i18n/i18n/en.i18n.json +++ b/apps/meteor/packages/rocketchat-i18n/i18n/en.i18n.json @@ -1452,6 +1452,7 @@ "CROWD_Reject_Unauthorized": "Reject Unauthorized", "Crowd_Remove_Orphaned_Users": "Remove Orphaned Users", "Crowd_sync_interval_Description": "The interval between synchronizations. Example `every 24 hours` or `on the first day of the week`, more examples at [Cron Text Parser](http://bunkat.github.io/later/parsers.html#text)", + "CSV": "CSV", "Current_Chats": "Current Chats", "Current_File": "Current File", "Current_Import_Operation": "Current Import Operation", @@ -2185,6 +2186,7 @@ "Failed_To_Load_Import_History": "Failed to load import history", "Failed_To_Load_Import_Operation": "Failed to load import operation", "Failed_To_Start_Import": "Failed to start import operation", + "Failed_To_upload_Import_File": "Failed to upload import file", "Failed_to_validate_invite_token": "Failed to validate invite token", "Failure": "Failure", "False": "False", @@ -2493,6 +2495,7 @@ "Highlights": "Highlights", "Highlights_How_To": "To be notified when someone mentions a word or phrase, add it here. You can separate words or phrases with commas. Highlight Words are not case sensitive.", "Highlights_List": "Highlight words", + "HipChat (tar.gz)": "HipChat (tar.gz)", "History": "History", "Hold_Time": "Hold Time", "Hold": "Hold", @@ -2548,6 +2551,7 @@ "Impersonate_user_description": "When enabled, integration posts as the user that triggered integration", "Import": "Import", "Import_New_File": "Import New File", + "Import_Operation_Failed": "Import operation failed", "Import_requested_successfully": "Import Requested Successfully", "Import_Type": "Import Type", "Importer_Archived": "Archived", @@ -4008,6 +4012,8 @@ "pdf_success_message": "PDF Transcript successfully generated", "pdf_error_message": "Error generating PDF Transcript", "Peer_Password": "Peer Password", + "Pending Avatars": "Pending Avatars", + "Pending Files": "Pending Files", "People": "People", "Permalink": "Permalink", "Permissions": "Permissions", @@ -4779,6 +4785,7 @@ "SLA_Policy": "SLA Policy", "SLA_Policies": "SLA Policies", "SLA_removed": "SLA removed", + "Slack": "Slack", "Slack_Users": "Slack's Users CSV", "SlackBridge_APIToken": "API Tokens (Legacy)", "SlackBridge_UseLegacy": "Use Legacy API Tokens", diff --git a/apps/meteor/server/models/raw/Imports.ts b/apps/meteor/server/models/raw/Imports.ts index e0e89d0cd878..b708c42e8119 100644 --- a/apps/meteor/server/models/raw/Imports.ts +++ b/apps/meteor/server/models/raw/Imports.ts @@ -17,11 +17,7 @@ export class ImportsModel extends BaseRaw implements IImportsModel { async findLastImport(): Promise { const imports = await this.find({}, { sort: { ts: -1 }, limit: 1 }).toArray(); - if (imports?.length) { - return imports.shift(); - } - - return undefined; + return imports.shift(); } async hasValidOperationInStatus(allowedStatus: IImport['status'][]): Promise { diff --git a/apps/meteor/server/modules/notifications/notifications.module.ts b/apps/meteor/server/modules/notifications/notifications.module.ts index ba66a31e5ee4..979603c92650 100644 --- a/apps/meteor/server/modules/notifications/notifications.module.ts +++ b/apps/meteor/server/modules/notifications/notifications.module.ts @@ -4,7 +4,7 @@ import { Rooms, Subscriptions, Users, Settings } from '@rocket.chat/models'; import type { StreamerCallbackArgs, StreamKeys, StreamNames } from '@rocket.chat/ui-contexts'; import type { IStreamer, IStreamerConstructor, IPublication } from 'meteor/rocketchat:streamer'; -import type { Progress } from '../../../app/importer/server/classes/ImporterProgress'; +import type { ImporterProgress } from '../../../app/importer/server/classes/ImporterProgress'; import { emit, StreamPresence } from '../../../app/notifications/server/lib/Presence'; import { SystemLogger } from '../../lib/logger/system'; @@ -536,7 +536,7 @@ export class NotificationsModule { return this.streamPresence.emitWithoutBroadcast(uid, args); } - progressUpdated(progress: { rate: number } | Progress): void { + progressUpdated(progress: { rate: number } | ImporterProgress): void { this.streamImporters.emit('progress', progress); } } diff --git a/apps/meteor/server/services/import/service.ts b/apps/meteor/server/services/import/service.ts index 2a61050fad33..cb95b2d1aa8f 100644 --- a/apps/meteor/server/services/import/service.ts +++ b/apps/meteor/server/services/import/service.ts @@ -4,8 +4,8 @@ import type { IImportUser, IImport, ImportStatus } from '@rocket.chat/core-typin import { Imports, ImportData } from '@rocket.chat/models'; import { ObjectId } from 'mongodb'; -import { Importers } from '../../../app/importer/lib/Importers'; -import { Selection } from '../../../app/importer/server/classes/ImporterSelection'; +import { Importers } from '../../../app/importer/server'; +import { ImporterSelection } from '../../../app/importer/server/classes/ImporterSelection'; import { settings } from '../../../app/settings/server'; import { validateRoleList } from '../../lib/roles/validateRoleList'; import { getNewUserRoles } from '../user/lib/getNewUserRoles'; @@ -92,7 +92,7 @@ export class ImportService extends ServiceClassInternal implements IImportServic }; } - private assertsValidStateForNewData(operation: IImport | null): asserts operation is IImport { + private assertsValidStateForNewData(operation: IImport | undefined): asserts operation is IImport { if (!operation?.valid) { throw new Error('Import operation not initialized.'); } @@ -172,12 +172,10 @@ export class ImportService extends ServiceClassInternal implements IImportServic skipDefaultChannels: true, enableEmail2fa: settings.get('Accounts_TwoFactorAuthentication_By_Email_Auto_Opt_In'), quickUserInsertion: true, - // Do not update the data of existing users, but add the importId to them if it's missing skipExistingUsers: true, - bindSkippedUsers: true, }); - const selection = new Selection(importer.name, [], [], 0); + const selection = new ImporterSelection(importer.name, [], [], 0); await instance.startImport(selection, userId); } } diff --git a/packages/core-typings/src/import/IImporterInfo.ts b/packages/core-typings/src/import/IImporterInfo.ts new file mode 100644 index 000000000000..cff4becc20df --- /dev/null +++ b/packages/core-typings/src/import/IImporterInfo.ts @@ -0,0 +1,4 @@ +export interface IImporterInfo { + key: string; + name: string; +} diff --git a/packages/core-typings/src/import/index.ts b/packages/core-typings/src/import/index.ts index 2f56f72575d4..00df59ff93b6 100644 --- a/packages/core-typings/src/import/index.ts +++ b/packages/core-typings/src/import/index.ts @@ -3,6 +3,7 @@ export * from './IImportUser'; export * from './IImportRecord'; export * from './IImportMessage'; export * from './IImportChannel'; +export * from './IImporterInfo'; export * from './IImportFileData'; export * from './IImportProgress'; export * from './IImporterSelection'; diff --git a/packages/model-typings/src/models/IImportsModel.ts b/packages/model-typings/src/models/IImportsModel.ts index a0a2055de588..309160d4c057 100644 --- a/packages/model-typings/src/models/IImportsModel.ts +++ b/packages/model-typings/src/models/IImportsModel.ts @@ -4,7 +4,7 @@ import type { UpdateResult, FindOptions, FindCursor, Document } from 'mongodb'; import type { IBaseModel } from './IBaseModel'; export interface IImportsModel extends IBaseModel { - findLastImport(): Promise; + findLastImport(): Promise; hasValidOperationInStatus(allowedStatus: IImport['status'][]): Promise; invalidateAllOperations(): Promise; invalidateOperationsExceptId(id: string): Promise; diff --git a/packages/rest-typings/src/v1/import/ImportersListParamsGET.ts b/packages/rest-typings/src/v1/import/ImportersListParamsGET.ts new file mode 100644 index 000000000000..976ff2e66794 --- /dev/null +++ b/packages/rest-typings/src/v1/import/ImportersListParamsGET.ts @@ -0,0 +1,16 @@ +import Ajv from 'ajv'; + +const ajv = new Ajv({ + coerceTypes: true, +}); + +export type ImportersListParamsGET = Record; + +const ImportersListParamsGETSchema = { + type: 'object', + properties: {}, + additionalProperties: false, + required: [], +}; + +export const isImportersListParamsGET = ajv.compile(ImportersListParamsGETSchema); diff --git a/packages/rest-typings/src/v1/import/StartImportParamsPOST.ts b/packages/rest-typings/src/v1/import/StartImportParamsPOST.ts index fded90645613..310ea8c1d61a 100644 --- a/packages/rest-typings/src/v1/import/StartImportParamsPOST.ts +++ b/packages/rest-typings/src/v1/import/StartImportParamsPOST.ts @@ -6,28 +6,24 @@ const ajv = new Ajv({ export type StartImportParamsPOST = { input: { - users: [ - { - user_id: string; - username: string; - email: string; - is_deleted: boolean; - is_bot: boolean; - do_import: boolean; - is_email_taken: boolean; - }, - ]; - channels: [ - { - channel_id: string; - name: string; - creator?: string; - is_archived: boolean; - do_import: boolean; - is_private: boolean; - is_direct: boolean; - }, - ]; + users: { + user_id: string; + username: string; + email: string; + is_deleted: boolean; + is_bot: boolean; + do_import: boolean; + is_email_taken: boolean; + }[]; + channels: { + channel_id: string; + name: string; + creator?: string; + is_archived: boolean; + do_import: boolean; + is_private: boolean; + is_direct: boolean; + }[]; }; }; diff --git a/packages/rest-typings/src/v1/import/import.ts b/packages/rest-typings/src/v1/import/import.ts index 1454200bab96..24d726d57e49 100644 --- a/packages/rest-typings/src/v1/import/import.ts +++ b/packages/rest-typings/src/v1/import/import.ts @@ -1,4 +1,4 @@ -import type { IImport, IImporterSelection, IImportProgress, ImportStatus, IImportUser } from '@rocket.chat/core-typings'; +import type { IImport, IImporterSelection, IImportProgress, IImporterInfo, ImportStatus, IImportUser } from '@rocket.chat/core-typings'; import type { DownloadPublicImportFileParamsPOST } from './DownloadPublicImportFileParamsPOST'; import type { StartImportParamsPOST } from './StartImportParamsPOST'; @@ -32,6 +32,9 @@ export type ImportEndpoints = { '/v1/getCurrentImportOperation': { GET: () => { operation: IImport }; }; + '/v1/importers.list': { + GET: () => Array; + }; '/v1/import.clear': { POST: () => void; }; diff --git a/packages/rest-typings/src/v1/import/index.ts b/packages/rest-typings/src/v1/import/index.ts index 622f4c1945b8..c1a6a39d144b 100644 --- a/packages/rest-typings/src/v1/import/index.ts +++ b/packages/rest-typings/src/v1/import/index.ts @@ -6,6 +6,7 @@ export * from './GetCurrentImportOperationParamsGET'; export * from './GetImportFileDataParamsGET'; export * from './GetImportProgressParamsGET'; export * from './GetLatestImportOperationsParamsGET'; +export * from './ImportersListParamsGET'; export * from './StartImportParamsPOST'; export * from './UploadImportFileParamsPOST'; export * from './ImportAddUsersParamsPOST';