From 3c05136811f16c98f6babb8436efa612452923a4 Mon Sep 17 00:00:00 2001 From: Pierre Lehnen <55164754+pierre-lehnen-rc@users.noreply.github.com> Date: Sat, 12 Oct 2024 04:35:32 -0300 Subject: [PATCH] chore: split ImportDataConverter into multiple classes and add unit testing (#33394) --- .../app/importer-csv/server/CsvImporter.ts | 4 +- .../server/HipChatEnterpriseImporter.js | 11 +- .../server/PendingFileImporter.ts | 4 +- .../server/SlackUsersImporter.ts | 4 +- .../server/classes/ImportDataConverter.ts | 1218 ++--------------- .../app/importer/server/classes/Importer.ts | 16 +- .../server/classes/VirtualDataConverter.ts | 169 --- .../classes/converters/ConverterCache.ts | 198 +++ .../classes/converters/MessageConverter.ts | 263 ++++ .../classes/converters/RecordConverter.ts | 237 ++++ .../classes/converters/RoomConverter.ts | 198 +++ .../classes/converters/UserConverter.ts | 419 ++++++ apps/meteor/ee/server/lib/ldap/Manager.ts | 16 +- apps/meteor/server/lib/ldap/Manager.ts | 8 +- .../{DataConverter.ts => UserConverter.ts} | 28 +- .../importer/server/messageConverter.spec.ts | 203 +++ .../importer/server/recordConverter.spec.ts | 137 ++ .../app/importer/server/roomConverter.spec.ts | 272 ++++ .../app/importer/server/userConverter.spec.ts | 612 +++++++++ 19 files changed, 2680 insertions(+), 1337 deletions(-) delete mode 100644 apps/meteor/app/importer/server/classes/VirtualDataConverter.ts create mode 100644 apps/meteor/app/importer/server/classes/converters/ConverterCache.ts create mode 100644 apps/meteor/app/importer/server/classes/converters/MessageConverter.ts create mode 100644 apps/meteor/app/importer/server/classes/converters/RecordConverter.ts create mode 100644 apps/meteor/app/importer/server/classes/converters/RoomConverter.ts create mode 100644 apps/meteor/app/importer/server/classes/converters/UserConverter.ts rename apps/meteor/server/lib/ldap/{DataConverter.ts => UserConverter.ts} (53%) create mode 100644 apps/meteor/tests/unit/app/importer/server/messageConverter.spec.ts create mode 100644 apps/meteor/tests/unit/app/importer/server/recordConverter.spec.ts create mode 100644 apps/meteor/tests/unit/app/importer/server/roomConverter.spec.ts create mode 100644 apps/meteor/tests/unit/app/importer/server/userConverter.spec.ts diff --git a/apps/meteor/app/importer-csv/server/CsvImporter.ts b/apps/meteor/app/importer-csv/server/CsvImporter.ts index 60c07c3288ce..a9844e747640 100644 --- a/apps/meteor/app/importer-csv/server/CsvImporter.ts +++ b/apps/meteor/app/importer-csv/server/CsvImporter.ts @@ -4,7 +4,7 @@ import { Random } from '@rocket.chat/random'; import { parse } from 'csv-parse/lib/sync'; import { Importer, ProgressStep, ImporterWebsocket } from '../../importer/server'; -import type { IConverterOptions } from '../../importer/server/classes/ImportDataConverter'; +import type { ConverterOptions } from '../../importer/server/classes/ImportDataConverter'; import type { ImporterProgress } from '../../importer/server/classes/ImporterProgress'; import type { ImporterInfo } from '../../importer/server/definitions/ImporterInfo'; import { notifyOnSettingChanged } from '../../lib/server/lib/notifyListener'; @@ -12,7 +12,7 @@ import { notifyOnSettingChanged } from '../../lib/server/lib/notifyListener'; export class CsvImporter extends Importer { private csvParser: (csv: string) => string[]; - constructor(info: ImporterInfo, importRecord: IImport, converterOptions: IConverterOptions = {}) { + constructor(info: ImporterInfo, importRecord: IImport, converterOptions: ConverterOptions = {}) { super(info, importRecord, converterOptions); this.csvParser = parse; diff --git a/apps/meteor/app/importer-hipchat-enterprise/server/HipChatEnterpriseImporter.js b/apps/meteor/app/importer-hipchat-enterprise/server/HipChatEnterpriseImporter.js index 663300e44154..ddabdfac4ee2 100644 --- a/apps/meteor/app/importer-hipchat-enterprise/server/HipChatEnterpriseImporter.js +++ b/apps/meteor/app/importer-hipchat-enterprise/server/HipChatEnterpriseImporter.js @@ -2,7 +2,7 @@ import fs from 'fs'; import path from 'path'; import { Readable } from 'stream'; -import { Settings } from '@rocket.chat/models'; +import { ImportData, Settings } from '@rocket.chat/models'; import { Meteor } from 'meteor/meteor'; import { Importer, ProgressStep } from '../../importer/server'; @@ -89,6 +89,13 @@ export class HipChatEnterpriseImporter extends Importer { await super.addCountToTotal(count); } + async findDMForImportedUsers(...users) { + const record = await ImportData.findDMForImportedUsers(...users); + if (record) { + return record.data; + } + } + async prepareUserMessagesFile(file) { this.logger.debug(`preparing room with ${file.length} messages `); let count = 0; @@ -110,7 +117,7 @@ export class HipChatEnterpriseImporter extends Importer { const users = [senderId, receiverId].sort(); if (!dmRooms[receiverId]) { - dmRooms[receiverId] = await this.converter.findDMForImportedUsers(senderId, receiverId); + dmRooms[receiverId] = await this.findDMForImportedUsers(senderId, receiverId); if (!dmRooms[receiverId]) { const room = { diff --git a/apps/meteor/app/importer-pending-files/server/PendingFileImporter.ts b/apps/meteor/app/importer-pending-files/server/PendingFileImporter.ts index da85e9b73296..400a9856c4e7 100644 --- a/apps/meteor/app/importer-pending-files/server/PendingFileImporter.ts +++ b/apps/meteor/app/importer-pending-files/server/PendingFileImporter.ts @@ -8,12 +8,12 @@ import { Random } from '@rocket.chat/random'; import { FileUpload } from '../../file-upload/server'; import { Importer, ProgressStep, Selection } from '../../importer/server'; -import type { IConverterOptions } from '../../importer/server/classes/ImportDataConverter'; +import type { ConverterOptions } from '../../importer/server/classes/ImportDataConverter'; import type { ImporterProgress } from '../../importer/server/classes/ImporterProgress'; import type { ImporterInfo } from '../../importer/server/definitions/ImporterInfo'; export class PendingFileImporter extends Importer { - constructor(info: ImporterInfo, importRecord: IImport, converterOptions: IConverterOptions = {}) { + constructor(info: ImporterInfo, importRecord: IImport, converterOptions: ConverterOptions = {}) { super(info, importRecord, converterOptions); } diff --git a/apps/meteor/app/importer-slack-users/server/SlackUsersImporter.ts b/apps/meteor/app/importer-slack-users/server/SlackUsersImporter.ts index 95461820bf2d..ae8df1859086 100644 --- a/apps/meteor/app/importer-slack-users/server/SlackUsersImporter.ts +++ b/apps/meteor/app/importer-slack-users/server/SlackUsersImporter.ts @@ -6,7 +6,7 @@ import { parse } from 'csv-parse/lib/sync'; import { RocketChatFile } from '../../file/server'; import { Importer, ProgressStep } from '../../importer/server'; -import type { IConverterOptions } from '../../importer/server/classes/ImportDataConverter'; +import type { ConverterOptions } from '../../importer/server/classes/ImportDataConverter'; import type { ImporterProgress } from '../../importer/server/classes/ImporterProgress'; import type { ImporterInfo } from '../../importer/server/definitions/ImporterInfo'; import { notifyOnSettingChanged } from '../../lib/server/lib/notifyListener'; @@ -14,7 +14,7 @@ import { notifyOnSettingChanged } from '../../lib/server/lib/notifyListener'; export class SlackUsersImporter extends Importer { private csvParser: (csv: string) => string[]; - constructor(info: ImporterInfo, importRecord: IImport, converterOptions: IConverterOptions = {}) { + constructor(info: ImporterInfo, importRecord: IImport, converterOptions: ConverterOptions = {}) { super(info, importRecord, converterOptions); this.csvParser = parse; diff --git a/apps/meteor/app/importer/server/classes/ImportDataConverter.ts b/apps/meteor/app/importer/server/classes/ImportDataConverter.ts index 6de47e33b2b6..64226f8752a1 100644 --- a/apps/meteor/app/importer/server/classes/ImportDataConverter.ts +++ b/apps/meteor/app/importer/server/classes/ImportDataConverter.ts @@ -1,1171 +1,119 @@ -import type { - IImportUser, - IImportMessage, - IImportMessageReaction, - IImportChannel, - IImportUserRecord, - IImportChannelRecord, - IImportMessageRecord, - IUser, - IUserEmail, - IImportData, - IImportRecordType, - IMessage as IDBMessage, -} from '@rocket.chat/core-typings'; +import type { IImportRecord, IImportUser, IImportMessage, IImportChannel } from '@rocket.chat/core-typings'; import type { Logger } from '@rocket.chat/logger'; -import { ImportData, Rooms, Users, Subscriptions } from '@rocket.chat/models'; -import { Random } from '@rocket.chat/random'; -import { SHA256 } from '@rocket.chat/sha256'; -import { hash as bcryptHash } from 'bcrypt'; -import { Accounts } from 'meteor/accounts-base'; -import { ObjectId } from 'mongodb'; +import { ImportData } from '@rocket.chat/models'; +import { pick } from '@rocket.chat/tools'; -import { callbacks } from '../../../../lib/callbacks'; -import { createDirectMessage } from '../../../../server/methods/createDirectMessage'; -import { saveRoomSettings } from '../../../channel-settings/server/methods/saveRoomSettings'; -import { addUserToDefaultChannels } from '../../../lib/server/functions/addUserToDefaultChannels'; -import { generateUsernameSuggestion } from '../../../lib/server/functions/getUsernameSuggestion'; -import { insertMessage } from '../../../lib/server/functions/insertMessage'; -import { saveUserIdentity } from '../../../lib/server/functions/saveUserIdentity'; -import { setUserActiveStatus } from '../../../lib/server/functions/setUserActiveStatus'; -import { notifyOnSubscriptionChangedByRoomId, notifyOnUserChange } from '../../../lib/server/lib/notifyListener'; -import { createChannelMethod } from '../../../lib/server/methods/createChannel'; -import { createPrivateGroupMethod } from '../../../lib/server/methods/createPrivateGroup'; -import { getValidRoomName } from '../../../utils/server/lib/getValidRoomName'; import type { IConversionCallbacks } from '../definitions/IConversionCallbacks'; +import { ConverterCache } from './converters/ConverterCache'; +import { type MessageConversionCallbacks, MessageConverter } from './converters/MessageConverter'; +import type { RecordConverter, RecordConverterOptions } from './converters/RecordConverter'; +import { RoomConverter } from './converters/RoomConverter'; +import { UserConverter, type UserConverterOptions } from './converters/UserConverter'; -type IRoom = Record; -type IMessage = Record; -type IUserIdentification = { - _id: string; - username: string | undefined; -}; -type IMentionedUser = { - _id: string; - username: string; - name?: string; -}; -type IMentionedChannel = { - _id: string; - name: string; -}; - -type IMessageReaction = { - name: string; - usernames: Array; -}; - -type IMessageReactions = Record; - -export type IConverterOptions = { - flagEmailsAsVerified?: boolean; - skipExistingUsers?: boolean; - skipNewUsers?: boolean; - skipUserCallbacks?: boolean; - skipDefaultChannels?: boolean; - - quickUserInsertion?: boolean; - enableEmail2fa?: boolean; -}; - -const guessNameFromUsername = (username: string): string => - username - .replace(/\W/g, ' ') - .replace(/\s(.)/g, (u) => u.toUpperCase()) - .replace(/^(.)/, (u) => u.toLowerCase()) - .replace(/^\w/, (u) => u.toUpperCase()); +export type ConverterOptions = UserConverterOptions & Omit; export class ImportDataConverter { - private _userCache: Map; + protected _options: ConverterOptions; - // display name uses a different cache because it's only used on mentions so we don't need to load it every time we load an user - private _userDisplayNameCache: Map; + protected _userConverter: UserConverter; - private _roomCache: Map; + protected _roomConverter: RoomConverter; - private _roomNameCache: Map; + protected _messageConverter: MessageConverter; - private _logger: Logger; + protected _cache = new ConverterCache(); - private _options: IConverterOptions; - - public get options(): IConverterOptions { + public get options(): ConverterOptions { return this._options; } - public aborted = false; - - constructor(options?: IConverterOptions) { - this._options = options || { - flagEmailsAsVerified: false, - skipExistingUsers: false, - skipNewUsers: false, - }; - this._userCache = new Map(); - this._userDisplayNameCache = new Map(); - this._roomCache = new Map(); - this._roomNameCache = new Map(); - } - - setLogger(logger: Logger): void { - this._logger = logger; - } - - addUserToCache(importId: string, _id: string, username: string | undefined): IUserIdentification { - const cache = { - _id, - username, + constructor(logger: Logger, options?: ConverterOptions) { + this._options = { + workInMemory: false, + ...(options || {}), }; - this._userCache.set(importId, cache); - return cache; + this.initializeUserConverter(logger); + this.initializeRoomConverter(logger); + this.initializeMessageConverter(logger); } - addUserDisplayNameToCache(importId: string, name: string): string { - this._userDisplayNameCache.set(importId, name); - return name; - } - - addRoomToCache(importId: string, rid: string): string { - this._roomCache.set(importId, rid); - return rid; - } - - addRoomNameToCache(importId: string, name: string): string { - this._roomNameCache.set(importId, name); - return name; - } - - addUserDataToCache(userData: IImportUser): void { - if (!userData._id) { - return; - } - if (!userData.importIds.length) { - return; - } - - this.addUserToCache(userData.importIds[0], userData._id, userData.username); - } - - protected async addObject(type: IImportRecordType, data: IImportData, options: Record = {}): Promise { - await ImportData.col.insertOne({ - _id: new ObjectId().toHexString(), - data, - dataType: type, - options, - }); - } - - async addUser(data: IImportUser): Promise { - await this.addObject('user', data); - } - - async addChannel(data: IImportChannel): Promise { - await this.addObject('channel', data); - } - - async addMessage(data: IImportMessage, useQuickInsert = false): Promise { - await this.addObject('message', data, { - useQuickInsert: useQuickInsert || undefined, - }); - } - - addUserImportId(updateData: Record, userData: IImportUser): void { - if (userData.importIds?.length) { - updateData.$addToSet = { - importIds: { - $each: userData.importIds, - }, - }; - } - } - - addUserEmails(updateData: Record, userData: IImportUser, existingEmails: Array): void { - if (!userData.emails?.length) { - return; - } - - const verifyEmails = Boolean(this.options.flagEmailsAsVerified); - const newEmailList: Array = []; - - for (const email of userData.emails) { - const verified = verifyEmails || existingEmails.find((ee) => ee.address === email)?.verified || false; - - newEmailList.push({ - address: email, - verified, - }); - } - - updateData.$set.emails = newEmailList; - } - - addUserServices(updateData: Record, userData: IImportUser): void { - if (!userData.services) { - return; - } - - for (const serviceKey in userData.services) { - if (!userData.services[serviceKey]) { - continue; - } - - const service = userData.services[serviceKey]; - - for (const key in service) { - if (!service[key]) { - continue; - } - - updateData.$set[`services.${serviceKey}.${key}`] = service[key]; - } - } - } - - addCustomFields(updateData: Record, userData: IImportUser): void { - if (!userData.customFields) { - return; - } - - const subset = (source: Record, currentPath: string): void => { - for (const key in source) { - if (!source.hasOwnProperty(key)) { - continue; - } - - const keyPath = `${currentPath}.${key}`; - if (typeof source[key] === 'object' && !Array.isArray(source[key])) { - subset(source[key], keyPath); - continue; - } - - updateData.$set = { - ...updateData.$set, - ...{ [keyPath]: source[key] }, - }; - } - }; - - subset(userData.customFields, 'customFields'); - } - - async updateUser(existingUser: IUser, userData: IImportUser): Promise { - const { _id } = existingUser; - if (!_id) { - return; - } - - userData._id = _id; - - if (!userData.roles && !existingUser.roles) { - userData.roles = ['user']; - } - if (!userData.type && !existingUser.type) { - userData.type = 'user'; - } - - const updateData: Record = Object.assign(Object.create(null), { - $set: Object.assign(Object.create(null), { - ...(userData.roles && { roles: userData.roles }), - ...(userData.type && { type: userData.type }), - ...(userData.statusText && { statusText: userData.statusText }), - ...(userData.bio && { bio: userData.bio }), - ...(userData.services?.ldap && { ldap: true }), - ...(userData.avatarUrl && { _pendingAvatarUrl: userData.avatarUrl }), - }), - }); - - this.addCustomFields(updateData, userData); - this.addUserServices(updateData, userData); - this.addUserImportId(updateData, userData); - this.addUserEmails(updateData, userData, existingUser.emails || []); - - if (Object.keys(updateData.$set).length === 0) { - delete updateData.$set; - } - if (Object.keys(updateData).length > 0) { - await Users.updateOne({ _id }, updateData); - } - - if (userData.utcOffset) { - await Users.setUtcOffset(_id, userData.utcOffset); - } - - if (userData.name || userData.username) { - await saveUserIdentity({ _id, name: userData.name, username: userData.username } as Parameters[0]); - } - - if (userData.importIds.length) { - this.addUserToCache(userData.importIds[0], existingUser._id, existingUser.username || userData.username); - } - - // Deleted users are 'inactive' users in Rocket.Chat - if (userData.deleted && existingUser?.active) { - await setUserActiveStatus(_id, false, true); - } else if (userData.deleted === false && existingUser?.active === false) { - await setUserActiveStatus(_id, true); - } - - void notifyOnUserChange({ clientAction: 'updated', id: _id, diff: updateData.$set }); - } - - private async hashPassword(password: string): Promise { - return bcryptHash(SHA256(password), Accounts._bcryptRounds()); - } - - private generateTempPassword(userData: IImportUser): string { - return `${Date.now()}${userData.name || ''}${userData.emails.length ? userData.emails[0].toUpperCase() : ''}`; - } - - private async buildNewUserObject(userData: IImportUser): Promise> { + protected getRecordConverterOptions(): RecordConverterOptions { return { - type: userData.type || 'user', - ...(userData.username && { username: userData.username }), - ...(userData.emails.length && { - emails: userData.emails.map((email) => ({ address: email, verified: !!this._options.flagEmailsAsVerified })), - }), - ...(userData.statusText && { statusText: userData.statusText }), - ...(userData.name && { name: userData.name }), - ...(userData.bio && { bio: userData.bio }), - ...(userData.avatarUrl && { _pendingAvatarUrl: userData.avatarUrl }), - ...(userData.utcOffset !== undefined && { utcOffset: userData.utcOffset }), - ...{ - services: { - // Add a password service if there's a password string, or if there's no service at all - ...((!!userData.password || !userData.services || !Object.keys(userData.services).length) && { - password: { bcrypt: await this.hashPassword(userData.password || this.generateTempPassword(userData)) }, - }), - ...(userData.services || {}), - }, - }, - ...(userData.services?.ldap && { ldap: true }), - ...(userData.importIds?.length && { importIds: userData.importIds }), - ...(!!userData.customFields && { customFields: userData.customFields }), - ...(userData.deleted !== undefined && { active: !userData.deleted }), + ...pick(this._options, 'workInMemory'), + // DbData is deleted by this class directly, so the converters don't need to do it individually + deleteDbData: false, }; } - private async buildUserBatch(usersData: IImportUser[]): Promise { - return Promise.all( - usersData.map(async (userData) => { - const user = await this.buildNewUserObject(userData); - return { - createdAt: new Date(), - _id: Random.id(), - - status: 'offline', - ...user, - roles: userData.roles?.length ? userData.roles : ['user'], - active: !userData.deleted, - services: { - ...user.services, - ...(this._options.enableEmail2fa - ? { - email2fa: { - enabled: true, - changedAt: new Date(), - }, - } - : {}), - }, - } as IUser; - }), - ); - } - - async insertUser(userData: IImportUser): Promise { - const user = await this.buildNewUserObject(userData); - - return Accounts.insertUserDoc( - { - joinDefaultChannels: false, - skipEmailValidation: true, - skipAdminCheck: true, - skipAdminEmail: true, - skipOnCreateUserCallback: this._options.skipUserCallbacks, - skipBeforeCreateUserCallback: this._options.skipUserCallbacks, - skipAfterCreateUserCallback: this._options.skipUserCallbacks, - skipDefaultAvatar: true, - skipAppsEngineEvent: !!process.env.IMPORTER_SKIP_APPS_EVENT, - }, - { - ...user, - ...(userData.roles?.length ? { globalRoles: userData.roles } : {}), - }, - ); - } - - protected async getUsersToImport(): Promise> { - return ImportData.getAllUsers().toArray(); - } - - async findExistingUser(data: IImportUser): Promise { - if (data.emails.length) { - const emailUser = await Users.findOneByEmailAddress(data.emails[0], {}); - - if (emailUser) { - return emailUser; - } - } - - // If we couldn't find one by their email address, try to find an existing user by their username - if (data.username) { - return Users.findOneByUsernameIgnoringCase(data.username, {}); - } - } - - private async insertUserBatch(users: IUser[], { afterBatchFn }: IConversionCallbacks): Promise { - let newIds: string[] | null = null; - - try { - newIds = Object.values((await Users.insertMany(users, { ordered: false })).insertedIds); - if (afterBatchFn) { - await afterBatchFn(newIds.length, 0); - } - } catch (e: any) { - newIds = (e.result?.result?.insertedIds || []) as string[]; - const errorCount = users.length - (e.result?.result?.nInserted || 0); - - if (afterBatchFn) { - await afterBatchFn(Math.min(newIds.length, users.length - errorCount), errorCount); - } - } - - return newIds; - } - - public async convertUsers({ beforeImportFn, afterImportFn, onErrorFn, afterBatchFn }: IConversionCallbacks = {}): Promise { - const users = (await this.getUsersToImport()) as IImportUserRecord[]; - - const insertedIds = new Set(); - const updatedIds = new Set(); - let skippedCount = 0; - let failedCount = 0; - - const batchToInsert = new Set(); - - for await (const record of users) { - const { data, _id } = record; - if (this.aborted) { - break; - } - - try { - if (beforeImportFn && !(await beforeImportFn(record))) { - await this.skipRecord(_id); - skippedCount++; - continue; - } - - const emails = data.emails.filter(Boolean).map((email) => ({ address: email })); - data.importIds = data.importIds.filter((item) => item); - - if (!data.emails.length && !data.username) { - throw new Error('importer-user-missing-email-and-username'); - } - - if (this.options.quickUserInsertion) { - batchToInsert.add(data); - - if (batchToInsert.size >= 50) { - const usersToInsert = await this.buildUserBatch([...batchToInsert]); - batchToInsert.clear(); - - const newIds = await this.insertUserBatch(usersToInsert, { afterBatchFn }); - newIds.forEach((id) => insertedIds.add(id)); - } - - continue; - } - - const existingUser = await this.findExistingUser(data); - if (existingUser && this._options.skipExistingUsers) { - await this.skipRecord(_id); - skippedCount++; - continue; - } - if (!existingUser && this._options.skipNewUsers) { - await this.skipRecord(_id); - skippedCount++; - continue; - } - - if (!data.username && !existingUser?.username) { - data.username = await generateUsernameSuggestion({ - name: data.name, - emails, - }); - } - - const isNewUser = !existingUser; - - if (existingUser) { - await this.updateUser(existingUser, data); - updatedIds.add(existingUser._id); - } else { - if (!data.name && data.username) { - data.name = guessNameFromUsername(data.username); - } - - const userId = await this.insertUser(data); - data._id = userId; - insertedIds.add(userId); - - if (!this._options.skipDefaultChannels) { - const insertedUser = await Users.findOneById(userId, {}); - if (!insertedUser) { - throw new Error(`User not found: ${userId}`); - } - - await addUserToDefaultChannels(insertedUser, true); - } - } - - if (afterImportFn) { - await afterImportFn(record, isNewUser); - } - } catch (e) { - this._logger.error(e); - await this.saveError(_id, e instanceof Error ? e : new Error(String(e))); - failedCount++; - - if (onErrorFn) { - await onErrorFn(); - } - } - } - - if (batchToInsert.size > 0) { - const usersToInsert = await this.buildUserBatch([...batchToInsert]); - const newIds = await this.insertUserBatch(usersToInsert, { afterBatchFn }); - newIds.forEach((id) => insertedIds.add(id)); - } - - await callbacks.run('afterUserImport', { - inserted: [...insertedIds], - updated: [...updatedIds], - skipped: skippedCount, - failed: failedCount, - }); - } - - protected async saveError(importId: string, error: Error): Promise { - this._logger.error(error); - await ImportData.updateOne( - { - _id: importId, - }, - { - $push: { - errors: { - message: error.message, - stack: error.stack, - }, - }, - }, - ); - } - - protected async skipRecord(_id: string): Promise { - await ImportData.updateOne( - { - _id, - }, - { - $set: { - skipped: true, - }, - }, - ); - } - - async convertMessageReactions(importedReactions: Record): Promise { - const reactions: IMessageReactions = {}; - - for await (const name of Object.keys(importedReactions)) { - if (!importedReactions.hasOwnProperty(name)) { - continue; - } - const { users } = importedReactions[name]; - - if (!users.length) { - continue; - } - - const reaction: IMessageReaction = { - name, - usernames: [], - }; - - for await (const importId of users) { - const username = await this.findImportedUsername(importId); - if (username && !reaction.usernames.includes(username)) { - reaction.usernames.push(username); - } - } - - if (reaction.usernames.length) { - reactions[name] = reaction; - } - } - - if (Object.keys(reactions).length > 0) { - return reactions; - } - } - - async convertMessageReplies(replies: Array): Promise> { - const result: Array = []; - for await (const importId of replies) { - const userId = await this.findImportedUserId(importId); - if (userId && !result.includes(userId)) { - result.push(userId); - } - } - return result; - } - - async convertMessageMentions(message: IImportMessage): Promise | undefined> { - const { mentions } = message; - if (!mentions) { - return undefined; - } - - const result: Array = []; - for await (const importId of mentions) { - if (importId === ('all' as 'string') || importId === 'here') { - result.push({ - _id: importId, - username: importId, - }); - continue; - } - - // Loading the name will also store the remaining data on the cache if it's missing, so this won't run two queries - const name = await this.findImportedUserDisplayName(importId); - const data = await this.findImportedUser(importId); - - if (!data) { - this._logger.warn(`Mentioned user not found: ${importId}`); - continue; - } - - if (!data.username) { - this._logger.debug(importId); - throw new Error('importer-message-mentioned-username-not-found'); - } - - message.msg = message.msg.replace(new RegExp(`\@${importId}`, 'gi'), `@${data.username}`); - - result.push({ - _id: data._id, - username: data.username as 'string', - name, - }); - } - return result; - } - - async getMentionedChannelData(importId: string): Promise { - // loading the name will also store the id on the cache if it's missing, so this won't run two queries - const name = await this.findImportedRoomName(importId); - const _id = await this.findImportedRoomId(importId); - - if (name && _id) { - return { - name, - _id, - }; - } - - // If the importId was not found, check if we have a room with that name - const roomName = await getValidRoomName(importId.trim(), undefined, { allowDuplicates: true }); - const room = await Rooms.findOneByNonValidatedName(roomName, { projection: { name: 1 } }); - if (room?.name) { - this.addRoomToCache(importId, room._id); - this.addRoomNameToCache(importId, room.name); - - return { - name: room.name, - _id: room._id, - }; - } - } - - async convertMessageChannels(message: IImportMessage): Promise { - const { channels } = message; - if (!channels) { - return; - } - - const result: Array = []; - for await (const importId of channels) { - const { name, _id } = (await this.getMentionedChannelData(importId)) || {}; - - if (!_id || !name) { - this._logger.warn(`Mentioned room not found: ${importId}`); - continue; - } - - message.msg = message.msg.replace(new RegExp(`\#${importId}`, 'gi'), `#${name}`); - - result.push({ - _id, - name, - }); - } - - return result; - } - - protected async getMessagesToImport(): Promise> { - return ImportData.getAllMessages().toArray(); - } - - async convertMessages({ - beforeImportFn, - afterImportFn, - onErrorFn, - afterImportAllMessagesFn, - }: IConversionCallbacks & { afterImportAllMessagesFn?: (roomIds: string[]) => Promise }): Promise { - const rids: Array = []; - const messages = await this.getMessagesToImport(); - - for await (const record of messages) { - const { data, _id } = record; - if (this.aborted) { - return; - } - - try { - if (beforeImportFn && !(await beforeImportFn(record))) { - await this.skipRecord(_id); - continue; - } - - if (!data.ts || isNaN(data.ts as unknown as number)) { - throw new Error('importer-message-invalid-timestamp'); - } - - const creator = await this.findImportedUser(data.u._id); - if (!creator) { - this._logger.warn(`Imported user not found: ${data.u._id}`); - throw new Error('importer-message-unknown-user'); - } - const rid = await this.findImportedRoomId(data.rid); - if (!rid) { - throw new Error('importer-message-unknown-room'); - } - if (!rids.includes(rid)) { - rids.push(rid); - } - - // Convert the mentions and channels first because these conversions can also modify the msg in the message object - const mentions = data.mentions && (await this.convertMessageMentions(data)); - const channels = data.channels && (await this.convertMessageChannels(data)); - - const msgObj: IMessage = { - rid, - u: { - _id: creator._id, - username: creator.username, - }, - msg: data.msg, - ts: data.ts, - t: data.t || undefined, - groupable: data.groupable, - tmid: data.tmid, - tlm: data.tlm, - tcount: data.tcount, - replies: data.replies && (await this.convertMessageReplies(data.replies)), - editedAt: data.editedAt, - editedBy: data.editedBy && ((await this.findImportedUser(data.editedBy)) || undefined), - mentions, - channels, - _importFile: data._importFile, - url: data.url, - attachments: data.attachments, - bot: data.bot, - emoji: data.emoji, - alias: data.alias, - }; - - if (data._id) { - msgObj._id = data._id; - } - - if (data.reactions) { - msgObj.reactions = await this.convertMessageReactions(data.reactions); - } - - try { - await insertMessage(creator, msgObj as unknown as IDBMessage, rid, true); - } catch (e) { - this._logger.warn(`Failed to import message with timestamp ${String(msgObj.ts)} to room ${rid}`); - this._logger.error(e); - } - - if (afterImportFn) { - await afterImportFn(record, true); - } - } catch (e) { - await this.saveError(_id, e instanceof Error ? e : new Error(String(e))); - if (onErrorFn) { - await onErrorFn(); - } - } - } - - for await (const rid of rids) { - try { - await Rooms.resetLastMessageById(rid, null); - } catch (e) { - this._logger.warn(`Failed to update last message of room ${rid}`); - this._logger.error(e); - } - } - if (afterImportAllMessagesFn) { - await afterImportAllMessagesFn(rids); - } - } - - async updateRoom(room: IRoom, roomData: IImportChannel, startedByUserId: string): Promise { - roomData._id = room._id; - - if ((roomData._id as string).toUpperCase() === 'GENERAL' && roomData.name !== room.name) { - await saveRoomSettings(startedByUserId, 'GENERAL', 'roomName', roomData.name); - } - - await this.updateRoomId(room._id, roomData); - } - - public async findDMForImportedUsers(...users: Array): Promise { - const record = await ImportData.findDMForImportedUsers(...users); - if (record) { - return record.data; - } - } - - async findImportedRoomId(importId: string): Promise { - if (this._roomCache.has(importId)) { - return this._roomCache.get(importId) as string; - } - - const options = { - projection: { - _id: 1, - }, - }; - - const room = await Rooms.findOneByImportId(importId, options); - if (room) { - return this.addRoomToCache(importId, room._id); - } - - return null; - } - - async findImportedRoomName(importId: string): Promise { - if (this._roomNameCache.has(importId)) { - return this._roomNameCache.get(importId) as string; - } + protected getUserConverterOptions(): UserConverterOptions { + return { + flagEmailsAsVerified: false, + skipExistingUsers: false, + skipNewUsers: false, - const options = { - projection: { - _id: 1, - name: 1, - }, + ...pick( + this._options, + 'flagEmailsAsVerified', + 'skipExistingUsers', + 'skipNewUsers', + 'skipUserCallbacks', + 'skipDefaultChannels', + 'quickUserInsertion', + 'enableEmail2fa', + ), }; - - const room = await Rooms.findOneByImportId(importId, options); - if (room) { - if (!this._roomCache.has(importId)) { - this.addRoomToCache(importId, room._id); - } - if (room?.name) { - return this.addRoomNameToCache(importId, room.name); - } - } } - async findImportedUser(importId: string): Promise { - const options = { - projection: { - _id: 1, - username: 1, - }, + protected initializeUserConverter(logger: Logger): void { + const userOptions = { + ...this.getRecordConverterOptions(), + ...this.getUserConverterOptions(), }; - if (importId === 'rocket.cat') { - return { - _id: 'rocket.cat', - username: 'rocket.cat', - }; - } - - if (this._userCache.has(importId)) { - return this._userCache.get(importId) as IUserIdentification; - } - - const user = await Users.findOneByImportId(importId, options); - if (user) { - return this.addUserToCache(importId, user._id, user.username); - } - - return null; + this._userConverter = new UserConverter(userOptions, logger, this._cache); } - async findImportedUserId(_id: string): Promise { - const data = await this.findImportedUser(_id); - return data?._id; - } - - async findImportedUsername(_id: string): Promise { - const data = await this.findImportedUser(_id); - return data?.username; - } - - async findImportedUserDisplayName(importId: string): Promise { - const options = { - projection: { - _id: 1, - name: 1, - username: 1, - }, + protected initializeRoomConverter(logger: Logger): void { + const roomOptions = { + ...this.getRecordConverterOptions(), }; - if (this._userDisplayNameCache.has(importId)) { - return this._userDisplayNameCache.get(importId); - } - - const user = - importId === 'rocket.cat' ? await Users.findOneById('rocket.cat', options) : await Users.findOneByImportId(importId, options); - if (user) { - if (!this._userCache.has(importId)) { - this.addUserToCache(importId, user._id, user.username); - } - - if (!user.name) { - return; - } - - return this.addUserDisplayNameToCache(importId, user.name); - } + this._roomConverter = new RoomConverter(roomOptions, logger, this._cache); } - async updateRoomId(_id: string, roomData: IImportChannel): Promise { - const set = { - ts: roomData.ts, - topic: roomData.topic, - description: roomData.description, + protected initializeMessageConverter(logger: Logger): void { + const messageOptions = { + ...this.getRecordConverterOptions(), }; - const roomUpdate: { $set?: Record; $addToSet?: Record } = {}; - - if (Object.keys(set).length > 0) { - roomUpdate.$set = set; - } - - if (roomData.importIds.length) { - roomUpdate.$addToSet = { - importIds: { - $each: roomData.importIds, - }, - }; - } - - if (roomUpdate.$set || roomUpdate.$addToSet) { - await Rooms.updateOne({ _id: roomData._id }, roomUpdate); - } + this._messageConverter = new MessageConverter(messageOptions, logger, this._cache); } - async getRoomCreatorId(roomData: IImportChannel, startedByUserId: string): Promise { - if (roomData.u) { - const creatorId = await this.findImportedUserId(roomData.u._id); - if (creatorId) { - return creatorId; - } - - if (roomData.t !== 'd') { - return startedByUserId; - } - - throw new Error('importer-channel-invalid-creator'); - } - - if (roomData.t === 'd') { - for await (const member of roomData.users) { - const userId = await this.findImportedUserId(member); - if (userId) { - return userId; - } - } - } - - throw new Error('importer-channel-invalid-creator'); - } - - async insertRoom(roomData: IImportChannel, startedByUserId: string): Promise { - // Find the rocketchatId of the user who created this channel - const creatorId = await this.getRoomCreatorId(roomData, startedByUserId); - const members = await this.convertImportedIdsToUsernames(roomData.users, roomData.t !== 'd' ? creatorId : undefined); - - if (roomData.t === 'd') { - if (members.length < roomData.users.length) { - this._logger.warn(`One or more imported users not found: ${roomData.users}`); - throw new Error('importer-channel-missing-users'); - } - } - - // Create the channel - try { - let roomInfo; - if (roomData.t === 'd') { - roomInfo = await createDirectMessage(members, startedByUserId, true); - } else { - if (!roomData.name) { - return; - } - if (roomData.t === 'p') { - const user = await Users.findOneById(creatorId); - if (!user) { - throw new Error('importer-channel-invalid-creator'); - } - roomInfo = await createPrivateGroupMethod(user, roomData.name, members, false, {}, {}); - } else { - roomInfo = await createChannelMethod(creatorId, roomData.name, members, false, {}, {}); - } - } - - roomData._id = roomInfo.rid; - } catch (e) { - this._logger.warn({ msg: 'Failed to create new room', name: roomData.name, members }); - this._logger.error(e); - throw e; - } - - await this.updateRoomId(roomData._id as 'string', roomData); + async addUser(data: IImportUser): Promise { + return this._userConverter.addObject(data); } - async convertImportedIdsToUsernames(importedIds: Array, idToRemove: string | undefined = undefined): Promise> { - return ( - await Promise.all( - importedIds.map(async (user) => { - if (user === 'rocket.cat') { - return user; - } - - if (this._userCache.has(user)) { - const cache = this._userCache.get(user); - if (cache) { - return cache.username; - } - } - - const obj = await Users.findOneByImportId(user, { projection: { _id: 1, username: 1 } }); - if (obj) { - this.addUserToCache(user, obj._id, obj.username); - - if (idToRemove && obj._id === idToRemove) { - return false; - } - - return obj.username; - } - - return false; - }), - ) - ).filter((user) => user) as string[]; + async addChannel(data: IImportChannel): Promise { + return this._roomConverter.addObject(data); } - async findExistingRoom(data: IImportChannel): Promise { - if (data._id && data._id.toUpperCase() === 'GENERAL') { - const room = await Rooms.findOneById('GENERAL', {}); - // Prevent the importer from trying to create a new general - if (!room) { - throw new Error('importer-channel-general-not-found'); - } - - return room; - } - - if (data.t === 'd') { - const users = await this.convertImportedIdsToUsernames(data.users); - if (users.length !== data.users.length) { - throw new Error('importer-channel-missing-users'); - } - - return Rooms.findDirectRoomContainingAllUsernames(users, {}); - } - - if (!data.name) { - return null; - } - - const roomName = await getValidRoomName(data.name.trim(), undefined, { allowDuplicates: true }); - return Rooms.findOneByNonValidatedName(roomName, {}); + async addMessage(data: IImportMessage, useQuickInsert = false): Promise { + return this._messageConverter.addObject(data, { + useQuickInsert: useQuickInsert || undefined, + }); } - protected async getChannelsToImport(): Promise> { - return ImportData.getAllChannels().toArray(); + async convertUsers(callbacks: IConversionCallbacks): Promise { + return this._userConverter.convertData(callbacks); } - async convertChannels(startedByUserId: string, { beforeImportFn, afterImportFn, onErrorFn }: IConversionCallbacks = {}): Promise { - const channels = await this.getChannelsToImport(); - for await (const record of channels) { - const { data, _id } = record; - if (this.aborted) { - return; - } - - try { - if (beforeImportFn && !(await beforeImportFn(record))) { - await this.skipRecord(_id); - continue; - } - - if (!data.name && data.t !== 'd') { - throw new Error('importer-channel-missing-name'); - } - - data.importIds = data.importIds.filter((item) => item); - data.users = [...new Set(data.users)]; - - if (!data.importIds.length) { - throw new Error('importer-channel-missing-import-id'); - } - - const existingRoom = await this.findExistingRoom(data); - - if (existingRoom) { - await this.updateRoom(existingRoom, data, startedByUserId); - } else { - await this.insertRoom(data, startedByUserId); - } - - if (data.archived && data._id) { - await this.archiveRoomById(data._id); - } - - if (afterImportFn) { - await afterImportFn(record, !existingRoom); - } - } catch (e) { - await this.saveError(_id, e instanceof Error ? e : new Error(String(e))); - if (onErrorFn) { - await onErrorFn(); - } - } - } + async convertChannels(startedByUserId: string, callbacks: IConversionCallbacks): Promise { + return this._roomConverter.convertChannels(startedByUserId, callbacks); } - async archiveRoomById(rid: string) { - const responses = await Promise.all([Rooms.archiveById(rid), Subscriptions.archiveByRoomId(rid)]); - - if (responses[1]?.modifiedCount) { - void notifyOnSubscriptionChangedByRoomId(rid); - } + async convertMessages(callbacks: MessageConversionCallbacks): Promise { + return this._messageConverter.convertData(callbacks); } async convertData(startedByUserId: string, callbacks: IConversionCallbacks = {}): Promise { @@ -1178,16 +126,34 @@ export class ImportDataConverter { }); } + protected getAllConverters(): RecordConverter[] { + return [this._userConverter, this._roomConverter, this._messageConverter]; + } + public async clearImportData(): Promise { - // Using raw collection since its faster - await ImportData.col.deleteMany({}); + if (!this._options.workInMemory) { + // Using raw collection since its faster + await ImportData.col.deleteMany({}); + } + + await Promise.all(this.getAllConverters().map((converter) => converter.clearImportData())); } async clearSuccessfullyImportedData(): Promise { - await ImportData.col.deleteMany({ - errors: { - $exists: false, - }, + if (!this._options.workInMemory) { + await ImportData.col.deleteMany({ + errors: { + $exists: false, + }, + }); + } + + await Promise.all(this.getAllConverters().map((converter) => converter.clearSuccessfullyImportedData())); + } + + public abort(): void { + this.getAllConverters().forEach((converter) => { + converter.aborted = true; }); } } diff --git a/apps/meteor/app/importer/server/classes/Importer.ts b/apps/meteor/app/importer/server/classes/Importer.ts index 846f9ef4b4f5..d89cb5f979f3 100644 --- a/apps/meteor/app/importer/server/classes/Importer.ts +++ b/apps/meteor/app/importer/server/classes/Importer.ts @@ -12,7 +12,7 @@ import { t } from '../../../utils/lib/i18n'; import { ProgressStep, ImportPreparingStartedStates } from '../../lib/ImporterProgressStep'; import type { ImporterInfo } from '../definitions/ImporterInfo'; import { ImportDataConverter } from './ImportDataConverter'; -import type { IConverterOptions } from './ImportDataConverter'; +import type { ConverterOptions } from './ImportDataConverter'; import { ImporterProgress } from './ImporterProgress'; import { ImporterWebsocket } from './ImporterWebsocket'; @@ -46,17 +46,15 @@ export class Importer { public progress: ImporterProgress; - constructor(info: ImporterInfo, importRecord: IImport, converterOptions: IConverterOptions = {}) { + constructor(info: ImporterInfo, importRecord: IImport, converterOptions: ConverterOptions = {}) { if (!info.key || !info.importer) { throw new Error('Information passed in must be a valid ImporterInfo instance.'); } - this.converter = new ImportDataConverter(converterOptions); - this.info = info; - this.logger = new Logger(`${this.info.name} Importer`); - this.converter.setLogger(this.logger); + + this.converter = new ImportDataConverter(this.logger, converterOptions); this.importRecord = importRecord; this.progress = new ImporterProgress(this.info.key, this.info.name); @@ -120,7 +118,7 @@ export class Importer { const beforeImportFn = async ({ data, dataType: type }: IImportRecord) => { if (this.importRecord.valid === false) { - this.converter.aborted = true; + this.converter.abort(); throw new Error('The import operation is no longer valid.'); } @@ -167,7 +165,7 @@ export class Importer { await this.addCountCompleted(1); if (this.importRecord.valid === false) { - this.converter.aborted = true; + this.converter.abort(); throw new Error('The import operation is no longer valid.'); } }; @@ -184,7 +182,7 @@ export class Importer { } if (this.importRecord.valid === false) { - this.converter.aborted = true; + this.converter.abort(); throw new Error('The import operation is no longer valid.'); } }; diff --git a/apps/meteor/app/importer/server/classes/VirtualDataConverter.ts b/apps/meteor/app/importer/server/classes/VirtualDataConverter.ts deleted file mode 100644 index ef850226be5c..000000000000 --- a/apps/meteor/app/importer/server/classes/VirtualDataConverter.ts +++ /dev/null @@ -1,169 +0,0 @@ -import type { - IImportUser, - IImportUserRecord, - IImportChannelRecord, - IImportMessageRecord, - IImportRecord, - IImportRecordType, - IImportData, - IImportChannel, -} from '@rocket.chat/core-typings'; -import { Random } from '@rocket.chat/random'; - -import { ImportDataConverter } from './ImportDataConverter'; -import type { IConverterOptions } from './ImportDataConverter'; - -export class VirtualDataConverter extends ImportDataConverter { - protected _userRecords: Array; - - protected _channelRecords: Array; - - protected _messageRecords: Array; - - protected useVirtual: boolean; - - constructor(virtual = true, options?: IConverterOptions) { - super(options); - - this.useVirtual = virtual; - if (virtual) { - this.clearVirtualData(); - } - } - - public async clearImportData(): Promise { - if (!this.useVirtual) { - return super.clearImportData(); - } - - this.clearVirtualData(); - } - - public async clearSuccessfullyImportedData(): Promise { - if (!this.useVirtual) { - return super.clearSuccessfullyImportedData(); - } - - this.clearVirtualData(); - } - - public async findDMForImportedUsers(...users: Array): Promise { - if (!this.useVirtual) { - return super.findDMForImportedUsers(...users); - } - - // The original method is only used by the hipchat importer so we probably don't need to implement this on the virtual converter. - return undefined; - } - - public addUserSync(data: IImportUser, options?: Record): void { - return this.addObjectSync('user', data, options); - } - - protected async addObject(type: IImportRecordType, data: IImportData, options: Record = {}): Promise { - if (!this.useVirtual) { - return super.addObject(type, data, options); - } - - this.addObjectSync(type, data, options); - } - - protected addObjectSync(type: IImportRecordType, data: IImportData, options: Record = {}): void { - if (!this.useVirtual) { - throw new Error('Sync operations can only be used on virtual converter'); - } - - const list = this.getObjectList(type); - - list.push({ - _id: Random.id(), - data, - dataType: type, - options, - }); - } - - protected async getUsersToImport(): Promise> { - if (!this.useVirtual) { - return super.getUsersToImport(); - } - - return this._userRecords; - } - - protected async saveError(importId: string, error: Error): Promise { - if (!this.useVirtual) { - return super.saveError(importId, error); - } - - const record = this.getVirtualRecordById(importId); - - if (!record) { - return; - } - - if (!record.errors) { - record.errors = []; - } - - record.errors.push({ - message: error.message, - stack: error.stack, - }); - } - - protected async skipRecord(_id: string): Promise { - if (!this.useVirtual) { - return super.skipRecord(_id); - } - - const record = this.getVirtualRecordById(_id); - - if (record) { - record.skipped = true; - } - } - - protected async getMessagesToImport(): Promise { - if (!this.useVirtual) { - return super.getMessagesToImport(); - } - - return this._messageRecords; - } - - protected async getChannelsToImport(): Promise { - if (!this.useVirtual) { - return super.getChannelsToImport(); - } - - return this._channelRecords; - } - - private clearVirtualData(): void { - this._userRecords = []; - this._channelRecords = []; - this._messageRecords = []; - } - - private getObjectList(type: IImportRecordType): Array { - switch (type) { - case 'user': - return this._userRecords; - case 'channel': - return this._channelRecords; - case 'message': - return this._messageRecords; - } - } - - private getVirtualRecordById(id: string): IImportRecord | undefined { - for (const store of [this._userRecords, this._channelRecords, this._messageRecords]) { - for (const record of store) { - if (record._id === id) { - return record; - } - } - } - } -} diff --git a/apps/meteor/app/importer/server/classes/converters/ConverterCache.ts b/apps/meteor/app/importer/server/classes/converters/ConverterCache.ts new file mode 100644 index 000000000000..cefbf9cc7dbb --- /dev/null +++ b/apps/meteor/app/importer/server/classes/converters/ConverterCache.ts @@ -0,0 +1,198 @@ +import type { IImportUser } from '@rocket.chat/core-typings'; +import { Rooms, Users } from '@rocket.chat/models'; + +export type UserIdentification = { + _id: string; + username: string | undefined; +}; + +export type MentionedChannel = { + _id: string; + name: string; +}; + +export class ConverterCache { + private _userCache = new Map(); + + // display name uses a different cache because it's only used on mentions so we don't need to load it every time we load an user + private _userDisplayNameCache = new Map(); + + private _roomCache = new Map(); + + private _roomNameCache = new Map(); + + addUser(importId: string, _id: string, username: string | undefined): UserIdentification { + const cache = { + _id, + username, + }; + + this._userCache.set(importId, cache); + return cache; + } + + addUserDisplayName(importId: string, name: string): string { + this._userDisplayNameCache.set(importId, name); + return name; + } + + addRoom(importId: string, rid: string): string { + this._roomCache.set(importId, rid); + return rid; + } + + addRoomName(importId: string, name: string): string { + this._roomNameCache.set(importId, name); + return name; + } + + addUserData(userData: IImportUser): void { + if (!userData._id) { + return; + } + if (!userData.importIds.length) { + return; + } + + this.addUser(userData.importIds[0], userData._id, userData.username); + } + + async findImportedRoomId(importId: string): Promise { + if (this._roomCache.has(importId)) { + return this._roomCache.get(importId) as string; + } + + const options = { + projection: { + _id: 1, + }, + }; + + const room = await Rooms.findOneByImportId(importId, options); + if (room) { + return this.addRoom(importId, room._id); + } + + return null; + } + + async findImportedRoomName(importId: string): Promise { + if (this._roomNameCache.has(importId)) { + return this._roomNameCache.get(importId) as string; + } + + const options = { + projection: { + _id: 1, + name: 1, + }, + }; + + const room = await Rooms.findOneByImportId(importId, options); + if (room) { + if (!this._roomCache.has(importId)) { + this.addRoom(importId, room._id); + } + if (room?.name) { + return this.addRoomName(importId, room.name); + } + } + } + + async findImportedUser(importId: string): Promise { + if (importId === 'rocket.cat') { + return { + _id: 'rocket.cat', + username: 'rocket.cat', + }; + } + + const options = { + projection: { + _id: 1, + username: 1, + }, + }; + + if (this._userCache.has(importId)) { + return this._userCache.get(importId) as UserIdentification; + } + + const user = await Users.findOneByImportId(importId, options); + if (user) { + return this.addUser(importId, user._id, user.username); + } + + return null; + } + + async findImportedUserId(_id: string): Promise { + const data = await this.findImportedUser(_id); + return data?._id; + } + + async findImportedUsername(_id: string): Promise { + const data = await this.findImportedUser(_id); + return data?.username; + } + + async findImportedUserDisplayName(importId: string): Promise { + const options = { + projection: { + _id: 1, + name: 1, + username: 1, + }, + }; + + if (this._userDisplayNameCache.has(importId)) { + return this._userDisplayNameCache.get(importId); + } + + const user = + importId === 'rocket.cat' ? await Users.findOneById('rocket.cat', options) : await Users.findOneByImportId(importId, options); + if (user) { + if (!this._userCache.has(importId)) { + this.addUser(importId, user._id, user.username); + } + + if (!user.name) { + return; + } + + return this.addUserDisplayName(importId, user.name); + } + } + + async convertImportedIdsToUsernames(importedIds: Array, idToRemove: string | undefined = undefined): Promise> { + return ( + await Promise.all( + importedIds.map(async (user) => { + if (user === 'rocket.cat') { + return user; + } + + if (this._userCache.has(user)) { + const cache = this._userCache.get(user); + if (cache) { + return cache.username; + } + } + + const obj = await Users.findOneByImportId(user, { projection: { _id: 1, username: 1 } }); + if (obj) { + this.addUser(user, obj._id, obj.username); + + if (idToRemove && obj._id === idToRemove) { + return false; + } + + return obj.username; + } + + return false; + }), + ) + ).filter((user) => user) as string[]; + } +} diff --git a/apps/meteor/app/importer/server/classes/converters/MessageConverter.ts b/apps/meteor/app/importer/server/classes/converters/MessageConverter.ts new file mode 100644 index 000000000000..b4540ed6182f --- /dev/null +++ b/apps/meteor/app/importer/server/classes/converters/MessageConverter.ts @@ -0,0 +1,263 @@ +import type { IImportMessageRecord, IMessage as IDBMessage, IImportMessage, IImportMessageReaction } from '@rocket.chat/core-typings'; +import { Rooms } from '@rocket.chat/models'; +import limax from 'limax'; + +import { insertMessage } from '../../../../lib/server/functions/insertMessage'; +import type { IConversionCallbacks } from '../../definitions/IConversionCallbacks'; +import type { UserIdentification, MentionedChannel } from './ConverterCache'; +import { RecordConverter } from './RecordConverter'; + +export type MessageConversionCallbacks = IConversionCallbacks & { afterImportAllMessagesFn?: (roomIds: string[]) => Promise }; + +type MessageObject = Record; + +type MentionedUser = { + _id: string; + username: string; + name?: string; +}; + +type IMessageReaction = { + name: string; + usernames: string[]; +}; + +type IMessageReactions = Record; + +export class MessageConverter extends RecordConverter { + private rids: string[] = []; + + async convertData({ afterImportAllMessagesFn, ...callbacks }: MessageConversionCallbacks = {}): Promise { + this.rids = []; + await super.convertData(callbacks); + + await this.resetLastMessages(); + if (afterImportAllMessagesFn) { + await afterImportAllMessagesFn(this.rids); + } + } + + protected async resetLastMessages(): Promise { + for await (const rid of this.rids) { + try { + await Rooms.resetLastMessageById(rid, null); + } catch (e) { + this._logger.warn(`Failed to update last message of room ${rid}`); + this._logger.error(e); + } + } + } + + protected async insertMessage(data: IImportMessage): Promise { + if (!data.ts || isNaN(data.ts as unknown as number)) { + throw new Error('importer-message-invalid-timestamp'); + } + + const creator = await this._cache.findImportedUser(data.u._id); + if (!creator) { + this._logger.warn(`Imported user not found: ${data.u._id}`); + throw new Error('importer-message-unknown-user'); + } + const rid = await this._cache.findImportedRoomId(data.rid); + if (!rid) { + throw new Error('importer-message-unknown-room'); + } + if (!this.rids.includes(rid)) { + this.rids.push(rid); + } + + const msgObj = await this.buildMessageObject(data, rid, creator); + + try { + await insertMessage(creator, msgObj as unknown as IDBMessage, rid, true); + } catch (e) { + this._logger.warn(`Failed to import message with timestamp ${String(msgObj.ts)} to room ${rid}`); + this._logger.error(e); + } + } + + protected async convertRecord(record: IImportMessageRecord): Promise { + await this.insertMessage(record.data); + return true; + } + + protected async buildMessageObject(data: IImportMessage, rid: string, creator: UserIdentification): Promise { + // Convert the mentions and channels first because these conversions can also modify the msg in the message object + const mentions = data.mentions && (await this.convertMessageMentions(data)); + const channels = data.channels && (await this.convertMessageChannels(data)); + + return { + rid, + u: { + _id: creator._id, + username: creator.username, + }, + msg: data.msg, + ts: data.ts, + t: data.t || undefined, + groupable: data.groupable, + tmid: data.tmid, + tlm: data.tlm, + tcount: data.tcount, + replies: data.replies && (await this.convertMessageReplies(data.replies)), + editedAt: data.editedAt, + editedBy: data.editedBy && ((await this._cache.findImportedUser(data.editedBy)) || undefined), + mentions, + channels, + _importFile: data._importFile, + url: data.url, + attachments: data.attachments, + bot: data.bot, + emoji: data.emoji, + alias: data.alias, + ...(data._id ? { _id: data._id } : {}), + ...(data.reactions ? { reactions: await this.convertMessageReactions(data.reactions) } : {}), + }; + } + + protected async convertMessageChannels(message: IImportMessage): Promise { + const { channels } = message; + if (!channels) { + return; + } + + const result: MentionedChannel[] = []; + for await (const importId of channels) { + const { name, _id } = (await this.getMentionedChannelData(importId)) || {}; + + if (!_id || !name) { + this._logger.warn(`Mentioned room not found: ${importId}`); + continue; + } + + message.msg = message.msg.replace(new RegExp(`\#${importId}`, 'gi'), `#${name}`); + + result.push({ + _id, + name, + }); + } + + return result; + } + + protected async convertMessageMentions(message: IImportMessage): Promise { + const { mentions } = message; + if (!mentions) { + return undefined; + } + + const result: MentionedUser[] = []; + for await (const importId of mentions) { + if (importId === ('all' as 'string') || importId === 'here') { + result.push({ + _id: importId, + username: importId, + }); + continue; + } + + // Loading the name will also store the remaining data on the cache if it's missing, so this won't run two queries + const name = await this._cache.findImportedUserDisplayName(importId); + const data = await this._cache.findImportedUser(importId); + + if (!data) { + this._logger.warn(`Mentioned user not found: ${importId}`); + continue; + } + + if (!data.username) { + this._logger.debug(importId); + throw new Error('importer-message-mentioned-username-not-found'); + } + + message.msg = message.msg.replace(new RegExp(`\@${importId}`, 'gi'), `@${data.username}`); + + result.push({ + _id: data._id, + username: data.username as 'string', + name, + }); + } + return result; + } + + protected async convertMessageReactions( + importedReactions: Record, + ): Promise { + const reactions: IMessageReactions = {}; + + for await (const name of Object.keys(importedReactions)) { + if (!importedReactions.hasOwnProperty(name)) { + continue; + } + const { users } = importedReactions[name]; + + if (!users.length) { + continue; + } + + const reaction: IMessageReaction = { + name, + usernames: [], + }; + + for await (const importId of users) { + const username = await this._cache.findImportedUsername(importId); + if (username && !reaction.usernames.includes(username)) { + reaction.usernames.push(username); + } + } + + if (reaction.usernames.length) { + reactions[name] = reaction; + } + } + + if (Object.keys(reactions).length > 0) { + return reactions; + } + } + + protected async convertMessageReplies(replies: string[]): Promise { + const result: string[] = []; + for await (const importId of replies) { + const userId = await this._cache.findImportedUserId(importId); + if (userId && !result.includes(userId)) { + result.push(userId); + } + } + return result; + } + + protected async getMentionedChannelData(importId: string): Promise { + // loading the name will also store the id on the cache if it's missing, so this won't run two queries + const name = await this._cache.findImportedRoomName(importId); + const _id = await this._cache.findImportedRoomId(importId); + + if (name && _id) { + return { + name, + _id, + }; + } + + // If the importId was not found, check if we have a room with that name + const roomName = limax(importId.trim(), { maintainCase: true }); + + const room = await Rooms.findOneByNonValidatedName(roomName, { projection: { name: 1 } }); + if (room?.name) { + this._cache.addRoom(importId, room._id); + this._cache.addRoomName(importId, room.name); + + return { + name: room.name, + _id: room._id, + }; + } + } + + protected getDataType(): 'message' { + return 'message'; + } +} diff --git a/apps/meteor/app/importer/server/classes/converters/RecordConverter.ts b/apps/meteor/app/importer/server/classes/converters/RecordConverter.ts new file mode 100644 index 000000000000..d0a6d60fa723 --- /dev/null +++ b/apps/meteor/app/importer/server/classes/converters/RecordConverter.ts @@ -0,0 +1,237 @@ +import type { IImportRecord } from '@rocket.chat/core-typings'; +import { Logger } from '@rocket.chat/logger'; +import { ImportData } from '@rocket.chat/models'; +import { Random } from '@rocket.chat/random'; +import { type FindCursor, ObjectId } from 'mongodb'; + +import type { IConversionCallbacks } from '../../definitions/IConversionCallbacks'; +import { ConverterCache } from './ConverterCache'; + +export type RecordConverterOptions = { + workInMemory?: boolean; + deleteDbData?: boolean; +}; + +export class RecordConverter { + protected _logger: Logger; + + protected _cache: ConverterCache; + + protected _converterOptions: RecordConverterOptions; + + protected _options: Omit; + + protected _records: R[]; + + protected skippedCount = 0; + + protected failedCount = 0; + + public aborted = false; + + constructor(options?: T, logger?: Logger, cache?: ConverterCache) { + const { workInMemory = false, deleteDbData = false, ...customOptions } = options || ({} as T); + this._converterOptions = { + workInMemory, + deleteDbData, + }; + this._options = customOptions; + + this._logger = logger || new Logger(`Data Importer - ${this.constructor.name}`); + this._cache = cache || new ConverterCache(); + this._records = []; + } + + private skipMemoryRecord(_id: string): void { + const record = this.getMemoryRecordById(_id); + if (!record) { + return; + } + + record.skipped = true; + } + + private async skipDatabaseRecord(_id: string): Promise { + await ImportData.updateOne( + { + _id, + }, + { + $set: { + skipped: true, + }, + }, + ); + } + + protected async skipRecord(_id: string): Promise { + this.skippedCount++; + this.skipMemoryRecord(_id); + if (!this._converterOptions.workInMemory) { + return this.skipDatabaseRecord(_id); + } + } + + private saveErrorToMemory(importId: string, error: Error): void { + const record = this.getMemoryRecordById(importId); + + if (!record) { + return; + } + + if (!record.errors) { + record.errors = []; + } + + record.errors.push({ + message: error.message, + stack: error.stack, + }); + } + + private async saveErrorToDatabase(importId: string, error: Error): Promise { + await ImportData.updateOne( + { + _id: importId, + }, + { + $push: { + errors: { + message: error.message, + stack: error.stack, + }, + }, + }, + ); + } + + protected async saveError(importId: string, error: Error): Promise { + this._logger.error(error); + this.saveErrorToMemory(importId, error); + + if (!this._converterOptions.workInMemory) { + return this.saveErrorToDatabase(importId, error); + } + } + + public async clearImportData(): Promise { + this._records = []; + + // On regular import operations this data will be deleted by the importer class with one single operation for all dataTypes (aka with no filter) + if (!this._converterOptions.workInMemory && this._converterOptions.deleteDbData) { + await ImportData.col.deleteMany({ dataType: this.getDataType() }); + } + } + + public async clearSuccessfullyImportedData(): Promise { + this._records = this._records.filter((record) => !record.errors?.length); + + // On regular import operations this data will be deleted by the importer class with one single operation for all dataTypes (aka with no filter) + if (!this._converterOptions.workInMemory && this._converterOptions.deleteDbData) { + await ImportData.col.deleteMany({ dataType: this.getDataType(), error: { $exists: false } }); + } + } + + private getMemoryRecordById(id: string): R | undefined { + for (const record of this._records) { + if (record._id === id) { + return record; + } + } + + return undefined; + } + + protected getDataType(): R['dataType'] { + throw new Error('Unspecified type'); + } + + protected async addObjectToDatabase(data: R['data'], options: R['options'] = {}): Promise { + await ImportData.col.insertOne({ + _id: new ObjectId().toHexString(), + data, + dataType: this.getDataType(), + options, + }); + } + + public addObjectToMemory(data: R['data'], options: R['options'] = {}): void { + this._records.push({ + _id: Random.id(), + data, + dataType: this.getDataType(), + options, + } as R); + } + + public async addObject(data: R['data'], options: R['options'] = {}): Promise { + if (this._converterOptions.workInMemory) { + return this.addObjectToMemory(data, options); + } + + return this.addObjectToDatabase(data, options); + } + + protected getDatabaseDataToImport(): Promise { + return (ImportData.find({ dataType: this.getDataType() }) as FindCursor).toArray(); + } + + protected async getDataToImport(): Promise { + if (this._converterOptions.workInMemory) { + return this._records; + } + + const dbRecords = await this.getDatabaseDataToImport(); + if (this._records.length) { + return [...this._records, ...dbRecords]; + } + + return dbRecords; + } + + protected async iterateRecords({ + beforeImportFn, + afterImportFn, + onErrorFn, + processRecord, + }: IConversionCallbacks & { processRecord?: (record: R) => Promise } = {}): Promise { + const records = await this.getDataToImport(); + + this.skippedCount = 0; + this.failedCount = 0; + + for await (const record of records) { + const { _id } = record; + if (this.aborted) { + return; + } + + try { + if (beforeImportFn && !(await beforeImportFn(record))) { + await this.skipRecord(_id); + continue; + } + + const isNew = await (processRecord || this.convertRecord).call(this, record); + + if (typeof isNew === 'boolean' && afterImportFn) { + await afterImportFn(record, isNew); + } + } catch (e) { + this.failedCount++; + await this.saveError(_id, e instanceof Error ? e : new Error(String(e))); + if (onErrorFn) { + await onErrorFn(); + } + } + } + } + + async convertData(callbacks: IConversionCallbacks = {}): Promise { + return this.iterateRecords(callbacks); + } + + protected async convertRecord(_record: R): Promise { + return undefined; + } +} diff --git a/apps/meteor/app/importer/server/classes/converters/RoomConverter.ts b/apps/meteor/app/importer/server/classes/converters/RoomConverter.ts new file mode 100644 index 000000000000..f57fa1a7cb88 --- /dev/null +++ b/apps/meteor/app/importer/server/classes/converters/RoomConverter.ts @@ -0,0 +1,198 @@ +import type { IImportChannel, IImportChannelRecord, IRoom } from '@rocket.chat/core-typings'; +import { Subscriptions, Rooms, Users } from '@rocket.chat/models'; +import limax from 'limax'; + +import { createDirectMessage } from '../../../../../server/methods/createDirectMessage'; +import { saveRoomSettings } from '../../../../channel-settings/server/methods/saveRoomSettings'; +import { notifyOnSubscriptionChangedByRoomId } from '../../../../lib/server/lib/notifyListener'; +import { createChannelMethod } from '../../../../lib/server/methods/createChannel'; +import { createPrivateGroupMethod } from '../../../../lib/server/methods/createPrivateGroup'; +import type { IConversionCallbacks } from '../../definitions/IConversionCallbacks'; +import { RecordConverter } from './RecordConverter'; + +export class RoomConverter extends RecordConverter { + public startedByUserId: string; + + async convertChannels(startedByUserId: string, callbacks: IConversionCallbacks = {}): Promise { + this.startedByUserId = startedByUserId; + + return this.convertData(callbacks); + } + + protected async convertRecord(record: IImportChannelRecord): Promise { + const { data } = record; + + if (!data.name && data.t !== 'd') { + throw new Error('importer-channel-missing-name'); + } + + data.importIds = data.importIds.filter((item) => item); + data.users = [...new Set(data.users)]; + + if (!data.importIds.length) { + throw new Error('importer-channel-missing-import-id'); + } + + const existingRoom = await this.findExistingRoom(data); + await this.insertOrUpdateRoom(existingRoom, data, this.startedByUserId); + + return !existingRoom; + } + + async insertOrUpdateRoom(existingRoom: IRoom | null, data: IImportChannel, startedByUserId: string): Promise { + if (existingRoom) { + await this.updateRoom(existingRoom, data, startedByUserId); + } else { + await this.insertRoom(data, startedByUserId); + } + + if (data.archived && data._id) { + await this.archiveRoomById(data._id); + } + } + + async findExistingRoom(data: IImportChannel): Promise { + if (data._id && data._id.toUpperCase() === 'GENERAL') { + const room = await Rooms.findOneById('GENERAL', {}); + // Prevent the importer from trying to create a new general + if (!room) { + throw new Error('importer-channel-general-not-found'); + } + + return room; + } + + if (data.t === 'd') { + const users = await this._cache.convertImportedIdsToUsernames(data.users); + if (users.length !== data.users.length) { + throw new Error('importer-channel-missing-users'); + } + + return Rooms.findDirectRoomContainingAllUsernames(users, {}); + } + + if (!data.name) { + return null; + } + + // Imported room names always allow special chars + const roomName = limax(data.name.trim(), { maintainCase: true }); + return Rooms.findOneByNonValidatedName(roomName, {}); + } + + async updateRoom(room: IRoom, roomData: IImportChannel, startedByUserId: string): Promise { + roomData._id = room._id; + + if ((roomData._id as string).toUpperCase() === 'GENERAL' && roomData.name !== room.name) { + await saveRoomSettings(startedByUserId, 'GENERAL', 'roomName', roomData.name); + } + + await this.updateRoomId(room._id, roomData); + } + + async insertRoom(roomData: IImportChannel, startedByUserId: string): Promise { + // Find the rocketchatId of the user who created this channel + const creatorId = await this.getRoomCreatorId(roomData, startedByUserId); + const members = await this._cache.convertImportedIdsToUsernames(roomData.users, roomData.t !== 'd' ? creatorId : undefined); + + if (roomData.t === 'd') { + if (members.length < roomData.users.length) { + this._logger.warn(`One or more imported users not found: ${roomData.users}`); + throw new Error('importer-channel-missing-users'); + } + } + + // Create the channel + try { + let roomInfo; + if (roomData.t === 'd') { + roomInfo = await createDirectMessage(members, startedByUserId, true); + } else { + if (!roomData.name) { + return; + } + if (roomData.t === 'p') { + const user = await Users.findOneById(creatorId); + if (!user) { + throw new Error('importer-channel-invalid-creator'); + } + roomInfo = await createPrivateGroupMethod(user, roomData.name, members, false, {}, {}); + } else { + roomInfo = await createChannelMethod(creatorId, roomData.name, members, false, {}, {}); + } + } + + roomData._id = roomInfo.rid; + } catch (e) { + this._logger.warn({ msg: 'Failed to create new room', name: roomData.name, members }); + this._logger.error(e); + throw e; + } + + await this.updateRoomId(roomData._id as 'string', roomData); + } + + async archiveRoomById(rid: string) { + const responses = await Promise.all([Rooms.archiveById(rid), Subscriptions.archiveByRoomId(rid)]); + + if (responses[1]?.modifiedCount) { + void notifyOnSubscriptionChangedByRoomId(rid); + } + } + + async updateRoomId(_id: string, roomData: IImportChannel): Promise { + const set = { + ts: roomData.ts, + topic: roomData.topic, + description: roomData.description, + }; + + const roomUpdate: { $set?: Record; $addToSet?: Record } = {}; + + if (Object.keys(set).length > 0) { + roomUpdate.$set = set; + } + + if (roomData.importIds.length) { + roomUpdate.$addToSet = { + importIds: { + $each: roomData.importIds, + }, + }; + } + + if (roomUpdate.$set || roomUpdate.$addToSet) { + await Rooms.updateOne({ _id: roomData._id }, roomUpdate); + } + } + + async getRoomCreatorId(roomData: IImportChannel, startedByUserId: string): Promise { + if (roomData.u) { + const creatorId = await this._cache.findImportedUserId(roomData.u._id); + if (creatorId) { + return creatorId; + } + + if (roomData.t !== 'd') { + return startedByUserId; + } + + throw new Error('importer-channel-invalid-creator'); + } + + if (roomData.t === 'd') { + for await (const member of roomData.users) { + const userId = await this._cache.findImportedUserId(member); + if (userId) { + return userId; + } + } + } + + throw new Error('importer-channel-invalid-creator'); + } + + protected getDataType(): 'channel' { + return 'channel'; + } +} diff --git a/apps/meteor/app/importer/server/classes/converters/UserConverter.ts b/apps/meteor/app/importer/server/classes/converters/UserConverter.ts new file mode 100644 index 000000000000..7401aea7c234 --- /dev/null +++ b/apps/meteor/app/importer/server/classes/converters/UserConverter.ts @@ -0,0 +1,419 @@ +import type { IImportUser, IImportUserRecord, IUser, IUserEmail } from '@rocket.chat/core-typings'; +import { Users } from '@rocket.chat/models'; +import { Random } from '@rocket.chat/random'; +import { SHA256 } from '@rocket.chat/sha256'; +import { hash as bcryptHash } from 'bcrypt'; +import { Accounts } from 'meteor/accounts-base'; + +import { callbacks as systemCallbacks } from '../../../../../lib/callbacks'; +import { addUserToDefaultChannels } from '../../../../lib/server/functions/addUserToDefaultChannels'; +import { generateUsernameSuggestion } from '../../../../lib/server/functions/getUsernameSuggestion'; +import { saveUserIdentity } from '../../../../lib/server/functions/saveUserIdentity'; +import { setUserActiveStatus } from '../../../../lib/server/functions/setUserActiveStatus'; +import { notifyOnUserChange } from '../../../../lib/server/lib/notifyListener'; +import type { IConversionCallbacks } from '../../definitions/IConversionCallbacks'; +import { RecordConverter, type RecordConverterOptions } from './RecordConverter'; + +export type UserConverterOptions = { + flagEmailsAsVerified?: boolean; + skipExistingUsers?: boolean; + skipNewUsers?: boolean; + skipUserCallbacks?: boolean; + skipDefaultChannels?: boolean; + + quickUserInsertion?: boolean; + enableEmail2fa?: boolean; +}; + +export type ConvertUsersResult = { + inserted: string[]; + updated: string[]; + skipped: number; + failed: number; +}; + +export class UserConverter extends RecordConverter { + private insertedIds = new Set(); + + private updatedIds = new Set(); + + protected async convertRecord(record: IImportUserRecord): Promise { + const { data, _id } = record; + + data.importIds = data.importIds.filter((item) => item); + + if (!data.emails.length && !data.username) { + throw new Error('importer-user-missing-email-and-username'); + } + + const existingUser = await this.findExistingUser(data); + if (existingUser && this._options.skipExistingUsers) { + await this.skipRecord(_id); + return; + } + if (!existingUser && this._options.skipNewUsers) { + await this.skipRecord(_id); + return; + } + + await this.insertOrUpdateUser(existingUser, data); + return !existingUser; + } + + async convertData(userCallbacks: IConversionCallbacks = {}): Promise { + this.insertedIds.clear(); + this.updatedIds.clear(); + + if (this._options.quickUserInsertion) { + await this.batchConversion(userCallbacks); + } else { + await super.convertData(userCallbacks); + } + + await systemCallbacks.run('afterUserImport', { + inserted: [...this.insertedIds], + updated: [...this.updatedIds], + skipped: this.skippedCount, + failed: this.failedCount, + }); + } + + public async batchConversion({ afterBatchFn, ...callbacks }: IConversionCallbacks = {}): Promise { + const batchToInsert = new Set(); + + await this.iterateRecords({ + ...callbacks, + processRecord: async (record: IImportUserRecord) => { + const { data } = record; + + data.importIds = data.importIds.filter((item) => item); + + if (!data.emails.length && !data.username) { + throw new Error('importer-user-missing-email-and-username'); + } + + batchToInsert.add(data); + + if (batchToInsert.size >= 50) { + const usersToInsert = await this.buildUserBatch([...batchToInsert]); + batchToInsert.clear(); + + const newIds = await this.insertUserBatch(usersToInsert, { afterBatchFn }); + newIds.forEach((id) => this.insertedIds.add(id)); + } + + return undefined; + }, + }); + + if (batchToInsert.size > 0) { + const usersToInsert = await this.buildUserBatch([...batchToInsert]); + const newIds = await this.insertUserBatch(usersToInsert, { afterBatchFn }); + newIds.forEach((id) => this.insertedIds.add(id)); + } + } + + private async insertUserBatch(users: IUser[], { afterBatchFn }: IConversionCallbacks): Promise { + let newIds: string[] | null = null; + + try { + newIds = Object.values((await Users.insertMany(users, { ordered: false })).insertedIds); + if (afterBatchFn) { + await afterBatchFn(newIds.length, 0); + } + } catch (e: any) { + newIds = (e.result?.result?.insertedIds || []) as string[]; + const errorCount = users.length - (e.result?.result?.nInserted || 0); + + if (afterBatchFn) { + await afterBatchFn(Math.min(newIds.length, users.length - errorCount), errorCount); + } + } + + return newIds; + } + + async findExistingUser(data: IImportUser): Promise { + if (data.emails.length) { + const emailUser = await Users.findOneByEmailAddress(data.emails[0], {}); + + if (emailUser) { + return emailUser; + } + } + + // If we couldn't find one by their email address, try to find an existing user by their username + if (data.username) { + return Users.findOneByUsernameIgnoringCase(data.username, {}); + } + } + + addUserImportId(updateData: Record, userData: IImportUser): void { + if (userData.importIds?.length) { + updateData.$addToSet = { + importIds: { + $each: userData.importIds, + }, + }; + } + } + + addUserEmails(updateData: Record, userData: IImportUser, existingEmails: Array): void { + if (!userData.emails?.length) { + return; + } + + const verifyEmails = Boolean(this._options.flagEmailsAsVerified); + const newEmailList: Array = []; + + for (const email of userData.emails) { + const verified = verifyEmails || existingEmails.find((ee) => ee.address === email)?.verified || false; + + newEmailList.push({ + address: email, + verified, + }); + } + + updateData.$set.emails = newEmailList; + } + + addUserServices(updateData: Record, userData: IImportUser): void { + if (!userData.services) { + return; + } + + for (const serviceKey in userData.services) { + if (!userData.services[serviceKey]) { + continue; + } + + const service = userData.services[serviceKey]; + + for (const key in service) { + if (!service[key]) { + continue; + } + + updateData.$set[`services.${serviceKey}.${key}`] = service[key]; + } + } + } + + addCustomFields(updateData: Record, userData: IImportUser): void { + if (!userData.customFields) { + return; + } + + const subset = (source: Record, currentPath: string): void => { + for (const key in source) { + if (!source.hasOwnProperty(key)) { + continue; + } + + const keyPath = `${currentPath}.${key}`; + if (typeof source[key] === 'object' && !Array.isArray(source[key])) { + subset(source[key], keyPath); + continue; + } + + updateData.$set = { + ...updateData.$set, + ...{ [keyPath]: source[key] }, + }; + } + }; + + subset(userData.customFields, 'customFields'); + } + + async insertOrUpdateUser(existingUser: IUser | undefined, data: IImportUser): Promise { + if (!data.username && !existingUser?.username) { + const emails = data.emails.filter(Boolean).map((email) => ({ address: email })); + data.username = await generateUsernameSuggestion({ + name: data.name, + emails, + }); + } + + if (existingUser) { + await this.updateUser(existingUser, data); + this.updatedIds.add(existingUser._id); + } else { + if (!data.name && data.username) { + data.name = this.guessNameFromUsername(data.username); + } + + const userId = await this.insertUser(data); + data._id = userId; + this.insertedIds.add(userId); + + if (!this._options.skipDefaultChannels) { + const insertedUser = await Users.findOneById(userId, {}); + if (!insertedUser) { + throw new Error(`User not found: ${userId}`); + } + + await addUserToDefaultChannels(insertedUser, true); + } + } + } + + async updateUser(existingUser: IUser, userData: IImportUser): Promise { + const { _id } = existingUser; + if (!_id) { + return; + } + + userData._id = _id; + + if (!userData.roles && !existingUser.roles) { + userData.roles = ['user']; + } + if (!userData.type && !existingUser.type) { + userData.type = 'user'; + } + + const updateData: Record = Object.assign(Object.create(null), { + $set: Object.assign(Object.create(null), { + ...(userData.roles && { roles: userData.roles }), + ...(userData.type && { type: userData.type }), + ...(userData.statusText && { statusText: userData.statusText }), + ...(userData.bio && { bio: userData.bio }), + ...(userData.services?.ldap && { ldap: true }), + ...(userData.avatarUrl && { _pendingAvatarUrl: userData.avatarUrl }), + }), + }); + + this.addCustomFields(updateData, userData); + this.addUserServices(updateData, userData); + this.addUserImportId(updateData, userData); + this.addUserEmails(updateData, userData, existingUser.emails || []); + + if (Object.keys(updateData.$set).length === 0) { + delete updateData.$set; + } + if (Object.keys(updateData).length > 0) { + await Users.updateOne({ _id }, updateData); + } + + if (userData.utcOffset) { + await Users.setUtcOffset(_id, userData.utcOffset); + } + + if (userData.name || userData.username) { + await saveUserIdentity({ _id, name: userData.name, username: userData.username } as Parameters[0]); + } + + if (userData.importIds.length) { + this._cache.addUser(userData.importIds[0], existingUser._id, existingUser.username || userData.username); + } + + // Deleted users are 'inactive' users in Rocket.Chat + if (userData.deleted && existingUser?.active) { + await setUserActiveStatus(_id, false, true); + } else if (userData.deleted === false && existingUser?.active === false) { + await setUserActiveStatus(_id, true); + } + + void notifyOnUserChange({ clientAction: 'updated', id: _id, diff: updateData.$set }); + } + + private async hashPassword(password: string): Promise { + return bcryptHash(SHA256(password), Accounts._bcryptRounds()); + } + + private generateTempPassword(userData: IImportUser): string { + return `${Date.now()}${userData.name || ''}${userData.emails.length ? userData.emails[0].toUpperCase() : ''}`; + } + + private async buildNewUserObject(userData: IImportUser): Promise> { + return { + type: userData.type || 'user', + ...(userData.username && { username: userData.username }), + ...(userData.emails.length && { + emails: userData.emails.map((email) => ({ address: email, verified: !!this._options.flagEmailsAsVerified })), + }), + ...(userData.statusText && { statusText: userData.statusText }), + ...(userData.name && { name: userData.name }), + ...(userData.bio && { bio: userData.bio }), + ...(userData.avatarUrl && { _pendingAvatarUrl: userData.avatarUrl }), + ...(userData.utcOffset !== undefined && { utcOffset: userData.utcOffset }), + ...{ + services: { + // Add a password service if there's a password string, or if there's no service at all + ...((!!userData.password || !userData.services || !Object.keys(userData.services).length) && { + password: { bcrypt: await this.hashPassword(userData.password || this.generateTempPassword(userData)) }, + }), + ...(userData.services || {}), + }, + }, + ...(userData.services?.ldap && { ldap: true }), + ...(userData.importIds?.length && { importIds: userData.importIds }), + ...(!!userData.customFields && { customFields: userData.customFields }), + ...(userData.deleted !== undefined && { active: !userData.deleted }), + }; + } + + private async buildUserBatch(usersData: IImportUser[]): Promise { + return Promise.all( + usersData.map(async (userData) => { + const user = await this.buildNewUserObject(userData); + return { + createdAt: new Date(), + _id: Random.id(), + + status: 'offline', + ...user, + roles: userData.roles?.length ? userData.roles : ['user'], + active: !userData.deleted, + services: { + ...user.services, + ...(this._options.enableEmail2fa + ? { + email2fa: { + enabled: true, + changedAt: new Date(), + }, + } + : {}), + }, + } as IUser; + }), + ); + } + + async insertUser(userData: IImportUser): Promise { + const user = await this.buildNewUserObject(userData); + + return Accounts.insertUserDoc( + { + joinDefaultChannels: false, + skipEmailValidation: true, + skipAdminCheck: true, + skipAdminEmail: true, + skipOnCreateUserCallback: this._options.skipUserCallbacks, + skipBeforeCreateUserCallback: this._options.skipUserCallbacks, + skipAfterCreateUserCallback: this._options.skipUserCallbacks, + skipDefaultAvatar: true, + skipAppsEngineEvent: !!process.env.IMPORTER_SKIP_APPS_EVENT, + }, + { + ...user, + ...(userData.roles?.length ? { globalRoles: userData.roles } : {}), + }, + ); + } + + protected guessNameFromUsername(username: string): string { + return username + .replace(/\W/g, ' ') + .replace(/\s(.)/g, (u) => u.toUpperCase()) + .replace(/^(.)/, (u) => u.toLowerCase()) + .replace(/^\w/, (u) => u.toUpperCase()); + } + + protected getDataType(): 'user' { + return 'user'; + } +} diff --git a/apps/meteor/ee/server/lib/ldap/Manager.ts b/apps/meteor/ee/server/lib/ldap/Manager.ts index eb96784c264f..61e0ba990082 100644 --- a/apps/meteor/ee/server/lib/ldap/Manager.ts +++ b/apps/meteor/ee/server/lib/ldap/Manager.ts @@ -15,9 +15,9 @@ import { settings } from '../../../../app/settings/server'; import { getValidRoomName } from '../../../../app/utils/server/lib/getValidRoomName'; import { ensureArray } from '../../../../lib/utils/arrayUtils'; import { LDAPConnection } from '../../../../server/lib/ldap/Connection'; -import { LDAPDataConverter } from '../../../../server/lib/ldap/DataConverter'; import { logger, searchLogger, mapLogger } from '../../../../server/lib/ldap/Logger'; import { LDAPManager } from '../../../../server/lib/ldap/Manager'; +import { LDAPUserConverter } from '../../../../server/lib/ldap/UserConverter'; import { syncUserRoles } from '../syncUserRoles'; import { copyCustomFieldsLDAP } from './copyCustomFieldsLDAP'; @@ -37,7 +37,7 @@ export class LDAPEEManager extends LDAPManager { options.skipNewUsers = !createNewUsers; const ldap = new LDAPConnection(); - const converter = new LDAPDataConverter(true, options); + const converter = new LDAPUserConverter(options); const touchedUsers = new Set(); try { @@ -53,7 +53,7 @@ export class LDAPEEManager extends LDAPManager { const membersOfGroupFilter = await ldap.searchMembersOfGroupFilter(); - await converter.convertUsers({ + await converter.convertData({ beforeImportFn: (async ({ options }: IImportRecord): Promise => { if (!ldap.options.groupFilterEnabled || !ldap.options.groupFilterGroupMemberFormat) { return true; @@ -156,7 +156,7 @@ export class LDAPEEManager extends LDAPManager { private static async advancedSync( ldap: LDAPConnection, importUser: IImportUser, - converter: LDAPDataConverter, + converter: LDAPUserConverter, isNewRecord: boolean, ): Promise { const user = await converter.findExistingUser(importUser); @@ -581,7 +581,7 @@ export class LDAPEEManager extends LDAPManager { ); } - private static async importNewUsers(ldap: LDAPConnection, converter: LDAPDataConverter): Promise { + private static async importNewUsers(ldap: LDAPConnection, converter: LDAPUserConverter): Promise { return new Promise((resolve, reject) => { let count = 0; @@ -591,7 +591,7 @@ export class LDAPEEManager extends LDAPManager { count++; const userData = this.mapUserData(data); - converter.addUserSync(userData, { dn: data.dn, username: this.getLdapUsername(data) }); + converter.addObjectToMemory(userData, { dn: data.dn, username: this.getLdapUsername(data) }); return userData; }, endCallback: (error: any): void => { @@ -608,14 +608,14 @@ export class LDAPEEManager extends LDAPManager { }); } - private static async updateExistingUsers(ldap: LDAPConnection, converter: LDAPDataConverter, disableMissingUsers = false): Promise { + private static async updateExistingUsers(ldap: LDAPConnection, converter: LDAPUserConverter, disableMissingUsers = false): Promise { const users = await Users.findLDAPUsers().toArray(); for await (const user of users) { const ldapUser = await this.findLDAPUser(ldap, user); if (ldapUser) { const userData = this.mapUserData(ldapUser, user.username); - converter.addUserSync(userData, { dn: ldapUser.dn, username: this.getLdapUsername(ldapUser) }); + converter.addObjectToMemory(userData, { dn: ldapUser.dn, username: this.getLdapUsername(ldapUser) }); } else if (disableMissingUsers) { await setUserActiveStatus(user._id, false, true); } diff --git a/apps/meteor/server/lib/ldap/Manager.ts b/apps/meteor/server/lib/ldap/Manager.ts index f0efcc04539d..b53146733571 100644 --- a/apps/meteor/server/lib/ldap/Manager.ts +++ b/apps/meteor/server/lib/ldap/Manager.ts @@ -8,14 +8,14 @@ import { Accounts } from 'meteor/accounts-base'; import { Meteor } from 'meteor/meteor'; import _ from 'underscore'; -import type { IConverterOptions } from '../../../app/importer/server/classes/ImportDataConverter'; +import type { UserConverterOptions } from '../../../app/importer/server/classes/converters/UserConverter'; import { setUserAvatar } from '../../../app/lib/server/functions/setUserAvatar'; import { settings } from '../../../app/settings/server'; import { callbacks } from '../../../lib/callbacks'; import { omit } from '../../../lib/utils/omit'; import { LDAPConnection } from './Connection'; -import { LDAPDataConverter } from './DataConverter'; import { logger, authLogger, connLogger } from './Logger'; +import { LDAPUserConverter } from './UserConverter'; import { getLDAPConditionalSetting } from './getLDAPConditionalSetting'; export class LDAPManager { @@ -149,7 +149,7 @@ export class LDAPManager { } } - protected static getConverterOptions(): IConverterOptions { + protected static getConverterOptions(): UserConverterOptions { return { flagEmailsAsVerified: settings.get('Accounts_Verify_Email_For_External_Accounts') ?? false, skipExistingUsers: false, @@ -360,7 +360,7 @@ export class LDAPManager { } const options = this.getConverterOptions(); - await LDAPDataConverter.convertSingleUser(userData, options); + await LDAPUserConverter.convertSingleUser(userData, options); return existingUser || this.findExistingLDAPUser(ldapUser); } diff --git a/apps/meteor/server/lib/ldap/DataConverter.ts b/apps/meteor/server/lib/ldap/UserConverter.ts similarity index 53% rename from apps/meteor/server/lib/ldap/DataConverter.ts rename to apps/meteor/server/lib/ldap/UserConverter.ts index 70f1f4451a50..1d94db88db3c 100644 --- a/apps/meteor/server/lib/ldap/DataConverter.ts +++ b/apps/meteor/server/lib/ldap/UserConverter.ts @@ -1,20 +1,22 @@ import type { IImportUser, IUser } from '@rocket.chat/core-typings'; -import { Logger } from '@rocket.chat/logger'; +import type { Logger } from '@rocket.chat/logger'; import { Users } from '@rocket.chat/models'; -import type { IConverterOptions } from '../../../app/importer/server/classes/ImportDataConverter'; -import { VirtualDataConverter } from '../../../app/importer/server/classes/VirtualDataConverter'; +import type { ConverterCache } from '../../../app/importer/server/classes/converters/ConverterCache'; +import { type RecordConverterOptions } from '../../../app/importer/server/classes/converters/RecordConverter'; +import { UserConverter, type UserConverterOptions } from '../../../app/importer/server/classes/converters/UserConverter'; import { settings } from '../../../app/settings/server'; -const logger = new Logger('LDAP Data Converter'); - -export class LDAPDataConverter extends VirtualDataConverter { +export class LDAPUserConverter extends UserConverter { private mergeExistingUsers: boolean; - constructor(virtual = true, options?: IConverterOptions) { - super(virtual, options); - this.setLogger(logger); + constructor(options?: UserConverterOptions & RecordConverterOptions, logger?: Logger, cache?: ConverterCache) { + const ldapOptions = { + workInMemory: true, + ...(options || {}), + }; + super(ldapOptions, logger, cache); this.mergeExistingUsers = settings.get('LDAP_Merge_Existing_Users') ?? true; } @@ -43,9 +45,9 @@ export class LDAPDataConverter extends VirtualDataConverter { } } - static async convertSingleUser(userData: IImportUser, options?: IConverterOptions): Promise { - const converter = new LDAPDataConverter(true, options); - await converter.addUser(userData); - await converter.convertUsers(); + static async convertSingleUser(userData: IImportUser, options?: UserConverterOptions): Promise { + const converter = new LDAPUserConverter(options); + await converter.addObject(userData); + await converter.convertData(); } } diff --git a/apps/meteor/tests/unit/app/importer/server/messageConverter.spec.ts b/apps/meteor/tests/unit/app/importer/server/messageConverter.spec.ts new file mode 100644 index 000000000000..dcf72bb9b50d --- /dev/null +++ b/apps/meteor/tests/unit/app/importer/server/messageConverter.spec.ts @@ -0,0 +1,203 @@ +import { expect } from 'chai'; +import proxyquire from 'proxyquire'; +import sinon from 'sinon'; + +const settingsStub = sinon.stub(); +const modelsMock = { + Rooms: { + findOneByImportId: sinon.stub(), + }, +}; +const insertMessage = sinon.stub(); + +const { MessageConverter } = proxyquire.noCallThru().load('../../../../../app/importer/server/classes/converters/MessageConverter', { + '../../../settings/server': { + settings: { get: settingsStub }, + }, + '../../../../lib/server/functions/insertMessage': { + insertMessage, + }, + 'meteor/check': sinon.stub(), + 'meteor/meteor': sinon.stub(), + '@rocket.chat/models': { ...modelsMock, '@global': true }, +}); + +describe('Message Converter', () => { + beforeEach(() => { + modelsMock.Rooms.findOneByImportId.reset(); + insertMessage.reset(); + settingsStub.reset(); + }); + + const messageToImport = { + ts: Date.now(), + u: { + _id: 'rocket.cat', + }, + rid: 'general', + msg: 'testing', + }; + + describe('[insertMessage]', () => { + it('function should be called by the converter', async () => { + const converter = new MessageConverter({ workInMemory: true }); + converter._cache.addRoom('general', 'general'); + + sinon.stub(converter, 'insertMessage'); + sinon.stub(converter, 'resetLastMessages'); + + await converter.addObject(messageToImport); + await converter.convertData(); + + expect(converter.insertMessage.getCalls()).to.be.an('array').with.lengthOf(1); + expect(converter.insertMessage.getCall(0).args).to.be.an('array').that.is.not.empty; + expect(converter.insertMessage.getCall(0).args[0]).to.be.deep.equal(messageToImport); + }); + + it('should call insertMessage lib function to save the message', async () => { + const converter = new MessageConverter({ workInMemory: true }); + converter._cache.addRoom('general', 'main'); + + await (converter as any).insertMessage(messageToImport); + + expect(insertMessage.getCalls()).to.be.an('array').with.lengthOf(1); + expect(insertMessage.getCall(0).args).to.be.an('array').with.lengthOf(4); + expect(insertMessage.getCall(0).args[0]).to.be.deep.equal({ + _id: 'rocket.cat', + username: 'rocket.cat', + }); + expect(insertMessage.getCall(0).args[1]).to.deep.include({ + ts: messageToImport.ts, + msg: messageToImport.msg, + rid: 'main', + }); + }); + }); + + describe('[buildMessageObject]', () => { + it('should have the basic info', async () => { + const converter = new MessageConverter({ workInMemory: true }); + + const converted = await converter.buildMessageObject(messageToImport, 'general', { _id: 'rocket.cat', username: 'rocket.cat' }); + + expect(converted) + .to.be.an('object') + .that.deep.includes({ + ts: messageToImport.ts, + msg: messageToImport.msg, + u: { + _id: 'rocket.cat', + username: 'rocket.cat', + }, + }); + }); + + // #TODO: Validate all message attributes + }); + + describe('callbacks', () => { + it('beforeImportFn should be triggered', async () => { + const beforeImportFn = sinon.stub(); + const converter = new MessageConverter({ workInMemory: true }); + converter._cache.addRoom('general', 'general'); + + sinon.stub(converter, 'insertMessage'); + sinon.stub(converter, 'resetLastMessages'); + + await converter.addObject(messageToImport); + await converter.convertData({ + beforeImportFn, + }); + + expect(beforeImportFn.getCalls()).to.be.an('array').with.lengthOf(1); + }); + + it('afterImportFn should be triggered', async () => { + const afterImportFn = sinon.stub(); + const converter = new MessageConverter({ workInMemory: true }); + converter._cache.addRoom('general', 'general'); + + sinon.stub(converter, 'insertMessage'); + sinon.stub(converter, 'resetLastMessages'); + + await converter.addObject(messageToImport); + await converter.convertData({ + afterImportFn, + }); + + expect(converter.insertMessage.getCalls()).to.be.an('array').with.lengthOf(1); + expect(afterImportFn.getCalls()).to.be.an('array').with.lengthOf(1); + }); + + it('should skip record if beforeImportFn returns false', async () => { + let recordId = null; + const beforeImportFn = sinon.stub(); + const afterImportFn = sinon.stub(); + + beforeImportFn.callsFake((record) => { + recordId = record._id; + return false; + }); + + const converter = new MessageConverter({ workInMemory: true }); + converter._cache.addRoom('general', 'general'); + + sinon.stub(converter, 'insertMessage'); + sinon.stub(converter, 'resetLastMessages'); + sinon.stub(converter, 'skipRecord'); + + await converter.addObject(messageToImport); + await converter.convertData({ + beforeImportFn, + afterImportFn, + }); + + expect(beforeImportFn.getCalls()).to.be.an('array').with.lengthOf(1); + expect(afterImportFn.getCalls()).to.be.an('array').with.lengthOf(0); + expect(converter.skipRecord.getCalls()).to.be.an('array').with.lengthOf(1); + expect(converter.skipRecord.getCall(0).args).to.be.an('array').that.is.deep.equal([recordId]); + expect(converter.insertMessage.getCalls()).to.be.an('array').with.lengthOf(0); + }); + + it('should not skip record if beforeImportFn returns true', async () => { + const beforeImportFn = sinon.stub(); + const afterImportFn = sinon.stub(); + + beforeImportFn.callsFake(() => true); + + const converter = new MessageConverter({ workInMemory: true }); + converter._cache.addRoom('general', 'general'); + + sinon.stub(converter, 'insertMessage'); + sinon.stub(converter, 'resetLastMessages'); + sinon.stub(converter, 'skipRecord'); + + await converter.addObject(messageToImport); + await converter.convertData({ + beforeImportFn, + afterImportFn, + }); + + expect(beforeImportFn.getCalls()).to.be.an('array').with.lengthOf(1); + expect(converter.skipRecord.getCalls()).to.be.an('array').with.lengthOf(0); + expect(converter.insertMessage.getCalls()).to.be.an('array').with.lengthOf(1); + expect(afterImportFn.getCalls()).to.be.an('array').with.lengthOf(1); + }); + + it('onErrorFn should be triggered if mandatory attributes are missing', async () => { + const converter = new MessageConverter({ workInMemory: true }); + converter._cache.addRoom('general', 'general'); + sinon.stub(converter, 'resetLastMessages'); + + const onErrorFn = sinon.stub(); + + sinon.stub(converter, 'saveError'); + + await converter.addObject({}); + await converter.convertData({ onErrorFn }); + + expect(onErrorFn.getCall(0)).to.not.be.null; + expect(converter.saveError.getCall(0)).to.not.be.null; + }); + }); +}); diff --git a/apps/meteor/tests/unit/app/importer/server/recordConverter.spec.ts b/apps/meteor/tests/unit/app/importer/server/recordConverter.spec.ts new file mode 100644 index 000000000000..71ebb277f3dd --- /dev/null +++ b/apps/meteor/tests/unit/app/importer/server/recordConverter.spec.ts @@ -0,0 +1,137 @@ +import type { IImportRecord, IImportRecordType } from '@rocket.chat/core-typings'; +import { expect } from 'chai'; +import proxyquire from 'proxyquire'; +import sinon from 'sinon'; + +const settingsStub = sinon.stub(); +const modelsMock = { + ImportData: { + find: sinon.stub(), + updateOne: sinon.stub(), + col: { + insertOne: sinon.stub(), + }, + }, +}; + +const { RecordConverter } = proxyquire.noCallThru().load('../../../../../app/importer/server/classes/converters/RecordConverter', { + '../../../settings/server': { + settings: { get: settingsStub }, + }, + 'meteor/check': sinon.stub(), + 'meteor/meteor': sinon.stub(), + '@rocket.chat/models': { ...modelsMock, '@global': true }, +}); + +class TestConverter extends RecordConverter { + constructor(workInMemory = true) { + super({ workInMemory }); + } + + protected getDataType(): IImportRecordType { + return 'user'; + } +} + +describe('Record Converter', () => { + const userToImport = { + name: 'user1', + emails: ['user1@domain.com'], + importIds: ['importId1'], + username: 'username1', + }; + + describe('Working with Mongo Collection', () => { + beforeEach(() => { + modelsMock.ImportData.col.insertOne.reset(); + modelsMock.ImportData.find.reset(); + modelsMock.ImportData.updateOne.reset(); + modelsMock.ImportData.find.callsFake(() => ({ toArray: () => [] })); + }); + + describe('Adding and Retrieving users', () => { + it('should store objects in the collection', async () => { + const converter = new TestConverter(false); + + await converter.addObject(userToImport); + expect(modelsMock.ImportData.col.insertOne.getCall(0)).to.not.be.null; + }); + + it('should read objects from the collection', async () => { + const converter = new TestConverter(false); + await converter.addObject(userToImport); + + await converter.getDataToImport(); + + expect(modelsMock.ImportData.find.getCall(0)).to.not.be.null; + }); + + it('should flag skipped records on the document', async () => { + const converter = new TestConverter(false); + await (converter as any).skipRecord('skippedId'); + + expect(modelsMock.ImportData.updateOne.getCall(0)).to.not.be.null; + expect(modelsMock.ImportData.updateOne.getCall(0).args).to.be.an('array').that.deep.contains({ _id: 'skippedId' }); + }); + + it('should store error information on the document', async () => { + const converter = new TestConverter(false); + await (converter as any).saveError('errorId', new Error()); + + expect(modelsMock.ImportData.updateOne.getCall(0)).to.not.be.null; + expect(modelsMock.ImportData.updateOne.getCall(0).args).to.be.an('array').that.deep.contains({ _id: 'errorId' }); + }); + }); + }); + + describe('Working in Memory', () => { + beforeEach(() => { + modelsMock.ImportData.col.insertOne.reset(); + modelsMock.ImportData.updateOne.reset(); + modelsMock.ImportData.find.reset(); + settingsStub.reset(); + }); + + describe('Adding and Retrieving users', () => { + it('should not store objects in the collection', async () => { + const converter = new TestConverter(true); + + await converter.addObject(userToImport); + expect(modelsMock.ImportData.col.insertOne.getCall(0)).to.be.null; + }); + + it('should not try to read objects from the collection', async () => { + const converter = new TestConverter(true); + await converter.addObject(userToImport); + + await converter.getDataToImport(); + + expect(modelsMock.ImportData.find.getCall(0)).to.be.null; + }); + + it('should properly retrieve the data added to memory', async () => { + const converter = new TestConverter(true); + + await converter.addObject(userToImport); + const dataToImport = await converter.getDataToImport(); + + expect(dataToImport.length).to.be.equal(1); + expect(dataToImport[0].data).to.be.equal(userToImport); + }); + + it('should not access the collection when flagging skipped records', async () => { + const converter = new TestConverter(true); + await (converter as any).skipRecord('skippedId'); + + expect(modelsMock.ImportData.updateOne.getCall(0)).to.be.null; + }); + + it('should not access the collection when storing error information', async () => { + const converter = new TestConverter(true); + await (converter as any).saveError('errorId', new Error()); + + expect(modelsMock.ImportData.updateOne.getCall(0)).to.be.null; + }); + }); + }); +}); diff --git a/apps/meteor/tests/unit/app/importer/server/roomConverter.spec.ts b/apps/meteor/tests/unit/app/importer/server/roomConverter.spec.ts new file mode 100644 index 000000000000..64502ed92a8f --- /dev/null +++ b/apps/meteor/tests/unit/app/importer/server/roomConverter.spec.ts @@ -0,0 +1,272 @@ +import { expect } from 'chai'; +import proxyquire from 'proxyquire'; +import sinon from 'sinon'; + +const settingsStub = sinon.stub(); +const modelsMock = { + Rooms: { + archiveById: sinon.stub(), + updateOne: sinon.stub(), + findOneById: sinon.stub(), + findDirectRoomContainingAllUsernames: sinon.stub(), + findOneByNonValidatedName: sinon.stub(), + }, + Subscriptions: { + archiveByRoomId: sinon.stub(), + }, +}; +const createDirectMessage = sinon.stub(); +const saveRoomSettings = sinon.stub(); + +const { RoomConverter } = proxyquire.noCallThru().load('../../../../../app/importer/server/classes/converters/RoomConverter', { + '../../../settings/server': { + settings: { get: settingsStub }, + }, + '../../../../../server/methods/createDirectMessage': { + createDirectMessage, + }, + '../../../../channel-settings/server/methods/saveRoomSettings': { + saveRoomSettings, + }, + '../../../../lib/server/lib/notifyListener': { + notifyOnSubscriptionChangedByRoomId: sinon.stub(), + }, + '../../../../lib/server/methods/createChannel': { + createChannelMethod: sinon.stub(), + }, + '../../../../lib/server/methods/createPrivateGroup': { + createPrivateGroupMethod: sinon.stub(), + }, + 'meteor/check': sinon.stub(), + 'meteor/meteor': sinon.stub(), + '@rocket.chat/models': { ...modelsMock, '@global': true }, +}); + +describe('Room Converter', () => { + beforeEach(() => { + modelsMock.Rooms.archiveById.reset(); + modelsMock.Rooms.updateOne.reset(); + modelsMock.Rooms.findOneById.reset(); + modelsMock.Rooms.findDirectRoomContainingAllUsernames.reset(); + modelsMock.Rooms.findOneByNonValidatedName.reset(); + modelsMock.Subscriptions.archiveByRoomId.reset(); + createDirectMessage.reset(); + saveRoomSettings.reset(); + settingsStub.reset(); + }); + + const roomToImport = { + name: 'room1', + importIds: ['importIdRoom1'], + }; + + describe('[findExistingRoom]', () => { + it('function should be called by the converter', async () => { + const converter = new RoomConverter({ workInMemory: true }); + + sinon.stub(converter, 'findExistingRoom'); + sinon.stub(converter, 'insertOrUpdateRoom'); + + await converter.addObject(roomToImport); + await converter.convertChannels('startedByUserId'); + + expect(converter.findExistingRoom.getCall(0)).to.not.be.null; + }); + + it('should search by name', async () => { + const converter = new RoomConverter({ workInMemory: true }); + + await converter.findExistingRoom(roomToImport); + expect(modelsMock.Rooms.findOneByNonValidatedName.getCalls()).to.be.an('array').with.lengthOf(1); + expect(modelsMock.Rooms.findOneByNonValidatedName.getCall(0).args).to.be.an('array').that.contains('room1'); + }); + + it('should not search by name if there is none', async () => { + const converter = new RoomConverter({ workInMemory: true }); + + await converter.findExistingRoom({}); + expect(modelsMock.Rooms.findOneByNonValidatedName.getCalls()).to.be.an('array').with.lengthOf(0); + }); + + it('should search DMs by usernames', async () => { + const converter = new RoomConverter({ workInMemory: true }); + converter._cache.addUser('importId1', 'userId1', 'username1'); + converter._cache.addUser('importId2', 'userId2', 'username2'); + + await converter.findExistingRoom({ + t: 'd', + users: ['importId1', 'importId2'], + importIds: ['importIdRoom1'], + }); + + expect(modelsMock.Rooms.findDirectRoomContainingAllUsernames.getCalls()).to.be.an('array').with.lengthOf(1); + expect(modelsMock.Rooms.findDirectRoomContainingAllUsernames.getCall(0).args) + .to.be.an('array') + .that.deep.includes(['username1', 'username2']); + }); + }); + + describe('[insertRoom]', () => { + it('function should be called by the converter', async () => { + const converter = new RoomConverter({ workInMemory: true }); + + sinon.stub(converter, 'findExistingRoom'); + sinon.stub(converter, 'insertRoom'); + sinon.stub(converter, 'updateRoom'); + + await converter.addObject(roomToImport); + await converter.convertChannels('startedByUserId'); + + expect(converter.updateRoom.getCalls()).to.be.an('array').with.lengthOf(0); + expect(converter.insertRoom.getCalls()).to.be.an('array').with.lengthOf(1); + expect(converter.insertRoom.getCall(0).args).to.be.an('array').that.is.not.empty; + expect(converter.insertRoom.getCall(0).args[0]).to.be.deep.equal(roomToImport); + }); + + it('function should not be called for existing rooms', async () => { + const converter = new RoomConverter({ workInMemory: true }); + + sinon.stub(converter, 'findExistingRoom'); + converter.findExistingRoom.returns({ _id: 'oldId' }); + sinon.stub(converter, 'insertRoom'); + sinon.stub(converter, 'updateRoom'); + + await converter.addObject(roomToImport); + await converter.convertChannels('startedByUserId'); + + expect(converter.insertRoom.getCall(0)).to.be.null; + }); + + it('should call createDirectMessage to create DM rooms', async () => { + const converter = new RoomConverter({ workInMemory: true }); + sinon.stub(converter, 'updateRoomId'); + + createDirectMessage.callsFake((_options, data) => { + return { + ...data, + _id: 'Id1', + }; + }); + + converter._cache.addUser('importId1', 'userId1', 'username1'); + converter._cache.addUser('importId2', 'userId2', 'username2'); + + await (converter as any).insertRoom( + { + t: 'd', + users: ['importId1', 'importId2'], + importIds: ['importIdRoom1'], + }, + 'startedByUserId', + ); + + expect(createDirectMessage.getCalls()).to.be.an('array').with.lengthOf(1); + expect(createDirectMessage.getCall(0).args).to.be.an('array').with.lengthOf(3).that.deep.includes(['username1', 'username2']); + }); + + // #TODO: Validate all room types + }); + + describe('callbacks', () => { + it('beforeImportFn should be triggered', async () => { + const beforeImportFn = sinon.stub(); + const converter = new RoomConverter({ workInMemory: true }); + + sinon.stub(converter, 'findExistingRoom'); + sinon.stub(converter, 'insertOrUpdateRoom'); + + await converter.addObject(roomToImport); + await converter.convertChannels('startedByUserId', { + beforeImportFn, + }); + + expect(beforeImportFn.getCalls()).to.be.an('array').with.lengthOf(1); + }); + + it('afterImportFn should be triggered', async () => { + const afterImportFn = sinon.stub(); + const converter = new RoomConverter({ workInMemory: true }); + + sinon.stub(converter, 'findExistingRoom'); + sinon.stub(converter, 'insertOrUpdateRoom'); + + await converter.addObject(roomToImport); + await converter.convertChannels('startedByUserId', { + afterImportFn, + }); + + expect(converter.insertOrUpdateRoom.getCalls()).to.be.an('array').with.lengthOf(1); + expect(afterImportFn.getCalls()).to.be.an('array').with.lengthOf(1); + }); + + it('should skip record if beforeImportFn returns false', async () => { + let recordId = null; + const beforeImportFn = sinon.stub(); + const afterImportFn = sinon.stub(); + + beforeImportFn.callsFake((record) => { + recordId = record._id; + return false; + }); + + const converter = new RoomConverter({ workInMemory: true }); + + sinon.stub(converter, 'findExistingRoom'); + sinon.stub(converter, 'insertOrUpdateRoom'); + sinon.stub(converter, 'skipRecord'); + + await converter.addObject(roomToImport); + await converter.convertChannels('startedByUserId', { + beforeImportFn, + afterImportFn, + }); + + expect(beforeImportFn.getCalls()).to.be.an('array').with.lengthOf(1); + expect(afterImportFn.getCalls()).to.be.an('array').with.lengthOf(0); + expect(converter.skipRecord.getCalls()).to.be.an('array').with.lengthOf(1); + expect(converter.skipRecord.getCall(0).args).to.be.an('array').that.is.deep.equal([recordId]); + expect(converter.insertOrUpdateRoom.getCalls()).to.be.an('array').with.lengthOf(0); + }); + + it('should not skip record if beforeImportFn returns true', async () => { + const beforeImportFn = sinon.stub(); + const afterImportFn = sinon.stub(); + + beforeImportFn.callsFake(() => true); + + const converter = new RoomConverter({ workInMemory: true }); + + sinon.stub(converter, 'findExistingRoom'); + sinon.stub(converter, 'insertOrUpdateRoom'); + sinon.stub(converter, 'skipRecord'); + + await converter.addObject(roomToImport); + await converter.convertChannels('startedByUserId', { + beforeImportFn, + afterImportFn, + }); + + expect(beforeImportFn.getCalls()).to.be.an('array').with.lengthOf(1); + expect(converter.skipRecord.getCalls()).to.be.an('array').with.lengthOf(0); + expect(converter.insertOrUpdateRoom.getCalls()).to.be.an('array').with.lengthOf(1); + expect(afterImportFn.getCalls()).to.be.an('array').with.lengthOf(1); + }); + + it('onErrorFn should be triggered if there is no name and is not a DM', async () => { + const converter = new RoomConverter({ workInMemory: true }); + + const onErrorFn = sinon.stub(); + + sinon.stub(converter, 'findExistingRoom'); + sinon.stub(converter, 'insertOrUpdateRoom'); + sinon.stub(converter, 'saveError'); + + await converter.addObject({}); + await converter.convertChannels('startedByUserId', { onErrorFn }); + + expect(converter.insertOrUpdateRoom.getCall(0)).to.be.null; + expect(onErrorFn.getCall(0)).to.not.be.null; + expect(converter.saveError.getCall(0)).to.not.be.null; + }); + }); +}); diff --git a/apps/meteor/tests/unit/app/importer/server/userConverter.spec.ts b/apps/meteor/tests/unit/app/importer/server/userConverter.spec.ts new file mode 100644 index 000000000000..3dd5a8a5e3c9 --- /dev/null +++ b/apps/meteor/tests/unit/app/importer/server/userConverter.spec.ts @@ -0,0 +1,612 @@ +import { expect } from 'chai'; +import proxyquire from 'proxyquire'; +import sinon from 'sinon'; + +const settingsStub = sinon.stub(); +const modelsMock = { + Users: { + findOneByEmailAddress: sinon.stub(), + findOneByUsernameIgnoringCase: sinon.stub(), + findOneById: sinon.stub(), + }, +}; +const addUserToDefaultChannels = sinon.stub(); +const generateUsernameSuggestion = sinon.stub(); +const insertUserDoc = sinon.stub(); +const callbacks = { + run: sinon.stub(), +}; + +const { UserConverter } = proxyquire.noCallThru().load('../../../../../app/importer/server/classes/converters/UserConverter', { + '../../../../../lib/callbacks': { + callbacks, + }, + '../../../settings/server': { + settings: { get: settingsStub }, + }, + '../../../../lib/server/functions/addUserToDefaultChannels': { + addUserToDefaultChannels, + }, + '../../../../lib/server/functions/getUsernameSuggestion': { + generateUsernameSuggestion, + }, + '../../../../lib/server/functions/saveUserIdentity': { + saveUserIdentity: sinon.stub(), + }, + '../../../../lib/server/functions/setUserActiveStatus': { + setUserActiveStatus: sinon.stub(), + }, + '../../../../lib/server/lib/notifyListener': { + notifyOnUserChange: sinon.stub(), + }, + 'meteor/check': sinon.stub(), + 'meteor/meteor': sinon.stub(), + 'meteor/accounts-base': { + Accounts: { + insertUserDoc, + _bcryptRounds: () => 10, + }, + }, + '@rocket.chat/models': { ...modelsMock, '@global': true }, +}); + +describe('User Converter', () => { + beforeEach(() => { + modelsMock.Users.findOneByEmailAddress.reset(); + modelsMock.Users.findOneByUsernameIgnoringCase.reset(); + modelsMock.Users.findOneById.reset(); + callbacks.run.reset(); + insertUserDoc.reset(); + addUserToDefaultChannels.reset(); + generateUsernameSuggestion.reset(); + settingsStub.reset(); + }); + + const userToImport = { + name: 'user1', + emails: ['user1@domain.com'], + importIds: ['importId1'], + username: 'username1', + }; + + describe('[findExistingUser]', () => { + it('function should be called by the converter', async () => { + const converter = new UserConverter({ workInMemory: true }); + const findExistingUser = sinon.stub(converter, 'findExistingUser'); + + findExistingUser.throws(); + + await converter.addObject(userToImport); + await converter.convertData(); + + expect(findExistingUser.getCall(0)).to.not.be.null; + expect(callbacks.run.getCall(0)).to.not.be.null; + expect(callbacks.run.getCall(0).args).to.be.deep.equal(['afterUserImport', { inserted: [], updated: [], skipped: 0, failed: 1 }]); + }); + + it('should search by email address', async () => { + const converter = new UserConverter({ workInMemory: true }); + + await converter.findExistingUser(userToImport); + expect(modelsMock.Users.findOneByEmailAddress.getCalls()).to.be.an('array').with.lengthOf(1); + expect(modelsMock.Users.findOneByEmailAddress.getCall(0).args).to.be.an('array').that.contains('user1@domain.com'); + }); + + it('should search by username', async () => { + const converter = new UserConverter({ workInMemory: true }); + + await converter.findExistingUser(userToImport); + expect(modelsMock.Users.findOneByUsernameIgnoringCase.getCalls()).to.be.an('array').with.lengthOf(1); + expect(modelsMock.Users.findOneByUsernameIgnoringCase.getCall(0).args).to.be.an('array').that.contains('username1'); + }); + + it('should not search by username if an user is found by email', async () => { + const converter = new UserConverter({ workInMemory: true }); + + modelsMock.Users.findOneByEmailAddress.resolves(userToImport); + + await converter.findExistingUser(userToImport); + expect(modelsMock.Users.findOneByUsernameIgnoringCase.getCall(0)).to.be.null; + }); + }); + + describe('[buildNewUserObject]', () => { + const mappedUser = (expectedData: Record) => ({ + type: 'user', + services: { + password: { + bcrypt: 'hashed=tempPassword', + }, + }, + ...expectedData, + }); + + const converter = new UserConverter({ workInMemory: true }); + sinon.stub(converter, 'generateTempPassword'); + sinon.stub(converter, 'hashPassword'); + converter.generateTempPassword.returns('tempPassword'); + converter.hashPassword.callsFake((pass: string) => `hashed=${pass}`); + + it('should map an empty object', async () => { + expect( + await (converter as any).buildNewUserObject({ + emails: [], + importIds: [], + }), + ).to.be.deep.equal(mappedUser({})); + }); + + it('should map the name and username', async () => { + expect( + await (converter as any).buildNewUserObject({ + emails: [], + importIds: [], + name: 'name1', + username: 'username1', + }), + ).to.be.deep.equal( + mappedUser({ + username: 'username1', + name: 'name1', + }), + ); + }); + + it('should map optional fields', async () => { + expect( + await (converter as any).buildNewUserObject({ + emails: [], + importIds: [], + statusText: 'statusText1', + bio: 'bio1', + avatarUrl: 'avatarUrl', + utcOffset: 3, + }), + ).to.be.deep.equal( + mappedUser({ + statusText: 'statusText1', + bio: 'bio1', + _pendingAvatarUrl: 'avatarUrl', + utcOffset: 3, + }), + ); + }); + + it('should map custom fields', async () => { + expect( + await (converter as any).buildNewUserObject({ + emails: [], + importIds: [], + customFields: { + age: 32, + nickname: 'stitch', + }, + }), + ).to.be.deep.equal( + mappedUser({ + customFields: { + age: 32, + nickname: 'stitch', + }, + }), + ); + }); + + it('should not map roles', async () => { + expect( + await (converter as any).buildNewUserObject({ + emails: [], + importIds: [], + roles: ['role1'], + }), + ).to.be.deep.equal(mappedUser({})); + }); + + it('should map identifiers', async () => { + expect( + await (converter as any).buildNewUserObject({ + name: 'user1', + emails: ['user1@domain.com'], + importIds: ['importId1'], + username: 'username1', + }), + ).to.be.deep.equal( + mappedUser({ + username: 'username1', + name: 'user1', + importIds: ['importId1'], + emails: [{ address: 'user1@domain.com', verified: false }], + }), + ); + }); + + it('should map password', async () => { + expect( + await (converter as any).buildNewUserObject({ + emails: [], + importIds: [], + password: 'batata', + }), + ).to.be.deep.equal( + mappedUser({ + services: { + password: { + bcrypt: 'hashed=batata', + }, + }, + }), + ); + }); + + it('should map ldap service data', async () => { + expect( + await (converter as any).buildNewUserObject({ + emails: [], + importIds: [], + services: { + ldap: { + id: 'id', + }, + }, + }), + ).to.be.deep.equal( + mappedUser({ + services: { + ldap: { + id: 'id', + }, + }, + ldap: true, + }), + ); + }); + + it('should map deleted users', async () => { + expect( + await (converter as any).buildNewUserObject({ + emails: [], + importIds: [], + deleted: true, + }), + ).to.be.deep.equal( + mappedUser({ + active: false, + }), + ); + }); + + it('should map restored users', async () => { + expect( + await (converter as any).buildNewUserObject({ + emails: [], + importIds: [], + deleted: false, + }), + ).to.be.deep.equal( + mappedUser({ + active: true, + }), + ); + }); + + it('should map user type', async () => { + expect( + await (converter as any).buildNewUserObject({ + emails: [], + importIds: [], + type: 'user', + }), + ).to.be.deep.equal(mappedUser({})); + }); + + it('should map bot type', async () => { + expect( + await (converter as any).buildNewUserObject({ + emails: [], + importIds: [], + type: 'bot', + }), + ).to.be.deep.equal( + mappedUser({ + type: 'bot', + services: { + password: { + bcrypt: 'hashed=tempPassword', + }, + }, + }), + ); + }); + }); + + describe('[insertUser]', () => { + it('function should be called by the converter', async () => { + const converter = new UserConverter({ workInMemory: true, skipDefaultChannels: true }); + + modelsMock.Users.findOneByEmailAddress.resolves(null); + modelsMock.Users.findOneByUsernameIgnoringCase.resolves(null); + + sinon.stub(converter, 'insertUser'); + sinon.stub(converter, 'updateUser'); + + await converter.addObject(userToImport); + await converter.convertData(); + + expect(converter.updateUser.getCalls()).to.be.an('array').with.lengthOf(0); + expect(converter.insertUser.getCalls()).to.be.an('array').with.lengthOf(1); + expect(converter.insertUser.getCall(0).args).to.be.an('array').that.is.not.empty; + expect(converter.insertUser.getCall(0).args[0]).to.be.deep.equal(userToImport); + expect(addUserToDefaultChannels.getCalls()).to.be.an('array').with.lengthOf(0); + }); + + it('function should not be called when skipNewUsers = true', async () => { + const converter = new UserConverter({ workInMemory: true, skipNewUsers: true }); + + sinon.stub(converter, 'findExistingUser'); + sinon.stub(converter, 'insertUser'); + sinon.stub(converter, 'updateUser'); + sinon.stub(converter, 'skipMemoryRecord'); + + await converter.addObject(userToImport); + await converter.convertData(); + + expect(converter.insertUser.getCall(0)).to.be.null; + expect(converter.skipMemoryRecord.getCall(0)).to.not.be.null; + expect(callbacks.run.getCall(0)).to.not.be.null; + expect(callbacks.run.getCall(0).args).to.be.deep.equal(['afterUserImport', { inserted: [], updated: [], skipped: 1, failed: 0 }]); + }); + + it('function should not be called for existing users', async () => { + const converter = new UserConverter({ workInMemory: true }); + + sinon.stub(converter, 'findExistingUser'); + converter.findExistingUser.returns({ _id: 'oldId' }); + sinon.stub(converter, 'insertUser'); + sinon.stub(converter, 'updateUser'); + + await converter.addObject(userToImport); + await converter.convertData(); + + expect(converter.insertUser.getCall(0)).to.be.null; + expect(callbacks.run.getCall(0)).to.not.be.null; + expect(callbacks.run.getCall(0).args).to.be.deep.equal([ + 'afterUserImport', + { inserted: [], updated: ['oldId'], skipped: 0, failed: 0 }, + ]); + }); + + it('addUserToDefaultChannels should be called by the converter on successful insert', async () => { + const converter = new UserConverter({ workInMemory: true, skipDefaultChannels: false }); + + modelsMock.Users.findOneByEmailAddress.resolves(null); + modelsMock.Users.findOneByUsernameIgnoringCase.resolves(null); + modelsMock.Users.findOneById.withArgs('newId').returns({ newUser: true }); + + sinon.stub(converter, 'insertUser'); + + converter.insertUser.callsFake(() => 'newId'); + + await converter.addObject(userToImport); + await converter.convertData(); + + expect(converter.insertUser.getCalls()).to.be.an('array').with.lengthOf(1); + expect(converter.insertUser.getCall(0).args).to.be.an('array').that.is.not.empty; + expect(converter.insertUser.getCall(0).args[0]).to.be.deep.equal(userToImport); + expect(addUserToDefaultChannels.getCalls()).to.be.an('array').with.lengthOf(1); + expect(addUserToDefaultChannels.getCall(0).args).to.be.an('array').that.deep.contains({ newUser: true }); + }); + + it('should call insertUserDoc with the mapped data and roles', async () => { + const converter = new UserConverter({ workInMemory: true }); + let insertedUser = null; + + insertUserDoc.callsFake((_options, data) => { + insertedUser = { + ...data, + _id: 'Id1', + }; + return 'Id1'; + }); + + modelsMock.Users.findOneById.withArgs('Id1').resolves(insertedUser); + + await (converter as any).insertUser({ ...userToImport, roles: ['role1', 'role2'] }); + + expect(insertUserDoc.getCalls()).to.be.an('array').with.lengthOf(1); + expect(insertUserDoc.getCall(0).args).to.be.an('array').with.lengthOf(2); + + const usedParams = insertUserDoc.getCall(0).args[1]; + expect(usedParams).to.deep.include({ + type: 'user', + username: 'username1', + name: 'user1', + importIds: ['importId1'], + emails: [{ address: 'user1@domain.com', verified: false }], + globalRoles: ['role1', 'role2'], + }); + }); + }); + + describe('[updateUser]', () => { + it('function should be called by the converter', async () => { + const converter = new UserConverter({ workInMemory: true }); + + sinon.stub(converter, 'findExistingUser'); + converter.findExistingUser.returns({ _id: 'oldId' }); + sinon.stub(converter, 'insertUser'); + sinon.stub(converter, 'updateUser'); + + await converter.addObject(userToImport); + await converter.convertData(); + + expect(converter.insertUser.getCalls()).to.be.an('array').with.lengthOf(0); + expect(converter.updateUser.getCalls()).to.be.an('array').with.lengthOf(1); + expect(converter.updateUser.getCall(0).args).to.be.an('array').that.is.not.empty; + expect(converter.updateUser.getCall(0).args[1]).to.be.deep.equal(userToImport); + }); + + it('function should not be called when skipExistingUsers = true', async () => { + const converter = new UserConverter({ workInMemory: true, skipExistingUsers: true }); + + sinon.stub(converter, 'findExistingUser'); + converter.findExistingUser.returns({ _id: 'oldId' }); + sinon.stub(converter, 'insertUser'); + sinon.stub(converter, 'updateUser'); + sinon.stub(converter, 'skipMemoryRecord'); + + await converter.addObject(userToImport); + await converter.convertData(); + + expect(converter.updateUser.getCall(0)).to.be.null; + expect(converter.skipMemoryRecord.getCall(0)).to.not.be.null; + expect(callbacks.run.getCall(0)).to.not.be.null; + expect(callbacks.run.getCall(0).args).to.be.deep.equal(['afterUserImport', { inserted: [], updated: [], skipped: 1, failed: 0 }]); + }); + + it('function should not be called for new users', async () => { + const converter = new UserConverter({ workInMemory: true }); + + sinon.stub(converter, 'findExistingUser'); + sinon.stub(converter, 'insertUser'); + sinon.stub(converter, 'updateUser'); + + await converter.addObject(userToImport); + await converter.convertData(); + + expect(converter.updateUser.getCall(0)).to.be.null; + }); + }); + + // #TODO: Validate batch conversions + + describe('callbacks', () => { + it('beforeImportFn should be triggered', async () => { + const beforeImportFn = sinon.stub(); + + beforeImportFn.callsFake(() => true); + + const converter = new UserConverter({ workInMemory: true, skipDefaultChannels: true }); + + sinon.stub(converter, 'findExistingUser'); + sinon.stub(converter, 'insertUser'); + sinon.stub(converter, 'updateUser'); + + await converter.addObject(userToImport); + await converter.convertData({ + beforeImportFn, + }); + + expect(beforeImportFn.getCalls()).to.be.an('array').with.lengthOf(1); + expect(converter.insertUser.getCalls()).to.be.an('array').with.lengthOf(1); + }); + + it('afterImportFn should be triggered', async () => { + const afterImportFn = sinon.stub(); + const converter = new UserConverter({ workInMemory: true, skipDefaultChannels: true }); + + sinon.stub(converter, 'findExistingUser'); + sinon.stub(converter, 'insertUser'); + sinon.stub(converter, 'updateUser'); + + await converter.addObject(userToImport); + await converter.convertData({ + afterImportFn, + }); + + expect(converter.insertUser.getCalls()).to.be.an('array').with.lengthOf(1); + expect(afterImportFn.getCalls()).to.be.an('array').with.lengthOf(1); + }); + + it('should skip record if beforeImportFn returns false', async () => { + let recordId = null; + const beforeImportFn = sinon.stub(); + const afterImportFn = sinon.stub(); + + beforeImportFn.callsFake((record) => { + recordId = record._id; + return false; + }); + + const converter = new UserConverter({ workInMemory: true, skipDefaultChannels: true }); + + sinon.stub(converter, 'findExistingUser'); + sinon.stub(converter, 'insertUser'); + sinon.stub(converter, 'updateUser'); + sinon.stub(converter, 'skipMemoryRecord'); + + await converter.addObject(userToImport); + await converter.convertData({ + beforeImportFn, + afterImportFn, + }); + + expect(beforeImportFn.getCalls()).to.be.an('array').with.lengthOf(1); + expect(converter.skipMemoryRecord.getCalls()).to.be.an('array').with.lengthOf(1); + expect(afterImportFn.getCalls()).to.be.an('array').with.lengthOf(0); + expect(converter.skipMemoryRecord.getCall(0).args).to.be.an('array').that.is.deep.equal([recordId]); + + expect(callbacks.run.getCall(0)).to.not.be.null; + expect(callbacks.run.getCall(0).args).to.be.deep.equal(['afterUserImport', { inserted: [], updated: [], skipped: 1, failed: 0 }]); + }); + + it('should not skip record if beforeImportFn returns true', async () => { + let userId = null; + const beforeImportFn = sinon.stub(); + const afterImportFn = sinon.stub(); + + beforeImportFn.callsFake(() => true); + + afterImportFn.callsFake((record) => { + userId = record.data._id; + }); + + const converter = new UserConverter({ workInMemory: true, skipDefaultChannels: true }); + + sinon.stub(converter, 'findExistingUser'); + sinon.stub(converter, 'insertUser'); + sinon.stub(converter, 'updateUser'); + + await converter.addObject(userToImport); + await converter.convertData({ + beforeImportFn, + afterImportFn, + }); + + expect(beforeImportFn.getCalls()).to.be.an('array').with.lengthOf(1); + expect(converter.insertUser.getCalls()).to.be.an('array').with.lengthOf(1); + expect(afterImportFn.getCalls()).to.be.an('array').with.lengthOf(1); + + expect(callbacks.run.getCall(0)).to.not.be.null; + expect(callbacks.run.getCall(0).args).to.be.deep.equal([ + 'afterUserImport', + { inserted: [userId], updated: [], skipped: 0, failed: 0 }, + ]); + }); + + it('onErrorFn should be triggered if there is no email and no username', async () => { + const converter = new UserConverter({ workInMemory: true, skipDefaultChannels: true }); + + const onErrorFn = sinon.stub(); + + sinon.stub(converter, 'findExistingUser'); + sinon.stub(converter, 'insertUser'); + sinon.stub(converter, 'updateUser'); + sinon.stub(converter, 'saveError'); + + await converter.addObject({ + name: 'user1', + emails: [], + importIds: [], + }); + await converter.convertData({ onErrorFn }); + + expect(converter.insertUser.getCall(0)).to.be.null; + expect(callbacks.run.getCall(0)).to.not.be.null; + expect(callbacks.run.getCall(0).args).to.be.deep.equal(['afterUserImport', { inserted: [], updated: [], skipped: 0, failed: 1 }]); + expect(onErrorFn.getCall(0)).to.not.be.null; + expect(converter.saveError.getCall(0)).to.not.be.null; + }); + + // #TODO: Validate afterBatchFn + }); +});