Cherry picked changes from PR #29714

apps/meteor/app/api/server/v1/import.ts

@@ -9,8 +9,10 @@ import {
 	isDownloadPendingFilesParamsPOST,
 	isDownloadPendingAvatarsParamsPOST,
 	isGetCurrentImportOperationParamsGET,
+	isImportAddUsersParamsPOST,
 } from '@rocket.chat/rest-typings';
 import { Imports } from '@rocket.chat/models';
+import { Import } from '@rocket.chat/core-services';
 
 import { API } from '../api';
 import { Importers } from '../../../importer/server';
@@ -66,7 +68,7 @@ API.v1.addRoute(
 		async post() {
 			const { input } = this.bodyParams;
 
-			await executeStartImport({ input });
+			await executeStartImport({ input }, this.userId);
 
 			return API.v1.success();
 		},
@@ -133,8 +135,9 @@ API.v1.addRoute(
 				throw new Meteor.Error('error-importer-not-defined', 'The Pending File Importer was not found.', 'downloadPendingFiles');
 			}
 
-			importer.instance = new importer.importer(importer); // eslint-disable-line new-cap
-			await importer.instance.build();
+			const operation = await Import.newOperation(this.userId, importer.name, importer.key);
+
+			importer.instance = new importer.importer(importer, operation); // eslint-disable-line new-cap
 			const count = await importer.instance.prepareFileCount();
 
 			return API.v1.success({
@@ -158,8 +161,8 @@ API.v1.addRoute(
 				throw new Meteor.Error('error-importer-not-defined', 'The Pending File Importer was not found.', 'downloadPendingAvatars');
 			}
 
-			importer.instance = new importer.importer(importer); // eslint-disable-line new-cap
-			await importer.instance.build();
+			const operation = await Import.newOperation(this.userId, importer.name, importer.key);
+			importer.instance = new importer.importer(importer, operation); // eslint-disable-line new-cap
 			const count = await importer.instance.prepareFileCount();
 
 			return API.v1.success({
@@ -185,3 +188,81 @@ API.v1.addRoute(
 		},
 	},
 );
+
+API.v1.addRoute(
+	'import.clear',
+	{
+		authRequired: true,
+		permissionsRequired: ['run-import'],
+	},
+	{
+		async post() {
+			await Import.clear();
+
+			return API.v1.success();
+		},
+	},
+);
+
+API.v1.addRoute(
+	'import.new',
+	{
+		authRequired: true,
+		permissionsRequired: ['run-import'],
+	},
+	{
+		async post() {
+			const operation = await Import.newOperation(this.userId, 'api', 'api');
+
+			return API.v1.success({ operation });
+		},
+	},
+);
+
+API.v1.addRoute(
+	'import.status',
+	{
+		authRequired: true,
+		permissionsRequired: ['run-import'],
+	},
+	{
+		async get() {
+			const status = await Import.status();
+
+			return API.v1.success(status);
+		},
+	},
+);
+
+API.v1.addRoute(
+	'import.addUsers',
+	{
+		authRequired: true,
+		validateParams: isImportAddUsersParamsPOST,
+		permissionsRequired: ['run-import'],
+	},
+	{
+		async post() {
+			const { users } = this.bodyParams;
+
+			await Import.addUsers(users);
+
+			return API.v1.success();
+		},
+	},
+);
+
+API.v1.addRoute(
+	'import.run',
+	{
+		authRequired: true,
+		permissionsRequired: ['run-import'],
+	},
+	{
+		async post() {
+			await Import.run(this.userId);
+
+			return API.v1.success();
+		},
+	},
+);

apps/meteor/app/api/server/v1/users.ts

@@ -29,7 +29,7 @@ import {
 	checkUsernameAvailability,
 	checkUsernameAvailabilityWithValidation,
 } from '../../../lib/server/functions/checkUsernameAvailability';
-import { getFullUserDataByIdOrUsername } from '../../../lib/server/functions/getFullUserData';
+import { getFullUserDataByIdOrUsernameOrImportId } from '../../../lib/server/functions/getFullUserData';
 import { setStatusText } from '../../../lib/server/functions/setStatusText';
 import { API } from '../api';
 import { findUsersToAutocomplete, getInclusiveFields, getNonEmptyFields, getNonEmptyQuery } from '../lib/users';
@@ -393,10 +393,16 @@ API.v1.addRoute(
 		async get() {
 			const { fields } = await this.parseJsonQuery();
 
-			const user = await getFullUserDataByIdOrUsername(this.userId, {
-				filterId: (this.queryParams as any).userId,
-				filterUsername: (this.queryParams as any).username,
-			});
+			const searchTerms: [string, 'id' | 'username' | 'importId'] | false =
+				('userId' in this.queryParams && !!this.queryParams.userId && [this.queryParams.userId, 'id']) ||
+				('username' in this.queryParams && !!this.queryParams.username && [this.queryParams.username, 'username']) ||
+				('importId' in this.queryParams && !!this.queryParams.importId && [this.queryParams.importId, 'importId']);
+
+			if (!searchTerms) {
+				return API.v1.failure('Invalid search query.');
+			}
+
+			const user = await getFullUserDataByIdOrUsernameOrImportId(this.userId, ...searchTerms);
 
 			if (!user) {
 				return API.v1.failure('User not found.');

apps/meteor/app/importer-csv/server/importer.js

@@ -59,13 +59,15 @@ export class CsvImporter extends Base {
 			// Ignore anything that has `__MACOSX` in it's name, as sadly these things seem to mess everything up
 			if (entry.entryName.indexOf('__MACOSX') > -1) {
 				this.logger.debug(`Ignoring the file: ${entry.entryName}`);
-				return increaseProgressCount();
+				increaseProgressCount();
+				continue;
 			}
 
 			// Directories are ignored, since they are "virtual" in a zip file
 			if (entry.isDirectory) {
 				this.logger.debug(`Ignoring the directory entry: ${entry.entryName}`);
-				return increaseProgressCount();
+				increaseProgressCount();
+				continue;
 			}
 
 			// Parse the channels
@@ -97,7 +99,8 @@ export class CsvImporter extends Base {
 				}
 
 				await super.updateRecord({ 'count.channels': channelsCount });
-				return increaseProgressCount();
+				increaseProgressCount();
+				continue;
 			}
 
 			// Parse the users
@@ -114,6 +117,7 @@ export class CsvImporter extends Base {
 					const name = u[2].trim();
 
 					await this.converter.addUser({
+						type: 'user',
 						importIds: [username],
 						emails: [email],
 						username,
@@ -122,7 +126,8 @@ export class CsvImporter extends Base {
 				}
 
 				await super.updateRecord({ 'count.users': usersCount });
-				return increaseProgressCount();
+				increaseProgressCount();
+				continue;
 			}
 
 			// Parse the messages
@@ -140,7 +145,8 @@ export class CsvImporter extends Base {
 					msgs = this.csvParser(entry.getData().toString());
 				} catch (e) {
 					this.logger.warn(`The file ${entry.entryName} contains invalid syntax`, e);
-					return increaseProgressCount();
+					increaseProgressCount();
+					continue;
 				}
 
 				let data;
@@ -211,7 +217,8 @@ export class CsvImporter extends Base {
 				}
 
 				await super.updateRecord({ 'count.messages': messagesCount, 'messagesstatus': null });
-				return increaseProgressCount();
+				increaseProgressCount();
+				continue;
 			}
 
 			increaseProgressCount();
@@ -243,7 +250,8 @@ export class CsvImporter extends Base {
 		if (usersCount === 0 && channelsCount === 0 && messagesCount === 0) {
 			this.logger.error('No users, channels, or messages found in the import file.');
 			await super.updateProgress(ProgressStep.ERROR);
-			return super.getProgress();
 		}
+
+		return super.getProgress();
 	}
 }

apps/meteor/app/importer-pending-avatars/server/importer.js

@@ -39,7 +39,7 @@ export class PendingAvatarImporter extends Base {
 
 					try {
 						if (!url || !url.startsWith('http')) {
-							return;
+							continue;
 						}
 
 						try {

apps/meteor/app/importer-slack-users/server/importer.js

@@ -1,3 +1,5 @@
+import fs from 'fs';
+
 import { Settings } from '@rocket.chat/models';
 
 import { Base, ProgressStep } from '../../importer/server';
@@ -13,14 +15,19 @@ export class SlackUsersImporter extends Base {
 	}
 
 	async prepareUsingLocalFile(fullFilePath) {
+		this.logger.debug('start preparing import operation');
 		await this.converter.clearImportData();
 
-		return super.prepareUsingLocalFile(fullFilePath);
-	}
+		const file = fs.readFileSync(fullFilePath);
+		const buffer = Buffer.isBuffer(file) ? file : Buffer.from(file);
 
-	async prepare(dataURI, sentContentType, fileName) {
-		this.logger.debug('start preparing import operation');
-		await super.prepare(dataURI, sentContentType, fileName, true);
+		const { contentType } = this.importRecord;
+		const fileName = this.importRecord.file;
+
+		const data = buffer.toString('base64');
+		const dataURI = `data:${contentType};base64,${data}`;
+
+		await this.updateRecord({ file: fileName });
 
 		await super.updateProgress(ProgressStep.PREPARING_USERS);
 		const uriResult = RocketChatFile.dataURIParse(dataURI);
@@ -76,6 +83,7 @@ export class SlackUsersImporter extends Base {
 		await super.updateProgress(ProgressStep.USER_SELECTION);
 		await super.addCountToTotal(userCount);
 		await Settings.incrementValueById('Slack_Users_Importer_Count', userCount);
-		return super.updateRecord({ 'count.users': userCount });
+		await super.updateRecord({ 'count.users': userCount });
+		return super.getProgress();
 	}
 }

apps/meteor/app/importer-slack/server/importer.js

@@ -295,6 +295,8 @@ export class SlackImporter extends Base {
 
 		ImporterWebsocket.progressUpdated({ rate: 100 });
 		await this.updateRecord({ 'count.messages': messagesCount, 'messagesstatus': null });
+
+		return this.progress;
 	}
 
 	parseMentions(newMessage) {
@@ -400,6 +402,8 @@ export class SlackImporter extends Base {
 				}
 				break;
 		}
+
+		return false;
 	}
 
 	makeSlackMessageId(channelId, ts, fileIndex = undefined) {

apps/meteor/app/importer/server/classes/ImportDataConverter.ts

@@ -80,6 +80,8 @@ export class ImportDataConverter {
 		return this._options;
 	}
 
+	public aborted = false;
+
 	constructor(options?: IConverterOptions) {
 		this._options = options || {
 			flagEmailsAsVerified: false,
@@ -285,7 +287,8 @@ export class ImportDataConverter {
 
 	// TODO
 	async insertUser(userData: IImportUser): Promise<IUser> {
-		const password = `${Date.now()}${userData.name || ''}${userData.emails.length ? userData.emails[0].toUpperCase() : ''}`;
+		const password =
+			userData.password || `${Date.now()}${userData.name || ''}${userData.emails.length ? userData.emails[0].toUpperCase() : ''}`;
 		const userId = userData.emails.length
 			? await Accounts.createUserAsync({
 					email: userData.emails[0],
@@ -329,6 +332,10 @@ export class ImportDataConverter {
 	public async convertUsers({ beforeImportFn, afterImportFn }: IConversionCallbacks = {}): Promise<void> {
 		const users = (await this.getUsersToImport()) as IImportUserRecord[];
 		for await (const { data, _id } of users) {
+			if (this.aborted) {
+				return;
+			}
+
 			try {
 				if (beforeImportFn && !(await beforeImportFn(data, 'user'))) {
 					await this.skipRecord(_id);
@@ -568,6 +575,10 @@ export class ImportDataConverter {
 		const messages = await this.getMessagesToImport();
 
 		for await (const { data, _id } of messages) {
+			if (this.aborted) {
+				return;
+			}
+
 			try {
 				if (beforeImportFn && !(await beforeImportFn(data, 'message'))) {
 					await this.skipRecord(_id);
@@ -937,6 +948,10 @@ export class ImportDataConverter {
 	async convertChannels(startedByUserId: string, { beforeImportFn, afterImportFn }: IConversionCallbacks = {}): Promise<void> {
 		const channels = await this.getChannelsToImport();
 		for await (const { data, _id } of channels) {
+			if (this.aborted) {
+				return;
+			}
+
 			try {
 				if (beforeImportFn && !(await beforeImportFn(data, 'channel'))) {
 					await this.skipRecord(_id);