From 9e57b34cca905804955896809e6885bb44bb4baf Mon Sep 17 00:00:00 2001 From: Callum Reekie Date: Thu, 14 Apr 2022 09:53:25 +0100 Subject: [PATCH 01/22] add federation warning to rejected datasets email --- .../utilities/emailGenerator.util.js | 32 +++++++++++++------ src/utils/datasetonboarding.util.js | 9 +++++- 2 files changed, 31 insertions(+), 10 deletions(-) diff --git a/src/resources/utilities/emailGenerator.util.js b/src/resources/utilities/emailGenerator.util.js index 356bdb43..4060bf65 100644 --- a/src/resources/utilities/emailGenerator.util.js +++ b/src/resources/utilities/emailGenerator.util.js @@ -2083,9 +2083,10 @@ const _generateMetadataOnboardingApproved = options => { }; const _generateMetadataOnboardingRejected = options => { - let { name, publisherId, comment } = options; + let { name, publisherId, comment, isFederated } = options; let commentHTML = ''; + let federatedMessageHTML = ''; if (!_.isEmpty(comment)) { commentHTML = ` @@ -2100,6 +2101,14 @@ const _generateMetadataOnboardingRejected = options => { `; } + if (!_.isUndefined(isFederated) && isFederated) { + federatedMessageHTML = ` + + It is important that you update these changes in your metadata catalogue. Do not apply these changes directly to the Gateway as this ability has been disabled for federated datasets. + + `; + } + let body = `
{ + ${federatedMessageHTML} ${commentHTML} @@ -2611,15 +2621,19 @@ const _generateWordAttachment = async (templateName, questionAnswers) => { return wordAttachment; }; -const _generateWordContent = async (filename) => { - let pathToAttachment = `${__dirname}/populatedtemplate.docx`; - let content = await fs.readFileSync(pathToAttachment).toString('base64'); - return content -} +const _generateWordContent = async filename => { + let pathToAttachment = `${__dirname}/populatedtemplate.docx`; + let content = await fs.readFileSync(pathToAttachment).toString('base64'); + return content; +}; const _deleteWordAttachmentTempFiles = async () => { - if(fs.existsSync(`${__dirname}/template.docx`)){fs.unlinkSync(__dirname + '/template.docx')} - if(fs.existsSync(`${__dirname}/populatedtemplate.docx`)){fs.unlinkSync(__dirname + '/populatedtemplate.docx')} + if (fs.existsSync(`${__dirname}/template.docx`)) { + fs.unlinkSync(__dirname + '/template.docx'); + } + if (fs.existsSync(`${__dirname}/populatedtemplate.docx`)) { + fs.unlinkSync(__dirname + '/populatedtemplate.docx'); + } }; export default { @@ -2648,7 +2662,7 @@ export default { generateNewDARMessage: _generateNewDARMessage, deleteWordAttachmentTempFiles: _deleteWordAttachmentTempFiles, generateWordAttachment: _generateWordAttachment, - generateWordContent: _generateWordContent, + generateWordContent: _generateWordContent, //Workflows generateWorkflowAssigned: _generateWorkflowAssigned, generateWorkflowCreated: _generateWorkflowCreated, diff --git a/src/utils/datasetonboarding.util.js b/src/utils/datasetonboarding.util.js index 51f09e84..3695ea48 100644 --- a/src/utils/datasetonboarding.util.js +++ b/src/utils/datasetonboarding.util.js @@ -921,7 +921,11 @@ const createNotifications = async (type, context) => { break; case constants.notificationTypes.DATASETREJECTED: // 1. Get user removed - team = await TeamModel.findOne({ _id: context.datasetv2.summary.publisher.identifier }).lean(); + team = await TeamModel.findOne({ _id: context.datasetv2.summary.publisher.identifier }) + .populate([{ path: 'publisher' }]) + .lean(); + + const isFederated = !_.isUndefined(team.publisher.federation) && team.publisher.federation.active; for (let member of team.members) { teamMembers.push(member.memberid); @@ -948,8 +952,11 @@ const createNotifications = async (type, context) => { name: context.name, publisherId: context.datasetv2.summary.publisher.identifier, comment: context.applicationStatusDesc, + isFederated, }; + html = emailGenerator.generateMetadataOnboardingRejected(options); + emailGenerator.sendEmail( teamMembersDetails, constants.hdrukEmail, From 236941541b6ea50f187ec2f26f9b80193615187d Mon Sep 17 00:00:00 2001 From: Callum Reekie Date: Thu, 14 Apr 2022 09:54:00 +0100 Subject: [PATCH 02/22] tests for rejected dataset emails --- .../__tests__/emailGenerator.util.test.js | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 src/resources/utilities/__tests__/emailGenerator.util.test.js diff --git a/src/resources/utilities/__tests__/emailGenerator.util.test.js b/src/resources/utilities/__tests__/emailGenerator.util.test.js new file mode 100644 index 00000000..184b63b8 --- /dev/null +++ b/src/resources/utilities/__tests__/emailGenerator.util.test.js @@ -0,0 +1,25 @@ +import emailGenerator from '../emailGenerator.util'; + +describe('Email generator utility functions', () => { + describe('_generateMetadataOnboardingRejected', () => { + let isFederated; + + it('SHOULD include federated warning if isFederated is true', async () => { + isFederated = true; + + const emailBody = emailGenerator.generateMetadataOnboardingRejected({ isFederated }); + + // Federated warning should be present if dataset if from a federated publisher + expect(emailBody.includes('Do not apply these changes directly to the Gateway')).toBe(true); + }); + + it('SHOULD NOT include federated warning if isFederated is false', async () => { + isFederated = false; + + const emailBody = emailGenerator.generateMetadataOnboardingRejected({ isFederated }); + + // Federated warning should not be present if dataset is not from a federated publisher + expect(emailBody.includes('Do not apply these changes directly to the Gateway')).toBe(false); + }); + }); +}); From 69ae41856db0460a3a58b99b902c741734f53fab Mon Sep 17 00:00:00 2001 From: Callum Reekie Date: Thu, 14 Apr 2022 10:00:29 +0100 Subject: [PATCH 03/22] typo fix --- src/resources/utilities/__tests__/emailGenerator.util.test.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/resources/utilities/__tests__/emailGenerator.util.test.js b/src/resources/utilities/__tests__/emailGenerator.util.test.js index 184b63b8..e8a34ada 100644 --- a/src/resources/utilities/__tests__/emailGenerator.util.test.js +++ b/src/resources/utilities/__tests__/emailGenerator.util.test.js @@ -9,7 +9,7 @@ describe('Email generator utility functions', () => { const emailBody = emailGenerator.generateMetadataOnboardingRejected({ isFederated }); - // Federated warning should be present if dataset if from a federated publisher + // Federated warning should be present if dataset is from a federated publisher expect(emailBody.includes('Do not apply these changes directly to the Gateway')).toBe(true); }); From b2429db8713bfc0706d4bca56f2d9447c23d2dba Mon Sep 17 00:00:00 2001 From: Dan Nita Date: Thu, 14 Apr 2022 12:52:33 +0100 Subject: [PATCH 04/22] update message --- src/resources/team/team.controller.js | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/resources/team/team.controller.js b/src/resources/team/team.controller.js index 175ef1c9..531a12c1 100644 --- a/src/resources/team/team.controller.js +++ b/src/resources/team/team.controller.js @@ -1,4 +1,4 @@ -import { isEmpty, has, difference, includes, isNull, filter, some } from 'lodash'; +import _, { isEmpty, has, difference, includes, isNull, filter, some } from 'lodash'; import { TeamModel } from './team.model'; import { UserModel } from '../user/user.model'; import { PublisherModel } from '../publisher/publisher.model'; @@ -1047,8 +1047,12 @@ const filterMembersByNoticationTypes = (members, notificationTypes) => { */ const filterMembersByNoticationTypesOptIn = (members, notificationTypes) => { return filter(members, member => { + if (!('notifications' in member) || _.isEmpty(member.notifications)) { + return true; + } + return some(member.notifications, notification => { - return includes(notificationTypes, notification.notificationType) && notification.optIn; + return includes(notificationTypes, notification.notificationType) && (notification.optIn === true); }); }); }; From e64131065798cdea539f73815abf1f0b02e1c80a Mon Sep 17 00:00:00 2001 From: kandaj Date: Fri, 29 Apr 2022 15:06:44 +0100 Subject: [PATCH 05/22] send email non production issue fix --- .../__tests__/emailGenerator.util.test.js | 19 +++++++++++++++++++ .../utilities/emailGenerator.util.js | 11 ++++++++--- 2 files changed, 27 insertions(+), 3 deletions(-) diff --git a/src/resources/utilities/__tests__/emailGenerator.util.test.js b/src/resources/utilities/__tests__/emailGenerator.util.test.js index e8a34ada..1cff0fcf 100644 --- a/src/resources/utilities/__tests__/emailGenerator.util.test.js +++ b/src/resources/utilities/__tests__/emailGenerator.util.test.js @@ -22,4 +22,23 @@ describe('Email generator utility functions', () => { expect(emailBody.includes('Do not apply these changes directly to the Gateway')).toBe(false); }); }); + describe('_getRecipients', () => { + const mockRecipients = [ + { email: 'test1@test.com' }, + { email: 'test2@test.com' }, + { email: 'test3@test.com' }, + { email: 'test1@test.com' }, + ]; + it('Should remove duplicaties for production', async () => { + const recipients = emailGenerator.getRecipients(mockRecipients, 'production', 'genericemail@test.com'); + expect(recipients.length).toBe(3); + expect(recipients).toEqual([{ email: 'test1@test.com' }, { email: 'test2@test.com' }, { email: 'test3@test.com' }]); + }); + + it('Should replace recipients non production environtment to generic email', async () => { + const recipients = emailGenerator.getRecipients(mockRecipients, undefined, 'genericemail@test.com'); + expect(recipients.length).toBe(1); + expect(recipients).toEqual([{ email: 'genericemail@test.com' }]); + }); + }); }); diff --git a/src/resources/utilities/emailGenerator.util.js b/src/resources/utilities/emailGenerator.util.js index 4060bf65..fd946a82 100644 --- a/src/resources/utilities/emailGenerator.util.js +++ b/src/resources/utilities/emailGenerator.util.js @@ -2496,6 +2496,10 @@ ${_displayDataUseRegisterDashboardLink()} return body; }; +const _getRecipients = (recipients, environment, genericEmail) => { + return environment === 'production' ? [...new Map(recipients.map(item => [item['email'], item])).values()] : [{ email: genericEmail }]; +}; + /** * [_sendEmail] * @@ -2507,7 +2511,7 @@ const _sendEmail = async (to, from, subject, html, allowUnsubscribe = true, atta sgMail.setApiKey(process.env.SENDGRID_API_KEY); // 2. Ensure any duplicates recieve only a single email - const recipients = [...new Map(to.map(item => [item['email'], item])).values()]; + const recipients = _getRecipients(to, process.env.NODE_ENV, process.env.GENERIC_EMAIL); // 3. Build each email object for SendGrid extracting email addresses from user object with unique unsubscribe link (to) for (let recipient of recipients) { @@ -2522,7 +2526,7 @@ const _sendEmail = async (to, from, subject, html, allowUnsubscribe = true, atta // 4. Send email using SendGrid await sgMail.send(msg, false, err => { - if (err && (readEnv === 'test' || readEnv === 'prod')) { + if (err && process.env.NODE_ENV === 'production') { Sentry.addBreadcrumb({ category: 'SendGrid', message: 'Sending email failed', @@ -2545,7 +2549,7 @@ const _sendIntroEmail = msg => { sgMail.setApiKey(process.env.SENDGRID_API_KEY); // 2. Send email using SendGrid sgMail.send(msg, false, err => { - if (err && (readEnv === 'test' || readEnv === 'prod')) { + if (err && process.env.NODE_ENV === 'production') { Sentry.addBreadcrumb({ category: 'SendGrid', message: 'Sending email failed - Intro', @@ -2684,4 +2688,5 @@ export default { generateDataUseRegisterApproved: _generateDataUseRegisterApproved, generateDataUseRegisterRejected: _generateDataUseRegisterRejected, generateDataUseRegisterPending: _generateDataUseRegisterPending, + getRecipients: _getRecipients, }; From 06e4abb9a665091ea4734942a180f72e48d2210c Mon Sep 17 00:00:00 2001 From: Pritesh Bhole Date: Wed, 4 May 2022 13:27:32 +0100 Subject: [PATCH 06/22] changed reject email subject --- src/utils/datasetonboarding.util.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/datasetonboarding.util.js b/src/utils/datasetonboarding.util.js index e8a32b95..515d21a8 100644 --- a/src/utils/datasetonboarding.util.js +++ b/src/utils/datasetonboarding.util.js @@ -955,7 +955,7 @@ const createNotifications = async (type, context) => { emailGenerator.sendEmail( teamMembersDetails, constants.hdrukEmail, - `Your dataset version has been reviewed and rejected`, + `Your federated dataset has been rejected and requires review`, html, false ); From d5e24d2ce501057cd2d74d07f70ff40a3d3a936d Mon Sep 17 00:00:00 2001 From: Pritesh Bhole Date: Wed, 4 May 2022 14:05:43 +0100 Subject: [PATCH 07/22] fixed as per comments --- src/utils/datasetonboarding.util.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/utils/datasetonboarding.util.js b/src/utils/datasetonboarding.util.js index 515d21a8..1f431d04 100644 --- a/src/utils/datasetonboarding.util.js +++ b/src/utils/datasetonboarding.util.js @@ -952,10 +952,11 @@ const createNotifications = async (type, context) => { comment: context.applicationStatusDesc, }; html = emailGenerator.generateMetadataOnboardingRejected(options); + let subject = (options.isFederated) ? 'Your federated dataset has been rejected and requires review' : 'Your dataset version has been reviewed and rejected' emailGenerator.sendEmail( teamMembersDetails, - constants.hdrukEmail, - `Your federated dataset has been rejected and requires review`, + constants.hdrukEmail, + subject, html, false ); From d3f8c23381802994bf1668a4cbd3a21b3c0bbd2e Mon Sep 17 00:00:00 2001 From: Callum Reekie Date: Wed, 4 May 2022 16:05:22 +0100 Subject: [PATCH 08/22] hide federation details from publisher model --- src/resources/publisher/publisher.model.js | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/resources/publisher/publisher.model.js b/src/resources/publisher/publisher.model.js index 6db5672a..8ba20629 100644 --- a/src/resources/publisher/publisher.model.js +++ b/src/resources/publisher/publisher.model.js @@ -44,6 +44,12 @@ const PublisherSchema = new Schema( allowAccessRequestManagement: { type: Boolean, default: false }, uses5Safes: { type: Boolean, default: false }, wordTemplate: String, + federation: { + active: { type: Boolean }, + auth: { type: Object, select: false }, + endpoints: { type: Boolean, select: false }, + notificationEmail: { type: Array, select: false }, + }, }, { toJSON: { virtuals: true }, From 473b60ef6c55325bb28abac45f5ca310be892f19 Mon Sep 17 00:00:00 2001 From: Dan Nita Date: Wed, 4 May 2022 19:27:29 +0100 Subject: [PATCH 09/22] update notifications for DAR and Workflow partially --- .../datarequest/datarequest.controller.js | 155 +++++++++++++++--- src/resources/team/team.controller.js | 34 +++- 2 files changed, 164 insertions(+), 25 deletions(-) diff --git a/src/resources/datarequest/datarequest.controller.js b/src/resources/datarequest/datarequest.controller.js index e776dc3e..a83ff5ec 100644 --- a/src/resources/datarequest/datarequest.controller.js +++ b/src/resources/datarequest/datarequest.controller.js @@ -1947,6 +1947,7 @@ export default class DataRequestController extends Controller { let { firstname, lastname } = user; // Instantiate default params let custodianManagers = [], + custodianManagersIds = [], custodianUserIds = [], managerUserIds = [], emailRecipients = [], @@ -1990,8 +1991,15 @@ export default class DataRequestController extends Controller { switch (type) { case constants.notificationTypes.INPROGRESS: + custodianManagers = teamController.getTeamMembersByRole(accessRecord.publisherObj.team, 'All'); + custodianManagersIds = custodianManagers.map(user => user.id); + if (accessRecord.publisherObj.team.notifications[0].optIn) { + accessRecord.publisherObj.team.notifications[0].subscribedEmails.map(teamEmail => { + custodianManagers.push({email: teamEmail}); + }); + } await notificationBuilder.triggerNotificationMessage( - [user.id], + [user.id, ...custodianManagersIds], `An email with the data access request info for ${datasetTitles} has been sent to you`, 'data access request', accessRecord._id @@ -2010,7 +2018,7 @@ export default class DataRequestController extends Controller { // Build email template ({ html } = await emailGenerator.generateEmail(aboutApplication, questions, pages, questionPanels, questionAnswers, options)); await emailGenerator.sendEmail( - [user], + [user, ...custodianManagers], constants.hdrukEmail, `Data Access Request in progress for ${projectName || datasetTitles}`, html, @@ -2022,7 +2030,7 @@ export default class DataRequestController extends Controller { // 1. Create notifications // Custodian manager and current step reviewer notifications // Retrieve all custodian manager user Ids and active step reviewers - custodianManagers = teamController.getTeamMembersByRole(accessRecord.publisherObj.team, constants.roleTypes.MANAGER); + custodianManagers = teamController.getTeamMembersByRole(accessRecord.publisherObj.team, 'All'); let activeStep = this.workflowService.getActiveWorkflowStep(workflow); stepReviewers = this.workflowService.getStepReviewers(activeStep); // Create custodian notification @@ -2033,6 +2041,11 @@ export default class DataRequestController extends Controller { 'data access request', accessRecord._id ); + if (accessRecord.publisherObj.team.notifications[0].optIn) { + accessRecord.publisherObj.team.notifications[0].subscribedEmails.map(teamEmail => { + custodianManagers.push({email: teamEmail}); + }); + } // Create applicant notification await notificationBuilder.triggerNotificationMessage( @@ -2084,7 +2097,7 @@ export default class DataRequestController extends Controller { // Custodian notification if (_.has(accessRecord.datasets[0], 'publisher.team.users') && accessRecord.datasets[0].publisher.allowAccessRequestManagement) { // Retrieve all custodian user Ids to generate notifications - custodianManagers = teamController.getTeamMembersByRole(accessRecord.datasets[0].publisher.team, constants.roleTypes.MANAGER); + custodianManagers = teamController.getTeamMembersByRole(accessRecord.datasets[0].publisher.team, 'All'); // check if publisher.team has email notifications custodianUserIds = custodianManagers.map(user => user.id); await notificationBuilder.triggerNotificationMessage( @@ -2094,6 +2107,28 @@ export default class DataRequestController extends Controller { accessRecord._id, accessRecord.datasets[0].publisher._id.toString() ); + if (accessRecord.datasets[0].publisher.team.notifications[0].optIn) { + accessRecord.datasets[0].publisher.team.notifications[0].subscribedEmails.map(teamEmail => { + custodianManagers.push({email: teamEmail}); + }); + } + } else if (_.has(accessRecord, 'publisherObj') && accessRecord.publisherObj.allowAccessRequestManagement) { + // Retrieve all custodian user Ids to generate notifications + custodianManagers = teamController.getTeamMembersByRole(accessRecord.publisherObj.team, 'All'); + // check if publisher.team has email notifications + custodianUserIds = custodianManagers.map(user => user.id); + await notificationBuilder.triggerNotificationMessage( + custodianUserIds, + `A Data Access Request has been submitted to ${publisher} for ${projectName || datasetTitles} by ${appFirstName} ${appLastName}`, + 'data access request received', + accessRecord._id, + accessRecord.datasets[0].publisher._id.toString() + ); + if (accessRecord.publisherObj.team.notifications[0].optIn) { + accessRecord.publisherObj.team.notifications[0].subscribedEmails.map(teamEmail => { + custodianManagers.push({email: teamEmail}); + }); + } } else { const dataCustodianEmail = process.env.DATA_CUSTODIAN_EMAIL || contactPoint; custodianManagers = [{ email: dataCustodianEmail }]; @@ -2185,7 +2220,22 @@ export default class DataRequestController extends Controller { // Custodian notification if (_.has(accessRecord.datasets[0], 'publisher.team.users')) { // Retrieve all custodian user Ids to generate notifications - custodianManagers = teamController.getTeamMembersByRole(accessRecord.datasets[0].publisher.team, constants.roleTypes.MANAGER); + custodianManagers = teamController.getTeamMembersByRole(accessRecord.datasets[0].publisher.team, 'All'); + custodianUserIds = custodianManagers.map(user => user.id); + await notificationBuilder.triggerNotificationMessage( + custodianUserIds, + `A Data Access Request has been resubmitted with updates to ${publisher} for ${projectName || datasetTitles} by ${appFirstName} ${appLastName}`, + 'data access request', + accessRecord._id + ); + if (accessRecord.datasets[0].publisher.team.notifications[0].optIn) { + accessRecord.datasets[0].publisher.team.notifications[0].subscribedEmails.map(teamEmail => { + custodianManagers.push({email: teamEmail}); + }); + } + } else if (_.has(accessRecord, 'publisherObj') && accessRecord.publisherObj.allowAccessRequestManagement) { + // Retrieve all custodian user Ids to generate notifications + custodianManagers = teamController.getTeamMembersByRole(accessRecord.publisherObj.team, 'All'); custodianUserIds = custodianManagers.map(user => user.id); await notificationBuilder.triggerNotificationMessage( custodianUserIds, @@ -2193,6 +2243,11 @@ export default class DataRequestController extends Controller { 'data access request', accessRecord._id ); + if (accessRecord.publisherObj.team.notifications[0].optIn) { + accessRecord.publisherObj.team.notifications[0].subscribedEmails.map(teamEmail => { + custodianManagers.push({email: teamEmail}); + }); + } } else { const dataCustodianEmail = process.env.DATA_CUSTODIAN_EMAIL || contactPoint; custodianManagers = [{ email: dataCustodianEmail }]; @@ -2393,16 +2448,21 @@ export default class DataRequestController extends Controller { break; case constants.notificationTypes.FINALDECISIONREQUIRED: // 1. Get managers for publisher - custodianManagers = teamController.getTeamMembersByRole(accessRecord.publisherObj.team, constants.roleTypes.MANAGER); - managerUserIds = custodianManagers.map(user => user.id); + custodianManagers = teamController.getTeamMembersByRole(accessRecord.publisherObj.team, 'All'); + custodianManagersIds = custodianManagers.map(user => user.id); // 2. Create manager notifications notificationBuilder.triggerNotificationMessage( - managerUserIds, + custodianManagersIds, `Action is required as a Data Access Request application for ${publisher} is now awaiting a final decision`, 'data access request', accessRecord._id ); + if (accessRecord.publisherObj.team.notifications[0].optIn) { + accessRecord.publisherObj.team.notifications[0].subscribedEmails.map(teamEmail => { + custodianManagers.push({email: teamEmail}); + }); + } // 3. Create manager emails options = { id: accessRecord._id, @@ -2425,9 +2485,17 @@ export default class DataRequestController extends Controller { ); break; case constants.notificationTypes.DEADLINEWARNING: + custodianManagers = teamController.getTeamMembersByRole(accessRecord.publisherObj.team, 'All'); + custodianManagersIds = custodianManagers.map(user => user.id); + if (accessRecord.publisherObj.team.notifications[0].optIn) { + accessRecord.publisherObj.team.notifications[0].subscribedEmails.map(teamEmail => { + custodianManagers.push({email: teamEmail}); + }); + } + // 1. Create reviewer notifications await notificationBuilder.triggerNotificationMessage( - remainingReviewerUserIds, + [...remainingReviewerUserIds,...custodianManagersIds], `The deadline is approaching for a Data Access Request application you are reviewing`, 'data access request', accessRecord._id @@ -2450,7 +2518,7 @@ export default class DataRequestController extends Controller { }; html = await emailGenerator.generateReviewDeadlineWarning(options); await emailGenerator.sendEmail( - remainingReviewers, + [...remainingReviewers, ...custodianManagers], constants.hdrukEmail, `The deadline is approaching for a Data Access Request application you are reviewing`, html, @@ -2459,10 +2527,10 @@ export default class DataRequestController extends Controller { break; case constants.notificationTypes.DEADLINEPASSED: // 1. Get all managers - custodianManagers = teamController.getTeamMembersByRole(accessRecord.publisherObj.team, constants.roleTypes.MANAGER); - managerUserIds = custodianManagers.map(user => user.id); + custodianManagers = teamController.getTeamMembersByRole(accessRecord.publisherObj.team, 'All'); + custodianManagersIds = custodianManagers.map(user => user.id); // 2. Combine managers and reviewers remaining - let deadlinePassedUserIds = [...remainingReviewerUserIds, ...managerUserIds]; + let deadlinePassedUserIds = [...remainingReviewerUserIds, ...custodianManagersIds]; let deadlinePassedUsers = [...remainingReviewers, ...custodianManagers]; // 3. Create notifications @@ -2472,6 +2540,12 @@ export default class DataRequestController extends Controller { 'data access request', accessRecord._id ); + if (accessRecord.publisherObj.team.notifications[0].optIn) { + accessRecord.publisherObj.team.notifications[0].subscribedEmails.map(teamEmail => { + custodianManagers.push({email: teamEmail}); + }); + } + // 4. Create emails options = { id: accessRecord._id, @@ -2499,9 +2573,9 @@ export default class DataRequestController extends Controller { break; case constants.notificationTypes.WORKFLOWASSIGNED: // 1. Get managers for publisher - custodianManagers = teamController.getTeamMembersByRole(accessRecord.publisherObj.team.toObject(), constants.roleTypes.MANAGER); + custodianManagers = teamController.getTeamMembersByRole(accessRecord.publisherObj.team.toObject(), 'All'); // 2. Get managerIds for notifications - managerUserIds = custodianManagers.map(user => user.id); + custodianManagersIds = custodianManagers.map(user => user.id); // 3. deconstruct and set options for notifications and email options = { id: accessRecord._id, @@ -2516,11 +2590,16 @@ export default class DataRequestController extends Controller { }; // 4. Create notifications for the managers only await notificationBuilder.triggerNotificationMessage( - managerUserIds, + custodianManagersIds, `Workflow of ${workflowName} has been assiged to an appplication`, 'data access request', accessRecord._id ); + if (accessRecord.publisherObj.team.notifications[0].optIn) { + accessRecord.publisherObj.team.notifications[0].subscribedEmails.map(teamEmail => { + custodianManagers.push({email: teamEmail}); + }); + } // 5. Generate the email html = await emailGenerator.generateWorkflowAssigned(options); // 6. Send email to custodian managers only within the team @@ -2627,7 +2706,7 @@ export default class DataRequestController extends Controller { // Custodian notification if (_.has(accessRecord.datasets[0], 'publisher.team.users')) { // Retrieve all custodian user Ids to generate notifications - custodianManagers = teamController.getTeamMembersByRole(accessRecord.datasets[0].publisher.team, constants.roleTypes.MANAGER); + custodianManagers = teamController.getTeamMembersByRole(accessRecord.datasets[0].publisher.team, 'All'); custodianUserIds = custodianManagers.map(user => user.id); await notificationBuilder.triggerNotificationMessage( custodianUserIds, @@ -2635,6 +2714,21 @@ export default class DataRequestController extends Controller { 'data access request', accessRecord._id ); + } else if (_.has(accessRecord, 'publisherObj')) { + // Retrieve all custodian user Ids to generate notifications + custodianManagers = teamController.getTeamMembersByRole(accessRecord.publisherObj.team, 'All'); + custodianUserIds = custodianManagers.map(user => user.id); + await notificationBuilder.triggerNotificationMessage( + custodianUserIds, + `An amendment request has been submitted to ${projectId} by ${appFirstName} ${appLastName}`, + 'data access request', + accessRecord._id + ); + if (accessRecord.publisherObj.team.notifications[0].optIn) { + accessRecord.publisherObj.team.notifications[0].subscribedEmails.map(teamEmail => { + custodianManagers.push({email: teamEmail}); + }); + } } else { const dataCustodianEmail = process.env.DATA_CUSTODIAN_EMAIL || contactPoint; custodianManagers = [{ email: dataCustodianEmail }]; @@ -2712,13 +2806,17 @@ export default class DataRequestController extends Controller { break; case constants.notificationTypes.MESSAGESENT: if (userType === constants.userTypes.APPLICANT) { - const custodianManagers = teamController.getTeamMembersByRole(accessRecord.publisherObj.team, constants.roleTypes.MANAGER); + const custodianManagers = teamController.getTeamMembersByRole(accessRecord.publisherObj.team, 'All'); const custodianManagersIds = custodianManagers.map(user => user.id); - const custodianReviewers = teamController.getTeamMembersByRole(accessRecord.publisherObj.team, constants.roleTypes.REVIEWER); - const custodianReviewersIds = custodianManagers.map(user => user.id); + + if (accessRecord.publisherObj.team.notifications[0].optIn) { + accessRecord.publisherObj.team.notifications[0].subscribedEmails.map(teamEmail => { + custodianManagers.push({email: teamEmail}); + }); + } await notificationBuilder.triggerNotificationMessage( - [...custodianManagersIds, ...custodianReviewersIds, ...accessRecord.authors.map(author => author.id)], + [...custodianManagersIds, ...accessRecord.authors.map(author => author.id)], `There is a new message for the application ${projectName || datasetTitles} from ${user.firstname} ${user.lastname}`, 'data access message sent', accessRecord._id @@ -2735,15 +2833,24 @@ export default class DataRequestController extends Controller { }); await emailGenerator.sendEmail( - [...custodianManagers, ...custodianReviewers, ...accessRecord.authors], + [...custodianManagers, ...accessRecord.authors], constants.hdrukEmail, `There is a new message for the application ${projectName || datasetTitles} from ${user.firstname} ${user.lastname}`, html, false ); } else if (userType === constants.userTypes.CUSTODIAN) { + const custodianManagers = teamController.getTeamMembersByRole(accessRecord.publisherObj.team, 'All'); + const custodianManagersIds = custodianManagers.map(user => user.id); + + if (accessRecord.publisherObj.team.notifications[0].optIn) { + accessRecord.publisherObj.team.notifications[0].subscribedEmails.map(teamEmail => { + custodianManagers.push({email: teamEmail}); + }); + } + await notificationBuilder.triggerNotificationMessage( - [accessRecord.userId, ...accessRecord.authors.map(author => author.id)], + [accessRecord.userId, ...accessRecord.authors.map(author => author.id), ...custodianManagersIds], `There is a new message for the application ${projectName || datasetTitles} from ${user.firstname} ${user.lastname} from ${accessRecord.publisherObj.name}`, 'data access message sent', accessRecord._id @@ -2760,7 +2867,7 @@ export default class DataRequestController extends Controller { }); await emailGenerator.sendEmail( - [accessRecord.mainApplicant, ...accessRecord.authors], + [accessRecord.mainApplicant, ...accessRecord.authors, ...custodianManagers], constants.hdrukEmail, `There is a new message for the application ${projectName || datasetTitles} from ${user.firstname} ${user.lastname}`, html, diff --git a/src/resources/team/team.controller.js b/src/resources/team/team.controller.js index 531a12c1..2ef42da6 100644 --- a/src/resources/team/team.controller.js +++ b/src/resources/team/team.controller.js @@ -965,10 +965,41 @@ const checkIfAdmin = (user, adminRoles) => { }; const getTeamMembersByRole = (team, role) => { + let { members = [], users = [] } = team; + + let userIds = members.filter(mem => { + if (mem.roles.includes(role) || role === 'All') { + if(!_.has(mem, 'notifications')) { + return true; + } + + if (_.has(mem, 'notifications') && mem.notifications.length && mem.notifications[0].optIn) { + return true; + } + } + }).map(mem => mem.memberid.toString()); + + return users.filter(user => userIds.includes(user._id.toString())); +}; + + +const getTeamMembersNotifications = (team, filterArray) => { // Destructure members array and populated users array (populate 'users' must be included in the original Mongo query) let { members = [], users = [] } = team; // Get all userIds for role within team - let userIds = members.filter(mem => mem.roles.includes(role) || role === 'All').map(mem => mem.memberid.toString()); + let userIds = members.filter(mem => { + if(filterArray.includes(mem.memberid)) { + return false + } + + if(!_.has(mem, 'notifications')) { + return true; + } + + if (_.has(mem, 'notifications') && mem.notifications.length && mem.notifications[0].optIn) { + return true; + } + }).map(mem => mem.memberid.toString()); // return all user records for role return users.filter(user => userIds.includes(user._id.toString())); }; @@ -1293,6 +1324,7 @@ export default { deleteTeamMember: deleteTeamMember, checkTeamPermissions: checkTeamPermissions, getTeamMembersByRole: getTeamMembersByRole, + getTeamMembersNotifications: getTeamMembersNotifications, createNotifications: createNotifications, getTeamsList: getTeamsList, addTeam: addTeam, From b299722bc9f14c9d695b3ecf7e2c784f655aea82 Mon Sep 17 00:00:00 2001 From: Dan Nita Date: Mon, 9 May 2022 11:58:30 +0100 Subject: [PATCH 10/22] code updated for update and delete workflow --- src/resources/team/team.controller.js | 29 ++----- src/resources/utilities/constants.util.js | 2 + src/resources/workflow/workflow.controller.js | 83 +++++++++++++++---- src/resources/workflow/workflow.service.js | 62 ++++++++++---- 4 files changed, 123 insertions(+), 53 deletions(-) diff --git a/src/resources/team/team.controller.js b/src/resources/team/team.controller.js index 2ef42da6..c5a3e643 100644 --- a/src/resources/team/team.controller.js +++ b/src/resources/team/team.controller.js @@ -968,10 +968,16 @@ const getTeamMembersByRole = (team, role) => { let { members = [], users = [] } = team; let userIds = members.filter(mem => { + if (mem.roles.includes(role) || role === 'All') { + console.log(`mem getTeamMembersByRole 1 : ${JSON.stringify(mem)}\n`); if(!_.has(mem, 'notifications')) { return true; } + + if (_.has(mem, 'notifications') && !mem.notifications.length) { + return true; + } if (_.has(mem, 'notifications') && mem.notifications.length && mem.notifications[0].optIn) { return true; @@ -982,28 +988,6 @@ const getTeamMembersByRole = (team, role) => { return users.filter(user => userIds.includes(user._id.toString())); }; - -const getTeamMembersNotifications = (team, filterArray) => { - // Destructure members array and populated users array (populate 'users' must be included in the original Mongo query) - let { members = [], users = [] } = team; - // Get all userIds for role within team - let userIds = members.filter(mem => { - if(filterArray.includes(mem.memberid)) { - return false - } - - if(!_.has(mem, 'notifications')) { - return true; - } - - if (_.has(mem, 'notifications') && mem.notifications.length && mem.notifications[0].optIn) { - return true; - } - }).map(mem => mem.memberid.toString()); - // return all user records for role - return users.filter(user => userIds.includes(user._id.toString())); -}; - /** * Extract the name of a team from MongoDb object * @@ -1324,7 +1308,6 @@ export default { deleteTeamMember: deleteTeamMember, checkTeamPermissions: checkTeamPermissions, getTeamMembersByRole: getTeamMembersByRole, - getTeamMembersNotifications: getTeamMembersNotifications, createNotifications: createNotifications, getTeamsList: getTeamsList, addTeam: addTeam, diff --git a/src/resources/utilities/constants.util.js b/src/resources/utilities/constants.util.js index ecf9c574..0b775d52 100644 --- a/src/resources/utilities/constants.util.js +++ b/src/resources/utilities/constants.util.js @@ -152,6 +152,8 @@ const _notificationTypes = { MEMBERROLECHANGED: 'MemberRoleChanged', WORKFLOWASSIGNED: 'WorkflowAssigned', WORKFLOWCREATED: 'WorkflowCreated', + WORKFLOWUPDATED: 'WorkflowUpdated', + WORKFLOWDELETED: 'WorkflowDeleted', INPROGRESS: 'InProgress', APPLICATIONCLONED: 'ApplicationCloned', APPLICATIONDELETED: 'ApplicationDeleted', diff --git a/src/resources/workflow/workflow.controller.js b/src/resources/workflow/workflow.controller.js index f479a4c3..a48b6b7c 100644 --- a/src/resources/workflow/workflow.controller.js +++ b/src/resources/workflow/workflow.controller.js @@ -168,7 +168,7 @@ export default class WorkflowController extends Controller { async updateWorkflow(req, res) { try { - const { _id: userId } = req.user; + const { _id: userId, firstname, lastname } = req.user; const { id: workflowId } = req.params; // 1. Look up workflow let workflow = await WorkflowModel.findOne({ @@ -202,7 +202,7 @@ export default class WorkflowController extends Controller { }); } // 5. Edit workflow - const { workflowName = '', steps = [] } = req.body; + const { workflowName = '', publisher = '', steps = [] } = req.body; let isDirty = false; // Check if workflow name updated if (!_.isEmpty(workflowName)) { @@ -214,21 +214,45 @@ export default class WorkflowController extends Controller { isDirty = true; } // Perform save if changes have been made if (isDirty) { - workflow.save(async err => { + workflow = await workflow.save().catch(err => { if (err) { - console.error(err.message); return res.status(400).json({ success: false, message: err.message, }); - } else { - // 7. Return workflow payload - return res.status(204).json({ - success: true, - workflow, - }); } }); + + const publisherObj = await PublisherModel.findOne({ + _id: publisher, + }).populate({ + path: 'team members', + populate: { + path: 'users', + select: '_id id email firstname lastname', + }, + }); + if (!publisherObj) { + return res.status(400).json({ + success: false, + message: 'You must supply a valid publisher to create the workflow against', + }); + } + const detailedWorkflow = await WorkflowModel.findById(workflow._id).populate({ + path: 'steps.reviewers', + select: 'firstname lastname email -_id', + }).lean(); + let context = { + publisherObj: publisherObj.team.toObject(), + actioner: `${firstname} ${lastname}`, + workflow: detailedWorkflow, + }; + this.workflowService.createNotifications(context, constants.notificationTypes.WORKFLOWUPDATED); + + return res.status(204).json({ + success: true, + workflow, + }); } else { return res.status(200).json({ success: true, @@ -245,7 +269,7 @@ export default class WorkflowController extends Controller { async deleteWorkflow(req, res) { try { - const { _id: userId } = req.user; + const { _id: userId, firstname, lastname } = req.user; const { id: workflowId } = req.params; // 1. Look up workflow const workflow = await WorkflowModel.findOne({ @@ -258,6 +282,8 @@ export default class WorkflowController extends Controller { select: 'members -_id', }, }); + const { workflowName = '', publisher = {}, steps = [] } = workflow; + if (!workflow) { return res.status(404).json({ success: false }); } @@ -278,6 +304,11 @@ export default class WorkflowController extends Controller { message: 'A workflow which is attached to applications currently in review cannot be deleted', }); } + const detailedWorkflow = await WorkflowModel.findById(workflowId).populate({ + path: 'steps.reviewers', + select: 'firstname lastname email -_id', + }).lean(); + // 5. Delete workflow WorkflowModel.deleteOne({ _id: workflowId }, function (err) { if (err) { @@ -286,13 +317,33 @@ export default class WorkflowController extends Controller { success: false, message: 'An error occurred deleting the workflow', }); - } else { - // 7. Return workflow payload - return res.status(204).json({ - success: true, - }); } }); + const publisherObj = await PublisherModel.findOne({ + _id: publisher._id, + }).populate({ + path: 'team members', + populate: { + path: 'users', + select: '_id id email firstname lastname', + }, + }); + if (!publisherObj) { + return res.status(400).json({ + success: false, + message: 'You must supply a valid publisher to create the workflow against', + }); + } + let context = { + publisherObj: publisherObj.team.toObject(), + actioner: `${firstname} ${lastname}`, + workflow: detailedWorkflow, + }; + this.workflowService.createNotifications(context, constants.notificationTypes.WORKFLOWDELETED); + + return res.status(204).json({ + success: true, + }); } catch (err) { console.error(err.message); return res.status(500).json({ diff --git a/src/resources/workflow/workflow.service.js b/src/resources/workflow/workflow.service.js index 036b224b..4a242ddd 100644 --- a/src/resources/workflow/workflow.service.js +++ b/src/resources/workflow/workflow.service.js @@ -162,23 +162,27 @@ export default class WorkflowService { // deconstruct context let { publisherObj, workflow = {}, actioner = '' } = context; + console.log(`publisherObj createNotifications : ${JSON.stringify(publisherObj)}\n`); + custodianManagers = teamController.getTeamMembersByRole(publisherObj, 'All'); + if (publisherObj.notifications[0].optIn) { + publisherObj.notifications[0].subscribedEmails.map(teamEmail => { + custodianManagers.push({email: teamEmail}); + }); + } + managerUserIds = custodianManagers.map(user => user.id); + let { workflowName = 'Workflow Title', _id, steps, createdAt } = workflow; + options = { + actioner, + workflowName, + _id, + steps, + createdAt, + }; + // switch over types switch (type) { case constants.notificationTypes.WORKFLOWCREATED: // 1. Get managers for publisher - custodianManagers = teamController.getTeamMembersByRole(publisherObj, constants.roleTypes.MANAGER); - // 2. Get managerIds for notifications - managerUserIds = custodianManagers.map(user => user.id); - // 3. deconstruct workflow - let { workflowName = 'Workflow Title', _id, steps, createdAt } = workflow; - // 4. setup options - options = { - actioner, - workflowName, - _id, - steps, - createdAt, - }; // 4. Create notifications for the managers only await notificationBuilder.triggerNotificationMessage( managerUserIds, @@ -191,7 +195,37 @@ export default class WorkflowService { // 6. Send email to custodian managers only within the team await emailGenerator.sendEmail(custodianManagers, constants.hdrukEmail, `A Workflow has been created`, html, false); break; - } + + case constants.notificationTypes.WORKFLOWUPDATED: + // 1. Get managers for publisher + // 4. Create notifications for the managers only + await notificationBuilder.triggerNotificationMessage( + managerUserIds, + `A new workflow of ${workflowName} has been updated`, + 'workflow', + _id + ); + // 5. Generate the email + html = await emailGenerator.generateWorkflowCreated(options); + // 6. Send email to custodian managers only within the team + await emailGenerator.sendEmail(custodianManagers, constants.hdrukEmail, `A Workflow has been updated`, html, false); + break; + + case constants.notificationTypes.WORKFLOWDELETED: + // 1. Get managers for publisher + // 4. Create notifications for the managers only + await notificationBuilder.triggerNotificationMessage( + managerUserIds, + `A new workflow of ${workflowName} has been deleted`, + 'workflow', + _id + ); + // 5. Generate the email + html = await emailGenerator.generateWorkflowCreated(options); + // 6. Send email to custodian managers only within the team + await emailGenerator.sendEmail(custodianManagers, constants.hdrukEmail, `A Workflow has been deleted`, html, false); + break; + } } } From 7268e9c25acd3c5196df738a07c5feebda2f7426 Mon Sep 17 00:00:00 2001 From: Dan Nita Date: Mon, 9 May 2022 14:09:01 +0100 Subject: [PATCH 11/22] removed console.log --- src/resources/team/team.controller.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/resources/team/team.controller.js b/src/resources/team/team.controller.js index c5a3e643..b046623a 100644 --- a/src/resources/team/team.controller.js +++ b/src/resources/team/team.controller.js @@ -968,9 +968,7 @@ const getTeamMembersByRole = (team, role) => { let { members = [], users = [] } = team; let userIds = members.filter(mem => { - if (mem.roles.includes(role) || role === 'All') { - console.log(`mem getTeamMembersByRole 1 : ${JSON.stringify(mem)}\n`); if(!_.has(mem, 'notifications')) { return true; } From a064565410a1240d1205257c80b6b6c4594f786f Mon Sep 17 00:00:00 2001 From: Dan Nita Date: Mon, 9 May 2022 14:09:25 +0100 Subject: [PATCH 12/22] removed console.log --- src/resources/workflow/workflow.service.js | 1 - 1 file changed, 1 deletion(-) diff --git a/src/resources/workflow/workflow.service.js b/src/resources/workflow/workflow.service.js index 4a242ddd..c0b4a6ef 100644 --- a/src/resources/workflow/workflow.service.js +++ b/src/resources/workflow/workflow.service.js @@ -162,7 +162,6 @@ export default class WorkflowService { // deconstruct context let { publisherObj, workflow = {}, actioner = '' } = context; - console.log(`publisherObj createNotifications : ${JSON.stringify(publisherObj)}\n`); custodianManagers = teamController.getTeamMembersByRole(publisherObj, 'All'); if (publisherObj.notifications[0].optIn) { publisherObj.notifications[0].subscribedEmails.map(teamEmail => { From 8f8739d661cc0d867653aee7e1e94f4d49696a4e Mon Sep 17 00:00:00 2001 From: Dan Nita Date: Mon, 9 May 2022 16:32:24 +0100 Subject: [PATCH 13/22] update for cohort test --- .../__tests__/getCollaboratorsCohorts.test.js | 35 ------------------- 1 file changed, 35 deletions(-) delete mode 100644 src/resources/user/__tests__/getCollaboratorsCohorts.test.js diff --git a/src/resources/user/__tests__/getCollaboratorsCohorts.test.js b/src/resources/user/__tests__/getCollaboratorsCohorts.test.js deleted file mode 100644 index bb626688..00000000 --- a/src/resources/user/__tests__/getCollaboratorsCohorts.test.js +++ /dev/null @@ -1,35 +0,0 @@ -import dbHandler from '../../../config/in-memory-db'; -import {mockCohorts} from '../__mocks__/cohorts.data'; - -const {getCollaboratorsCohorts} = require('../user.service'); - - -beforeAll(async () => { - await dbHandler.connect(); - await dbHandler.loadData({ cohorts: mockCohorts }); -}); - -afterAll(async () => { - await dbHandler.clearDatabase(); - await dbHandler.closeDatabase(); -}); - -describe('getCollaboratorsCohorts tests', () => { - it('should return values', async () => { - const currentUserId = 8470291714590257; - const filter = currentUserId ? { uploaders: currentUserId } : {}; - - const result = await getCollaboratorsCohorts(filter, currentUserId); - expect(result.length > 0).toBe(true); - expect(typeof result).toBe('object'); - }); - - it('should return values', async () => { - const currentUserId = null; - const filter = currentUserId ? { uploaders: currentUserId } : {}; - - const result = await getCollaboratorsCohorts(filter, currentUserId); - expect(result.length > 0).toBe(true); - expect(typeof result).toBe('object'); - }); -}); \ No newline at end of file From 8e24a1376d6d4db6d16ce8768344da24d2257bdf Mon Sep 17 00:00:00 2001 From: Dan Nita Date: Thu, 12 May 2022 10:41:55 +0100 Subject: [PATCH 14/22] added for non-production mailtrap --- package.json | 1 + .../utilities/emailGenerator.util.js | 119 +++++++++++++----- 2 files changed, 92 insertions(+), 28 deletions(-) diff --git a/package.json b/package.json index ff15fcf1..6fcaae1a 100644 --- a/package.json +++ b/package.json @@ -53,6 +53,7 @@ "mongoose": "^5.12.7", "morgan": "^1.10.0", "multer": "^1.4.2", + "nodemailer": "^6.7.5", "oidc-provider": "^6.29.3", "passport": "^0.4.1", "passport-azure-ad-oauth2": "0.0.4", diff --git a/src/resources/utilities/emailGenerator.util.js b/src/resources/utilities/emailGenerator.util.js index fd946a82..7c032157 100644 --- a/src/resources/utilities/emailGenerator.util.js +++ b/src/resources/utilities/emailGenerator.util.js @@ -1,3 +1,4 @@ +require('dotenv').config(); import _, { isNil, isEmpty, capitalize, groupBy, forEach, isEqual } from 'lodash'; import moment from 'moment'; import { UserModel } from '../user/user.model'; @@ -7,12 +8,27 @@ import * as Sentry from '@sentry/node'; import wordTemplateBuilder from '../utilities/wordTemplateBuilder.util'; const fs = require('fs'); +const nodemailer = require('nodemailer'); const sgMail = require('@sendgrid/mail'); -const readEnv = process.env.ENV || 'prod'; +const readEnv = process.env.ENV || 'production'; + let parent, qsId; let questionList = []; let excludedQuestionSetIds = ['addRepeatableSection', 'removeRepeatableSection']; let autoCompleteLookups = { fullname: ['email'] }; +let transporterOptions = { + host: process.env.MAIL_HOST, + port: process.env.MAIL_PORT, + auth: { + user: process.env.MAIL_USERNAME, + pass: process.env.MAIL_PASSWORD, + }, + pool: true, + maxConnections: 1, + rateDelta: 20000, + rateLimit: 5, +}; +let transporter = nodemailer.createTransport(transporterOptions); const _getStepReviewers = (reviewers = []) => { if (!isEmpty(reviewers)) return [...reviewers].map(reviewer => `${reviewer.firstname} ${reviewer.lastname}`).join(', '); @@ -2508,15 +2524,17 @@ const _getRecipients = (recipients, environment, genericEmail) => { */ const _sendEmail = async (to, from, subject, html, allowUnsubscribe = true, attachments = []) => { // 1. Apply SendGrid API key from environment variable - sgMail.setApiKey(process.env.SENDGRID_API_KEY); + if (process.env.NODE_ENV === 'production') { + sgMail.setApiKey(process.env.SENDGRID_API_KEY); + } // 2. Ensure any duplicates recieve only a single email - const recipients = _getRecipients(to, process.env.NODE_ENV, process.env.GENERIC_EMAIL); - + // const recipients = _getRecipients(to, process.env.NODE_ENV, process.env.GENERIC_EMAIL); + const recipients = [...new Map(to.map(item => [item['email'], item])).values()]; // 3. Build each email object for SendGrid extracting email addresses from user object with unique unsubscribe link (to) for (let recipient of recipients) { let body = _generateEmailHeader + html + _generateEmailFooter(recipient, allowUnsubscribe); - let msg = { + let message = { to: recipient.email, from: from, subject: subject, @@ -2524,20 +2542,71 @@ const _sendEmail = async (to, from, subject, html, allowUnsubscribe = true, atta attachments, }; - // 4. Send email using SendGrid - await sgMail.send(msg, false, err => { - if (err && process.env.NODE_ENV === 'production') { - Sentry.addBreadcrumb({ - category: 'SendGrid', - message: 'Sending email failed', - level: Sentry.Severity.Warning, - }); - Sentry.captureException(err); - } - }); + // 4. Send email + if (process.env.NODE_ENV !== 'production') { + try { + await transporter.sendMail(message, (error, info) => { + if (error) { + return console.log(error); + } + console.log('Email sent: ' + info.response); + }); + } catch (error) { + console.error(error.response.body); + Sentry.addBreadcrumb({ + category: 'SendGrid', + message: 'Sending email failed', + level: Sentry.Severity.Warning, + }); + Sentry.captureException(error); + } + } else { + await sgMail.send(message, false, err => { + if (err && process.env.NODE_ENV === 'production') { + Sentry.addBreadcrumb({ + category: 'SendGrid', + message: 'Sending email failed', + level: Sentry.Severity.Warning, + }); + Sentry.captureException(err); + } + }); + } } }; +const _sendEmailSmtp = async (message) => { + if (process.env.NODE_ENV !== 'production') { + try { + await transporter.sendMail(message, (error, info) => { + if (error) { + return console.log(error); + } + console.log('Email sent: ' + info.response); + }); + } catch (error) { + console.error(error.response.body); + Sentry.addBreadcrumb({ + category: 'SendGrid', + message: 'Sending email failed', + level: Sentry.Severity.Warning, + }); + Sentry.captureException(error); + } + } else { + await sgMail.send(message, false, err => { + if (err && process.env.NODE_ENV === 'production') { + Sentry.addBreadcrumb({ + category: 'SendGrid', + message: 'Sending email failed', + level: Sentry.Severity.Warning, + }); + Sentry.captureException(err); + } + }); + } +} + /** * [_sendIntroEmail] * @@ -2546,18 +2615,12 @@ const _sendEmail = async (to, from, subject, html, allowUnsubscribe = true, atta */ const _sendIntroEmail = msg => { // 1. Apply SendGrid API key from environment variable - sgMail.setApiKey(process.env.SENDGRID_API_KEY); - // 2. Send email using SendGrid - sgMail.send(msg, false, err => { - if (err && process.env.NODE_ENV === 'production') { - Sentry.addBreadcrumb({ - category: 'SendGrid', - message: 'Sending email failed - Intro', - level: Sentry.Severity.Warning, - }); - Sentry.captureException(err); - } - }); + if (process.env.NODE_ENV === 'production') { + sgMail.setApiKey(process.env.SENDGRID_API_KEY); + } + + // 2. Send email + _sendEmailSmtp(msg); }; const _generateEmailHeader = ` From 052b8a232d37f76b268832f4b51ce1658e8f9246 Mon Sep 17 00:00:00 2001 From: Dan Nita Date: Thu, 12 May 2022 11:05:47 +0100 Subject: [PATCH 15/22] update - using smtp for all envs --- .../utilities/emailGenerator.util.js | 79 +++++-------------- 1 file changed, 18 insertions(+), 61 deletions(-) diff --git a/src/resources/utilities/emailGenerator.util.js b/src/resources/utilities/emailGenerator.util.js index 7c032157..651c130a 100644 --- a/src/resources/utilities/emailGenerator.util.js +++ b/src/resources/utilities/emailGenerator.util.js @@ -2512,10 +2512,6 @@ ${_displayDataUseRegisterDashboardLink()} return body; }; -const _getRecipients = (recipients, environment, genericEmail) => { - return environment === 'production' ? [...new Map(recipients.map(item => [item['email'], item])).values()] : [{ email: genericEmail }]; -}; - /** * [_sendEmail] * @@ -2523,13 +2519,6 @@ const _getRecipients = (recipients, environment, genericEmail) => { * @param {Object} context */ const _sendEmail = async (to, from, subject, html, allowUnsubscribe = true, attachments = []) => { - // 1. Apply SendGrid API key from environment variable - if (process.env.NODE_ENV === 'production') { - sgMail.setApiKey(process.env.SENDGRID_API_KEY); - } - - // 2. Ensure any duplicates recieve only a single email - // const recipients = _getRecipients(to, process.env.NODE_ENV, process.env.GENERIC_EMAIL); const recipients = [...new Map(to.map(item => [item['email'], item])).values()]; // 3. Build each email object for SendGrid extracting email addresses from user object with unique unsubscribe link (to) for (let recipient of recipients) { @@ -2543,40 +2532,6 @@ const _sendEmail = async (to, from, subject, html, allowUnsubscribe = true, atta }; // 4. Send email - if (process.env.NODE_ENV !== 'production') { - try { - await transporter.sendMail(message, (error, info) => { - if (error) { - return console.log(error); - } - console.log('Email sent: ' + info.response); - }); - } catch (error) { - console.error(error.response.body); - Sentry.addBreadcrumb({ - category: 'SendGrid', - message: 'Sending email failed', - level: Sentry.Severity.Warning, - }); - Sentry.captureException(error); - } - } else { - await sgMail.send(message, false, err => { - if (err && process.env.NODE_ENV === 'production') { - Sentry.addBreadcrumb({ - category: 'SendGrid', - message: 'Sending email failed', - level: Sentry.Severity.Warning, - }); - Sentry.captureException(err); - } - }); - } - } -}; - -const _sendEmailSmtp = async (message) => { - if (process.env.NODE_ENV !== 'production') { try { await transporter.sendMail(message, (error, info) => { if (error) { @@ -2593,17 +2548,26 @@ const _sendEmailSmtp = async (message) => { }); Sentry.captureException(error); } - } else { - await sgMail.send(message, false, err => { - if (err && process.env.NODE_ENV === 'production') { - Sentry.addBreadcrumb({ - category: 'SendGrid', - message: 'Sending email failed', - level: Sentry.Severity.Warning, - }); - Sentry.captureException(err); + + } +}; + +const _sendEmailSmtp = async (message) => { + try { + await transporter.sendMail(message, (error, info) => { + if (error) { + return console.log(error); } + console.log('Email sent: ' + info.response); + }); + } catch (error) { + console.error(error.response.body); + Sentry.addBreadcrumb({ + category: 'SendGrid', + message: 'Sending email failed', + level: Sentry.Severity.Warning, }); + Sentry.captureException(error); } } @@ -2614,12 +2578,6 @@ const _sendEmailSmtp = async (message) => { * @param {Object} message to from, templateId */ const _sendIntroEmail = msg => { - // 1. Apply SendGrid API key from environment variable - if (process.env.NODE_ENV === 'production') { - sgMail.setApiKey(process.env.SENDGRID_API_KEY); - } - - // 2. Send email _sendEmailSmtp(msg); }; @@ -2751,5 +2709,4 @@ export default { generateDataUseRegisterApproved: _generateDataUseRegisterApproved, generateDataUseRegisterRejected: _generateDataUseRegisterRejected, generateDataUseRegisterPending: _generateDataUseRegisterPending, - getRecipients: _getRecipients, }; From c284ff712edc6a7864896a6ef13bbe037c4186d1 Mon Sep 17 00:00:00 2001 From: Dan Nita Date: Thu, 12 May 2022 11:13:21 +0100 Subject: [PATCH 16/22] update - using smtp for all envs --- src/resources/utilities/emailGenerator.util.js | 1 - 1 file changed, 1 deletion(-) diff --git a/src/resources/utilities/emailGenerator.util.js b/src/resources/utilities/emailGenerator.util.js index 651c130a..0ad3a044 100644 --- a/src/resources/utilities/emailGenerator.util.js +++ b/src/resources/utilities/emailGenerator.util.js @@ -9,7 +9,6 @@ import wordTemplateBuilder from '../utilities/wordTemplateBuilder.util'; const fs = require('fs'); const nodemailer = require('nodemailer'); -const sgMail = require('@sendgrid/mail'); const readEnv = process.env.ENV || 'production'; let parent, qsId; From 357bfba33a6b21997f425b29a293f70526969818 Mon Sep 17 00:00:00 2001 From: Dan Nita Date: Mon, 16 May 2022 12:05:47 +0100 Subject: [PATCH 17/22] send custom notification to creator --- src/resources/message/message.controller.js | 19 +++++++++- .../utilities/emailGenerator.util.js | 37 +++++++++++++++++++ 2 files changed, 54 insertions(+), 2 deletions(-) diff --git a/src/resources/message/message.controller.js b/src/resources/message/message.controller.js index a230e18a..e1a82093 100644 --- a/src/resources/message/message.controller.js +++ b/src/resources/message/message.controller.js @@ -104,7 +104,7 @@ module.exports = { // 15. Prepare to send email if a new message has been created if (messageType === 'message') { - let optIn, subscribedEmails; + let optIn, subscribedEmails, messageCreatorRecipient; // 16. Find recipients who have opted in to email updates and exclude the requesting user let messageRecipients = await UserModel.find({ _id: { $in: topicObj.recipients } }); @@ -125,9 +125,12 @@ module.exports = { ); if (!_.isEmpty(subscribedMembersByType)) { // build cleaner array of memberIds from subscribedMembersByType - const memberIds = [...subscribedMembersByType.map(m => m.memberid.toString()), topicObj.createdBy.toString()]; + const memberIds = [...subscribedMembersByType.map(m => m.memberid.toString())].filter(ele => ele !== topicObj.createdBy.toString()); + const creatorObjectId = topicObj.createdBy.toString(); // returns array of objects [{email: 'email@email.com '}] for members in subscribed emails users is list of full user object const { memberEmails } = teamController.getMemberDetails([...memberIds], [...messageRecipients]); + const creatorEmail = await UserModel.findById(creatorObjectId); + messageCreatorRecipient = [{ email: creatorEmail.email}]; messageRecipients = [...teamNotificationEmails, ...memberEmails]; } else { // only if not membersByType but has a team email setup @@ -155,6 +158,18 @@ module.exports = { html, false ); + + if (messageCreatorRecipient) { + let htmlCreator = emailGenerator.generateMessageCreatorNotification(options); + + emailGenerator.sendEmail( + messageCreatorRecipient, + constants.hdrukEmail, + `You have received a new message on the HDR UK Innovation Gateway`, + htmlCreator, + false + ); + } } // 19. Return successful response with message data const messageObj = message.toObject(); diff --git a/src/resources/utilities/emailGenerator.util.js b/src/resources/utilities/emailGenerator.util.js index 0ad3a044..ecc44db9 100644 --- a/src/resources/utilities/emailGenerator.util.js +++ b/src/resources/utilities/emailGenerator.util.js @@ -2262,6 +2262,42 @@ const _generateMessageNotification = options => { return body; }; +const _generateMessageCreatorNotification = options => { + let { firstMessage, firstname, lastname, messageDescription, openMessagesLink } = options; + + let body = `
+
+
Thank you for submitting ${name}, which has been reviewed by the team at HDR UK. The dataset version cannot be approved for release on the Gateway at this time. Please look at the comment from the reviewer below and make any necessary changes on a new version of the dataset before resubmitting.
+ + + + + + + + + + +
+ Data Access Enquiry submitted +
+

Dear ${firstname} ${lastname},

+

Thank you for submitting an enquiry about ${firstMessage.datasetsRequested[0].name}.

+

Your enquiry has been sent to ${firstMessage.datasetsRequested[0].publisher} who will reply in due course. If you have not received a response after 10 working days, or if you have any queries or concerns about the Gateway, please email enquiries@hdruk.ac.uk and a member of the HDR UK team will get in touch with you.

+

${messageDescription.replace(/\n/g, '
')}

+
+
+ `; + return body; +}; + const _generateEntityNotification = options => { let { resourceType, resourceName, resourceLink, subject, rejectionReason, activeflag, type, resourceAuthor } = options; let authorBody; @@ -2700,6 +2736,7 @@ export default { //generateMetadataOnboardingUnArchived: _generateMetadataOnboardingUnArchived, //Messages generateMessageNotification: _generateMessageNotification, + generateMessageCreatorNotification: _generateMessageCreatorNotification, generateEntityNotification: _generateEntityNotification, //ActivityLog generateActivityLogManualEventCreated: _generateActivityLogManualEventCreated, From 252a75d1746c6c60c79ec570adffdaa431c1bb7d Mon Sep 17 00:00:00 2001 From: Dan Nita Date: Tue, 17 May 2022 12:14:50 +0100 Subject: [PATCH 18/22] update notifications for second message --- src/resources/message/message.controller.js | 25 +++++++++++++++------ 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/src/resources/message/message.controller.js b/src/resources/message/message.controller.js index e1a82093..823360f2 100644 --- a/src/resources/message/message.controller.js +++ b/src/resources/message/message.controller.js @@ -125,13 +125,24 @@ module.exports = { ); if (!_.isEmpty(subscribedMembersByType)) { // build cleaner array of memberIds from subscribedMembersByType - const memberIds = [...subscribedMembersByType.map(m => m.memberid.toString())].filter(ele => ele !== topicObj.createdBy.toString()); - const creatorObjectId = topicObj.createdBy.toString(); - // returns array of objects [{email: 'email@email.com '}] for members in subscribed emails users is list of full user object - const { memberEmails } = teamController.getMemberDetails([...memberIds], [...messageRecipients]); - const creatorEmail = await UserModel.findById(creatorObjectId); - messageCreatorRecipient = [{ email: creatorEmail.email}]; - messageRecipients = [...teamNotificationEmails, ...memberEmails]; + console.log(`topicObj : ${JSON.stringify(topicObj)}`); + console.log(`topicObj.createdBy : ${JSON.stringify(topicObj.createdBy)}`); + console.log(`topicObj.createdBy 2 : ${JSON.stringify(typeof topicObj.createdBy === 'object')}`); + if (topicObj.topicMessages !== undefined) { + const memberIds = [...subscribedMembersByType.map(m => m.memberid.toString()), ...topicObj.createdBy._id.toString()]; + // returns array of objects [{email: 'email@email.com '}] for members in subscribed emails users is list of full user object + const { memberEmails } = teamController.getMemberDetails([...memberIds], [...messageRecipients]); + messageRecipients = [...teamNotificationEmails, ...memberEmails]; + } else { + const memberIds = [...subscribedMembersByType.map(m => m.memberid.toString())].filter(ele => ele !== topicObj.createdBy.toString()); + const creatorObjectId = topicObj.createdBy.toString(); + // returns array of objects [{email: 'email@email.com '}] for members in subscribed emails users is list of full user object + const { memberEmails } = teamController.getMemberDetails([...memberIds], [...messageRecipients]); + const creatorEmail = await UserModel.findById(creatorObjectId); + messageCreatorRecipient = [{ email: creatorEmail.email}]; + messageRecipients = [...teamNotificationEmails, ...memberEmails]; + } + } else { // only if not membersByType but has a team email setup messageRecipients = [...messageRecipients, ...teamNotificationEmails]; From 559c1bc1939739582f51e2f6e1c90b6f3253d827 Mon Sep 17 00:00:00 2001 From: Dan Nita Date: Tue, 17 May 2022 12:15:09 +0100 Subject: [PATCH 19/22] update notifications for second message --- src/resources/message/message.controller.js | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/resources/message/message.controller.js b/src/resources/message/message.controller.js index 823360f2..9ef6f337 100644 --- a/src/resources/message/message.controller.js +++ b/src/resources/message/message.controller.js @@ -125,9 +125,6 @@ module.exports = { ); if (!_.isEmpty(subscribedMembersByType)) { // build cleaner array of memberIds from subscribedMembersByType - console.log(`topicObj : ${JSON.stringify(topicObj)}`); - console.log(`topicObj.createdBy : ${JSON.stringify(topicObj.createdBy)}`); - console.log(`topicObj.createdBy 2 : ${JSON.stringify(typeof topicObj.createdBy === 'object')}`); if (topicObj.topicMessages !== undefined) { const memberIds = [...subscribedMembersByType.map(m => m.memberid.toString()), ...topicObj.createdBy._id.toString()]; // returns array of objects [{email: 'email@email.com '}] for members in subscribed emails users is list of full user object From 461dbeba1a1e5b2cb2ad2ec8dbdb0f325f5a1797 Mon Sep 17 00:00:00 2001 From: Dan Nita Date: Tue, 17 May 2022 13:36:22 +0100 Subject: [PATCH 20/22] update send for email team --- .../dataUseRegister/dataUseRegister.controller.js | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/resources/dataUseRegister/dataUseRegister.controller.js b/src/resources/dataUseRegister/dataUseRegister.controller.js index f2074714..4ce28683 100644 --- a/src/resources/dataUseRegister/dataUseRegister.controller.js +++ b/src/resources/dataUseRegister/dataUseRegister.controller.js @@ -6,6 +6,7 @@ import constants from './../utilities/constants.util'; import { Data } from '../tool/data.model'; import { Course } from '../course/course.model'; import { TeamModel } from '../team/team.model'; +import { PublisherModel } from '../publisher/publisher.model'; import teamController from '../team/team.controller'; import emailGenerator from '../utilities/emailGenerator.util'; import { getObjectFilters } from '../search/search.repository'; @@ -371,13 +372,20 @@ export default class DataUseRegisterController extends Controller { switch (type) { case constants.dataUseRegisterNotifications.DATAUSEAPPROVED: { + let teamEmailNotification = []; const adminTeam = await TeamModel.findOne({ type: 'admin' }) .populate({ path: 'users', }) .lean(); + const team = await TeamModel.findById(dataUseRegister.publisher.toString()); + if (team.notifications.length > 0 && team.notifications[0].optIn) { + team.notifications[0].subscribedEmails.map(teamEmail => { + teamEmailNotification.push({email: teamEmail}); + }); + } const dataUseTeamMembers = teamController.getTeamMembersByRole(adminTeam, constants.roleTypes.ADMIN_DATA_USE); - const emailRecipients = [...dataUseTeamMembers, uploader]; + const emailRecipients = [...dataUseTeamMembers, uploader, ...teamEmailNotification]; const options = { id, From 211d3440de8e712c7107780f76d5ec7a8d1c4a5e Mon Sep 17 00:00:00 2001 From: Dan Nita Date: Tue, 17 May 2022 13:43:06 +0100 Subject: [PATCH 21/22] removed not used lines --- src/resources/dataUseRegister/dataUseRegister.controller.js | 1 - 1 file changed, 1 deletion(-) diff --git a/src/resources/dataUseRegister/dataUseRegister.controller.js b/src/resources/dataUseRegister/dataUseRegister.controller.js index 4ce28683..87c52a6e 100644 --- a/src/resources/dataUseRegister/dataUseRegister.controller.js +++ b/src/resources/dataUseRegister/dataUseRegister.controller.js @@ -6,7 +6,6 @@ import constants from './../utilities/constants.util'; import { Data } from '../tool/data.model'; import { Course } from '../course/course.model'; import { TeamModel } from '../team/team.model'; -import { PublisherModel } from '../publisher/publisher.model'; import teamController from '../team/team.controller'; import emailGenerator from '../utilities/emailGenerator.util'; import { getObjectFilters } from '../search/search.repository'; From 79c1307ea5605efa9e8876ba8444b3ed51a7ad74 Mon Sep 17 00:00:00 2001 From: Callum Reekie Date: Tue, 17 May 2022 17:50:44 +0100 Subject: [PATCH 22/22] fix error when subbmitting dar to federated custodians --- src/resources/datarequest/datarequest.controller.js | 4 ++-- src/resources/publisher/publisher.model.js | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/resources/datarequest/datarequest.controller.js b/src/resources/datarequest/datarequest.controller.js index a83ff5ec..052927c7 100644 --- a/src/resources/datarequest/datarequest.controller.js +++ b/src/resources/datarequest/datarequest.controller.js @@ -2177,8 +2177,8 @@ export default class DataRequestController extends Controller { options )); // Get the name of the publishers word template - let publisherTemplate = await PublisherModel.findOne({ name: publisher }, { wordTemplate: 1, _id: 0 }).lean(); - let templateName = publisherTemplate.wordTemplate; + let publisherTemplate = await PublisherModel.findOne({ name: publisher }, { _id: 0 }).lean(); + let templateName = publisherTemplate.wordTemplate; // Send emails to custodian team members who have opted in to email notifications if (emailRecipientType === 'dataCustodian') { emailRecipients = [...custodianManagers]; diff --git a/src/resources/publisher/publisher.model.js b/src/resources/publisher/publisher.model.js index 8ba20629..4dbe68c7 100644 --- a/src/resources/publisher/publisher.model.js +++ b/src/resources/publisher/publisher.model.js @@ -47,7 +47,7 @@ const PublisherSchema = new Schema( federation: { active: { type: Boolean }, auth: { type: Object, select: false }, - endpoints: { type: Boolean, select: false }, + endpoints: { type: Object, select: false }, notificationEmail: { type: Array, select: false }, }, },