From 8873281db54b43a66a2f32485bf90436eb3843ae Mon Sep 17 00:00:00 2001 From: massimocianfroccaPA Date: Thu, 9 Sep 2021 11:24:45 +0100 Subject: [PATCH 001/116] add repository patterns components for data use register --- src/config/server.js | 1 + .../dataUseRegister.controller.js | 88 +++++++++++++++++++ .../dataUseRegister/dataUseRegister.entity.js | 8 ++ .../dataUseRegister/dataUseRegister.model.js | 28 ++++++ .../dataUseRegister.repository.js | 19 ++++ .../dataUseRegister/dataUseRegister.route.js | 57 ++++++++++++ .../dataUseRegister.service.js | 25 ++++++ src/resources/dataUseRegister/dependency.js | 5 ++ src/resources/utilities/constants.util.js | 8 ++ 9 files changed, 239 insertions(+) create mode 100644 src/resources/dataUseRegister/dataUseRegister.controller.js create mode 100644 src/resources/dataUseRegister/dataUseRegister.entity.js create mode 100644 src/resources/dataUseRegister/dataUseRegister.model.js create mode 100644 src/resources/dataUseRegister/dataUseRegister.repository.js create mode 100644 src/resources/dataUseRegister/dataUseRegister.route.js create mode 100644 src/resources/dataUseRegister/dataUseRegister.service.js create mode 100644 src/resources/dataUseRegister/dependency.js diff --git a/src/config/server.js b/src/config/server.js index c7ed035e..15cea342 100644 --- a/src/config/server.js +++ b/src/config/server.js @@ -258,6 +258,7 @@ app.use('/api/v1/cohortprofiling', require('../resources/cohortprofiling/cohortp app.use('/api/v1/search-preferences', require('../resources/searchpreferences/searchpreferences.route')); +app.use('/api/v2/data-use-registers', require('../resources/datauseregister/datauseregister.route')); initialiseAuthentication(app); diff --git a/src/resources/dataUseRegister/dataUseRegister.controller.js b/src/resources/dataUseRegister/dataUseRegister.controller.js new file mode 100644 index 00000000..340bfa3f --- /dev/null +++ b/src/resources/dataUseRegister/dataUseRegister.controller.js @@ -0,0 +1,88 @@ +import Controller from '../base/controller'; +import { logger } from '../utilities/logger'; + +const logCategory = 'dataUseRegister'; + +export default class DataUseRegisterController extends Controller { + constructor(dataUseRegisterService) { + super(dataUseRegisterService); + this.dataUseRegisterService = dataUseRegisterService; + } + + async getDataUseRegister(req, res) { + try { + // Extract id parameter from query string + const { id } = req.params; + // If no id provided, it is a bad request + if (!id) { + return res.status(400).json({ + success: false, + message: 'You must provide a dataUseRegister identifier', + }); + } + // Find the dataUseRegister + const options = { lean: true }; + const dataUseRegister = await this.dataUseRegisterService.getDataUseRegister(id, req.query, options); + // Return if no dataUseRegister found + if (!dataUseRegister) { + return res.status(404).json({ + success: false, + message: 'A dataUseRegister could not be found with the provided id', + }); + } + // Return the dataUseRegister + return res.status(200).json({ + success: true, + ...dataUseRegister, + }); + } catch (err) { + // Return error response if something goes wrong + console.error(err.message); + return res.status(500).json({ + success: false, + message: 'A server error occurred, please try again', + }); + } + } + + async getDataUseRegisters(req, res) { + try { + // Find the relevant dataUseRegisters + const dataUseRegisters = await this.dataUseRegisterService.getDataUseRegisters(req.query).catch(err => { + logger.logError(err, logCategory); + }); + // Return the dataUseRegisters + return res.status(200).json({ + success: true, + data: dataUseRegisters, + }); + } catch (err) { + // Return error response if something goes wrong + logger.logError(err, logCategory); + return res.status(500).json({ + success: false, + message: 'A server error occurred, please try again', + }); + } + } + + async updateDataUseRegister(req, res) { + try { + // Find the relevant dataUseRegisters + const dataUseRegisters = await this.dataUseRegisterService.getDataUseRegisters(req.query).catch(err => { + logger.logError(err, logCategory); + }); + // Return the dataUseRegisters + return res.status(200).json({ + success: true, + }); + } catch (err) { + // Return error response if something goes wrong + logger.logError(err, logCategory); + return res.status(500).json({ + success: false, + message: 'A server error occurred, please try again', + }); + } + } +} diff --git a/src/resources/dataUseRegister/dataUseRegister.entity.js b/src/resources/dataUseRegister/dataUseRegister.entity.js new file mode 100644 index 00000000..8f1bbd8d --- /dev/null +++ b/src/resources/dataUseRegister/dataUseRegister.entity.js @@ -0,0 +1,8 @@ +import Entity from '../base/entity'; + +export default class DataUseRegisterClass extends Entity { + constructor(obj) { + super(); + Object.assign(this, obj); + } +} diff --git a/src/resources/dataUseRegister/dataUseRegister.model.js b/src/resources/dataUseRegister/dataUseRegister.model.js new file mode 100644 index 00000000..b9fae567 --- /dev/null +++ b/src/resources/dataUseRegister/dataUseRegister.model.js @@ -0,0 +1,28 @@ +import { model, Schema } from 'mongoose'; + +import DataUseRegisterClass from './dataUseRegister.entity'; +import constants from './../../resources/utilities/constants.util'; + +const dataUseRegisterSchema = new Schema( + { + lastActivity: Date, + projectTitle: String, + projectId: { type: Schema.Types.ObjectId, ref: 'data_request' }, + datasetTitles: [{ type: String }], + datasetIds: [{ type: String }], + publisher: { type: Schema.Types.ObjectId, ref: 'Publisher', required: true }, + user: { type: Schema.Types.ObjectId, ref: 'User', required: true }, + status: { type: String, required: true, enum: Object.values(constants.dataUseRegisterStatus) }, + }, + + { + timestamps: true, + toJSON: { virtuals: true }, + toObject: { virtuals: true }, + } +); + +// Load entity class +dataUseRegisterSchema.loadClass(DataUseRegisterClass); + +export const DataUseRegister = model('DataUseRegister', dataUseRegisterSchema); diff --git a/src/resources/dataUseRegister/dataUseRegister.repository.js b/src/resources/dataUseRegister/dataUseRegister.repository.js new file mode 100644 index 00000000..c87f2c55 --- /dev/null +++ b/src/resources/dataUseRegister/dataUseRegister.repository.js @@ -0,0 +1,19 @@ +import Repository from '../base/repository'; +import { DataUseRegister } from './dataUseRegister.model'; +import constants from '../utilities/constants.util'; + +export default class DataUseRegisterRepository extends Repository { + constructor() { + super(DataUseRegister); + this.dataUseRegister = DataUseRegister; + } + + async getDataUseRegister(query, options) { + return this.findOne(query, options); + } + + async getDataUseRegisters(query) { + const options = { lean: true }; + return this.find(query, options); + } +} diff --git a/src/resources/dataUseRegister/dataUseRegister.route.js b/src/resources/dataUseRegister/dataUseRegister.route.js new file mode 100644 index 00000000..9f1c44d8 --- /dev/null +++ b/src/resources/dataUseRegister/dataUseRegister.route.js @@ -0,0 +1,57 @@ +import express from 'express'; +import DataUseRegisterController from './dataUseRegister.controller'; +import { dataUseRegisterService } from './dependency'; +import { logger } from '../utilities/logger'; +import passport from 'passport'; +import { TheatersRounded } from '@material-ui/icons'; + +const router = express.Router(); +const dataUseRegisterController = new DataUseRegisterController(dataUseRegisterService); +const logCategory = 'dataUseRegister'; + +// @route GET /api/v2/data-use-registers/id +// @desc Returns a dataUseRegister based on dataUseRegister ID provided +// @access Public +router.get( + '/:id', + // passport.authenticate('jwt'), + logger.logRequestMiddleware({ logCategory, action: 'Viewed dataUseRegister data' }), + (req, res) => dataUseRegisterController.getDataUseRegister(req, res) +); + +// @route GET /api/v2/data-use-registers +// @desc Returns a collection of dataUseRegisters based on supplied query parameters +// @access Public +router.get( + '/', + passport.authenticate('jwt'), + + (req, res, next) => { + const { user } = req.user; + const { publisher } = req.query; + + if (publisher && isUserMemberOfTeam(user, publisher)) { + next(); + } else { + console.log('not'); + } + }, + logger.logRequestMiddleware({ logCategory, action: 'Viewed dataUseRegisters data' }), + (req, res) => dataUseRegisterController.getDataUseRegisters(req, res) +); + +// @route PUT /api/v2/data-use-registers/id +// @desc Update the content of the data user register based on dataUseRegister ID provided +// @access Public +router.patch( + '/:id', + // passport.authenticate('jwt'), + logger.logRequestMiddleware({ logCategory, action: 'Updated dataUseRegister data' }), + (req, res) => dataUseRegisterController.updateDataUseRegister(req, res) +); + +function isUserMemberOfTeam(user, teamId) { + return user.teams.exists(team => team.publisher._id === teamId); +} + +module.exports = router; diff --git a/src/resources/dataUseRegister/dataUseRegister.service.js b/src/resources/dataUseRegister/dataUseRegister.service.js new file mode 100644 index 00000000..7d737f02 --- /dev/null +++ b/src/resources/dataUseRegister/dataUseRegister.service.js @@ -0,0 +1,25 @@ +export default class DataUseRegisterService { + constructor(dataUseRegisterRepository) { + this.dataUseRegisterRepository = dataUseRegisterRepository; + } + + getDataUseRegister(id, query = {}, options = {}) { + // Protect for no id passed + if (!id) return; + + query = { ...query, id }; + return this.dataUseRegisterRepository.getDataUseRegister(query, options); + } + + getDataUseRegisters(query = {}) { + return this.dataUseRegisterRepository.getDataUseRegisters(query); + } + + updateDataUseRegister(id, query = {}, options = {}) { + // Protect for no id passed + if (!id) return; + + query = { ...query, id }; + return this.dataUseRegisterRepository.update(query, options); + } +} diff --git a/src/resources/dataUseRegister/dependency.js b/src/resources/dataUseRegister/dependency.js new file mode 100644 index 00000000..2c22d9f8 --- /dev/null +++ b/src/resources/dataUseRegister/dependency.js @@ -0,0 +1,5 @@ +import DataUseRegisterRepository from './dataUseRegister.repository'; +import DataUseRegisterService from './dataUseRegister.service'; + +export const dataUseRegisterRepository = new DataUseRegisterRepository(); +export const dataUseRegisterService = new DataUseRegisterService(dataUseRegisterRepository); diff --git a/src/resources/utilities/constants.util.js b/src/resources/utilities/constants.util.js index caa273ea..258326c5 100644 --- a/src/resources/utilities/constants.util.js +++ b/src/resources/utilities/constants.util.js @@ -242,6 +242,13 @@ const _logTypes = { USER: 'User', }; +const _dataUseRegisterStatus = { + ACTIVE: 'active', + PENDING_APPROVAL: 'pendingApproval', + REJECTED: 'rejected', + ARCHIVED: 'archived', +}; + export default { userTypes: _userTypes, enquiryFormId: _enquiryFormId, @@ -267,4 +274,5 @@ export default { datatsetStatuses: _datatsetStatuses, logTypes: _logTypes, DARMessageTypes: _DARMessageTypes, + dataUseRegisterStatus: _dataUseRegisterStatus, }; From 313f82873e4725cb850aa9133138cba3e55a1289 Mon Sep 17 00:00:00 2001 From: massimocianfroccaPA Date: Thu, 9 Sep 2021 14:31:51 +0100 Subject: [PATCH 002/116] implement get all and patch API in v2 style --- .../dataUseRegister.controller.js | 12 ++- .../dataUseRegister.repository.js | 5 +- .../dataUseRegister/dataUseRegister.route.js | 80 ++++++++++++++----- .../dataUseRegister.service.js | 7 +- 4 files changed, 77 insertions(+), 27 deletions(-) diff --git a/src/resources/dataUseRegister/dataUseRegister.controller.js b/src/resources/dataUseRegister/dataUseRegister.controller.js index 340bfa3f..ffdda4e7 100644 --- a/src/resources/dataUseRegister/dataUseRegister.controller.js +++ b/src/resources/dataUseRegister/dataUseRegister.controller.js @@ -1,5 +1,6 @@ import Controller from '../base/controller'; import { logger } from '../utilities/logger'; +import _ from 'lodash'; const logCategory = 'dataUseRegister'; @@ -48,7 +49,8 @@ export default class DataUseRegisterController extends Controller { async getDataUseRegisters(req, res) { try { // Find the relevant dataUseRegisters - const dataUseRegisters = await this.dataUseRegisterService.getDataUseRegisters(req.query).catch(err => { + const query = _.isEmpty(req.query) ? { user: req.user._id } : req.query; + const dataUseRegisters = await this.dataUseRegisterService.getDataUseRegisters(query).catch(err => { logger.logError(err, logCategory); }); // Return the dataUseRegisters @@ -68,11 +70,13 @@ export default class DataUseRegisterController extends Controller { async updateDataUseRegister(req, res) { try { - // Find the relevant dataUseRegisters - const dataUseRegisters = await this.dataUseRegisterService.getDataUseRegisters(req.query).catch(err => { + const id = req.params.id; + const body = req.body; + + this.dataUseRegisterService.updateDataUseRegister(id, body).catch(err => { logger.logError(err, logCategory); }); - // Return the dataUseRegisters + // Return success return res.status(200).json({ success: true, }); diff --git a/src/resources/dataUseRegister/dataUseRegister.repository.js b/src/resources/dataUseRegister/dataUseRegister.repository.js index c87f2c55..709fa10d 100644 --- a/src/resources/dataUseRegister/dataUseRegister.repository.js +++ b/src/resources/dataUseRegister/dataUseRegister.repository.js @@ -1,6 +1,5 @@ import Repository from '../base/repository'; import { DataUseRegister } from './dataUseRegister.model'; -import constants from '../utilities/constants.util'; export default class DataUseRegisterRepository extends Repository { constructor() { @@ -16,4 +15,8 @@ export default class DataUseRegisterRepository extends Repository { const options = { lean: true }; return this.find(query, options); } + + async updateDataUseRegister(id, body) { + return this.update(id, body); + } } diff --git a/src/resources/dataUseRegister/dataUseRegister.route.js b/src/resources/dataUseRegister/dataUseRegister.route.js index 9f1c44d8..4f1940d3 100644 --- a/src/resources/dataUseRegister/dataUseRegister.route.js +++ b/src/resources/dataUseRegister/dataUseRegister.route.js @@ -3,18 +3,74 @@ import DataUseRegisterController from './dataUseRegister.controller'; import { dataUseRegisterService } from './dependency'; import { logger } from '../utilities/logger'; import passport from 'passport'; -import { TheatersRounded } from '@material-ui/icons'; +import _ from 'lodash'; const router = express.Router(); const dataUseRegisterController = new DataUseRegisterController(dataUseRegisterService); const logCategory = 'dataUseRegister'; +function isUserMemberOfTeam(user, publisherId) { + const { teams } = user; + return teams.some(team => team.publisher._id.equals(publisherId)); +} + +const validateRequest = (req, res, next) => { + const { id } = req.params; + if (!id) { + return res.status(400).json({ + success: false, + message: 'You must provide a log event identifier', + }); + } + next(); +}; + +const authorizeView = async (req, res, next) => { + const requestingUser = req.user; + const { publisher } = req.query; + + const authorised = _.isUndefined(publisher) || isUserMemberOfTeam(requestingUser, publisher); + if (!authorised) { + return res.status(401).json({ + success: false, + message: 'You are not authorised to perform this action', + }); + } + + next(); +}; + +const authorizeUpdate = async (req, res, next) => { + const requestingUser = req.user; + const { id } = req.params; + + const dataUseRegister = await dataUseRegisterService.getDataUseRegister(id); + + if (!dataUseRegister) { + return res.status(404).json({ + success: false, + message: 'The requested data use register entry could not be found', + }); + } + + const { publisher } = dataUseRegister; + const authorised = isUserMemberOfTeam(requestingUser, publisher._id); + if (!authorised) { + return res.status(401).json({ + success: false, + message: 'You are not authorised to perform this action', + }); + } + + next(); +}; + // @route GET /api/v2/data-use-registers/id // @desc Returns a dataUseRegister based on dataUseRegister ID provided // @access Public router.get( '/:id', - // passport.authenticate('jwt'), + passport.authenticate('jwt'), logger.logRequestMiddleware({ logCategory, action: 'Viewed dataUseRegister data' }), (req, res) => dataUseRegisterController.getDataUseRegister(req, res) ); @@ -25,17 +81,7 @@ router.get( router.get( '/', passport.authenticate('jwt'), - - (req, res, next) => { - const { user } = req.user; - const { publisher } = req.query; - - if (publisher && isUserMemberOfTeam(user, publisher)) { - next(); - } else { - console.log('not'); - } - }, + authorizeView, logger.logRequestMiddleware({ logCategory, action: 'Viewed dataUseRegisters data' }), (req, res) => dataUseRegisterController.getDataUseRegisters(req, res) ); @@ -45,13 +91,11 @@ router.get( // @access Public router.patch( '/:id', - // passport.authenticate('jwt'), + passport.authenticate('jwt'), + validateRequest, + authorizeUpdate, logger.logRequestMiddleware({ logCategory, action: 'Updated dataUseRegister data' }), (req, res) => dataUseRegisterController.updateDataUseRegister(req, res) ); -function isUserMemberOfTeam(user, teamId) { - return user.teams.exists(team => team.publisher._id === teamId); -} - module.exports = router; diff --git a/src/resources/dataUseRegister/dataUseRegister.service.js b/src/resources/dataUseRegister/dataUseRegister.service.js index 7d737f02..0c95ea5c 100644 --- a/src/resources/dataUseRegister/dataUseRegister.service.js +++ b/src/resources/dataUseRegister/dataUseRegister.service.js @@ -7,7 +7,7 @@ export default class DataUseRegisterService { // Protect for no id passed if (!id) return; - query = { ...query, id }; + query = { ...query, _id: id }; return this.dataUseRegisterRepository.getDataUseRegister(query, options); } @@ -15,11 +15,10 @@ export default class DataUseRegisterService { return this.dataUseRegisterRepository.getDataUseRegisters(query); } - updateDataUseRegister(id, query = {}, options = {}) { + updateDataUseRegister(id, body = {}) { // Protect for no id passed if (!id) return; - query = { ...query, id }; - return this.dataUseRegisterRepository.update(query, options); + return this.dataUseRegisterRepository.updateDataUseRegister({ _id: id }, body); } } From 70c48909a1f112df1a83fe23944974498898ee46 Mon Sep 17 00:00:00 2001 From: massimocianfroccaPA Date: Thu, 9 Sep 2021 15:55:13 +0100 Subject: [PATCH 003/116] fix log message --- src/resources/dataUseRegister/dataUseRegister.route.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/resources/dataUseRegister/dataUseRegister.route.js b/src/resources/dataUseRegister/dataUseRegister.route.js index 4f1940d3..12f17cbb 100644 --- a/src/resources/dataUseRegister/dataUseRegister.route.js +++ b/src/resources/dataUseRegister/dataUseRegister.route.js @@ -19,7 +19,7 @@ const validateRequest = (req, res, next) => { if (!id) { return res.status(400).json({ success: false, - message: 'You must provide a log event identifier', + message: 'You must provide a data user register identifier', }); } next(); From fd91bab876167cab964f1ae79af64d45f3ae7869 Mon Sep 17 00:00:00 2001 From: massimocianfroccaPA Date: Fri, 10 Sep 2021 15:27:02 +0100 Subject: [PATCH 004/116] add logic to differentiate among user, team and publisher --- .../dataUseRegister.controller.js | 18 ++++++- .../dataUseRegister/dataUseRegister.route.js | 52 ++++++++++++++++--- src/resources/utilities/constants.util.js | 1 + 3 files changed, 62 insertions(+), 9 deletions(-) diff --git a/src/resources/dataUseRegister/dataUseRegister.controller.js b/src/resources/dataUseRegister/dataUseRegister.controller.js index ffdda4e7..d8e6979a 100644 --- a/src/resources/dataUseRegister/dataUseRegister.controller.js +++ b/src/resources/dataUseRegister/dataUseRegister.controller.js @@ -1,6 +1,7 @@ import Controller from '../base/controller'; import { logger } from '../utilities/logger'; import _ from 'lodash'; +import constants from './../utilities/constants.util'; const logCategory = 'dataUseRegister'; @@ -48,8 +49,21 @@ export default class DataUseRegisterController extends Controller { async getDataUseRegisters(req, res) { try { - // Find the relevant dataUseRegisters - const query = _.isEmpty(req.query) ? { user: req.user._id } : req.query; + const { team } = req.query; + const requestingUser = req.user; + + let query = ''; + switch (team) { + case 'user': + query = { user: requestingUser._id }; + break; + case 'admin': + query = { status: constants.dataUseRegisterStatus.PENDING_APPROVAL }; + break; + default: + query = { publisher: team }; + } + const dataUseRegisters = await this.dataUseRegisterService.getDataUseRegisters(query).catch(err => { logger.logError(err, logCategory); }); diff --git a/src/resources/dataUseRegister/dataUseRegister.route.js b/src/resources/dataUseRegister/dataUseRegister.route.js index 12f17cbb..7918d236 100644 --- a/src/resources/dataUseRegister/dataUseRegister.route.js +++ b/src/resources/dataUseRegister/dataUseRegister.route.js @@ -3,6 +3,7 @@ import DataUseRegisterController from './dataUseRegister.controller'; import { dataUseRegisterService } from './dependency'; import { logger } from '../utilities/logger'; import passport from 'passport'; +import constants from './../utilities/constants.util'; import _ from 'lodash'; const router = express.Router(); @@ -10,26 +11,62 @@ const dataUseRegisterController = new DataUseRegisterController(dataUseRegisterS const logCategory = 'dataUseRegister'; function isUserMemberOfTeam(user, publisherId) { - const { teams } = user; - return teams.some(team => team.publisher._id.equals(publisherId)); + let { teams } = user; + return teams.filter(team => !_.isNull(team.publisher)).some(team => team.publisher._id.equals(publisherId)); } -const validateRequest = (req, res, next) => { +function isUserDataUseAdmin(user) { + let { teams } = user; + + if (teams) { + teams = teams.map(team => { + let { publisher, type, members } = team; + let member = members.find(member => { + return member.memberid.toString() === user._id.toString(); + }); + let { roles } = member; + return { ...publisher, type, roles }; + }); + } + + return teams + .filter(team => team.type === constants.teamTypes.ADMIN) + .some(team => team.roles.includes(constants.roleTypes.ADMIN_DATA_USE)); +} + +const validateUpdateRequest = (req, res, next) => { const { id } = req.params; + if (!id) { return res.status(400).json({ success: false, message: 'You must provide a data user register identifier', }); } + + next(); +}; + +const validateViewRequest = (req, res, next) => { + const { team } = req.query; + + if (!team) { + return res.status(400).json({ + success: false, + message: 'You must provide a team parameter', + }); + } + next(); }; const authorizeView = async (req, res, next) => { const requestingUser = req.user; - const { publisher } = req.query; + const { team } = req.query; + + const authorised = + team === 'user' || (team === 'admin' && isUserDataUseAdmin(requestingUser)) || isUserMemberOfTeam(requestingUser, team); - const authorised = _.isUndefined(publisher) || isUserMemberOfTeam(requestingUser, publisher); if (!authorised) { return res.status(401).json({ success: false, @@ -54,7 +91,7 @@ const authorizeUpdate = async (req, res, next) => { } const { publisher } = dataUseRegister; - const authorised = isUserMemberOfTeam(requestingUser, publisher._id); + const authorised = isUserMemberOfTeam(requestingUser, publisher._id) || isUserDataUseAdmin(requestingUser); if (!authorised) { return res.status(401).json({ success: false, @@ -81,6 +118,7 @@ router.get( router.get( '/', passport.authenticate('jwt'), + validateViewRequest, authorizeView, logger.logRequestMiddleware({ logCategory, action: 'Viewed dataUseRegisters data' }), (req, res) => dataUseRegisterController.getDataUseRegisters(req, res) @@ -92,7 +130,7 @@ router.get( router.patch( '/:id', passport.authenticate('jwt'), - validateRequest, + validateUpdateRequest, authorizeUpdate, logger.logRequestMiddleware({ logCategory, action: 'Updated dataUseRegister data' }), (req, res) => dataUseRegisterController.updateDataUseRegister(req, res) diff --git a/src/resources/utilities/constants.util.js b/src/resources/utilities/constants.util.js index 258326c5..dcd82a99 100644 --- a/src/resources/utilities/constants.util.js +++ b/src/resources/utilities/constants.util.js @@ -210,6 +210,7 @@ const _roleTypes = { REVIEWER: 'reviewer', METADATA_EDITOR: 'metadata_editor', ADMIN_DATASET: 'admin_dataset', + ADMIN_DATA_USE: 'admin_data_use', }; // From 09d72459d626357d9c4ad44e8916a81c06f8ebd0 Mon Sep 17 00:00:00 2001 From: massimocianfroccaPA Date: Fri, 10 Sep 2021 15:55:01 +0100 Subject: [PATCH 005/116] change data use statue pending to in review --- src/resources/dataUseRegister/dataUseRegister.controller.js | 2 +- src/resources/utilities/constants.util.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/resources/dataUseRegister/dataUseRegister.controller.js b/src/resources/dataUseRegister/dataUseRegister.controller.js index d8e6979a..fd49ca21 100644 --- a/src/resources/dataUseRegister/dataUseRegister.controller.js +++ b/src/resources/dataUseRegister/dataUseRegister.controller.js @@ -58,7 +58,7 @@ export default class DataUseRegisterController extends Controller { query = { user: requestingUser._id }; break; case 'admin': - query = { status: constants.dataUseRegisterStatus.PENDING_APPROVAL }; + query = { status: constants.dataUseRegisterStatus.INREVIEW }; break; default: query = { publisher: team }; diff --git a/src/resources/utilities/constants.util.js b/src/resources/utilities/constants.util.js index dcd82a99..ec04a547 100644 --- a/src/resources/utilities/constants.util.js +++ b/src/resources/utilities/constants.util.js @@ -245,7 +245,7 @@ const _logTypes = { const _dataUseRegisterStatus = { ACTIVE: 'active', - PENDING_APPROVAL: 'pendingApproval', + INREVIEW: 'inReview', REJECTED: 'rejected', ARCHIVED: 'archived', }; From 7c059f1ee61f0637d171316845ca9c7f7a6cd680 Mon Sep 17 00:00:00 2001 From: Robin Kavanagh Date: Fri, 17 Sep 2021 15:22:48 +0100 Subject: [PATCH 006/116] Stubbed out upload endpoint --- .../dataUseRegister.controller.js | 31 +++++++++++++++++++ .../dataUseRegister/dataUseRegister.route.js | 30 ++++++++++++++++++ .../dataUseRegister.service.js | 4 +++ 3 files changed, 65 insertions(+) diff --git a/src/resources/dataUseRegister/dataUseRegister.controller.js b/src/resources/dataUseRegister/dataUseRegister.controller.js index fd49ca21..8d4d3956 100644 --- a/src/resources/dataUseRegister/dataUseRegister.controller.js +++ b/src/resources/dataUseRegister/dataUseRegister.controller.js @@ -103,4 +103,35 @@ export default class DataUseRegisterController extends Controller { }); } } + + async uploadDataUseRegisters(req, res) { + // Model for the data use entity + + // Flag if uploaded version + + // POST to create the data use entries + + // Pre-populate the related resources with a fixed message + + // POST to check for duplicates and return any found datasets, applicants or outputs + try { + const { placeholder } = req; + + const result = await this.dataUseRegisterService.uploadDataUseRegister(placeholder).catch(err => { + logger.logError(err, logCategory); + }); + + // Return success + return res.status(200).json({ + success: true, + }); + } catch (err) { + // Return error response if something goes wrong + logger.logError(err, logCategory); + return res.status(500).json({ + success: false, + message: 'A server error occurred, please try again', + }); + } + } } diff --git a/src/resources/dataUseRegister/dataUseRegister.route.js b/src/resources/dataUseRegister/dataUseRegister.route.js index 7918d236..9b5e51bf 100644 --- a/src/resources/dataUseRegister/dataUseRegister.route.js +++ b/src/resources/dataUseRegister/dataUseRegister.route.js @@ -60,6 +60,19 @@ const validateViewRequest = (req, res, next) => { next(); }; +const validateUploadRequest = (req, res, next) => { + const { placeholder } = req.body; + + if (!req) { + return res.status(400).json({ + success: false, + message: 'You must provide...', + }); + } + + next(); +}; + const authorizeView = async (req, res, next) => { const requestingUser = req.user; const { team } = req.query; @@ -102,6 +115,11 @@ const authorizeUpdate = async (req, res, next) => { next(); }; +const authorizeUpload = async (req, res, next) => { + + next(); +} + // @route GET /api/v2/data-use-registers/id // @desc Returns a dataUseRegister based on dataUseRegister ID provided // @access Public @@ -136,4 +154,16 @@ router.patch( (req, res) => dataUseRegisterController.updateDataUseRegister(req, res) ); +// @route POST /api/v2/data-use-registers/upload +// @desc Accepts a bulk upload of data uses with built-in duplicate checking and rejection +// @access Public +router.post( + '/upload', + passport.authenticate('jwt'), + validateUploadRequest, + authorizeUpload, + logger.logRequestMiddleware({ logCategory, action: 'Bulk uploaded data uses' }), + (req, res) => dataUseRegisterController.uploadDataUseRegisters(req, res) +); + module.exports = router; diff --git a/src/resources/dataUseRegister/dataUseRegister.service.js b/src/resources/dataUseRegister/dataUseRegister.service.js index 0c95ea5c..30499ff1 100644 --- a/src/resources/dataUseRegister/dataUseRegister.service.js +++ b/src/resources/dataUseRegister/dataUseRegister.service.js @@ -21,4 +21,8 @@ export default class DataUseRegisterService { return this.dataUseRegisterRepository.updateDataUseRegister({ _id: id }, body); } + + uploadDataUseRegister(placeholder) { + return; + } } From 7acf495686b0b875d53141b32cb9df9bb9742b55 Mon Sep 17 00:00:00 2001 From: Robin Kavanagh Date: Tue, 21 Sep 2021 20:59:56 +0100 Subject: [PATCH 007/116] Updated data use model for manual upload --- .../dataUseRegister/dataUseRegister.model.js | 36 +++++++++++++++++-- 1 file changed, 34 insertions(+), 2 deletions(-) diff --git a/src/resources/dataUseRegister/dataUseRegister.model.js b/src/resources/dataUseRegister/dataUseRegister.model.js index b9fae567..30ff5a3a 100644 --- a/src/resources/dataUseRegister/dataUseRegister.model.js +++ b/src/resources/dataUseRegister/dataUseRegister.model.js @@ -8,13 +8,45 @@ const dataUseRegisterSchema = new Schema( lastActivity: Date, projectTitle: String, projectId: { type: Schema.Types.ObjectId, ref: 'data_request' }, - datasetTitles: [{ type: String }], + projectIdText: String, //Project ID + datasetTitles: [{ type: String }], //Dataset Name(s) datasetIds: [{ type: String }], publisher: { type: Schema.Types.ObjectId, ref: 'Publisher', required: true }, user: { type: Schema.Types.ObjectId, ref: 'User', required: true }, status: { type: String, required: true, enum: Object.values(constants.dataUseRegisterStatus) }, + organisationName: String, //Organisation Name + organisationSector: String, //Organisation Sector + gatewayApplicants: [ + { + type: Schema.Types.ObjectId, + ref: 'User', + }, + ], + nonGatewayApplicants: [{ type: String }], //Applicant Name(s) + applicantId: String, //Applicant ID + fundersAndSponsors: [{ type: String }], // Funders/Sponsors + accreditedResearcherStatus: String, //Accredited Researcher Status + sublicenceArrangements: String, //Sub-Licence Arrangements (if any)? + laySummary: String, //Lay Summary + publicBenefitStatement: String, //Public Benefit Statement + requestCategoryType: String, //Request Category Type + technicalSummary: String, //Technical Summary + otherApprovalCommittees: String, //Other Approval Committees + projectStartDate: Date, //Project Start Date + projectEndDate: Date, //Project End Date + latestApprovalDate: Date, //Latest Approval Date + dataSensitivityLevel: String, //Data Sensitivity Level + legalBasisForData: String, //Legal Basis For Provision Of Data + dutyOfConfidentiality: String, //Common Law Duty Of Confidentiality + nationalDataOptOut: String, //National Data Opt-Out Applied + requestFrequency: String, //Request Frequency + dataProcessingDescription: String, //Description Of How The Data Will Be Processed + confidentialDataDescription: String, //Description Of The Confidential Data Being Used + accessDate: Date, //Release/Access Date + dataLocation: String, //TRE Or Any Other Specified Location + privacyEnhancements: String, //How Has Data Been Processed To Enhance Privacy + researchOutputs: String, //Link To Research Outputs }, - { timestamps: true, toJSON: { virtuals: true }, From a2107cddb43300bef0e1c5a35577d895744c8c0d Mon Sep 17 00:00:00 2001 From: umma-pa Date: Wed, 22 Sep 2021 09:18:48 +0100 Subject: [PATCH 008/116] First commit --- src/resources/dataUseRegister/dataUseRegister.model.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/resources/dataUseRegister/dataUseRegister.model.js b/src/resources/dataUseRegister/dataUseRegister.model.js index b9fae567..49ee6077 100644 --- a/src/resources/dataUseRegister/dataUseRegister.model.js +++ b/src/resources/dataUseRegister/dataUseRegister.model.js @@ -12,6 +12,9 @@ const dataUseRegisterSchema = new Schema( datasetIds: [{ type: String }], publisher: { type: Schema.Types.ObjectId, ref: 'Publisher', required: true }, user: { type: Schema.Types.ObjectId, ref: 'User', required: true }, + userID: Number, + keywords: [{ type: String }], + fiveSafeFormAnswers: [{ type: String }], status: { type: String, required: true, enum: Object.values(constants.dataUseRegisterStatus) }, }, From 416882af79d53d04cbf876d4a5bcc2e7ada3261c Mon Sep 17 00:00:00 2001 From: umma-pa Date: Wed, 22 Sep 2021 11:52:33 +0100 Subject: [PATCH 009/116] Keyword/five safe form answers removed --- src/resources/dataUseRegister/dataUseRegister.model.js | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/resources/dataUseRegister/dataUseRegister.model.js b/src/resources/dataUseRegister/dataUseRegister.model.js index 58e19bc5..30ff5a3a 100644 --- a/src/resources/dataUseRegister/dataUseRegister.model.js +++ b/src/resources/dataUseRegister/dataUseRegister.model.js @@ -13,9 +13,6 @@ const dataUseRegisterSchema = new Schema( datasetIds: [{ type: String }], publisher: { type: Schema.Types.ObjectId, ref: 'Publisher', required: true }, user: { type: Schema.Types.ObjectId, ref: 'User', required: true }, - userID: Number, - keywords: [{ type: String }], - fiveSafeFormAnswers: [{ type: String }], status: { type: String, required: true, enum: Object.values(constants.dataUseRegisterStatus) }, organisationName: String, //Organisation Name organisationSector: String, //Organisation Sector From 3aaf93d25e56ab7173dfb4c228edc2d2aa4a8354 Mon Sep 17 00:00:00 2001 From: Robin Kavanagh Date: Wed, 22 Sep 2021 16:16:06 +0100 Subject: [PATCH 010/116] Updated model to support original entity properties --- .../dataUseRegister/dataUseRegister.model.js | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/src/resources/dataUseRegister/dataUseRegister.model.js b/src/resources/dataUseRegister/dataUseRegister.model.js index 30ff5a3a..bfd88360 100644 --- a/src/resources/dataUseRegister/dataUseRegister.model.js +++ b/src/resources/dataUseRegister/dataUseRegister.model.js @@ -5,6 +5,24 @@ import constants from './../../resources/utilities/constants.util'; const dataUseRegisterSchema = new Schema( { + id: Number, + type: String, + activeflag: { type: String, required: true, enum: Object.values(constants.dataUseRegisterStatus) }, + updatedon: Date, + counter: Number, + discourseTopicId: Number, + relatedObjects: [ + { + objectId: String, + reason: String, + objectType: String, + pid: String, + user: String, + updated: String, + }, + ], + keywords: [String], + lastActivity: Date, projectTitle: String, projectId: { type: Schema.Types.ObjectId, ref: 'data_request' }, @@ -13,7 +31,6 @@ const dataUseRegisterSchema = new Schema( datasetIds: [{ type: String }], publisher: { type: Schema.Types.ObjectId, ref: 'Publisher', required: true }, user: { type: Schema.Types.ObjectId, ref: 'User', required: true }, - status: { type: String, required: true, enum: Object.values(constants.dataUseRegisterStatus) }, organisationName: String, //Organisation Name organisationSector: String, //Organisation Sector gatewayApplicants: [ From 043e6a98f9af83f8b0449f7a38fca10c8ef3d45d Mon Sep 17 00:00:00 2001 From: Callum Reekie Date: Thu, 23 Sep 2021 14:58:21 +0100 Subject: [PATCH 011/116] CR - added ORCID passport strategy for SSO --- package.json | 1 + src/resources/auth/index.js | 1 + src/resources/auth/strategies/index.js | 3 +- src/resources/auth/strategies/orcid.js | 157 +++++++++++++++++++++++++ 4 files changed, 161 insertions(+), 1 deletion(-) create mode 100644 src/resources/auth/strategies/orcid.js diff --git a/package.json b/package.json index 56d99e48..1cc23db3 100644 --- a/package.json +++ b/package.json @@ -57,6 +57,7 @@ "passport-jwt": "^4.0.0", "passport-linkedin-oauth2": "^2.0.0", "passport-openidconnect": "0.0.2", + "passport-orcid": "0.0.4", "prettier": "^2.2.1", "query-string": "^6.12.1", "randomstring": "^1.1.5", diff --git a/src/resources/auth/index.js b/src/resources/auth/index.js index 3edecf4a..d51545b3 100644 --- a/src/resources/auth/index.js +++ b/src/resources/auth/index.js @@ -11,6 +11,7 @@ const initialiseAuthentication = app => { strategies.LinkedinStrategy, strategies.GoogleStrategy, strategies.AzureStrategy, + strategies.OrcidStrategy, strategies.JWTStrategy )(app); }; diff --git a/src/resources/auth/strategies/index.js b/src/resources/auth/strategies/index.js index 28c09cc7..527dcf85 100644 --- a/src/resources/auth/strategies/index.js +++ b/src/resources/auth/strategies/index.js @@ -3,5 +3,6 @@ import { strategy as GoogleStrategy } from './google'; import { strategy as LinkedinStrategy } from './linkedin'; import { strategy as OdicStrategy } from './oidc'; import { strategy as AzureStrategy } from './azure'; +import { strategy as OrcidStrategy } from './orcid'; -export { JWTStrategy, GoogleStrategy, LinkedinStrategy, OdicStrategy, AzureStrategy }; +export { JWTStrategy, GoogleStrategy, LinkedinStrategy, OdicStrategy, AzureStrategy, OrcidStrategy }; diff --git a/src/resources/auth/strategies/orcid.js b/src/resources/auth/strategies/orcid.js new file mode 100644 index 00000000..10f992fd --- /dev/null +++ b/src/resources/auth/strategies/orcid.js @@ -0,0 +1,157 @@ +import passport from 'passport'; +import passportOrcid from 'passport-orcid'; +import { to } from 'await-to-js'; +import axios from 'axios'; + +import { getUserByProviderId } from '../../user/user.repository'; +import { updateRedirectURL } from '../../user/user.service'; +import { getObjectById } from '../../tool/data.repository'; +import { createUser } from '../../user/user.service'; +import { signToken } from '../utils'; +import { ROLES } from '../../user/user.roles'; +import queryString from 'query-string'; +import Url from 'url'; +import { discourseLogin } from '../sso/sso.discourse.service'; + +const eventLogController = require('../../eventlog/eventlog.controller'); +const OrcidStrategy = passportOrcid.Strategy + +const strategy = app => { + const strategyOptions = { + sandbox: process.env.ORCID_SSO_ENV, + clientID: process.env.ORCID_SSO_CLIENT_ID, + clientSecret: process.env.ORCID_SSO_CLIENT_SECRET, + callbackURL: `/auth/orcid/callback`, + scope: `/authenticate /read-limited`, + proxy: true + }; + + const verifyCallback = async (accessToken, refreshToken, params, profile, done) => { + + if (!params.orcid || params.orcid === '') return done('loginError'); + + let [err, user] = await to(getUserByProviderId(params.orcid)); + if (err || user) { + return done(err, user); + } + // Orcid does not include email natively + const requestedEmail = + await axios.get( + `${process.env.ORCID_SSO_BASE_URL}/v3.0/${params.orcid}/email`, + { headers: { 'Authorization': `Bearer ` + accessToken, 'Accept': 'application/json' }} + ).then((response) => { + const email = response.data.email[0].email + return ( email == undefined || !/\b[a-zA-Z0-9-_.]+\@[a-zA-Z0-9-_]+\.\w+(?:\.\w+)?\b/.test(email) ) ? '' : email + }).catch((err) => { + console.log(err); + return ''; + }); + + const [createdError, createdUser] = await to( + createUser({ + provider: 'orcid', + providerId: params.orcid, + firstname: params.name.split(' ')[0], + lastname: params.name.split(' ')[1], + password: null, + email: requestedEmail, + role: ROLES.Creator, + }) + ); + + return done(createdError, createdUser); + }; + + passport.use('orcid', new OrcidStrategy(strategyOptions, verifyCallback)); + + app.get( + `/auth/orcid`, + (req, res, next) => { + // Save the url of the user's current page so the app can redirect back to it after authorization + if (req.headers.referer) { + req.param.returnpage = req.headers.referer; + } + next(); + }, + passport.authenticate('orcid') + ); + + app.get('/auth/orcid/callback', (req, res, next) => { + passport.authenticate('orcid', (err, user, info) => { + + if (err || !user) { + //loginError + if (err === 'loginError') return res.status(200).redirect(process.env.homeURL + '/loginerror'); + + // failureRedirect + var redirect = '/'; + let returnPage = null; + if (req.param.returnpage) { + returnPage = Url.parse(req.param.returnpage); + redirect = returnPage.path; + delete req.param.returnpage; + }; + + let redirectUrl = process.env.homeURL + redirect; + return res.status(200).redirect(redirectUrl); + }; + + req.login(user, async err => { + + if (err) { + return next(err); + }; + + var redirect = '/'; + let returnPage = null; + let queryStringParsed = null; + if (req.param.returnpage) { + returnPage = Url.parse(req.param.returnpage); + redirect = returnPage.path; + queryStringParsed = queryString.parse(returnPage.query); + }; + + let [profileErr, profile] = await to(getObjectById(req.user.id)); + if (!profile) { + await to(updateRedirectURL({ id: req.user.id, redirectURL: redirect })); + return res.redirect(process.env.homeURL + '/completeRegistration/' + req.user.id); + }; + + if (req.param.returnpage) { + delete req.param.returnpage; + }; + + let redirectUrl = process.env.homeURL + redirect; + if (queryStringParsed && queryStringParsed.sso && queryStringParsed.sig) { + try { + console.log(req.user) + redirectUrl = discourseLogin(queryStringParsed.sso, queryStringParsed.sig, req.user); + } catch (err) { + console.error(err.message); + return res.status(500).send('Error authenticating the user.'); + } + }; + + //Build event object for user login and log it to DB + let eventObj = { + userId: req.user.id, + event: `user_login_${req.user.provider}`, + timestamp: Date.now(), + }; + + await eventLogController.logEvent(eventObj); + + return res + .status(200) + .cookie('jwt', signToken({ _id: req.user._id, id: req.user.id, timeStamp: Date.now() }), { + httpOnly: true, + secure: process.env.api_url ? true : false, + }) + .redirect(redirectUrl); + }); + })(req, res, next); + }); + return app; +}; + +export { strategy }; \ No newline at end of file From 8f3019946287c548929451a84cc4a2f9f5e3c174 Mon Sep 17 00:00:00 2001 From: umma-pa Date: Thu, 23 Sep 2021 15:56:30 +0100 Subject: [PATCH 012/116] Reverse look up --- .../dataUseRegister/dataUseRegister.route.js | 55 +++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/src/resources/dataUseRegister/dataUseRegister.route.js b/src/resources/dataUseRegister/dataUseRegister.route.js index 7918d236..9c82d0a1 100644 --- a/src/resources/dataUseRegister/dataUseRegister.route.js +++ b/src/resources/dataUseRegister/dataUseRegister.route.js @@ -1,5 +1,6 @@ import express from 'express'; import DataUseRegisterController from './dataUseRegister.controller'; +import { Data } from '../tool/data.model'; import { dataUseRegisterService } from './dependency'; import { logger } from '../utilities/logger'; import passport from 'passport'; @@ -124,6 +125,60 @@ router.get( (req, res) => dataUseRegisterController.getDataUseRegisters(req, res) ); +// {get} /api/v2/data-use-registers/{datauseregisterID} Data Use Register +// Return the details on the Data Use Register based on the Course ID(?) +// +router.get('/:datauseid', async (req, res) => { + let id = parseInt(req.params.id); + var query = Data.aggregate([ + { $match: { id: parseInt(req.params.id) } }, + { + $lookup: { + from: 'tools', + localField: 'creator', + foreignField: 'id', + as: 'creator', + }, + }, + ]); + query.exec((err, data) => { + if (data.length > 0) { + var p = Data.aggregate([ + { + $match: { + $and: [{ relatedObjects: { $elemMatch: { objectId: req.params.id } } }], + }, + }, + ]); + p.exec((err, relatedData) => { + relatedData.forEach(dat => { + dat.relatedObjects.forEach(x => { + if (x.objectId === req.params.id && dat.id !== req.params.id) { + let relatedObject = { + objectId: dat.id, + reason: x.reason, + objectType: dat.type, + user: x.user, + updated: x.updated, + }; + data[0].relatedObjects = [relatedObject, ...(data[0].relatedObjects || [])]; + } + }); + }); + + if (err) return res.json({ success: false, error: err }); + + return res.json({ + success: true, + data: data, + }); + }); + } else { + return res.status(404).send(`Data Use Register not found for Id: ${escape(id)}`); + } + }); +}); + // @route PUT /api/v2/data-use-registers/id // @desc Update the content of the data user register based on dataUseRegister ID provided // @access Public From d67758c97fa72fa9a23ba7eaa288b924d7ec5ba4 Mon Sep 17 00:00:00 2001 From: umma-pa Date: Fri, 24 Sep 2021 16:15:55 +0100 Subject: [PATCH 013/116] Reverse look up id --- src/resources/dataUseRegister/dataUseRegister.route.js | 2 +- src/resources/dataUseRegister/dataUseRegister.service.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/resources/dataUseRegister/dataUseRegister.route.js b/src/resources/dataUseRegister/dataUseRegister.route.js index 9c82d0a1..7535461f 100644 --- a/src/resources/dataUseRegister/dataUseRegister.route.js +++ b/src/resources/dataUseRegister/dataUseRegister.route.js @@ -128,7 +128,7 @@ router.get( // {get} /api/v2/data-use-registers/{datauseregisterID} Data Use Register // Return the details on the Data Use Register based on the Course ID(?) // -router.get('/:datauseid', async (req, res) => { +router.get('/:id', async (req, res) => { let id = parseInt(req.params.id); var query = Data.aggregate([ { $match: { id: parseInt(req.params.id) } }, diff --git a/src/resources/dataUseRegister/dataUseRegister.service.js b/src/resources/dataUseRegister/dataUseRegister.service.js index 0c95ea5c..d605aa25 100644 --- a/src/resources/dataUseRegister/dataUseRegister.service.js +++ b/src/resources/dataUseRegister/dataUseRegister.service.js @@ -7,7 +7,7 @@ export default class DataUseRegisterService { // Protect for no id passed if (!id) return; - query = { ...query, _id: id }; + query = { ...query, id: id }; return this.dataUseRegisterRepository.getDataUseRegister(query, options); } From 6c1e327646ce7528b41aedc2b29fd5fff147a8fd Mon Sep 17 00:00:00 2001 From: Robin Kavanagh Date: Sun, 26 Sep 2021 17:26:30 +0100 Subject: [PATCH 014/116] Completed bulk upload plus unit tests --- src/resources/base/entity.js | 10 +- .../__mocks__/dataUseRegisterUsers.js | 11 + .../__mocks__/dataUseRegisters.js | 363 ++++++++++++++++++ .../__tests__/dataUseRegister.service.test.js | 58 +++ .../__tests__/dataUseRegister.util.test.js | 81 ++++ .../dataUseRegister.controller.js | 21 +- .../dataUseRegister/dataUseRegister.entity.js | 2 + .../dataUseRegister/dataUseRegister.model.js | 17 +- .../dataUseRegister.repository.js | 28 +- .../dataUseRegister/dataUseRegister.route.js | 32 +- .../dataUseRegister.service.js | 74 +++- .../dataUseRegister/dataUseRegister.util.js | 217 +++++++++++ src/resources/dataset/dataset.repository.js | 11 +- src/resources/dataset/dataset.service.js | 4 + src/resources/user/user.repository.js | 4 + 15 files changed, 894 insertions(+), 39 deletions(-) create mode 100644 src/resources/dataUseRegister/__mocks__/dataUseRegisterUsers.js create mode 100644 src/resources/dataUseRegister/__mocks__/dataUseRegisters.js create mode 100644 src/resources/dataUseRegister/__tests__/dataUseRegister.service.test.js create mode 100644 src/resources/dataUseRegister/__tests__/dataUseRegister.util.test.js create mode 100644 src/resources/dataUseRegister/dataUseRegister.util.js diff --git a/src/resources/base/entity.js b/src/resources/base/entity.js index 2b21d3fe..7efee620 100644 --- a/src/resources/base/entity.js +++ b/src/resources/base/entity.js @@ -1,6 +1,8 @@ +import helper from '../utilities/helper.util'; + const transform = require('transformobject').transform; -class Entity { +export default class Entity { equals (other) { if (other instanceof Entity === false) { @@ -24,6 +26,8 @@ class Entity { transformTo(format, {strict} = {strict: false}) { return transform(this, format, { strict }); } -} -module.exports = Entity; \ No newline at end of file + generateId () { + return helper.generatedNumericId(); + } +} \ No newline at end of file diff --git a/src/resources/dataUseRegister/__mocks__/dataUseRegisterUsers.js b/src/resources/dataUseRegister/__mocks__/dataUseRegisterUsers.js new file mode 100644 index 00000000..93295172 --- /dev/null +++ b/src/resources/dataUseRegister/__mocks__/dataUseRegisterUsers.js @@ -0,0 +1,11 @@ +import mongoose from 'mongoose'; + +const uploader = { + _id: new mongoose.Types.ObjectId(), + firstname: 'James', + lastname: 'Smith', +}; + +export { + uploader +}; diff --git a/src/resources/dataUseRegister/__mocks__/dataUseRegisters.js b/src/resources/dataUseRegister/__mocks__/dataUseRegisters.js new file mode 100644 index 00000000..ee519a6e --- /dev/null +++ b/src/resources/dataUseRegister/__mocks__/dataUseRegisters.js @@ -0,0 +1,363 @@ +export const dataUseRegisterUploads = [ + { + "projectTitle": "This a test data use register", + "projectIdText": "this is the project id", + "datasetNames": [ + "This is the dataset title", "http://localhost:3000/dataset/f725187f-7352-482b-a43b-64ebc96e66f2", "http://localhost:3000/dataset/c6d6bbd3-74ed-46af-841d-ac5e05f4da41", "http://localhost:3000/dataset/e55df485-5acd-4606-bbb8-668d4c06380a" + ], + "applicantNames": [ + " Michael Donnelly", "http://localhost:3000/person/8495781222000176", "http://localhost:3000/person/4495285946631793" + ], + "organisationName": "organisation name", + "organisationSector": "organisation sector", + "applicantId": "applicant id", + "fundersAndSponsors": "funder1 , funder2 , funder3 ", + "accreditedResearcherStatus": "accredited Researcher Status", + "sublicenceArrangements": "sublicence Arrangements", + "laySummary": "lay Summary", + "publicBenefitStatement": "public Benefit Statement", + "requestCategoryType": "request Category Type", + "technicalSummary": "technical Summary", + "otherApprovalCommittees": "other Approval Committees", + "projectStartDate": "2021-09-25", + "projectEndDate": "2021-09-30", + "latestApprovalDate": "2021-09-21", + "dataSensitivityLevel": "data Sensitivity Level", + "legalBasisForData": "legal Basis For Data", + "dutyOfConfidentiality": "duty Of Confidentiality", + "nationalDataOptOut": "national Data Opt Out", + "requestFrequency": "request Frequency", + "dataProcessingDescription": "data Processing Description", + "confidentialDataDescription": "confidential Data Description", + "accessDate": "2021-09-26", + "dataLocation": "data Location", + "privacyEnhancements": "privacy Enhancements", + "researchOutputs": "research Outputs" + }, + { + "projectTitle": "This is another test data use register", + "projectIdText": "this is the other project id", + "datasetNames": [ + "This is the dataset title", "http://localhost:3000/dataset/f725187f-7352-482b-a43b-64ebc96e66f2", "http://localhost:3000/dataset/c6d6bbd3-74ed-46af-841d-ac5e05f4da41", "http://localhost:3000/dataset/e55df485-5acd-4606-bbb8-668d4c06380a" + ], + "applicantNames": [ + " Michael Donnelly", "http://localhost:3000/person/8495781222000176", "http://localhost:3000/person/4495285946631793" + ], + "organisationName": "organisation name", + "organisationSector": "organisation sector", + "applicantId": "applicant id", + "fundersAndSponsors": "funder1 , funder2 , funder3 ", + "accreditedResearcherStatus": "accredited Researcher Status", + "sublicenceArrangements": "sublicence Arrangements", + "laySummary": "other lay Summary", + "publicBenefitStatement": "public Benefit Statement", + "requestCategoryType": "request Category Type", + "technicalSummary": "technical Summary", + "otherApprovalCommittees": "other Approval Committees", + "projectStartDate": "2021-09-25", + "projectEndDate": "2021-09-30", + "latestApprovalDate": "2021-09-21", + "dataSensitivityLevel": "data Sensitivity Level", + "legalBasisForData": "legal Basis For Data", + "dutyOfConfidentiality": "duty Of Confidentiality", + "nationalDataOptOut": "national Data Opt Out", + "requestFrequency": "request Frequency", + "dataProcessingDescription": "data Processing Description", + "confidentialDataDescription": "confidential Data Description", + "accessDate": "2021-09-26", + "dataLocation": "data Location", + "privacyEnhancements": "privacy Enhancements", + "researchOutputs": "research Outputs" + } +] + +export const dataUseRegisterUploadsWithDuplicates = [ + { + "projectTitle": "This a test data use register", + "projectIdText": "this is the project id", + "datasetNames": [ + "This is the dataset title", "http://localhost:3000/dataset/f725187f-7352-482b-a43b-64ebc96e66f2", "http://localhost:3000/dataset/c6d6bbd3-74ed-46af-841d-ac5e05f4da41", "http://localhost:3000/dataset/e55df485-5acd-4606-bbb8-668d4c06380a" + ], + "applicantNames": [ + " Michael Donnelly", "http://localhost:3000/person/8495781222000176", "http://localhost:3000/person/4495285946631793" + ], + "organisationName": "organisation name", + "organisationSector": "organisation sector", + "applicantId": "applicant id", + "fundersAndSponsors": "funder1 , funder2 , funder3 ", + "accreditedResearcherStatus": "accredited Researcher Status", + "sublicenceArrangements": "sublicence Arrangements", + "laySummary": "lay Summary", + "publicBenefitStatement": "public Benefit Statement", + "requestCategoryType": "request Category Type", + "technicalSummary": "technical Summary", + "otherApprovalCommittees": "other Approval Committees", + "projectStartDate": "2021-09-25", + "projectEndDate": "2021-09-30", + "latestApprovalDate": "2021-09-21", + "dataSensitivityLevel": "data Sensitivity Level", + "legalBasisForData": "legal Basis For Data", + "dutyOfConfidentiality": "duty Of Confidentiality", + "nationalDataOptOut": "national Data Opt Out", + "requestFrequency": "request Frequency", + "dataProcessingDescription": "data Processing Description", + "confidentialDataDescription": "confidential Data Description", + "accessDate": "2021-09-26", + "dataLocation": "data Location", + "privacyEnhancements": "privacy Enhancements", + "researchOutputs": "research Outputs" + }, + { + "projectTitle": "This a test data use register", + "projectIdText": "this is the project id", + "datasetNames": [ + "This is the dataset title", "http://localhost:3000/dataset/f725187f-7352-482b-a43b-64ebc96e66f2", "http://localhost:3000/dataset/c6d6bbd3-74ed-46af-841d-ac5e05f4da41", "http://localhost:3000/dataset/e55df485-5acd-4606-bbb8-668d4c06380a" + ], + "applicantNames": [ + " Michael Donnelly", "http://localhost:3000/person/8495781222000176", "http://localhost:3000/person/4495285946631793" + ], + "organisationName": "organisation name", + "organisationSector": "organisation sector", + "applicantId": "applicant id", + "fundersAndSponsors": "funder1 , funder2 , funder3 ", + "accreditedResearcherStatus": "accredited Researcher Status", + "sublicenceArrangements": "sublicence Arrangements", + "laySummary": "lay Summary", + "publicBenefitStatement": "public Benefit Statement", + "requestCategoryType": "request Category Type", + "technicalSummary": "technical Summary", + "otherApprovalCommittees": "other Approval Committees", + "projectStartDate": "2021-09-25", + "projectEndDate": "2021-09-30", + "latestApprovalDate": "2021-09-21", + "dataSensitivityLevel": "data Sensitivity Level", + "legalBasisForData": "legal Basis For Data", + "dutyOfConfidentiality": "duty Of Confidentiality", + "nationalDataOptOut": "national Data Opt Out", + "requestFrequency": "request Frequency", + "dataProcessingDescription": "data Processing Description", + "confidentialDataDescription": "confidential Data Description", + "accessDate": "2021-09-26", + "dataLocation": "data Location", + "privacyEnhancements": "privacy Enhancements", + "researchOutputs": "research Outputs" + }, + { + "projectTitle": "This a test data use register", + "projectIdText": "this is the project id", + "datasetNames": [ + "This is the dataset title", "http://localhost:3000/dataset/f725187f-7352-482b-a43b-64ebc96e66f2", "http://localhost:3000/dataset/c6d6bbd3-74ed-46af-841d-ac5e05f4da41", "http://localhost:3000/dataset/e55df485-5acd-4606-bbb8-668d4c06380a" + ], + "applicantNames": [ + " Michael Donnelly", "http://localhost:3000/person/8495781222000176", "http://localhost:3000/person/4495285946631793" + ], + "organisationName": "organisation name", + "organisationSector": "organisation sector", + "applicantId": "applicant id", + "fundersAndSponsors": "funder1 , funder2 , funder3 ", + "accreditedResearcherStatus": "accredited Researcher Status", + "sublicenceArrangements": "sublicence Arrangements", + "laySummary": "lay Summary", + "publicBenefitStatement": "public Benefit Statement", + "requestCategoryType": "request Category Type", + "technicalSummary": "technical Summary", + "otherApprovalCommittees": "other Approval Committees", + "projectStartDate": "2021-09-25", + "projectEndDate": "2021-09-30", + "latestApprovalDate": "2021-09-21", + "dataSensitivityLevel": "data Sensitivity Level", + "legalBasisForData": "legal Basis For Data", + "dutyOfConfidentiality": "duty Of Confidentiality", + "nationalDataOptOut": "national Data Opt Out", + "requestFrequency": "request Frequency", + "dataProcessingDescription": "data Processing Description", + "confidentialDataDescription": "confidential Data Description", + "accessDate": "2021-09-26", + "dataLocation": "data Location", + "privacyEnhancements": "privacy Enhancements", + "researchOutputs": "research Outputs" + }, + { + "projectTitle": "This a test data use register", + "projectIdText": "this is the project id", + "datasetNames": [ + "This is the dataset title", "http://localhost:3000/dataset/f725187f-7352-482b-a43b-64ebc96e66f2", "http://localhost:3000/dataset/c6d6bbd3-74ed-46af-841d-ac5e05f4da41", "http://localhost:3000/dataset/e55df485-5acd-4606-bbb8-668d4c06380a" + ], + "applicantNames": [ + " Michael Donnelly", "http://localhost:3000/person/8495781222000176", "http://localhost:3000/person/4495285946631793" + ], + "organisationName": "organisation name", + "organisationSector": "organisation sector", + "applicantId": "applicant id", + "fundersAndSponsors": "funder1 , funder2 , funder3 ", + "accreditedResearcherStatus": "accredited Researcher Status", + "sublicenceArrangements": "sublicence Arrangements", + "laySummary": "lay Summary", + "publicBenefitStatement": "public Benefit Statement", + "requestCategoryType": "request Category Type", + "technicalSummary": "technical Summary", + "otherApprovalCommittees": "other Approval Committees", + "projectStartDate": "2021-09-25", + "projectEndDate": "2021-09-30", + "latestApprovalDate": "2021-09-21", + "dataSensitivityLevel": "data Sensitivity Level", + "legalBasisForData": "legal Basis For Data", + "dutyOfConfidentiality": "duty Of Confidentiality", + "nationalDataOptOut": "national Data Opt Out", + "requestFrequency": "request Frequency", + "dataProcessingDescription": "data Processing Description", + "confidentialDataDescription": "confidential Data Description", + "accessDate": "2021-09-26", + "dataLocation": "data Location", + "privacyEnhancements": "privacy Enhancements", + "researchOutputs": "research Outputs" + }, + { + "projectTitle": "This a test data use register", + "projectIdText": "this is another project id", + "datasetNames": [ + "This is the dataset title", "http://localhost:3000/dataset/f725187f-7352-482b-a43b-64ebc96e66f2", "http://localhost:3000/dataset/c6d6bbd3-74ed-46af-841d-ac5e05f4da41", "http://localhost:3000/dataset/e55df485-5acd-4606-bbb8-668d4c06380a" + ], + "applicantNames": [ + " Michael Donnelly", "http://localhost:3000/person/8495781222000176", "http://localhost:3000/person/4495285946631793" + ], + "organisationName": "another organisation name", + "organisationSector": "organisation sector", + "applicantId": "applicant id", + "fundersAndSponsors": "funder1 , funder2 , funder3 ", + "accreditedResearcherStatus": "accredited Researcher Status", + "sublicenceArrangements": "sublicence Arrangements", + "laySummary": "lay Summary", + "publicBenefitStatement": "public Benefit Statement", + "requestCategoryType": "request Category Type", + "technicalSummary": "technical Summary", + "otherApprovalCommittees": "other Approval Committees", + "projectStartDate": "2021-09-25", + "projectEndDate": "2021-09-30", + "latestApprovalDate": "2021-09-21", + "dataSensitivityLevel": "data Sensitivity Level", + "legalBasisForData": "legal Basis For Data", + "dutyOfConfidentiality": "duty Of Confidentiality", + "nationalDataOptOut": "national Data Opt Out", + "requestFrequency": "request Frequency", + "dataProcessingDescription": "data Processing Description", + "confidentialDataDescription": "confidential Data Description", + "accessDate": "2021-09-26", + "dataLocation": "data Location", + "privacyEnhancements": "privacy Enhancements", + "researchOutputs": "research Outputs" + }, + { + "projectTitle": "This a test data use register", + "projectIdText": "this is another project id", + "datasetNames": [ + "This is the dataset title", "http://localhost:3000/dataset/f725187f-7352-482b-a43b-64ebc96e66f2", "http://localhost:3000/dataset/c6d6bbd3-74ed-46af-841d-ac5e05f4da41", "http://localhost:3000/dataset/e55df485-5acd-4606-bbb8-668d4c06380a" + ], + "applicantNames": [ + " Michael Donnelly", "http://localhost:3000/person/8495781222000176", "http://localhost:3000/person/4495285946631793" + ], + "organisationName": "another organisation name", + "organisationSector": "organisation sector", + "applicantId": "applicant id", + "fundersAndSponsors": "funder1 , funder2 , funder3 ", + "accreditedResearcherStatus": "accredited Researcher Status", + "sublicenceArrangements": "sublicence Arrangements", + "laySummary": "another lay Summary", + "publicBenefitStatement": "public Benefit Statement", + "requestCategoryType": "request Category Type", + "technicalSummary": "technical Summary", + "otherApprovalCommittees": "other Approval Committees", + "projectStartDate": "2021-09-25", + "projectEndDate": "2021-09-30", + "latestApprovalDate": "2021-09-21", + "dataSensitivityLevel": "data Sensitivity Level", + "legalBasisForData": "legal Basis For Data", + "dutyOfConfidentiality": "duty Of Confidentiality", + "nationalDataOptOut": "national Data Opt Out", + "requestFrequency": "request Frequency", + "dataProcessingDescription": "data Processing Description", + "confidentialDataDescription": "confidential Data Description", + "accessDate": "2021-09-26", + "dataLocation": "data Location", + "privacyEnhancements": "privacy Enhancements", + "researchOutputs": "research Outputs" + } +] + +export const datasets = [ + { + datasetId: "70b4d407-288a-4945-a4d5-506d60715110", + pid: "e55df485-5acd-4606-bbb8-668d4c06380a" + }, + { + datasetId: "82ef7d1a-98d8-48b6-9acd-461bf2a399c3", + pid: "e55df485-5acd-4606-bbb8-668d4c06380a" + }, + { + datasetId: "673626f3-bdac-4d32-9bb8-c890b727c0d1", + pid: "594d79a4-92b9-4a7f-b991-abf850bf2b67" + }, + { + datasetId: "89e57932-ac48-48ac-a6e5-29795bc38b94", + pid: "efbd4275-70e2-4887-8499-18b1fb24ce5b" + } +] + +export const relatedObjectDatasets = [ + { + objectId: "70b4d407-288a-4945-a4d5-506d60715110", + pid: "e55df485-5acd-4606-bbb8-668d4c06380a", + objectType: "dataset", + user: "James Smith", + updated: "2021-24-09T11:01:58.135Z" + }, + { + objectId: "82ef7d1a-98d8-48b6-9acd-461bf2a399c3", + pid: "e55df485-5acd-4606-bbb8-668d4c06380a", + objectType: "dataset", + user: "James Smith", + updated: "2021-24-09T11:01:58.135Z" + }, + { + objectId: "673626f3-bdac-4d32-9bb8-c890b727c0d1", + pid: "594d79a4-92b9-4a7f-b991-abf850bf2b67", + objectType: "dataset", + user: "James Smith", + updated: "2021-24-09T11:01:58.135Z" + }, + { + objectId: "89e57932-ac48-48ac-a6e5-29795bc38b94", + pid: "efbd4275-70e2-4887-8499-18b1fb24ce5b", + objectType: "dataset", + user: "James Smith", + updated: "2021-24-09T11:01:58.135Z" + } +] + +export const nonGatewayDatasetNames = [ + "dataset one", "dataset two", " dataset three", "dataset four" +] + +export const gatewayDatasetNames = [ + "http://localhost:3000/dataset/f725187f-7352-482b-a43b-64ebc96e66f2", + "http://localhost:3000/dataset/c6d6bbd3-74ed-46af-841d-ac5e05f4da41", + "http://localhost:3000/dataset/e55df485-5acd-4606-bbb8-668d4c06380a" +] + +export const expectedGatewayDatasets = [ + { datasetid: "1", name: "dataset 1", pid:"111" }, + { datasetid: "2", name: "dataset 2", pid:"222" }, + { datasetid: "3", name: "dataset 3", pid:"333" } +] + +export const nonGatewayApplicantNames = [ + "applicant one", "applicant two", "applicant three", "applicant four" +] + +export const gatewayApplicantNames = [ + "http://localhost:3000/person/8495781222000176", "http://localhost:3000/person/4495285946631793" +] + +export const expectedGatewayApplicants = [ + "89e57932-ac48-48ac-a6e5-29795bc38b94", "0cfe60cd-038d-4c03-9a95-894c52135922" +] \ No newline at end of file diff --git a/src/resources/dataUseRegister/__tests__/dataUseRegister.service.test.js b/src/resources/dataUseRegister/__tests__/dataUseRegister.service.test.js new file mode 100644 index 00000000..b1eeb4d2 --- /dev/null +++ b/src/resources/dataUseRegister/__tests__/dataUseRegister.service.test.js @@ -0,0 +1,58 @@ +import sinon from 'sinon'; + +import DataUseRegisterService from '../dataUseRegister.service'; +import DataUseRegisterRepository from '../dataUseRegister.repository'; +import { dataUseRegisterUploadsWithDuplicates, dataUseRegisterUploads } from '../__mocks__/dataUseRegisters'; + +describe('DataUseRegisterService', function () { + describe('filterDuplicateDataUseRegisters', function () { + it('filters out data uses that have matching project Ids', async function () { + // Arrange + const dataUseRegisterRepository = new DataUseRegisterRepository(); + const dataUseRegisterService = new DataUseRegisterService(dataUseRegisterRepository); + + // Act + const result = dataUseRegisterService.filterDuplicateDataUseRegisters(dataUseRegisterUploadsWithDuplicates); + + // Assert + expect(dataUseRegisterUploadsWithDuplicates.length).toEqual(6); + expect(result.length).toEqual(2); + expect(result[0].projectIdText).not.toEqual(result[1].projectIdText); + expect(result[0]).toEqual(dataUseRegisterUploadsWithDuplicates[0]); + }); + it('filters out duplicate data uses that match across the following fields: project title, lay summary, organisation name, dataset names and latest approval date', async function () { + // Arrange + const dataUseRegisterRepository = new DataUseRegisterRepository(); + const dataUseRegisterService = new DataUseRegisterService(dataUseRegisterRepository); + + // Act + const result = dataUseRegisterService.filterDuplicateDataUseRegisters(dataUseRegisterUploadsWithDuplicates); + + // Assert + expect(dataUseRegisterUploadsWithDuplicates.length).toEqual(6); + expect(result.length).toEqual(2); + expect(result[1]).toEqual(dataUseRegisterUploadsWithDuplicates[4]); + }); + }); + + describe('filterExistingDataUseRegisters', function () { + it('filters out data uses that are found to already exist in the database', async function () { + // Arrange + const dataUseRegisterRepository = new DataUseRegisterRepository(); + const dataUseRegisterService = new DataUseRegisterService(dataUseRegisterRepository); + + const checkDataUseRegisterExistsStub = sinon.stub(dataUseRegisterRepository, 'checkDataUseRegisterExists'); + checkDataUseRegisterExistsStub.onCall(0).returns(false); + checkDataUseRegisterExistsStub.onCall(1).returns(true); + + // Act + const result = await dataUseRegisterService.filterExistingDataUseRegisters(dataUseRegisterUploads); + + // Assert + expect(checkDataUseRegisterExistsStub.calledTwice).toBe(true); + expect(dataUseRegisterUploads.length).toBe(2); + expect(result.length).toBe(1); + expect(result[0].projectIdText).toEqual(dataUseRegisterUploads[0].projectIdText); + }); + }); +}); diff --git a/src/resources/dataUseRegister/__tests__/dataUseRegister.util.test.js b/src/resources/dataUseRegister/__tests__/dataUseRegister.util.test.js new file mode 100644 index 00000000..4555be2c --- /dev/null +++ b/src/resources/dataUseRegister/__tests__/dataUseRegister.util.test.js @@ -0,0 +1,81 @@ +import sinon from 'sinon'; +import { cloneDeep } from 'lodash'; + +import dataUseRegisterUtil from '../dataUseRegister.util'; +import { datasets, relatedObjectDatasets, nonGatewayDatasetNames, gatewayDatasetNames, expectedGatewayDatasets, nonGatewayApplicantNames, gatewayApplicantNames, expectedGatewayApplicants } from '../__mocks__/dataUseRegisters'; +import { uploader } from '../__mocks__/dataUseRegisterUsers'; +import * as userRepository from '../../user/user.repository'; +import { datasetService } from '../../dataset/dependency'; + +describe('DataUseRegisterUtil', function () { + beforeAll(function () { + process.env.homeURL = 'http://localhost:3000'; + }); + + describe('getLinkedDatasets', function () { + it('returns the names of the datasets that could not be found on the Gateway as named datasets', async function () { + // Act + const result = await dataUseRegisterUtil.getLinkedDatasets(nonGatewayDatasetNames); + + // Assert + expect(result).toEqual({ linkedDatasets: [], namedDatasets: nonGatewayDatasetNames }); + }); + it('returns the details of datasets that could be found on the Gateway when valid URLs are given', async function () { + // Arrange + const getDatasetsByPidsStub = sinon.stub(datasetService, 'getDatasetsByPids'); + getDatasetsByPidsStub.returns(expectedGatewayDatasets); + + // Act + const result = await dataUseRegisterUtil.getLinkedDatasets(gatewayDatasetNames); + + // Assert + expect(getDatasetsByPidsStub.calledOnce).toBe(true); + expect(result).toEqual({ linkedDatasets: expectedGatewayDatasets, namedDatasets: [] }); + }); + }); + + describe('getLinkedApplicants', function () { + it('returns the names of the applicants that could not be found on the Gateway', async function () { + // Act + const result = await dataUseRegisterUtil.getLinkedApplicants(nonGatewayApplicantNames); + + // Assert + expect(result).toEqual({ gatewayApplicants: [], nonGatewayApplicants: nonGatewayApplicantNames }); + }); + it('returns the details of applicants that could be found on the Gateway when valid profile URLs are given', async function () { + // Arrange + const getUsersByIdsStub = sinon.stub(userRepository, 'getUsersByIds'); + getUsersByIdsStub.returns([{_id:'89e57932-ac48-48ac-a6e5-29795bc38b94'}, {_id:'0cfe60cd-038d-4c03-9a95-894c52135922'}]); + + // Act + const result = await dataUseRegisterUtil.getLinkedApplicants(gatewayApplicantNames); + + // Assert + expect(getUsersByIdsStub.calledOnce).toBe(true); + expect(result).toEqual({ gatewayApplicants: expectedGatewayApplicants, nonGatewayApplicants: [] }); + }); + }); + + describe('buildRelatedObjects', function () { + it('filters out data uses that are found to already exist in the database', async function () { + // Arrange + const data = cloneDeep(datasets); + sinon.stub(Date, 'now').returns('2021-24-09T11:01:58.135Z'); + + // Act + const result = dataUseRegisterUtil.buildRelatedObjects(uploader, data); + + // Assert + expect(result.length).toBe(data.length); + expect(result).toEqual(relatedObjectDatasets); + }); + + afterEach(function () { + sinon.restore(); + }); + }); + + afterAll(function () { + delete process.env.homeURL; + }); +}); diff --git a/src/resources/dataUseRegister/dataUseRegister.controller.js b/src/resources/dataUseRegister/dataUseRegister.controller.js index 8d4d3956..f22579b3 100644 --- a/src/resources/dataUseRegister/dataUseRegister.controller.js +++ b/src/resources/dataUseRegister/dataUseRegister.controller.js @@ -105,25 +105,14 @@ export default class DataUseRegisterController extends Controller { } async uploadDataUseRegisters(req, res) { - // Model for the data use entity - - // Flag if uploaded version - - // POST to create the data use entries - - // Pre-populate the related resources with a fixed message - - // POST to check for duplicates and return any found datasets, applicants or outputs try { - const { placeholder } = req; - - const result = await this.dataUseRegisterService.uploadDataUseRegister(placeholder).catch(err => { - logger.logError(err, logCategory); - }); - + const { teamId, dataUses } = req.body; + const requestingUser = req.user; + const result = await this.dataUseRegisterService.uploadDataUseRegisters(requestingUser, teamId, dataUses); // Return success - return res.status(200).json({ + return res.status(result.uploadedCount > 0 ? 201 : 200).json({ success: true, + result, }); } catch (err) { // Return error response if something goes wrong diff --git a/src/resources/dataUseRegister/dataUseRegister.entity.js b/src/resources/dataUseRegister/dataUseRegister.entity.js index 8f1bbd8d..3af49d17 100644 --- a/src/resources/dataUseRegister/dataUseRegister.entity.js +++ b/src/resources/dataUseRegister/dataUseRegister.entity.js @@ -3,6 +3,8 @@ import Entity from '../base/entity'; export default class DataUseRegisterClass extends Entity { constructor(obj) { super(); + if(!obj.id) obj.id = this.generateId(); + obj.type = 'dataUseRegister'; Object.assign(this, obj); } } diff --git a/src/resources/dataUseRegister/dataUseRegister.model.js b/src/resources/dataUseRegister/dataUseRegister.model.js index bfd88360..98d3c622 100644 --- a/src/resources/dataUseRegister/dataUseRegister.model.js +++ b/src/resources/dataUseRegister/dataUseRegister.model.js @@ -5,11 +5,11 @@ import constants from './../../resources/utilities/constants.util'; const dataUseRegisterSchema = new Schema( { - id: Number, - type: String, + id: { type: Number, required: true }, + type: { type: String, required: true }, activeflag: { type: String, required: true, enum: Object.values(constants.dataUseRegisterStatus) }, updatedon: Date, - counter: Number, + counter: { type: Number, default: 0 }, discourseTopicId: Number, relatedObjects: [ { @@ -22,16 +22,18 @@ const dataUseRegisterSchema = new Schema( }, ], keywords: [String], + manualUpload: Boolean, lastActivity: Date, - projectTitle: String, + projectTitle: { type: String }, projectId: { type: Schema.Types.ObjectId, ref: 'data_request' }, projectIdText: String, //Project ID datasetTitles: [{ type: String }], //Dataset Name(s) datasetIds: [{ type: String }], + datasetPids: [{ type: String }], publisher: { type: Schema.Types.ObjectId, ref: 'Publisher', required: true }, user: { type: Schema.Types.ObjectId, ref: 'User', required: true }, - organisationName: String, //Organisation Name + organisationName: { type: String }, //Organisation Name organisationSector: String, //Organisation Sector gatewayApplicants: [ { @@ -48,7 +50,7 @@ const dataUseRegisterSchema = new Schema( publicBenefitStatement: String, //Public Benefit Statement requestCategoryType: String, //Request Category Type technicalSummary: String, //Technical Summary - otherApprovalCommittees: String, //Other Approval Committees + otherApprovalCommittees: [{type: String}], //Other Approval Committees projectStartDate: Date, //Project Start Date projectEndDate: Date, //Project End Date latestApprovalDate: Date, //Latest Approval Date @@ -62,12 +64,13 @@ const dataUseRegisterSchema = new Schema( accessDate: Date, //Release/Access Date dataLocation: String, //TRE Or Any Other Specified Location privacyEnhancements: String, //How Has Data Been Processed To Enhance Privacy - researchOutputs: String, //Link To Research Outputs + researchOutputs: [{type: String}], //Link To Research Outputs }, { timestamps: true, toJSON: { virtuals: true }, toObject: { virtuals: true }, + strict: false } ); diff --git a/src/resources/dataUseRegister/dataUseRegister.repository.js b/src/resources/dataUseRegister/dataUseRegister.repository.js index 709fa10d..b7a63f2f 100644 --- a/src/resources/dataUseRegister/dataUseRegister.repository.js +++ b/src/resources/dataUseRegister/dataUseRegister.repository.js @@ -7,16 +7,38 @@ export default class DataUseRegisterRepository extends Repository { this.dataUseRegister = DataUseRegister; } - async getDataUseRegister(query, options) { + getDataUseRegister(query, options) { return this.findOne(query, options); } - async getDataUseRegisters(query) { + getDataUseRegisters(query) { const options = { lean: true }; return this.find(query, options); } - async updateDataUseRegister(id, body) { + updateDataUseRegister(id, body) { return this.update(id, body); } + + uploadDataUseRegisters(dataUseRegisters) { + return this.dataUseRegister.insertMany(dataUseRegisters); + } + + async checkDataUseRegisterExists(dataUseRegister) { + const { projectIdText, projectTitle, laySummary, organisationName, datasetTitles, latestApprovalDate } = dataUseRegister; + const duplicatesFound = await this.dataUseRegister.countDocuments({ + $or: [ + { projectIdText }, + { + projectTitle, + laySummary, + organisationName, + datasetTitles, + latestApprovalDate, + }, + ], + }); + + return duplicatesFound > 0; + } } diff --git a/src/resources/dataUseRegister/dataUseRegister.route.js b/src/resources/dataUseRegister/dataUseRegister.route.js index 9b5e51bf..55a223bd 100644 --- a/src/resources/dataUseRegister/dataUseRegister.route.js +++ b/src/resources/dataUseRegister/dataUseRegister.route.js @@ -4,15 +4,15 @@ import { dataUseRegisterService } from './dependency'; import { logger } from '../utilities/logger'; import passport from 'passport'; import constants from './../utilities/constants.util'; -import _ from 'lodash'; +import { isEmpty, isNull } from 'lodash'; const router = express.Router(); const dataUseRegisterController = new DataUseRegisterController(dataUseRegisterService); const logCategory = 'dataUseRegister'; -function isUserMemberOfTeam(user, publisherId) { +function isUserMemberOfTeam(user, teamId) { let { teams } = user; - return teams.filter(team => !_.isNull(team.publisher)).some(team => team.publisher._id.equals(publisherId)); + return teams.filter(team => !isNull(team.publisher)).some(team => team.publisher._id.equals(teamId)); } function isUserDataUseAdmin(user) { @@ -61,12 +61,21 @@ const validateViewRequest = (req, res, next) => { }; const validateUploadRequest = (req, res, next) => { - const { placeholder } = req.body; + const { teamId, dataUses } = req.body; + let errors = []; - if (!req) { + if (!teamId) { + errors.push('You must provide the custodian team identifier to associate the data uses to'); + } + + if(!dataUses || isEmpty(dataUses)) { + errors.push('You must provide data uses to upload'); + } + + if(!isEmpty(errors)){ return res.status(400).json({ success: false, - message: 'You must provide...', + message: errors.join(', '), }); } @@ -116,6 +125,17 @@ const authorizeUpdate = async (req, res, next) => { }; const authorizeUpload = async (req, res, next) => { + const requestingUser = req.user; + const { teamId } = req.body; + + const authorised = isUserDataUseAdmin(requestingUser) || isUserMemberOfTeam(requestingUser, teamId); + + if (!authorised) { + return res.status(401).json({ + success: false, + message: 'You are not authorised to perform this action', + }); + } next(); } diff --git a/src/resources/dataUseRegister/dataUseRegister.service.js b/src/resources/dataUseRegister/dataUseRegister.service.js index 30499ff1..55abfe3d 100644 --- a/src/resources/dataUseRegister/dataUseRegister.service.js +++ b/src/resources/dataUseRegister/dataUseRegister.service.js @@ -1,3 +1,5 @@ +import dataUseRegisterUtil from './dataUseRegister.util'; + export default class DataUseRegisterService { constructor(dataUseRegisterRepository) { this.dataUseRegisterRepository = dataUseRegisterRepository; @@ -18,11 +20,77 @@ export default class DataUseRegisterService { updateDataUseRegister(id, body = {}) { // Protect for no id passed if (!id) return; - + return this.dataUseRegisterRepository.updateDataUseRegister({ _id: id }, body); } - uploadDataUseRegister(placeholder) { - return; + /** + * Upload Data Use Registers + * + * @desc Accepts multiple data uses to upload and a team identifier indicating which Custodian team to add the data uses to. + * + * @param {String} teamId Array of data use objects to filter until uniqueness exists + * @param {Array} dataUseUploads Array of data use objects to filter until uniqueness exists + * @returns {Object} Object containing the details of the upload operation including number of duplicates found in payload, database and number successfully added + */ + async uploadDataUseRegisters(creatorUser, teamId, dataUseRegisterUploads = []) { + const dedupedDataUseRegisters = this.filterDuplicateDataUseRegisters(dataUseRegisterUploads); + + const dataUseRegisters = await dataUseRegisterUtil.buildDataUseRegisters(creatorUser, teamId, dedupedDataUseRegisters); + + const newDataUseRegisters = await this.filterExistingDataUseRegisters(dataUseRegisters); + + const uploadedDataUseRegisters = await this.dataUseRegisterRepository.uploadDataUseRegisters(newDataUseRegisters); + + return { + uploadedCount: uploadedDataUseRegisters.length, + duplicateCount: dataUseRegisterUploads.length - newDataUseRegisters.length, + uploaded: uploadedDataUseRegisters, + }; + } + + /** + * Filter Duplicate Data Uses + * + * @desc Accepts multiple data uses and outputs a unique list of data uses based on each entities properties. + * A duplicate project id is automatically indicates a duplicate entry as the id must be unique. + * Alternatively, a combination of matching title, summary, organisation name, dataset titles and latest approval date indicates a duplicate entry. + * @param {Array} dataUses Array of data use objects to filter until uniqueness exists + * @returns {Array} Filtered array of data uses assumed unique based on filter criteria + */ + filterDuplicateDataUseRegisters(dataUses) { + return dataUses.reduce((arr, dataUse) => { + const isDuplicate = arr.some( + el => + el.projectIdText === dataUse.projectIdText || + (el.projectTitle === dataUse.projectTitle && + el.laySummary === dataUse.laySummary && + el.organisationName === dataUse.organisationName && + el.datasetTitles === dataUse.datasetTitles && + el.latestApprovalDate === dataUse.latestApprovalDate) + ); + if (!isDuplicate) arr = [...arr, dataUse]; + return arr; + }, []); + } + + /** + * Filter Existing Data Uses + * + * @desc Accepts multiple data uses, verifying each in turn is considered 'new' to the database, then outputs the list of data uses. + * A duplicate project id is automatically indicates a duplicate entry as the id must be unique. + * Alternatively, a combination of matching title, summary, organisation name and dataset titles indicates a duplicate entry. + * @param {Array} dataUses Array of data use objects to iterate through and check for existence in database + * @returns {Array} Filtered array of data uses assumed to be 'new' to the database based on filter criteria + */ + async filterExistingDataUseRegisters(dataUses) { + const newDataUses = []; + + for (const dataUse of dataUses) { + const exists = await this.dataUseRegisterRepository.checkDataUseRegisterExists(dataUse); + if (exists === false) newDataUses.push(dataUse); + } + + return newDataUses; } } diff --git a/src/resources/dataUseRegister/dataUseRegister.util.js b/src/resources/dataUseRegister/dataUseRegister.util.js new file mode 100644 index 00000000..a76cf8d4 --- /dev/null +++ b/src/resources/dataUseRegister/dataUseRegister.util.js @@ -0,0 +1,217 @@ +import moment from 'moment'; +import { isEmpty } from 'lodash'; +import DataUseRegister from './dataUseRegister.entity'; +import { getUsersByIds } from '../user/user.repository'; +import { datasetService } from '../dataset/dependency'; + +/** + * Build Data Use Registers + * + * @desc Accepts a creator user object, the custodian/publisher team identifier to create the data use registers against and an array of data use POJOs to map to data use models. + * The function drops out invalid dates, empty fields and removes white space from all strings before constructing the model instances. + * @param {String} creatorUser User object from the authenticated request who is creating the data use registers + * @param {String} teamId Custodian/publisher team identifier to identify who to create the data uses against + * @param {String} dataUses Array of data use register shaped POJOs to map to data use models + * @returns {Array} Array of data use register models + */ +const buildDataUseRegisters = async (creatorUser, teamId, dataUses = []) => { + const dataUseRegisters = []; + + for (const obj of dataUses) { + // Handle dataset linkages + const { linkedDatasets = [], namedDatasets = [] } = await getLinkedDatasets( + obj.datasetNames && + obj.datasetNames + .toString() + .split(',') + .map(el => { + if (!isEmpty(el)) return el.trim(); + }) + ); + const datasetTitles = [...linkedDatasets.map(dataset => dataset.name), ...namedDatasets]; + const datasetIds = [...linkedDatasets.map(dataset => dataset.datasetid)]; + const datasetPids = [...linkedDatasets.map(dataset => dataset.pid)]; + + // Handle applicant linkages + const { gatewayApplicants, nonGatewayApplicants } = await getLinkedApplicants( + obj.applicantNames && + obj.applicantNames + .toString() + .split(',') + .map(el => { + if (!isEmpty(el)) return el.trim(); + }) + ); + + // Create related objects + const relatedObjects = buildRelatedObjects(creatorUser, linkedDatasets); + + // Handle comma separated fields + const fundersAndSponsors = + obj.fundersAndSponsors && + obj.fundersAndSponsors + .toString() + .split(',') + .map(el => { + if (!isEmpty(el)) return el.trim(); + }); + const researchOutputs = + obj.researchOutputs && + obj.researchOutputs + .toString() + .split(',') + .map(el => { + if (!isEmpty(el)) return el.trim(); + }); + const otherApprovalCommittees = + obj.otherApprovalCommittees && + obj.otherApprovalCommittees + .toString() + .split(',') + .map(el => { + if (!isEmpty(el)) return el.trim(); + }); + + // Handle expected dates + const projectStartDate = moment(obj.projectStartDate, 'YYYY-MM-DD'); + const projectEndDate = moment(obj.projectEndDate, 'YYYY-MM-DD'); + const latestApprovalDate = moment(obj.latestApprovalDate, 'YYYY-MM-DD'); + const accessDate = moment(obj.accessDate, 'YYYY-MM-DD'); + + // Clean and assign to model + dataUseRegisters.push( + new DataUseRegister({ + ...(obj.projectTitle && { projectTitle: obj.projectTitle.toString().trim() }), + ...(obj.projectIdText && { projectIdText: obj.projectIdText.toString().trim() }), + ...(obj.organisationName && { organisationName: obj.organisationName.toString().trim() }), + ...(obj.organisationSector && { organisationSector: obj.organisationSector.toString().trim() }), + ...(obj.applicantId && { applicantId: obj.applicantId.toString().trim() }), + ...(obj.accreditedResearcherStatus && { accreditedResearcherStatus: obj.accreditedResearcherStatus.toString().trim() }), + ...(obj.sublicenceArrangements && { sublicenceArrangements: obj.sublicenceArrangements.toString().trim() }), + ...(obj.laySummary && { laySummary: obj.laySummary.toString().trim() }), + ...(obj.publicBenefitStatement && { publicBenefitStatement: obj.publicBenefitStatement.toString().trim() }), + ...(obj.requestCategoryType && { requestCategoryType: obj.requestCategoryType.toString().trim() }), + ...(obj.technicalSummary && { technicalSummary: obj.technicalSummary.toString().trim() }), + ...(obj.dataSensitivityLevel && { dataSensitivityLevel: obj.dataSensitivityLevel.toString().trim() }), + ...(obj.legalBasisForData && { legalBasisForData: obj.legalBasisForData.toString().trim() }), + ...(obj.nationalDataOptOut && { nationalDataOptOut: obj.nationalDataOptOut.toString().trim() }), + ...(obj.requestFrequency && { requestFrequency: obj.requestFrequency.toString().trim() }), + ...(obj.dataProcessingDescription && { dataProcessingDescription: obj.dataProcessingDescription.toString().trim() }), + ...(obj.confidentialDataDescription && { confidentialDataDescription: obj.confidentialDataDescription.toString().trim() }), + ...(obj.dataLocation && { dataLocation: obj.dataLocation.toString().trim() }), + ...(obj.privacyEnhancements && { privacyEnhancements: obj.privacyEnhancements.toString().trim() }), + ...(projectStartDate.isValid() && { projectStartDate }), + ...(projectEndDate.isValid() && { projectEndDate }), + ...(latestApprovalDate.isValid() && { latestApprovalDate }), + ...(accessDate.isValid() && { accessDate }), + ...(!isEmpty(datasetTitles) && { datasetTitles }), + ...(!isEmpty(datasetIds) && { datasetIds }), + ...(!isEmpty(datasetPids) && { datasetPids }), + ...(!isEmpty(gatewayApplicants) && { gatewayApplicants }), + ...(!isEmpty(nonGatewayApplicants) && { nonGatewayApplicants }), + ...(!isEmpty(fundersAndSponsors) && { fundersAndSponsors }), + ...(!isEmpty(researchOutputs) && { researchOutputs }), + ...(!isEmpty(otherApprovalCommittees) && { otherApprovalCommittees }), + ...(!isEmpty(relatedObjects) && { relatedObjects }), + activeflag: 'inReview', + publisher: teamId, + user: creatorUser._id, + updatedon: Date.now(), + lastActivity: Date.now(), + manualUpload: true + }) + ); + } + + return dataUseRegisters; +}; + +/** + * Get Linked Datasets + * + * @desc Accepts a comma separated string containing dataset names which can be in the form of text based names or URLs belonging to the Gateway which resolve to a dataset page, or a mix of both. + * The function separates URLs and uses regex to locate a suspected dataset PID to use in a search against the Gateway database. If a match is found, the entry is considered a linked dataset. + * Entries which cannot be matched are returned as named datasets. + * @param {String} datasetNames A comma separated string representation of the dataset names to attempt to find and link to existing Gateway datasets + * @returns {Object} An object containing linked and named datasets in separate arrays + */ +const getLinkedDatasets = async (datasetNames = []) => { + const unverifiedDatasetPids = []; + const namedDatasets = []; + const validLinkRegexp = new RegExp(`^${process.env.homeURL.replace('//', '//')}\/dataset\/([a-f|\\d|-]+)\/?$`, 'i'); + + for (const datasetName of datasetNames) { + const [, datasetPid] = validLinkRegexp.exec(datasetName) || []; + if (datasetPid) { + unverifiedDatasetPids.push(datasetPid); + } else { + namedDatasets.push(datasetName); + } + } + + const linkedDatasets = isEmpty(unverifiedDatasetPids) + ? [] + : (await datasetService.getDatasetsByPids(unverifiedDatasetPids)).map(dataset => { + return { datasetid: dataset.datasetid, name: dataset.name, pid: dataset.pid }; + }); + + return { linkedDatasets, namedDatasets }; +}; + +/** + * Get Linked Applicants + * + * @desc Accepts a comma separated string containing applicant names which can be in the form of text based names or URLs belonging to the Gateway which resolve to a users profile page, or a mix of both. + * The function separates URLs and uses regex to locate a suspected user ID to use in a search against the Gateway database. If a match is found, the entry is considered a Gateway applicant. + * Entries which cannot be matched are returned as non Gateway applicants. Failed attempts at adding URLs which do not resolve are excluded. + * @param {String} datasetNames A comma separated string representation of the applicant(s) names to attempt to find and link to existing Gateway users + * @returns {Object} An object containing Gateway applicants and non Gateway applicants in separate arrays + */ +const getLinkedApplicants = async (applicantNames = []) => { + const unverifiedUserIds = []; + const nonGatewayApplicants = []; + const validLinkRegexp = new RegExp(`^${process.env.homeURL.replace('//', '//')}\/person\/(\\d+)\/?$`, 'i'); + + for (const applicantName of applicantNames) { + const [, userId] = validLinkRegexp.exec(applicantName) || []; + if (userId) { + unverifiedUserIds.push(userId); + } else { + nonGatewayApplicants.push(applicantName); + } + } + + const gatewayApplicants = isEmpty(unverifiedUserIds) ? [] : (await getUsersByIds(unverifiedUserIds)).map(el => el._id); + + return { gatewayApplicants, nonGatewayApplicants }; +}; + +/** + * Build Related Objects + * + * @desc Accepts an array of datasets and outputs an array of related objects which can be assigned to an entity to show the relationship to the datasets. + * Related objects contain the 'objectId' (dataset version identifier), 'pid', 'objectType' (dataset), 'updated' date and 'user' that created the linkage. + * @param {Object} creatorUser A user object to allow the assignment of their name to the creator of the linkage + * @param {Array} datasets An array of dataset objects containing the necessary properties to assemble a related object record reference + * @returns {Array} An array containing the assembled related objects relative to the datasets provided + */ +const buildRelatedObjects = (creatorUser, datasets = []) => { + const { firstname, lastname } = creatorUser; + return datasets.map(dataset => { + const { datasetId: objectId, pid } = dataset; + return { + objectId, + pid, + objectType: 'dataset', + user: `${firstname} ${lastname}`, + updated: Date.now(), + }; + }); +}; + +export default { + buildDataUseRegisters, + getLinkedDatasets, + getLinkedApplicants, + buildRelatedObjects +}; diff --git a/src/resources/dataset/dataset.repository.js b/src/resources/dataset/dataset.repository.js index df4e1db3..ff7a13c7 100644 --- a/src/resources/dataset/dataset.repository.js +++ b/src/resources/dataset/dataset.repository.js @@ -20,7 +20,7 @@ export default class DatasetRepository extends Repository { return {}; } // Get dataset versions using pid - const query = { pid, fields:'datasetid,datasetVersion,activeflag' }; + const query = { pid, fields: 'datasetid,datasetVersion,activeflag' }; const options = { lean: true }; const datasets = await this.find(query, options); // Create revision structure @@ -34,4 +34,13 @@ export default class DatasetRepository extends Repository { return obj; }, {}); } + + getDatasetsByPids(pids) { + return this.dataset.aggregate([ + { $match: { pid: { $in: pids } } }, + { $project: { pid: 1, datasetid: 1, name: 1, createdAt: 1 } }, + { $sort: { createdAt: -1 } }, + { $group: { _id: '$pid', pid: { $first: '$pid' }, datasetid: { $first: '$datasetid' }, name: { $first: '$name' } } }, + ]); + } } diff --git a/src/resources/dataset/dataset.service.js b/src/resources/dataset/dataset.service.js index 5bd76087..6b09bfd7 100644 --- a/src/resources/dataset/dataset.service.js +++ b/src/resources/dataset/dataset.service.js @@ -110,4 +110,8 @@ export default class DatasetService { }); return dataset; } + + getDatasetsByPids(pids) { + return this.datasetRepository.getDatasetsByPids(pids); + } } diff --git a/src/resources/user/user.repository.js b/src/resources/user/user.repository.js index a72ab0ed..a67e041a 100644 --- a/src/resources/user/user.repository.js +++ b/src/resources/user/user.repository.js @@ -25,6 +25,10 @@ export async function getUserByUserId(id) { return await UserModel.findOne({ id }).exec(); } +export async function getUsersByIds(userIds) { + return await UserModel.find({ id: { $in: userIds } }, '_id').lean(); +} + export async function getServiceAccountByClientCredentials(clientId, clientSecret) { // 1. Locate service account by clientId, return undefined if no document located const id = clientId.toString(); From cb3b7b0b5cca2f23892d10719e6e0ff0becd6886 Mon Sep 17 00:00:00 2001 From: umma-pa Date: Mon, 27 Sep 2021 13:45:50 +0100 Subject: [PATCH 015/116] Data use reverse look up comment --- src/resources/dataUseRegister/dataUseRegister.route.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/resources/dataUseRegister/dataUseRegister.route.js b/src/resources/dataUseRegister/dataUseRegister.route.js index 7535461f..5303f947 100644 --- a/src/resources/dataUseRegister/dataUseRegister.route.js +++ b/src/resources/dataUseRegister/dataUseRegister.route.js @@ -125,9 +125,9 @@ router.get( (req, res) => dataUseRegisterController.getDataUseRegisters(req, res) ); -// {get} /api/v2/data-use-registers/{datauseregisterID} Data Use Register -// Return the details on the Data Use Register based on the Course ID(?) -// +// @route GET /api/v2/data-use-registers/{id} +// @desc Return the details on the Data Use Register based on the Course ID +// @access Public router.get('/:id', async (req, res) => { let id = parseInt(req.params.id); var query = Data.aggregate([ From 1e580c76fae5c3615f6f72708047449e072236e0 Mon Sep 17 00:00:00 2001 From: umma-pa Date: Mon, 27 Sep 2021 16:04:45 +0100 Subject: [PATCH 016/116] Reverse look up moved --- .../dataUseRegister.controller.js | 50 ++++++++++++++++- .../dataUseRegister/dataUseRegister.route.js | 56 +------------------ 2 files changed, 50 insertions(+), 56 deletions(-) diff --git a/src/resources/dataUseRegister/dataUseRegister.controller.js b/src/resources/dataUseRegister/dataUseRegister.controller.js index fd49ca21..ee379477 100644 --- a/src/resources/dataUseRegister/dataUseRegister.controller.js +++ b/src/resources/dataUseRegister/dataUseRegister.controller.js @@ -2,7 +2,7 @@ import Controller from '../base/controller'; import { logger } from '../utilities/logger'; import _ from 'lodash'; import constants from './../utilities/constants.util'; - +import { Data } from '../tool/data.model'; const logCategory = 'dataUseRegister'; export default class DataUseRegisterController extends Controller { @@ -25,6 +25,54 @@ export default class DataUseRegisterController extends Controller { // Find the dataUseRegister const options = { lean: true }; const dataUseRegister = await this.dataUseRegisterService.getDataUseRegister(id, req.query, options); + // Reverse look up + var query = Data.aggregate([ + { $match: { id: parseInt(req.params.id) } }, + { + $lookup: { + from: 'tools', + localField: 'creator', + foreignField: 'id', + as: 'creator', + }, + }, + ]); + query.exec((err, data) => { + if (data.length > 0) { + var p = Data.aggregate([ + { + $match: { + $and: [{ relatedObjects: { $elemMatch: { objectId: req.params.id } } }], + }, + }, + ]); + p.exec((err, relatedData) => { + relatedData.forEach(dat => { + dat.relatedObjects.forEach(x => { + if (x.objectId === req.params.id && dat.id !== req.params.id) { + let relatedObject = { + objectId: dat.id, + reason: x.reason, + objectType: dat.type, + user: x.user, + updated: x.updated, + }; + data[0].relatedObjects = [relatedObject, ...(data[0].relatedObjects || [])]; + } + }); + }); + + if (err) return res.json({ success: false, error: err }); + + return res.json({ + success: true, + data: data, + }); + }); + } else { + return res.status(404).send(`Data Use Register not found for Id: ${escape(id)}`); + } + }); // Return if no dataUseRegister found if (!dataUseRegister) { return res.status(404).json({ diff --git a/src/resources/dataUseRegister/dataUseRegister.route.js b/src/resources/dataUseRegister/dataUseRegister.route.js index 5303f947..5a35fb86 100644 --- a/src/resources/dataUseRegister/dataUseRegister.route.js +++ b/src/resources/dataUseRegister/dataUseRegister.route.js @@ -1,6 +1,6 @@ import express from 'express'; import DataUseRegisterController from './dataUseRegister.controller'; -import { Data } from '../tool/data.model'; + import { dataUseRegisterService } from './dependency'; import { logger } from '../utilities/logger'; import passport from 'passport'; @@ -125,60 +125,6 @@ router.get( (req, res) => dataUseRegisterController.getDataUseRegisters(req, res) ); -// @route GET /api/v2/data-use-registers/{id} -// @desc Return the details on the Data Use Register based on the Course ID -// @access Public -router.get('/:id', async (req, res) => { - let id = parseInt(req.params.id); - var query = Data.aggregate([ - { $match: { id: parseInt(req.params.id) } }, - { - $lookup: { - from: 'tools', - localField: 'creator', - foreignField: 'id', - as: 'creator', - }, - }, - ]); - query.exec((err, data) => { - if (data.length > 0) { - var p = Data.aggregate([ - { - $match: { - $and: [{ relatedObjects: { $elemMatch: { objectId: req.params.id } } }], - }, - }, - ]); - p.exec((err, relatedData) => { - relatedData.forEach(dat => { - dat.relatedObjects.forEach(x => { - if (x.objectId === req.params.id && dat.id !== req.params.id) { - let relatedObject = { - objectId: dat.id, - reason: x.reason, - objectType: dat.type, - user: x.user, - updated: x.updated, - }; - data[0].relatedObjects = [relatedObject, ...(data[0].relatedObjects || [])]; - } - }); - }); - - if (err) return res.json({ success: false, error: err }); - - return res.json({ - success: true, - data: data, - }); - }); - } else { - return res.status(404).send(`Data Use Register not found for Id: ${escape(id)}`); - } - }); -}); - // @route PUT /api/v2/data-use-registers/id // @desc Update the content of the data user register based on dataUseRegister ID provided // @access Public From f9c6affe3ba8270b414c9f725b0afa665ba938cb Mon Sep 17 00:00:00 2001 From: Robin Kavanagh Date: Wed, 29 Sep 2021 16:02:15 +0100 Subject: [PATCH 017/116] Removed unnecessary replace function before regex check --- src/resources/dataUseRegister/dataUseRegister.util.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/resources/dataUseRegister/dataUseRegister.util.js b/src/resources/dataUseRegister/dataUseRegister.util.js index a76cf8d4..4f5673f5 100644 --- a/src/resources/dataUseRegister/dataUseRegister.util.js +++ b/src/resources/dataUseRegister/dataUseRegister.util.js @@ -138,7 +138,7 @@ const buildDataUseRegisters = async (creatorUser, teamId, dataUses = []) => { const getLinkedDatasets = async (datasetNames = []) => { const unverifiedDatasetPids = []; const namedDatasets = []; - const validLinkRegexp = new RegExp(`^${process.env.homeURL.replace('//', '//')}\/dataset\/([a-f|\\d|-]+)\/?$`, 'i'); + const validLinkRegexp = new RegExp(`^${process.env.homeURL}\/dataset\/([a-f|\\d|-]+)\/?$`, 'i'); for (const datasetName of datasetNames) { const [, datasetPid] = validLinkRegexp.exec(datasetName) || []; @@ -170,7 +170,7 @@ const getLinkedDatasets = async (datasetNames = []) => { const getLinkedApplicants = async (applicantNames = []) => { const unverifiedUserIds = []; const nonGatewayApplicants = []; - const validLinkRegexp = new RegExp(`^${process.env.homeURL.replace('//', '//')}\/person\/(\\d+)\/?$`, 'i'); + const validLinkRegexp = new RegExp(`^${process.env.homeURL}\/person\/(\\d+)\/?$`, 'i'); for (const applicantName of applicantNames) { const [, userId] = validLinkRegexp.exec(applicantName) || []; From 05a22119f4fc7de7c2f2ad04dfc6ab3bb3d8dc9b Mon Sep 17 00:00:00 2001 From: Paul McCafferty <> Date: Thu, 30 Sep 2021 11:53:24 +0100 Subject: [PATCH 018/116] Adding back in the statics --- src/config/account.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/config/account.js b/src/config/account.js index 4f4d103a..01c5be29 100755 --- a/src/config/account.js +++ b/src/config/account.js @@ -49,7 +49,7 @@ class Account { return claim; } - async findByFederated(provider, claims) { + static async findByFederated(provider, claims) { const id = `${provider}.${claims.sub}`; if (!logins.get(id)) { logins.set(id, new Account(id, claims)); @@ -57,7 +57,7 @@ class Account { return logins.get(id); } - async findByLogin(login) { + static async findByLogin(login) { if (!logins.get(login)) { logins.set(login, new Account(login)); } @@ -65,7 +65,7 @@ class Account { return logins.get(login); } - async findAccount(ctx, id) { + static async findAccount(ctx, id) { // eslint-disable-line no-unused-vars // token is a reference to the token used for which a given account is being loaded, // it is undefined in scenarios where account claims are returned from authorization endpoint From 40c8f2303e51d586b12e1be9fe4798793bb3a9d8 Mon Sep 17 00:00:00 2001 From: Richard Date: Thu, 30 Sep 2021 12:31:44 +0100 Subject: [PATCH 019/116] Adding missing filterService import --- src/resources/course/course.repository.js | 1 + 1 file changed, 1 insertion(+) diff --git a/src/resources/course/course.repository.js b/src/resources/course/course.repository.js index 22ba1425..488b21bc 100644 --- a/src/resources/course/course.repository.js +++ b/src/resources/course/course.repository.js @@ -6,6 +6,7 @@ import emailGenerator from '../utilities/emailGenerator.util'; import helper from '../utilities/helper.util'; import { utils } from '../auth'; import { ROLES } from '../user/user.roles'; +import { filtersService } from '../filters/dependency'; const hdrukEmail = `enquiry@healthdatagateway.org`; const urlValidator = require('../utilities/urlValidator'); const inputSanitizer = require('../utilities/inputSanitizer'); From 509d89affddd69331cf355c60ebd6a16a23850b1 Mon Sep 17 00:00:00 2001 From: Robin Kavanagh Date: Fri, 1 Oct 2021 07:02:02 +0100 Subject: [PATCH 020/116] Added generic middleware for handling result limit in v2 endpoints --- src/config/middleware.js | 24 +++++++++++++++++++ src/resources/base/repository.js | 2 +- src/resources/cohort/cohort.route.js | 3 ++- .../cohortprofiling/cohortprofiling.route.js | 4 +++- src/resources/course/v2/course.route.js | 3 ++- src/resources/dataset/v2/dataset.route.js | 3 ++- src/resources/paper/v2/paper.route.js | 3 ++- src/resources/project/v2/project.route.js | 3 ++- src/resources/tool/v2/tool.route.js | 3 ++- 9 files changed, 40 insertions(+), 8 deletions(-) create mode 100644 src/config/middleware.js diff --git a/src/config/middleware.js b/src/config/middleware.js new file mode 100644 index 00000000..d2e915ef --- /dev/null +++ b/src/config/middleware.js @@ -0,0 +1,24 @@ +import { has, isNaN } from 'lodash'; + +export const resultLimit = (req, res, next, allowedLimit) => { + let error; + if(has(req.query, 'limit')) { + const requestedLimit = parseInt(req.query.limit); + + if(isNaN(requestedLimit)) { + error = `The result limit parameter provided must be a numeric value.`; + } + else if (requestedLimit > allowedLimit){ + error = `Maximum request limit exceeded. You may only request up to a maximum of ${allowedLimit} records per page. Please use the page query parameter to request further data.`; + } + } + + if (error) { + return res.status(400).json({ + success: false, + message: error, + }); + } + + next(); +}; diff --git a/src/resources/base/repository.js b/src/resources/base/repository.js index 3294ab52..ea73d430 100644 --- a/src/resources/base/repository.js +++ b/src/resources/base/repository.js @@ -48,7 +48,7 @@ export default class Repository { // Pagination const page = query.page * 1 || 1; - const limit = query.limit * 1 || null; + const limit = query.limit * 1 || 500; const skip = (page - 1) * limit; results = results.skip(skip).limit(limit); diff --git a/src/resources/cohort/cohort.route.js b/src/resources/cohort/cohort.route.js index 6debae9f..705d578a 100644 --- a/src/resources/cohort/cohort.route.js +++ b/src/resources/cohort/cohort.route.js @@ -2,6 +2,7 @@ import express from 'express'; import CohortController from './cohort.controller'; import { cohortService } from './dependency'; import { logger } from '../utilities/logger'; +import { resultLimit } from '../../config/middleware'; const router = express.Router(); const cohortController = new CohortController(cohortService); @@ -17,7 +18,7 @@ router.get('/:id', logger.logRequestMiddleware({ logCategory, action: 'Viewed co // @route GET /api/v1/cohorts // @desc Returns a collection of cohorts based on supplied query parameters // @access Public -router.get('/', logger.logRequestMiddleware({ logCategory, action: 'Viewed cohorts data' }), (req, res) => +router.get('/', (req, res, next) => resultLimit(req, res, next, 100), logger.logRequestMiddleware({ logCategory, action: 'Viewed cohorts data' }), (req, res) => cohortController.getCohorts(req, res) ); diff --git a/src/resources/cohortprofiling/cohortprofiling.route.js b/src/resources/cohortprofiling/cohortprofiling.route.js index 49bb0563..ce7df438 100644 --- a/src/resources/cohortprofiling/cohortprofiling.route.js +++ b/src/resources/cohortprofiling/cohortprofiling.route.js @@ -1,7 +1,9 @@ import express from 'express'; import CohortProfilingController from './cohortprofiling.controller'; import { cohortProfilingService } from './dependency'; +import { resultLimit } from '../../config/middleware'; import multer from 'multer'; + const upload = multer(); const cohortProfilingController = new CohortProfilingController(cohortProfilingService); @@ -15,7 +17,7 @@ router.get('/:pid/:tableName/:variable', (req, res) => cohortProfilingController // @route GET api/v1/cohortprofiling // @desc Returns a collection of cohort profiling data based on supplied query parameters // @access Public -router.get('/', (req, res) => cohortProfilingController.getCohortProfiling(req, res)); +router.get('/', (req, res, next) => resultLimit(req, res, next, 100), (req, res) => cohortProfilingController.getCohortProfiling(req, res)); // @route POST api/v1/cohortprofiling // @desc Consumes a JSON file containing cohort profiling data, transforms it and saves to MongoDB. diff --git a/src/resources/course/v2/course.route.js b/src/resources/course/v2/course.route.js index 8c6aab39..121e6739 100644 --- a/src/resources/course/v2/course.route.js +++ b/src/resources/course/v2/course.route.js @@ -1,6 +1,7 @@ import express from 'express'; import CourseController from './course.controller'; import { courseService } from './dependency'; +import { resultLimit } from '../../../config/middleware'; const router = express.Router(); const courseController = new CourseController(courseService); @@ -13,6 +14,6 @@ router.get('/:id', (req, res) => courseController.getCourse(req, res)); // @route GET /api/v2/courses // @desc Returns a collection of courses based on supplied query parameters // @access Public -router.get('/', (req, res) => courseController.getCourses(req, res)); +router.get('/', (req, res, next) => resultLimit(req, res, next, 100), (req, res) => courseController.getCourses(req, res)); module.exports = router; diff --git a/src/resources/dataset/v2/dataset.route.js b/src/resources/dataset/v2/dataset.route.js index b0df0f01..6acac939 100644 --- a/src/resources/dataset/v2/dataset.route.js +++ b/src/resources/dataset/v2/dataset.route.js @@ -1,6 +1,7 @@ import express from 'express'; import DatasetController from '../dataset.controller'; import { datasetService } from '../dependency'; +import { resultLimit } from '../../../config/middleware'; const router = express.Router(); const datasetController = new DatasetController(datasetService); @@ -13,6 +14,6 @@ router.get('/:id', (req, res) => datasetController.getDataset(req, res)); // @route GET /api/v2/datasets // @desc Returns a collection of datasets based on supplied query parameters // @access Public -router.get('/', (req, res) => datasetController.getDatasets(req, res)); +router.get('/', (req, res, next) => resultLimit(req, res, next, 100), (req, res) => datasetController.getDatasets(req, res)); module.exports = router; diff --git a/src/resources/paper/v2/paper.route.js b/src/resources/paper/v2/paper.route.js index 40c14ab0..ef60cfbf 100644 --- a/src/resources/paper/v2/paper.route.js +++ b/src/resources/paper/v2/paper.route.js @@ -1,6 +1,7 @@ import express from 'express'; import PaperController from '../paper.controller'; import { paperService } from '../dependency'; +import { resultLimit } from '../../../config/middleware'; const router = express.Router(); const paperController = new PaperController(paperService); @@ -13,6 +14,6 @@ router.get('/:id', (req, res) => paperController.getPaper(req, res)); // @route GET /api/v2/papers // @desc Returns a collection of papers based on supplied query parameters // @access Public -router.get('/', (req, res) => paperController.getPapers(req, res)); +router.get('/', (req, res, next) => resultLimit(req, res, next, 100), (req, res) => paperController.getPapers(req, res)); module.exports = router; diff --git a/src/resources/project/v2/project.route.js b/src/resources/project/v2/project.route.js index fed79dcb..a96a8ede 100644 --- a/src/resources/project/v2/project.route.js +++ b/src/resources/project/v2/project.route.js @@ -1,6 +1,7 @@ import express from 'express'; import ProjectController from '../project.controller'; import { projectService } from '../dependency'; +import { resultLimit } from '../../../config/middleware'; const router = express.Router(); const projectController = new ProjectController(projectService); @@ -13,6 +14,6 @@ router.get('/:id', (req, res) => projectController.getProject(req, res)); // @route GET /api/v2/projects // @desc Returns a collection of projects based on supplied query parameters // @access Public -router.get('/', (req, res) => projectController.getProjects(req, res)); +router.get('/', (req, res, next) => resultLimit(req, res, next, 100), (req, res) => projectController.getProjects(req, res)); module.exports = router; diff --git a/src/resources/tool/v2/tool.route.js b/src/resources/tool/v2/tool.route.js index 4eb0a7de..ec4d1ca4 100644 --- a/src/resources/tool/v2/tool.route.js +++ b/src/resources/tool/v2/tool.route.js @@ -1,6 +1,7 @@ import express from 'express'; import ToolController from './tool.controller'; import { toolService } from './dependency'; +import { resultLimit } from '../../../config/middleware'; const router = express.Router(); const toolController = new ToolController(toolService); @@ -13,6 +14,6 @@ router.get('/:id', (req, res) => toolController.getTool(req, res)); // @route GET /api/v2/tools // @desc Returns a collection of tools based on supplied query parameters // @access Public -router.get('/', (req, res) => toolController.getTools(req, res)); +router.get('/', (req, res, next) => resultLimit(req, res, next, 100), (req, res) => toolController.getTools(req, res)); module.exports = router; From 414c3a8a0c700f5fc990561780db68f72b1c1276 Mon Sep 17 00:00:00 2001 From: Robin Kavanagh Date: Fri, 1 Oct 2021 08:58:42 +0100 Subject: [PATCH 021/116] Added unit tests for middleware limiter --- test/middleware.test.js | 66 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 test/middleware.test.js diff --git a/test/middleware.test.js b/test/middleware.test.js new file mode 100644 index 00000000..9dd41fbb --- /dev/null +++ b/test/middleware.test.js @@ -0,0 +1,66 @@ +import { resultLimit } from '../src/config/middleware'; + +describe('resultLimit', () => { + const nextFunction = jest.fn(); + + const mockResponse = () => { + const res = {}; + res.status = jest.fn().mockReturnValue(res); + res.json = jest.fn().mockReturnValue(res); + return res; + }; + + const allowedLimit = 100; + + it('should return a 400 response code with the correct reason when the requested limit is non numeric', () => { + const expectedResponse = { + success: false, + message: 'The result limit parameter provided must be a numeric value.', + }; + + const req = { query: { limit: 'one hundred' } }; + const res = mockResponse(); + + resultLimit(req, res, nextFunction, allowedLimit); + + expect(res.status).toHaveBeenCalledWith(400); + expect(res.json).toHaveBeenCalledWith(expectedResponse); + }); + + it('should return a 400 response code with the correct reason when the maximum allowed limit is exceeded', () => { + const expectedResponse = { + success: false, + message: `Maximum request limit exceeded. You may only request up to a maximum of ${allowedLimit} records per page. Please use the page query parameter to request further data.`, + }; + + const req = { query: { limit: 101 } }; + const res = mockResponse(); + + resultLimit(req, res, nextFunction, allowedLimit); + + expect(res.status).toHaveBeenCalledWith(400); + expect(res.json).toHaveBeenCalledWith(expectedResponse); + }); + + it('should invoke the next function when no request limit is provided', () => { + const req = {}; + const res = mockResponse(); + + resultLimit(req, res, nextFunction, allowedLimit); + + expect(res.status.mock.calls.length).toBe(0); + expect(res.json.mock.calls.length).toBe(0); + expect(nextFunction.mock.calls.length).toBe(1); + }); + + it('should invoke the next function when the requested limit is valid', () => { + const req = { query: { limit: 100 } }; + const res = mockResponse(); + + resultLimit(req, res, nextFunction, allowedLimit); + + expect(res.status.mock.calls.length).toBe(0); + expect(res.json.mock.calls.length).toBe(0); + expect(nextFunction.mock.calls.length).toBe(1); + }); +}); From 7f79b42a9e64e75c204463ae9afcdbdc01a7f7a0 Mon Sep 17 00:00:00 2001 From: Richard Date: Fri, 1 Oct 2021 09:21:03 +0100 Subject: [PATCH 022/116] Populate relatedObjects with datasetid of dataset used in query --- src/resources/cohort/cohort.service.js | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/resources/cohort/cohort.service.js b/src/resources/cohort/cohort.service.js index 9429889b..f84ae55f 100644 --- a/src/resources/cohort/cohort.service.js +++ b/src/resources/cohort/cohort.service.js @@ -1,4 +1,5 @@ import { v4 as uuidv4 } from 'uuid'; +import { Data } from '../tool/data.model'; export default class CohortService { constructor(cohortRepository) { @@ -32,14 +33,17 @@ export default class CohortService { } // 3. Extract PIDs from cohort object so we can build up related objects - let pids = body.cohort.input.collections.map(collection => { - return collection.external_id; + let datasetIdentifiersPromises = await body.cohort.input.collections.map(async collection => { + let dataset = await Data.findOne({ pid: collection.external_id, activeflag: 'active' }, { datasetid: 1 }).lean(); + return { pid: collection.external_id, datasetId: dataset.datasetid }; }); + let datasetIdentifiers = await Promise.all(datasetIdentifiersPromises); let relatedObjects = []; - pids.forEach(pid => { + datasetIdentifiers.forEach(datasetIdentifier => { relatedObjects.push({ objectType: 'dataset', - pid, + pid: datasetIdentifier.pid, + objectId: datasetIdentifier.datasetId, isLocked: true, }); }); From 74180b1fc5452f325e9af7eb84e651d5d8ba54dd Mon Sep 17 00:00:00 2001 From: Robin Kavanagh Date: Fri, 1 Oct 2021 09:41:31 +0100 Subject: [PATCH 023/116] Updated swagger docs to include limitation of 100 records by default --- swagger.yaml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/swagger.yaml b/swagger.yaml index f572bea9..613451b0 100644 --- a/swagger.yaml +++ b/swagger.yaml @@ -1823,7 +1823,7 @@ paths: summary: Returns a list of dataset objects tags: - Datasets v2.0 - description: Version 2.0 of the datasets API introduces a large number of parameterised query string options to aid requests in collecting the data that is most relevant for a given use case. The query parameters defined below support a variety of comparison operators such as equals, contains, greater than, and less than. Using dot notation, any field can be queried, please see some examples below. + description: Version 2.0 of the datasets API introduces a large number of parameterised query string options to aid requests in collecting the data that is most relevant for a given use case. The query parameters defined below support a variety of comparison operators such as equals, contains, greater than, and less than. Using dot notation, any field can be queried, please see some examples below. Note - This response is limited to 100 records by default. Please use the 'page' query parameter to access records beyond the first 100. The 'limit' query parameter can therefore only be specified up to a maximum of 100. parameters: - name: search in: query @@ -2201,7 +2201,7 @@ paths: schema: type: boolean example: true - description: Version 2.0 of the courses API introduces a large number of parameterised query string options to aid requests in collecting the data that is most relevant for a given use case. The query parameters defined below support a variety of comparison operators such as equals, contains, greater than, and less than. Using dot notation, any field can be queried, please see some examples below. + description: Version 2.0 of the courses API introduces a large number of parameterised query string options to aid requests in collecting the data that is most relevant for a given use case. The query parameters defined below support a variety of comparison operators such as equals, contains, greater than, and less than. Using dot notation, any field can be queried, please see some examples below. Note - This response is limited to 100 records by default. Please use the 'page' query parameter to access records beyond the first 100. The 'limit' query parameter can therefore only be specified up to a maximum of 100. responses: '200': description: Successful response containing a list of projects matching query parameters @@ -2489,7 +2489,7 @@ paths: schema: type: boolean example: true - description: Version 2.0 of the courses API introduces a large number of parameterised query string options to aid requests in collecting the data that is most relevant for a given use case. The query parameters defined below support a variety of comparison operators such as equals, contains, greater than, and less than. Using dot notation, any field can be queried, please see some examples below. + description: Version 2.0 of the courses API introduces a large number of parameterised query string options to aid requests in collecting the data that is most relevant for a given use case. The query parameters defined below support a variety of comparison operators such as equals, contains, greater than, and less than. Using dot notation, any field can be queried, please see some examples below. Note - This response is limited to 100 records by default. Please use the 'page' query parameter to access records beyond the first 100. The 'limit' query parameter can therefore only be specified up to a maximum of 100. responses: '200': description: Successful response containing a list of papers matching query parameters @@ -2771,7 +2771,7 @@ paths: schema: type: boolean example: true - description: Version 2.0 of the courses API introduces a large number of parameterised query string options to aid requests in collecting the data that is most relevant for a given use case. The query parameters defined below support a variety of comparison operators such as equals, contains, greater than, and less than. Using dot notation, any field can be queried, please see some examples below. + description: Version 2.0 of the courses API introduces a large number of parameterised query string options to aid requests in collecting the data that is most relevant for a given use case. The query parameters defined below support a variety of comparison operators such as equals, contains, greater than, and less than. Using dot notation, any field can be queried, please see some examples below. Note - This response is limited to 100 records by default. Please use the 'page' query parameter to access records beyond the first 100. The 'limit' query parameter can therefore only be specified up to a maximum of 100. responses: '200': description: Successful response containing a list of tools matching query parameters @@ -2836,7 +2836,7 @@ paths: schema: type: boolean example: true - description: Version 2.0 of the courses API introduces a large number of parameterised query string options to aid requests in collecting the data that is most relevant for a given use case. The query parameters defined below support a variety of comparison operators such as equals, contains, greater than, and less than. Using dot notation, any field can be queried, please see some examples below. + description: Version 2.0 of the courses API introduces a large number of parameterised query string options to aid requests in collecting the data that is most relevant for a given use case. The query parameters defined below support a variety of comparison operators such as equals, contains, greater than, and less than. Using dot notation, any field can be queried, please see some examples below. Note - This response is limited to 100 records by default. Please use the 'page' query parameter to access records beyond the first 100. The 'limit' query parameter can therefore only be specified up to a maximum of 100. tags: - Courses v2.0 responses: @@ -2871,6 +2871,7 @@ components: flows: clientCredentials: tokenUrl: 'https://api.www.healthdatagateway.org/oauth/token' + scopes: {} cookieAuth: type: http scheme: jwt From 64cf8cd05dca1a25f0a6b6b5c186db197f532861 Mon Sep 17 00:00:00 2001 From: Robin Kavanagh Date: Fri, 1 Oct 2021 13:50:44 +0100 Subject: [PATCH 024/116] Adding entry point for DUR creation --- src/resources/datarequest/datarequest.controller.js | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/resources/datarequest/datarequest.controller.js b/src/resources/datarequest/datarequest.controller.js index fa558f58..fe35c462 100644 --- a/src/resources/datarequest/datarequest.controller.js +++ b/src/resources/datarequest/datarequest.controller.js @@ -671,25 +671,28 @@ export default class DataRequestController extends Controller { //Update any connected version trees this.dataRequestService.updateVersionStatus(accessRecord, accessRecord.applicationStatus); - if (accessRecord.applicationStatus === constants.applicationStatuses.APPROVED) - await this.activityLogService.logActivity(constants.activityLogEvents.APPLICATION_APPROVED, { + if (accessRecord.applicationStatus === constants.applicationStatuses.APPROVED) { + this.dataUseRegisterService.createDataUseRegister(accessRecord); + this.activityLogService.logActivity(constants.activityLogEvents.APPLICATION_APPROVED, { accessRequest: accessRecord, user: req.user, }); + } else if (accessRecord.applicationStatus === constants.applicationStatuses.APPROVEDWITHCONDITIONS) { - await this.activityLogService.logActivity(constants.activityLogEvents.APPLICATION_APPROVED_WITH_CONDITIONS, { + this.dataUseRegisterService.createDataUseRegister(accessRecord); + this.activityLogService.logActivity(constants.activityLogEvents.APPLICATION_APPROVED_WITH_CONDITIONS, { accessRequest: accessRecord, user: req.user, }); } else if (accessRecord.applicationStatus === constants.applicationStatuses.REJECTED) { - await this.activityLogService.logActivity(constants.activityLogEvents.APPLICATION_REJECTED, { + this.activityLogService.logActivity(constants.activityLogEvents.APPLICATION_REJECTED, { accessRequest: accessRecord, user: req.user, }); } // Send notifications to custodian team, main applicant and contributors regarding status change - await this.createNotifications( + this.createNotifications( constants.notificationTypes.STATUSCHANGE, { applicationStatus, applicationStatusDesc }, accessRecord, From 94ceff76d8ccb2050f4d9bc7da835f7062df238c Mon Sep 17 00:00:00 2001 From: Richard Date: Fri, 1 Oct 2021 15:21:43 +0100 Subject: [PATCH 025/116] Added datasetPids to Cohort model to easily capture pids of datasets used for cohort query --- src/resources/cohort/cohort.model.js | 1 + src/resources/cohort/cohort.service.js | 3 +++ 2 files changed, 4 insertions(+) diff --git a/src/resources/cohort/cohort.model.js b/src/resources/cohort/cohort.model.js index 5c8676f8..de7310e8 100644 --- a/src/resources/cohort/cohort.model.js +++ b/src/resources/cohort/cohort.model.js @@ -16,6 +16,7 @@ const cohortSchema = new Schema( changeLog: String, updatedAt: Date, lastRefresh: Date, + datasetPids: [], // fields from RQuest request_id: String, diff --git a/src/resources/cohort/cohort.service.js b/src/resources/cohort/cohort.service.js index f84ae55f..546ce388 100644 --- a/src/resources/cohort/cohort.service.js +++ b/src/resources/cohort/cohort.service.js @@ -39,7 +39,9 @@ export default class CohortService { }); let datasetIdentifiers = await Promise.all(datasetIdentifiersPromises); let relatedObjects = []; + let datasetPids = []; datasetIdentifiers.forEach(datasetIdentifier => { + datasetPids.push(datasetIdentifier.pid); relatedObjects.push({ objectType: 'dataset', pid: datasetIdentifier.pid, @@ -63,6 +65,7 @@ export default class CohortService { cohort: body.cohort, items: body.items, rquestRelatedObjects: body.relatedObjects, + datasetPids, relatedObjects, }; return this.cohortRepository.addCohort(document); From f46d7bf660675eb58be7d0e1a518d5956c6c40a7 Mon Sep 17 00:00:00 2001 From: Robin Kavanagh Date: Tue, 5 Oct 2021 07:24:25 +0100 Subject: [PATCH 026/116] Updated JSDocs --- .../dataUseRegister/dataUseRegister.util.js | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/src/resources/dataUseRegister/dataUseRegister.util.js b/src/resources/dataUseRegister/dataUseRegister.util.js index 9e671cc8..a0573566 100644 --- a/src/resources/dataUseRegister/dataUseRegister.util.js +++ b/src/resources/dataUseRegister/dataUseRegister.util.js @@ -3,7 +3,6 @@ import { isEmpty } from 'lodash'; import DataUseRegister from './dataUseRegister.entity'; import { getUsersByIds } from '../user/user.repository'; import { datasetService } from '../dataset/dependency'; -import constantsUtil from '../utilities/constants.util'; /** * Build Data Use Registers @@ -214,6 +213,16 @@ const buildRelatedDatasets = (creatorUser, datasets = [], manualUpload = true) = }); }; +/** + * Extract Form Applicants + * + * @desc Accepts an array of authors and object containing answers from a Data Access Request application and extracts the names of non Gateway applicants as provided in the form, + * and extracts registered Gateway applicants, combining them before de-duplicating where match is found. + * @param {Array} authors An array of user documents representing contributors and the main applicant to a Data Access Request application + * @param {Object} applicationQuestionAnswers An object of key pairs containing the question identifiers and answers to the questions taken from a Data Access Request application + * @returns {Object} An object containing two arrays, the first being representative of registered Gateway users in the form of their identifying _id + * and the second array being the names of applicants who were extracted from the question answers object passed in but did not match any of the registered users provided in authors + */ const extractFormApplicants = (authors = [], applicationQuestionAnswers = {}) => { const gatewayApplicants = authors.map(el => el._id); const gatewayApplicantsNames = authors.map(el => `${el.firstname.trim()} ${el.lastname.trim()}`); @@ -229,6 +238,13 @@ const extractFormApplicants = (authors = [], applicationQuestionAnswers = {}) => return { gatewayApplicants, nonGatewayApplicants }; }; +/** + * Extract Funders And Sponsors + * + * @desc Accepts an object containing answers from a Data Access Request application and extracts funders and sponsors names from the specific sections where these questions are asked. + * @param {Object} applicationQuestionAnswers An object of key pairs containing the question identifiers and answers to the questions taken from a Data Access Request application + * @returns {Array} An array containing the organisation names provided as funders and sponsors + */ const extractFundersAndSponsors = (applicationQuestionAnswers = {}) => { return Object.keys(applicationQuestionAnswers) .filter( From d3e4d606e356e006ff33ff9419696f34b9de45cd Mon Sep 17 00:00:00 2001 From: Richard Date: Tue, 5 Oct 2021 13:17:39 +0100 Subject: [PATCH 027/116] Adding filterCriteria array for easy access to criteria used in cohort query --- src/resources/cohort/cohort.model.js | 2 ++ src/resources/cohort/cohort.service.js | 15 ++++++++++++++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/src/resources/cohort/cohort.model.js b/src/resources/cohort/cohort.model.js index de7310e8..6cd0a9fe 100644 --- a/src/resources/cohort/cohort.model.js +++ b/src/resources/cohort/cohort.model.js @@ -8,6 +8,7 @@ const cohortSchema = new Schema( pid: String, type: String, name: String, + description: String, activeflag: String, userId: Number, uploaders: [], @@ -17,6 +18,7 @@ const cohortSchema = new Schema( updatedAt: Date, lastRefresh: Date, datasetPids: [], + filterCriteria: [], // fields from RQuest request_id: String, diff --git a/src/resources/cohort/cohort.service.js b/src/resources/cohort/cohort.service.js index 546ce388..fbb7d3b1 100644 --- a/src/resources/cohort/cohort.service.js +++ b/src/resources/cohort/cohort.service.js @@ -50,7 +50,17 @@ export default class CohortService { }); }); - // 4. Build document object and save to DB + // 4. Extract filter criteria used in query + let filterCriteria = []; + body.cohort.input.cohorts.forEach(cohort => { + cohort.groups.forEach(group => { + group.rules.forEach(rule => { + filterCriteria.push(rule.value); + }); + }); + }); + + // 5. Build document object and save to DB const document = { id: uniqueId, pid: uuid, @@ -66,7 +76,10 @@ export default class CohortService { items: body.items, rquestRelatedObjects: body.relatedObjects, datasetPids, + filterCriteria, relatedObjects, + description: '', + publicflag: true, }; return this.cohortRepository.addCohort(document); } From db0d4945e2e11a3796d03245dc643017574c9840 Mon Sep 17 00:00:00 2001 From: Robin Kavanagh Date: Tue, 5 Oct 2021 13:43:46 +0100 Subject: [PATCH 028/116] Added unit tests for extracting authors/applicants/funders/sponsros from application form --- .../__mocks__/dataUseRegisters.js | 725 +++++++++--------- .../__tests__/dataUseRegister.util.test.js | 63 +- 2 files changed, 438 insertions(+), 350 deletions(-) diff --git a/src/resources/dataUseRegister/__mocks__/dataUseRegisters.js b/src/resources/dataUseRegister/__mocks__/dataUseRegisters.js index 0015e603..abbdbf87 100644 --- a/src/resources/dataUseRegister/__mocks__/dataUseRegisters.js +++ b/src/resources/dataUseRegister/__mocks__/dataUseRegisters.js @@ -1,363 +1,392 @@ export const dataUseRegisterUploads = [ - { - "projectTitle": "This a test data use register", - "projectIdText": "this is the project id", - "datasetNames": [ - "This is the dataset title", "http://localhost:3000/dataset/f725187f-7352-482b-a43b-64ebc96e66f2", "http://localhost:3000/dataset/c6d6bbd3-74ed-46af-841d-ac5e05f4da41", "http://localhost:3000/dataset/e55df485-5acd-4606-bbb8-668d4c06380a" - ], - "applicantNames": [ - " Michael Donnelly", "http://localhost:3000/person/8495781222000176", "http://localhost:3000/person/4495285946631793" - ], - "organisationName": "organisation name", - "organisationSector": "organisation sector", - "applicantId": "applicant id", - "fundersAndSponsors": "funder1 , funder2 , funder3 ", - "accreditedResearcherStatus": "accredited Researcher Status", - "sublicenceArrangements": "sublicence Arrangements", - "laySummary": "lay Summary", - "publicBenefitStatement": "public Benefit Statement", - "requestCategoryType": "request Category Type", - "technicalSummary": "technical Summary", - "otherApprovalCommittees": "other Approval Committees", - "projectStartDate": "2021-09-25", - "projectEndDate": "2021-09-30", - "latestApprovalDate": "2021-09-21", - "dataSensitivityLevel": "data Sensitivity Level", - "legalBasisForData": "legal Basis For Data", - "dutyOfConfidentiality": "duty Of Confidentiality", - "nationalDataOptOut": "national Data Opt Out", - "requestFrequency": "request Frequency", - "dataProcessingDescription": "data Processing Description", - "confidentialDataDescription": "confidential Data Description", - "accessDate": "2021-09-26", - "dataLocation": "data Location", - "privacyEnhancements": "privacy Enhancements", - "researchOutputs": "research Outputs" - }, - { - "projectTitle": "This is another test data use register", - "projectIdText": "this is the other project id", - "datasetNames": [ - "This is the dataset title", "http://localhost:3000/dataset/f725187f-7352-482b-a43b-64ebc96e66f2", "http://localhost:3000/dataset/c6d6bbd3-74ed-46af-841d-ac5e05f4da41", "http://localhost:3000/dataset/e55df485-5acd-4606-bbb8-668d4c06380a" - ], - "applicantNames": [ - " Michael Donnelly", "http://localhost:3000/person/8495781222000176", "http://localhost:3000/person/4495285946631793" - ], - "organisationName": "organisation name", - "organisationSector": "organisation sector", - "applicantId": "applicant id", - "fundersAndSponsors": "funder1 , funder2 , funder3 ", - "accreditedResearcherStatus": "accredited Researcher Status", - "sublicenceArrangements": "sublicence Arrangements", - "laySummary": "other lay Summary", - "publicBenefitStatement": "public Benefit Statement", - "requestCategoryType": "request Category Type", - "technicalSummary": "technical Summary", - "otherApprovalCommittees": "other Approval Committees", - "projectStartDate": "2021-09-25", - "projectEndDate": "2021-09-30", - "latestApprovalDate": "2021-09-21", - "dataSensitivityLevel": "data Sensitivity Level", - "legalBasisForData": "legal Basis For Data", - "dutyOfConfidentiality": "duty Of Confidentiality", - "nationalDataOptOut": "national Data Opt Out", - "requestFrequency": "request Frequency", - "dataProcessingDescription": "data Processing Description", - "confidentialDataDescription": "confidential Data Description", - "accessDate": "2021-09-26", - "dataLocation": "data Location", - "privacyEnhancements": "privacy Enhancements", - "researchOutputs": "research Outputs" - } -] + { + projectTitle: 'This a test data use register', + projectIdText: 'this is the project id', + datasetNames: [ + 'This is the dataset title', + 'http://localhost:3000/dataset/f725187f-7352-482b-a43b-64ebc96e66f2', + 'http://localhost:3000/dataset/c6d6bbd3-74ed-46af-841d-ac5e05f4da41', + 'http://localhost:3000/dataset/e55df485-5acd-4606-bbb8-668d4c06380a', + ], + applicantNames: [' Michael Donnelly', 'http://localhost:3000/person/8495781222000176', 'http://localhost:3000/person/4495285946631793'], + organisationName: 'organisation name', + organisationSector: 'organisation sector', + applicantId: 'applicant id', + fundersAndSponsors: 'funder1 , funder2 , funder3 ', + accreditedResearcherStatus: 'accredited Researcher Status', + sublicenceArrangements: 'sublicence Arrangements', + laySummary: 'lay Summary', + publicBenefitStatement: 'public Benefit Statement', + requestCategoryType: 'request Category Type', + technicalSummary: 'technical Summary', + otherApprovalCommittees: 'other Approval Committees', + projectStartDate: '2021-09-25', + projectEndDate: '2021-09-30', + latestApprovalDate: '2021-09-21', + dataSensitivityLevel: 'data Sensitivity Level', + legalBasisForData: 'legal Basis For Data', + dutyOfConfidentiality: 'duty Of Confidentiality', + nationalDataOptOut: 'national Data Opt Out', + requestFrequency: 'request Frequency', + dataProcessingDescription: 'data Processing Description', + confidentialDataDescription: 'confidential Data Description', + accessDate: '2021-09-26', + dataLocation: 'data Location', + privacyEnhancements: 'privacy Enhancements', + researchOutputs: 'research Outputs', + }, + { + projectTitle: 'This is another test data use register', + projectIdText: 'this is the other project id', + datasetNames: [ + 'This is the dataset title', + 'http://localhost:3000/dataset/f725187f-7352-482b-a43b-64ebc96e66f2', + 'http://localhost:3000/dataset/c6d6bbd3-74ed-46af-841d-ac5e05f4da41', + 'http://localhost:3000/dataset/e55df485-5acd-4606-bbb8-668d4c06380a', + ], + applicantNames: [' Michael Donnelly', 'http://localhost:3000/person/8495781222000176', 'http://localhost:3000/person/4495285946631793'], + organisationName: 'organisation name', + organisationSector: 'organisation sector', + applicantId: 'applicant id', + fundersAndSponsors: 'funder1 , funder2 , funder3 ', + accreditedResearcherStatus: 'accredited Researcher Status', + sublicenceArrangements: 'sublicence Arrangements', + laySummary: 'other lay Summary', + publicBenefitStatement: 'public Benefit Statement', + requestCategoryType: 'request Category Type', + technicalSummary: 'technical Summary', + otherApprovalCommittees: 'other Approval Committees', + projectStartDate: '2021-09-25', + projectEndDate: '2021-09-30', + latestApprovalDate: '2021-09-21', + dataSensitivityLevel: 'data Sensitivity Level', + legalBasisForData: 'legal Basis For Data', + dutyOfConfidentiality: 'duty Of Confidentiality', + nationalDataOptOut: 'national Data Opt Out', + requestFrequency: 'request Frequency', + dataProcessingDescription: 'data Processing Description', + confidentialDataDescription: 'confidential Data Description', + accessDate: '2021-09-26', + dataLocation: 'data Location', + privacyEnhancements: 'privacy Enhancements', + researchOutputs: 'research Outputs', + }, +]; export const dataUseRegisterUploadsWithDuplicates = [ - { - "projectTitle": "This a test data use register", - "projectIdText": "this is the project id", - "datasetNames": [ - "This is the dataset title", "http://localhost:3000/dataset/f725187f-7352-482b-a43b-64ebc96e66f2", "http://localhost:3000/dataset/c6d6bbd3-74ed-46af-841d-ac5e05f4da41", "http://localhost:3000/dataset/e55df485-5acd-4606-bbb8-668d4c06380a" - ], - "applicantNames": [ - " Michael Donnelly", "http://localhost:3000/person/8495781222000176", "http://localhost:3000/person/4495285946631793" - ], - "organisationName": "organisation name", - "organisationSector": "organisation sector", - "applicantId": "applicant id", - "fundersAndSponsors": "funder1 , funder2 , funder3 ", - "accreditedResearcherStatus": "accredited Researcher Status", - "sublicenceArrangements": "sublicence Arrangements", - "laySummary": "lay Summary", - "publicBenefitStatement": "public Benefit Statement", - "requestCategoryType": "request Category Type", - "technicalSummary": "technical Summary", - "otherApprovalCommittees": "other Approval Committees", - "projectStartDate": "2021-09-25", - "projectEndDate": "2021-09-30", - "latestApprovalDate": "2021-09-21", - "dataSensitivityLevel": "data Sensitivity Level", - "legalBasisForData": "legal Basis For Data", - "dutyOfConfidentiality": "duty Of Confidentiality", - "nationalDataOptOut": "national Data Opt Out", - "requestFrequency": "request Frequency", - "dataProcessingDescription": "data Processing Description", - "confidentialDataDescription": "confidential Data Description", - "accessDate": "2021-09-26", - "dataLocation": "data Location", - "privacyEnhancements": "privacy Enhancements", - "researchOutputs": "research Outputs" - }, - { - "projectTitle": "This a test data use register", - "projectIdText": "this is the project id", - "datasetNames": [ - "This is the dataset title", "http://localhost:3000/dataset/f725187f-7352-482b-a43b-64ebc96e66f2", "http://localhost:3000/dataset/c6d6bbd3-74ed-46af-841d-ac5e05f4da41", "http://localhost:3000/dataset/e55df485-5acd-4606-bbb8-668d4c06380a" - ], - "applicantNames": [ - " Michael Donnelly", "http://localhost:3000/person/8495781222000176", "http://localhost:3000/person/4495285946631793" - ], - "organisationName": "organisation name", - "organisationSector": "organisation sector", - "applicantId": "applicant id", - "fundersAndSponsors": "funder1 , funder2 , funder3 ", - "accreditedResearcherStatus": "accredited Researcher Status", - "sublicenceArrangements": "sublicence Arrangements", - "laySummary": "lay Summary", - "publicBenefitStatement": "public Benefit Statement", - "requestCategoryType": "request Category Type", - "technicalSummary": "technical Summary", - "otherApprovalCommittees": "other Approval Committees", - "projectStartDate": "2021-09-25", - "projectEndDate": "2021-09-30", - "latestApprovalDate": "2021-09-21", - "dataSensitivityLevel": "data Sensitivity Level", - "legalBasisForData": "legal Basis For Data", - "dutyOfConfidentiality": "duty Of Confidentiality", - "nationalDataOptOut": "national Data Opt Out", - "requestFrequency": "request Frequency", - "dataProcessingDescription": "data Processing Description", - "confidentialDataDescription": "confidential Data Description", - "accessDate": "2021-09-26", - "dataLocation": "data Location", - "privacyEnhancements": "privacy Enhancements", - "researchOutputs": "research Outputs" - }, - { - "projectTitle": "This a test data use register", - "projectIdText": "this is the project id", - "datasetNames": [ - "This is the dataset title", "http://localhost:3000/dataset/f725187f-7352-482b-a43b-64ebc96e66f2", "http://localhost:3000/dataset/c6d6bbd3-74ed-46af-841d-ac5e05f4da41", "http://localhost:3000/dataset/e55df485-5acd-4606-bbb8-668d4c06380a" - ], - "applicantNames": [ - " Michael Donnelly", "http://localhost:3000/person/8495781222000176", "http://localhost:3000/person/4495285946631793" - ], - "organisationName": "organisation name", - "organisationSector": "organisation sector", - "applicantId": "applicant id", - "fundersAndSponsors": "funder1 , funder2 , funder3 ", - "accreditedResearcherStatus": "accredited Researcher Status", - "sublicenceArrangements": "sublicence Arrangements", - "laySummary": "lay Summary", - "publicBenefitStatement": "public Benefit Statement", - "requestCategoryType": "request Category Type", - "technicalSummary": "technical Summary", - "otherApprovalCommittees": "other Approval Committees", - "projectStartDate": "2021-09-25", - "projectEndDate": "2021-09-30", - "latestApprovalDate": "2021-09-21", - "dataSensitivityLevel": "data Sensitivity Level", - "legalBasisForData": "legal Basis For Data", - "dutyOfConfidentiality": "duty Of Confidentiality", - "nationalDataOptOut": "national Data Opt Out", - "requestFrequency": "request Frequency", - "dataProcessingDescription": "data Processing Description", - "confidentialDataDescription": "confidential Data Description", - "accessDate": "2021-09-26", - "dataLocation": "data Location", - "privacyEnhancements": "privacy Enhancements", - "researchOutputs": "research Outputs" - }, - { - "projectTitle": "This a test data use register", - "projectIdText": "this is the project id", - "datasetNames": [ - "This is the dataset title", "http://localhost:3000/dataset/f725187f-7352-482b-a43b-64ebc96e66f2", "http://localhost:3000/dataset/c6d6bbd3-74ed-46af-841d-ac5e05f4da41", "http://localhost:3000/dataset/e55df485-5acd-4606-bbb8-668d4c06380a" - ], - "applicantNames": [ - " Michael Donnelly", "http://localhost:3000/person/8495781222000176", "http://localhost:3000/person/4495285946631793" - ], - "organisationName": "organisation name", - "organisationSector": "organisation sector", - "applicantId": "applicant id", - "fundersAndSponsors": "funder1 , funder2 , funder3 ", - "accreditedResearcherStatus": "accredited Researcher Status", - "sublicenceArrangements": "sublicence Arrangements", - "laySummary": "lay Summary", - "publicBenefitStatement": "public Benefit Statement", - "requestCategoryType": "request Category Type", - "technicalSummary": "technical Summary", - "otherApprovalCommittees": "other Approval Committees", - "projectStartDate": "2021-09-25", - "projectEndDate": "2021-09-30", - "latestApprovalDate": "2021-09-21", - "dataSensitivityLevel": "data Sensitivity Level", - "legalBasisForData": "legal Basis For Data", - "dutyOfConfidentiality": "duty Of Confidentiality", - "nationalDataOptOut": "national Data Opt Out", - "requestFrequency": "request Frequency", - "dataProcessingDescription": "data Processing Description", - "confidentialDataDescription": "confidential Data Description", - "accessDate": "2021-09-26", - "dataLocation": "data Location", - "privacyEnhancements": "privacy Enhancements", - "researchOutputs": "research Outputs" - }, - { - "projectTitle": "This a test data use register", - "projectIdText": "this is another project id", - "datasetNames": [ - "This is the dataset title", "http://localhost:3000/dataset/f725187f-7352-482b-a43b-64ebc96e66f2", "http://localhost:3000/dataset/c6d6bbd3-74ed-46af-841d-ac5e05f4da41", "http://localhost:3000/dataset/e55df485-5acd-4606-bbb8-668d4c06380a" - ], - "applicantNames": [ - " Michael Donnelly", "http://localhost:3000/person/8495781222000176", "http://localhost:3000/person/4495285946631793" - ], - "organisationName": "another organisation name", - "organisationSector": "organisation sector", - "applicantId": "applicant id", - "fundersAndSponsors": "funder1 , funder2 , funder3 ", - "accreditedResearcherStatus": "accredited Researcher Status", - "sublicenceArrangements": "sublicence Arrangements", - "laySummary": "lay Summary", - "publicBenefitStatement": "public Benefit Statement", - "requestCategoryType": "request Category Type", - "technicalSummary": "technical Summary", - "otherApprovalCommittees": "other Approval Committees", - "projectStartDate": "2021-09-25", - "projectEndDate": "2021-09-30", - "latestApprovalDate": "2021-09-21", - "dataSensitivityLevel": "data Sensitivity Level", - "legalBasisForData": "legal Basis For Data", - "dutyOfConfidentiality": "duty Of Confidentiality", - "nationalDataOptOut": "national Data Opt Out", - "requestFrequency": "request Frequency", - "dataProcessingDescription": "data Processing Description", - "confidentialDataDescription": "confidential Data Description", - "accessDate": "2021-09-26", - "dataLocation": "data Location", - "privacyEnhancements": "privacy Enhancements", - "researchOutputs": "research Outputs" - }, - { - "projectTitle": "This a test data use register", - "projectIdText": "this is another project id", - "datasetNames": [ - "This is the dataset title", "http://localhost:3000/dataset/f725187f-7352-482b-a43b-64ebc96e66f2", "http://localhost:3000/dataset/c6d6bbd3-74ed-46af-841d-ac5e05f4da41", "http://localhost:3000/dataset/e55df485-5acd-4606-bbb8-668d4c06380a" - ], - "applicantNames": [ - " Michael Donnelly", "http://localhost:3000/person/8495781222000176", "http://localhost:3000/person/4495285946631793" - ], - "organisationName": "another organisation name", - "organisationSector": "organisation sector", - "applicantId": "applicant id", - "fundersAndSponsors": "funder1 , funder2 , funder3 ", - "accreditedResearcherStatus": "accredited Researcher Status", - "sublicenceArrangements": "sublicence Arrangements", - "laySummary": "another lay Summary", - "publicBenefitStatement": "public Benefit Statement", - "requestCategoryType": "request Category Type", - "technicalSummary": "technical Summary", - "otherApprovalCommittees": "other Approval Committees", - "projectStartDate": "2021-09-25", - "projectEndDate": "2021-09-30", - "latestApprovalDate": "2021-09-21", - "dataSensitivityLevel": "data Sensitivity Level", - "legalBasisForData": "legal Basis For Data", - "dutyOfConfidentiality": "duty Of Confidentiality", - "nationalDataOptOut": "national Data Opt Out", - "requestFrequency": "request Frequency", - "dataProcessingDescription": "data Processing Description", - "confidentialDataDescription": "confidential Data Description", - "accessDate": "2021-09-26", - "dataLocation": "data Location", - "privacyEnhancements": "privacy Enhancements", - "researchOutputs": "research Outputs" - } -] + { + projectTitle: 'This a test data use register', + projectIdText: 'this is the project id', + datasetNames: [ + 'This is the dataset title', + 'http://localhost:3000/dataset/f725187f-7352-482b-a43b-64ebc96e66f2', + 'http://localhost:3000/dataset/c6d6bbd3-74ed-46af-841d-ac5e05f4da41', + 'http://localhost:3000/dataset/e55df485-5acd-4606-bbb8-668d4c06380a', + ], + applicantNames: [' Michael Donnelly', 'http://localhost:3000/person/8495781222000176', 'http://localhost:3000/person/4495285946631793'], + organisationName: 'organisation name', + organisationSector: 'organisation sector', + applicantId: 'applicant id', + fundersAndSponsors: 'funder1 , funder2 , funder3 ', + accreditedResearcherStatus: 'accredited Researcher Status', + sublicenceArrangements: 'sublicence Arrangements', + laySummary: 'lay Summary', + publicBenefitStatement: 'public Benefit Statement', + requestCategoryType: 'request Category Type', + technicalSummary: 'technical Summary', + otherApprovalCommittees: 'other Approval Committees', + projectStartDate: '2021-09-25', + projectEndDate: '2021-09-30', + latestApprovalDate: '2021-09-21', + dataSensitivityLevel: 'data Sensitivity Level', + legalBasisForData: 'legal Basis For Data', + dutyOfConfidentiality: 'duty Of Confidentiality', + nationalDataOptOut: 'national Data Opt Out', + requestFrequency: 'request Frequency', + dataProcessingDescription: 'data Processing Description', + confidentialDataDescription: 'confidential Data Description', + accessDate: '2021-09-26', + dataLocation: 'data Location', + privacyEnhancements: 'privacy Enhancements', + researchOutputs: 'research Outputs', + }, + { + projectTitle: 'This a test data use register', + projectIdText: 'this is the project id', + datasetNames: [ + 'This is the dataset title', + 'http://localhost:3000/dataset/f725187f-7352-482b-a43b-64ebc96e66f2', + 'http://localhost:3000/dataset/c6d6bbd3-74ed-46af-841d-ac5e05f4da41', + 'http://localhost:3000/dataset/e55df485-5acd-4606-bbb8-668d4c06380a', + ], + applicantNames: [' Michael Donnelly', 'http://localhost:3000/person/8495781222000176', 'http://localhost:3000/person/4495285946631793'], + organisationName: 'organisation name', + organisationSector: 'organisation sector', + applicantId: 'applicant id', + fundersAndSponsors: 'funder1 , funder2 , funder3 ', + accreditedResearcherStatus: 'accredited Researcher Status', + sublicenceArrangements: 'sublicence Arrangements', + laySummary: 'lay Summary', + publicBenefitStatement: 'public Benefit Statement', + requestCategoryType: 'request Category Type', + technicalSummary: 'technical Summary', + otherApprovalCommittees: 'other Approval Committees', + projectStartDate: '2021-09-25', + projectEndDate: '2021-09-30', + latestApprovalDate: '2021-09-21', + dataSensitivityLevel: 'data Sensitivity Level', + legalBasisForData: 'legal Basis For Data', + dutyOfConfidentiality: 'duty Of Confidentiality', + nationalDataOptOut: 'national Data Opt Out', + requestFrequency: 'request Frequency', + dataProcessingDescription: 'data Processing Description', + confidentialDataDescription: 'confidential Data Description', + accessDate: '2021-09-26', + dataLocation: 'data Location', + privacyEnhancements: 'privacy Enhancements', + researchOutputs: 'research Outputs', + }, + { + projectTitle: 'This a test data use register', + projectIdText: 'this is the project id', + datasetNames: [ + 'This is the dataset title', + 'http://localhost:3000/dataset/f725187f-7352-482b-a43b-64ebc96e66f2', + 'http://localhost:3000/dataset/c6d6bbd3-74ed-46af-841d-ac5e05f4da41', + 'http://localhost:3000/dataset/e55df485-5acd-4606-bbb8-668d4c06380a', + ], + applicantNames: [' Michael Donnelly', 'http://localhost:3000/person/8495781222000176', 'http://localhost:3000/person/4495285946631793'], + organisationName: 'organisation name', + organisationSector: 'organisation sector', + applicantId: 'applicant id', + fundersAndSponsors: 'funder1 , funder2 , funder3 ', + accreditedResearcherStatus: 'accredited Researcher Status', + sublicenceArrangements: 'sublicence Arrangements', + laySummary: 'lay Summary', + publicBenefitStatement: 'public Benefit Statement', + requestCategoryType: 'request Category Type', + technicalSummary: 'technical Summary', + otherApprovalCommittees: 'other Approval Committees', + projectStartDate: '2021-09-25', + projectEndDate: '2021-09-30', + latestApprovalDate: '2021-09-21', + dataSensitivityLevel: 'data Sensitivity Level', + legalBasisForData: 'legal Basis For Data', + dutyOfConfidentiality: 'duty Of Confidentiality', + nationalDataOptOut: 'national Data Opt Out', + requestFrequency: 'request Frequency', + dataProcessingDescription: 'data Processing Description', + confidentialDataDescription: 'confidential Data Description', + accessDate: '2021-09-26', + dataLocation: 'data Location', + privacyEnhancements: 'privacy Enhancements', + researchOutputs: 'research Outputs', + }, + { + projectTitle: 'This a test data use register', + projectIdText: 'this is the project id', + datasetNames: [ + 'This is the dataset title', + 'http://localhost:3000/dataset/f725187f-7352-482b-a43b-64ebc96e66f2', + 'http://localhost:3000/dataset/c6d6bbd3-74ed-46af-841d-ac5e05f4da41', + 'http://localhost:3000/dataset/e55df485-5acd-4606-bbb8-668d4c06380a', + ], + applicantNames: [' Michael Donnelly', 'http://localhost:3000/person/8495781222000176', 'http://localhost:3000/person/4495285946631793'], + organisationName: 'organisation name', + organisationSector: 'organisation sector', + applicantId: 'applicant id', + fundersAndSponsors: 'funder1 , funder2 , funder3 ', + accreditedResearcherStatus: 'accredited Researcher Status', + sublicenceArrangements: 'sublicence Arrangements', + laySummary: 'lay Summary', + publicBenefitStatement: 'public Benefit Statement', + requestCategoryType: 'request Category Type', + technicalSummary: 'technical Summary', + otherApprovalCommittees: 'other Approval Committees', + projectStartDate: '2021-09-25', + projectEndDate: '2021-09-30', + latestApprovalDate: '2021-09-21', + dataSensitivityLevel: 'data Sensitivity Level', + legalBasisForData: 'legal Basis For Data', + dutyOfConfidentiality: 'duty Of Confidentiality', + nationalDataOptOut: 'national Data Opt Out', + requestFrequency: 'request Frequency', + dataProcessingDescription: 'data Processing Description', + confidentialDataDescription: 'confidential Data Description', + accessDate: '2021-09-26', + dataLocation: 'data Location', + privacyEnhancements: 'privacy Enhancements', + researchOutputs: 'research Outputs', + }, + { + projectTitle: 'This a test data use register', + projectIdText: 'this is another project id', + datasetNames: [ + 'This is the dataset title', + 'http://localhost:3000/dataset/f725187f-7352-482b-a43b-64ebc96e66f2', + 'http://localhost:3000/dataset/c6d6bbd3-74ed-46af-841d-ac5e05f4da41', + 'http://localhost:3000/dataset/e55df485-5acd-4606-bbb8-668d4c06380a', + ], + applicantNames: [' Michael Donnelly', 'http://localhost:3000/person/8495781222000176', 'http://localhost:3000/person/4495285946631793'], + organisationName: 'another organisation name', + organisationSector: 'organisation sector', + applicantId: 'applicant id', + fundersAndSponsors: 'funder1 , funder2 , funder3 ', + accreditedResearcherStatus: 'accredited Researcher Status', + sublicenceArrangements: 'sublicence Arrangements', + laySummary: 'lay Summary', + publicBenefitStatement: 'public Benefit Statement', + requestCategoryType: 'request Category Type', + technicalSummary: 'technical Summary', + otherApprovalCommittees: 'other Approval Committees', + projectStartDate: '2021-09-25', + projectEndDate: '2021-09-30', + latestApprovalDate: '2021-09-21', + dataSensitivityLevel: 'data Sensitivity Level', + legalBasisForData: 'legal Basis For Data', + dutyOfConfidentiality: 'duty Of Confidentiality', + nationalDataOptOut: 'national Data Opt Out', + requestFrequency: 'request Frequency', + dataProcessingDescription: 'data Processing Description', + confidentialDataDescription: 'confidential Data Description', + accessDate: '2021-09-26', + dataLocation: 'data Location', + privacyEnhancements: 'privacy Enhancements', + researchOutputs: 'research Outputs', + }, + { + projectTitle: 'This a test data use register', + projectIdText: 'this is another project id', + datasetNames: [ + 'This is the dataset title', + 'http://localhost:3000/dataset/f725187f-7352-482b-a43b-64ebc96e66f2', + 'http://localhost:3000/dataset/c6d6bbd3-74ed-46af-841d-ac5e05f4da41', + 'http://localhost:3000/dataset/e55df485-5acd-4606-bbb8-668d4c06380a', + ], + applicantNames: [' Michael Donnelly', 'http://localhost:3000/person/8495781222000176', 'http://localhost:3000/person/4495285946631793'], + organisationName: 'another organisation name', + organisationSector: 'organisation sector', + applicantId: 'applicant id', + fundersAndSponsors: 'funder1 , funder2 , funder3 ', + accreditedResearcherStatus: 'accredited Researcher Status', + sublicenceArrangements: 'sublicence Arrangements', + laySummary: 'another lay Summary', + publicBenefitStatement: 'public Benefit Statement', + requestCategoryType: 'request Category Type', + technicalSummary: 'technical Summary', + otherApprovalCommittees: 'other Approval Committees', + projectStartDate: '2021-09-25', + projectEndDate: '2021-09-30', + latestApprovalDate: '2021-09-21', + dataSensitivityLevel: 'data Sensitivity Level', + legalBasisForData: 'legal Basis For Data', + dutyOfConfidentiality: 'duty Of Confidentiality', + nationalDataOptOut: 'national Data Opt Out', + requestFrequency: 'request Frequency', + dataProcessingDescription: 'data Processing Description', + confidentialDataDescription: 'confidential Data Description', + accessDate: '2021-09-26', + dataLocation: 'data Location', + privacyEnhancements: 'privacy Enhancements', + researchOutputs: 'research Outputs', + }, +]; export const datasets = [ - { - datasetid: "70b4d407-288a-4945-a4d5-506d60715110", - pid: "e55df485-5acd-4606-bbb8-668d4c06380a" - }, - { - datasetid: "82ef7d1a-98d8-48b6-9acd-461bf2a399c3", - pid: "e55df485-5acd-4606-bbb8-668d4c06380a" - }, - { - datasetid: "673626f3-bdac-4d32-9bb8-c890b727c0d1", - pid: "594d79a4-92b9-4a7f-b991-abf850bf2b67" - }, - { - datasetid: "89e57932-ac48-48ac-a6e5-29795bc38b94", - pid: "efbd4275-70e2-4887-8499-18b1fb24ce5b" - } -] + { + datasetid: '70b4d407-288a-4945-a4d5-506d60715110', + pid: 'e55df485-5acd-4606-bbb8-668d4c06380a', + }, + { + datasetid: '82ef7d1a-98d8-48b6-9acd-461bf2a399c3', + pid: 'e55df485-5acd-4606-bbb8-668d4c06380a', + }, + { + datasetid: '673626f3-bdac-4d32-9bb8-c890b727c0d1', + pid: '594d79a4-92b9-4a7f-b991-abf850bf2b67', + }, + { + datasetid: '89e57932-ac48-48ac-a6e5-29795bc38b94', + pid: 'efbd4275-70e2-4887-8499-18b1fb24ce5b', + }, +]; export const relatedObjectDatasets = [ - { - objectId: "70b4d407-288a-4945-a4d5-506d60715110", - pid: "e55df485-5acd-4606-bbb8-668d4c06380a", - objectType: "dataset", - user: "James Smith", - updated: "2021-24-09T11:01:58.135Z" - }, - { - objectId: "82ef7d1a-98d8-48b6-9acd-461bf2a399c3", - pid: "e55df485-5acd-4606-bbb8-668d4c06380a", - objectType: "dataset", - user: "James Smith", - updated: "2021-24-09T11:01:58.135Z" - }, - { - objectId: "673626f3-bdac-4d32-9bb8-c890b727c0d1", - pid: "594d79a4-92b9-4a7f-b991-abf850bf2b67", - objectType: "dataset", - user: "James Smith", - updated: "2021-24-09T11:01:58.135Z" - }, - { - objectId: "89e57932-ac48-48ac-a6e5-29795bc38b94", - pid: "efbd4275-70e2-4887-8499-18b1fb24ce5b", - objectType: "dataset", - user: "James Smith", - updated: "2021-24-09T11:01:58.135Z" - } -] + { + objectId: '70b4d407-288a-4945-a4d5-506d60715110', + pid: 'e55df485-5acd-4606-bbb8-668d4c06380a', + objectType: 'dataset', + user: 'James Smith', + updated: '2021-24-09T11:01:58.135Z', + isLocked: true, + reason: 'This dataset was added automatically during the manual upload of this data use register', + }, + { + objectId: '82ef7d1a-98d8-48b6-9acd-461bf2a399c3', + pid: 'e55df485-5acd-4606-bbb8-668d4c06380a', + objectType: 'dataset', + user: 'James Smith', + updated: '2021-24-09T11:01:58.135Z', + isLocked: true, + reason: 'This dataset was added automatically during the manual upload of this data use register', + }, + { + objectId: '673626f3-bdac-4d32-9bb8-c890b727c0d1', + pid: '594d79a4-92b9-4a7f-b991-abf850bf2b67', + objectType: 'dataset', + user: 'James Smith', + updated: '2021-24-09T11:01:58.135Z', + isLocked: true, + reason: 'This dataset was added automatically during the manual upload of this data use register', + }, + { + objectId: '89e57932-ac48-48ac-a6e5-29795bc38b94', + pid: 'efbd4275-70e2-4887-8499-18b1fb24ce5b', + objectType: 'dataset', + user: 'James Smith', + updated: '2021-24-09T11:01:58.135Z', + isLocked: true, + reason: 'This dataset was added automatically during the manual upload of this data use register', + }, +]; -export const nonGatewayDatasetNames = [ - "dataset one", "dataset two", " dataset three", "dataset four" -] +export const nonGatewayDatasetNames = ['dataset one', 'dataset two', ' dataset three', 'dataset four']; export const gatewayDatasetNames = [ - "http://localhost:3000/dataset/f725187f-7352-482b-a43b-64ebc96e66f2", - "http://localhost:3000/dataset/c6d6bbd3-74ed-46af-841d-ac5e05f4da41", - "http://localhost:3000/dataset/e55df485-5acd-4606-bbb8-668d4c06380a" -] + 'http://localhost:3000/dataset/f725187f-7352-482b-a43b-64ebc96e66f2', + 'http://localhost:3000/dataset/c6d6bbd3-74ed-46af-841d-ac5e05f4da41', + 'http://localhost:3000/dataset/e55df485-5acd-4606-bbb8-668d4c06380a', +]; export const expectedGatewayDatasets = [ - { datasetid: "1", name: "dataset 1", pid:"111" }, - { datasetid: "2", name: "dataset 2", pid:"222" }, - { datasetid: "3", name: "dataset 3", pid:"333" } -] + { datasetid: '1', name: 'dataset 1', pid: '111' }, + { datasetid: '2', name: 'dataset 2', pid: '222' }, + { datasetid: '3', name: 'dataset 3', pid: '333' }, +]; -export const nonGatewayApplicantNames = [ - "applicant one", "applicant two", "applicant three", "applicant four" -] +export const nonGatewayApplicantNames = ['applicant one', 'applicant two', 'applicant three', 'applicant four']; -export const gatewayApplicantNames = [ - "http://localhost:3000/person/8495781222000176", "http://localhost:3000/person/4495285946631793" -] +export const gatewayApplicantNames = ['http://localhost:3000/person/8495781222000176', 'http://localhost:3000/person/4495285946631793']; -export const expectedGatewayApplicants = [ - "89e57932-ac48-48ac-a6e5-29795bc38b94", "0cfe60cd-038d-4c03-9a95-894c52135922" -] \ No newline at end of file +export const expectedGatewayApplicants = ['89e57932-ac48-48ac-a6e5-29795bc38b94', '0cfe60cd-038d-4c03-9a95-894c52135922']; + +export const applications = [ + { + questionAnswers: { + safepeopleprimaryapplicantfullname: 'applicant name', + safeprojectfunderinformationprojecthasfundername: 'funder 1', + safeprojectfunderinformationprojecthasfundername_gRvcG: 'funder 2', + safeprojectsponsorinformationprojecthassponsororganisationname: 'sponsor 1', + safeprojectsponsorinformationprojecthassponsororganisationname_2gixm: 'sponsor 2', + safepeopleprimaryapplicantfullname: 'James Smith', + safepeopleprimaryapplicantfullname_xRtvc: 'Michael Howard', + safepeopleotherindividualsfullname: 'Colin Devlin', + safepeopleotherindividualsfullname_3uGds: 'Graham Patterson', + }, + }, +]; + +export const authors = [ + { _id: '607db9c6e1f9d3704d570d93', firstname: 'James', lastname: 'Smith' }, + { _id: '5fb628de6f3f9767bd2d9281', firstname: 'Michael', lastname: 'Howard' }, +]; diff --git a/src/resources/dataUseRegister/__tests__/dataUseRegister.util.test.js b/src/resources/dataUseRegister/__tests__/dataUseRegister.util.test.js index db4886bc..49b2aaf1 100644 --- a/src/resources/dataUseRegister/__tests__/dataUseRegister.util.test.js +++ b/src/resources/dataUseRegister/__tests__/dataUseRegister.util.test.js @@ -2,7 +2,18 @@ import sinon from 'sinon'; import { cloneDeep } from 'lodash'; import dataUseRegisterUtil from '../dataUseRegister.util'; -import { datasets, relatedObjectDatasets, nonGatewayDatasetNames, gatewayDatasetNames, expectedGatewayDatasets, nonGatewayApplicantNames, gatewayApplicantNames, expectedGatewayApplicants } from '../__mocks__/dataUseRegisters'; +import { + datasets, + relatedObjectDatasets, + nonGatewayDatasetNames, + gatewayDatasetNames, + expectedGatewayDatasets, + nonGatewayApplicantNames, + gatewayApplicantNames, + expectedGatewayApplicants, + applications, + authors +} from '../__mocks__/dataUseRegisters'; import { uploader } from '../__mocks__/dataUseRegisterUsers'; import * as userRepository from '../../user/user.repository'; import { datasetService } from '../../dataset/dependency'; @@ -45,7 +56,7 @@ describe('DataUseRegisterUtil', function () { it('returns the details of applicants that could be found on the Gateway when valid profile URLs are given', async function () { // Arrange const getUsersByIdsStub = sinon.stub(userRepository, 'getUsersByIds'); - getUsersByIdsStub.returns([{_id:'89e57932-ac48-48ac-a6e5-29795bc38b94'}, {_id:'0cfe60cd-038d-4c03-9a95-894c52135922'}]); + getUsersByIdsStub.returns([{ _id: '89e57932-ac48-48ac-a6e5-29795bc38b94' }, { _id: '0cfe60cd-038d-4c03-9a95-894c52135922' }]); // Act const result = await dataUseRegisterUtil.getLinkedApplicants(gatewayApplicantNames); @@ -75,6 +86,54 @@ describe('DataUseRegisterUtil', function () { }); }); + describe('extractFormApplicants', function () { + it('identifies and combines gateway and non gateway applicants in the correct format', function () { + // Arrange + const questionAnswersStub = cloneDeep(applications[0].questionAnswers); + const authorsStub = cloneDeep(authors); + + // Act + const result = dataUseRegisterUtil.extractFormApplicants(authorsStub, questionAnswersStub); + + // Assert + expect(result.gatewayApplicants.length).toBe(2); + expect(result.gatewayApplicants).toEqual(expect.arrayContaining(['607db9c6e1f9d3704d570d93', '5fb628de6f3f9767bd2d9281'])); + + expect(result.nonGatewayApplicants.length).toBe(2); + expect(result.nonGatewayApplicants).toEqual(expect.arrayContaining(['Colin Devlin', 'Graham Patterson'])); + }); + + it('removes duplicate applicants who are both authors of the application and named in the questions answers', function () { + // Arrange + const questionAnswersStub = cloneDeep(applications[0].questionAnswers); + const authorsStub = cloneDeep(authors); + + // Act + const result = dataUseRegisterUtil.extractFormApplicants(authorsStub, questionAnswersStub); + + // Assert + expect(result.gatewayApplicants.length).toBe(2); + expect(result.gatewayApplicants).toEqual(expect.arrayContaining(['607db9c6e1f9d3704d570d93', '5fb628de6f3f9767bd2d9281'])); + + expect(result.nonGatewayApplicants.length).toBe(2); + expect(result.nonGatewayApplicants).toEqual(expect.arrayContaining(['Colin Devlin', 'Graham Patterson'])); + }); + }); + + describe('extractFundersAndSponsors', function () { + it('identifies and combines funder and sponsor organisations named in the question answers ', function () { + // Arrange + const questionAnswersStub = cloneDeep(applications[0].questionAnswers); + + // Act + const result = dataUseRegisterUtil.extractFundersAndSponsors(questionAnswersStub); + + // Assert + expect(result.length).toBe(4); + expect(result).toEqual(expect.arrayContaining(['funder 1', 'funder 2', 'sponsor 1', 'sponsor 2'])); + }); + }); + afterAll(function () { delete process.env.homeURL; }); From a60e4dd0b5f012f681def5fd53215f128cc920f5 Mon Sep 17 00:00:00 2001 From: Paul McCafferty <> Date: Tue, 5 Oct 2021 15:01:18 +0100 Subject: [PATCH 029/116] Removing eq from mongo update queries --- .../collections/collectioncounter.route.js | 2 +- .../collections/collections.route.js | 18 ++++---- src/resources/course/course.repository.js | 34 +++++++------- src/resources/course/coursecounter.route.js | 2 +- .../dataset/datasetonboarding.controller.js | 6 +-- src/resources/tool/counter.route.js | 4 +- src/resources/tool/data.repository.js | 44 +++++++++---------- src/resources/tool/v1/tool.route.js | 6 +-- 8 files changed, 58 insertions(+), 58 deletions(-) diff --git a/src/resources/collections/collectioncounter.route.js b/src/resources/collections/collectioncounter.route.js index 67464084..52e8628c 100644 --- a/src/resources/collections/collectioncounter.route.js +++ b/src/resources/collections/collectioncounter.route.js @@ -12,7 +12,7 @@ const datasetLimiter = rateLimit({ router.post('/update', datasetLimiter, async (req, res) => { const { id, counter } = req.body; - Collections.findOneAndUpdate({ id: { $eq: id } }, { counter: { $eq: counter } }, err => { + Collections.findOneAndUpdate({ id: { $eq: id } }, { counter }, err => { if (err) return res.json({ success: false, error: err }); return res.json({ success: true }); }); diff --git a/src/resources/collections/collections.route.js b/src/resources/collections/collections.route.js index f9d285ca..04166403 100644 --- a/src/resources/collections/collections.route.js +++ b/src/resources/collections/collections.route.js @@ -135,16 +135,16 @@ router.put('/edit/:id', passport.authenticate('jwt'), utils.checkAllowedToAccess let collectionId = parseInt(id); await Collections.findOneAndUpdate( - { id: collectionId }, + { id: { $eq: collectionId } }, { - name: { $eq: inputSanitizer.removeNonBreakingSpaces(name) }, - description: { $eq: inputSanitizer.removeNonBreakingSpaces(description) }, - imageLink: { $eq: imageLink }, - authors: { $eq: authors }, - relatedObjects: { $eq: relatedObjects }, - publicflag: { $eq: publicflag }, - keywords: { $eq: keywords }, - updatedon: { $eq: updatedon }, + name: inputSanitizer.removeNonBreakingSpaces(name), + description: inputSanitizer.removeNonBreakingSpaces(description), + imageLink, + authors, + relatedObjects, + publicflag, + keywords, + updatedon, }, err => { if (err) { diff --git a/src/resources/course/course.repository.js b/src/resources/course/course.repository.js index 488b21bc..0cee771b 100644 --- a/src/resources/course/course.repository.js +++ b/src/resources/course/course.repository.js @@ -104,24 +104,24 @@ const editCourse = async req => { let updatedon = Date.now(); Course.findOneAndUpdate( - { id: id }, + { id: { $eq: id } }, { - title: { $eq: inputSanitizer.removeNonBreakingSpaces(req.body.title) }, - link: { $eq: urlValidator.validateURL(inputSanitizer.removeNonBreakingSpaces(req.body.link)) }, - provider: { $eq: inputSanitizer.removeNonBreakingSpaces(req.body.provider) }, - description: { $eq: inputSanitizer.removeNonBreakingSpaces(req.body.description) }, - courseDelivery: { $eq: inputSanitizer.removeNonBreakingSpaces(req.body.courseDelivery) }, - location: { $eq: inputSanitizer.removeNonBreakingSpaces(req.body.location) }, - keywords: { $eq: inputSanitizer.removeNonBreakingSpaces(req.body.keywords) }, - domains: { $eq: inputSanitizer.removeNonBreakingSpaces(req.body.domains) }, - relatedObjects: { $eq: relatedObjects }, - courseOptions: { $eq: courseOptions }, - entries: { $eq: entries }, - restrictions: { $eq: inputSanitizer.removeNonBreakingSpaces(req.body.restrictions) }, - award: { $eq: inputSanitizer.removeNonBreakingSpaces(req.body.award) }, - competencyFramework: { $eq: inputSanitizer.removeNonBreakingSpaces(req.body.competencyFramework) }, - nationalPriority: { $eq: inputSanitizer.removeNonBreakingSpaces(req.body.nationalPriority) }, - updatedon: { $eq: updatedon }, + title: inputSanitizer.removeNonBreakingSpaces(req.body.title), + link: urlValidator.validateURL(inputSanitizer.removeNonBreakingSpaces(req.body.link)), + provider: inputSanitizer.removeNonBreakingSpaces(req.body.provider), + description: inputSanitizer.removeNonBreakingSpaces(req.body.description), + courseDelivery: inputSanitizer.removeNonBreakingSpaces(req.body.courseDelivery), + location: inputSanitizer.removeNonBreakingSpaces(req.body.location), + keywords: inputSanitizer.removeNonBreakingSpaces(req.body.keywords), + domains: inputSanitizer.removeNonBreakingSpaces(req.body.domains), + relatedObjects, + courseOptions, + entries, + restrictions: inputSanitizer.removeNonBreakingSpaces(req.body.restrictions), + award: inputSanitizer.removeNonBreakingSpaces(req.body.award), + competencyFramework: inputSanitizer.removeNonBreakingSpaces(req.body.competencyFramework), + nationalPriority: inputSanitizer.removeNonBreakingSpaces(req.body.nationalPriority), + updatedon, }, err => { if (err) { diff --git a/src/resources/course/coursecounter.route.js b/src/resources/course/coursecounter.route.js index 23ea9a1a..092ba179 100644 --- a/src/resources/course/coursecounter.route.js +++ b/src/resources/course/coursecounter.route.js @@ -12,7 +12,7 @@ const datasetLimiter = rateLimit({ router.post('/update', datasetLimiter, async (req, res) => { const { id, counter } = req.body; - Course.findOneAndUpdate({ id: { $eq: id } }, { counter: { $eq: counter } }, err => { + Course.findOneAndUpdate({ id: { $eq: id } }, { counter }, err => { if (err) return res.json({ success: false, error: err }); return res.json({ success: true }); }); diff --git a/src/resources/dataset/datasetonboarding.controller.js b/src/resources/dataset/datasetonboarding.controller.js index 8d473f07..aa078500 100644 --- a/src/resources/dataset/datasetonboarding.controller.js +++ b/src/resources/dataset/datasetonboarding.controller.js @@ -260,8 +260,8 @@ module.exports = { Data.findByIdAndUpdate( { _id: id }, { - structuralMetadata: { $eq: structuralMetadata }, - percentageCompleted: { $eq: data.percentageCompleted }, + structuralMetadata, + percentageCompleted, 'timestamps.updated': Date.now(), }, { new: true } @@ -284,7 +284,7 @@ module.exports = { let title = questionAnswers['properties/summary/title']; if (title && title.length >= 2) { - Data.findByIdAndUpdate({ _id: id }, { name: { $eq: title }, 'timestamps.updated': Date.now() }, { new: true }).catch(err => { + Data.findByIdAndUpdate({ _id: id }, { name: title, 'timestamps.updated': Date.now() }, { new: true }).catch(err => { console.error(err); throw err; }); diff --git a/src/resources/tool/counter.route.js b/src/resources/tool/counter.route.js index 18579845..eda04cd6 100644 --- a/src/resources/tool/counter.route.js +++ b/src/resources/tool/counter.route.js @@ -7,12 +7,12 @@ router.post('/update', async (req, res) => { const { id, counter } = req.body; if (isNaN(id)) { - Data.findOneAndUpdate({ datasetid: { $eq: id } }, { counter: { $eq: counter } }, err => { + Data.findOneAndUpdate({ datasetid: { $eq: id } }, { counter }, err => { if (err) return res.json({ success: false, error: err }); return res.json({ success: true }); }); } else { - Data.findOneAndUpdate({ id: { $eq: id } }, { counter: { $eq: counter } }, err => { + Data.findOneAndUpdate({ id: { $eq: id } }, { counter }, err => { if (err) return res.json({ success: false, error: err }); return res.json({ success: true }); }); diff --git a/src/resources/tool/data.repository.js b/src/resources/tool/data.repository.js index 3eacb712..4e47130d 100644 --- a/src/resources/tool/data.repository.js +++ b/src/resources/tool/data.repository.js @@ -171,33 +171,33 @@ const editTool = async (req, res) => { }; Data.findOneAndUpdate( - { id: id }, + { id: { $eq: id } }, { - type: { $eq: inputSanitizer.removeNonBreakingSpaces(type) }, - name: { $eq: inputSanitizer.removeNonBreakingSpaces(name) }, - link: { $eq: urlValidator.validateURL(inputSanitizer.removeNonBreakingSpaces(link)) }, - description: { $eq: inputSanitizer.removeNonBreakingSpaces(description) }, - resultsInsights: { $eq: inputSanitizer.removeNonBreakingSpaces(resultsInsights) }, - authorsNew: { $eq: inputSanitizer.removeNonBreakingSpaces(authorsNew) }, - leadResearcher: { $eq: inputSanitizer.removeNonBreakingSpaces(leadResearcher) }, - journal: { $eq: inputSanitizer.removeNonBreakingSpaces(journal) }, - journalYear: { $eq: inputSanitizer.removeNonBreakingSpaces(journalYear) }, + type: inputSanitizer.removeNonBreakingSpaces(type), + name: inputSanitizer.removeNonBreakingSpaces(name), + link: urlValidator.validateURL(inputSanitizer.removeNonBreakingSpaces(link)), + description: inputSanitizer.removeNonBreakingSpaces(description), + resultsInsights: inputSanitizer.removeNonBreakingSpaces(resultsInsights), + authorsNew: inputSanitizer.removeNonBreakingSpaces(authorsNew), + leadResearcher: inputSanitizer.removeNonBreakingSpaces(leadResearcher), + journal: inputSanitizer.removeNonBreakingSpaces(journal), + journalYear: inputSanitizer.removeNonBreakingSpaces(journalYear), categories: { - category: { $eq: inputSanitizer.removeNonBreakingSpaces(categories.category) }, - programmingLanguage: { $eq: categories.programmingLanguage }, - programmingLanguageVersion: { $eq: categories.programmingLanguageVersion }, + category: inputSanitizer.removeNonBreakingSpaces(categories.category), + programmingLanguage: categories.programmingLanguage, + programmingLanguageVersion: categories.programmingLanguageVersion, }, - license: { $eq: inputSanitizer.removeNonBreakingSpaces(license) }, - authors: { $eq: authors }, - programmingLanguage: { $eq: programmingLanguage }, + license: inputSanitizer.removeNonBreakingSpaces(license), + authors, + programmingLanguage, tags: { - features: { $eq: inputSanitizer.removeNonBreakingSpaces(tags.features) }, - topics: { $eq: inputSanitizer.removeNonBreakingSpaces(tags.topics) }, + features: inputSanitizer.removeNonBreakingSpaces(tags.features), + topics: inputSanitizer.removeNonBreakingSpaces(tags.topics), }, - relatedObjects: { $eq: relatedObjects }, - isPreprint: { $eq: isPreprint }, - document_links: { $eq: documentLinksValidated }, - updatedon: { $eq: updatedon }, + relatedObjects, + isPreprint, + document_links: documentLinksValidated, + updatedon, }, err => { if (err) { diff --git a/src/resources/tool/v1/tool.route.js b/src/resources/tool/v1/tool.route.js index ea9f892f..aa2ca4eb 100644 --- a/src/resources/tool/v1/tool.route.js +++ b/src/resources/tool/v1/tool.route.js @@ -269,8 +269,8 @@ router.post('/reply', passport.authenticate('jwt'), async (req, res) => { Reviews.findOneAndUpdate( { reviewID: { $eq: reviewID } }, { - replierID: { $eq: replierID }, - reply: { $eq: inputSanitizer.removeNonBreakingSpaces(reply) }, + replierID, + reply: inputSanitizer.removeNonBreakingSpaces(reply), replydate: Date.now(), }, err => { @@ -290,7 +290,7 @@ router.put('/review/approve', passport.authenticate('jwt'), utils.checkIsInRole( Reviews.findOneAndUpdate( { reviewID: { $eq: id } }, { - activeflag: { $eq: activeflag }, + activeflag, }, err => { if (err) return res.json({ success: false, error: err }); From 2895cf47596aee47b35216171e79fdaea6647485 Mon Sep 17 00:00:00 2001 From: Callum Reekie Date: Tue, 5 Oct 2021 19:38:01 +0100 Subject: [PATCH 030/116] CR - refactor and auth strategies --- src/resources/auth/strategies/azure.js | 206 +++++++------------- src/resources/auth/strategies/google.js | 97 ++-------- src/resources/auth/strategies/linkedin.js | 97 ++-------- src/resources/auth/strategies/oidc.js | 100 ++-------- src/resources/auth/strategies/orcid.js | 219 ++++++++-------------- src/resources/auth/utils.js | 121 ++++++++++-- 6 files changed, 292 insertions(+), 548 deletions(-) diff --git a/src/resources/auth/strategies/azure.js b/src/resources/auth/strategies/azure.js index 9787d0cc..fbc6ab21 100644 --- a/src/resources/auth/strategies/azure.js +++ b/src/resources/auth/strategies/azure.js @@ -3,151 +3,79 @@ import passportAzure from 'passport-azure-ad-oauth2'; import { to } from 'await-to-js'; import jwt from 'jsonwebtoken'; +import { catchLoginErrorAndRedirect, loginAndSignToken } from '../utils'; import { getUserByProviderId } from '../../user/user.repository'; -import { updateRedirectURL } from '../../user/user.service'; -import { getObjectById } from '../../tool/data.repository'; import { createUser } from '../../user/user.service'; -import { signToken } from '../utils'; import { ROLES } from '../../user/user.roles'; -import queryString from 'query-string'; -import Url from 'url'; -import { discourseLogin } from '../sso/sso.discourse.service'; -const eventLogController = require('../../eventlog/eventlog.controller'); const AzureStrategy = passportAzure.Strategy; const strategy = app => { - const strategyOptions = { - clientID: process.env.AZURE_SSO_CLIENT_ID, - clientSecret: process.env.AZURE_SSO_CLIENT_SECRET, - callbackURL: `/auth/azure/callback`, - proxy: true - }; - - const verifyCallback = async (accessToken, refreshToken, params, profile, done) => { - - let decodedToken; - - try { - decodedToken = jwt.decode(params.id_token); - } catch(err) { - return done('loginError'); - }; - - if ( !decodedToken.oid || decodedToken.oid === '' ) return done('loginError'); - - let [err, user] = await to(getUserByProviderId(decodedToken.oid)); - if (err || user) { - return done(err, user) - }; - - const [createdError, createdUser] = await to( - createUser({ - provider: 'azure', - providerId: decodedToken.oid, - firstname: decodedToken.given_name, - lastname: decodedToken.family_name, - password: null, - email: decodedToken.email, - role: ROLES.Creator - }) - ); - - return done(createdError, createdUser); - }; - - passport.use('azure_ad_oauth2', new AzureStrategy(strategyOptions, verifyCallback)); - - app.get( - `/auth/azure`, - (req, res, next) => { - // Save the url of the user's current page so the app can redirect back to it after authorization - if (req.headers.referer) { - req.param.returnpage = req.headers.referer; - } - next(); - }, - passport.authenticate('azure_ad_oauth2') - ); - - app.get( - `/auth/azure/callback`, (req, res, next) => { - passport.authenticate('azure_ad_oauth2', (err, user, info) => { - - if (err || !user) { - //loginError - if (err === 'loginError') return res.status(200).redirect(process.env.homeURL + '/loginerror'); - - // failureRedirect - var redirect = '/'; - let returnPage = null; - if (req.param.returnpage) { - returnPage = Url.parse(req.param.returnpage); - redirect = returnPage.path; - delete req.param.returnpage; - }; - - let redirectUrl = process.env.homeURL + redirect; - - return res.status(200).redirect(redirectUrl); - }; - - req.login(user, async err => { - if (err) { - return next(err); - } - - var redirect = '/'; - let returnPage = null; - let queryStringParsed = null; - if (req.param.returnpage) { - returnPage = Url.parse(req.param.returnpage); - redirect = returnPage.path; - queryStringParsed = queryString.parse(returnPage.query); - }; - - let [profileErr, profile] = await to(getObjectById(req.user.id)); - if (!profile) { - await to(updateRedirectURL({ id: req.user.id, redirectURL: redirect })); - return res.redirect(process.env.homeURL + '/completeRegistration/' + req.user.id); - }; - - if (req.param.returnpage) { - delete req.param.returnpage; - }; - - let redirectUrl = process.env.homeURL + redirect; - if (queryStringParsed && queryStringParsed.sso && queryStringParsed.sig) { - try { - console.log(req.user) - redirectUrl = discourseLogin(queryStringParsed.sso, queryStringParsed.sig, req.user); - } catch (err) { - console.error(err.message); - return res.status(500).send('Error authenticating the user.'); - } - }; - - //Build event object for user login and log it to DB - let eventObj = { - userId: req.user.id, - event: `user_login_${req.user.provider}`, - timestamp: Date.now(), - }; - - await eventLogController.logEvent(eventObj); - - return res - .status(200) - .cookie('jwt', signToken({ _id: req.user._id, id: req.user.id, timeStamp: Date.now() }), { - httpOnly: true, - secure: process.env.api_url ? true : false, - }) - .redirect(redirectUrl); - }) - })(req, res, next); - } - ) - return app; + const strategyOptions = { + clientID: process.env.AZURE_SSO_CLIENT_ID, + clientSecret: process.env.AZURE_SSO_CLIENT_SECRET, + callbackURL: `/auth/azure/callback`, + proxy: true, + }; + + const verifyCallback = async (accessToken, refreshToken, params, profile, done) => { + let decodedToken; + + try { + decodedToken = jwt.decode(params.id_token); + } catch (err) { + return done('loginError'); + } + + if (!decodedToken.oid || decodedToken.oid === '') return done('loginError'); + + let [err, user] = await to(getUserByProviderId(decodedToken.oid)); + if (err || user) { + return done(err, user); + } + + const [createdError, createdUser] = await to( + createUser({ + provider: 'azure', + providerId: decodedToken.oid, + firstname: decodedToken.given_name, + lastname: decodedToken.family_name, + password: null, + email: decodedToken.email, + role: ROLES.Creator, + }) + ); + + return done(createdError, createdUser); + }; + + passport.use('azure_ad_oauth2', new AzureStrategy(strategyOptions, verifyCallback)); + + app.get( + `/auth/azure`, + (req, res, next) => { + // Save the url of the user's current page so the app can redirect back to it after authorization + if (req.headers.referer) { + req.param.returnpage = req.headers.referer; + } + next(); + }, + passport.authenticate('azure_ad_oauth2') + ); + + app.get( + `/auth/azure/callback`, + (req, res, next) => { + passport.authenticate('azure_ad_oauth2', (err, user) => { + req.err = err; + req.user = user; + next(); + })(req, res, next); + }, + catchLoginErrorAndRedirect, + loginAndSignToken + ); + return app; }; -export { strategy } \ No newline at end of file +export { strategy }; diff --git a/src/resources/auth/strategies/google.js b/src/resources/auth/strategies/google.js index 9104c3c5..4209a9a6 100644 --- a/src/resources/auth/strategies/google.js +++ b/src/resources/auth/strategies/google.js @@ -2,17 +2,11 @@ import passport from 'passport'; import passportGoogle from 'passport-google-oauth'; import { to } from 'await-to-js'; +import { catchLoginErrorAndRedirect, loginAndSignToken } from '../utils'; import { getUserByProviderId } from '../../user/user.repository'; -import { updateRedirectURL } from '../../user/user.service'; -import { getObjectById } from '../../tool/data.repository'; import { createUser } from '../../user/user.service'; -import { signToken } from '../utils'; import { ROLES } from '../../user/user.roles'; -import queryString from 'query-string'; -import Url from 'url'; -import { discourseLogin } from '../sso/sso.discourse.service'; -const eventLogController = require('../../eventlog/eventlog.controller'); const GoogleStrategy = passportGoogle.OAuth2Strategy; const strategy = app => { @@ -64,83 +58,18 @@ const strategy = app => { }) ); - app.get('/auth/google/callback', (req, res, next) => { - passport.authenticate('google', (err, user) => { - if (err || !user) { - //loginError - if (err === 'loginError') return res.status(200).redirect(process.env.homeURL + '/loginerror'); - - // failureRedirect - var redirect = '/'; - let returnPage = null; - - if (req.param.returnpage) { - returnPage = Url.parse(req.param.returnpage); - redirect = returnPage.path; - delete req.param.returnpage; - } - - let redirectUrl = process.env.homeURL + redirect; - - return res.status(200).redirect(redirectUrl); - } - - req.login(user, async err => { - if (err) { - return next(err); - } - - var redirect = '/'; - - let returnPage = null; - let queryStringParsed = null; - if (req.param.returnpage) { - returnPage = Url.parse(req.param.returnpage); - redirect = returnPage.path; - queryStringParsed = queryString.parse(returnPage.query); - } - - let [, profile] = await to(getObjectById(req.user.id)); - - if (!profile) { - await to(updateRedirectURL({ id: req.user.id, redirectURL: redirect })); - return res.redirect(process.env.homeURL + '/completeRegistration/' + req.user.id); - } - - if (req.param.returnpage) { - delete req.param.returnpage; - } - - let redirectUrl = process.env.homeURL + redirect; - - if (queryStringParsed && queryStringParsed.sso && queryStringParsed.sig) { - try { - redirectUrl = discourseLogin(queryStringParsed.sso, queryStringParsed.sig, req.user); - } catch (err) { - console.error(err.message); - return res.status(500).send('Error authenticating the user.'); - } - } - - //Build event object for user login and log it to DB - let eventObj = { - userId: req.user.id, - event: `user_login_${req.user.provider}`, - timestamp: Date.now(), - }; - await eventLogController.logEvent(eventObj); - - return res - .status(200) - .cookie('jwt', signToken({ _id: req.user._id, id: req.user.id, timeStamp: Date.now() }), { - httpOnly: true, - secure: process.env.api_url ? true : false, - }) - .redirect(redirectUrl); - }); - })(req, res, next); - }); - + app.get( + '/auth/google/callback', + (req, res, next) => { + passport.authenticate('google', (err, user) => { + req.err = err; + req.user = user; + next(); + })(req, res, next); + }, + catchLoginErrorAndRedirect, + loginAndSignToken + ); return app; }; diff --git a/src/resources/auth/strategies/linkedin.js b/src/resources/auth/strategies/linkedin.js index 80cabb0b..c5bc4386 100644 --- a/src/resources/auth/strategies/linkedin.js +++ b/src/resources/auth/strategies/linkedin.js @@ -2,17 +2,11 @@ import passport from 'passport'; import passportLinkedin from 'passport-linkedin-oauth2'; import { to } from 'await-to-js'; +import { catchLoginErrorAndRedirect, loginAndSignToken } from '../utils'; import { getUserByProviderId } from '../../user/user.repository'; -import { getObjectById } from '../../tool/data.repository'; -import { updateRedirectURL } from '../../user/user.service'; import { createUser } from '../../user/user.service'; -import { signToken } from '../utils'; import { ROLES } from '../../user/user.roles'; -import queryString from 'query-string'; -import Url from 'url'; -import { discourseLogin } from '../sso/sso.discourse.service'; -const eventLogController = require('../../eventlog/eventlog.controller'); const LinkedinStrategy = passportLinkedin.OAuth2Strategy; const strategy = app => { @@ -62,83 +56,18 @@ const strategy = app => { }) ); - app.get('/auth/linkedin/callback', (req, res, next) => { - passport.authenticate('linkedin', (err, user) => { - if (err || !user) { - //loginError - if (err === 'loginError') return res.status(200).redirect(process.env.homeURL + '/loginerror'); - - // failureRedirect - var redirect = '/'; - let returnPage = null; - - if (req.param.returnpage) { - returnPage = Url.parse(req.param.returnpage); - redirect = returnPage.path; - delete req.param.returnpage; - } - - let redirectUrl = process.env.homeURL + redirect; - - return res.status(200).redirect(redirectUrl); - } - - req.login(user, async err => { - if (err) { - return next(err); - } - - var redirect = '/'; - - let returnPage = null; - let queryStringParsed = null; - if (req.param.returnpage) { - returnPage = Url.parse(req.param.returnpage); - redirect = returnPage.path; - queryStringParsed = queryString.parse(returnPage.query); - } - - let [, profile] = await to(getObjectById(req.user.id)); - - if (!profile) { - await to(updateRedirectURL({ id: req.user.id, redirectURL: redirect })); - return res.redirect(process.env.homeURL + '/completeRegistration/' + req.user.id); - } - - if (req.param.returnpage) { - delete req.param.returnpage; - } - - let redirectUrl = process.env.homeURL + redirect; - - if (queryStringParsed && queryStringParsed.sso && queryStringParsed.sig) { - try { - redirectUrl = discourseLogin(queryStringParsed.sso, queryStringParsed.sig, req.user); - } catch (err) { - console.error(err.message); - return res.status(500).send('Error authenticating the user.'); - } - } - - //Build event object for user login and log it to DB - let eventObj = { - userId: req.user.id, - event: `user_login_${req.user.provider}`, - timestamp: Date.now(), - }; - await eventLogController.logEvent(eventObj); - - return res - .status(200) - .cookie('jwt', signToken({ _id: req.user._id, id: req.user.id, timeStamp: Date.now() }), { - httpOnly: true, - secure: process.env.api_url ? true : false, - }) - .redirect(redirectUrl); - }); - })(req, res, next); - }); - + app.get( + '/auth/linkedin/callback', + (req, res, next) => { + passport.authenticate('linkedin', (err, user) => { + req.err = err; + req.user = user; + next(); + })(req, res, next); + }, + catchLoginErrorAndRedirect, + loginAndSignToken + ); return app; }; diff --git a/src/resources/auth/strategies/oidc.js b/src/resources/auth/strategies/oidc.js index 6493bb92..b8602d15 100644 --- a/src/resources/auth/strategies/oidc.js +++ b/src/resources/auth/strategies/oidc.js @@ -2,20 +2,15 @@ import passport from 'passport'; import passportOidc from 'passport-openidconnect'; import { to } from 'await-to-js'; +import { catchLoginErrorAndRedirect, loginAndSignToken } from '../utils'; import { getUserByProviderId } from '../../user/user.repository'; -import { getObjectById } from '../../tool/data.repository'; -import { updateRedirectURL } from '../../user/user.service'; import { createUser } from '../../user/user.service'; -import { signToken } from '../utils'; +import { UserModel } from '../../user/user.model'; import { ROLES } from '../../user/user.roles'; -import queryString from 'query-string'; -import Url from 'url'; -import { discourseLogin } from '../sso/sso.discourse.service'; import { isNil } from 'lodash'; + const OidcStrategy = passportOidc.Strategy; const baseAuthUrl = process.env.AUTH_PROVIDER_URI; -const eventLogController = require('../../eventlog/eventlog.controller'); -import { UserModel } from '../../user/user.model'; const strategy = app => { const strategyOptions = { @@ -71,83 +66,18 @@ const strategy = app => { passport.authenticate('oidc') ); - app.get('/auth/oidc/callback', (req, res, next) => { - passport.authenticate('oidc', (err, user) => { - if (err || !user) { - //loginError - if (err === 'loginError') return res.status(200).redirect(process.env.homeURL + '/loginerror'); - - // failureRedirect - var redirect = '/'; - let returnPage = null; - - if (req.param.returnpage) { - returnPage = Url.parse(req.param.returnpage); - redirect = returnPage.path; - delete req.param.returnpage; - } - - let redirectUrl = process.env.homeURL + redirect; - - return res.status(200).redirect(redirectUrl); - } - - req.login(user, async err => { - if (err) { - return next(err); - } - - var redirect = '/'; - - let returnPage = null; - let queryStringParsed = null; - if (req.param.returnpage) { - returnPage = Url.parse(req.param.returnpage); - redirect = returnPage.path; - queryStringParsed = queryString.parse(returnPage.query); - } - - let [, profile] = await to(getObjectById(req.user.id)); - - if (!profile) { - await to(updateRedirectURL({ id: req.user.id, redirectURL: redirect })); - return res.redirect(process.env.homeURL + '/completeRegistration/' + req.user.id); - } - - if (req.param.returnpage) { - delete req.param.returnpage; - } - - let redirectUrl = process.env.homeURL + redirect; - - if (queryStringParsed && queryStringParsed.sso && queryStringParsed.sig) { - try { - redirectUrl = discourseLogin(queryStringParsed.sso, queryStringParsed.sig, req.user); - } catch (err) { - console.error(err.message); - return res.status(500).send('Error authenticating the user.'); - } - } - - //Build event object for user login and log it to DB - let eventObj = { - userId: req.user.id, - event: `user_login_${req.user.provider}`, - timestamp: Date.now(), - }; - await eventLogController.logEvent(eventObj); - - return res - .status(200) - .cookie('jwt', signToken({ _id: req.user._id, id: req.user.id, timeStamp: Date.now() }), { - httpOnly: true, - secure: process.env.api_url ? true : false, - }) - .redirect(redirectUrl); - }); - })(req, res, next); - }); - + app.get( + '/auth/oidc/callback', + (req, res, next) => { + passport.authenticate('oidc', (err, user) => { + req.err = err; + req.user = user; + next(); + })(req, res, next); + }, + catchLoginErrorAndRedirect, + loginAndSignToken + ); return app; }; diff --git a/src/resources/auth/strategies/orcid.js b/src/resources/auth/strategies/orcid.js index 10f992fd..b7e0536e 100644 --- a/src/resources/auth/strategies/orcid.js +++ b/src/resources/auth/strategies/orcid.js @@ -3,155 +3,86 @@ import passportOrcid from 'passport-orcid'; import { to } from 'await-to-js'; import axios from 'axios'; +import { catchLoginErrorAndRedirect, loginAndSignToken } from '../utils'; import { getUserByProviderId } from '../../user/user.repository'; -import { updateRedirectURL } from '../../user/user.service'; -import { getObjectById } from '../../tool/data.repository'; import { createUser } from '../../user/user.service'; -import { signToken } from '../utils'; import { ROLES } from '../../user/user.roles'; -import queryString from 'query-string'; -import Url from 'url'; -import { discourseLogin } from '../sso/sso.discourse.service'; -const eventLogController = require('../../eventlog/eventlog.controller'); -const OrcidStrategy = passportOrcid.Strategy +const OrcidStrategy = passportOrcid.Strategy; const strategy = app => { - const strategyOptions = { - sandbox: process.env.ORCID_SSO_ENV, - clientID: process.env.ORCID_SSO_CLIENT_ID, - clientSecret: process.env.ORCID_SSO_CLIENT_SECRET, - callbackURL: `/auth/orcid/callback`, - scope: `/authenticate /read-limited`, - proxy: true - }; - - const verifyCallback = async (accessToken, refreshToken, params, profile, done) => { - - if (!params.orcid || params.orcid === '') return done('loginError'); - - let [err, user] = await to(getUserByProviderId(params.orcid)); - if (err || user) { - return done(err, user); - } - // Orcid does not include email natively - const requestedEmail = - await axios.get( - `${process.env.ORCID_SSO_BASE_URL}/v3.0/${params.orcid}/email`, - { headers: { 'Authorization': `Bearer ` + accessToken, 'Accept': 'application/json' }} - ).then((response) => { - const email = response.data.email[0].email - return ( email == undefined || !/\b[a-zA-Z0-9-_.]+\@[a-zA-Z0-9-_]+\.\w+(?:\.\w+)?\b/.test(email) ) ? '' : email - }).catch((err) => { - console.log(err); - return ''; - }); - - const [createdError, createdUser] = await to( - createUser({ - provider: 'orcid', - providerId: params.orcid, - firstname: params.name.split(' ')[0], - lastname: params.name.split(' ')[1], - password: null, - email: requestedEmail, - role: ROLES.Creator, - }) - ); - - return done(createdError, createdUser); - }; - - passport.use('orcid', new OrcidStrategy(strategyOptions, verifyCallback)); - - app.get( - `/auth/orcid`, - (req, res, next) => { - // Save the url of the user's current page so the app can redirect back to it after authorization - if (req.headers.referer) { - req.param.returnpage = req.headers.referer; - } - next(); - }, - passport.authenticate('orcid') - ); - - app.get('/auth/orcid/callback', (req, res, next) => { - passport.authenticate('orcid', (err, user, info) => { - - if (err || !user) { - //loginError - if (err === 'loginError') return res.status(200).redirect(process.env.homeURL + '/loginerror'); - - // failureRedirect - var redirect = '/'; - let returnPage = null; - if (req.param.returnpage) { - returnPage = Url.parse(req.param.returnpage); - redirect = returnPage.path; - delete req.param.returnpage; - }; - - let redirectUrl = process.env.homeURL + redirect; - return res.status(200).redirect(redirectUrl); - }; - - req.login(user, async err => { - - if (err) { - return next(err); - }; - - var redirect = '/'; - let returnPage = null; - let queryStringParsed = null; - if (req.param.returnpage) { - returnPage = Url.parse(req.param.returnpage); - redirect = returnPage.path; - queryStringParsed = queryString.parse(returnPage.query); - }; - - let [profileErr, profile] = await to(getObjectById(req.user.id)); - if (!profile) { - await to(updateRedirectURL({ id: req.user.id, redirectURL: redirect })); - return res.redirect(process.env.homeURL + '/completeRegistration/' + req.user.id); - }; - - if (req.param.returnpage) { - delete req.param.returnpage; - }; - - let redirectUrl = process.env.homeURL + redirect; - if (queryStringParsed && queryStringParsed.sso && queryStringParsed.sig) { - try { - console.log(req.user) - redirectUrl = discourseLogin(queryStringParsed.sso, queryStringParsed.sig, req.user); - } catch (err) { - console.error(err.message); - return res.status(500).send('Error authenticating the user.'); - } - }; - - //Build event object for user login and log it to DB - let eventObj = { - userId: req.user.id, - event: `user_login_${req.user.provider}`, - timestamp: Date.now(), - }; - - await eventLogController.logEvent(eventObj); - - return res - .status(200) - .cookie('jwt', signToken({ _id: req.user._id, id: req.user.id, timeStamp: Date.now() }), { - httpOnly: true, - secure: process.env.api_url ? true : false, - }) - .redirect(redirectUrl); - }); - })(req, res, next); - }); - return app; + const strategyOptions = { + sandbox: process.env.ORCID_SSO_ENV, + clientID: process.env.ORCID_SSO_CLIENT_ID, + clientSecret: process.env.ORCID_SSO_CLIENT_SECRET, + callbackURL: `/auth/orcid/callback`, + scope: `/authenticate /read-limited`, + proxy: true, + }; + + const verifyCallback = async (accessToken, refreshToken, params, profile, done) => { + if (!params.orcid || params.orcid === '') return done('loginError'); + + let [err, user] = await to(getUserByProviderId(params.orcid)); + if (err || user) { + return done(err, user); + } + // Orcid does not include email natively + const requestedEmail = await axios + .get(`${process.env.ORCID_SSO_BASE_URL}/v3.0/${params.orcid}/email`, { + headers: { Authorization: `Bearer ` + accessToken, Accept: 'application/json' }, + }) + .then(response => { + const email = response.data.email[0].email; + return email == undefined || !/\b[a-zA-Z0-9-_.]+\@[a-zA-Z0-9-_]+\.\w+(?:\.\w+)?\b/.test(email) ? '' : email; + }) + .catch(err => { + console.log(err); + return ''; + }); + + const [createdError, createdUser] = await to( + createUser({ + provider: 'orcid', + providerId: params.orcid, + firstname: params.name.split(' ')[0], + lastname: params.name.split(' ')[1], + password: null, + email: requestedEmail, + role: ROLES.Creator, + }) + ); + + return done(createdError, createdUser); + }; + + passport.use('orcid', new OrcidStrategy(strategyOptions, verifyCallback)); + + app.get( + `/auth/orcid`, + (req, res, next) => { + // Save the url of the user's current page so the app can redirect back to it after authorization + if (req.headers.referer) { + req.param.returnpage = req.headers.referer; + } + next(); + }, + passport.authenticate('orcid') + ); + + app.get( + '/auth/orcid/callback', + (req, res, next) => { + passport.authenticate('orcid', (err, user, info) => { + req.err = err; + req.user = user; + next(); + })(req, res, next); + }, + catchLoginErrorAndRedirect, + loginAndSignToken + ); + return app; }; -export { strategy }; \ No newline at end of file +export { strategy }; diff --git a/src/resources/auth/utils.js b/src/resources/auth/utils.js index e6bc69cb..b1679822 100644 --- a/src/resources/auth/utils.js +++ b/src/resources/auth/utils.js @@ -1,6 +1,11 @@ /* eslint-disable no-undef */ import passport from 'passport'; import jwt from 'jsonwebtoken'; +import { to } from 'await-to-js'; +import Url from 'url'; +import { isEmpty } from 'lodash'; +import queryString from 'query-string'; + import { ROLES } from '../user/user.roles'; import { UserModel } from '../user/user.model'; import { Course } from '../course/course.model'; @@ -8,7 +13,11 @@ import { Collections } from '../collections/collections.model'; import { Data } from '../tool/data.model'; import { TeamModel } from '../team/team.model'; import constants from '../utilities/constants.util'; -import { isEmpty } from 'lodash'; +import { discourseLogin } from './sso/sso.discourse.service'; +import { updateRedirectURL } from './../user/user.service'; +import { getObjectById } from './../tool/data.repository'; + +const eventLogController = require('./../eventlog/eventlog.controller'); const setup = () => { passport.serializeUser((user, done) => done(null, user._id)); @@ -46,18 +55,20 @@ const camundaToken = () => { ); }; -const checkIsInRole = (...roles) => (req, res, next) => { - if (!req.user) { - return res.redirect('/login'); - } +const checkIsInRole = + (...roles) => + (req, res, next) => { + if (!req.user) { + return res.redirect('/login'); + } - const hasRole = roles.find(role => req.user.role === role); - if (!hasRole) { - return res.redirect('/login'); - } + const hasRole = roles.find(role => req.user.role === role); + if (!hasRole) { + return res.redirect('/login'); + } - return next(); -}; + return next(); + }; const whatIsRole = req => { if (!req.user) { @@ -116,4 +127,90 @@ const getTeams = async () => { return teams; }; -export { setup, signToken, camundaToken, checkIsInRole, whatIsRole, checkIsUser, checkAllowedToAccess, getTeams }; +const catchLoginErrorAndRedirect = (req, res, next) => { + if (req.err || !req.user) { + if (req.err === 'loginError') { + return res.status(200).redirect(process.env.homeURL + '/loginerror'); + } + + let redirect = '/'; + let returnPage = null; + if (req.param.returnpage) { + returnPage = Url.parse(req.param.returnpage); + redirect = returnPage.path; + delete req.param.returnpage; + } + + let redirectUrl = process.env.homeURL + redirect; + + return res.status(200).redirect(redirectUrl); + } + next(); +}; + +const loginAndSignToken = (req, res, next) => { + req.login(req.user, async err => { + if (err) { + return next(err); + } + + let redirect = '/'; + let returnPage = null; + let queryStringParsed = null; + if (req.param.returnpage) { + returnPage = Url.parse(req.param.returnpage); + redirect = returnPage.path; + queryStringParsed = queryString.parse(returnPage.query); + } + + let [, profile] = await to(getObjectById(req.user.id)); + if (!profile) { + await to(updateRedirectURL({ id: req.user.id, redirectURL: redirect })); + return res.redirect(process.env.homeURL + '/completeRegistration/' + req.user.id); + } + + if (req.param.returnpage) { + delete req.param.returnpage; + } + + let redirectUrl = process.env.homeURL + redirect; + if (queryStringParsed && queryStringParsed.sso && queryStringParsed.sig) { + try { + redirectUrl = discourseLogin(queryStringParsed.sso, queryStringParsed.sig, req.user); + } catch (err) { + console.error(err.message); + return res.status(500).send('Error authenticating the user.'); + } + } + + //Build event object for user login and log it to DB + let eventObj = { + userId: req.user.id, + event: `user_login_${req.user.provider}`, + timestamp: Date.now(), + }; + + await eventLogController.logEvent(eventObj); + + return res + .status(200) + .cookie('jwt', signToken({ _id: req.user._id, id: req.user.id, timeStamp: Date.now() }), { + httpOnly: true, + secure: process.env.api_url ? true : false, + }) + .redirect(redirectUrl); + }); +}; + +export { + setup, + signToken, + camundaToken, + checkIsInRole, + whatIsRole, + checkIsUser, + checkAllowedToAccess, + getTeams, + catchLoginErrorAndRedirect, + loginAndSignToken, +}; From 7a557deb46058090ddb7f5631fd7b8caffa6fa87 Mon Sep 17 00:00:00 2001 From: Robin Kavanagh Date: Wed, 6 Oct 2021 07:19:01 +0100 Subject: [PATCH 031/116] Made modifications to support recent DUR change request --- .../__mocks__/dataUseRegisters.js | 40 +++++++++++-------- .../dataUseRegister/dataUseRegister.model.js | 5 ++- .../dataUseRegister.service.js | 30 ++++++++++++-- .../dataUseRegister/dataUseRegister.util.js | 5 ++- 4 files changed, 56 insertions(+), 24 deletions(-) diff --git a/src/resources/dataUseRegister/__mocks__/dataUseRegisters.js b/src/resources/dataUseRegister/__mocks__/dataUseRegisters.js index abbdbf87..7bb54d15 100644 --- a/src/resources/dataUseRegister/__mocks__/dataUseRegisters.js +++ b/src/resources/dataUseRegister/__mocks__/dataUseRegisters.js @@ -24,11 +24,12 @@ export const dataUseRegisterUploads = [ projectEndDate: '2021-09-30', latestApprovalDate: '2021-09-21', dataSensitivityLevel: 'data Sensitivity Level', - legalBasisForData: 'legal Basis For Data', + legalBasisForDataArticle6: 'legal Basis For Data 6', + legalBasisForDataArticle9: 'legal Basis For Data 9', dutyOfConfidentiality: 'duty Of Confidentiality', nationalDataOptOut: 'national Data Opt Out', requestFrequency: 'request Frequency', - dataProcessingDescription: 'data Processing Description', + datasetLinkageDescription: 'data Processing Description', confidentialDataDescription: 'confidential Data Description', accessDate: '2021-09-26', dataLocation: 'data Location', @@ -60,11 +61,12 @@ export const dataUseRegisterUploads = [ projectEndDate: '2021-09-30', latestApprovalDate: '2021-09-21', dataSensitivityLevel: 'data Sensitivity Level', - legalBasisForData: 'legal Basis For Data', + legalBasisForDataArticle6: 'legal Basis For Data 6', + legalBasisForDataArticle9: 'legal Basis For Data 9', dutyOfConfidentiality: 'duty Of Confidentiality', nationalDataOptOut: 'national Data Opt Out', requestFrequency: 'request Frequency', - dataProcessingDescription: 'data Processing Description', + datasetLinkageDescription: 'data Processing Description', confidentialDataDescription: 'confidential Data Description', accessDate: '2021-09-26', dataLocation: 'data Location', @@ -99,11 +101,12 @@ export const dataUseRegisterUploadsWithDuplicates = [ projectEndDate: '2021-09-30', latestApprovalDate: '2021-09-21', dataSensitivityLevel: 'data Sensitivity Level', - legalBasisForData: 'legal Basis For Data', + legalBasisForDataArticle6: 'legal Basis For Data 6', + legalBasisForDataArticle9: 'legal Basis For Data 9', dutyOfConfidentiality: 'duty Of Confidentiality', nationalDataOptOut: 'national Data Opt Out', requestFrequency: 'request Frequency', - dataProcessingDescription: 'data Processing Description', + datasetLinkageDescription: 'data Processing Description', confidentialDataDescription: 'confidential Data Description', accessDate: '2021-09-26', dataLocation: 'data Location', @@ -135,11 +138,12 @@ export const dataUseRegisterUploadsWithDuplicates = [ projectEndDate: '2021-09-30', latestApprovalDate: '2021-09-21', dataSensitivityLevel: 'data Sensitivity Level', - legalBasisForData: 'legal Basis For Data', + legalBasisForDataArticle6: 'legal Basis For Data 6', + legalBasisForDataArticle9: 'legal Basis For Data 9', dutyOfConfidentiality: 'duty Of Confidentiality', nationalDataOptOut: 'national Data Opt Out', requestFrequency: 'request Frequency', - dataProcessingDescription: 'data Processing Description', + datasetLinkageDescription: 'data Processing Description', confidentialDataDescription: 'confidential Data Description', accessDate: '2021-09-26', dataLocation: 'data Location', @@ -171,11 +175,12 @@ export const dataUseRegisterUploadsWithDuplicates = [ projectEndDate: '2021-09-30', latestApprovalDate: '2021-09-21', dataSensitivityLevel: 'data Sensitivity Level', - legalBasisForData: 'legal Basis For Data', + legalBasisForDataArticle6: 'legal Basis For Data 6', + legalBasisForDataArticle9: 'legal Basis For Data 9', dutyOfConfidentiality: 'duty Of Confidentiality', nationalDataOptOut: 'national Data Opt Out', requestFrequency: 'request Frequency', - dataProcessingDescription: 'data Processing Description', + datasetLinkageDescription: 'data Processing Description', confidentialDataDescription: 'confidential Data Description', accessDate: '2021-09-26', dataLocation: 'data Location', @@ -207,11 +212,12 @@ export const dataUseRegisterUploadsWithDuplicates = [ projectEndDate: '2021-09-30', latestApprovalDate: '2021-09-21', dataSensitivityLevel: 'data Sensitivity Level', - legalBasisForData: 'legal Basis For Data', + legalBasisForDataArticle6: 'legal Basis For Data 6', + legalBasisForDataArticle9: 'legal Basis For Data 9', dutyOfConfidentiality: 'duty Of Confidentiality', nationalDataOptOut: 'national Data Opt Out', requestFrequency: 'request Frequency', - dataProcessingDescription: 'data Processing Description', + datasetLinkageDescription: 'data Processing Description', confidentialDataDescription: 'confidential Data Description', accessDate: '2021-09-26', dataLocation: 'data Location', @@ -243,11 +249,12 @@ export const dataUseRegisterUploadsWithDuplicates = [ projectEndDate: '2021-09-30', latestApprovalDate: '2021-09-21', dataSensitivityLevel: 'data Sensitivity Level', - legalBasisForData: 'legal Basis For Data', + legalBasisForDataArticle6: 'legal Basis For Data 6', + legalBasisForDataArticle9: 'legal Basis For Data 9', dutyOfConfidentiality: 'duty Of Confidentiality', nationalDataOptOut: 'national Data Opt Out', requestFrequency: 'request Frequency', - dataProcessingDescription: 'data Processing Description', + datasetLinkageDescription: 'data Processing Description', confidentialDataDescription: 'confidential Data Description', accessDate: '2021-09-26', dataLocation: 'data Location', @@ -279,11 +286,12 @@ export const dataUseRegisterUploadsWithDuplicates = [ projectEndDate: '2021-09-30', latestApprovalDate: '2021-09-21', dataSensitivityLevel: 'data Sensitivity Level', - legalBasisForData: 'legal Basis For Data', + legalBasisForDataArticle6: 'legal Basis For Data 6', + legalBasisForDataArticle9: 'legal Basis For Data 9', dutyOfConfidentiality: 'duty Of Confidentiality', nationalDataOptOut: 'national Data Opt Out', requestFrequency: 'request Frequency', - dataProcessingDescription: 'data Processing Description', + datasetLinkageDescription: 'data Processing Description', confidentialDataDescription: 'confidential Data Description', accessDate: '2021-09-26', dataLocation: 'data Location', diff --git a/src/resources/dataUseRegister/dataUseRegister.model.js b/src/resources/dataUseRegister/dataUseRegister.model.js index 98d3c622..3fdc006d 100644 --- a/src/resources/dataUseRegister/dataUseRegister.model.js +++ b/src/resources/dataUseRegister/dataUseRegister.model.js @@ -55,11 +55,12 @@ const dataUseRegisterSchema = new Schema( projectEndDate: Date, //Project End Date latestApprovalDate: Date, //Latest Approval Date dataSensitivityLevel: String, //Data Sensitivity Level - legalBasisForData: String, //Legal Basis For Provision Of Data + legalBasisForDataArticle6: String, //Legal Basis For Provision Of Data (changed to 'Legal basis for provision of data under Article 6') + legalBasisForDataArticle9: String, //Added 'Lawful conditions for provision of data under Article 9' dutyOfConfidentiality: String, //Common Law Duty Of Confidentiality nationalDataOptOut: String, //National Data Opt-Out Applied requestFrequency: String, //Request Frequency - dataProcessingDescription: String, //Description Of How The Data Will Be Processed + datasetLinkageDescription: String, //Description Of How The Data Will Be Processed (changed to 'For linked datasets, specify how the linkage will take place') confidentialDataDescription: String, //Description Of The Confidential Data Being Used accessDate: Date, //Release/Access Date dataLocation: String, //TRE Or Any Other Specified Location diff --git a/src/resources/dataUseRegister/dataUseRegister.service.js b/src/resources/dataUseRegister/dataUseRegister.service.js index 283afb89..c5028288 100644 --- a/src/resources/dataUseRegister/dataUseRegister.service.js +++ b/src/resources/dataUseRegister/dataUseRegister.service.js @@ -128,13 +128,25 @@ export default class DataUseRegisterService { ['safeproject-projectdetails-startdate']: startDate, ['safeproject-projectdetails-enddate']: endDate, safedatastorageandprocessingaccessmethodtrustedresearchenvironment: dataLocation, + safedataconfidentialityavenuelegalbasisconfidentialinformation: dutyOfConfidentiality, + safedataotherdatasetslinkadditionaldatasetslinkagedetails: datasetLinkageDetails = '', + safedataotherdatasetsrisksmitigations: datasetLinkageRiskMitigation = '', + safedatalawfulbasisgdprarticle6basis: legalBasisForDataArticle6, + safedatalawfulbasisgdprarticle9conditions: legalBasisForDataArticle9, + safedatadatafieldsdatarefreshrequired: dataRefreshRequired = '', + safeoutputsoutputsdisseminationplansdisclosurecontrolpolicy: privacyEnhancements, }, } = accessRecord; const fundersAndSponsors = dataUseRegisterUtil.extractFundersAndSponsors(questionAnswers); - const { gatewayApplicants = [], nonGatewayApplicants = [] } = dataUseRegisterUtil.extractFormApplicants([...authors, mainApplicant], questionAnswers); + const { gatewayApplicants = [], nonGatewayApplicants = [] } = dataUseRegisterUtil.extractFormApplicants( + [...authors, mainApplicant], + questionAnswers + ); const relatedDatasets = dataUseRegisterUtil.buildRelatedDatasets(creatorUser, datasets, false); const relatedApplications = await this.buildRelatedDataUseRegisters(creatorUser, versionTree, applicationId); + const datasetLinkageDescription = `${datasetLinkageDetails.toString().trim()} ${datasetLinkageRiskMitigation.toString().trim()}`; + const requestFrequency = dataRefreshRequired === 'Yes' ? 'Recurring' : dataRefreshRequired === 'No' ? 'One-off' : ''; const projectStartDate = moment(startDate, 'DD/MM/YYYY'); const projectEndDate = moment(endDate, 'DD/MM/YYYY'); @@ -145,12 +157,18 @@ export default class DataUseRegisterService { projectIdText: projectId, projectId: applicationId, applicantId: applicantId.trim(), + accreditedResearcherStatus: isNil(accreditedResearcherStatus) ? 'Unknown' : accreditedResearcherStatus.toString().trim(), ...(projectTitle && { projectTitle: projectTitle.toString().trim() }), - ...(accreditedResearcherStatus && { accreditedResearcherStatus: accreditedResearcherStatus.toString().trim() }), ...(organisationName && { organisationName: organisationName.toString().trim() }), ...(laySummary && { laySummary: laySummary.toString().trim() }), ...(publicBenefitStatement && { publicBenefitStatement: publicBenefitStatement.toString().trim() }), ...(dataLocation && { dataLocation: dataLocation.toString().trim() }), + ...(dutyOfConfidentiality && { dutyOfConfidentiality: dutyOfConfidentiality.toString().trim() }), + ...(!isEmpty(datasetLinkageDescription) && { datasetLinkageDescription: datasetLinkageDescription.trim() }), + ...(!isEmpty(requestFrequency) && { requestFrequency }), + ...(legalBasisForDataArticle6 && { legalBasisForDataArticle6: legalBasisForDataArticle6.toString().trim() }), + ...(legalBasisForDataArticle9 && { legalBasisForDataArticle9: legalBasisForDataArticle9.toString().trim() }), + ...(privacyEnhancements && { privacyEnhancements: privacyEnhancements.toString().trim() }), ...(projectStartDate.isValid() && { projectStartDate }), ...(projectEndDate.isValid() && { projectEndDate }), ...(latestApprovalDate.isValid() && { latestApprovalDate }), @@ -169,7 +187,7 @@ export default class DataUseRegisterService { lastActivity: Date.now(), manualUpload: false, }); - + this.dataUseRegisterRepository.createDataUseRegister(dataUseRegister); } @@ -189,7 +207,11 @@ export default class DataUseRegisterService { const ignoredApplicationTypes = [constants.submissionTypes.INPROGRESS, constants.submissionTypes.RESUBMISSION]; for (const key of Object.keys(versionTree)) { - if (versionTree[key].applicationType && !ignoredApplicationTypes.includes(versionTree[key].applicationType) && versionTree[key].toString() !== applicationId.toString()) { + if ( + versionTree[key].applicationType && + !ignoredApplicationTypes.includes(versionTree[key].applicationType) && + versionTree[key].toString() !== applicationId.toString() + ) { const { applicationId } = versionTree[key]; const dataUseRegister = await this.dataUseRegisterRepository.getDataUseRegisterByApplicationId(applicationId); diff --git a/src/resources/dataUseRegister/dataUseRegister.util.js b/src/resources/dataUseRegister/dataUseRegister.util.js index a0573566..8f3924e0 100644 --- a/src/resources/dataUseRegister/dataUseRegister.util.js +++ b/src/resources/dataUseRegister/dataUseRegister.util.js @@ -93,10 +93,11 @@ const buildDataUseRegisters = async (creatorUser, teamId, dataUses = []) => { ...(obj.requestCategoryType && { requestCategoryType: obj.requestCategoryType.toString().trim() }), ...(obj.technicalSummary && { technicalSummary: obj.technicalSummary.toString().trim() }), ...(obj.dataSensitivityLevel && { dataSensitivityLevel: obj.dataSensitivityLevel.toString().trim() }), - ...(obj.legalBasisForData && { legalBasisForData: obj.legalBasisForData.toString().trim() }), + ...(obj.legalBasisForDataArticle6 && { legalBasisForDataArticle6: obj.legalBasisForDataArticle6.toString().trim() }), + ...(obj.legalBasisForDataArticle9 && { legalBasisForDataArticle9: obj.legalBasisForDataArticle9.toString().trim() }), ...(obj.nationalDataOptOut && { nationalDataOptOut: obj.nationalDataOptOut.toString().trim() }), ...(obj.requestFrequency && { requestFrequency: obj.requestFrequency.toString().trim() }), - ...(obj.dataProcessingDescription && { dataProcessingDescription: obj.dataProcessingDescription.toString().trim() }), + ...(obj.datasetLinkageDescription && { datasetLinkageDescription: obj.datasetLinkageDescription.toString().trim() }), ...(obj.confidentialDataDescription && { confidentialDataDescription: obj.confidentialDataDescription.toString().trim() }), ...(obj.dataLocation && { dataLocation: obj.dataLocation.toString().trim() }), ...(obj.privacyEnhancements && { privacyEnhancements: obj.privacyEnhancements.toString().trim() }), From b48fd6a7f40f73476bbd4fc54b1759a40bd407b4 Mon Sep 17 00:00:00 2001 From: Callum Reekie Date: Wed, 6 Oct 2021 09:19:27 +0100 Subject: [PATCH 032/116] rattach params to request object --- src/resources/auth/strategies/azure.js | 5 +++-- src/resources/auth/strategies/google.js | 5 +++-- src/resources/auth/strategies/linkedin.js | 5 +++-- src/resources/auth/strategies/oidc.js | 5 +++-- src/resources/auth/strategies/orcid.js | 5 +++-- src/resources/auth/utils.js | 6 +++--- 6 files changed, 18 insertions(+), 13 deletions(-) diff --git a/src/resources/auth/strategies/azure.js b/src/resources/auth/strategies/azure.js index fbc6ab21..d1adcf44 100644 --- a/src/resources/auth/strategies/azure.js +++ b/src/resources/auth/strategies/azure.js @@ -67,8 +67,9 @@ const strategy = app => { `/auth/azure/callback`, (req, res, next) => { passport.authenticate('azure_ad_oauth2', (err, user) => { - req.err = err; - req.user = user; + req.auth = {}; + req.auth.err = err; + req.auth.user = user; next(); })(req, res, next); }, diff --git a/src/resources/auth/strategies/google.js b/src/resources/auth/strategies/google.js index 4209a9a6..9ddb4d63 100644 --- a/src/resources/auth/strategies/google.js +++ b/src/resources/auth/strategies/google.js @@ -62,8 +62,9 @@ const strategy = app => { '/auth/google/callback', (req, res, next) => { passport.authenticate('google', (err, user) => { - req.err = err; - req.user = user; + req.auth = {}; + req.auth.err = err; + req.auth.user = user; next(); })(req, res, next); }, diff --git a/src/resources/auth/strategies/linkedin.js b/src/resources/auth/strategies/linkedin.js index c5bc4386..b53372bf 100644 --- a/src/resources/auth/strategies/linkedin.js +++ b/src/resources/auth/strategies/linkedin.js @@ -60,8 +60,9 @@ const strategy = app => { '/auth/linkedin/callback', (req, res, next) => { passport.authenticate('linkedin', (err, user) => { - req.err = err; - req.user = user; + req.auth = {}; + req.auth.err = err; + req.auth.user = user; next(); })(req, res, next); }, diff --git a/src/resources/auth/strategies/oidc.js b/src/resources/auth/strategies/oidc.js index b8602d15..7d0c30da 100644 --- a/src/resources/auth/strategies/oidc.js +++ b/src/resources/auth/strategies/oidc.js @@ -70,8 +70,9 @@ const strategy = app => { '/auth/oidc/callback', (req, res, next) => { passport.authenticate('oidc', (err, user) => { - req.err = err; - req.user = user; + req.auth = {}; + req.auth.err = err; + req.auth.user = user; next(); })(req, res, next); }, diff --git a/src/resources/auth/strategies/orcid.js b/src/resources/auth/strategies/orcid.js index b7e0536e..f9dc7911 100644 --- a/src/resources/auth/strategies/orcid.js +++ b/src/resources/auth/strategies/orcid.js @@ -74,8 +74,9 @@ const strategy = app => { '/auth/orcid/callback', (req, res, next) => { passport.authenticate('orcid', (err, user, info) => { - req.err = err; - req.user = user; + req.auth = {}; + req.auth.err = err; + req.auth.user = user; next(); })(req, res, next); }, diff --git a/src/resources/auth/utils.js b/src/resources/auth/utils.js index b1679822..97cb301b 100644 --- a/src/resources/auth/utils.js +++ b/src/resources/auth/utils.js @@ -128,8 +128,8 @@ const getTeams = async () => { }; const catchLoginErrorAndRedirect = (req, res, next) => { - if (req.err || !req.user) { - if (req.err === 'loginError') { + if (req.auth.err || !req.auth.user) { + if (req.auth.err === 'loginError') { return res.status(200).redirect(process.env.homeURL + '/loginerror'); } @@ -149,7 +149,7 @@ const catchLoginErrorAndRedirect = (req, res, next) => { }; const loginAndSignToken = (req, res, next) => { - req.login(req.user, async err => { + req.login(req.auth.user, async err => { if (err) { return next(err); } From 71db988eab74945ecfe6233abd842b8fb5026c14 Mon Sep 17 00:00:00 2001 From: Callum Reekie Date: Wed, 6 Oct 2021 09:39:42 +0100 Subject: [PATCH 033/116] change to how req.auth is initialised --- src/resources/auth/strategies/azure.js | 7 ++++--- src/resources/auth/strategies/google.js | 7 ++++--- src/resources/auth/strategies/linkedin.js | 7 ++++--- src/resources/auth/strategies/oidc.js | 7 ++++--- src/resources/auth/strategies/orcid.js | 7 ++++--- 5 files changed, 20 insertions(+), 15 deletions(-) diff --git a/src/resources/auth/strategies/azure.js b/src/resources/auth/strategies/azure.js index d1adcf44..b0b2be03 100644 --- a/src/resources/auth/strategies/azure.js +++ b/src/resources/auth/strategies/azure.js @@ -67,9 +67,10 @@ const strategy = app => { `/auth/azure/callback`, (req, res, next) => { passport.authenticate('azure_ad_oauth2', (err, user) => { - req.auth = {}; - req.auth.err = err; - req.auth.user = user; + req.auth = { + err: err, + user: user, + }; next(); })(req, res, next); }, diff --git a/src/resources/auth/strategies/google.js b/src/resources/auth/strategies/google.js index 9ddb4d63..1af2697b 100644 --- a/src/resources/auth/strategies/google.js +++ b/src/resources/auth/strategies/google.js @@ -62,9 +62,10 @@ const strategy = app => { '/auth/google/callback', (req, res, next) => { passport.authenticate('google', (err, user) => { - req.auth = {}; - req.auth.err = err; - req.auth.user = user; + req.auth = { + err: err, + user: user, + }; next(); })(req, res, next); }, diff --git a/src/resources/auth/strategies/linkedin.js b/src/resources/auth/strategies/linkedin.js index b53372bf..584d639c 100644 --- a/src/resources/auth/strategies/linkedin.js +++ b/src/resources/auth/strategies/linkedin.js @@ -60,9 +60,10 @@ const strategy = app => { '/auth/linkedin/callback', (req, res, next) => { passport.authenticate('linkedin', (err, user) => { - req.auth = {}; - req.auth.err = err; - req.auth.user = user; + req.auth = { + err: err, + user: user, + }; next(); })(req, res, next); }, diff --git a/src/resources/auth/strategies/oidc.js b/src/resources/auth/strategies/oidc.js index 7d0c30da..3748e8fd 100644 --- a/src/resources/auth/strategies/oidc.js +++ b/src/resources/auth/strategies/oidc.js @@ -70,9 +70,10 @@ const strategy = app => { '/auth/oidc/callback', (req, res, next) => { passport.authenticate('oidc', (err, user) => { - req.auth = {}; - req.auth.err = err; - req.auth.user = user; + req.auth = { + err: err, + user: user, + }; next(); })(req, res, next); }, diff --git a/src/resources/auth/strategies/orcid.js b/src/resources/auth/strategies/orcid.js index f9dc7911..68ca546a 100644 --- a/src/resources/auth/strategies/orcid.js +++ b/src/resources/auth/strategies/orcid.js @@ -74,9 +74,10 @@ const strategy = app => { '/auth/orcid/callback', (req, res, next) => { passport.authenticate('orcid', (err, user, info) => { - req.auth = {}; - req.auth.err = err; - req.auth.user = user; + req.auth = { + err: err, + user: user, + }; next(); })(req, res, next); }, From 4e0719a565900b62ec395522e69b027aa2b1fd50 Mon Sep 17 00:00:00 2001 From: Callum Reekie Date: Wed, 6 Oct 2021 10:55:15 +0100 Subject: [PATCH 034/116] added appropriate tests --- .../auth/__tests__/auth.utilities.test.js | 72 +++++++++++++++++++ src/resources/auth/strategies/orcid.js | 2 +- 2 files changed, 73 insertions(+), 1 deletion(-) create mode 100644 src/resources/auth/__tests__/auth.utilities.test.js diff --git a/src/resources/auth/__tests__/auth.utilities.test.js b/src/resources/auth/__tests__/auth.utilities.test.js new file mode 100644 index 00000000..75da1a4f --- /dev/null +++ b/src/resources/auth/__tests__/auth.utilities.test.js @@ -0,0 +1,72 @@ +import { catchLoginErrorAndRedirect, loginAndSignToken } from '../utils'; + +describe('Utilities', () => { + describe('catchErrorAndRedirect middleware', () => { + it('should be a function', () => { + expect(typeof catchLoginErrorAndRedirect).toBe('function'); + }); + + it('should call next once when ( req.auth.err || !req.auth.user ) == false', () => { + let res = {}; + let req = { + auth: { + user: 'someUser', + err: null, + }, + }; + const next = jest.fn(); + + catchLoginErrorAndRedirect(req, res, next); + + // assert + expect(next.mock.calls.length).toBe(1); + }); + + it('should not call next when ( req.auth.err || !req.auth.user ) == true', () => { + let res = {}; + res.status = jest.fn().mockReturnValue(res); + res.redirect = jest.fn().mockReturnValue(res); + let req = { + auth: { + user: {}, + err: 'someErr', + }, + param: { + returnpage: 'somePage', + }, + }; + const next = jest.fn(); + + catchLoginErrorAndRedirect(req, res, next); + + // assert + expect(next.mock.calls.length).toBe(0); + expect(res.status.mock.calls.length).toBe(1); + expect(res.redirect.mock.calls.length).toBe(1); + }); + }); + + describe('loginAndSignToken middleware', () => { + it('should be a function', () => { + expect(typeof loginAndSignToken).toBe('function'); + }); + + it('should call res.login once', () => { + let res = {}; + res.status = jest.fn().mockReturnValue(res); + res.redirect = jest.fn().mockReturnValue(res); + let req = { + auth: { + user: 'someUser', + }, + }; + req.login = jest.fn().mockReturnValue(req); + const next = jest.fn(); + + loginAndSignToken(req, res, next); + + // assert + expect(req.login.mock.calls.length).toBe(1); + }); + }); +}); diff --git a/src/resources/auth/strategies/orcid.js b/src/resources/auth/strategies/orcid.js index 68ca546a..4638cc5d 100644 --- a/src/resources/auth/strategies/orcid.js +++ b/src/resources/auth/strategies/orcid.js @@ -73,7 +73,7 @@ const strategy = app => { app.get( '/auth/orcid/callback', (req, res, next) => { - passport.authenticate('orcid', (err, user, info) => { + passport.authenticate('orcid', (err, user) => { req.auth = { err: err, user: user, From 43449be094a17dbfe4e276eac71ab98cee0447b9 Mon Sep 17 00:00:00 2001 From: massimocianfroccaPA Date: Wed, 6 Oct 2021 11:53:13 +0100 Subject: [PATCH 035/116] Add data use. API check --- .../dataUseRegister.controller.js | 18 ++++++ .../dataUseRegister/dataUseRegister.route.js | 15 +++-- .../dataUseRegister.service.js | 57 ++++++++++++++++++- .../dataUseRegister/dataUseRegister.util.js | 12 ++-- src/resources/user/user.repository.js | 2 +- 5 files changed, 94 insertions(+), 10 deletions(-) diff --git a/src/resources/dataUseRegister/dataUseRegister.controller.js b/src/resources/dataUseRegister/dataUseRegister.controller.js index f22579b3..567dfdf9 100644 --- a/src/resources/dataUseRegister/dataUseRegister.controller.js +++ b/src/resources/dataUseRegister/dataUseRegister.controller.js @@ -2,6 +2,7 @@ import Controller from '../base/controller'; import { logger } from '../utilities/logger'; import _ from 'lodash'; import constants from './../utilities/constants.util'; +import dataUseRegisterUtil from './dataUseRegister.util'; const logCategory = 'dataUseRegister'; @@ -123,4 +124,21 @@ export default class DataUseRegisterController extends Controller { }); } } + + async checkDataUseRegister(req, res) { + try { + const { dataUses } = req.body; + + const result = await this.dataUseRegisterService.checkDataUseRegisters(dataUses); + + return res.status(200).json({ success: true, result }); + } catch (err) { + // Return error response if something goes wrong + logger.logError(err, logCategory); + return res.status(500).json({ + success: false, + message: 'A server error occurred, please try again', + }); + } + } } diff --git a/src/resources/dataUseRegister/dataUseRegister.route.js b/src/resources/dataUseRegister/dataUseRegister.route.js index 55a223bd..2ed60be6 100644 --- a/src/resources/dataUseRegister/dataUseRegister.route.js +++ b/src/resources/dataUseRegister/dataUseRegister.route.js @@ -68,11 +68,11 @@ const validateUploadRequest = (req, res, next) => { errors.push('You must provide the custodian team identifier to associate the data uses to'); } - if(!dataUses || isEmpty(dataUses)) { + if (!dataUses || isEmpty(dataUses)) { errors.push('You must provide data uses to upload'); } - if(!isEmpty(errors)){ + if (!isEmpty(errors)) { return res.status(400).json({ success: false, message: errors.join(', '), @@ -136,9 +136,9 @@ const authorizeUpload = async (req, res, next) => { message: 'You are not authorised to perform this action', }); } - + next(); -} +}; // @route GET /api/v2/data-use-registers/id // @desc Returns a dataUseRegister based on dataUseRegister ID provided @@ -174,6 +174,13 @@ router.patch( (req, res) => dataUseRegisterController.updateDataUseRegister(req, res) ); +// @route POST /api/v2/data-use-registers/check +// @desc Check the submitted data uses for duplicates and returns links to Gatway entities (datasets, users) +// @access Public +router.post('/check', passport.authenticate('jwt'), logger.logRequestMiddleware({ logCategory, action: 'Check data uses' }), (req, res) => + dataUseRegisterController.checkDataUseRegister(req, res) +); + // @route POST /api/v2/data-use-registers/upload // @desc Accepts a bulk upload of data uses with built-in duplicate checking and rejection // @access Public diff --git a/src/resources/dataUseRegister/dataUseRegister.service.js b/src/resources/dataUseRegister/dataUseRegister.service.js index 55abfe3d..7aa31ff0 100644 --- a/src/resources/dataUseRegister/dataUseRegister.service.js +++ b/src/resources/dataUseRegister/dataUseRegister.service.js @@ -1,4 +1,5 @@ import dataUseRegisterUtil from './dataUseRegister.util'; +import { isEmpty } from 'lodash'; export default class DataUseRegisterService { constructor(dataUseRegisterRepository) { @@ -20,7 +21,7 @@ export default class DataUseRegisterService { updateDataUseRegister(id, body = {}) { // Protect for no id passed if (!id) return; - + return this.dataUseRegisterRepository.updateDataUseRegister({ _id: id }, body); } @@ -93,4 +94,58 @@ export default class DataUseRegisterService { return newDataUses; } + + /** + * Filter Existing Data Uses + * + * @desc Accepts multiple data uses, verifying each in turn is considered 'new' to the database, then outputs the list of data uses. + * A duplicate project id is automatically indicates a duplicate entry as the id must be unique. + * Alternatively, a combination of matching title, summary, organisation name and dataset titles indicates a duplicate entry. + * @param {Array} dataUses Array of data use objects to iterate through and check for existence in database + * @returns {Array} Filtered array of data uses linked entites and flat to indicates a duplicate entry + */ + async checkDataUseRegisters(dataUses = []) { + const dataUsesChecks = []; + + for (const obj of dataUses) { + const { linkedDatasets = [], namedDatasets = [] } = await dataUseRegisterUtil.getLinkedDatasets( + obj.datasetNames && + obj.datasetNames + .toString() + .split(',') + .map(el => { + if (!isEmpty(el)) return el.trim(); + }) + ); + + const { gatewayApplicants, nonGatewayApplicants } = await dataUseRegisterUtil.getLinkedApplicants( + obj.applicantNames && + obj.applicantNames + .toString() + .split(',') + .map(el => { + if (!isEmpty(el)) return el.trim(); + }) + ); + + const exists = await this.dataUseRegisterRepository.checkDataUseRegisterExists(obj); + + //Add new data use with linked entities + dataUsesChecks.push({ + projectIdText: obj.projectIdText, + projectTitle: obj.projectTitle, + laySummary: obj.laySummary, + organisationName: obj.organisationName, + datasetTitles: obj.datasetTitles, + latestApprovalDate: obj.latestApprovalDate, + linkedDatasets, + namedDatasets, + gatewayApplicants, + nonGatewayApplicants, + isDuplicated: exists, + }); + } + + return dataUsesChecks; + } } diff --git a/src/resources/dataUseRegister/dataUseRegister.util.js b/src/resources/dataUseRegister/dataUseRegister.util.js index 4f5673f5..257e3082 100644 --- a/src/resources/dataUseRegister/dataUseRegister.util.js +++ b/src/resources/dataUseRegister/dataUseRegister.util.js @@ -107,7 +107,7 @@ const buildDataUseRegisters = async (creatorUser, teamId, dataUses = []) => { ...(!isEmpty(datasetTitles) && { datasetTitles }), ...(!isEmpty(datasetIds) && { datasetIds }), ...(!isEmpty(datasetPids) && { datasetPids }), - ...(!isEmpty(gatewayApplicants) && { gatewayApplicants }), + ...(!isEmpty(gatewayApplicants) && { gatewayApplicants: gatewayApplicants.map(gatewayApplicant => gatewayApplicant._id) }), ...(!isEmpty(nonGatewayApplicants) && { nonGatewayApplicants }), ...(!isEmpty(fundersAndSponsors) && { fundersAndSponsors }), ...(!isEmpty(researchOutputs) && { researchOutputs }), @@ -118,7 +118,7 @@ const buildDataUseRegisters = async (creatorUser, teamId, dataUses = []) => { user: creatorUser._id, updatedon: Date.now(), lastActivity: Date.now(), - manualUpload: true + manualUpload: true, }) ); } @@ -181,7 +181,11 @@ const getLinkedApplicants = async (applicantNames = []) => { } } - const gatewayApplicants = isEmpty(unverifiedUserIds) ? [] : (await getUsersByIds(unverifiedUserIds)).map(el => el._id); + const gatewayApplicants = isEmpty(unverifiedUserIds) + ? [] + : (await getUsersByIds(unverifiedUserIds)).map(el => { + return { _id: el._id, id: el.id, firstname: el.firstname, lastname: el.lastname }; + }); return { gatewayApplicants, nonGatewayApplicants }; }; @@ -213,5 +217,5 @@ export default { buildDataUseRegisters, getLinkedDatasets, getLinkedApplicants, - buildRelatedObjects + buildRelatedObjects, }; diff --git a/src/resources/user/user.repository.js b/src/resources/user/user.repository.js index a67e041a..3f7d4aae 100644 --- a/src/resources/user/user.repository.js +++ b/src/resources/user/user.repository.js @@ -26,7 +26,7 @@ export async function getUserByUserId(id) { } export async function getUsersByIds(userIds) { - return await UserModel.find({ id: { $in: userIds } }, '_id').lean(); + return await UserModel.find({ id: { $in: userIds } }).lean(); } export async function getServiceAccountByClientCredentials(clientId, clientSecret) { From ad54fca569f75f373641ecd22d9bc818e6bb504d Mon Sep 17 00:00:00 2001 From: Robin Kavanagh Date: Wed, 6 Oct 2021 14:44:03 +0100 Subject: [PATCH 036/116] Adding missing repository functions --- .../dataUseRegister/dataUseRegister.repository.js | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/resources/dataUseRegister/dataUseRegister.repository.js b/src/resources/dataUseRegister/dataUseRegister.repository.js index b7a63f2f..622e211d 100644 --- a/src/resources/dataUseRegister/dataUseRegister.repository.js +++ b/src/resources/dataUseRegister/dataUseRegister.repository.js @@ -16,6 +16,10 @@ export default class DataUseRegisterRepository extends Repository { return this.find(query, options); } + getDataUseRegisterByApplicationId(applicationId) { + return this.dataUseRegister.findOne({ projectId: applicationId }, 'id').lean(); + } + updateDataUseRegister(id, body) { return this.update(id, body); } @@ -24,6 +28,11 @@ export default class DataUseRegisterRepository extends Repository { return this.dataUseRegister.insertMany(dataUseRegisters); } + async createDataUseRegister(dataUseRegister) { + await this.linkRelatedDataUseRegisters(dataUseRegister); + return await this.create(dataUseRegister); + } + async checkDataUseRegisterExists(dataUseRegister) { const { projectIdText, projectTitle, laySummary, organisationName, datasetTitles, latestApprovalDate } = dataUseRegister; const duplicatesFound = await this.dataUseRegister.countDocuments({ From c2997200ab145921332d02fc0fd02b811c7abd34 Mon Sep 17 00:00:00 2001 From: Pritesh Bhole Date: Wed, 6 Oct 2021 16:06:20 +0100 Subject: [PATCH 037/116] added script for publisher data update --- ...525344331-Ig_2354_replace_hubs_with_hub.js | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 migrations/1633525344331-Ig_2354_replace_hubs_with_hub.js diff --git a/migrations/1633525344331-Ig_2354_replace_hubs_with_hub.js b/migrations/1633525344331-Ig_2354_replace_hubs_with_hub.js new file mode 100644 index 00000000..dff23ae6 --- /dev/null +++ b/migrations/1633525344331-Ig_2354_replace_hubs_with_hub.js @@ -0,0 +1,40 @@ +import { firebaseml_v1beta2, toolresults_v1beta3 } from 'googleapis'; +import { PublisherModel } from '../src/resources/publisher/publisher.model'; + +// import { Data as ToolModel } from '../src/resources/tool/data.model'; + +/** + * Make any changes you need to make to the database here + */ +async function up () { + const publishers = await PublisherModel.find({ "publisherDetails.memberOf": "HUBS" }).lean(); + let tmp = []; + publishers.forEach((pub => { + const { _id } = pub; + const { name } = pub; + const { memberOf } = pub.publisherDetails + tmp.push({ + updateOne: { + filter: { _id }, + update: { + "publisherDetails.memberOf": replaceHubs(memberOf), + name : replaceHubs(name), + } + }, + }); + })); + await PublisherModel.bulkWrite(tmp); +} + +function replaceHubs(input) { + return input.replace('HUBS','HUB') +} + +/** + * Make any changes that UNDO the up function side effects here (if possible) + */ +async function down () { + // Write migration here +} + +module.exports = { up, down }; From d831cd52f3ce24bd68d604e8b39a51c36518888e Mon Sep 17 00:00:00 2001 From: Pritesh Bhole Date: Thu, 7 Oct 2021 12:52:35 +0100 Subject: [PATCH 038/116] added tools model --- ...525344331-Ig_2354_replace_hubs_with_hub.js | 43 ++++++++++++++----- 1 file changed, 33 insertions(+), 10 deletions(-) diff --git a/migrations/1633525344331-Ig_2354_replace_hubs_with_hub.js b/migrations/1633525344331-Ig_2354_replace_hubs_with_hub.js index dff23ae6..b9b5854c 100644 --- a/migrations/1633525344331-Ig_2354_replace_hubs_with_hub.js +++ b/migrations/1633525344331-Ig_2354_replace_hubs_with_hub.js @@ -1,31 +1,54 @@ -import { firebaseml_v1beta2, toolresults_v1beta3 } from 'googleapis'; import { PublisherModel } from '../src/resources/publisher/publisher.model'; - -// import { Data as ToolModel } from '../src/resources/tool/data.model'; +import { Data as ToolModel } from '../src/resources/tool/data.model'; /** * Make any changes you need to make to the database here */ async function up () { + await toolsUpdate(); + await publisherUpdate(); + +} + +async function toolsUpdate() { + const tools = await ToolModel.find({ type: "dataset", "datasetfields.publisher": { $regex: "HUBS" } }).lean(); + let tmpTool = []; + tools.forEach((tool => { + const { _id } = tool; + tmpTool.push({ + updateOne: { + filter: { _id }, + update: { + "datasetfields.publisher": replaceHubs(tool.datasetfields.publisher), + "datasetfields.metadataquality.publisher": replaceHubs(tool.datasetfields.metadataquality.publisher), + "datasetv2.summary.publisher.memberOf": replaceHubs(tool.datasetv2.summary.publisher.memberOf), + } + }, + }); + })); + await ToolModel.bulkWrite(tmpTool); +} + + +async function publisherUpdate() { const publishers = await PublisherModel.find({ "publisherDetails.memberOf": "HUBS" }).lean(); - let tmp = []; + let tmpPub = []; publishers.forEach((pub => { const { _id } = pub; - const { name } = pub; - const { memberOf } = pub.publisherDetails - tmp.push({ + tmpPub.push({ updateOne: { filter: { _id }, update: { - "publisherDetails.memberOf": replaceHubs(memberOf), - name : replaceHubs(name), + "publisherDetails.memberOf": replaceHubs(pub.publisherDetails.memberOf), + "name" : replaceHubs(pub.name), } }, }); })); - await PublisherModel.bulkWrite(tmp); + await PublisherModel.bulkWrite(tmpPub); } + function replaceHubs(input) { return input.replace('HUBS','HUB') } From fc275baf7b6f1ea0c6550cc1127defd4ebb2e210 Mon Sep 17 00:00:00 2001 From: Paul McCafferty <> Date: Fri, 8 Oct 2021 11:20:25 +0100 Subject: [PATCH 039/116] Updates to the backend for searching and filtering on data use --- src/resources/filters/dependency.js | 5 +- src/resources/filters/filters.controller.js | 24 +++--- src/resources/filters/filters.mapper.js | 78 ++++++++++++++++++++ src/resources/filters/filters.service.js | 28 +++++-- src/resources/search/search.repository.js | 81 ++++++++++++++++++--- src/resources/search/search.router.js | 13 ++++ 6 files changed, 200 insertions(+), 29 deletions(-) diff --git a/src/resources/filters/dependency.js b/src/resources/filters/dependency.js index 6244ad43..050bb6ab 100644 --- a/src/resources/filters/dependency.js +++ b/src/resources/filters/dependency.js @@ -6,6 +6,7 @@ import ProjectRepository from '../project/project.repository'; import PaperRepository from '../paper/paper.repository'; import CollectionsRepository from '../collections/v2/collection.repository'; import CourseRepository from '../course/v2/course.repository'; +import DataUseRegisterRepository from '../dataUseRegister/dataUseRegister.repository'; const datasetRepository = new DatasetRepository(); const toolRepository = new ToolRepository(); @@ -13,6 +14,7 @@ const projectRepository = new ProjectRepository(); const paperRepository = new PaperRepository(); const collectionsRepository = new CollectionsRepository(); const courseRepository = new CourseRepository(); +const dataUseRegisterRepository = new DataUseRegisterRepository(); export const filtersRepository = new FiltersRepository(); export const filtersService = new FiltersService( @@ -22,5 +24,6 @@ export const filtersService = new FiltersService( projectRepository, paperRepository, collectionsRepository, - courseRepository + courseRepository, + dataUseRegisterRepository ); diff --git a/src/resources/filters/filters.controller.js b/src/resources/filters/filters.controller.js index d6e86fe8..7f5cc2ba 100644 --- a/src/resources/filters/filters.controller.js +++ b/src/resources/filters/filters.controller.js @@ -2,42 +2,42 @@ import Controller from '../base/controller'; export default class FiltersController extends Controller { constructor(filtersService) { - super(filtersService); + super(filtersService); this.filtersService = filtersService; } async getFilters(req, res) { try { - // Extract id parameter from query string + // Extract id parameter from query string const { id } = req.params; - // If no id provided, it is a bad request + // If no id provided, it is a bad request if (!id) { return res.status(400).json({ success: false, message: 'You must provide a filters identifier', }); } - // Find the filters + // Find the filters let filters = await this.filtersService.getFilters(id, req.query); - // Return if no filters found + // Return if no filters found if (!filters) { return res.status(404).json({ success: false, message: 'A filter could not be found with the provided id', }); - } - // Return the filters + } + // Return the filters return res.status(200).json({ success: true, data: filters, }); } catch (err) { - // Return error response if something goes wrong - console.error(err.message); - return res.status(500).json({ + // Return error response if something goes wrong + console.error(err.message); + return res.status(500).json({ success: false, message: 'A server error occurred, please try again', }); - } - } + } + } } diff --git a/src/resources/filters/filters.mapper.js b/src/resources/filters/filters.mapper.js index 3f245d93..5bf4a3b3 100644 --- a/src/resources/filters/filters.mapper.js +++ b/src/resources/filters/filters.mapper.js @@ -854,3 +854,81 @@ export const courseFilters = [ beta: false, }, ]; + +export const dataUseRegisterFilters = [ + { + id: 1, + label: 'Data custodian', + key: 'dataCustodian', + alias: 'datausedatacustodian', + dataPath: 'datacustodian.datacustodian', + type: 'contains', + tooltip: null, + closed: true, + isSearchable: false, + selectedCount: 0, + filters: [], + highlighted: [], + beta: false, + }, + { + id: 2, + label: 'Lead applicant organisation', + key: 'organisationName', + alias: 'datauseorganisationname', + dataPath: 'organisationName', + type: 'contains', + tooltip: null, + closed: true, + isSearchable: false, + selectedCount: 0, + filters: [], + highlighted: [], + beta: false, + }, + { + id: 3, + label: 'Organisation sector', + key: 'organisationSector', + alias: 'datauserganisationsector', + dataPath: 'organisationSector', + type: 'contains', + tooltip: null, + closed: true, + isSearchable: false, + selectedCount: 0, + filters: [], + highlighted: [], + beta: false, + }, + { + id: 4, + label: 'Funders/Sponsor', + key: 'fundersSponsor', + alias: 'datausefunderssponsor', + dataPath: 'funderssponsor.funderssponsor', + type: 'contains', + tooltip: null, + closed: true, + isSearchable: false, + selectedCount: 0, + filters: [], + highlighted: [], + beta: false, + }, + { + id: 5, + label: 'Keywords', + key: 'keywords', + alias: 'datausekeywords', + dataPath: 'keywords', + type: 'contains', + tooltip: null, + closed: true, + isSearchable: false, + selectedCount: 0, + filters: [], + highlighted: [], + beta: false, + }, +]; diff --git a/src/resources/filters/filters.service.js b/src/resources/filters/filters.service.js index 51152334..874539ea 100644 --- a/src/resources/filters/filters.service.js +++ b/src/resources/filters/filters.service.js @@ -10,7 +10,8 @@ export default class FiltersService { projectRepository, paperRepository, collectionRepository, - courseRepository + courseRepository, + DataUseRegisterRepository ) { this.filtersRepository = filtersRepository; this.datasetRepository = datasetRepository; @@ -19,6 +20,7 @@ export default class FiltersService { this.paperRepository = paperRepository; this.collectionRepository = collectionRepository; this.courseRepository = courseRepository; + this.DataUseRegisterRepository = DataUseRegisterRepository; } async getFilters(id, query = {}) { @@ -146,6 +148,10 @@ export default class FiltersService { fields = `courseOptions.startDate, provider,location,courseOptions.studyMode,award,entries.level,domains,keywords,competencyFramework,nationalPriority`; entities = await this.courseRepository.getCourses({ ...query, fields }, { lean: true, dateFormat: 'DD MMM YYYY' }); break; + case 'dataUseRegister': + fields = `organisationName organisationSector keywords`; + entities = await this.DataUseRegisterRepository.getDataUseRegisters({ ...query, fields }, { lean: true }); + break; } // 3. Loop over each entity entities.forEach(entity => { @@ -176,12 +182,12 @@ export default class FiltersService { Object.keys(filters).forEach(filterKey => { // 9. Set filter values to title case (all except publisher) / upper case (publisher) and remove white space if (filterKey === 'publisher') { - filters[filterKey] = filters[filterKey].map(value => value.includes(">") - ? value.split(" > ")[1].toString().toUpperCase().trim() - : value.toString().toUpperCase().trim()); + filters[filterKey] = filters[filterKey].map(value => + value.includes('>') ? value.split(' > ')[1].toString().toUpperCase().trim() : value.toString().toUpperCase().trim() + ); } else { filters[filterKey] = filters[filterKey].map(value => helper.toTitleCase(value.toString().trim())); - }; + } // 10. Distinct filter values const distinctFilter = uniq(filters[filterKey]); // 11. Sort filter values and update final object @@ -304,6 +310,18 @@ export default class FiltersService { }; break; } + case 'dataUseRegister': { + // 2. Extract all properties used for filtering + let { keywords = [], organisationName = '', organisationSector = '' } = entity; + + // 3. Create flattened filter props object + filterValues = { + keywords, + organisationName, + organisationSector, + }; + break; + } } // 4. Return filter values return filterValues; diff --git a/src/resources/search/search.repository.js b/src/resources/search/search.repository.js index 54448236..e6cb0990 100644 --- a/src/resources/search/search.repository.js +++ b/src/resources/search/search.repository.js @@ -1,8 +1,17 @@ import { Data } from '../tool/data.model'; import { Course } from '../course/course.model'; import { Collections } from '../collections/collections.model'; +import { DataUseRegister } from '../dataUseRegister/dataUseRegister.model'; import { findNodeInTree } from '../filters/utils/filters.util'; -import { datasetFilters, toolFilters, projectFilters, paperFilters, collectionFilters, courseFilters } from '../filters/filters.mapper'; +import { + datasetFilters, + toolFilters, + projectFilters, + paperFilters, + collectionFilters, + courseFilters, + dataUseRegisterFilters, +} from '../filters/filters.mapper'; import _ from 'lodash'; import moment from 'moment'; import helperUtil from '../utilities/helper.util'; @@ -13,6 +22,8 @@ export async function getObjectResult(type, searchAll, searchQuery, startIndex, collection = Course; } else if (type === 'collection') { collection = Collections; + } else if (type === 'dataUseRegister') { + collection = DataUseRegister; } // ie copy deep object let newSearchQuery = _.cloneDeep(searchQuery); @@ -65,13 +76,12 @@ export async function getObjectResult(type, searchAll, searchQuery, startIndex, }, ]; } else if (type === 'collection') { - - const searchTerm = newSearchQuery && newSearchQuery['$and'] && newSearchQuery['$and'].find(exp => !_.isNil(exp['$text'])) || {}; + const searchTerm = (newSearchQuery && newSearchQuery['$and'] && newSearchQuery['$and'].find(exp => !_.isNil(exp['$text']))) || {}; - if(searchTerm) { + if (searchTerm) { newSearchQuery['$and'] = newSearchQuery['$and'].filter(exp => !exp['$text']); } - + queryObject = [ { $match: searchTerm }, { $lookup: { from: 'tools', localField: 'authors', foreignField: 'id', as: 'persons' } }, @@ -119,6 +129,47 @@ export async function getObjectResult(type, searchAll, searchQuery, startIndex, }, }, ]; + } else if (type === 'dataUseRegister') { + const searchTerm = (newSearchQuery && newSearchQuery['$and'] && newSearchQuery['$and'].find(exp => !_.isNil(exp['$text']))) || {}; + + if (searchTerm) { + newSearchQuery['$and'] = newSearchQuery['$and'].filter(exp => !exp['$text']); + } + + queryObject = [ + { $match: searchTerm }, + { $lookup: { from: 'tools', localField: 'authors', foreignField: 'id', as: 'persons' } }, + { + $addFields: { + persons: { + $map: { + input: '$persons', + as: 'row', + in: { + id: '$$row.id', + firstname: '$$row.firstname', + lastname: '$$row.lastname', + fullName: { $concat: ['$$row.firstname', ' ', '$$row.lastname'] }, + }, + }, + }, + }, + }, + { $match: newSearchQuery }, + { + $project: { + _id: 0, + id: 1, + projectTitle: 1, + organisationName: 1, + keywords: 1, + datasetTitles: 1, + activeflag: 1, + counter: 1, + type: 1, + }, + }, + ]; } else if (type === 'dataset') { queryObject = [ { $match: newSearchQuery }, @@ -374,9 +425,13 @@ export async function getObjectResult(type, searchAll, searchQuery, startIndex, } // Get paged results based on query params - const searchResults = await collection.aggregate(queryObject).skip(parseInt(startIndex)).limit(parseInt(maxResults)).catch(err => { - console.log(err); - }); + const searchResults = await collection + .aggregate(queryObject) + .skip(parseInt(startIndex)) + .limit(parseInt(maxResults)) + .catch(err => { + console.log(err); + }); // Return data return { data: searchResults }; } @@ -387,6 +442,8 @@ export function getObjectCount(type, searchAll, searchQuery) { collection = Course; } else if (type === 'collection') { collection = Collections; + } else if (type === 'dataUseRegister') { + collection = DataUseRegister; } let newSearchQuery = JSON.parse(JSON.stringify(searchQuery)); if (type !== 'collection') { @@ -451,12 +508,12 @@ export function getObjectCount(type, searchAll, searchQuery) { .sort({ score: { $meta: 'textScore' } }); } } else if (type === 'collection') { - const searchTerm = newSearchQuery && newSearchQuery['$and'] && newSearchQuery['$and'].find(exp => !_.isNil(exp['$text'])) || {}; + const searchTerm = (newSearchQuery && newSearchQuery['$and'] && newSearchQuery['$and'].find(exp => !_.isNil(exp['$text']))) || {}; - if(searchTerm) { + if (searchTerm) { newSearchQuery['$and'] = newSearchQuery['$and'].filter(exp => !exp['$text']); } - + if (searchAll) { q = collection.aggregate([ { $match: searchTerm }, @@ -723,6 +780,8 @@ export function getObjectFilters(searchQueryStart, req, type) { filterNode = findNodeInTree(collectionFilters, key); } else if (type === 'course') { filterNode = findNodeInTree(courseFilters, key); + } else if (type === 'dataUseRegister') { + filterNode = findNodeInTree(dataUseRegisterFilters, key); } if (filterNode) { diff --git a/src/resources/search/search.router.js b/src/resources/search/search.router.js index 60547789..d8268d25 100644 --- a/src/resources/search/search.router.js +++ b/src/resources/search/search.router.js @@ -56,6 +56,7 @@ router.get('/', async (req, res) => { People: 'person', Courses: 'course', Collections: 'collection', + Datauses: 'dataUseRegister', }; const entityType = typeMapper[`${tab}`]; @@ -125,6 +126,14 @@ router.get('/', async (req, res) => { req.query.maxResults || 40, req.query.collectionSort || '' ), + getObjectResult( + 'dataUseRegister', + searchAll, + getObjectFilters(searchQuery, req, 'dataUseRegister'), + req.query.dataUseRegisterIndex || 0, + req.query.maxResults || 40, + req.query.dataUseRegisterSort || '' + ), ]); } else { const sort = entityType === 'course' ? 'startdate' : req.query[`${entityType}Sort`] || ''; @@ -146,6 +155,7 @@ router.get('/', async (req, res) => { getObjectCount('person', searchAll, searchQuery), getObjectCount('course', searchAll, getObjectFilters(searchQuery, req, 'course')), getObjectCount('collection', searchAll, getObjectFilters(searchQuery, req, 'collection')), + getObjectCount('dataUseRegister', searchAll, getObjectFilters(searchQuery, req, ' dataUseRegister')), ]); const summary = { @@ -156,6 +166,7 @@ router.get('/', async (req, res) => { personCount: summaryCounts[4][0] !== undefined ? summaryCounts[4][0].count : 0, courseCount: summaryCounts[5][0] !== undefined ? summaryCounts[5][0].count : 0, collectionCount: summaryCounts[6][0] !== undefined ? summaryCounts[6][0].count : 0, + dataUseRegisterCount: summaryCounts[7][0] !== undefined ? summaryCounts[7][0].count : 0, }; let myEntitiesSummary = {}; @@ -184,6 +195,7 @@ router.get('/', async (req, res) => { recordSearchData.returned.person = summaryCounts[4][0] !== undefined ? summaryCounts[4][0].count : 0; recordSearchData.returned.course = summaryCounts[5][0] !== undefined ? summaryCounts[5][0].count : 0; recordSearchData.returned.collection = summaryCounts[6][0] !== undefined ? summaryCounts[6][0].count : 0; + recordSearchData.returned.datause = summaryCounts[7][0] !== undefined ? summaryCounts[7][0].count : 0; recordSearchData.datesearched = Date.now(); recordSearchData.save(err => {}); @@ -197,6 +209,7 @@ router.get('/', async (req, res) => { personResults: allResults[4].data, courseResults: allResults[5].data, collectionResults: allResults[6].data, + dataUseRegisterResults: allResults[7].data, summary: summary, myEntitiesSummary: myEntitiesSummary, }); From 4ae3db27e7b01f5cfcd982fdce1229d05332756c Mon Sep 17 00:00:00 2001 From: Paul McCafferty <> Date: Tue, 12 Oct 2021 09:01:24 +0100 Subject: [PATCH 040/116] DUR 12-10 fixes --- .../dataUseRegister/dataUseRegister.model.js | 13 ++- .../dataUseRegister.repository.js | 50 +++++++++- .../dataUseRegister/dataUseRegister.route.js | 8 +- src/resources/filters/filters.mapper.js | 10 +- src/resources/filters/filters.service.js | 10 +- src/resources/search/filter.route.js | 98 ++++++------------- src/resources/search/search.repository.js | 21 ++-- 7 files changed, 115 insertions(+), 95 deletions(-) diff --git a/src/resources/dataUseRegister/dataUseRegister.model.js b/src/resources/dataUseRegister/dataUseRegister.model.js index 3fdc006d..e6715f24 100644 --- a/src/resources/dataUseRegister/dataUseRegister.model.js +++ b/src/resources/dataUseRegister/dataUseRegister.model.js @@ -50,7 +50,7 @@ const dataUseRegisterSchema = new Schema( publicBenefitStatement: String, //Public Benefit Statement requestCategoryType: String, //Request Category Type technicalSummary: String, //Technical Summary - otherApprovalCommittees: [{type: String}], //Other Approval Committees + otherApprovalCommittees: [{ type: String }], //Other Approval Committees projectStartDate: Date, //Project Start Date projectEndDate: Date, //Project End Date latestApprovalDate: Date, //Latest Approval Date @@ -65,16 +65,23 @@ const dataUseRegisterSchema = new Schema( accessDate: Date, //Release/Access Date dataLocation: String, //TRE Or Any Other Specified Location privacyEnhancements: String, //How Has Data Been Processed To Enhance Privacy - researchOutputs: [{type: String}], //Link To Research Outputs + researchOutputs: [{ type: String }], //Link To Research Outputs }, { timestamps: true, toJSON: { virtuals: true }, toObject: { virtuals: true }, - strict: false + strict: false, } ); +dataUseRegisterSchema.virtual('gatewayApplicantsNames', { + ref: 'User', + foreignField: '_id', + localField: 'gatewayApplicants', + justOne: false, +}); + // Load entity class dataUseRegisterSchema.loadClass(DataUseRegisterClass); diff --git a/src/resources/dataUseRegister/dataUseRegister.repository.js b/src/resources/dataUseRegister/dataUseRegister.repository.js index bf20282a..af50bd49 100644 --- a/src/resources/dataUseRegister/dataUseRegister.repository.js +++ b/src/resources/dataUseRegister/dataUseRegister.repository.js @@ -1,5 +1,6 @@ import Repository from '../base/repository'; import { DataUseRegister } from './dataUseRegister.model'; +import { isNil } from 'lodash'; export default class DataUseRegisterRepository extends Repository { constructor() { @@ -11,9 +12,52 @@ export default class DataUseRegisterRepository extends Repository { return this.findOne(query, options); } - getDataUseRegisters(query) { - const options = { lean: true }; - return this.find(query, options); + async getDataUseRegisters(query, options = {}) { + if (options.aggregate) { + const searchTerm = (query && query['$and'] && query['$and'].find(exp => !isNil(exp['$text']))) || {}; + + if (searchTerm) { + query['$and'] = query['$and'].filter(exp => !exp['$text']); + } + + const aggregateQuery = [ + { $match: searchTerm }, + { + $lookup: { + from: 'publishers', + localField: 'publisher', + foreignField: '_id', + as: 'publisherDetails', + }, + }, + { + $addFields: { + publisherDetails: { + $map: { + input: '$publisherDetails', + as: 'row', + in: { + name: '$$row.name', + }, + }, + }, + }, + }, + { $match: { $and: [...query['$and']] } }, + ]; + + if (query.fields) { + aggregateQuery.push({ + $project: query.fields.split(',').reduce((obj, key) => { + return { ...obj, [key]: 1 }; + }, {}), + }); + } + return DataUseRegister.aggregate(aggregateQuery); + } else { + const options = { lean: true }; + return this.find(query, options); + } } getDataUseRegisterByApplicationId(applicationId) { diff --git a/src/resources/dataUseRegister/dataUseRegister.route.js b/src/resources/dataUseRegister/dataUseRegister.route.js index f30c5a7f..cb05b68c 100644 --- a/src/resources/dataUseRegister/dataUseRegister.route.js +++ b/src/resources/dataUseRegister/dataUseRegister.route.js @@ -69,11 +69,11 @@ const validateUploadRequest = (req, res, next) => { errors.push('You must provide the custodian team identifier to associate the data uses to'); } - if(!dataUses || isEmpty(dataUses)) { + if (!dataUses || isEmpty(dataUses)) { errors.push('You must provide data uses to upload'); } - if(!isEmpty(errors)){ + if (!isEmpty(errors)) { return res.status(400).json({ success: false, message: errors.join(', '), @@ -137,9 +137,9 @@ const authorizeUpload = async (req, res, next) => { message: 'You are not authorised to perform this action', }); } - + next(); -} +}; // @route GET /api/v2/data-use-registers/id // @desc Returns a dataUseRegister based on dataUseRegister ID provided diff --git a/src/resources/filters/filters.mapper.js b/src/resources/filters/filters.mapper.js index 5bf4a3b3..31b24f68 100644 --- a/src/resources/filters/filters.mapper.js +++ b/src/resources/filters/filters.mapper.js @@ -859,9 +859,9 @@ export const dataUseRegisterFilters = [ { id: 1, label: 'Data custodian', - key: 'dataCustodian', + key: 'publisher', alias: 'datausedatacustodian', - dataPath: 'datacustodian.datacustodian', + dataPath: 'publisher', type: 'contains', tooltip: null, closed: true, @@ -904,9 +904,9 @@ export const dataUseRegisterFilters = [ { id: 4, label: 'Funders/Sponsor', - key: 'fundersSponsor', - alias: 'datausefunderssponsor', - dataPath: 'funderssponsor.funderssponsor', + key: 'fundersAndSponsors', + alias: 'datausefundersandsponsors', + dataPath: 'fundersAndSponsors', type: 'contains', tooltip: null, closed: true, diff --git a/src/resources/filters/filters.service.js b/src/resources/filters/filters.service.js index 874539ea..101061e0 100644 --- a/src/resources/filters/filters.service.js +++ b/src/resources/filters/filters.service.js @@ -145,12 +145,12 @@ export default class FiltersService { entities = await this.collectionRepository.getCollections({ ...query, fields }, { aggregate: true }); break; case 'course': - fields = `courseOptions.startDate, provider,location,courseOptions.studyMode,award,entries.level,domains,keywords,competencyFramework,nationalPriority`; + fields = `courseOptions.startDate,provider,location,courseOptions.studyMode,award,entries.level,domains,keywords,competencyFramework,nationalPriority`; entities = await this.courseRepository.getCourses({ ...query, fields }, { lean: true, dateFormat: 'DD MMM YYYY' }); break; case 'dataUseRegister': - fields = `organisationName organisationSector keywords`; - entities = await this.DataUseRegisterRepository.getDataUseRegisters({ ...query, fields }, { lean: true }); + fields = `organisationName,organisationSector,keywords,publisherDetails.name,fundersAndSponsors`; + entities = await this.DataUseRegisterRepository.getDataUseRegisters({ ...query, fields }, { aggregate: true }); break; } // 3. Loop over each entity @@ -312,13 +312,15 @@ export default class FiltersService { } case 'dataUseRegister': { // 2. Extract all properties used for filtering - let { keywords = [], organisationName = '', organisationSector = '' } = entity; + let { keywords = [], organisationName = '', organisationSector = '', publisherDetails = '', fundersAndSponsors = [] } = entity; // 3. Create flattened filter props object filterValues = { keywords, organisationName, organisationSector, + publisher: publisherDetails[0].name, + fundersAndSponsors, }; break; } diff --git a/src/resources/search/filter.route.js b/src/resources/search/filter.route.js index b8c97f92..9e4cb71c 100644 --- a/src/resources/search/filter.route.js +++ b/src/resources/search/filter.route.js @@ -1,84 +1,48 @@ import express from 'express'; import { getObjectFilters, getFilter } from './search.repository'; import { filtersService } from '../filters/dependency'; -import { isEqual, lowerCase, isEmpty } from 'lodash'; +import { isEqual, isEmpty } from 'lodash'; const router = express.Router(); +const typeMapper = { + Datasets: 'dataset', + Tools: 'tool', + Projects: 'project', + Papers: 'paper', + People: 'person', + Courses: 'course', + Collections: 'collection', + Datauses: 'dataUseRegister', +}; + // @route GET api/v1/search/filter // @desc GET Get filters // @access Public router.get('/', async (req, res) => { let searchString = req.query.search || ''; //If blank then return all let tab = req.query.tab || ''; //If blank then return all - if (tab === '') { - let searchQuery = { $and: [{ activeflag: 'active' }] }; - if (searchString.length > 0) searchQuery['$and'].push({ $text: { $search: searchString } }); - - await Promise.all([ - getFilter(searchString, 'tool', 'tags.topic', true, getObjectFilters(searchQuery, req, 'tool')), - getFilter(searchString, 'tool', 'tags.features', true, getObjectFilters(searchQuery, req, 'tool')), - getFilter(searchString, 'tool', 'programmingLanguage.programmingLanguage', true, getObjectFilters(searchQuery, req, 'tool')), - getFilter(searchString, 'tool', 'categories.category', false, getObjectFilters(searchQuery, req, 'tool')), - - getFilter(searchString, 'project', 'tags.topics', true, getObjectFilters(searchQuery, req, 'project')), - getFilter(searchString, 'project', 'tags.features', true, getObjectFilters(searchQuery, req, 'project')), - getFilter(searchString, 'project', 'categories.category', false, getObjectFilters(searchQuery, req, 'project')), - - getFilter(searchString, 'paper', 'tags.topics', true, getObjectFilters(searchQuery, req, 'project')), - getFilter(searchString, 'paper', 'tags.features', true, getObjectFilters(searchQuery, req, 'project')), - ]).then(values => { - return res.json({ - success: true, - allFilters: { - toolTopicFilter: values[0][0], - toolFeatureFilter: values[1][0], - toolLanguageFilter: values[2][0], - toolCategoryFilter: values[3][0], - - projectTopicFilter: values[4][0], - projectFeatureFilter: values[5][0], - projectCategoryFilter: values[6][0], - - paperTopicFilter: values[7][0], - paperFeatureFilter: values[8][0], - }, - filterOptions: { - toolTopicsFilterOptions: values[0][1], - featuresFilterOptions: values[1][1], - programmingLanguageFilterOptions: values[2][1], - toolCategoriesFilterOptions: values[3][1], - - projectTopicsFilterOptions: values[4][1], - projectFeaturesFilterOptions: values[5][1], - projectCategoriesFilterOptions: values[6][1], - - paperTopicsFilterOptions: values[7][1], - paperFeaturesFilterOptions: values[8][1], - }, - }); - }); - } else { - const type = !isEmpty(tab) && typeof tab === 'string' ? lowerCase(tab.substring(0, tab.length - 1)) : ''; - let defaultQuery = { $and: [{ activeflag: 'active' }] }; - if (type === 'collection') { - defaultQuery['$and'].push({ publicflag: true }); - } else if (type === 'course') { - defaultQuery['$and'].push({ - $or: [{ 'courseOptions.startDate': { $gte: new Date(Date.now()) } }, { 'courseOptions.flexibleDates': true }], - }); - } - - if (searchString.length > 0) defaultQuery['$and'].push({ $text: { $search: searchString } }); - const filterQuery = getObjectFilters(defaultQuery, req, type); - const useCachedFilters = isEqual(defaultQuery, filterQuery) && searchString.length === 0; - - const filters = await filtersService.buildFilters(type, filterQuery, useCachedFilters); - return res.json({ - success: true, - filters, + + const type = !isEmpty(tab) && typeof tab === 'string' ? typeMapper[`${tab}`] : ''; + + let defaultQuery = { $and: [{ activeflag: 'active' }] }; + if (type === 'collection') { + defaultQuery['$and'].push({ publicflag: true }); + } else if (type === 'course') { + defaultQuery['$and'].push({ + $or: [{ 'courseOptions.startDate': { $gte: new Date(Date.now()) } }, { 'courseOptions.flexibleDates': true }], }); } + + if (searchString.length > 0) defaultQuery['$and'].push({ $text: { $search: searchString } }); + const filterQuery = getObjectFilters(defaultQuery, req, type); + const useCachedFilters = isEqual(defaultQuery, filterQuery) && searchString.length === 0; + + const filters = await filtersService.buildFilters(type, filterQuery, useCachedFilters); + return res.json({ + success: true, + filters, + }); }); // @route GET api/v1/search/filter/topic/:type diff --git a/src/resources/search/search.repository.js b/src/resources/search/search.repository.js index e6cb0990..fd972d07 100644 --- a/src/resources/search/search.repository.js +++ b/src/resources/search/search.repository.js @@ -137,25 +137,27 @@ export async function getObjectResult(type, searchAll, searchQuery, startIndex, } queryObject = [ - { $match: searchTerm }, - { $lookup: { from: 'tools', localField: 'authors', foreignField: 'id', as: 'persons' } }, + { + $lookup: { + from: 'publishers', + localField: 'publisher', + foreignField: '_id', + as: 'publisherDetails', + }, + }, { $addFields: { - persons: { + publisherDetails: { $map: { - input: '$persons', + input: '$publisherDetails', as: 'row', in: { - id: '$$row.id', - firstname: '$$row.firstname', - lastname: '$$row.lastname', - fullName: { $concat: ['$$row.firstname', ' ', '$$row.lastname'] }, + name: '$$row.name', }, }, }, }, }, - { $match: newSearchQuery }, { $project: { _id: 0, @@ -164,6 +166,7 @@ export async function getObjectResult(type, searchAll, searchQuery, startIndex, organisationName: 1, keywords: 1, datasetTitles: 1, + publisherDetails: 1, activeflag: 1, counter: 1, type: 1, From 5cbced834d0b6edc5c320e13f2860eed3775721e Mon Sep 17 00:00:00 2001 From: Callum Reekie Date: Tue, 12 Oct 2021 14:33:56 +0100 Subject: [PATCH 041/116] CR - split Swagger docs into constituent parts --- docs/index.docs.js | 80 +++ docs/resources/auth.docs.js | 122 ++++ docs/resources/course.docs.js | 102 ++++ docs/resources/datarequest.docs.js | 882 +++++++++++++++++++++++++++++ docs/resources/dataset.docs.js | 255 +++++++++ docs/resources/message.docs.js | 134 +++++ docs/resources/paper.docs.js | 435 ++++++++++++++ docs/resources/person.docs.js | 134 +++++ docs/resources/project.docs.js | 429 ++++++++++++++ docs/resources/publisher.docs.js | 717 +++++++++++++++++++++++ docs/resources/search.docs.js | 58 ++ docs/resources/stats.docs.js | 122 ++++ docs/resources/tool.docs.js | 422 ++++++++++++++ docs/resources/topic.docs.js | 159 ++++++ src/config/server.js | 4 +- 15 files changed, 4052 insertions(+), 3 deletions(-) create mode 100644 docs/index.docs.js create mode 100644 docs/resources/auth.docs.js create mode 100644 docs/resources/course.docs.js create mode 100644 docs/resources/datarequest.docs.js create mode 100644 docs/resources/dataset.docs.js create mode 100644 docs/resources/message.docs.js create mode 100644 docs/resources/paper.docs.js create mode 100644 docs/resources/person.docs.js create mode 100644 docs/resources/project.docs.js create mode 100644 docs/resources/publisher.docs.js create mode 100644 docs/resources/search.docs.js create mode 100644 docs/resources/stats.docs.js create mode 100644 docs/resources/tool.docs.js create mode 100644 docs/resources/topic.docs.js diff --git a/docs/index.docs.js b/docs/index.docs.js new file mode 100644 index 00000000..6dfa4f65 --- /dev/null +++ b/docs/index.docs.js @@ -0,0 +1,80 @@ +import auth from './resources/auth.docs'; +import datarequest from './resources/datarequest.docs'; +import publisher from './resources/publisher.docs'; +import person from './resources/person.docs'; +import search from './resources/search.docs'; +import stats from './resources/stats.docs'; +import message from './resources/message.docs'; +import topic from './resources/topic.docs'; +import dataset from './resources/dataset.docs'; +import project from './resources/project.docs'; +import paper from './resources/paper.docs'; +import tool from './resources/tool.docs'; +import course from './resources/course.docs'; + +module.exports = { + openapi: '3.0.1', + info: { + title: 'HDR UK API', + description: 'API for Tools and artefacts repository.', + version: '1.0.0', + }, + servers: [ + { + url: 'https://api.www.healthdatagateway.org/', + }, + { + url: 'http://localhost:3001/', + }, + { + url: 'https://api.{environment}.healthdatagateway.org:{port}/', + variables: { + environment: { + default: 'latest', + description: 'The Environment name.', + }, + port: { + enum: ['443'], + default: '443', + }, + }, + }, + ], + security: [ + { + oauth2: [], + }, + ], + paths: { + ...auth, + ...datarequest, + ...publisher, + ...person, + ...search, + ...stats, + ...message, + ...topic, + ...dataset, + ...project, + ...paper, + ...tool, + ...course, + }, + components: { + securitySchemes: { + oauth2: { + type: 'oauth2', + flows: { + clientCredentials: { + tokenUrl: 'https://api.www.healthdatagateway.org/oauth/token', + scopes: {}, + }, + }, + }, + cookieAuth: { + type: 'http', + scheme: 'jwt', + }, + }, + }, +}; diff --git a/docs/resources/auth.docs.js b/docs/resources/auth.docs.js new file mode 100644 index 00000000..b0ce0a67 --- /dev/null +++ b/docs/resources/auth.docs.js @@ -0,0 +1,122 @@ +module.exports = { + '/oauth/token': { + post: { + tags: ['Authorization'], + description: + 'OAuth2.0 token endpoint responsible for issuing short-lived json web tokens (JWT) for access to secure Gateway APIs. For client credentials grant flow, a valid client id and secret must be provided to identify your application and provide the expected permissions. This type of authentication is reserved for team based connectivity through client applications and is not provided for human user access. For more information, contact the HDR-UK team.', + requestBody: { + required: true, + content: { + 'application/json': { + schema: { + type: 'object', + properties: { + grant_type: { + type: 'string', + description: 'The OAuth2.0 grant type that will be used to provide authentication.', + }, + client_id: { + type: 'string', + description: + 'A unique identifer provided to your team by the HDR-UK team at the time of onboarding to the Gateway. Contact the HDR-UK team for issue of new credentials.', + }, + client_secret: { + type: 'string', + description: + 'A long (50 character) string provided by the HDR-UK team at the time of onboarding to the Gateway. Contact the HDR-UK team for issue of new credentials.', + }, + }, + required: ['grant_type', 'client_secret', 'client_id'], + }, + examples: { + 'Client Credentials Grant Flow': { + value: { + grant_type: 'client_credentials', + client_id: '2ca1f61a90e3547', + client_secret: '3f80fecbf781b6da280a8d17aa1a22066fb66daa415d8befc1', + }, + }, + }, + }, + }, + }, + responses: { + 200: { + description: 'Successful response containing json web token (JWT) that will authorize an HTTP request against secured resources.', + content: { + 'application/json': { + schema: { + type: 'object', + properties: { + access_token: { + type: 'string', + description: + 'The encoded json web token (JWT) that must be appended to the Authorization of subsequent API HTTP requests in order to access secured resources.', + }, + token_type: { + type: 'string', + description: 'The type of token issued, in this case, a json web token (JWT).', + }, + expires_in: { + type: 'integer', + description: 'The length of time in seconds before the issued JWT expires, defaulted to 900 seconds (15 minutes).', + }, + }, + }, + examples: { + 'Client Credentials Grant Flow': { + value: { + access_token: + 'Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJkYXRhIjp7Il9pZCI6IjYwMGJmYzk5YzhiZjcwMGYyYzdkNWMzNiIsInRpbWVTdGFtcCI2MTYxMjM4MzkwMzE5Nn0sImlhdCI6MTYxMjM4MzkwMywiZXhwIjoxNjEyMzg0ODAzfQ.-YvUBdjtJvdrRacz6E8-cYPQlum4TrEmiCFl8jO5a-M', + token_type: 'jwt', + expires_in: 900, + }, + }, + }, + }, + }, + }, + 400: { + description: 'Failure response caused by incomplete or invalid client credentials being passed to the endpoint.', + content: { + 'application/json': { + schema: { + type: 'object', + properties: { + success: { + type: 'boolean', + description: 'A field that indicates the API request failed.', + }, + message: { + type: 'string', + description: 'A message indicating that the request failed for a given reason.', + }, + }, + }, + examples: { + 'Invalid Client Credentials': { + value: { + success: false, + message: 'Invalid client credentials were provided for the authorisation attempt', + }, + }, + 'Incomplete Client Credentials': { + value: { + success: false, + message: 'Incomplete client credentials were provided for the authorisation attempt', + }, + }, + 'Invalid Grant Type': { + value: { + success: false, + message: 'An invalid grant type has been specified', + }, + }, + }, + }, + }, + }, + }, + }, + }, +}; diff --git a/docs/resources/course.docs.js b/docs/resources/course.docs.js new file mode 100644 index 00000000..a745b132 --- /dev/null +++ b/docs/resources/course.docs.js @@ -0,0 +1,102 @@ +module.exports = { + '/api/v2/courses': { + get: { + summary: 'Returns a list of courses', + parameters: [ + { + name: 'search', + in: 'query', + description: + 'Full text index search function which searches for partial matches in various fields including name and description. The response will contain a metascore indicating the relevancy of the match, by default results are sorted by the most relevant first unless a manual sort query parameter has been added.', + schema: { + type: 'string', + }, + example: 'Research', + }, + { + name: 'page', + in: 'query', + description: 'A specific page of results to retrieve', + schema: { + type: 'number', + }, + example: 1, + }, + { + name: 'limit', + in: 'query', + description: 'Maximum number of results returned per page', + schema: { + type: 'number', + }, + example: 10, + }, + { + name: 'sort', + in: 'query', + description: + 'Fields to apply sort operations to. Accepts multiple fields in ascending and descending. E.g. provider for ascending or -provider for descending. Multiple fields should be comma separated as shown in the example below.', + schema: { + type: 'string', + }, + example: 'provider,-counter', + }, + { + name: 'fields', + in: 'query', + description: + 'Limit the size of the response by requesting only certain fields. Note that some additional derived fields are always returned. Multiple fields should be comma separate as shown in the example below.', + schema: { + type: 'string', + }, + example: 'provider,counter,description', + }, + { + name: 'count', + in: 'query', + description: 'Returns the number of the number of entities matching the query parameters provided instead of the result payload', + schema: { + type: 'boolean', + }, + example: true, + }, + ], + description: + "Version 2.0 of the courses API introduces a large number of parameterised query string options to aid requests in collecting the data that is most relevant for a given use case. The query parameters defined below support a variety of comparison operators such as equals, contains, greater than, and less than. Using dot notation, any field can be queried, please see some examples below. Note - This response is limited to 100 records by default. Please use the 'page' query parameter to access records beyond the first 100. The 'limit' query parameter can therefore only be specified up to a maximum of 100.", + tags: ['Courses v2.0'], + responses: { + 200: { + description: 'Successful response containing a list of course objects matching query parameters', + }, + }, + }, + }, + '/api/v2/courses/{id}': { + summary: 'summary', + get: { + summary: 'Returns a course object', + description: 'Returns a course object by matching unique identifier in the default format that is stored as within the Gateway', + tags: ['Courses v2.0'], + parameters: [ + { + in: 'path', + name: 'id', + required: true, + description: 'The ID of the course', + schema: { + type: 'number', + example: 5540794872521069, + }, + }, + ], + responses: { + 200: { + description: 'Successful response containing a single course object', + }, + 404: { + description: 'A course could not be found by the provided course identifier', + }, + }, + }, + }, +}; diff --git a/docs/resources/datarequest.docs.js b/docs/resources/datarequest.docs.js new file mode 100644 index 00000000..f67f1d55 --- /dev/null +++ b/docs/resources/datarequest.docs.js @@ -0,0 +1,882 @@ +module.exports = { + '/api/v1/data-access-request/{id}': { + get: { + tags: ['Data Access Request'], + parameters: [ + { + in: 'path', + name: 'id', + required: true, + description: 'The unique identifier for a single data access request application.', + schema: { + type: 'string', + example: '5ee249426136805fbf094eef', + }, + }, + ], + description: 'Retrieve a single Data Access Request application using a supplied identifer', + responses: { + 200: { + description: 'Successful response containing a full data access request application.', + content: { + 'application/json': { + schema: { + type: 'object', + properties: { + status: { + type: 'string', + }, + data: { + type: 'object', + properties: { + id: { + type: 'string', + description: 'The unique identifier for the application.', + }, + aboutApplication: { + description: + "An object which holds data relating to the 'about application' section of the application form including details of whether the project is an NCS project or not.", + type: 'object', + properties: { + isNationalCoreStudies: { + type: 'boolean', + description: 'A flag to indicate if this application is in relation to a National Core Studies Project.', + }, + nationalCoreStudiesProjectId: { + type: 'integer', + description: + 'The unique identifier correlating to a Gateway Project entity indicating that this application is relating to a National Core Studies project.', + }, + projectName: { + type: 'string', + description: 'The project name that has been assigned to the application by the applicant(s).', + }, + }, + }, + authorIds: { + type: 'array', + items: { + type: 'integer', + }, + description: + "An array of values correlating to specific user's via their numeric identifiers. An author is also known as a contributor to an application and can view, edit or submit.", + }, + datasetIds: { + type: 'array', + items: { + type: 'string', + }, + description: + 'An array of values correlating to datasets selected for the application via their identifier, which is unique per version.', + }, + datasetTitles: { + type: 'array', + items: { + type: 'string', + }, + description: 'An array of strings correlating to the dataset titles that have been selected for the application.', + }, + applicationStatus: { + type: 'string', + enum: ['inProgress', 'submitted', 'inReview', 'approved', 'rejected', 'approved with conditions'], + description: 'The current status of the application.', + }, + jsonSchema: { + type: 'object', + description: + 'The object containing the json definition that renders the application form using the Winterfell library. This contains the details of questions, questions sets, question panels, headings and navigation items that appear.', + }, + questionAnswers: { + type: 'object', + description: + 'The object containing the answers provided on the application form. This consists of a series of key pairs, where the key is the unqiue question Id, and the value the is the answer provided to the question. In the case of a multi select on the form, the value may be an array.', + }, + publisher: { + type: 'string', + description: 'The name of the Custodian that holds the dataset and is processing the application.', + }, + publisherObj: { + type: 'object', + description: 'The object containing details regarding the Custodian/publisher relating to the application.', + }, + userId: { + type: 'integer', + description: + 'The unique identifier that correlates to the user account of the main applicant. This is always the user that started the application.', + }, + schemaId: { + type: 'string', + description: 'The unique identifier that correlates to the schema from which the application form was generated.', + }, + files: { + type: 'array', + items: { + type: 'object', + }, + description: + 'An array containing the links to files that have been uploaded to the application form and are held within the Gateway ecosystem.', + }, + amendmentIterations: { + type: 'array', + items: { + type: 'object', + }, + description: + 'An array containing an object with details for each iteration the application has passed through. An iteration is defined as an application which has been returned by the Custodian for correction, corrected by the applicant(s) and resubmitted. The object contains dates that the application was returned, and resubmitted as well as reference to any questions that were highlighted for amendment.', + }, + createdAt: { + type: 'string', + description: 'The date and time that the application was started.', + }, + updatedAt: { + type: 'string', + description: 'The date and time that the application was last updated by any party.', + }, + projectId: { + type: 'string', + description: + 'The unique identifier for the application converted to a more human friendly format in uppercase and hypenated.', + }, + dateSubmitted: { + type: 'string', + description: + 'The date and time that the application was originally submitted by the applicant(s) to the Custodian for review.', + }, + dateReviewStart: { + type: 'string', + description: + 'The date and time that the review process was commenced by a Custodian manager. The review starts from the moment the manager opens the application to triage it.', + }, + dateFinalStatus: { + type: 'string', + description: + 'The date and time that the Custodian triggered a status change to the application once a final decision was made. E.g. when application was approved. This date can be used in conjunction with the dateReviewStart date to calculate the length of time the Custodian took to make a decision through their review process.', + }, + datasets: { + type: 'array', + items: { + type: 'object', + }, + description: + 'An array containing the full metadata for each of the datasets that have been applied for through this application.', + }, + mainApplicant: { + type: 'object', + description: + 'An object containing the details of the main applicant of the application as referenced by the userId field.', + }, + authors: { + type: 'array', + items: { + type: 'object', + }, + description: + 'An array containing the details of the contributors of the application as referenced by the authorIds field.', + }, + readOnly: { + type: 'boolean', + description: + 'A value to indicate if the requesting party is able to modify the application in its present state. For example, this will be false for a Custodian, but true for applicants if the applicant(s) are working on resubmitting the application following a request for amendments.', + }, + unansweredAmendments: { + type: 'integer', + description: + 'The number of amendments that have been requested by the Custodian in the current amendment iteration.', + }, + answeredAmendments: { + type: 'integer', + description: + 'The number of requested amendments that the applicant(s) have fixed in the current amendment iteration.', + }, + userType: { + type: 'string', + enum: ['custodian', 'applicant'], + description: + 'The type of user that has requested the Data Access Request application based on their permissions. It is either an applicant or a Custodian user.', + }, + activeParty: { + type: 'string', + enum: ['custodian', 'applicant'], + description: + 'The party that is currently handling the application. This is the applicant during presubmission, then the Custodian following submission. The active party then fluctuates between parties during amendment iterations.', + }, + inReviewMode: { + type: 'boolean', + description: + 'A flag to indicate if the current user is a reviewer of the application. This value will be false unless the requesting user is an assigned reviewer to a currently active workflow step. When this value is true, the requesting user is able to recommend approval or rejection of the application.', + }, + reviewSections: { + type: 'array', + items: { + type: 'string', + }, + description: + "An array containing the sections of the application form that the current user is required to review if they are a reviewer of the current workflow step that the application is in. E.g. ['Safe People','Safe Data']", + }, + hasRecommended: { + type: 'boolean', + description: + 'A flag to indicate if the current user as a reviewer of the current workflow phase has submitted their recommendation for approval or rejection based on their review of the review sections assigned to them.', + }, + workflow: { + type: 'object', + description: + 'The full details of the workflow that has been assigned to the Data Access Request application. This includes information such as the review phases that the application will pass through and associated metadata.', + }, + }, + }, + }, + }, + examples: { + 'Approved Application': { + value: { + status: 'success', + data: { + aboutApplication: { + selectedDatasets: [ + { + _id: '5fc31a18d98e4f4cff7e9315', + datasetId: 'd5faf9c6-6c34-46d7-93c4-7706a5436ed9', + name: 'HDR UK Papers & Preprints', + description: + 'Publications that mention HDR-UK (or any variant thereof) in Acknowledgements or Author Affiliations\n\nThis will include:\n- Papers\n- COVID-19 Papers\n- COVID-19 Preprint', + abstract: + 'Publications that mention HDR-UK (or any variant thereof) in Acknowledgements or Author Affiliations', + publisher: 'OTHER > HEALTH DATA RESEARCH UK', + contactPoint: 'hdr.hdr@hdruk.ac.uk', + publisherObj: { + dataRequestModalContent: { + header: ' ', + body: '{omitted for brevity...}', + footer: '', + }, + active: true, + allowsMessaging: true, + workflowEnabled: true, + _id: '5f7b1a2bce9f65e6ed83e7da', + name: 'OTHER > HEALTH DATA RESEARCH UK', + imageURL: '', + team: { + active: true, + _id: '5f7b1a2bce9f65e6ed83e7da', + members: [ + { + roles: ['manager'], + memberid: '5f1a98861a821b4a53e44d15', + }, + { + roles: ['manager'], + memberid: '600bfc99c8bf700f2c7d5c36', + }, + ], + type: 'publisher', + __v: 3, + createdAt: '2020-11-30T21:12:40.855Z', + updatedAt: '2020-12-02T13:33:45.232Z', + }, + }, + }, + ], + isNationalCoreStudies: true, + nationalCoreStudiesProjectId: '4324836585275824', + projectName: 'Test application title', + completedDatasetSelection: true, + completedInviteCollaborators: true, + completedReadAdvice: true, + completedCommunicateAdvice: true, + completedApprovalsAdvice: true, + completedSubmitAdvice: true, + }, + authorIds: [], + datasetIds: ['d5faf9c6-6c34-46d7-93c4-7706a5436ed9'], + datasetTitles: [], + applicationStatus: 'approved', + jsonSchema: '{omitted for brevity...}', + questionAnswers: { + 'fullname-892140ec730145dc5a28b8fe139c2876': 'James Smith', + 'jobtitle-ff1d692a04b4bb9a2babe9093339136f': 'Consultant', + 'organisation-65c06905b8319ffa29919732a197d581': 'Consulting Inc.', + }, + publisher: 'OTHER > HEALTH DATA RESEARCH UK', + _id: '60142c5b4316a0e0fcd47c56', + version: 1, + userId: 9190228196797084, + schemaId: '5f55e87e780ba204b0a98eb8', + files: [], + amendmentIterations: [], + createdAt: '2021-01-29T15:40:11.943Z', + updatedAt: '2021-02-03T14:38:22.688Z', + __v: 0, + projectId: '6014-2C5B-4316-A0E0-FCD4-7C56', + dateSubmitted: '2021-01-29T16:30:27.351Z', + dateReviewStart: '2021-02-03T14:36:22.341Z', + dateFinalStatus: '2021-02-03T14:38:22.680Z', + datasets: ['{omitted for brevity...}'], + dataset: null, + mainApplicant: { + _id: '5f1a98861a821b4a53e44d15', + firstname: 'James', + lastname: 'Smith', + }, + authors: [], + id: '60142c5b4316a0e0fcd47c56', + readOnly: true, + unansweredAmendments: 0, + answeredAmendments: 0, + userType: 'custodian', + activeParty: 'custodian', + inReviewMode: false, + reviewSections: [], + hasRecommended: false, + workflow: {}, + }, + }, + }, + }, + }, + }, + }, + 401: { + description: 'Unauthorised attempt to access an application.', + content: { + 'application/json': { + schema: { + type: 'object', + properties: { + status: { + type: 'string', + }, + message: { + type: 'string', + }, + }, + }, + examples: { + Unauthorised: { + value: { + status: 'failure', + message: 'Unauthorised', + }, + }, + }, + }, + }, + }, + 404: { + description: 'Failed to find the application requested.', + content: { + 'application/json': { + schema: { + type: 'object', + properties: { + status: { + type: 'string', + }, + message: { + type: 'string', + }, + }, + }, + examples: { + 'Not Found': { + value: { + status: 'error', + message: 'Application not found.', + }, + }, + }, + }, + }, + }, + }, + }, + put: { + tags: ['Data Access Request'], + parameters: [ + { + in: 'path', + name: 'id', + required: true, + description: 'The unique identifier for a single Data Access Request application.', + schema: { + type: 'string', + example: '5ee249426136805fbf094eef', + }, + }, + ], + description: 'Update a single Data Access Request application.', + requestBody: { + content: { + 'application/json': { + schema: { + type: 'object', + properties: { + applicationStatus: { + type: 'string', + }, + applicationStatusDesc: { + type: 'string', + }, + }, + }, + examples: { + 'Update Application Status': { + value: { + applicationStatus: 'approved', + applicationStatusDesc: 'This application meets all the requirements.', + }, + }, + }, + }, + }, + }, + responses: { + 200: { + description: 'Successful response containing the full, updated data access request application.', + content: { + 'application/json': { + schema: { + type: 'object', + properties: { + status: { + type: 'string', + }, + data: { + type: 'object', + properties: { + id: { + type: 'string', + description: 'The unique identifier for the application.', + }, + aboutApplication: { + description: + "An object which holds data relating to the 'about application' section of the application form including details of whether the project is an NCS project or not.", + type: 'object', + properties: { + isNationalCoreStudies: { + type: 'boolean', + description: 'A flag to indicate if this application is in relation to a National Core Studies Project.', + }, + nationalCoreStudiesProjectId: { + type: 'integer', + description: + 'The unique identifier correlating to a Gateway Project entity indicating that this application is relating to a National Core Studies project.', + }, + projectName: { + type: 'string', + description: 'The project name that has been assigned to the application by the applicant(s).', + }, + }, + }, + authorIds: { + type: 'array', + items: { + type: 'integer', + }, + description: + "An array of values correlating to specific user's via their numeric identifiers. An author is also known as a contributor to an application and can view, edit or submit.", + }, + datasetIds: { + type: 'array', + items: { + type: 'string', + }, + description: + 'An array of values correlating to datasets selected for the application via their identifier, which is unique per version.', + }, + datasetTitles: { + type: 'array', + items: { + type: 'string', + }, + description: 'An array of strings correlating to the dataset titles that have been selected for the application.', + }, + applicationStatus: { + type: 'string', + enum: ['inProgress', 'submitted', 'inReview', 'approved', 'rejected', 'approved with conditions'], + description: 'The current status of the application.', + }, + jsonSchema: { + type: 'object', + description: + 'The object containing the json definition that renders the application form using the Winterfell library. This contains the details of questions, questions sets, question panels, headings and navigation items that appear.', + }, + questionAnswers: { + type: 'object', + description: + 'The object containing the answers provided on the application form. This consists of a series of key pairs, where the key is the unqiue question Id, and the value the is the answer provided to the question. In the case of a multi select on the form, the value may be an array.', + }, + publisher: { + type: 'string', + description: 'The name of the Custodian that holds the dataset and is processing the application.', + }, + publisherObj: { + type: 'object', + description: 'The object containing details regarding the Custodian/publisher relating to the application.', + }, + userId: { + type: 'integer', + description: + 'The unique identifier that correlates to the user account of the main applicant. This is always the user that started the application.', + }, + schemaId: { + type: 'string', + description: 'The unique identifier that correlates to the schema from which the application form was generated.', + }, + files: { + type: 'array', + items: { + type: 'object', + }, + description: + 'An array containing the links to files that have been uploaded to the application form and are held within the Gateway ecosystem.', + }, + amendmentIterations: { + type: 'array', + items: { + type: 'object', + }, + description: + 'An array containing an object with details for each iteration the application has passed through. An iteration is defined as an application which has been returned by the Custodian for correction, corrected by the applicant(s) and resubmitted. The object contains dates that the application was returned, and resubmitted as well as reference to any questions that were highlighted for amendment.', + }, + createdAt: { + type: 'string', + description: 'The date and time that the application was started.', + }, + updatedAt: { + type: 'string', + description: 'The date and time that the application was last updated by any party.', + }, + projectId: { + type: 'string', + description: + 'The unique identifier for the application converted to a more human friendly format in uppercase and hypenated.', + }, + dateSubmitted: { + type: 'string', + description: + 'The date and time that the application was originally submitted by the applicant(s) to the Custodian for review.', + }, + dateReviewStart: { + type: 'string', + description: + 'The date and time that the review process was commenced by a Custodian manager. The review starts from the moment the manager opens the application to triage it.', + }, + dateFinalStatus: { + type: 'string', + description: + 'The date and time that the Custodian triggered a status change to the application once a final decision was made. E.g. when application was approved. This date can be used in conjunction with the dateReviewStart date to calculate the length of time the Custodian took to make a decision through their review process.', + }, + datasets: { + type: 'array', + items: { + type: 'object', + }, + description: + 'An array containing the full metadata for each of the datasets that have been applied for through this application.', + }, + mainApplicant: { + type: 'object', + description: + 'An object containing the details of the main applicant of the application as referenced by the userId field.', + }, + authors: { + type: 'array', + items: { + type: 'object', + }, + description: + 'An array containing the details of the contributors of the application as referenced by the authorIds field.', + }, + readOnly: { + type: 'boolean', + description: + 'A value to indicate if the requesting party is able to modify the application in its present state. For example, this will be false for a Custodian, but true for applicants if the applicant(s) are working on resubmitting the application following a request for amendments.', + }, + unansweredAmendments: { + type: 'integer', + description: + 'The number of amendments that have been requested by the Custodian in the current amendment iteration.', + }, + answeredAmendments: { + type: 'integer', + description: + 'The number of requested amendments that the applicant(s) have fixed in the current amendment iteration.', + }, + userType: { + type: 'string', + enum: ['custodian', 'applicant'], + description: + 'The type of user that has requested the Data Access Request application based on their permissions. It is either an applicant or a Custodian user.', + }, + activeParty: { + type: 'string', + enum: ['custodian', 'applicant'], + description: + 'The party that is currently handling the application. This is the applicant during presubmission, then the Custodian following submission. The active party then fluctuates between parties during amendment iterations.', + }, + inReviewMode: { + type: 'boolean', + description: + 'A flag to indicate if the current user is a reviewer of the application. This value will be false unless the requesting user is an assigned reviewer to a currently active workflow step. When this value is true, the requesting user is able to recommend approval or rejection of the application.', + }, + reviewSections: { + type: 'array', + items: { + type: 'string', + }, + description: + "An array containing the sections of the application form that the current user is required to review if they are a reviewer of the current workflow step that the application is in. E.g. ['Safe People','Safe Data']", + }, + hasRecommended: { + type: 'boolean', + description: + 'A flag to indicate if the current user as a reviewer of the current workflow phase has submitted their recommendation for approval or rejection based on their review of the review sections assigned to them.', + }, + workflow: { + type: 'object', + description: + 'The full details of the workflow that has been assigned to the Data Access Request application. This includes information such as the review phases that the application will pass through and associated metadata.', + }, + }, + }, + }, + }, + examples: { + 'Approved Application': { + value: { + status: 'success', + data: { + aboutApplication: { + selectedDatasets: [ + { + _id: '5fc31a18d98e4f4cff7e9315', + datasetId: 'd5faf9c6-6c34-46d7-93c4-7706a5436ed9', + name: 'HDR UK Papers & Preprints', + description: + 'Publications that mention HDR-UK (or any variant thereof) in Acknowledgements or Author Affiliations\n\nThis will include:\n- Papers\n- COVID-19 Papers\n- COVID-19 Preprint', + abstract: + 'Publications that mention HDR-UK (or any variant thereof) in Acknowledgements or Author Affiliations', + publisher: 'OTHER > HEALTH DATA RESEARCH UK', + contactPoint: 'hdr.hdr@hdruk.ac.uk', + publisherObj: { + dataRequestModalContent: { + header: ' ', + body: '{omitted for brevity...}', + footer: '', + }, + active: true, + allowsMessaging: true, + workflowEnabled: true, + _id: '5f7b1a2bce9f65e6ed83e7da', + name: 'OTHER > HEALTH DATA RESEARCH UK', + imageURL: '', + team: { + active: true, + _id: '5f7b1a2bce9f65e6ed83e7da', + members: [ + { + roles: ['manager'], + memberid: '5f1a98861a821b4a53e44d15', + }, + { + roles: ['manager'], + memberid: '600bfc99c8bf700f2c7d5c36', + }, + ], + type: 'publisher', + __v: 3, + createdAt: '2020-11-30T21:12:40.855Z', + updatedAt: '2020-12-02T13:33:45.232Z', + }, + }, + }, + ], + isNationalCoreStudies: true, + nationalCoreStudiesProjectId: '4324836585275824', + projectName: 'Test application title', + completedDatasetSelection: true, + completedInviteCollaborators: true, + completedReadAdvice: true, + completedCommunicateAdvice: true, + completedApprovalsAdvice: true, + completedSubmitAdvice: true, + }, + authorIds: [], + datasetIds: ['d5faf9c6-6c34-46d7-93c4-7706a5436ed9'], + datasetTitles: [], + applicationStatus: 'approved', + jsonSchema: '{omitted for brevity...}', + questionAnswers: { + 'fullname-892140ec730145dc5a28b8fe139c2876': 'James Smith', + 'jobtitle-ff1d692a04b4bb9a2babe9093339136f': 'Consultant', + 'organisation-65c06905b8319ffa29919732a197d581': 'Consulting Inc.', + }, + publisher: 'OTHER > HEALTH DATA RESEARCH UK', + _id: '60142c5b4316a0e0fcd47c56', + version: 1, + userId: 9190228196797084, + schemaId: '5f55e87e780ba204b0a98eb8', + files: [], + amendmentIterations: [], + createdAt: '2021-01-29T15:40:11.943Z', + updatedAt: '2021-02-03T14:38:22.688Z', + __v: 0, + projectId: '6014-2C5B-4316-A0E0-FCD4-7C56', + dateSubmitted: '2021-01-29T16:30:27.351Z', + dateReviewStart: '2021-02-03T14:36:22.341Z', + dateFinalStatus: '2021-02-03T14:38:22.680Z', + datasets: ['{omitted for brevity...}'], + dataset: null, + mainApplicant: { + _id: '5f1a98861a821b4a53e44d15', + firstname: 'James', + lastname: 'Smith', + }, + authors: [], + id: '60142c5b4316a0e0fcd47c56', + readOnly: true, + unansweredAmendments: 0, + answeredAmendments: 0, + userType: 'custodian', + activeParty: 'custodian', + inReviewMode: false, + reviewSections: [], + hasRecommended: false, + workflow: {}, + }, + }, + }, + }, + }, + }, + }, + 401: { + description: 'Unauthorised attempt to update an application.', + content: { + 'application/json': { + schema: { + type: 'object', + properties: { + status: { + type: 'string', + }, + message: { + type: 'string', + }, + }, + }, + examples: { + Unauthorised: { + value: { + status: 'error', + message: 'Unauthorised to perform this update.', + }, + }, + }, + }, + }, + }, + 404: { + description: 'Failed to find the application requested.', + content: { + 'application/json': { + schema: { + type: 'object', + properties: { + status: { + type: 'string', + }, + message: { + type: 'string', + }, + }, + }, + examples: { + 'Not Found': { + value: { + status: 'error', + message: 'Application not found.', + }, + }, + }, + }, + }, + }, + }, + }, + patch: { + summary: 'Update a users question answers for access request.', + security: [ + { + cookieAuth: [], + }, + ], + tags: ['Data Access Request'], + parameters: [ + { + in: 'path', + name: 'id', + required: true, + description: 'The ID of the datset', + schema: { + type: 'string', + example: '5ee249426136805fbf094eef', + }, + }, + ], + requestBody: { + content: { + 'application/json': { + schema: { + type: 'object', + properties: { + questionAnswers: { + type: 'object', + }, + }, + }, + examples: { + 0: { + value: '{\n "firstName": "Roger"\n}', + }, + }, + }, + }, + }, + responses: { + 200: { + description: 'OK', + }, + }, + }, + }, + '/api/v1/data-access-request/{datasetID}': { + get: { + summary: 'Returns access request template.', + security: [ + { + cookieAuth: [], + }, + ], + tags: ['Data Access Request'], + parameters: [ + { + in: 'path', + name: 'datasetID', + required: true, + description: 'The ID of the datset', + schema: { + type: 'string', + example: '6efbc62f-6ebb-4f18-959b-1ec6fd0cc6fb', + }, + }, + ], + responses: { + 200: { + description: 'OK', + }, + }, + }, + }, +}; diff --git a/docs/resources/dataset.docs.js b/docs/resources/dataset.docs.js new file mode 100644 index 00000000..a2efbdee --- /dev/null +++ b/docs/resources/dataset.docs.js @@ -0,0 +1,255 @@ +module.exports = { + '/api/v1/datasets/{datasetID}': { + get: { + summary: 'Returns Dataset object.', + tags: ['Datasets'], + parameters: [ + { + in: 'path', + name: 'datasetID', + required: true, + description: 'The ID of the datset', + schema: { + type: 'string', + example: '756daeaa-6e47-4269-9df5-477c01cdd271', + }, + }, + ], + responses: { + 200: { + description: 'OK', + }, + }, + }, + }, + '/api/v1/datasets': { + get: { + summary: 'Returns List of Dataset objects.', + tags: ['Datasets'], + parameters: [ + { + in: 'query', + name: 'limit', + required: false, + description: 'Limit the number of results', + schema: { + type: 'integer', + example: 3, + }, + }, + { + in: 'query', + name: 'offset', + required: false, + description: 'Index to offset the search results', + schema: { + type: 'integer', + example: 1, + }, + }, + { + in: 'query', + name: 'q', + required: false, + description: 'Filter using search query', + schema: { + type: 'string', + example: 'epilepsy', + }, + }, + ], + responses: { + 200: { + description: 'OK', + }, + }, + }, + }, + '/api/v2/datasets': { + get: { + summary: 'Returns a list of dataset objects', + tags: ['Datasets v2.0'], + description: + "Version 2.0 of the datasets API introduces a large number of parameterised query string options to aid requests in collecting the data that is most relevant for a given use case. The query parameters defined below support a variety of comparison operators such as equals, contains, greater than, and less than. Using dot notation, any field can be queried, please see some examples below. Note - This response is limited to 100 records by default. Please use the 'page' query parameter to access records beyond the first 100. The 'limit' query parameter can therefore only be specified up to a maximum of 100.", + parameters: [ + { + name: 'search', + in: 'query', + description: + 'Full text index search function which searches for partial matches in various dataset fields including name, description and abstract. The response will contain a metascore indicating the relevancy of the match, by default results are sorted by the most relevant first unless a manual sort query parameter has been added.', + schema: { + type: 'string', + }, + example: 'COVID-19', + }, + { + name: 'page', + in: 'query', + description: 'A specific page of results to retrieve', + schema: { + type: 'number', + }, + example: 1, + }, + { + name: 'limit', + in: 'query', + description: 'Maximum number of results returned per page', + schema: { + type: 'number', + }, + example: 10, + }, + { + name: 'sort', + in: 'query', + description: + 'Fields to apply sort operations to. Accepts multiple fields in ascending and descending. E.g. name for ascending or -name for descending. Multiple fields should be comma separated as shown in the example below.', + schema: { + type: 'string', + }, + example: 'datasetfields.publisher,name,-counter', + }, + { + name: 'fields', + in: 'query', + description: + 'Limit the size of the response by requesting only certain fields. Note that some additional derived fields are always returned. Multiple fields should be comma separate as shown in the example below.', + schema: { + type: 'string', + }, + example: 'name,counter,datasetid', + }, + { + name: 'count', + in: 'query', + description: 'Returns the number of the number of entities matching the query parameters provided instead of the result payload', + schema: { + type: 'boolean', + }, + example: true, + }, + { + name: 'datasetid', + in: 'query', + description: 'Filter by the unique identifier for a single version of a dataset', + schema: { + type: 'string', + }, + example: '0cfe60cd-038d-4c03-9a95-894c52135922', + }, + { + name: 'pid', + in: 'query', + description: 'Filter by the identifier for a dataset that persists across versions', + schema: { + type: 'string', + }, + example: '621dd611-adcf-4434-b538-eecdbe5f72cf', + }, + { + name: 'name', + in: 'query', + description: 'Filter by dataset name', + schema: { + type: 'string', + }, + example: 'ARIA Dataset', + }, + { + name: 'activeflag', + in: 'query', + description: 'Filter by the status of a single dataset version', + schema: { + type: 'string', + enum: ['active', 'archive'], + }, + example: 'active', + }, + { + name: 'datasetfields.publisher', + in: 'query', + description: 'Filter by the name of the Custodian holding the dataset', + schema: { + type: 'string', + }, + example: 'ALLIANCE > BARTS HEALTH NHS TRUST', + }, + { + name: 'metadataquality.completeness_percent[gte]', + in: 'query', + description: + 'Filter by the metadata quality completeness percentage using an operator [gte] for greater than or equal to, [gt] for greater than, [lte] for less than or equal to, [lt] for less than, and [eq] for equal to.', + schema: { + type: 'number', + }, + example: 90.5, + }, + { + name: 'metadataquality.weighted_completeness_percent[gte]', + in: 'query', + description: + 'Filter by the metadata quality weighted completeness percentage using an operator [gte] for greater than or equal to, [gt] for greater than, [lte] for less than or equal to, [lt] for less than, and [eq] for equal to.', + schema: { + type: 'number', + }, + example: 71.2, + }, + { + name: 'metadataquality.weighted_quality_score[gte]', + in: 'query', + description: + 'Filter by the metadata quality score using an operator [gte] for greater than or equal to, [gt] for greater than, [lte] for less than or equal to, [lt] for less than, and [eq] for equal to.', + schema: { + type: 'number', + }, + example: 35.3, + }, + ], + responses: { + 200: { + description: 'Successful response containing a list of datasets matching query parameters', + }, + }, + }, + }, + '/api/v2/datasets/{datasetid}': { + get: { + summary: 'Returns a dataset object.', + tags: ['Datasets v2.0'], + parameters: [ + { + in: 'path', + name: 'datasetid', + required: true, + description: 'The unqiue identifier for a specific version of a dataset', + schema: { + type: 'string', + example: 'af20ebb2-018a-4557-8ced-0bec75dba150', + }, + }, + { + in: 'query', + name: 'raw', + required: false, + description: + 'A flag which determines if the response triggered is the raw structure in which the data is stored rather than the dataset v2.0 standard', + schema: { + type: 'boolean', + example: false, + }, + }, + ], + description: + 'Version 2.0 of the datasets API introduces the agreed dataset v2.0 schema as defined at the following link - https://github.com/HDRUK/schemata/edit/master/schema/dataset/2.0.0/dataset.schema.json', + responses: { + 200: { + description: 'Successful response containing a single dataset object', + }, + 404: { + description: 'A dataset could not be found by the provided dataset identifier', + }, + }, + }, + }, +}; diff --git a/docs/resources/message.docs.js b/docs/resources/message.docs.js new file mode 100644 index 00000000..d0f63212 --- /dev/null +++ b/docs/resources/message.docs.js @@ -0,0 +1,134 @@ +module.exports = { + '/api/v1/messages/{id}': { + delete: { + summary: 'Delete a Message', + security: [ + { + cookieAuth: [], + }, + ], + tags: ['Messages'], + parameters: [ + { + in: 'path', + name: 'id', + required: true, + description: 'The ID of the Message', + schema: { + type: 'string', + example: '5ee249426136805fbf094eef', + }, + }, + ], + responses: { + 204: { + description: 'Ok', + }, + }, + }, + put: { + summary: 'Update a single Message', + security: [ + { + cookieAuth: [], + }, + ], + tags: ['Messages'], + parameters: [ + { + in: 'path', + name: 'id', + required: true, + description: 'The ID of the Message', + schema: { + type: 'string', + example: '5ee249426136805fbf094eef', + }, + }, + ], + requestBody: { + content: { + 'application/json': { + schema: { + type: 'object', + properties: { + isRead: { + type: 'boolean', + }, + }, + }, + examples: { + 'Update message to read': { + value: '{\n "isRead": true\n}', + }, + }, + }, + }, + }, + responses: { + 204: { + description: 'OK', + }, + }, + }, + }, + '/api/v1/messages/unread/count': { + get: { + summary: 'Returns the number of unread messages for the authenticated user', + security: [ + { + cookieAuth: [], + }, + ], + tags: ['Messages'], + responses: { + 200: { + description: 'OK', + }, + }, + }, + }, + '/api/v1/messages': { + post: { + summary: 'Returns a new Message object and creates an associated parent Topic if a Topic is not specified in request body', + security: [ + { + cookieAuth: [], + }, + ], + tags: ['Messages'], + requestBody: { + required: true, + content: { + 'application/json': { + schema: { + type: 'object', + properties: { + isRead: { + type: 'boolean', + }, + messageDescription: { + type: 'string', + }, + messageType: { + type: 'string', + }, + }, + required: ['isRead', 'messageDescription', 'messageType'], + }, + examples: { + 'Create new message': { + value: '{\n "isRead": false,\n "messageDescription": "this is an example",\n "messageType": "message"\n}', + }, + }, + }, + }, + }, + responses: { + 201: { + description: 'OK', + }, + }, + }, + }, +}; diff --git a/docs/resources/paper.docs.js b/docs/resources/paper.docs.js new file mode 100644 index 00000000..8134739d --- /dev/null +++ b/docs/resources/paper.docs.js @@ -0,0 +1,435 @@ +module.exports = { + '/api/v1/papers': { + post: { + summary: 'Returns a Paper object with ID.', + security: [ + { + cookieAuth: [], + }, + ], + tags: ['Papers'], + requestBody: { + content: { + 'application/json': { + schema: { + type: 'object', + required: ['name'], + properties: { + type: { + type: 'string', + }, + name: { + type: 'string', + }, + link: { + type: 'string', + }, + description: { + type: 'string', + }, + categories: { + type: 'object', + properties: { + category: { + type: 'string', + }, + programmingLanguage: { + type: 'array', + items: { + type: 'string', + }, + }, + programmingLanguageVersion: { + type: 'string', + }, + }, + }, + licence: { + type: 'string', + }, + authors: { + type: 'array', + items: { + type: 'number', + }, + }, + tags: { + type: 'object', + properties: { + features: { + type: 'array', + items: { + type: 'string', + }, + }, + topics: { + type: 'array', + items: { + type: 'string', + }, + }, + }, + }, + }, + example: { + type: 'paper', + name: 'Epilepsy data research', + link: 'http://epilepsy.org', + description: 'Epilespy data research description', + categories: { + category: 'API', + programmingLanguage: ['Javascript'], + programmingLanguageVersion: '0.0.0', + }, + licence: 'MIT licence', + authors: [4495285946631793], + tags: { + features: ['Arbitrage'], + topics: ['Epilepsy'], + }, + }, + }, + }, + }, + }, + responses: { + 200: { + description: 'OK', + }, + }, + }, + get: { + summary: 'Return List of Paper objects.', + tags: ['Papers'], + parameters: [ + { + in: 'query', + name: 'limit', + required: false, + description: 'Limit the number of results', + schema: { + type: 'integer', + example: 3, + }, + }, + { + in: 'query', + name: 'offset', + required: false, + description: 'Index to offset the search results', + schema: { + type: 'integer', + example: 1, + }, + }, + { + in: 'query', + name: 'q', + required: false, + description: 'Filter using search query', + schema: { + type: 'string', + example: 'epilepsy', + }, + }, + ], + responses: { + 200: { + description: 'OK', + }, + }, + }, + }, + '/api/v1/papers/{id}': { + get: { + summary: 'Returns Paper object.', + tags: ['Papers'], + parameters: [ + { + name: 'id', + in: 'path', + required: true, + description: 'The ID of the user', + schema: { + type: 'integer', + format: 'int64', + minimum: 1, + example: 8370396016757367, + }, + }, + ], + responses: { + 200: { + description: 'OK', + }, + }, + }, + patch: { + summary: 'Change status of the Paper object.', + security: [ + { + cookieAuth: [], + }, + ], + tags: ['Papers'], + parameters: [ + { + in: 'path', + name: 'id', + required: true, + schema: { + type: 'integer', + example: 7485531672584456, + }, + }, + ], + requestBody: { + content: { + 'application/json': { + schema: { + type: 'object', + required: ['name'], + properties: { + id: { + type: 'number', + }, + activeflag: { + type: 'string', + }, + }, + example: { + activeflag: 'active', + }, + }, + }, + }, + }, + responses: { + 200: { + description: 'OK', + }, + }, + }, + put: { + summary: 'Returns edited Paper object.', + security: [ + { + cookieAuth: [], + }, + ], + tags: ['Papers'], + parameters: [ + { + in: 'path', + name: 'id', + required: true, + description: 'The ID of the paper', + schema: { + type: 'integer', + format: 'int64', + example: 7485531672584456, + }, + }, + ], + requestBody: { + content: { + 'application/json': { + schema: { + type: 'object', + required: ['name'], + properties: { + id: { + type: 'number', + }, + type: { + type: 'string', + }, + name: { + type: 'string', + }, + link: { + type: 'string', + }, + description: { + type: 'string', + }, + categories: { + type: 'object', + properties: { + category: { + type: 'string', + }, + programmingLanguage: { + type: 'array', + items: { + type: 'string', + }, + }, + programmingLanguageVersion: { + type: 'string', + }, + }, + }, + licence: { + type: 'string', + }, + authors: { + type: 'array', + items: { + type: 'number', + }, + }, + tags: { + type: 'object', + properties: { + features: { + type: 'array', + items: { + type: 'string', + }, + }, + topics: { + type: 'array', + items: { + type: 'string', + }, + }, + }, + }, + toolids: { + type: 'array', + items: { + type: 'string', + }, + }, + }, + example: { + id: 7485531672584456, + type: 'paper', + name: 'Test Paper Title 2', + link: 'http://localhost:8080/epilepsy', + description: 'Test abstract 2', + categories: { + category: 'API', + programmingLanguage: ['Javascript'], + programmingLanguageVersion: '1.0.0', + }, + licence: 'MIT licence', + authors: [4495285946631793], + tags: { + features: ['Arbitrage'], + topics: ['Epilepsy'], + }, + toolids: [], + }, + }, + }, + }, + }, + responses: { + 200: { + description: 'OK', + }, + }, + }, + }, + '/api/v2/papers': { + get: { + summary: 'Returns a list of paper objects', + tags: ['Papers v2.0'], + parameters: [ + { + name: 'search', + in: 'query', + description: + 'Full text index search function which searches for partial matches in various fields including name and description. The response will contain a metascore indicating the relevancy of the match, by default results are sorted by the most relevant first unless a manual sort query parameter has been added.', + schema: { + type: 'string', + }, + example: 'Exploration', + }, + { + name: 'page', + in: 'query', + description: 'A specific page of results to retrieve', + schema: { + type: 'number', + }, + example: 1, + }, + { + name: 'limit', + in: 'query', + description: 'Maximum number of results returned per page', + schema: { + type: 'number', + }, + example: 10, + }, + { + name: 'sort', + in: 'query', + description: + 'Fields to apply sort operations to. Accepts multiple fields in ascending and descending. E.g. name for ascending or -name for descending. Multiple fields should be comma separated as shown in the example below.', + schema: { + type: 'string', + }, + example: 'name,-counter', + }, + { + name: 'fields', + in: 'query', + description: + 'Limit the size of the response by requesting only certain fields. Note that some additional derived fields are always returned. Multiple fields should be comma separate as shown in the example below.', + schema: { + type: 'string', + }, + example: 'name,counter,description', + }, + { + name: 'count', + in: 'query', + description: 'Returns the number of the number of entities matching the query parameters provided instead of the result payload', + schema: { + type: 'boolean', + }, + example: true, + }, + ], + description: + "Version 2.0 of the courses API introduces a large number of parameterised query string options to aid requests in collecting the data that is most relevant for a given use case. The query parameters defined below support a variety of comparison operators such as equals, contains, greater than, and less than. Using dot notation, any field can be queried, please see some examples below. Note - This response is limited to 100 records by default. Please use the 'page' query parameter to access records beyond the first 100. The 'limit' query parameter can therefore only be specified up to a maximum of 100.", + responses: { + 200: { + description: 'Successful response containing a list of papers matching query parameters', + }, + }, + }, + }, + '/api/v2/papers/{id}': { + get: { + summary: 'Returns paper object', + tags: ['Papers v2.0'], + parameters: [ + { + in: 'path', + name: 'id', + required: true, + description: 'The ID of the paper', + schema: { + type: 'number', + example: 13296138992670704, + }, + }, + ], + description: 'Returns a paper object by matching unique identifier in the default format that is stored as within the Gateway', + responses: { + 200: { + description: 'Successful response containing a single paper object', + }, + 404: { + description: 'A paper could not be found by the provided paper identifier', + }, + }, + }, + }, +}; diff --git a/docs/resources/person.docs.js b/docs/resources/person.docs.js new file mode 100644 index 00000000..4df4cf29 --- /dev/null +++ b/docs/resources/person.docs.js @@ -0,0 +1,134 @@ +module.exports = { + '/api/v1/person/{id}': { + get: { + summary: 'Returns details for a person.', + tags: ['Person'], + parameters: [ + { + in: 'path', + name: 'id', + required: true, + description: 'The ID of the person', + schema: { + type: 'string', + example: 900000014, + }, + }, + ], + responses: { + 200: { + description: 'OK', + }, + }, + }, + }, + '/api/v1/person': { + get: { + summary: 'Returns an array of person objects.', + tags: ['Person'], + responses: { + 200: { + description: 'OK', + }, + }, + }, + post: { + summary: 'Returns a new person object.', + tags: ['Person'], + requestBody: { + content: { + 'application/json': { + schema: { + type: 'object', + required: ['firstname', 'lastname', 'bio', 'link', 'orcid', 'emailNotifications', 'terms'], + properties: { + firstname: { + type: 'string', + }, + lastname: { + type: 'string', + }, + bio: { + type: 'string', + }, + link: { + type: 'string', + }, + orcid: { + type: 'string', + }, + emailNotifications: { + type: 'boolean', + }, + terms: { + type: 'boolean', + }, + }, + example: { + firstname: 'John', + lastname: 'Smith', + bio: 'Researcher', + link: 'http://google.com', + orcid: 'https://orcid.org/123456789', + emailNotifications: false, + terms: true, + }, + }, + }, + }, + }, + responses: { + 200: { + description: 'OK', + }, + }, + }, + put: { + summary: 'Returns edited person object.', + tags: ['Person'], + responses: { + 200: { + description: 'OK', + }, + }, + requestBody: { + content: { + 'application/json': { + schema: { + type: 'object', + required: ['id', 'bio', 'link', 'orcid', 'emailNotifications', 'terms'], + properties: { + id: { + type: 'string', + }, + bio: { + type: 'string', + }, + link: { + type: 'string', + }, + orcid: { + type: 'string', + }, + emailNotifications: { + type: 'boolean', + }, + terms: { + type: 'boolean', + }, + }, + example: { + id: '5268590523943617', + bio: 'Research assistant', + link: 'http://google.com', + orcid: 'https://orcid.org/123456789', + emailNotifications: false, + terms: true, + }, + }, + }, + }, + }, + }, + }, +}; diff --git a/docs/resources/project.docs.js b/docs/resources/project.docs.js new file mode 100644 index 00000000..abc37dd3 --- /dev/null +++ b/docs/resources/project.docs.js @@ -0,0 +1,429 @@ +module.exports = { + '/api/v1/projects': { + post: { + summary: 'Returns a Project object with ID.', + security: [ + { + cookieAuth: [], + }, + ], + tags: ['Projects'], + requestBody: { + content: { + 'application/json': { + schema: { + type: 'object', + required: ['name'], + properties: { + type: { + type: 'string', + }, + name: { + type: 'string', + }, + link: { + type: 'string', + }, + description: { + type: 'string', + }, + categories: { + type: 'object', + properties: { + category: { + type: 'string', + }, + programmingLanguage: { + type: 'array', + items: { + type: 'string', + }, + }, + programmingLanguageVersion: { + type: 'string', + }, + }, + }, + licence: { + type: 'string', + }, + authors: { + type: 'array', + items: { + type: 'number', + }, + }, + tags: { + type: 'object', + properties: { + features: { + type: 'array', + items: { + type: 'string', + }, + }, + topics: { + type: 'array', + items: { + type: 'string', + }, + }, + }, + }, + }, + example: { + type: 'project', + name: 'Epilepsy data research', + link: 'http://epilepsy.org', + description: 'Epilespy data research description', + categories: { + category: 'API', + programmingLanguage: ['Javascript'], + programmingLanguageVersion: '0.0.0', + }, + licence: 'MIT licence', + authors: [4495285946631793], + tags: { + features: ['Arbitrage'], + topics: ['Epilepsy'], + }, + }, + }, + }, + }, + }, + responses: { + 200: { + description: 'OK', + }, + }, + }, + get: { + summary: 'Returns List of Project objects.', + tags: ['Projects'], + parameters: [ + { + in: 'query', + name: 'limit', + required: false, + description: 'Limit the number of results', + schema: { + type: 'integer', + example: 3, + }, + }, + { + in: 'query', + name: 'offset', + required: false, + description: 'Index to offset the search results', + schema: { + type: 'integer', + example: 1, + }, + }, + { + in: 'query', + name: 'q', + required: false, + description: 'Filter using search query', + schema: { + type: 'string', + example: 'epilepsy', + }, + }, + ], + responses: { + 200: { + description: 'OK', + }, + }, + }, + }, + '/api/v1/projects/{id}': { + get: { + summary: 'Returns Project object.', + tags: ['Projects'], + parameters: [ + { + in: 'path', + name: 'id', + required: true, + schema: { + type: 'integer', + example: 441788967946948, + }, + }, + ], + responses: { + 200: { + description: 'OK', + }, + }, + }, + patch: { + summary: 'Change status of the Project object.', + security: [ + { + cookieAuth: [], + }, + ], + tags: ['Projects'], + parameters: [ + { + in: 'path', + name: 'id', + required: true, + schema: { + type: 'integer', + example: 662346984100503, + }, + }, + ], + requestBody: { + content: { + 'application/json': { + schema: { + type: 'object', + required: ['name'], + properties: { + activeflag: { + type: 'string', + }, + }, + example: { + activeflag: 'active', + }, + }, + }, + }, + }, + responses: { + 200: { + description: 'OK', + }, + }, + }, + put: { + summary: 'Returns edited Project object.', + security: [ + { + cookieAuth: [], + }, + ], + tags: ['Projects'], + parameters: [ + { + in: 'path', + name: 'id', + required: true, + description: 'The ID of the project', + schema: { + type: 'integer', + format: 'int64', + example: 26542005388306332, + }, + }, + ], + requestBody: { + content: { + 'application/json': { + schema: { + type: 'object', + required: ['name'], + properties: { + id: { + type: 'number', + }, + type: { + type: 'string', + }, + name: { + type: 'string', + }, + link: { + type: 'string', + }, + description: { + type: 'string', + }, + categories: { + type: 'object', + properties: { + category: { + type: 'string', + }, + programmingLanguage: { + type: 'array', + items: { + type: 'string', + }, + }, + programmingLanguageVersion: { + type: 'string', + }, + }, + }, + licence: { + type: 'string', + }, + authors: { + type: 'array', + items: { + type: 'number', + }, + }, + tags: { + type: 'object', + properties: { + features: { + type: 'array', + items: { + type: 'string', + }, + }, + topics: { + type: 'array', + items: { + type: 'string', + }, + }, + }, + }, + toolids: { + type: 'array', + items: { + type: 'string', + }, + }, + }, + example: { + id: 26542005388306332, + type: 'project', + name: 'Research Data TEST EPILEPSY', + link: 'http://localhost:8080/epilepsy', + description: 'Epilespy data research description', + categories: { + category: 'API', + programmingLanguage: ['Javascript'], + programmingLanguageVersion: '1.0.0', + }, + licence: 'MIT licence', + authors: [4495285946631793], + tags: { + features: ['Arbitrage'], + topics: ['Epilepsy'], + }, + toolids: [], + }, + }, + }, + }, + }, + responses: { + 200: { + description: 'OK', + }, + }, + }, + }, + '/api/v2/projects': { + get: { + summary: 'Returns a list of project objects', + tags: ['Projects v2.0'], + parameters: [ + { + name: 'search', + in: 'query', + description: + 'Full text index search function which searches for partial matches in various fields including name and description. The response will contain a metascore indicating the relevancy of the match, by default results are sorted by the most relevant first unless a manual sort query parameter has been added.', + schema: { + type: 'string', + }, + example: 'health service', + }, + { + name: 'page', + in: 'query', + description: 'A specific page of results to retrieve', + schema: { + type: 'number', + }, + example: 1, + }, + { + name: 'limit', + in: 'query', + description: 'Maximum number of results returned per page', + schema: { + type: 'number', + }, + example: 10, + }, + { + name: 'sort', + in: 'query', + description: + 'Fields to apply sort operations to. Accepts multiple fields in ascending and descending. E.g. name for ascending or -name for descending. Multiple fields should be comma separated as shown in the example below.', + schema: { + type: 'string', + }, + example: 'name,-counter', + }, + { + name: 'fields', + in: 'query', + description: + 'Limit the size of the response by requesting only certain fields. Note that some additional derived fields are always returned. Multiple fields should be comma separate as shown in the example below.', + schema: { + type: 'string', + }, + example: 'name,counter,description', + }, + { + name: 'count', + in: 'query', + description: 'Returns the number of the number of entities matching the query parameters provided instead of the result payload', + schema: { + type: 'boolean', + }, + example: true, + }, + ], + description: + "Version 2.0 of the courses API introduces a large number of parameterised query string options to aid requests in collecting the data that is most relevant for a given use case. The query parameters defined below support a variety of comparison operators such as equals, contains, greater than, and less than. Using dot notation, any field can be queried, please see some examples below. Note - This response is limited to 100 records by default. Please use the 'page' query parameter to access records beyond the first 100. The 'limit' query parameter can therefore only be specified up to a maximum of 100.", + responses: { + 200: { + description: 'Successful response containing a list of projects matching query parameters', + }, + }, + }, + }, + '/api/v2/projects/{id}': { + get: { + summary: 'Returns a project object', + tags: ['Projects v2.0'], + parameters: [ + { + in: 'path', + name: 'id', + required: true, + description: 'The ID of the project', + schema: { + type: 'number', + example: 100000001, + }, + }, + ], + description: 'Returns a project object by matching unique identifier in the default format that is stored as within the Gateway', + responses: { + 200: { + description: 'Successful response containing a single project object', + }, + 404: { + description: 'A project could not be found by the provided project identifier', + }, + }, + }, + }, +}; diff --git a/docs/resources/publisher.docs.js b/docs/resources/publisher.docs.js new file mode 100644 index 00000000..e15fb371 --- /dev/null +++ b/docs/resources/publisher.docs.js @@ -0,0 +1,717 @@ +module.exports = { + '/api/v1/publishers/{publisher}/dataaccessrequests': { + get: { + tags: ['Publishers'], + parameters: [ + { + in: 'path', + name: 'publisher', + required: true, + description: 'The full name of the Custodian/Publisher, as registered on the Gateway.', + schema: { + type: 'string', + example: 'OTHER > HEALTH DATA RESEARCH UK', + }, + }, + ], + description: 'Returns a collection of all Data Access Requests that have been submitted to the Custodian team for review.', + responses: { + 200: { + description: 'Successful response containing a collection of Data Access Request applications.', + content: { + 'application/json': { + schema: { + type: 'object', + properties: { + avgDecisionTime: { + type: 'string', + description: 'The average number of days the Custodian has taken to process applications from submission to decision.', + }, + canViewSubmitted: { + type: 'boolean', + description: + 'A flag to indicate if the requesting user has permissions to view submitted applications, which are visible only to managers of the Custodian team. Using OAuth2.0 client credentials will return this value as true.', + }, + status: { + type: 'string', + }, + data: { + type: 'array', + items: { + type: 'object', + properties: { + aboutApplication: { + description: + "An object which holds data relating to the 'about application' section of the application form including details of whether the project is an NCS project or not.", + type: 'object', + properties: { + isNationalCoreStudies: { + type: 'boolean', + description: 'A flag to indicate if this application is in relation to a National Core Studies Project.', + }, + nationalCoreStudiesProjectId: { + type: 'integer', + description: + 'The unique identifier correlating to a Gateway Project entity indicating that this application is relating to a National Core Studies project.', + }, + projectName: { + type: 'string', + description: 'The project name that has been assigned to the application by the applicant(s).', + }, + }, + }, + amendmentIterations: { + type: 'array', + items: { + type: 'object', + }, + description: + 'An array containing an object with details for each iteration the application has passed through. An iteration is defined as an application which has been returned by the Custodian for correction, corrected by the applicant(s) and resubmitted. The object contains dates that the application was returned, and resubmitted as well as reference to any questions that were highlighted for amendment.', + }, + amendmentStatus: { + type: 'string', + description: + 'A textual indicator of what state the application is in relating to updates made by the Custodian e.g. if it is awaiting updates from the applicant or if new updates have been submitted by the applicant(s).', + }, + applicants: { + type: 'string', + description: 'Concatenated list of applicants names who are contributing to the application.', + }, + applicationStatus: { + type: 'string', + enum: ['inProgress', 'submitted', 'inReview', 'approved', 'rejected', 'approved with conditions'], + description: 'The current status of the application.', + }, + authorIds: { + type: 'array', + items: { + type: 'integer', + description: + "An array of values correlating to specific user's via their numeric identifiers. An author is also known as a contributor to an application and can view, edit or submit.", + }, + }, + createdAt: { + type: 'string', + description: 'The date and time that the application was started.', + }, + datasetIds: { + type: 'array', + items: { + type: 'string', + }, + description: + 'An array of values correlating to datasets selected for the application via their identifier, which is unique per version.', + }, + datasetTitles: { + type: 'array', + items: { + type: 'string', + }, + description: 'An array of strings correlating to the dataset titles that have been selected for the application.', + }, + datasets: { + type: 'array', + items: { + type: 'object', + }, + description: + 'An array containing the full metadata for each of the datasets that have been applied for through this application.', + }, + dateSubmitted: { + type: 'string', + description: + 'The date and time that the application was originally submitted by the applicant(s) to the Custodian for review.', + }, + files: { + type: 'array', + items: { + type: 'object', + }, + description: + 'An array containing the links to files that have been uploaded to the application form and are held within the Gateway ecosystem.', + }, + id: { + type: 'string', + description: 'The unique identifier for the application.', + }, + jsonSchema: { + type: 'object', + description: + 'The object containing the json definition that renders the application form using the Winterfell library. This contains the details of questions, questions sets, question panels, headings and navigation items that appear.', + }, + questionAnswers: { + type: 'object', + description: + 'The object containing the answers provided on the application form. This consists of a series of key pairs, where the key is the unqiue question Id, and the value the is the answer provided to the question. In the case of a multi select on the form, the value may be an array.', + }, + mainApplicant: { + type: 'object', + description: + 'An object containing the details of the main applicant of the application as referenced by the userId field.', + }, + projectId: { + type: 'string', + description: + 'The unique identifier for the application converted to a more human friendly format in uppercase and hypenated.', + }, + projectName: { + type: 'string', + description: 'The project name that has been assigned to the application by the applicant(s).', + }, + publisher: { + type: 'string', + description: 'The name of the Custodian that holds the dataset and is processing the application.', + }, + publisherObj: { + type: 'object', + description: 'The object containing details regarding the Custodian/publisher relating to the application.', + }, + reviewPanels: { + type: 'array', + items: { + type: 'string', + }, + description: + "An array containing the sections of the application form that the current user is required to review if they are a reviewer of the current workflow step that the application is in. E.g. ['Safe People','Safe Data']", + }, + schemaId: { + type: 'string', + description: 'The unique identifier that correlates to the schema from which the application form was generated.', + }, + updatedAt: { + type: 'string', + description: 'The date and time that the application was last updated by any party.', + }, + userId: { + type: 'integer', + description: + 'The unique identifier that correlates to the user account of the main applicant. This is always the user that started the application.', + }, + deadlinePassed: { + type: 'boolean', + description: 'A flag to indicate if the deadline has passed for the current review phase for this application.', + }, + decisionApproved: { + type: 'boolean', + description: + 'A flag to indicate if the request users decision as a reviewer of the current workflow phase was positive or negative. i.e. correlating to approval or rejection recommendation.', + }, + decisionComments: { + type: 'string', + description: + 'A supporting note or comment made by the requesting user as context to their decision as a reviewer of the current workflow phase.', + }, + decisionDate: { + type: 'string', + description: 'The date that the requesting user made their decision as a reviewer of the current workflow phase.', + }, + decisionDuration: { + type: 'integer', + description: + "The number of days from submission until a final decision was made on the application. i.e. the application status was changed to a final status e.g. 'Approved'.", + }, + decisionMade: { + type: 'boolean', + description: + 'A flag to indicate if the requesting user has made an expected decision as a reviewer of the current workflow phase.', + }, + decisionStatus: { + type: 'string', + description: + 'A message indicating if the requesting user as a reviewer of the application has made a decision or is still required to make a decision for the current work flow.', + }, + isReviewer: { + type: 'boolean', + description: + 'A flag to indicate if the requesting user is a reviewer of the current workflow step for the application.', + }, + remainingActioners: { + type: 'array', + items: { + type: 'string', + }, + description: + 'An array containing the names of Custodian team reviewers expected to complete a review for the current workflow phase, or a list of managers if the application is awaiting a final decision.', + }, + reviewStatus: { + type: 'string', + description: + "A message indicating the current status of the application review in relation to the assigned workflow. E.g. 'Final decision required' or 'Deadline is today'. This message changes based on the requesting user's relationship to the application. E.g. if they are a reviewer or manager.", + }, + stepName: { + type: 'string', + description: 'The name of the current workflow step that the application is in.', + }, + workflowCompleted: { + type: 'boolean', + description: 'A flag to indicate if the assigned workflow for the review process has been completed.', + }, + workflowName: { + type: 'string', + description: + 'The name of the workflow the Custodian team have assigned to the application for the review process.', + }, + }, + }, + }, + }, + }, + examples: { + 'Single Request Received': { + value: { + success: true, + data: [ + { + authorIds: [], + datasetIds: ['d5faf9c6-6c34-46d7-93c4-7706a5436ed9'], + datasetTitles: [], + applicationStatus: 'submitted', + jsonSchema: '{omitted for brevity...}', + questionAnswers: '{omitted for brevity...}', + publisher: 'OTHER > HEALTH DATA RESEARCH UK', + _id: '601853db22dc004f9adfaa24', + version: 1, + userId: 7584453789581072, + schemaId: '5f55e87e780ba204b0a98eb8', + files: [ + { + error: '', + _id: '601aacf8ecdfa66e5cbc2742', + status: 'UPLOADED', + description: 'QuestionAnswers', + fileId: '9e76ee1a676f423b9b5c7aabf59c69db', + size: 509984, + name: 'QuestionAnswersFlags.png', + owner: '5ec7f1b39219d627e5cafae3', + }, + { + error: '', + _id: '601aadbcecdfa6c532bc2743', + status: 'UPLOADED', + description: 'Notifications', + fileId: 'adb1718dcc094b9cb4b0ab347ad2ee94', + size: 54346, + name: 'HQIP-Workflow-Assigned-Notification.png', + owner: '5ec7f1b39219d627e5cafae3', + }, + ], + amendmentIterations: [], + createdAt: '2021-02-01T19:17:47.470Z', + updatedAt: '2021-02-03T16:36:36.720Z', + __v: 2, + projectId: '6018-53DB-22DC-004F-9ADF-AA24', + aboutApplication: { + selectedDatasets: [ + { + _id: '5fc31a18d98e4f4cff7e9315', + datasetId: 'd5faf9c6-6c34-46d7-93c4-7706a5436ed9', + name: 'HDR UK Papers & Preprints', + description: + 'Publications that mention HDR-UK (or any variant thereof) in Acknowledgements or Author Affiliations\n\nThis will include:\n- Papers\n- COVID-19 Papers\n- COVID-19 Preprint', + abstract: + 'Publications that mention HDR-UK (or any variant thereof) in Acknowledgements or Author Affiliations', + publisher: 'OTHER > HEALTH DATA RESEARCH UK', + contactPoint: 'hdr.hdr@hdruk.ac.uk', + publisherObj: { + dataRequestModalContent: { + header: ' ', + body: '{omitted for brevity...}', + footer: '', + }, + active: true, + allowsMessaging: true, + workflowEnabled: true, + _id: '5f7b1a2bce9f65e6ed83e7da', + name: 'OTHER > HEALTH DATA RESEARCH UK', + imageURL: '', + team: { + active: true, + _id: '5f7b1a2bce9f65e6ed83e7da', + members: [ + { + roles: ['manager'], + memberid: '5f1a98861a821b4a53e44d15', + }, + { + roles: ['manager'], + memberid: '600bfc99c8bf700f2c7d5c36', + }, + ], + type: 'publisher', + __v: 3, + createdAt: '2020-11-30T21:12:40.855Z', + updatedAt: '2020-12-02T13:33:45.232Z', + }, + }, + }, + ], + isNationalCoreStudies: true, + nationalCoreStudiesProjectId: '4324836585275824', + projectName: 'Test application title', + completedDatasetSelection: true, + completedInviteCollaborators: true, + completedReadAdvice: true, + completedCommunicateAdvice: true, + completedApprovalsAdvice: true, + completedSubmitAdvice: true, + }, + dateSubmitted: '2021-02-03T16:37:36.081Z', + datasets: [ + { + categories: { + programmingLanguage: [], + }, + tags: { + features: ['Preprints', 'Papers', 'HDR UK'], + topics: [], + }, + datasetfields: { + geographicCoverage: ['https://www.geonames.org/countries/GB/united-kingdom.html'], + physicalSampleAvailability: ['Not Available'], + technicaldetails: '{omitted for brevity...}', + versionLinks: [ + { + id: '142b1618-2691-4019-97b4-16b1e27c5f95', + linkType: 'Superseded By', + domainType: 'CatalogueSemanticLink', + source: { + id: '9e798632-442a-427b-8d0e-456f754d28dc', + domainType: 'DataModel', + label: 'HDR UK Papers & Preprints', + documentationVersion: '0.0.1', + }, + target: { + id: 'd5faf9c6-6c34-46d7-93c4-7706a5436ed9', + domainType: 'DataModel', + label: 'HDR UK Papers & Preprints', + documentationVersion: '1.0.0', + }, + }, + ], + phenotypes: [], + publisher: 'OTHER > HEALTH DATA RESEARCH UK', + abstract: + 'Publications that mention HDR-UK (or any variant thereof) in Acknowledgements or Author Affiliations', + releaseDate: '2020-11-27T00:00:00Z', + accessRequestDuration: 'Other', + conformsTo: 'OTHER', + accessRights: 'https://github.com/HDRUK/papers/blob/master/LICENSE', + jurisdiction: 'GB-ENG', + datasetStartDate: '2020-03-31', + datasetEndDate: '2022-04-30', + statisticalPopulation: '0', + ageBand: '0-0', + contactPoint: 'hdr.hdr@hdruk.ac.uk', + periodicity: 'Daily', + metadataquality: { + id: 'd5faf9c6-6c34-46d7-93c4-7706a5436ed9', + publisher: 'OTHER > HEALTH DATA RESEARCH UK', + title: 'HDR UK Papers & Preprints', + completeness_percent: 95.24, + weighted_completeness_percent: 100, + error_percent: 11.63, + weighted_error_percent: 19.05, + quality_score: 91.81, + quality_rating: 'Gold', + weighted_quality_score: 90.47, + weighted_quality_rating: 'Gold', + }, + datautility: { + id: 'd5faf9c6-6c34-46d7-93c4-7706a5436ed9', + publisher: 'OTHER > HEALTH DATA RESEARCH UK', + title: 'HDR UK Papers & Preprints', + metadata_richness: 'Gold', + availability_of_additional_documentation_and_support: '', + data_model: '', + data_dictionary: '', + provenance: '', + data_quality_management_process: '', + dama_quality_dimensions: '', + pathway_coverage: '', + length_of_follow_up: '', + allowable_uses: '', + research_environment: '', + time_lag: '', + timeliness: '', + linkages: '', + data_enrichments: '', + }, + metadataschema: { + '@context': 'http://schema.org/', + '@type': 'Dataset', + identifier: 'd5faf9c6-6c34-46d7-93c4-7706a5436ed9', + url: 'https://healthdatagateway.org/detail/d5faf9c6-6c34-46d7-93c4-7706a5436ed9', + name: 'HDR UK Papers & Preprints', + description: + 'Publications that mention HDR-UK (or any variant thereof) in Acknowledgements or Author Affiliations\n\nThis will include:\n- Papers\n- COVID-19 Papers\n- COVID-19 Preprint', + license: 'Open Access', + keywords: [ + 'Preprints,Papers,HDR UK', + 'OTHER > HEALTH DATA RESEARCH UK', + 'NOT APPLICABLE', + 'GB-ENG', + 'https://www.geonames.org/countries/GB/united-kingdom.html', + ], + includedinDataCatalog: [ + { + '@type': 'DataCatalog', + name: 'OTHER > HEALTH DATA RESEARCH UK', + url: 'hdr.hdr@hdruk.ac.uk', + }, + { + '@type': 'DataCatalog', + name: 'HDR UK Health Data Gateway', + url: 'http://healthdatagateway.org', + }, + ], + }, + }, + authors: [], + showOrganisation: false, + toolids: [], + datasetids: [], + _id: '5fc31a18d98e4f4cff7e9315', + relatedObjects: [], + programmingLanguage: [], + pid: 'b7a62c6d-ed00-4423-ad27-e90b71222d8e', + datasetVersion: '1.0.0', + id: 9816147066244124, + datasetid: 'd5faf9c6-6c34-46d7-93c4-7706a5436ed9', + type: 'dataset', + activeflag: 'active', + name: 'HDR UK Papers & Preprints', + description: + 'Publications that mention HDR-UK (or any variant thereof) in Acknowledgements or Author Affiliations\n\nThis will include:\n- Papers\n- COVID-19 Papers\n- COVID-19 Preprint', + license: 'Open Access', + datasetv2: { + identifier: '', + version: '', + issued: '', + modified: '', + revisions: [], + summary: { + title: '', + abstract: + 'Publications that mention HDR-UK (or any variant thereof) in Acknowledgements or Author Affiliations', + publisher: { + identifier: '', + name: 'HEALTH DATA RESEARCH UK', + logo: '', + description: '', + contactPoint: 'hdr.hdr@hdruk.ac.uk', + memberOf: 'OTHER', + accessRights: [], + deliveryLeadTime: '', + accessService: '', + accessRequestCost: '', + dataUseLimitation: [], + dataUseRequirements: [], + }, + contactPoint: 'hdr.hdr@hdruk.ac.uk', + keywords: ['Preprints', 'Papers', 'HDR UK'], + alternateIdentifiers: [], + doiName: 'https://doi.org/10.5281/zenodo.326615', + }, + documentation: { + description: '', + associatedMedia: ['https://github.com/HDRUK/papers'], + isPartOf: 'NOT APPLICABLE', + }, + coverage: { + spatial: 'GB', + typicalAgeRange: '0-0', + physicalSampleAvailability: ['NOT AVAILABLE'], + followup: 'UNKNOWN', + pathway: 'NOT APPLICABLE', + }, + provenance: { + origin: { + purpose: 'OTHER', + source: 'MACHINE GENERATED', + collectionSituation: 'OTHER', + }, + temporal: { + accrualPeriodicity: 'DAILY', + distributionReleaseDate: '2020-11-27', + startDate: '2020-03-31', + endDate: '2022-04-30', + timeLag: 'NO TIMELAG', + }, + }, + accessibility: { + usage: { + dataUseLimitation: 'GENERAL RESEARCH USE', + dataUseRequirements: 'RETURN TO DATABASE OR RESOURCE', + resourceCreator: 'HDR UK Using Team', + investigations: ['https://github.com/HDRUK/papers'], + isReferencedBy: ['Not Available'], + }, + access: { + accessRights: ['Open Access'], + accessService: 'https://github.com/HDRUK/papers', + accessRequestCost: 'Free', + deliveryLeadTime: 'OTHER', + jurisdiction: 'GB-ENG', + dataProcessor: 'HDR UK', + dataController: 'HDR UK', + }, + formatAndStandards: { + vocabularyEncodingScheme: 'OTHER', + conformsTo: 'OTHER', + language: 'en', + format: ['csv', 'JSON'], + }, + }, + enrichmentAndLinkage: { + qualifiedRelation: ['Not Available'], + derivation: ['Not Available'], + tools: ['https://github.com/HDRUK/papers'], + }, + observations: [], + }, + createdAt: '2020-11-29T03:48:41.794Z', + updatedAt: '2021-02-02T10:09:57.030Z', + __v: 0, + counter: 20, + }, + ], + dataset: null, + mainApplicant: { + isServiceAccount: false, + _id: '5ec7f1b39219d627e5cafae3', + id: 7584453789581072, + providerId: '112563375053074694443', + provider: 'google', + firstname: 'Chris', + lastname: 'Marks', + email: 'chris.marks@paconsulting.com', + role: 'Admin', + __v: 0, + redirectURL: '/tool/100000012', + discourseKey: '2f52ecaa21a0d0223a119da5a09f8f8b09459e7b69ec3f981102d09f66488d99', + discourseUsername: 'chris.marks', + updatedAt: '2021-02-01T12:39:56.372Z', + }, + publisherObj: { + dataRequestModalContent: { + header: '', + body: '', + footer: '', + }, + active: true, + allowsMessaging: true, + workflowEnabled: true, + _id: '5f7b1a2bce9f65e6ed83e7da', + name: 'OTHER > HEALTH DATA RESEARCH UK', + imageURL: '', + team: { + active: true, + _id: '5f7b1a2bce9f65e6ed83e7da', + members: [ + { + roles: ['manager'], + memberid: '5f1a98861a821b4a53e44d15', + }, + { + roles: ['manager'], + memberid: '600bfc99c8bf700f2c7d5c36', + }, + ], + type: 'publisher', + __v: 3, + createdAt: '2020-11-30T21:12:40.855Z', + updatedAt: '2020-12-02T13:33:45.232Z', + users: [ + { + _id: '5f1a98861a821b4a53e44d15', + firstname: 'Robin', + lastname: 'Kavanagh', + }, + { + _id: '600bfc99c8bf700f2c7d5c36', + firstname: 'HDR-UK', + lastname: 'Service Account', + }, + ], + }, + }, + id: '601853db22dc004f9adfaa24', + projectName: 'PA Paper', + applicants: 'Chris Marks', + workflowName: '', + workflowCompleted: false, + decisionDuration: '', + decisionMade: false, + decisionStatus: '', + decisionComments: '', + decisionDate: '', + decisionApproved: false, + remainingActioners: 'Robin Kavanagh (you), HDR-UK Service Account', + stepName: '', + deadlinePassed: '', + reviewStatus: '', + isReviewer: false, + reviewPanels: [], + amendmentStatus: '', + }, + ], + avgDecisionTime: 1, + canViewSubmitted: true, + }, + }, + }, + }, + }, + }, + 401: { + description: 'Unauthorised attempt to access an application.', + content: { + 'application/json': { + schema: { + type: 'object', + properties: { + status: { + type: 'string', + }, + message: { + type: 'string', + }, + }, + }, + examples: { + Unauthorised: { + value: { + status: 'failure', + message: 'Unauthorised', + }, + }, + }, + }, + }, + }, + 404: { + description: 'Failed to find the application requested.', + content: { + 'application/json': { + schema: { + type: 'object', + properties: { + success: { + type: 'boolean', + }, + }, + }, + examples: { + 'Not Found': { + value: { + success: false, + }, + }, + }, + }, + }, + }, + }, + }, + }, +}; diff --git a/docs/resources/search.docs.js b/docs/resources/search.docs.js new file mode 100644 index 00000000..f75f3d3c --- /dev/null +++ b/docs/resources/search.docs.js @@ -0,0 +1,58 @@ +module.exports = { + '/api/v1/search': { + get: { + tags: ['Search'], + summary: 'Search for HDRUK /search?search', + parameters: [ + { + in: 'query', + name: 'params', + schema: { + type: 'object', + properties: { + search: { + type: 'string', + example: 'Epilepsy', + }, + type: { + type: 'string', + example: 'all', + }, + category: { + type: 'string', + example: 'API', + }, + programmingLanguage: { + type: 'string', + example: 'Javascript', + }, + features: { + type: 'string', + example: 'Arbitrage', + }, + topics: { + type: 'string', + example: 'Epilepsy', + }, + startIndex: { + type: 'string', + example: 0, + }, + maxResults: { + type: 'string', + example: 10, + }, + }, + }, + style: 'form', + explode: true, + }, + ], + responses: { + 200: { + description: 'OK', + }, + }, + }, + }, +}; diff --git a/docs/resources/stats.docs.js b/docs/resources/stats.docs.js new file mode 100644 index 00000000..28659afa --- /dev/null +++ b/docs/resources/stats.docs.js @@ -0,0 +1,122 @@ +module.exports = { + '/api/v1/stats/topSearches': { + get: { + summary: 'Returns top searches for a given month and year.', + tags: ['Stats'], + parameters: [ + { + name: 'month', + in: 'query', + required: true, + description: 'Month number.', + schema: { + type: 'string', + example: 7, + }, + }, + { + name: 'year', + in: 'query', + required: true, + description: 'Year.', + schema: { + type: 'string', + example: 2020, + }, + }, + ], + responses: { + 200: { + description: 'OK', + }, + }, + }, + }, + '/api/v1/stats': { + get: { + summary: + 'Returns the details on recent searches, popular objects, unmet demands or recently updated objects based on the rank query parameter.', + tags: ['Stats'], + parameters: [ + { + name: 'rank', + in: 'query', + required: true, + description: 'The type of stat.', + schema: { + type: 'string', + example: 'unmet', + }, + }, + { + name: 'type', + in: 'query', + required: true, + description: 'Resource type.', + schema: { + type: 'string', + example: 'Tools', + }, + }, + { + name: 'month', + in: 'query', + required: true, + description: 'Month number.', + schema: { + type: 'string', + example: 7, + }, + }, + { + name: 'year', + in: 'query', + required: true, + description: 'Year.', + schema: { + type: 'string', + example: 2020, + }, + }, + ], + responses: { + 200: { + description: 'OK', + }, + }, + }, + }, + '/api/v1/kpis': { + get: { + summary: 'Returns information for KPIs, based on the KPI type and selectedDate parameters.', + tags: ['KPIs'], + parameters: [ + { + name: 'type', + in: 'query', + required: true, + description: 'The type of KPI.', + schema: { + type: 'string', + example: 'uptime', + }, + }, + { + name: 'selectedDate', + in: 'query', + required: true, + description: 'Full date time string.', + schema: { + type: 'string', + example: 'Wed Jul 01 2020 01:00:00 GMT 0100 (British Summer Time)', + }, + }, + ], + responses: { + 200: { + description: 'OK', + }, + }, + }, + }, +}; diff --git a/docs/resources/tool.docs.js b/docs/resources/tool.docs.js new file mode 100644 index 00000000..e2c4d92e --- /dev/null +++ b/docs/resources/tool.docs.js @@ -0,0 +1,422 @@ +module.exports = { + '/api/v1/tools': { + get: { + summary: 'Return List of Tool objects.', + tags: ['Tools'], + parameters: [ + { + in: 'query', + name: 'limit', + required: false, + description: 'Limit the number of results', + schema: { + type: 'integer', + example: 3, + }, + }, + { + in: 'query', + name: 'offset', + required: false, + description: 'Index to offset the search results', + schema: { + type: 'integer', + example: 1, + }, + }, + { + in: 'query', + name: 'q', + required: false, + description: 'Filter using search query', + schema: { + type: 'string', + example: 'epilepsy', + }, + }, + ], + responses: { + 200: { + description: 'OK', + }, + }, + }, + post: { + summary: 'Returns new Tool object with ID.', + security: [ + { + cookieAuth: [], + }, + ], + tags: ['Tools'], + requestBody: { + content: { + 'application/json': { + schema: { + type: 'object', + required: ['name'], + properties: { + type: { + type: 'string', + }, + name: { + type: 'string', + }, + link: { + type: 'string', + }, + description: { + type: 'string', + }, + categories: { + type: 'object', + properties: { + category: { + type: 'string', + }, + programmingLanguage: { + type: 'array', + items: { + type: 'string', + }, + }, + programmingLanguageVersion: { + type: 'string', + }, + }, + }, + licence: { + type: 'string', + }, + authors: { + type: 'array', + items: { + type: 'number', + }, + }, + tags: { + type: 'object', + properties: { + features: { + type: 'array', + items: { + type: 'string', + }, + }, + topics: { + type: 'array', + items: { + type: 'string', + }, + }, + }, + }, + }, + example: { + id: 26542005388306332, + }, + }, + }, + }, + }, + responses: { + 200: { + description: 'OK', + }, + }, + }, + }, + '/api/v1/tools/{id}': { + get: { + summary: 'Returns Tool object', + tags: ['Tools'], + parameters: [ + { + in: 'path', + name: 'id', + required: true, + description: 'The ID of the tool', + schema: { + type: 'integer', + format: 'int64', + minimum: 1, + example: 19009, + }, + }, + ], + responses: { + 200: { + description: 'OK', + }, + }, + }, + put: { + summary: 'Returns edited Tool object.', + security: [ + { + cookieAuth: [], + }, + ], + tags: ['Tools'], + parameters: [ + { + in: 'path', + name: 'id', + required: true, + schema: { + type: 'integer', + example: 123, + }, + }, + ], + requestBody: { + content: { + 'application/json': { + schema: { + type: 'object', + required: ['name'], + properties: { + id: { + type: 'number', + }, + type: { + type: 'string', + }, + name: { + type: 'string', + }, + link: { + type: 'string', + }, + description: { + type: 'string', + }, + categories: { + type: 'object', + properties: { + category: { + type: 'string', + }, + programmingLanguage: { + type: 'array', + items: { + type: 'string', + }, + }, + programmingLanguageVersion: { + type: 'string', + }, + }, + }, + licence: { + type: 'string', + }, + authors: { + type: 'array', + items: { + type: 'number', + }, + }, + tags: { + type: 'object', + properties: { + features: { + type: 'array', + items: { + type: 'string', + }, + }, + topics: { + type: 'array', + items: { + type: 'string', + }, + }, + }, + }, + toolids: { + type: 'array', + items: { + type: 'string', + }, + }, + }, + example: { + id: 26542005388306332, + type: 'tool', + name: 'Research Data TEST EPILEPSY', + link: 'http://localhost:8080/epilepsy', + description: 'Epilespy data research description', + categories: { + category: 'API', + programmingLanguage: ['Javascript'], + programmingLanguageVersion: '1.0.0', + }, + licence: 'MIT licence', + authors: [4495285946631793], + tags: { + features: ['Arbitrage'], + topics: ['Epilepsy'], + }, + toolids: [], + }, + }, + }, + }, + }, + responses: { + 200: { + description: 'OK', + }, + }, + }, + patch: { + summary: 'Change status of Tool object.', + security: [ + { + cookieAuth: [], + }, + ], + tags: ['Tools'], + parameters: [ + { + name: 'id', + in: 'path', + required: true, + description: 'The ID of the tool', + schema: { + type: 'integer', + format: 'int64', + example: 5032687830560181, + }, + }, + ], + requestBody: { + content: { + 'application/json': { + schema: { + type: 'object', + required: ['name'], + properties: { + id: { + type: 'number', + }, + activeflag: { + type: 'string', + }, + }, + example: { + id: 662346984100503, + activeflag: 'active', + }, + }, + }, + }, + }, + responses: { + 200: { + description: 'OK', + }, + }, + }, + }, + '/api/v2/tools': { + get: { + summary: 'Returns a list of tool objects', + tags: ['Tools v2.0'], + parameters: [ + { + name: 'search', + in: 'query', + description: + 'Full text index search function which searches for partial matches in various fields including name and description. The response will contain a metascore indicating the relevancy of the match, by default results are sorted by the most relevant first unless a manual sort query parameter has been added.', + schema: { + type: 'string', + }, + example: 'Regulation', + }, + { + name: 'page', + in: 'query', + description: 'A specific page of results to retrieve', + schema: { + type: 'number', + }, + example: 1, + }, + { + name: 'limit', + in: 'query', + description: 'Maximum number of results returned per page', + schema: { + type: 'number', + }, + example: 10, + }, + { + name: 'sort', + in: 'query', + description: + 'Fields to apply sort operations to. Accepts multiple fields in ascending and descending. E.g. name for ascending or -name for descending. Multiple fields should be comma separated as shown in the example below.', + schema: { + type: 'string', + }, + example: 'name,-counter', + }, + { + name: 'fields', + in: 'query', + description: + 'Limit the size of the response by requesting only certain fields. Note that some additional derived fields are always returned. Multiple fields should be comma separate as shown in the example below.', + schema: { + type: 'string', + }, + example: 'name,counter, description', + }, + { + name: 'count', + in: 'query', + description: 'Returns the number of the number of entities matching the query parameters provided instead of the result payload', + schema: { + type: 'boolean', + }, + example: true, + }, + ], + description: + "Version 2.0 of the courses API introduces a large number of parameterised query string options to aid requests in collecting the data that is most relevant for a given use case. The query parameters defined below support a variety of comparison operators such as equals, contains, greater than, and less than. Using dot notation, any field can be queried, please see some examples below. Note - This response is limited to 100 records by default. Please use the 'page' query parameter to access records beyond the first 100. The 'limit' query parameter can therefore only be specified up to a maximum of 100.", + responses: { + 200: { + description: 'Successful response containing a list of tools matching query parameters', + }, + }, + }, + }, + '/api/v2/tools/{id}': { + get: { + summary: 'Returns a tool object', + tags: ['Tools v2.0'], + parameters: [ + { + in: 'path', + name: 'id', + required: true, + description: 'The ID of the tool', + schema: { + type: 'number', + example: 100000006, + }, + }, + ], + description: 'Returns a tool object by matching unique identifier in the default format that is stored as within the Gateway', + responses: { + 200: { + description: 'Successful response containing a single tool object', + }, + 404: { + description: 'A tool could not be found by the provided tool identifier', + }, + }, + }, + }, +}; diff --git a/docs/resources/topic.docs.js b/docs/resources/topic.docs.js new file mode 100644 index 00000000..1939b5d0 --- /dev/null +++ b/docs/resources/topic.docs.js @@ -0,0 +1,159 @@ +module.exports = { + '/api/v1/topics': { + post: { + summary: 'Returns a new Topic object with ID (Does not create any associated messages)', + security: [ + { + cookieAuth: [], + }, + ], + tags: ['Topics'], + requestBody: { + required: true, + content: { + 'application/json': { + schema: { + type: 'object', + properties: { + relatedObjectIds: { + type: 'array', + items: { + type: 'string', + }, + }, + }, + }, + examples: { + 'Create a new topic': { + value: "{\n \"relatedObjectIds\": \"['1','2','3']\"\n}", + }, + }, + }, + }, + }, + responses: { + 201: { + description: 'A new Topic', + content: { + 'application/json': { + schema: { + type: 'object', + properties: { + _id: { + type: 'object', + description: 'Generated ID', + }, + title: { + type: 'string', + description: 'Title of message', + }, + subtitle: { + type: 'string', + description: 'Subtitle of message', + }, + relatedObjectIds: { + type: 'array', + items: { + type: 'string', + }, + description: 'Object ID this Topic is related to', + }, + createdBy: { + type: 'object', + description: 'User that created the topic', + }, + createdDate: { + type: 'string', + description: 'Date the topic was created', + }, + recipients: { + type: 'array', + items: { + type: 'string', + }, + description: 'Collection of user IDs', + }, + tags: { + type: 'array', + items: { + type: 'string', + }, + description: 'Collection of tags to describe topic', + }, + }, + }, + }, + }, + }, + }, + }, + get: { + summary: 'Returns a list of all topics that the authenticated user is a recipient or member of', + security: [ + { + cookieAuth: [], + }, + ], + tags: ['Topics'], + responses: { + 200: { + description: 'Ok', + }, + }, + }, + }, + '/api/v1/topics/{id}': { + get: { + summary: 'Returns Topic object by ID', + security: [ + { + cookieAuth: [], + }, + ], + tags: ['Topics'], + parameters: [ + { + in: 'path', + name: 'id', + required: true, + description: 'The ID of the topic', + schema: { + type: 'string', + example: '5ee249426136805fbf094eef', + }, + }, + ], + responses: { + 200: { + description: 'Ok', + }, + }, + }, + delete: { + summary: 'Soft deletes a message Topic but does not affect associated messages', + security: [ + { + cookieAuth: [], + }, + ], + tags: ['Topics'], + parameters: [ + { + in: 'path', + name: 'id', + required: true, + description: 'The ID of the Topic', + schema: { + type: 'string', + example: '5ee249426136805fbf094eef', + }, + }, + ], + responses: { + 204: { + description: 'Ok', + }, + }, + }, + }, +}; diff --git a/src/config/server.js b/src/config/server.js index eacf8cdd..39fbad97 100644 --- a/src/config/server.js +++ b/src/config/server.js @@ -3,8 +3,6 @@ import express from 'express'; import Provider from 'oidc-provider'; import swaggerUi from 'swagger-ui-express'; -import YAML from 'yamljs'; -const swaggerDocument = YAML.load('./swagger.yaml'); import cors from 'cors'; import logger from 'morgan'; import passport from 'passport'; @@ -180,7 +178,7 @@ app.get('/api/v1/openid/interaction/:uid', setNoCache, (req, res, next) => { app.use('/api/v1/openid', oidc.callback); app.use('/api', router); -app.use('/api-docs', swaggerUi.serve, swaggerUi.setup(swaggerDocument)); +app.use('/api-docs', swaggerUi.serve, swaggerUi.setup(require('../../docs/index.docs'))); app.use('/oauth', require('../resources/auth/oauth.route')); app.use('/api/v1/auth/sso/discourse', require('../resources/auth/sso/sso.discourse.router')); From 60fc320e0dd494ae0157bbd0849188a70d9b411a Mon Sep 17 00:00:00 2001 From: Callum Reekie Date: Tue, 12 Oct 2021 14:35:20 +0100 Subject: [PATCH 042/116] CR - deleted swagger.yaml --- swagger.yaml | 2877 -------------------------------------------------- 1 file changed, 2877 deletions(-) delete mode 100644 swagger.yaml diff --git a/swagger.yaml b/swagger.yaml deleted file mode 100644 index 613451b0..00000000 --- a/swagger.yaml +++ /dev/null @@ -1,2877 +0,0 @@ -openapi: 3.0.1 -info: - title: HDR UK API - description: API for Tools and artefacts repository. - version: 1.0.0 -servers: - - url: https://api.www.healthdatagateway.org/ - - url: http://localhost:3001/ - - url: https://api.{environment}.healthdatagateway.org:{port}/ - variables: - environment: - default: latest - description: The Environment name. - port: - enum: - - '443' - default: '443' -security: - - oauth2: [] -paths: - /oauth/token: - post: - tags: - - Authorization - description: OAuth2.0 token endpoint responsible for issuing short-lived json web tokens (JWT) for access to secure Gateway APIs. For client credentials grant flow, a valid client id and secret must be provided to identify your application and provide the expected permissions. This type of authentication is reserved for team based connectivity through client applications and is not provided for human user access. For more information, contact the HDR-UK team. - requestBody: - required: true - content: - application/json: - schema: - type: object - properties: - grant_type: - type: string - description: The OAuth2.0 grant type that will be used to provide authentication. - client_id: - type: string - description: A unique identifer provided to your team by the HDR-UK team at the time of onboarding to the Gateway. Contact the HDR-UK team for issue of new credentials. - client_secret: - type: string - description: A long (50 character) string provided by the HDR-UK team at the time of onboarding to the Gateway. Contact the HDR-UK team for issue of new credentials. - required: - - grant_type - - client_secret - - client_id - examples: - 'Client Credentials Grant Flow': - value: - { - 'grant_type': 'client_credentials', - 'client_id': '2ca1f61a90e3547', - 'client_secret': '3f80fecbf781b6da280a8d17aa1a22066fb66daa415d8befc1', - } - responses: - '200': - description: Successful response containing json web token (JWT) that will authorize an HTTP request against secured resources. - content: - application/json: - schema: - type: object - properties: - access_token: - type: string - description: The encoded json web token (JWT) that must be appended to the Authorization of subsequent API HTTP requests in order to access secured resources. - token_type: - type: string - description: The type of token issued, in this case, a json web token (JWT). - expires_in: - type: integer - description: The length of time in seconds before the issued JWT expires, defaulted to 900 seconds (15 minutes). - examples: - 'Client Credentials Grant Flow': - value: - { - 'access_token': 'Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJkYXRhIjp7Il9pZCI6IjYwMGJmYzk5YzhiZjcwMGYyYzdkNWMzNiIsInRpbWVTdGFtcCI2MTYxMjM4MzkwMzE5Nn0sImlhdCI6MTYxMjM4MzkwMywiZXhwIjoxNjEyMzg0ODAzfQ.-YvUBdjtJvdrRacz6E8-cYPQlum4TrEmiCFl8jO5a-M', - 'token_type': 'jwt', - 'expires_in': 900, - } - '400': - description: Failure response caused by incomplete or invalid client credentials being passed to the endpoint. - content: - application/json: - schema: - type: object - properties: - success: - type: boolean - description: A field that indicates the API request failed. - message: - type: string - description: A message indicating that the request failed for a given reason. - examples: - 'Invalid Client Credentials': - value: { 'success': false, 'message': 'Invalid client credentials were provided for the authorisation attempt' } - 'Incomplete Client Credentials': - value: { 'success': false, 'message': 'Incomplete client credentials were provided for the authorisation attempt' } - 'Invalid Grant Type': - value: { 'success': false, 'message': 'An invalid grant type has been specified' } - - /api/v1/data-access-request/{id}: - get: - tags: - - Data Access Request - parameters: - - in: path - name: id - required: true - description: The unique identifier for a single data access request application. - schema: - type: string - example: 5ee249426136805fbf094eef - description: Retrieve a single Data Access Request application using a supplied identifer - responses: - '200': - description: Successful response containing a full data access request application. - content: - application/json: - schema: - type: object - properties: - status: - type: string - data: - type: object - properties: - id: - type: string - description: The unique identifier for the application. - aboutApplication: - description: An object which holds data relating to the 'about application' section of the application form including details of whether the project is an NCS project or not. - type: object - properties: - isNationalCoreStudies: - type: boolean - description: A flag to indicate if this application is in relation to a National Core Studies Project. - nationalCoreStudiesProjectId: - type: integer - description: The unique identifier correlating to a Gateway Project entity indicating that this application is relating to a National Core Studies project. - projectName: - type: string - description: The project name that has been assigned to the application by the applicant(s). - authorIds: - type: array - items: - type: integer - description: An array of values correlating to specific user's via their numeric identifiers. An author is also known as a contributor to an application and can view, edit or submit. - datasetIds: - type: array - items: - type: string - description: An array of values correlating to datasets selected for the application via their identifier, which is unique per version. - datasetTitles: - type: array - items: - type: string - description: An array of strings correlating to the dataset titles that have been selected for the application. - applicationStatus: - type: string - enum: - - inProgress - - submitted - - inReview - - approved - - rejected - - approved with conditions - description: The current status of the application. - jsonSchema: - type: object - description: The object containing the json definition that renders the application form using the Winterfell library. This contains the details of questions, questions sets, question panels, headings and navigation items that appear. - questionAnswers: - type: object - description: The object containing the answers provided on the application form. This consists of a series of key pairs, where the key is the unqiue question Id, and the value the is the answer provided to the question. In the case of a multi select on the form, the value may be an array. - publisher: - type: string - description: The name of the Custodian that holds the dataset and is processing the application. - publisherObj: - type: object - description: The object containing details regarding the Custodian/publisher relating to the application. - userId: - type: integer - description: The unique identifier that correlates to the user account of the main applicant. This is always the user that started the application. - schemaId: - type: string - description: The unique identifier that correlates to the schema from which the application form was generated. - files: - type: array - items: - type: object - description: An array containing the links to files that have been uploaded to the application form and are held within the Gateway ecosystem. - amendmentIterations: - type: array - items: - type: object - description: An array containing an object with details for each iteration the application has passed through. An iteration is defined as an application which has been returned by the Custodian for correction, corrected by the applicant(s) and resubmitted. The object contains dates that the application was returned, and resubmitted as well as reference to any questions that were highlighted for amendment. - createdAt: - type: string - description: The date and time that the application was started. - updatedAt: - type: string - description: The date and time that the application was last updated by any party. - projectId: - type: string - description: The unique identifier for the application converted to a more human friendly format in uppercase and hypenated. - dateSubmitted: - type: string - description: The date and time that the application was originally submitted by the applicant(s) to the Custodian for review. - dateReviewStart: - type: string - description: The date and time that the review process was commenced by a Custodian manager. The review starts from the moment the manager opens the application to triage it. - dateFinalStatus: - type: string - description: The date and time that the Custodian triggered a status change to the application once a final decision was made. E.g. when application was approved. This date can be used in conjunction with the dateReviewStart date to calculate the length of time the Custodian took to make a decision through their review process. - datasets: - type: array - items: - type: object - description: An array containing the full metadata for each of the datasets that have been applied for through this application. - mainApplicant: - type: object - description: An object containing the details of the main applicant of the application as referenced by the userId field. - authors: - type: array - items: - type: object - description: An array containing the details of the contributors of the application as referenced by the authorIds field. - readOnly: - type: boolean - description: A value to indicate if the requesting party is able to modify the application in its present state. For example, this will be false for a Custodian, but true for applicants if the applicant(s) are working on resubmitting the application following a request for amendments. - unansweredAmendments: - type: integer - description: The number of amendments that have been requested by the Custodian in the current amendment iteration. - answeredAmendments: - type: integer - description: The number of requested amendments that the applicant(s) have fixed in the current amendment iteration. - userType: - type: string - enum: - - custodian - - applicant - description: The type of user that has requested the Data Access Request application based on their permissions. It is either an applicant or a Custodian user. - activeParty: - type: string - enum: - - custodian - - applicant - description: The party that is currently handling the application. This is the applicant during presubmission, then the Custodian following submission. The active party then fluctuates between parties during amendment iterations. - inReviewMode: - type: boolean - description: A flag to indicate if the current user is a reviewer of the application. This value will be false unless the requesting user is an assigned reviewer to a currently active workflow step. When this value is true, the requesting user is able to recommend approval or rejection of the application. - reviewSections: - type: array - items: - type: string - description: An array containing the sections of the application form that the current user is required to review if they are a reviewer of the current workflow step that the application is in. E.g. ['Safe People','Safe Data'] - hasRecommended: - type: boolean - description: A flag to indicate if the current user as a reviewer of the current workflow phase has submitted their recommendation for approval or rejection based on their review of the review sections assigned to them. - workflow: - type: object - description: The full details of the workflow that has been assigned to the Data Access Request application. This includes information such as the review phases that the application will pass through and associated metadata. - examples: - 'Approved Application': - value: - { - 'status': 'success', - 'data': - { - 'aboutApplication': - { - 'selectedDatasets': - [ - { - '_id': '5fc31a18d98e4f4cff7e9315', - 'datasetId': 'd5faf9c6-6c34-46d7-93c4-7706a5436ed9', - 'name': 'HDR UK Papers & Preprints', - 'description': "Publications that mention HDR-UK (or any variant thereof) in Acknowledgements or Author Affiliations\n\nThis will include:\n- Papers\n- COVID-19 Papers\n- COVID-19 Preprint", - 'abstract': 'Publications that mention HDR-UK (or any variant thereof) in Acknowledgements or Author Affiliations', - 'publisher': 'OTHER > HEALTH DATA RESEARCH UK', - 'contactPoint': 'hdr.hdr@hdruk.ac.uk', - 'publisherObj': - { - 'dataRequestModalContent': { 'header': ' ', 'body': '{omitted for brevity...}', 'footer': '' }, - 'active': true, - 'allowsMessaging': true, - 'workflowEnabled': true, - '_id': '5f7b1a2bce9f65e6ed83e7da', - 'name': 'OTHER > HEALTH DATA RESEARCH UK', - 'imageURL': '', - 'team': - { - 'active': true, - '_id': '5f7b1a2bce9f65e6ed83e7da', - 'members': - [ - { 'roles': ['manager'], 'memberid': '5f1a98861a821b4a53e44d15' }, - { 'roles': ['manager'], 'memberid': '600bfc99c8bf700f2c7d5c36' }, - ], - 'type': 'publisher', - '__v': 3, - 'createdAt': '2020-11-30T21:12:40.855Z', - 'updatedAt': '2020-12-02T13:33:45.232Z', - }, - }, - }, - ], - 'isNationalCoreStudies': true, - 'nationalCoreStudiesProjectId': '4324836585275824', - 'projectName': 'Test application title', - 'completedDatasetSelection': true, - 'completedInviteCollaborators': true, - 'completedReadAdvice': true, - 'completedCommunicateAdvice': true, - 'completedApprovalsAdvice': true, - 'completedSubmitAdvice': true, - }, - 'authorIds': [], - 'datasetIds': ['d5faf9c6-6c34-46d7-93c4-7706a5436ed9'], - 'datasetTitles': [], - 'applicationStatus': 'approved', - 'jsonSchema': '{omitted for brevity...}', - 'questionAnswers': - { - 'fullname-892140ec730145dc5a28b8fe139c2876': 'James Smith', - 'jobtitle-ff1d692a04b4bb9a2babe9093339136f': 'Consultant', - 'organisation-65c06905b8319ffa29919732a197d581': 'Consulting Inc.', - }, - 'publisher': 'OTHER > HEALTH DATA RESEARCH UK', - '_id': '60142c5b4316a0e0fcd47c56', - 'version': 1, - 'userId': 9190228196797084, - 'schemaId': '5f55e87e780ba204b0a98eb8', - 'files': [], - 'amendmentIterations': [], - 'createdAt': '2021-01-29T15:40:11.943Z', - 'updatedAt': '2021-02-03T14:38:22.688Z', - '__v': 0, - 'projectId': '6014-2C5B-4316-A0E0-FCD4-7C56', - 'dateSubmitted': '2021-01-29T16:30:27.351Z', - 'dateReviewStart': '2021-02-03T14:36:22.341Z', - 'dateFinalStatus': '2021-02-03T14:38:22.680Z', - 'datasets': ['{omitted for brevity...}'], - 'dataset': null, - 'mainApplicant': { '_id': '5f1a98861a821b4a53e44d15', 'firstname': 'James', 'lastname': 'Smith' }, - 'authors': [], - 'id': '60142c5b4316a0e0fcd47c56', - 'readOnly': true, - 'unansweredAmendments': 0, - 'answeredAmendments': 0, - 'userType': 'custodian', - 'activeParty': 'custodian', - 'inReviewMode': false, - 'reviewSections': [], - 'hasRecommended': false, - 'workflow': {}, - }, - } - '404': - description: Failed to find the application requested. - content: - application/json: - schema: - type: object - properties: - status: - type: string - message: - type: string - examples: - 'Not Found': - value: { 'status': 'error', 'message': 'Application not found.' } - '401': - description: Unauthorised attempt to access an application. - content: - application/json: - schema: - type: object - properties: - status: - type: string - message: - type: string - examples: - 'Unauthorised': - value: { 'status': 'failure', 'message': 'Unauthorised' } - put: - tags: - - Data Access Request - parameters: - - in: path - name: id - required: true - description: The unique identifier for a single Data Access Request application. - schema: - type: string - example: 5ee249426136805fbf094eef - description: Update a single Data Access Request application. - requestBody: - content: - application/json: - schema: - type: object - properties: - applicationStatus: - type: string - applicationStatusDesc: - type: string - examples: - 'Update Application Status': - value: { 'applicationStatus': 'approved', 'applicationStatusDesc': 'This application meets all the requirements.' } - responses: - '200': - description: Successful response containing the full, updated data access request application. - content: - application/json: - schema: - type: object - properties: - status: - type: string - data: - type: object - properties: - id: - type: string - description: The unique identifier for the application. - aboutApplication: - description: An object which holds data relating to the 'about application' section of the application form including details of whether the project is an NCS project or not. - type: object - properties: - isNationalCoreStudies: - type: boolean - description: A flag to indicate if this application is in relation to a National Core Studies Project. - nationalCoreStudiesProjectId: - type: integer - description: The unique identifier correlating to a Gateway Project entity indicating that this application is relating to a National Core Studies project. - projectName: - type: string - description: The project name that has been assigned to the application by the applicant(s). - authorIds: - type: array - items: - type: integer - description: An array of values correlating to specific user's via their numeric identifiers. An author is also known as a contributor to an application and can view, edit or submit. - datasetIds: - type: array - items: - type: string - description: An array of values correlating to datasets selected for the application via their identifier, which is unique per version. - datasetTitles: - type: array - items: - type: string - description: An array of strings correlating to the dataset titles that have been selected for the application. - applicationStatus: - type: string - enum: - - inProgress - - submitted - - inReview - - approved - - rejected - - approved with conditions - description: The current status of the application. - jsonSchema: - type: object - description: The object containing the json definition that renders the application form using the Winterfell library. This contains the details of questions, questions sets, question panels, headings and navigation items that appear. - questionAnswers: - type: object - description: The object containing the answers provided on the application form. This consists of a series of key pairs, where the key is the unqiue question Id, and the value the is the answer provided to the question. In the case of a multi select on the form, the value may be an array. - publisher: - type: string - description: The name of the Custodian that holds the dataset and is processing the application. - publisherObj: - type: object - description: The object containing details regarding the Custodian/publisher relating to the application. - userId: - type: integer - description: The unique identifier that correlates to the user account of the main applicant. This is always the user that started the application. - schemaId: - type: string - description: The unique identifier that correlates to the schema from which the application form was generated. - files: - type: array - items: - type: object - description: An array containing the links to files that have been uploaded to the application form and are held within the Gateway ecosystem. - amendmentIterations: - type: array - items: - type: object - description: An array containing an object with details for each iteration the application has passed through. An iteration is defined as an application which has been returned by the Custodian for correction, corrected by the applicant(s) and resubmitted. The object contains dates that the application was returned, and resubmitted as well as reference to any questions that were highlighted for amendment. - createdAt: - type: string - description: The date and time that the application was started. - updatedAt: - type: string - description: The date and time that the application was last updated by any party. - projectId: - type: string - description: The unique identifier for the application converted to a more human friendly format in uppercase and hypenated. - dateSubmitted: - type: string - description: The date and time that the application was originally submitted by the applicant(s) to the Custodian for review. - dateReviewStart: - type: string - description: The date and time that the review process was commenced by a Custodian manager. The review starts from the moment the manager opens the application to triage it. - dateFinalStatus: - type: string - description: The date and time that the Custodian triggered a status change to the application once a final decision was made. E.g. when application was approved. This date can be used in conjunction with the dateReviewStart date to calculate the length of time the Custodian took to make a decision through their review process. - datasets: - type: array - items: - type: object - description: An array containing the full metadata for each of the datasets that have been applied for through this application. - mainApplicant: - type: object - description: An object containing the details of the main applicant of the application as referenced by the userId field. - authors: - type: array - items: - type: object - description: An array containing the details of the contributors of the application as referenced by the authorIds field. - readOnly: - type: boolean - description: A value to indicate if the requesting party is able to modify the application in its present state. For example, this will be false for a Custodian, but true for applicants if the applicant(s) are working on resubmitting the application following a request for amendments. - unansweredAmendments: - type: integer - description: The number of amendments that have been requested by the Custodian in the current amendment iteration. - answeredAmendments: - type: integer - description: The number of requested amendments that the applicant(s) have fixed in the current amendment iteration. - userType: - type: string - enum: - - custodian - - applicant - description: The type of user that has requested the Data Access Request application based on their permissions. It is either an applicant or a Custodian user. - activeParty: - type: string - enum: - - custodian - - applicant - description: The party that is currently handling the application. This is the applicant during presubmission, then the Custodian following submission. The active party then fluctuates between parties during amendment iterations. - inReviewMode: - type: boolean - description: A flag to indicate if the current user is a reviewer of the application. This value will be false unless the requesting user is an assigned reviewer to a currently active workflow step. When this value is true, the requesting user is able to recommend approval or rejection of the application. - reviewSections: - type: array - items: - type: string - description: An array containing the sections of the application form that the current user is required to review if they are a reviewer of the current workflow step that the application is in. E.g. ['Safe People','Safe Data'] - hasRecommended: - type: boolean - description: A flag to indicate if the current user as a reviewer of the current workflow phase has submitted their recommendation for approval or rejection based on their review of the review sections assigned to them. - workflow: - type: object - description: The full details of the workflow that has been assigned to the Data Access Request application. This includes information such as the review phases that the application will pass through and associated metadata. - examples: - 'Approved Application': - value: - { - 'status': 'success', - 'data': - { - 'aboutApplication': - { - 'selectedDatasets': - [ - { - '_id': '5fc31a18d98e4f4cff7e9315', - 'datasetId': 'd5faf9c6-6c34-46d7-93c4-7706a5436ed9', - 'name': 'HDR UK Papers & Preprints', - 'description': "Publications that mention HDR-UK (or any variant thereof) in Acknowledgements or Author Affiliations\n\nThis will include:\n- Papers\n- COVID-19 Papers\n- COVID-19 Preprint", - 'abstract': 'Publications that mention HDR-UK (or any variant thereof) in Acknowledgements or Author Affiliations', - 'publisher': 'OTHER > HEALTH DATA RESEARCH UK', - 'contactPoint': 'hdr.hdr@hdruk.ac.uk', - 'publisherObj': - { - 'dataRequestModalContent': { 'header': ' ', 'body': '{omitted for brevity...}', 'footer': '' }, - 'active': true, - 'allowsMessaging': true, - 'workflowEnabled': true, - '_id': '5f7b1a2bce9f65e6ed83e7da', - 'name': 'OTHER > HEALTH DATA RESEARCH UK', - 'imageURL': '', - 'team': - { - 'active': true, - '_id': '5f7b1a2bce9f65e6ed83e7da', - 'members': - [ - { 'roles': ['manager'], 'memberid': '5f1a98861a821b4a53e44d15' }, - { 'roles': ['manager'], 'memberid': '600bfc99c8bf700f2c7d5c36' }, - ], - 'type': 'publisher', - '__v': 3, - 'createdAt': '2020-11-30T21:12:40.855Z', - 'updatedAt': '2020-12-02T13:33:45.232Z', - }, - }, - }, - ], - 'isNationalCoreStudies': true, - 'nationalCoreStudiesProjectId': '4324836585275824', - 'projectName': 'Test application title', - 'completedDatasetSelection': true, - 'completedInviteCollaborators': true, - 'completedReadAdvice': true, - 'completedCommunicateAdvice': true, - 'completedApprovalsAdvice': true, - 'completedSubmitAdvice': true, - }, - 'authorIds': [], - 'datasetIds': ['d5faf9c6-6c34-46d7-93c4-7706a5436ed9'], - 'datasetTitles': [], - 'applicationStatus': 'approved', - 'jsonSchema': '{omitted for brevity...}', - 'questionAnswers': - { - 'fullname-892140ec730145dc5a28b8fe139c2876': 'James Smith', - 'jobtitle-ff1d692a04b4bb9a2babe9093339136f': 'Consultant', - 'organisation-65c06905b8319ffa29919732a197d581': 'Consulting Inc.', - }, - 'publisher': 'OTHER > HEALTH DATA RESEARCH UK', - '_id': '60142c5b4316a0e0fcd47c56', - 'version': 1, - 'userId': 9190228196797084, - 'schemaId': '5f55e87e780ba204b0a98eb8', - 'files': [], - 'amendmentIterations': [], - 'createdAt': '2021-01-29T15:40:11.943Z', - 'updatedAt': '2021-02-03T14:38:22.688Z', - '__v': 0, - 'projectId': '6014-2C5B-4316-A0E0-FCD4-7C56', - 'dateSubmitted': '2021-01-29T16:30:27.351Z', - 'dateReviewStart': '2021-02-03T14:36:22.341Z', - 'dateFinalStatus': '2021-02-03T14:38:22.680Z', - 'datasets': ['{omitted for brevity...}'], - 'dataset': null, - 'mainApplicant': { '_id': '5f1a98861a821b4a53e44d15', 'firstname': 'James', 'lastname': 'Smith' }, - 'authors': [], - 'id': '60142c5b4316a0e0fcd47c56', - 'readOnly': true, - 'unansweredAmendments': 0, - 'answeredAmendments': 0, - 'userType': 'custodian', - 'activeParty': 'custodian', - 'inReviewMode': false, - 'reviewSections': [], - 'hasRecommended': false, - 'workflow': {}, - }, - } - '404': - description: Failed to find the application requested. - content: - application/json: - schema: - type: object - properties: - status: - type: string - message: - type: string - examples: - 'Not Found': - value: { 'status': 'error', 'message': 'Application not found.' } - '401': - description: Unauthorised attempt to update an application. - content: - application/json: - schema: - type: object - properties: - status: - type: string - message: - type: string - examples: - 'Unauthorised': - value: { 'status': 'error', 'message': 'Unauthorised to perform this update.' } - patch: - summary: Update a users question answers for access request. - security: - - cookieAuth: [] - tags: - - Data Access Request - parameters: - - in: path - name: id - required: true - description: The ID of the datset - schema: - type: string - example: 5ee249426136805fbf094eef - requestBody: - content: - application/json: - schema: - type: object - properties: - questionAnswers: - type: object - examples: - '0': - value: |- - { - "firstName": "Roger" - } - responses: - '200': - description: OK - - /api/v1/publishers/{publisher}/dataaccessrequests: - get: - tags: - - Publishers - parameters: - - in: path - name: publisher - required: true - description: The full name of the Custodian/Publisher, as registered on the Gateway. - schema: - type: string - example: OTHER > HEALTH DATA RESEARCH UK - description: Returns a collection of all Data Access Requests that have been submitted to the Custodian team for review. - responses: - '200': - description: Successful response containing a collection of Data Access Request applications. - content: - application/json: - schema: - type: object - properties: - avgDecisionTime: - type: string - description: The average number of days the Custodian has taken to process applications from submission to decision. - canViewSubmitted: - type: boolean - description: A flag to indicate if the requesting user has permissions to view submitted applications, which are visible only to managers of the Custodian team. Using OAuth2.0 client credentials will return this value as true. - status: - type: string - data: - type: array - items: - type: object - properties: - aboutApplication: - description: An object which holds data relating to the 'about application' section of the application form including details of whether the project is an NCS project or not. - type: object - properties: - isNationalCoreStudies: - type: boolean - description: A flag to indicate if this application is in relation to a National Core Studies Project. - nationalCoreStudiesProjectId: - type: integer - description: The unique identifier correlating to a Gateway Project entity indicating that this application is relating to a National Core Studies project. - projectName: - type: string - description: The project name that has been assigned to the application by the applicant(s). - amendmentIterations: - type: array - items: - type: object - description: An array containing an object with details for each iteration the application has passed through. An iteration is defined as an application which has been returned by the Custodian for correction, corrected by the applicant(s) and resubmitted. The object contains dates that the application was returned, and resubmitted as well as reference to any questions that were highlighted for amendment. - amendmentStatus: - type: string - description: A textual indicator of what state the application is in relating to updates made by the Custodian e.g. if it is awaiting updates from the applicant or if new updates have been submitted by the applicant(s). - applicants: - type: string - description: Concatenated list of applicants names who are contributing to the application. - applicationStatus: - type: string - enum: - - inProgress - - submitted - - inReview - - approved - - rejected - - approved with conditions - description: The current status of the application. - authorIds: - type: array - items: - type: integer - description: An array of values correlating to specific user's via their numeric identifiers. An author is also known as a contributor to an application and can view, edit or submit. - createdAt: - type: string - description: The date and time that the application was started. - datasetIds: - type: array - items: - type: string - description: An array of values correlating to datasets selected for the application via their identifier, which is unique per version. - datasetTitles: - type: array - items: - type: string - description: An array of strings correlating to the dataset titles that have been selected for the application. - datasets: - type: array - items: - type: object - description: An array containing the full metadata for each of the datasets that have been applied for through this application. - dateSubmitted: - type: string - description: The date and time that the application was originally submitted by the applicant(s) to the Custodian for review. - files: - type: array - items: - type: object - description: An array containing the links to files that have been uploaded to the application form and are held within the Gateway ecosystem. - id: - type: string - description: The unique identifier for the application. - - jsonSchema: - type: object - description: The object containing the json definition that renders the application form using the Winterfell library. This contains the details of questions, questions sets, question panels, headings and navigation items that appear. - questionAnswers: - type: object - description: The object containing the answers provided on the application form. This consists of a series of key pairs, where the key is the unqiue question Id, and the value the is the answer provided to the question. In the case of a multi select on the form, the value may be an array. - mainApplicant: - type: object - description: An object containing the details of the main applicant of the application as referenced by the userId field. - projectId: - type: string - description: The unique identifier for the application converted to a more human friendly format in uppercase and hypenated. - projectName: - type: string - description: The project name that has been assigned to the application by the applicant(s). - publisher: - type: string - description: The name of the Custodian that holds the dataset and is processing the application. - publisherObj: - type: object - description: The object containing details regarding the Custodian/publisher relating to the application. - reviewPanels: - type: array - items: - type: string - description: An array containing the sections of the application form that the current user is required to review if they are a reviewer of the current workflow step that the application is in. E.g. ['Safe People','Safe Data'] - schemaId: - type: string - description: The unique identifier that correlates to the schema from which the application form was generated. - updatedAt: - type: string - description: The date and time that the application was last updated by any party. - userId: - type: integer - description: The unique identifier that correlates to the user account of the main applicant. This is always the user that started the application. - deadlinePassed: - type: boolean - description: A flag to indicate if the deadline has passed for the current review phase for this application. - decisionApproved: - type: boolean - description: A flag to indicate if the request users decision as a reviewer of the current workflow phase was positive or negative. i.e. correlating to approval or rejection recommendation. - decisionComments: - type: string - description: A supporting note or comment made by the requesting user as context to their decision as a reviewer of the current workflow phase. - decisionDate: - type: string - description: The date that the requesting user made their decision as a reviewer of the current workflow phase. - decisionDuration: - type: integer - description: The number of days from submission until a final decision was made on the application. i.e. the application status was changed to a final status e.g. 'Approved'. - decisionMade: - type: boolean - description: A flag to indicate if the requesting user has made an expected decision as a reviewer of the current workflow phase. - decisionStatus: - type: string - description: A message indicating if the requesting user as a reviewer of the application has made a decision or is still required to make a decision for the current work flow. - isReviewer: - type: boolean - description: A flag to indicate if the requesting user is a reviewer of the current workflow step for the application. - remainingActioners: - type: array - items: - type: string - description: An array containing the names of Custodian team reviewers expected to complete a review for the current workflow phase, or a list of managers if the application is awaiting a final decision. - reviewStatus: - type: string - description: A message indicating the current status of the application review in relation to the assigned workflow. E.g. 'Final decision required' or 'Deadline is today'. This message changes based on the requesting user's relationship to the application. E.g. if they are a reviewer or manager. - stepName: - type: string - description: The name of the current workflow step that the application is in. - workflowCompleted: - type: boolean - description: A flag to indicate if the assigned workflow for the review process has been completed. - workflowName: - type: string - description: The name of the workflow the Custodian team have assigned to the application for the review process. - examples: - 'Single Request Received': - value: - { - 'success': true, - 'data': - [ - { - 'authorIds': [], - 'datasetIds': ['d5faf9c6-6c34-46d7-93c4-7706a5436ed9'], - 'datasetTitles': [], - 'applicationStatus': 'submitted', - 'jsonSchema': '{omitted for brevity...}', - 'questionAnswers': '{omitted for brevity...}', - 'publisher': 'OTHER > HEALTH DATA RESEARCH UK', - '_id': '601853db22dc004f9adfaa24', - 'version': 1, - 'userId': 7584453789581072, - 'schemaId': '5f55e87e780ba204b0a98eb8', - 'files': - [ - { - 'error': '', - '_id': '601aacf8ecdfa66e5cbc2742', - 'status': 'UPLOADED', - 'description': 'QuestionAnswers', - 'fileId': '9e76ee1a676f423b9b5c7aabf59c69db', - 'size': 509984, - 'name': 'QuestionAnswersFlags.png', - 'owner': '5ec7f1b39219d627e5cafae3', - }, - { - 'error': '', - '_id': '601aadbcecdfa6c532bc2743', - 'status': 'UPLOADED', - 'description': 'Notifications', - 'fileId': 'adb1718dcc094b9cb4b0ab347ad2ee94', - 'size': 54346, - 'name': 'HQIP-Workflow-Assigned-Notification.png', - 'owner': '5ec7f1b39219d627e5cafae3', - }, - ], - 'amendmentIterations': [], - 'createdAt': '2021-02-01T19:17:47.470Z', - 'updatedAt': '2021-02-03T16:36:36.720Z', - '__v': 2, - 'projectId': '6018-53DB-22DC-004F-9ADF-AA24', - 'aboutApplication': - { - 'selectedDatasets': - [ - { - '_id': '5fc31a18d98e4f4cff7e9315', - 'datasetId': 'd5faf9c6-6c34-46d7-93c4-7706a5436ed9', - 'name': 'HDR UK Papers & Preprints', - 'description': "Publications that mention HDR-UK (or any variant thereof) in Acknowledgements or Author Affiliations\n\nThis will include:\n- Papers\n- COVID-19 Papers\n- COVID-19 Preprint", - 'abstract': 'Publications that mention HDR-UK (or any variant thereof) in Acknowledgements or Author Affiliations', - 'publisher': 'OTHER > HEALTH DATA RESEARCH UK', - 'contactPoint': 'hdr.hdr@hdruk.ac.uk', - 'publisherObj': - { - 'dataRequestModalContent': { 'header': ' ', 'body': '{omitted for brevity...}', 'footer': '' }, - 'active': true, - 'allowsMessaging': true, - 'workflowEnabled': true, - '_id': '5f7b1a2bce9f65e6ed83e7da', - 'name': 'OTHER > HEALTH DATA RESEARCH UK', - 'imageURL': '', - 'team': - { - 'active': true, - '_id': '5f7b1a2bce9f65e6ed83e7da', - 'members': - [ - { 'roles': ['manager'], 'memberid': '5f1a98861a821b4a53e44d15' }, - { 'roles': ['manager'], 'memberid': '600bfc99c8bf700f2c7d5c36' }, - ], - 'type': 'publisher', - '__v': 3, - 'createdAt': '2020-11-30T21:12:40.855Z', - 'updatedAt': '2020-12-02T13:33:45.232Z', - }, - }, - }, - ], - 'isNationalCoreStudies': true, - 'nationalCoreStudiesProjectId': '4324836585275824', - 'projectName': 'Test application title', - 'completedDatasetSelection': true, - 'completedInviteCollaborators': true, - 'completedReadAdvice': true, - 'completedCommunicateAdvice': true, - 'completedApprovalsAdvice': true, - 'completedSubmitAdvice': true, - }, - 'dateSubmitted': '2021-02-03T16:37:36.081Z', - 'datasets': - [ - { - 'categories': { 'programmingLanguage': [] }, - 'tags': { 'features': ['Preprints', 'Papers', 'HDR UK'], 'topics': [] }, - 'datasetfields': - { - 'geographicCoverage': ['https://www.geonames.org/countries/GB/united-kingdom.html'], - 'physicalSampleAvailability': ['Not Available'], - 'technicaldetails': '{omitted for brevity...}', - 'versionLinks': - [ - { - 'id': '142b1618-2691-4019-97b4-16b1e27c5f95', - 'linkType': 'Superseded By', - 'domainType': 'CatalogueSemanticLink', - 'source': - { - 'id': '9e798632-442a-427b-8d0e-456f754d28dc', - 'domainType': 'DataModel', - 'label': 'HDR UK Papers & Preprints', - 'documentationVersion': '0.0.1', - }, - 'target': - { - 'id': 'd5faf9c6-6c34-46d7-93c4-7706a5436ed9', - 'domainType': 'DataModel', - 'label': 'HDR UK Papers & Preprints', - 'documentationVersion': '1.0.0', - }, - }, - ], - 'phenotypes': [], - 'publisher': 'OTHER > HEALTH DATA RESEARCH UK', - 'abstract': 'Publications that mention HDR-UK (or any variant thereof) in Acknowledgements or Author Affiliations', - 'releaseDate': '2020-11-27T00:00:00Z', - 'accessRequestDuration': 'Other', - 'conformsTo': 'OTHER', - 'accessRights': 'https://github.com/HDRUK/papers/blob/master/LICENSE', - 'jurisdiction': 'GB-ENG', - 'datasetStartDate': '2020-03-31', - 'datasetEndDate': '2022-04-30', - 'statisticalPopulation': '0', - 'ageBand': '0-0', - 'contactPoint': 'hdr.hdr@hdruk.ac.uk', - 'periodicity': 'Daily', - 'metadataquality': - { - 'id': 'd5faf9c6-6c34-46d7-93c4-7706a5436ed9', - 'publisher': 'OTHER > HEALTH DATA RESEARCH UK', - 'title': 'HDR UK Papers & Preprints', - 'completeness_percent': 95.24, - 'weighted_completeness_percent': 100, - 'error_percent': 11.63, - 'weighted_error_percent': 19.05, - 'quality_score': 91.81, - 'quality_rating': 'Gold', - 'weighted_quality_score': 90.47, - 'weighted_quality_rating': 'Gold', - }, - 'datautility': - { - 'id': 'd5faf9c6-6c34-46d7-93c4-7706a5436ed9', - 'publisher': 'OTHER > HEALTH DATA RESEARCH UK', - 'title': 'HDR UK Papers & Preprints', - 'metadata_richness': 'Gold', - 'availability_of_additional_documentation_and_support': '', - 'data_model': '', - 'data_dictionary': '', - 'provenance': '', - 'data_quality_management_process': '', - 'dama_quality_dimensions': '', - 'pathway_coverage': '', - 'length_of_follow_up': '', - 'allowable_uses': '', - 'research_environment': '', - 'time_lag': '', - 'timeliness': '', - 'linkages': '', - 'data_enrichments': '', - }, - 'metadataschema': - { - '@context': 'http://schema.org/', - '@type': 'Dataset', - 'identifier': 'd5faf9c6-6c34-46d7-93c4-7706a5436ed9', - 'url': 'https://healthdatagateway.org/detail/d5faf9c6-6c34-46d7-93c4-7706a5436ed9', - 'name': 'HDR UK Papers & Preprints', - 'description': "Publications that mention HDR-UK (or any variant thereof) in Acknowledgements or Author Affiliations\n\nThis will include:\n- Papers\n- COVID-19 Papers\n- COVID-19 Preprint", - 'license': 'Open Access', - 'keywords': - [ - 'Preprints,Papers,HDR UK', - 'OTHER > HEALTH DATA RESEARCH UK', - 'NOT APPLICABLE', - 'GB-ENG', - 'https://www.geonames.org/countries/GB/united-kingdom.html', - ], - 'includedinDataCatalog': - [ - { - '@type': 'DataCatalog', - 'name': 'OTHER > HEALTH DATA RESEARCH UK', - 'url': 'hdr.hdr@hdruk.ac.uk', - }, - { - '@type': 'DataCatalog', - 'name': 'HDR UK Health Data Gateway', - 'url': 'http://healthdatagateway.org', - }, - ], - }, - }, - 'authors': [], - 'showOrganisation': false, - 'toolids': [], - 'datasetids': [], - '_id': '5fc31a18d98e4f4cff7e9315', - 'relatedObjects': [], - 'programmingLanguage': [], - 'pid': 'b7a62c6d-ed00-4423-ad27-e90b71222d8e', - 'datasetVersion': '1.0.0', - 'id': 9816147066244124, - 'datasetid': 'd5faf9c6-6c34-46d7-93c4-7706a5436ed9', - 'type': 'dataset', - 'activeflag': 'active', - 'name': 'HDR UK Papers & Preprints', - 'description': "Publications that mention HDR-UK (or any variant thereof) in Acknowledgements or Author Affiliations\n\nThis will include:\n- Papers\n- COVID-19 Papers\n- COVID-19 Preprint", - 'license': 'Open Access', - 'datasetv2': - { - 'identifier': '', - 'version': '', - 'issued': '', - 'modified': '', - 'revisions': [], - 'summary': - { - 'title': '', - 'abstract': 'Publications that mention HDR-UK (or any variant thereof) in Acknowledgements or Author Affiliations', - 'publisher': - { - 'identifier': '', - 'name': 'HEALTH DATA RESEARCH UK', - 'logo': '', - 'description': '', - 'contactPoint': 'hdr.hdr@hdruk.ac.uk', - 'memberOf': 'OTHER', - 'accessRights': [], - 'deliveryLeadTime': '', - 'accessService': '', - 'accessRequestCost': '', - 'dataUseLimitation': [], - 'dataUseRequirements': [], - }, - 'contactPoint': 'hdr.hdr@hdruk.ac.uk', - 'keywords': ['Preprints', 'Papers', 'HDR UK'], - 'alternateIdentifiers': [], - 'doiName': 'https://doi.org/10.5281/zenodo.326615', - }, - 'documentation': - { - 'description': '', - 'associatedMedia': ['https://github.com/HDRUK/papers'], - 'isPartOf': 'NOT APPLICABLE', - }, - 'coverage': - { - 'spatial': 'GB', - 'typicalAgeRange': '0-0', - 'physicalSampleAvailability': ['NOT AVAILABLE'], - 'followup': 'UNKNOWN', - 'pathway': 'NOT APPLICABLE', - }, - 'provenance': - { - 'origin': { 'purpose': 'OTHER', 'source': 'MACHINE GENERATED', 'collectionSituation': 'OTHER' }, - 'temporal': - { - 'accrualPeriodicity': 'DAILY', - 'distributionReleaseDate': '2020-11-27', - 'startDate': '2020-03-31', - 'endDate': '2022-04-30', - 'timeLag': 'NO TIMELAG', - }, - }, - 'accessibility': - { - 'usage': - { - 'dataUseLimitation': 'GENERAL RESEARCH USE', - 'dataUseRequirements': 'RETURN TO DATABASE OR RESOURCE', - 'resourceCreator': 'HDR UK Using Team', - 'investigations': ['https://github.com/HDRUK/papers'], - 'isReferencedBy': ['Not Available'], - }, - 'access': - { - 'accessRights': ['Open Access'], - 'accessService': 'https://github.com/HDRUK/papers', - 'accessRequestCost': 'Free', - 'deliveryLeadTime': 'OTHER', - 'jurisdiction': 'GB-ENG', - 'dataProcessor': 'HDR UK', - 'dataController': 'HDR UK', - }, - 'formatAndStandards': - { - 'vocabularyEncodingScheme': 'OTHER', - 'conformsTo': 'OTHER', - 'language': 'en', - 'format': ['csv', 'JSON'], - }, - }, - 'enrichmentAndLinkage': - { - 'qualifiedRelation': ['Not Available'], - 'derivation': ['Not Available'], - 'tools': ['https://github.com/HDRUK/papers'], - }, - 'observations': [], - }, - 'createdAt': '2020-11-29T03:48:41.794Z', - 'updatedAt': '2021-02-02T10:09:57.030Z', - '__v': 0, - 'counter': 20, - }, - ], - 'dataset': null, - 'mainApplicant': - { - 'isServiceAccount': false, - '_id': '5ec7f1b39219d627e5cafae3', - 'id': 7584453789581072, - 'providerId': '112563375053074694443', - 'provider': 'google', - 'firstname': 'Chris', - 'lastname': 'Marks', - 'email': 'chris.marks@paconsulting.com', - 'role': 'Admin', - '__v': 0, - 'redirectURL': '/tool/100000012', - 'discourseKey': '2f52ecaa21a0d0223a119da5a09f8f8b09459e7b69ec3f981102d09f66488d99', - 'discourseUsername': 'chris.marks', - 'updatedAt': '2021-02-01T12:39:56.372Z', - }, - 'publisherObj': - { - 'dataRequestModalContent': { 'header': '', 'body': '', 'footer': '' }, - 'active': true, - 'allowsMessaging': true, - 'workflowEnabled': true, - '_id': '5f7b1a2bce9f65e6ed83e7da', - 'name': 'OTHER > HEALTH DATA RESEARCH UK', - 'imageURL': '', - 'team': - { - 'active': true, - '_id': '5f7b1a2bce9f65e6ed83e7da', - 'members': - [ - { 'roles': ['manager'], 'memberid': '5f1a98861a821b4a53e44d15' }, - { 'roles': ['manager'], 'memberid': '600bfc99c8bf700f2c7d5c36' }, - ], - 'type': 'publisher', - '__v': 3, - 'createdAt': '2020-11-30T21:12:40.855Z', - 'updatedAt': '2020-12-02T13:33:45.232Z', - 'users': - [ - { '_id': '5f1a98861a821b4a53e44d15', 'firstname': 'Robin', 'lastname': 'Kavanagh' }, - { '_id': '600bfc99c8bf700f2c7d5c36', 'firstname': 'HDR-UK', 'lastname': 'Service Account' }, - ], - }, - }, - 'id': '601853db22dc004f9adfaa24', - 'projectName': 'PA Paper', - 'applicants': 'Chris Marks', - 'workflowName': '', - 'workflowCompleted': false, - 'decisionDuration': '', - 'decisionMade': false, - 'decisionStatus': '', - 'decisionComments': '', - 'decisionDate': '', - 'decisionApproved': false, - 'remainingActioners': 'Robin Kavanagh (you), HDR-UK Service Account', - 'stepName': '', - 'deadlinePassed': '', - 'reviewStatus': '', - 'isReviewer': false, - 'reviewPanels': [], - 'amendmentStatus': '', - }, - ], - 'avgDecisionTime': 1, - 'canViewSubmitted': true, - } - '404': - description: Failed to find the application requested. - content: - application/json: - schema: - type: object - properties: - success: - type: boolean - examples: - 'Not Found': - value: { 'success': false } - '401': - description: Unauthorised attempt to access an application. - content: - application/json: - schema: - type: object - properties: - status: - type: string - message: - type: string - examples: - 'Unauthorised': - value: { 'status': 'failure', 'message': 'Unauthorised' } - - /api/v1/data-access-request/{datasetID}: - get: - summary: Returns access request template. - security: - - cookieAuth: [] - tags: - - Data Access Request - parameters: - - in: path - name: datasetID - required: true - description: The ID of the datset - schema: - type: string - example: 6efbc62f-6ebb-4f18-959b-1ec6fd0cc6fb - responses: - '200': - description: OK - - /api/v1/person/{id}: - get: - summary: Returns details for a person. - tags: - - Person - parameters: - - in: path - name: id - required: true - description: The ID of the person - schema: - type: string - example: 900000014 - responses: - '200': - description: OK - - /api/v1/person: - get: - summary: Returns an array of person objects. - tags: - - Person - responses: - '200': - description: OK - post: - summary: Returns a new person object. - tags: - - Person - requestBody: - content: - application/json: - schema: # Request body contents - type: object - required: - - firstname - - lastname - - bio - - link - - orcid - - emailNotifications - - terms - properties: - firstname: - type: string - lastname: - type: string - bio: - type: string - link: - type: string - orcid: - type: string - emailNotifications: - type: boolean - terms: - type: boolean - example: - firstname: 'John' - lastname: 'Smith' - bio: 'Researcher' - link: 'http://google.com' - orcid: 'https://orcid.org/123456789' - emailNotifications: false - terms: true - responses: - '200': - description: OK - put: - summary: Returns edited person object. - tags: - - Person - responses: - '200': - description: OK - requestBody: - content: - application/json: - schema: # Request body contents - type: object - required: - - id - - bio - - link - - orcid - - emailNotifications - - terms - properties: - id: - type: string - bio: - type: string - link: - type: string - orcid: - type: string - emailNotifications: - type: boolean - terms: - type: boolean - example: - id: '5268590523943617' - bio: 'Research assistant' - link: 'http://google.com' - orcid: 'https://orcid.org/123456789' - emailNotifications: false - terms: true - - /api/v1/search: - get: - tags: - - Search - summary: Search for HDRUK /search?search - parameters: - - in: query - name: params - schema: - type: object - properties: - search: - type: string - example: Epilepsy - type: - type: string - example: all - category: - type: string - example: API - programmingLanguage: - type: string - example: Javascript - features: - type: string - example: Arbitrage - topics: - type: string - example: Epilepsy - startIndex: - type: string - example: 0 - maxResults: - type: string - example: 10 - style: form - explode: true - responses: - '200': - description: OK - - /api/v1/stats/topSearches: - get: - summary: Returns top searches for a given month and year. - tags: - - Stats - parameters: - - name: month - in: query - required: true - description: Month number. - schema: - type: string - example: 7 - - name: year - in: query - required: true - description: Year. - schema: - type: string - example: 2020 - responses: - '200': - description: OK - - /api/v1/stats: - get: - summary: Returns the details on recent searches, popular objects, unmet demands or recently updated objects based on the rank query parameter. - tags: - - Stats - parameters: - - name: rank - in: query - required: true - description: The type of stat. - schema: - type: string - example: unmet - - name: type - in: query - required: true - description: Resource type. - schema: - type: string - example: Tools - - name: month - in: query - required: true - description: Month number. - schema: - type: string - example: 7 - - name: year - in: query - required: true - description: Year. - schema: - type: string - example: 2020 - responses: - '200': - description: OK - - /api/v1/kpis: - get: - summary: Returns information for KPIs, based on the KPI type and selectedDate parameters. - tags: - - KPIs - parameters: - - name: type - in: query - required: true - description: The type of KPI. - schema: - type: string - example: uptime - - name: selectedDate - in: query - required: true - description: Full date time string. - schema: - type: string - example: Wed Jul 01 2020 01:00:00 GMT 0100 (British Summer Time) - responses: - '200': - description: OK - - /api/v1/messages/{id}: - delete: - summary: Delete a Message - security: - - cookieAuth: [] - tags: - - Messages - parameters: - - in: path - name: id - required: true - description: The ID of the Message - schema: - type: string - example: '5ee249426136805fbf094eef' - responses: - '204': - description: Ok - put: - summary: Update a single Message - security: - - cookieAuth: [] - tags: - - Messages - parameters: - - in: path - name: id - required: true - description: The ID of the Message - schema: - type: string - example: '5ee249426136805fbf094eef' - requestBody: - content: - application/json: - schema: - type: object - properties: - isRead: - type: boolean - examples: - 'Update message to read': - value: |- - { - "isRead": true - } - responses: - '204': - description: OK - - /api/v1/messages/unread/count: - get: - summary: Returns the number of unread messages for the authenticated user - security: - - cookieAuth: [] - tags: - - Messages - responses: - '200': - description: OK - - /api/v1/messages: - post: - summary: Returns a new Message object and creates an associated parent Topic if a Topic is not specified in request body - security: - - cookieAuth: [] - tags: - - Messages - requestBody: - required: true - content: - application/json: - schema: - type: object - properties: - isRead: - type: boolean - messageDescription: - type: string - messageType: - type: string - required: - - isRead - - messageDescription - - messageType - examples: - 'Create new message': - value: |- - { - "isRead": false, - "messageDescription": "this is an example", - "messageType": "message" - } - responses: - '201': - description: OK - - /api/v1/topics: - post: - summary: Returns a new Topic object with ID (Does not create any associated messages) - security: - - cookieAuth: [] - tags: - - Topics - requestBody: - required: true - content: - application/json: - schema: - type: object - properties: - relatedObjectIds: - type: array - items: - type: string - examples: - 'Create a new topic': - value: |- - { - "relatedObjectIds": "['1','2','3']" - } - responses: - '201': - description: A new Topic - content: - application/json: - schema: - type: object - properties: - _id: - type: object - description: Generated ID - title: - type: string - description: Title of message - subtitle: - type: string - description: Subtitle of message - relatedObjectIds: - type: array - items: - type: string - description: Object ID this Topic is related to - createdBy: - type: object - description: User that created the topic - createdDate: - type: string - description: Date the topic was created - recipients: - type: array - items: - type: string - description: Collection of user IDs - tags: - type: array - items: - type: string - description: Collection of tags to describe topic - get: - summary: Returns a list of all topics that the authenticated user is a recipient or member of - security: - - cookieAuth: [] - tags: - - Topics - responses: - '200': - description: Ok - - /api/v1/topics/{id}: - get: - summary: Returns Topic object by ID - security: - - cookieAuth: [] - tags: - - Topics - parameters: - - in: path - name: id - required: true - description: The ID of the topic - schema: - type: string - example: '5ee249426136805fbf094eef' - responses: - '200': - description: Ok - delete: - summary: Soft deletes a message Topic but does not affect associated messages - security: - - cookieAuth: [] - tags: - - Topics - parameters: - - in: path - name: id - required: true - description: The ID of the Topic - schema: - type: string - example: '5ee249426136805fbf094eef' - responses: - '204': - description: Ok - - /api/v1/datasets/{datasetID}: - get: - summary: Returns Dataset object. - tags: - - Datasets - parameters: - - in: path - name: datasetID - required: true - description: The ID of the datset - schema: - type: string - example: '756daeaa-6e47-4269-9df5-477c01cdd271' - responses: - '200': - description: OK - - /api/v1/datasets: - get: - summary: Returns List of Dataset objects. - tags: - - Datasets - parameters: - - in: query - name: limit - required: false - description: Limit the number of results - schema: - type: integer - example: 3 - - in: query - name: offset - required: false - description: Index to offset the search results - schema: - type: integer - example: 1 - - in: query - name: q - required: false - description: Filter using search query - schema: - type: string - example: epilepsy - responses: - '200': - description: OK - - /api/v2/datasets: - get: - summary: Returns a list of dataset objects - tags: - - Datasets v2.0 - description: Version 2.0 of the datasets API introduces a large number of parameterised query string options to aid requests in collecting the data that is most relevant for a given use case. The query parameters defined below support a variety of comparison operators such as equals, contains, greater than, and less than. Using dot notation, any field can be queried, please see some examples below. Note - This response is limited to 100 records by default. Please use the 'page' query parameter to access records beyond the first 100. The 'limit' query parameter can therefore only be specified up to a maximum of 100. - parameters: - - name: search - in: query - description: Full text index search function which searches for partial matches in various dataset fields including name, description and abstract. The response will contain a metascore indicating the relevancy of the match, by default results are sorted by the most relevant first unless a manual sort query parameter has been added. - schema: - type: string - example: COVID-19 - - name: page - in: query - description: A specific page of results to retrieve - schema: - type: number - example: 1 - - name: limit - in: query - description: Maximum number of results returned per page - schema: - type: number - example: 10 - - name: sort - in: query - description: Fields to apply sort operations to. Accepts multiple fields in ascending and descending. E.g. name for ascending or -name for descending. Multiple fields should be comma separated as shown in the example below. - schema: - type: string - example: datasetfields.publisher,name,-counter - - name: fields - in: query - description: Limit the size of the response by requesting only certain fields. Note that some additional derived fields are always returned. Multiple fields should be comma separate as shown in the example below. - schema: - type: string - example: name,counter,datasetid - - name: count - in: query - description: Returns the number of the number of entities matching the query parameters provided instead of the result payload - schema: - type: boolean - example: true - - name: datasetid - in: query - description: Filter by the unique identifier for a single version of a dataset - schema: - type: string - example: 0cfe60cd-038d-4c03-9a95-894c52135922 - - name: pid - in: query - description: Filter by the identifier for a dataset that persists across versions - schema: - type: string - example: 621dd611-adcf-4434-b538-eecdbe5f72cf - - name: name - in: query - description: Filter by dataset name - schema: - type: string - example: ARIA Dataset - - name: activeflag - in: query - description: Filter by the status of a single dataset version - schema: - type: string - enum: - - active - - archive - example: active - - name: datasetfields.publisher - in: query - description: Filter by the name of the Custodian holding the dataset - schema: - type: string - example: ALLIANCE > BARTS HEALTH NHS TRUST - - name: metadataquality.completeness_percent[gte] - in: query - description: Filter by the metadata quality completeness percentage using an operator [gte] for greater than or equal to, [gt] for greater than, [lte] for less than or equal to, [lt] for less than, and [eq] for equal to. - schema: - type: number - example: 90.5 - - name: metadataquality.weighted_completeness_percent[gte] - in: query - description: Filter by the metadata quality weighted completeness percentage using an operator [gte] for greater than or equal to, [gt] for greater than, [lte] for less than or equal to, [lt] for less than, and [eq] for equal to. - schema: - type: number - example: 71.2 - - name: metadataquality.weighted_quality_score[gte] - in: query - description: Filter by the metadata quality score using an operator [gte] for greater than or equal to, [gt] for greater than, [lte] for less than or equal to, [lt] for less than, and [eq] for equal to. - schema: - type: number - example: 35.3 - responses: - '200': - description: Successful response containing a list of datasets matching query parameters - - /api/v2/datasets/{datasetid}: - get: - summary: Returns a dataset object. - tags: - - Datasets v2.0 - parameters: - - in: path - name: datasetid - required: true - description: The unqiue identifier for a specific version of a dataset - schema: - type: string - example: af20ebb2-018a-4557-8ced-0bec75dba150 - - in: query - name: raw - required: false - description: A flag which determines if the response triggered is the raw structure in which the data is stored rather than the dataset v2.0 standard - schema: - type: boolean - example: false - description: Version 2.0 of the datasets API introduces the agreed dataset v2.0 schema as defined at the following link - https://github.com/HDRUK/schemata/edit/master/schema/dataset/2.0.0/dataset.schema.json - responses: - '200': - description: Successful response containing a single dataset object - '404': - description: A dataset could not be found by the provided dataset identifier - - /api/v1/projects: - post: - summary: Returns a Project object with ID. - security: - - cookieAuth: [] - tags: - - Projects - requestBody: - content: - application/json: - schema: # Request body contents - type: object - required: - - name - properties: - type: - type: string - name: - type: string - link: - type: string - description: - type: string - categories: - type: object - properties: - category: - type: string - programmingLanguage: - type: array - items: - type: string - programmingLanguageVersion: - type: string - licence: - type: string - authors: - type: array - items: - type: number - tags: - type: object - properties: - features: - type: array - items: - type: string - topics: - type: array - items: - type: string - example: # Sample object - type: 'project' - name: 'Epilepsy data research' - link: 'http://epilepsy.org' - description: 'Epilespy data research description' - categories: { category: 'API', programmingLanguage: ['Javascript'], programmingLanguageVersion: '0.0.0' } - licence: 'MIT licence' - authors: [4495285946631793] - tags: { features: ['Arbitrage'], topics: ['Epilepsy'] } - responses: - '200': - description: OK - get: - summary: Returns List of Project objects. - tags: - - Projects - parameters: - - in: query - name: limit - required: false - description: Limit the number of results - schema: - type: integer - example: 3 - - in: query - name: offset - required: false - description: Index to offset the search results - schema: - type: integer - example: 1 - - in: query - name: q - required: false - description: Filter using search query - schema: - type: string - example: epilepsy - responses: - '200': - description: OK - - /api/v1/projects/{id}: - get: - summary: Returns Project object. - tags: - - Projects - parameters: - - in: path - name: id - required: true - schema: - type: integer - example: 441788967946948 - responses: - '200': - description: OK - patch: - summary: Change status of the Project object. - security: - - cookieAuth: [] - tags: - - Projects - parameters: - - in: path - name: id - required: true - schema: - type: integer - example: 662346984100503 - requestBody: - content: - application/json: - schema: # Request body contents - type: object - required: - - name - properties: - activeflag: - type: string - example: # Sample object - activeflag: 'active' - responses: - '200': - description: OK - put: - summary: Returns edited Project object. - security: - - cookieAuth: [] - tags: - - Projects - parameters: - - in: path - name: id - required: true - description: The ID of the project - schema: - type: integer - format: int64 - example: 26542005388306332 - requestBody: - content: - application/json: - schema: # Request body contents - type: object - required: - - name - properties: - id: - type: number - type: - type: string - name: - type: string - link: - type: string - description: - type: string - categories: - type: object - properties: - category: - type: string - programmingLanguage: - type: array - items: - type: string - programmingLanguageVersion: - type: string - licence: - type: string - authors: - type: array - items: - type: number - tags: - type: object - properties: - features: - type: array - items: - type: string - topics: - type: array - items: - type: string - toolids: - type: array - items: - type: string - example: # Sample object - id: 26542005388306332 - type: 'project' - name: 'Research Data TEST EPILEPSY' - link: 'http://localhost:8080/epilepsy' - description: 'Epilespy data research description' - categories: { category: 'API', programmingLanguage: ['Javascript'], programmingLanguageVersion: '1.0.0' } - licence: 'MIT licence' - authors: [4495285946631793] - tags: { features: ['Arbitrage'], topics: ['Epilepsy'] } - toolids: [] - responses: - '200': - description: OK - - /api/v2/projects: - get: - summary: Returns a list of project objects - tags: - - Projects v2.0 - parameters: - - name: search - in: query - description: Full text index search function which searches for partial matches in various fields including name and description. The response will contain a metascore indicating the relevancy of the match, by default results are sorted by the most relevant first unless a manual sort query parameter has been added. - schema: - type: string - example: health service - - name: page - in: query - description: A specific page of results to retrieve - schema: - type: number - example: 1 - - name: limit - in: query - description: Maximum number of results returned per page - schema: - type: number - example: 10 - - name: sort - in: query - description: Fields to apply sort operations to. Accepts multiple fields in ascending and descending. E.g. name for ascending or -name for descending. Multiple fields should be comma separated as shown in the example below. - schema: - type: string - example: name,-counter - - name: fields - in: query - description: Limit the size of the response by requesting only certain fields. Note that some additional derived fields are always returned. Multiple fields should be comma separate as shown in the example below. - schema: - type: string - example: name,counter,description - - name: count - in: query - description: Returns the number of the number of entities matching the query parameters provided instead of the result payload - schema: - type: boolean - example: true - description: Version 2.0 of the courses API introduces a large number of parameterised query string options to aid requests in collecting the data that is most relevant for a given use case. The query parameters defined below support a variety of comparison operators such as equals, contains, greater than, and less than. Using dot notation, any field can be queried, please see some examples below. Note - This response is limited to 100 records by default. Please use the 'page' query parameter to access records beyond the first 100. The 'limit' query parameter can therefore only be specified up to a maximum of 100. - responses: - '200': - description: Successful response containing a list of projects matching query parameters - - /api/v2/projects/{id}: - get: - summary: Returns a project object - tags: - - Projects v2.0 - parameters: - - in: path - name: id - required: true - description: The ID of the project - schema: - type: number - example: 100000001 - description: Returns a project object by matching unique identifier in the default format that is stored as within the Gateway - responses: - '200': - description: Successful response containing a single project object - '404': - description: A project could not be found by the provided project identifier - - /api/v1/papers: - post: - summary: Returns a Paper object with ID. - security: - - cookieAuth: [] - tags: - - Papers - requestBody: - content: - application/json: - schema: # Request body contents - type: object - required: - - name - properties: - type: - type: string - name: - type: string - link: - type: string - description: - type: string - categories: - type: object - properties: - category: - type: string - programmingLanguage: - type: array - items: - type: string - programmingLanguageVersion: - type: string - licence: - type: string - authors: - type: array - items: - type: number - tags: - type: object - properties: - features: - type: array - items: - type: string - topics: - type: array - items: - type: string - example: # Sample object - type: 'paper' - name: 'Epilepsy data research' - link: 'http://epilepsy.org' - description: 'Epilespy data research description' - categories: { category: 'API', programmingLanguage: ['Javascript'], programmingLanguageVersion: '0.0.0' } - licence: 'MIT licence' - authors: [4495285946631793] - tags: { features: ['Arbitrage'], topics: ['Epilepsy'] } - responses: - '200': - description: OK - get: - summary: Return List of Paper objects. - tags: - - Papers - parameters: - - in: query - name: limit - required: false - description: Limit the number of results - schema: - type: integer - example: 3 - - in: query - name: offset - required: false - description: Index to offset the search results - schema: - type: integer - example: 1 - - in: query - name: q - required: false - description: Filter using search query - schema: - type: string - example: epilepsy - responses: - '200': - description: OK - - /api/v1/papers/{id}: - get: - summary: Returns Paper object. - tags: - - Papers - parameters: - - name: id - in: path - required: true - description: The ID of the user - schema: - type: integer - format: int64 - minimum: 1 - example: 8370396016757367 - responses: - '200': - description: OK - patch: - summary: Change status of the Paper object. - security: - - cookieAuth: [] - tags: - - Papers - parameters: - - in: path - name: id - required: true - schema: - type: integer - example: 7485531672584456 - requestBody: - content: - application/json: - schema: # Request body contents - type: object - required: - - name - properties: - id: - type: number - activeflag: - type: string - example: # Sample object - activeflag: 'active' - responses: - '200': - description: OK - put: - summary: Returns edited Paper object. - security: - - cookieAuth: [] - tags: - - Papers - parameters: - - in: path - name: id - required: true - description: The ID of the paper - schema: - type: integer - format: int64 - example: 7485531672584456 - requestBody: - content: - application/json: - schema: # Request body contents - type: object - required: - - name - properties: - id: - type: number - type: - type: string - name: - type: string - link: - type: string - description: - type: string - categories: - type: object - properties: - category: - type: string - programmingLanguage: - type: array - items: - type: string - programmingLanguageVersion: - type: string - licence: - type: string - authors: - type: array - items: - type: number - tags: - type: object - properties: - features: - type: array - items: - type: string - topics: - type: array - items: - type: string - toolids: - type: array - items: - type: string - example: # Sample object - id: 7485531672584456 - type: 'paper' - name: 'Test Paper Title 2' - link: 'http://localhost:8080/epilepsy' - description: 'Test abstract 2' - categories: { category: 'API', programmingLanguage: ['Javascript'], programmingLanguageVersion: '1.0.0' } - licence: 'MIT licence' - authors: [4495285946631793] - tags: { features: ['Arbitrage'], topics: ['Epilepsy'] } - toolids: [] - responses: - '200': - description: OK - - /api/v2/papers: - get: - summary: Returns a list of paper objects - tags: - - Papers v2.0 - parameters: - - name: search - in: query - description: Full text index search function which searches for partial matches in various fields including name and description. The response will contain a metascore indicating the relevancy of the match, by default results are sorted by the most relevant first unless a manual sort query parameter has been added. - schema: - type: string - example: Exploration - - name: page - in: query - description: A specific page of results to retrieve - schema: - type: number - example: 1 - - name: limit - in: query - description: Maximum number of results returned per page - schema: - type: number - example: 10 - - name: sort - in: query - description: Fields to apply sort operations to. Accepts multiple fields in ascending and descending. E.g. name for ascending or -name for descending. Multiple fields should be comma separated as shown in the example below. - schema: - type: string - example: name,-counter - - name: fields - in: query - description: Limit the size of the response by requesting only certain fields. Note that some additional derived fields are always returned. Multiple fields should be comma separate as shown in the example below. - schema: - type: string - example: name,counter,description - - name: count - in: query - description: Returns the number of the number of entities matching the query parameters provided instead of the result payload - schema: - type: boolean - example: true - description: Version 2.0 of the courses API introduces a large number of parameterised query string options to aid requests in collecting the data that is most relevant for a given use case. The query parameters defined below support a variety of comparison operators such as equals, contains, greater than, and less than. Using dot notation, any field can be queried, please see some examples below. Note - This response is limited to 100 records by default. Please use the 'page' query parameter to access records beyond the first 100. The 'limit' query parameter can therefore only be specified up to a maximum of 100. - responses: - '200': - description: Successful response containing a list of papers matching query parameters - - /api/v2/papers/{id}: - get: - summary: Returns paper object - tags: - - Papers v2.0 - parameters: - - in: path - name: id - required: true - description: The ID of the paper - schema: - type: number - example: 13296138992670704 - description: Returns a paper object by matching unique identifier in the default format that is stored as within the Gateway - responses: - '200': - description: Successful response containing a single paper object - '404': - description: A paper could not be found by the provided paper identifier - - /api/v1/tools: - get: - summary: Return List of Tool objects. - tags: - - Tools - parameters: - - in: query - name: limit - required: false - description: Limit the number of results - schema: - type: integer - example: 3 - - in: query - name: offset - required: false - description: Index to offset the search results - schema: - type: integer - example: 1 - - in: query - name: q - required: false - description: Filter using search query - schema: - type: string - example: epilepsy - responses: - '200': - description: OK - post: - summary: Returns new Tool object with ID. - security: - - cookieAuth: [] - tags: - - Tools - requestBody: - content: - application/json: - schema: # Request body contents - type: object - required: - - name - properties: - type: - type: string - name: - type: string - link: - type: string - description: - type: string - categories: - type: object - properties: - category: - type: string - programmingLanguage: - type: array - items: - type: string - programmingLanguageVersion: - type: string - licence: - type: string - authors: - type: array - items: - type: number - tags: - type: object - properties: - features: - type: array - items: - type: string - topics: - type: array - items: - type: string - example: # Sample object - id: 26542005388306332 - responses: - '200': - description: OK - - /api/v1/tools/{id}: - get: - summary: Returns Tool object - tags: - - Tools - parameters: - - in: path - name: id - required: true - description: The ID of the tool - schema: - type: integer - format: int64 - minimum: 1 - example: 19009 - responses: - '200': - description: OK - put: - summary: Returns edited Tool object. - security: - - cookieAuth: [] - tags: - - Tools - parameters: - - in: path - name: id - required: true - schema: - type: integer - example: 123 - requestBody: - content: - application/json: - schema: # Request body contents - type: object - required: - - name - properties: - id: - type: number - type: - type: string - name: - type: string - link: - type: string - description: - type: string - categories: - type: object - properties: - category: - type: string - programmingLanguage: - type: array - items: - type: string - programmingLanguageVersion: - type: string - licence: - type: string - authors: - type: array - items: - type: number - tags: - type: object - properties: - features: - type: array - items: - type: string - topics: - type: array - items: - type: string - toolids: - type: array - items: - type: string - example: # Sample object - id: 26542005388306332 - type: 'tool' - name: 'Research Data TEST EPILEPSY' - link: 'http://localhost:8080/epilepsy' - description: 'Epilespy data research description' - categories: { category: 'API', programmingLanguage: ['Javascript'], programmingLanguageVersion: '1.0.0' } - licence: 'MIT licence' - authors: [4495285946631793] - tags: { features: ['Arbitrage'], topics: ['Epilepsy'] } - toolids: [] - responses: - '200': - description: OK - patch: - summary: Change status of Tool object. - security: - - cookieAuth: [] - tags: - - Tools - parameters: - - name: id - in: path - required: true - description: The ID of the tool - schema: - type: integer - format: int64 - example: 5032687830560181 - requestBody: - content: - application/json: - schema: # Request body contents - type: object - required: - - name - properties: - id: - type: number - activeflag: - type: string - example: # Sample object - id: 662346984100503 - activeflag: 'active' - responses: - '200': - description: OK - - /api/v2/tools: - get: - summary: Returns a list of tool objects - tags: - - Tools v2.0 - parameters: - - name: search - in: query - description: Full text index search function which searches for partial matches in various fields including name and description. The response will contain a metascore indicating the relevancy of the match, by default results are sorted by the most relevant first unless a manual sort query parameter has been added. - schema: - type: string - example: Regulation - - name: page - in: query - description: A specific page of results to retrieve - schema: - type: number - example: 1 - - name: limit - in: query - description: Maximum number of results returned per page - schema: - type: number - example: 10 - - name: sort - in: query - description: Fields to apply sort operations to. Accepts multiple fields in ascending and descending. E.g. name for ascending or -name for descending. Multiple fields should be comma separated as shown in the example below. - schema: - type: string - example: name,-counter - - name: fields - in: query - description: Limit the size of the response by requesting only certain fields. Note that some additional derived fields are always returned. Multiple fields should be comma separate as shown in the example below. - schema: - type: string - example: name,counter, description - - name: count - in: query - description: Returns the number of the number of entities matching the query parameters provided instead of the result payload - schema: - type: boolean - example: true - description: Version 2.0 of the courses API introduces a large number of parameterised query string options to aid requests in collecting the data that is most relevant for a given use case. The query parameters defined below support a variety of comparison operators such as equals, contains, greater than, and less than. Using dot notation, any field can be queried, please see some examples below. Note - This response is limited to 100 records by default. Please use the 'page' query parameter to access records beyond the first 100. The 'limit' query parameter can therefore only be specified up to a maximum of 100. - responses: - '200': - description: Successful response containing a list of tools matching query parameters - - /api/v2/tools/{id}: - get: - summary: Returns a tool object - tags: - - Tools v2.0 - parameters: - - in: path - name: id - required: true - description: The ID of the tool - schema: - type: number - example: 100000006 - description: Returns a tool object by matching unique identifier in the default format that is stored as within the Gateway - responses: - '200': - description: Successful response containing a single tool object - '404': - description: A tool could not be found by the provided tool identifier - - /api/v2/courses: - get: - summary: Returns a list of courses - parameters: - - name: search - in: query - description: Full text index search function which searches for partial matches in various fields including name and description. The response will contain a metascore indicating the relevancy of the match, by default results are sorted by the most relevant first unless a manual sort query parameter has been added. - schema: - type: string - example: Research - - name: page - in: query - description: A specific page of results to retrieve - schema: - type: number - example: 1 - - name: limit - in: query - description: Maximum number of results returned per page - schema: - type: number - example: 10 - - name: sort - in: query - description: Fields to apply sort operations to. Accepts multiple fields in ascending and descending. E.g. provider for ascending or -provider for descending. Multiple fields should be comma separated as shown in the example below. - schema: - type: string - example: provider,-counter - - name: fields - in: query - description: Limit the size of the response by requesting only certain fields. Note that some additional derived fields are always returned. Multiple fields should be comma separate as shown in the example below. - schema: - type: string - example: provider,counter,description - - name: count - in: query - description: Returns the number of the number of entities matching the query parameters provided instead of the result payload - schema: - type: boolean - example: true - description: Version 2.0 of the courses API introduces a large number of parameterised query string options to aid requests in collecting the data that is most relevant for a given use case. The query parameters defined below support a variety of comparison operators such as equals, contains, greater than, and less than. Using dot notation, any field can be queried, please see some examples below. Note - This response is limited to 100 records by default. Please use the 'page' query parameter to access records beyond the first 100. The 'limit' query parameter can therefore only be specified up to a maximum of 100. - tags: - - Courses v2.0 - responses: - '200': - description: Successful response containing a list of course objects matching query parameters - - /api/v2/courses/{id}: - summary: summary - get: - summary: Returns a course object - description: Returns a course object by matching unique identifier in the default format that is stored as within the Gateway - tags: - - Courses v2.0 - parameters: - - in: path - name: id - required: true - description: The ID of the course - schema: - type: number - example: 5540794872521069 - responses: - '200': - description: Successful response containing a single course object - '404': - description: A course could not be found by the provided course identifier - -components: - securitySchemes: - oauth2: - type: oauth2 - flows: - clientCredentials: - tokenUrl: 'https://api.www.healthdatagateway.org/oauth/token' - scopes: {} - cookieAuth: - type: http - scheme: jwt From 1fea98913c86359d34cd1aca825d5ef47ccd3668 Mon Sep 17 00:00:00 2001 From: Pritesh Bhole Date: Tue, 12 Oct 2021 17:21:56 +0100 Subject: [PATCH 043/116] added check for undefined --- .../auth/__tests__/auth.utilities.test.js | 25 ++++++++++++++++++- src/resources/auth/utils.js | 2 +- 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/src/resources/auth/__tests__/auth.utilities.test.js b/src/resources/auth/__tests__/auth.utilities.test.js index 75da1a4f..6ba4a187 100644 --- a/src/resources/auth/__tests__/auth.utilities.test.js +++ b/src/resources/auth/__tests__/auth.utilities.test.js @@ -11,7 +11,7 @@ describe('Utilities', () => { let req = { auth: { user: 'someUser', - err: null, + err: '', }, }; const next = jest.fn(); @@ -44,6 +44,29 @@ describe('Utilities', () => { expect(res.status.mock.calls.length).toBe(1); expect(res.redirect.mock.calls.length).toBe(1); }); + + it('should not call next when (req.auth.err === loginError || req.auth.user === undefined) == true', () => { + let res = {}; + res.status = jest.fn().mockReturnValue(res); + res.redirect = jest.fn().mockReturnValue(res); + let req = { + auth: { + user: undefined, + err: 'loginError', + }, + param: { + returnpage: 'somePage', + }, + }; + const next = jest.fn(); + + catchLoginErrorAndRedirect(req, res, next); + + // assert + expect(next.mock.calls.length).toBe(0); + expect(res.status.mock.calls.length).toBe(1); + expect(res.redirect.mock.calls.length).toBe(1); + }); }); describe('loginAndSignToken middleware', () => { diff --git a/src/resources/auth/utils.js b/src/resources/auth/utils.js index 97cb301b..f1e236ce 100644 --- a/src/resources/auth/utils.js +++ b/src/resources/auth/utils.js @@ -129,7 +129,7 @@ const getTeams = async () => { const catchLoginErrorAndRedirect = (req, res, next) => { if (req.auth.err || !req.auth.user) { - if (req.auth.err === 'loginError') { + if (req.auth.err === 'loginError' || req.auth.user === undefined) { return res.status(200).redirect(process.env.homeURL + '/loginerror'); } From 9397faf9176e835c4c6a4f3ae78523f0906b1be2 Mon Sep 17 00:00:00 2001 From: Pritesh Bhole Date: Tue, 12 Oct 2021 17:27:20 +0100 Subject: [PATCH 044/116] reverted test back to null --- src/resources/auth/__tests__/auth.utilities.test.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/resources/auth/__tests__/auth.utilities.test.js b/src/resources/auth/__tests__/auth.utilities.test.js index 6ba4a187..cb0043c1 100644 --- a/src/resources/auth/__tests__/auth.utilities.test.js +++ b/src/resources/auth/__tests__/auth.utilities.test.js @@ -11,7 +11,7 @@ describe('Utilities', () => { let req = { auth: { user: 'someUser', - err: '', + err: null }, }; const next = jest.fn(); From 824e88bad223d3b6d4773927be5da527a5870404 Mon Sep 17 00:00:00 2001 From: Pritesh Bhole Date: Tue, 12 Oct 2021 17:28:02 +0100 Subject: [PATCH 045/116] reverted test back to null --- src/resources/auth/__tests__/auth.utilities.test.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/resources/auth/__tests__/auth.utilities.test.js b/src/resources/auth/__tests__/auth.utilities.test.js index cb0043c1..10ebd8f0 100644 --- a/src/resources/auth/__tests__/auth.utilities.test.js +++ b/src/resources/auth/__tests__/auth.utilities.test.js @@ -11,7 +11,7 @@ describe('Utilities', () => { let req = { auth: { user: 'someUser', - err: null + err: null, }, }; const next = jest.fn(); From eb5329604bbc8812392a7ee43e175a153f896edc Mon Sep 17 00:00:00 2001 From: Callum Reekie Date: Wed, 13 Oct 2021 12:43:54 +0100 Subject: [PATCH 046/116] CR - modified ORCiD strategy to work in production --- src/resources/auth/strategies/orcid.js | 22 ++++++---------------- 1 file changed, 6 insertions(+), 16 deletions(-) diff --git a/src/resources/auth/strategies/orcid.js b/src/resources/auth/strategies/orcid.js index 4638cc5d..c2faa996 100644 --- a/src/resources/auth/strategies/orcid.js +++ b/src/resources/auth/strategies/orcid.js @@ -12,14 +12,17 @@ const OrcidStrategy = passportOrcid.Strategy; const strategy = app => { const strategyOptions = { - sandbox: process.env.ORCID_SSO_ENV, clientID: process.env.ORCID_SSO_CLIENT_ID, clientSecret: process.env.ORCID_SSO_CLIENT_SECRET, callbackURL: `/auth/orcid/callback`, - scope: `/authenticate /read-limited`, + scope: `/authenticate`, proxy: true, }; + if (process.env.ORCID_SSO_ENV) { + strategyOptions.sandbox = process.env.ORCID_SSO_ENV; + } + const verifyCallback = async (accessToken, refreshToken, params, profile, done) => { if (!params.orcid || params.orcid === '') return done('loginError'); @@ -27,19 +30,6 @@ const strategy = app => { if (err || user) { return done(err, user); } - // Orcid does not include email natively - const requestedEmail = await axios - .get(`${process.env.ORCID_SSO_BASE_URL}/v3.0/${params.orcid}/email`, { - headers: { Authorization: `Bearer ` + accessToken, Accept: 'application/json' }, - }) - .then(response => { - const email = response.data.email[0].email; - return email == undefined || !/\b[a-zA-Z0-9-_.]+\@[a-zA-Z0-9-_]+\.\w+(?:\.\w+)?\b/.test(email) ? '' : email; - }) - .catch(err => { - console.log(err); - return ''; - }); const [createdError, createdUser] = await to( createUser({ @@ -48,7 +38,7 @@ const strategy = app => { firstname: params.name.split(' ')[0], lastname: params.name.split(' ')[1], password: null, - email: requestedEmail, + email: '', role: ROLES.Creator, }) ); From 0f513c81b88d4e5d184979d0e43286d1549e7a75 Mon Sep 17 00:00:00 2001 From: Callum Reekie Date: Wed, 13 Oct 2021 13:05:32 +0100 Subject: [PATCH 047/116] CR - removed unused axios import --- src/resources/auth/strategies/orcid.js | 1 - 1 file changed, 1 deletion(-) diff --git a/src/resources/auth/strategies/orcid.js b/src/resources/auth/strategies/orcid.js index c2faa996..38f8fa10 100644 --- a/src/resources/auth/strategies/orcid.js +++ b/src/resources/auth/strategies/orcid.js @@ -1,7 +1,6 @@ import passport from 'passport'; import passportOrcid from 'passport-orcid'; import { to } from 'await-to-js'; -import axios from 'axios'; import { catchLoginErrorAndRedirect, loginAndSignToken } from '../utils'; import { getUserByProviderId } from '../../user/user.repository'; From 1e379c0a38a80d85ca024e808cce9b87986cd8ff Mon Sep 17 00:00:00 2001 From: CiaraWardPA Date: Wed, 13 Oct 2021 16:07:30 +0100 Subject: [PATCH 048/116] IG-2278 Stats api updated to return active data uses count, most popular and latest updated data uses --- .../dataUseRegister/dataUseRegister.model.js | 13 +++- src/resources/stats/stats.repository.js | 60 +++++++++++++++++++ src/resources/stats/stats.service.js | 8 +++ src/resources/stats/v1/stats.route.js | 4 +- 4 files changed, 81 insertions(+), 4 deletions(-) diff --git a/src/resources/dataUseRegister/dataUseRegister.model.js b/src/resources/dataUseRegister/dataUseRegister.model.js index 3fdc006d..b27b5bd4 100644 --- a/src/resources/dataUseRegister/dataUseRegister.model.js +++ b/src/resources/dataUseRegister/dataUseRegister.model.js @@ -50,7 +50,7 @@ const dataUseRegisterSchema = new Schema( publicBenefitStatement: String, //Public Benefit Statement requestCategoryType: String, //Request Category Type technicalSummary: String, //Technical Summary - otherApprovalCommittees: [{type: String}], //Other Approval Committees + otherApprovalCommittees: [{ type: String }], //Other Approval Committees projectStartDate: Date, //Project Start Date projectEndDate: Date, //Project End Date latestApprovalDate: Date, //Latest Approval Date @@ -65,17 +65,24 @@ const dataUseRegisterSchema = new Schema( accessDate: Date, //Release/Access Date dataLocation: String, //TRE Or Any Other Specified Location privacyEnhancements: String, //How Has Data Been Processed To Enhance Privacy - researchOutputs: [{type: String}], //Link To Research Outputs + researchOutputs: [{ type: String }], //Link To Research Outputs }, { timestamps: true, toJSON: { virtuals: true }, toObject: { virtuals: true }, - strict: false + strict: false, } ); // Load entity class dataUseRegisterSchema.loadClass(DataUseRegisterClass); +dataUseRegisterSchema.virtual('publisherInfo', { + ref: 'Publisher', + foreignField: '_id', + localField: 'publisher', + justOne: true, +}); + export const DataUseRegister = model('DataUseRegister', dataUseRegisterSchema); diff --git a/src/resources/stats/stats.repository.js b/src/resources/stats/stats.repository.js index 034c7b2b..eeeeb993 100644 --- a/src/resources/stats/stats.repository.js +++ b/src/resources/stats/stats.repository.js @@ -5,6 +5,7 @@ import { RecordSearchData } from '../search/record.search.model'; import { DataRequestModel } from '../datarequest/datarequest.model'; import { Course } from '../course/course.model'; import { MessagesModel } from '../message/message.model'; +import { DataUseRegister } from '../dataUseRegister/dataUseRegister.model'; import constants from '../utilities/constants.util'; export default class StatsRepository extends Repository { @@ -469,10 +470,44 @@ export default class StatsRepository extends Repository { ]); } + async getPopularDataUses() { + return DataUseRegister.find( + { + activeflag: 'active', + counter: { + $gt: 0, + }, + }, + { + _id: 0, + type: 1, + projectTitle: 1, + organisationName: 1, + keywords: 1, + datasetTitles: 1, + publisher: 1, + id: 1, + counter: 1, + updatedon: 1, + } + ) + .populate({ + path: 'publisherInfo', + select: { name: 1, _id: 0 }, + }) + .sort({ counter: -1, title: 1 }) + .limit(10) + .lean(); + } + async getActiveCourseCount() { return Course.countDocuments({ activeflag: 'active' }); } + async getActiveDataUsesCount() { + return DataUseRegister.countDocuments({ activeflag: 'active' }); + } + async getPopularEntitiesByType(entityType) { let entityTypeFilter = {}; if (entityType) entityTypeFilter = { type: entityType }; @@ -604,6 +639,31 @@ export default class StatsRepository extends Repository { .lean(); } + async getRecentlyUpdatedDataUses() { + return DataUseRegister.find( + { activeflag: 'active' }, + { + _id: 0, + type: 1, + projectTitle: 1, + organisationName: 1, + keywords: 1, + datasetTitles: 1, + publisher: 1, + id: 1, + counter: 1, + updatedon: 1, + } + ) + .populate({ + path: 'publisherInfo', + select: { name: 1, _id: 0 }, + }) + .sort({ updatedon: -1, title: 1 }) + .limit(10) + .lean(); + } + async getRecentlyUpdatedEntitiesByType(entityType) { if (entityType) { return Data.find( diff --git a/src/resources/stats/stats.service.js b/src/resources/stats/stats.service.js index 726abc7e..2aec5aec 100644 --- a/src/resources/stats/stats.service.js +++ b/src/resources/stats/stats.service.js @@ -118,6 +118,8 @@ export default class StatsService { switch (entityType) { case 'course': return this.statsRepository.getPopularCourses(); + case 'dataUseRegister': + return this.statsRepository.getPopularDataUses(); default: return this.statsRepository.getPopularEntitiesByType(entityType); } @@ -127,12 +129,18 @@ export default class StatsService { return this.statsRepository.getActiveCourseCount(); } + async getActiveDataUsesCount() { + return this.statsRepository.getActiveDataUsesCount(); + } + async getRecentlyUpdatedEntitiesByType(entityType) { switch (entityType) { case 'course': return this.statsRepository.getRecentlyUpdatedCourses(); case 'dataset': return this.statsRepository.getRecentlyUpdatedDatasets(); + case 'dataUseRegister': + return this.statsRepository.getRecentlyUpdatedDataUses(); default: return this.statsRepository.getRecentlyUpdatedEntitiesByType(entityType); } diff --git a/src/resources/stats/v1/stats.route.js b/src/resources/stats/v1/stats.route.js index ebd0f9a6..4e339078 100644 --- a/src/resources/stats/v1/stats.route.js +++ b/src/resources/stats/v1/stats.route.js @@ -61,11 +61,12 @@ router.get('', logger.logRequestMiddleware({ logCategory, action: 'Viewed stats' break; default: - const [searchCounts, accessRequestCount, entityTotalCounts, coursesActiveCount] = await Promise.all([ + const [searchCounts, accessRequestCount, entityTotalCounts, coursesActiveCount, dataUsesActiveCount] = await Promise.all([ statsService.getTotalSearchesByUsers(), statsService.getDataAccessRequestStats(), statsService.getTotalEntityCounts(), statsService.getActiveCourseCount(), + statsService.getActiveDataUsesCount(), ]).catch(err => { logger.logError(err, logCategory); }); @@ -75,6 +76,7 @@ router.get('', logger.logRequestMiddleware({ logCategory, action: 'Viewed stats' ...entityTotalCounts, accessRequests: accessRequestCount, course: coursesActiveCount, + dataUses: dataUsesActiveCount, }, daycounts: searchCounts, }; From 531e41915e4f5f0c29943e7c5e2f8d0de4722014 Mon Sep 17 00:00:00 2001 From: Callum Reekie Date: Thu, 14 Oct 2021 21:08:39 +0100 Subject: [PATCH 049/116] CR - refactor collections --- .../collections/__mocks__/multi.collection.js | 76 +++ .../__mocks__/single.collection.js | 68 +++ .../__tests__/collections.controller.test.js | 343 +++++++++++ .../collections/collections.controller.js | 228 ++++++++ .../collections/collections.repository.js | 505 +--------------- .../collections/collections.route.js | 257 +------- .../collections/collections.service.js | 551 ++++++++++++++++++ src/resources/collections/dependency.js | 6 + .../__tests__/course.repository.it.test.js | 2 +- .../__tests__/course.repository.test.js | 6 +- 10 files changed, 1315 insertions(+), 727 deletions(-) create mode 100644 src/resources/collections/__mocks__/multi.collection.js create mode 100644 src/resources/collections/__mocks__/single.collection.js create mode 100644 src/resources/collections/__tests__/collections.controller.test.js create mode 100644 src/resources/collections/collections.controller.js create mode 100644 src/resources/collections/collections.service.js create mode 100644 src/resources/collections/dependency.js diff --git a/src/resources/collections/__mocks__/multi.collection.js b/src/resources/collections/__mocks__/multi.collection.js new file mode 100644 index 00000000..61375ed0 --- /dev/null +++ b/src/resources/collections/__mocks__/multi.collection.js @@ -0,0 +1,76 @@ +export const mock_collections = [ + { + _id: { + oid: '6168030b0e24c03595166261', + }, + authors: [12345], + keywords: [], + relatedObjects: [ + { + _id: { + $oid: '6168030b0e24c03595166262', + }, + objectId: 'af434b05-52a7-4ff1-92f5-e2dd38a574aa', + reason: '', + objectType: 'dataset', + pid: 'fdd9e5ab-442f-45d0-a004-f581a3ac809c', + user: 'John Doe', + updated: '14 Oct 2021', + }, + ], + id: 138879762298581, + name: 'Test collection 1', + description: 'A test collection', + imageLink: '', + activeflag: 'active', + publicflag: true, + updatedon: { + $date: '2021-10-14T12:10:13.817Z', + }, + createdAt: { + $date: '2021-10-14T10:14:35.308Z', + }, + updatedAt: { + $date: '2021-10-14T12:10:14.563Z', + }, + __v: 0, + counter: 1, + }, + { + _id: { + oid: '6168030b0e24c03595166262', + }, + authors: [12345], + keywords: [], + relatedObjects: [ + { + _id: { + $oid: '6168030b0e24c03595166262', + }, + objectId: 'af434b05-52a7-4ff1-92f5-e2dd38a574aa', + reason: '', + objectType: 'dataset', + pid: 'fdd9e5ab-442f-45d0-a004-f581a3ac809c', + user: 'John Doe', + updated: '14 Oct 2021', + }, + ], + id: 138879762298582, + name: 'Test collection 2', + description: 'A test collection', + imageLink: '', + activeflag: 'active', + publicflag: true, + updatedon: { + $date: '2021-10-14T12:10:13.817Z', + }, + createdAt: { + $date: '2021-10-14T10:14:35.308Z', + }, + updatedAt: { + $date: '2021-10-14T12:10:14.563Z', + }, + __v: 0, + counter: 1, + }, +]; diff --git a/src/resources/collections/__mocks__/single.collection.js b/src/resources/collections/__mocks__/single.collection.js new file mode 100644 index 00000000..ff799f0f --- /dev/null +++ b/src/resources/collections/__mocks__/single.collection.js @@ -0,0 +1,68 @@ +export const mock_collection = [ + { + _id: '612e0d035671f75be2461dfa', + authors: [8470291714590257], + keywords: [], + relatedObjects: [ + { + _id: '612e0d035671f75be2461dfb', + objectId: '6ec3a47b-447a-4b22-9b7a-43acae5d408f', + reason: '', + objectType: 'dataset', + pid: 'fce78329-0de1-45f2-9ff1-e1b4af50528e', + user: 'John Doe', + updated: '31 Aug 2021', + }, + ], + id: 20905331408744290, + name: 'Test', + description: 'TestTestTestTestTestTestTestTestTestTest', + imageLink: '', + activeflag: 'active', + publicflag: true, + updatedon: '2021-08-31T11:06:19.329Z', + createdAt: '2021-08-31T11:05:39.129Z', + updatedAt: '2021-10-14T14:38:21.800Z', + __v: 0, + counter: 3, + persons: [ + { + _id: '6128a6f9dd361d15499db644', + categories: { programmingLanguage: [] }, + tags: { features: [], topics: [] }, + document_links: { doi: [], pdf: [], html: [] }, + datasetfields: { geographicCoverage: [], physicalSampleAvailability: [], technicaldetails: [], versionLinks: [], phenotypes: [] }, + authors: [], + emailNotifications: true, + showOrganisation: true, + structuralMetadata: [], + datasetVersionIsV1: false, + toolids: [], + datasetids: [], + id: 8470291714590257, + type: 'person', + firstname: 'John', + lastname: 'Doe', + bio: '', + link: '', + orcid: 'https://orcid.org/', + activeflag: 'active', + terms: true, + sector: 'Academia', + organisation: '', + showSector: true, + showBio: true, + showLink: true, + showOrcid: true, + showDomain: true, + profileComplete: true, + relatedObjects: [], + programmingLanguage: [], + createdAt: '2021-08-27T08:48:57.710Z', + updatedAt: '2021-08-27T10:23:11.582Z', + __v: 0, + counter: 1, + }, + ], + }, +]; diff --git a/src/resources/collections/__tests__/collections.controller.test.js b/src/resources/collections/__tests__/collections.controller.test.js new file mode 100644 index 00000000..9048e336 --- /dev/null +++ b/src/resources/collections/__tests__/collections.controller.test.js @@ -0,0 +1,343 @@ +import sinon from 'sinon'; + +import { mock_collections } from '../__mocks__/multi.collection'; +import { mock_collection } from '../__mocks__/single.collection'; +import CollectionsController from '../collections.controller'; +import CollectionsService from '../collections.service'; +import { Data } from '../../tool/data.model'; +import { filtersService } from '../../filters/dependency'; +import { Collections } from '../collections.model'; + +afterEach(function () { + sinon.restore(); +}); + +describe('With the Collections controller class', () => { + const collectionsService = new CollectionsService(); + const collectionsController = new CollectionsController(collectionsService); + + describe('Using the getList method', () => { + describe('As an ADMIN user', () => { + let req = { + user: { + role: 'Admin', + }, + query: {}, + }; + let res, json; + json = sinon.spy(); + res = { json }; + + it('Should return a list of collections for an Admin user', async () => { + let stub = sinon.stub(collectionsService, 'getCollectionsAdmin').returns(mock_collections); + + await collectionsController.getList(req, res); + expect(stub.calledOnce).toBe(true); + expect(json.calledWith({ success: true, data: mock_collections })).toBe(true); + }); + + it('Should return an error if the service call fails for an Admin user', async () => { + let stub = sinon.stub(collectionsService, 'getCollectionsAdmin').returns(Promise.reject('error')); + + await collectionsController.getList(req, res); + expect(stub.calledOnce).toBe(true); + expect(json.calledWith({ success: false, error: 'error' })).toBe(true); + }); + }); + + describe('As a CREATOR user', () => { + let req = { + user: { + role: 'Creator', + id: 12345, + }, + query: {}, + }; + let res, json; + json = sinon.spy(); + res = { json }; + + it('Should return a list of collections for a Creator user', async () => { + let stub = sinon.stub(collectionsService, 'getCollections').returns(mock_collections); + + await collectionsController.getList(req, res); + expect(stub.calledOnce).toBe(true); + expect(json.calledWith({ success: true, data: mock_collections })).toBe(true); + }); + + it('Should return an error if the service call fails for a Creator user', async () => { + let stub = sinon.stub(collectionsService, 'getCollections').throws(); + + const badCall = async () => { + await collectionsController.getList(req, res); + }; + + try { + badCall(); + } catch (error) { + expect(stub.calledOnce).toBe(true); + expect(badCall).to.have.been.calledWith(error); + } + }); + }); + }); + + describe('Using the getCollection method', () => { + let req = { + user: { + role: 'Creator', + }, + params: { + id: 138879762298581, + }, + query: {}, + }; + let res, json, status; + json = sinon.spy(); + status = sinon.spy(); + res = { json, status }; + + it('Should call the getCollection service and return data, if data is exists', async () => { + let stub = sinon.stub(collectionsService, 'getCollection').returns(mock_collection); + + await collectionsController.getCollection(req, res); + expect(stub.calledOnce).toBe(true); + expect(json.calledWith({ success: true, data: mock_collection })).toBe(true); + }); + + it('Should return a 404 error if no data exists', async () => { + let stub = sinon.stub(collectionsService, 'getCollection').returns([]); + + await collectionsController.getCollection(req, res); + expect(stub.calledOnce).toBe(true); + expect(status.calledOnce).toBe(true); + expect(status.calledWith(404)).toBe(true); + }); + + it('Should return an error if the service call fails', async () => { + let stub = sinon.stub(collectionsService, 'getCollection').returns(Promise.reject('error')); + + await collectionsController.getCollection(req, res); + expect(stub.calledOnce).toBe(true); + expect(json.calledWith({ success: false, error: 'error' })).toBe(true); + }); + }); + + describe('Using the getCollectionRelatedResources method', () => { + let req = { + user: { + role: 'Creator', + }, + params: { + collectionID: 138879762298581, + }, + query: {}, + }; + let res, json; + json = sinon.spy(); + res = { json }; + + it('Should call the getCollectionsObject service and return data', async () => { + let stub = sinon.stub(collectionsService, 'getCollectionObjects').returns(mock_collections[0].relatedObjects[0]); + + await collectionsController.getCollectionRelatedResources(req, res); + expect(stub.calledOnce).toBe(true); + expect(json.calledWith({ success: true, data: mock_collections[0].relatedObjects[0] })).toBe(true); + }); + + it('Should return an error if the service call fails', async () => { + let stub = sinon.stub(collectionsService, 'getCollectionObjects').returns(Promise.reject('error')); + + await collectionsController.getCollectionRelatedResources(req, res); + expect(stub.calledOnce).toBe(true); + expect(json.calledWith({ success: false, error: 'error' })).toBe(true); + }); + }); + + describe('Using the getCollectionByEntity method', () => { + let req = { + user: { + role: 'Creator', + }, + params: { + entityID: 12345, + }, + query: {}, + }; + let res, json; + json = sinon.spy(); + res = { json }; + + it('Should call the getCollectionByEntity service and return data', async () => { + let stub = sinon.stub(collectionsService, 'getCollectionByEntity').returns(mock_collection); + let dataStub = sinon.stub(Data, 'find').returns([]); + + await collectionsController.getCollectionByEntity(req, res); + expect(stub.calledOnce).toBe(true); + expect(dataStub.calledOnce).toBe(true); + expect(json.calledWith({ success: true, data: mock_collection })).toBe(true); + }); + + it('Should return an error if the service call fails', async () => { + let stub = sinon.stub(collectionsService, 'getCollectionByEntity').returns(Promise.reject('error')); + let dataStub = sinon.stub(Data, 'find').returns([]); + + await collectionsController.getCollectionByEntity(req, res); + expect(stub.calledOnce).toBe(true); + expect(dataStub.calledOnce).toBe(true); + expect(json.calledWith({ success: false, error: 'error' })).toBe(true); + }); + }); + + describe('Using the editCollection method', () => { + let req = { + user: { + role: 'Creator', + }, + params: { + id: 12345, + }, + query: {}, + body: { + publicflag: true, + previousPublicFlag: false, + }, + }; + let res, json; + json = sinon.spy(); + res = { json }; + + it('Should call the editCollection service, return data, optimise filters and send notifications', async () => { + let stub = sinon.stub(collectionsService, 'editCollection'); + let collectionStub = sinon.stub(Collections, 'find').returns(mock_collections[0]); + let filterStub = sinon.stub(filtersService, 'optimiseFilters'); + + await collectionsController.editCollection(req, res); + expect(stub.calledOnce).toBe(true); + expect(collectionStub.calledOnce).toBe(true); + expect(filterStub.calledOnce).toBe(true); + }); + + it('Should return an error if the service call fails', async () => { + let stub = sinon.stub(collectionsService, 'editCollection').returns(Promise.reject('error')); + + await collectionsController.editCollection(req, res); + expect(stub.calledOnce).toBe(true); + expect(json.calledWith({ success: false, error: 'error' })).toBe(true); + }); + }); + + describe('Using the addCollection method', () => { + let req = { + user: { + role: 'Creator', + }, + params: { + id: 12345, + }, + query: {}, + body: { + name: 'test', + description: 'test', + imageLink: '', + authors: [123, 456], + relatedObjects: [], + publicflag: true, + keywords: [], + }, + }; + let res, json; + json = sinon.spy(); + res = { json }; + + it('Should call the addCollection service, return the ID and send notifications', async () => { + let stub = sinon.stub(collectionsService, 'addCollection'); + let messageStub = sinon.stub(collectionsController, 'createMessage'); + let emailStub = sinon.stub(collectionsService, 'sendEmailNotifications'); + + await collectionsController.addCollection(req, res); + expect(stub.calledOnce).toBe(true); + expect(messageStub.callCount).toBe(3); + expect(emailStub.calledOnce).toBe(true); + }); + + it('Should return an error if the service call fails', async () => { + let stub = sinon.stub(collectionsService, 'addCollection').returns(Promise.reject('error')); + let messageStub = sinon.stub(collectionsController, 'createMessage'); + let emailStub = sinon.stub(collectionsService, 'sendEmailNotifications'); + + await collectionsController.addCollection(req, res); + expect(stub.calledOnce).toBe(true); + expect(messageStub.callCount).toBe(3); + expect(emailStub.calledOnce).toBe(true); + expect(json.calledWith({ success: false, error: 'error' })).toBe(true); + }); + }); + + describe('Using the deleteCollection method', () => { + let req = { + user: { + role: 'Creator', + }, + params: { + id: 12345, + }, + query: {}, + }; + let res, json; + json = sinon.spy(); + res = { json }; + + it('Should call the deleteCollection service', async () => { + let stub = sinon.stub(collectionsService, 'deleteCollection'); + + await collectionsController.deleteCollection(req, res); + expect(stub.calledOnce).toBe(true); + expect(json.calledWith({ success: true })).toBe(true); + }); + + it('Should return an error if the service call fails', async () => { + let stub = sinon.stub(collectionsService, 'deleteCollection').returns(Promise.reject('error')); + + await collectionsController.deleteCollection(req, res); + expect(stub.calledOnce).toBe(true); + expect(json.calledWith({ success: false, error: 'error' })).toBe(true); + }); + }); + + describe('Using the changeStatus method', () => { + let req = { + user: { + role: 'Creator', + }, + params: { + id: 12345, + }, + query: {}, + body: { + activeflag: 'archive', + }, + }; + let res, json; + json = sinon.spy(); + res = { json }; + + it('Should call the changeStatus service', async () => { + let stub = sinon.stub(collectionsService, 'changeStatus'); + let filterStub = sinon.stub(filtersService, 'optimiseFilters'); + + await collectionsController.changeStatus(req, res); + expect(stub.calledOnce).toBe(true); + expect(filterStub.calledOnce).toBe(true); + expect(json.calledWith({ success: true })).toBe(true); + }); + + it('Should return an error if the service call fails', async () => { + let stub = sinon.stub(collectionsService, 'changeStatus').returns(Promise.reject('error')); + + await collectionsController.changeStatus(req, res); + expect(stub.calledOnce).toBe(true); + expect(json.calledWith({ success: false, error: 'error' })).toBe(true); + }); + }); +}); diff --git a/src/resources/collections/collections.controller.js b/src/resources/collections/collections.controller.js new file mode 100644 index 00000000..632cc4da --- /dev/null +++ b/src/resources/collections/collections.controller.js @@ -0,0 +1,228 @@ +import _ from 'lodash'; +import escape from 'escape-html'; + +import Controller from '../base/controller'; +import inputSanitizer from '../utilities/inputSanitizer'; +import urlValidator from '../utilities/urlValidator'; +import { filtersService } from '../filters/dependency'; +import helper from '../utilities/helper.util'; +import { Collections } from '../collections/collections.model'; +import { Data } from '../tool/data.model'; +import { ROLES } from '../user/user.roles'; +import { MessagesModel } from '../message/message.model'; +import { UserModel } from '../user/user.model'; + +export default class CollectionsController extends Controller { + constructor(collectionsService) { + super(collectionsService); + this.collectionsService = collectionsService; + } + + async getList(req, res) { + let role = req.user.role; + let startIndex = 0; + let limit = 40; + let searchString = ''; + let status = 'all'; + + if (req.query.offset) { + startIndex = req.query.offset; + } + if (req.query.limit) { + limit = req.query.limit; + } + if (req.query.q) { + searchString = req.query.q || ''; + } + if (req.query.status) { + status = req.query.status; + } + + if (role === ROLES.Admin) { + try { + const data = await this.collectionsService.getCollectionsAdmin(searchString, status, startIndex, limit); + return res.json({ success: true, data: data }); + } catch (err) { + return res.json({ success: false, error: err }); + } + } else if (role === ROLES.Creator) { + try { + let idString = req.user.id; + if (req.query.id) { + idString = req.query.id; + } + const data = await this.collectionsService.getCollections(idString, status, startIndex, limit); + return res.json({ success: true, data: data }); + } catch (err) { + return res.json({ success: false, error: err }); + } + } + } + + async getCollection(req, res) { + let collectionID = parseInt(req.params.collectionID); + + try { + const data = await this.collectionsService.getCollection(collectionID); + if (_.isEmpty(data)) { + return res.status(404).send(`Collection not found for ID: ${escape(collectionID)}`); + } + data[0].persons = helper.hidePrivateProfileDetails(data[0].persons); + return res.json({ success: true, data: data }); + } catch (err) { + return res.json({ success: false, error: err }); + } + } + + async getCollectionRelatedResources(req, res) { + let collectionID = parseInt(req.params.collectionID); + + try { + const data = await this.collectionsService.getCollectionObjects(collectionID); + return res.json({ success: true, data: data }); + } catch (err) { + return res.json({ success: false, error: err }); + } + } + + async getCollectionByEntity(req, res) { + let entityID = req.params.entityID; + let dataVersions = await Data.find({ pid: entityID }, { _id: 0, datasetid: 1 }); + let dataVersionsArray = dataVersions.map(a => a.datasetid); + dataVersionsArray.push(entityID); + + try { + const data = await this.collectionsService.getCollectionByEntity(entityID, dataVersionsArray); + return res.json({ success: true, data: data }); + } catch (err) { + res.json({ success: false, error: err }); + } + } + + async editCollection(req, res) { + let collectionID = parseInt(req.params.id); + let { name, description, imageLink, authors, relatedObjects, publicflag, keywords, previousPublicFlag, collectionCreator } = req.body; + imageLink = urlValidator.validateURL(imageLink); + + let updatedCollection = { name, description, imageLink, authors, relatedObjects, publicflag, keywords }; + + try { + await this.collectionsService.editCollection(collectionID, updatedCollection); + filtersService.optimiseFilters('collection'); + await Collections.find({ id: collectionID }, { publicflag: 1, id: 1, activeflag: 1, authors: 1, name: 1 }).then(async res => { + if (previousPublicFlag === false && publicflag === true) { + await this.collectionsService.sendEmailNotifications(res[0], res[0].activeflag, collectionCreator, true); + + if (res[0].authors) { + res[0].authors.forEach(async authorId => { + await this.createMessage(authorId, res[0], res[0].activeflag, collectionCreator, true); + }); + } + + await this.createMessage(0, res[0], res[0].activeflag, collectionCreator, true); + } + }); + return res.json({ success: true }); + } catch (err) { + return res.json({ success: false, error: err }); + } + } + + async addCollection(req, res) { + let collections = new Collections(); + const collectionCreator = req.body.collectionCreator; + const { name, description, imageLink, authors, relatedObjects, publicflag, keywords } = req.body; + + collections.id = parseInt(Math.random().toString().replace('0.', '')); + collections.name = inputSanitizer.removeNonBreakingSpaces(name); + collections.description = inputSanitizer.removeNonBreakingSpaces(description); + collections.imageLink = imageLink; + collections.authors = authors; + collections.relatedObjects = relatedObjects; + collections.activeflag = 'active'; + collections.publicflag = publicflag; + collections.keywords = keywords; + collections.updatedon = Date.now(); + + if (collections.authors) { + collections.authors.forEach(async authorId => { + await this.createMessage(authorId, collections, collections.activeflag, collectionCreator); + }); + } + + await this.createMessage(0, collections, collections.activeflag, collectionCreator); + + await this.collectionsService.sendEmailNotifications(collections, collections.activeflag, collectionCreator); + + try { + await this.collectionsService.addCollection(collections); + res.json({ success: true, id: collections.id }); + } catch (err) { + res.json({ success: false, error: err }); + } + } + + async changeStatus(req, res) { + const collectionID = parseInt(req.params.id); + let { activeflag } = req.body; + activeflag = activeflag.toString(); + + try { + await this.collectionsService.changeStatus(collectionID, activeflag); + filtersService.optimiseFilters('collection'); + return res.json({ success: true }); + } catch (err) { + return res.json({ success: false, error: err }); + } + } + + async deleteCollection(req, res) { + const collectionID = parseInt(req.params.id); + try { + await this.collectionsService.deleteCollection(collectionID); + res.json({ success: true }); + } catch (err) { + res.json({ success: false, error: err }); + } + } + + async createMessage(authorId, collections, activeflag, collectionCreator, isEdit) { + let message = new MessagesModel(); + + const messageRecipients = await UserModel.find({ $or: [{ role: 'Admin' }, { id: { $in: collections.authors } }] }); + async function saveMessage() { + message.messageID = parseInt(Math.random().toString().replace('0.', '')); + message.messageTo = authorId; + message.messageObjectID = collections.id; + message.messageSent = Date.now(); + message.isRead = false; + await message.save(); + } + + if (authorId === 0) { + message.messageType = 'added collection'; + message.messageDescription = this.collectionsService.generateCollectionEmailSubject( + 'Admin', + collections.publicflag, + collections.name, + false, + isEdit + ); + saveMessage(); + } + + for (let messageRecipient of messageRecipients) { + if (activeflag === 'active' && authorId === messageRecipient.id) { + message.messageType = 'added collection'; + message.messageDescription = this.collectionsService.generateCollectionEmailSubject( + 'Creator', + collections.publicflag, + collections.name, + authorId === collectionCreator.id ? true : false, + isEdit + ); + saveMessage(); + } + } + } +} diff --git a/src/resources/collections/collections.repository.js b/src/resources/collections/collections.repository.js index 2c9d2816..70841db8 100644 --- a/src/resources/collections/collections.repository.js +++ b/src/resources/collections/collections.repository.js @@ -1,496 +1,31 @@ -/* eslint-disable no-undef */ -import { Data } from '../tool/data.model'; -import { Course } from '../course/course.model'; +import Repository from '../base/repository'; import { Collections } from './collections.model'; -import { UserModel } from '../user/user.model'; -import emailGenerator from '../utilities/emailGenerator.util'; -import _ from 'lodash'; import helper from '../utilities/helper.util'; -const hdrukEmail = `enquiry@healthdatagateway.org`; - -const getCollectionObjects = async req => { - let relatedObjects = []; - await Collections.find( - { id: parseInt(req.params.collectionID) }, - { - 'relatedObjects._id': 1, - 'relatedObjects.objectId': 1, - 'relatedObjects.objectType': 1, - 'relatedObjects.pid': 1, - 'relatedObjects.updated': 1, - } - ).then(async res => { - await new Promise(async (resolve, reject) => { - if (_.isEmpty(res)) { - reject(`Collection not found for Id: ${req.params.collectionID}.`); - } else { - for (let object of res[0].relatedObjects) { - let relatedObject = await getCollectionObject(object.objectId, object.objectType, object.pid, object.updated); - if (!_.isUndefined(relatedObject)) { - relatedObjects.push(relatedObject); - } else { - await Collections.findOneAndUpdate( - { id: parseInt(req.params.collectionID) }, - { $pull: { relatedObjects: { _id: object._id } } } - ); - } - } - resolve(relatedObjects); - } - }); - }); - - return relatedObjects.sort((a, b) => b.updated - a.updated); -}; - -function getCollectionObject(objectId, objectType, pid, updated) { - let id = pid && pid.length > 0 ? pid : objectId; - - return new Promise(async resolve => { - let data; - if (objectType !== 'dataset' && objectType !== 'course') { - data = await Data.find( - { id: parseInt(id) }, - { - id: 1, - type: 1, - activeflag: 1, - tags: 1, - description: 1, - name: 1, - persons: 1, - categories: 1, - programmingLanguage: 1, - firstname: 1, - lastname: 1, - bio: 1, - authors: 1, - counter: { $ifNull: ['$counter', 0] }, - relatedresources: { $cond: { if: { $isArray: '$relatedObjects' }, then: { $size: '$relatedObjects' }, else: 0 } }, - } - ) - .populate([{ path: 'persons', options: { select: { id: 1, firstname: 1, lastname: 1 } } }]) - .lean(); - } else if (!isNaN(id) && objectType === 'course') { - data = await Course.find( - { id: parseInt(id) }, - { - id: 1, - type: 1, - activeflag: 1, - title: 1, - provider: 1, - courseOptions: 1, - award: 1, - domains: 1, - tags: 1, - description: 1, - counter: { $ifNull: ['$counter', 0] }, - relatedresources: { $cond: { if: { $isArray: '$relatedObjects' }, then: { $size: '$relatedObjects' }, else: 0 } }, - } - ).lean(); - } else { - const datasetRelatedResources = { - $lookup: { - from: 'tools', - let: { - pid: '$pid', - }, - pipeline: [ - { $unwind: '$relatedObjects' }, - { - $match: { - $expr: { - $and: [ - { - $eq: ['$relatedObjects.pid', '$$pid'], - }, - { - $eq: ['$activeflag', 'active'], - }, - ], - }, - }, - }, - { $group: { _id: null, count: { $sum: 1 } } }, - ], - as: 'relatedResourcesTools', - }, - }; - - const datasetRelatedCourses = { - $lookup: { - from: 'course', - let: { - pid: '$pid', - }, - pipeline: [ - { $unwind: '$relatedObjects' }, - { - $match: { - $expr: { - $and: [ - { - $eq: ['$relatedObjects.pid', '$$pid'], - }, - { - $eq: ['$activeflag', 'active'], - }, - ], - }, - }, - }, - { $group: { _id: null, count: { $sum: 1 } } }, - ], - as: 'relatedResourcesCourses', - }, - }; - - const datasetProjectFields = { - $project: { - id: 1, - datasetid: 1, - pid: 1, - type: 1, - activeflag: 1, - name: 1, - datasetv2: 1, - datasetfields: 1, - tags: 1, - description: 1, - counter: { $ifNull: ['$counter', 0] }, - relatedresources: { - $add: [ - { - $cond: { - if: { $eq: [{ $size: '$relatedResourcesTools' }, 0] }, - then: 0, - else: { $first: '$relatedResourcesTools.count' }, - }, - }, - { - $cond: { - if: { $eq: [{ $size: '$relatedResourcesCourses' }, 0] }, - then: 0, - else: { $first: '$relatedResourcesCourses.count' }, - }, - }, - ], - }, - }, - }; - - // 1. Search for a dataset based on pid - data = await Data.aggregate([ - { $match: { $and: [{ pid: id }, { activeflag: 'active' }] } }, - datasetRelatedResources, - datasetRelatedCourses, - datasetProjectFields, - ]); - - // 2. If dataset not found search for a dataset based on datasetID - if (!data || data.length <= 0) { - data = await Data.find({ datasetid: objectId }, { datasetid: 1, pid: 1 }).lean(); - // 3. Use retrieved dataset's pid to search by pid again - data = await Data.aggregate([ - { $match: { $and: [{ pid: data[0].pid }, { activeflag: 'active' }] } }, - datasetRelatedResources, - datasetRelatedCourses, - datasetProjectFields, - ]); - } - - // 4. If dataset still not found search for deleted dataset by pid - if (!data || data.length <= 0) { - data = await Data.aggregate([ - { $match: { $and: [{ pid: id }, { activeflag: 'archive' }] } }, - datasetRelatedResources, - datasetRelatedCourses, - datasetProjectFields, - ]); - } - } - - let relatedObject = { ...data[0], updated: Date.parse(updated) }; - resolve(relatedObject); - }); -} - -async function sendEmailNotifications(collections, activeflag, collectionCreator, isEdit) { - // Generate URL for linking collection in email - const collectionLink = process.env.homeURL + '/collection/' + collections.id; - - // Query Db for all admins or authors of the collection - var q = UserModel.aggregate([ - { $match: { $or: [{ role: 'Admin' }, { id: { $in: collections.authors } }] } }, - { $lookup: { from: 'tools', localField: 'id', foreignField: 'id', as: 'tool' } }, - { - $project: { - _id: 1, - firstname: 1, - lastname: 1, - email: 1, - role: 1, - id: 1, - }, - }, - ]); - - // Use the returned array of email recipients to generate and send emails with SendGrid - q.exec((err, emailRecipients) => { - if (err) { - return new Error({ success: false, error: err }); - } else { - let subject; - let html; - - emailRecipients.map(emailRecipient => { - if (collections.authors.includes(emailRecipient.id)) { - let author = Number(collections.authors.filter(author => author === emailRecipient.id)); - - if (activeflag === 'active') { - subject = generateCollectionEmailSubject( - 'Creator', - collections.publicflag, - collections.name, - author === collectionCreator.id ? true : false, - isEdit - ); - html = generateCollectionEmailContent( - 'Creator', - collections.publicflag, - collections.name, - collectionLink, - author === collectionCreator.id ? true : false, - isEdit - ); - } - } else if (activeflag === 'active' && emailRecipient.role === 'Admin') { - subject = generateCollectionEmailSubject('Admin', collections.publicflag, collections.name, false, isEdit); - html = generateCollectionEmailContent('Admin', collections.publicflag, collections.name, collectionLink, false, isEdit); - } - - emailGenerator.sendEmail([emailRecipient], `${hdrukEmail}`, subject, html, false); - }); - } - }); -} - -function generateCollectionEmailSubject(role, publicflag, collectionName, isCreator, isEdit) { - let emailSubject; - - if (role !== 'Admin' && isCreator !== true) { - if (isEdit === true) { - emailSubject = `The ${ - publicflag === true ? 'public' : 'private' - } collection ${collectionName} that you are a collaborator on has been edited and is now live`; - } else { - emailSubject = `You have been added as a collaborator on the ${ - publicflag === true ? 'public' : 'private' - } collection ${collectionName}`; - } - } else { - emailSubject = `${role === 'Admin' ? 'A' : 'Your'} ${ - publicflag === true ? 'public' : 'private' - } collection ${collectionName} has been ${isEdit === true ? 'edited' : 'published'} and is now live`; +export default class CollectionsRepository extends Repository { + constructor() { + super(Collections); + this.collections = Collections; } - return emailSubject; -} - -function generateCollectionEmailContent(role, publicflag, collectionName, collectionLink, isCreator, isEdit) { - return `
-
- - - - - - - - - - - - - - -
- ${generateCollectionEmailSubject(role, publicflag, collectionName, isCreator, isEdit)} -
- ${ - publicflag === true - ? `${role === 'Admin' ? 'A' : 'Your'} public collection has been ${ - isEdit === true ? 'edited on' : 'published to' - } the Gateway. The collection is searchable on the Gateway and can be viewed by all users.` - : `${role === 'Admin' ? 'A' : 'Your'} private collection has been ${ - isEdit === true ? 'edited on' : 'published to' - } the Gateway. Only those who you share the collection link with will be able to view the collection.` - } -
- View Collection -
-
-
`; -} - -const getCollectionsAdmin = async req => { - return new Promise(async resolve => { - let startIndex = 0; - let limit = 40; - let searchString = ''; - let status = 'all'; - - if (req.query.offset) { - startIndex = req.query.offset; - } - if (req.query.limit) { - limit = req.query.limit; - } - if (req.query.q) { - searchString = req.query.q || ''; - } - if (req.query.status) { - status = req.query.status; - } - - let searchQuery; - if (status === 'all') { - searchQuery = {}; - } else { - searchQuery = { $and: [{ activeflag: status }] }; - } - - let searchAll = false; - - if (searchString.length > 0) { - searchQuery['$and'].push({ $text: { $search: searchString } }); - } else { - searchAll = true; - } - - await Promise.all([getObjectResult(searchAll, searchQuery, startIndex, limit), getCountsByStatus()]).then(values => { - resolve(values); - }); - }); -}; - -const getCollections = async req => { - return new Promise(async resolve => { - let startIndex = 0; - let limit = 40; - let idString = req.user.id; - let status = 'all'; - - if (req.query.offset) { - startIndex = req.query.offset; - } - if (req.query.limit) { - limit = req.query.limit; - } - if (req.query.id) { - idString = req.query.id; - } - if (req.query.status) { - status = req.query.status; - } - - let searchQuery; - if (status === 'all') { - searchQuery = [{ authors: parseInt(idString) }]; - } else { - searchQuery = [{ authors: parseInt(idString) }, { activeflag: status }]; - } - - let query = Collections.aggregate([ - { $match: { $and: searchQuery } }, - { $lookup: { from: 'tools', localField: 'authors', foreignField: 'id', as: 'persons' } }, - { $sort: { updatedAt: -1, _id: 1 } }, - ]) - .skip(parseInt(startIndex)) - .limit(parseInt(limit)); - - await Promise.all([getUserCollections(query), getCountsByStatus(idString)]).then(values => { - resolve(values); - }); - - function getUserCollections(query) { - return new Promise(resolve => { - query.exec((err, data) => { - data && - data.map(dat => { - dat.persons = helper.hidePrivateProfileDetails(dat.persons); - }); - if (typeof data === 'undefined') resolve([]); - else resolve(data); - }); - }); - } - }); -}; - -function getObjectResult(searchAll, searchQuery, startIndex, limit) { - let newSearchQuery = JSON.parse(JSON.stringify(searchQuery)); - let q = ''; - - if (searchAll) { - q = Collections.aggregate([ - { $match: newSearchQuery }, - { $lookup: { from: 'tools', localField: 'authors', foreignField: 'id', as: 'persons' } }, - ]) - .sort({ updatedAt: -1, _id: 1 }) - .skip(parseInt(startIndex)) - .limit(parseInt(limit)); - } else { - q = Collections.aggregate([ - { $match: newSearchQuery }, - { $lookup: { from: 'tools', localField: 'authors', foreignField: 'id', as: 'persons' } }, - ]) - .sort({ score: { $meta: 'textScore' } }) - .skip(parseInt(startIndex)) - .limit(parseInt(limit)); + async getCollections(query, options) { + return this.find(query, options); } - return new Promise(resolve => { - q.exec((err, data) => { - if (typeof data === 'undefined') { - resolve([]); - } else { - data.map(dat => { - dat.persons = helper.hidePrivateProfileDetails(dat.persons); - }); - resolve(data); - } - }); - }); -} -function getCountsByStatus(idString) { - let q; - - if (_.isUndefined(idString)) { - q = Collections.find({}, { id: 1, name: 1, activeflag: 1 }); - } else { - q = Collections.find({ authors: parseInt(idString) }, { id: 1, name: 1, activeflag: 1 }); + async updateCollection(query, options) { + return this.updateByQuery(query, options); } - return new Promise(resolve => { - q.exec((err, data) => { - const activeCount = data.filter(dat => dat.activeflag === 'active').length; - const archiveCount = data.filter(dat => dat.activeflag === 'archive').length; - - let countSummary = { activeCount: activeCount, archiveCount: archiveCount }; - - resolve(countSummary); + async searchCollections(query) { + return new Promise(resolve => { + query.exec((err, data) => { + data && + data.map(dat => { + dat.persons = helper.hidePrivateProfileDetails(dat.persons); + }); + if (typeof data === 'undefined') resolve([]); + else resolve(data); + }); }); - }); + } } - -export { getCollectionObjects, getCollectionsAdmin, getCollections, sendEmailNotifications, generateCollectionEmailSubject }; diff --git a/src/resources/collections/collections.route.js b/src/resources/collections/collections.route.js index 04166403..adcc6c2c 100644 --- a/src/resources/collections/collections.route.js +++ b/src/resources/collections/collections.route.js @@ -1,278 +1,59 @@ import express from 'express'; -import { ROLES } from '../user/user.roles'; import passport from 'passport'; -import { utils } from '../auth'; -import { Collections } from '../collections/collections.model'; -import { Data } from '../tool/data.model'; -import { MessagesModel } from '../message/message.model'; -import { UserModel } from '../user/user.model'; -import helper from '../utilities/helper.util'; import _ from 'lodash'; -import escape from 'escape-html'; -import { - getCollectionObjects, - getCollectionsAdmin, - getCollections, - sendEmailNotifications, - generateCollectionEmailSubject, -} from './collections.repository'; -import inputSanitizer from '../utilities/inputSanitizer'; -import urlValidator from '../utilities/urlValidator'; -import { filtersService } from '../filters/dependency'; +import { utils } from '../auth'; +import CollectionsController from './collections.controller'; +import { collectionsService } from './dependency'; + +const collectionsController = new CollectionsController(collectionsService); const router = express.Router(); // @router GET /api/v1/collections/getList // @desc Returns List of Collections // @access Private -router.get('/getList', passport.authenticate('jwt'), async (req, res) => { - let role = req.user.role; - - if (role === ROLES.Admin) { - await getCollectionsAdmin(req) - .then(data => { - return res.json({ success: true, data }); - }) - .catch(err => { - return res.json({ success: false, err }); - }); - } else if (role === ROLES.Creator) { - await getCollections(req) - .then(data => { - return res.json({ success: true, data }); - }) - .catch(err => { - return res.json({ success: false, err }); - }); - } -}); +router.get('/getList', passport.authenticate('jwt'), (req, res) => collectionsController.getList(req, res)); // @router GET /api/v1/collections/{collectionID} // @desc Returns collection based on id // @access Public -router.get('/:collectionID', async (req, res) => { - var q = Collections.aggregate([ - { $match: { $and: [{ id: parseInt(req.params.collectionID) }] } }, - - { $lookup: { from: 'tools', localField: 'authors', foreignField: 'id', as: 'persons' } }, - ]); - q.exec((err, data) => { - if (err) return res.json({ success: false, error: err }); - - if (_.isEmpty(data)) return res.status(404).send(`Collection not found for Id: ${escape(req.params.collectionID)}`); - - data[0].persons = helper.hidePrivateProfileDetails(data[0].persons); - return res.json({ success: true, data: data }); - }); -}); +router.get('/:collectionID', (req, res) => collectionsController.getCollection(req, res)); // @router GET /api/v1/collections/relatedobjects/{collectionID} // @desc Returns related resources for collection based on id // @access Public -router.get('/relatedobjects/:collectionID', async (req, res) => { - await getCollectionObjects(req) - .then(data => { - return res.json({ success: true, data }); - }) - .catch(err => { - return res.json({ success: false, err }); - }); -}); +router.get('/relatedobjects/:collectionID', (req, res) => collectionsController.getCollectionRelatedResources(req, res)); // @router GET /api/v1/collections/entityid/{entityID} // @desc Returns collections that contant the entity id // @access Public -router.get('/entityid/:entityID', async (req, res) => { - let entityID = req.params.entityID; - let dataVersions = await Data.find({ pid: entityID }, { _id: 0, datasetid: 1 }); - let dataVersionsArray = dataVersions.map(a => a.datasetid); - dataVersionsArray.push(entityID); - - var q = Collections.aggregate([ - { - $match: { - $and: [ - { - relatedObjects: { - $elemMatch: { - $or: [ - { - objectId: { $in: dataVersionsArray }, - }, - { - pid: entityID, - }, - ], - }, - }, - }, - { publicflag: true }, - { activeflag: 'active' }, - ], - }, - }, - { $lookup: { from: 'tools', localField: 'authors', foreignField: 'id', as: 'persons' } }, - { - $project: { _id: 1, id: 1, name: 1, description: 1, imageLink: 1, relatedObjects: 1, 'persons.firstname': 1, 'persons.lastname': 1 }, - }, - ]); - - q.exec((err, data) => { - if (err) return res.json({ success: false, error: err }); - return res.json({ success: true, data: data }); - }); -}); +router.get('/entityid/:entityID', (req, res) => collectionsController.getCollectionByEntity(req, res)); // @router PUT /api/v1/collections/edit/{id} // @desc Edit Collection // @access Private -router.put('/edit/:id', passport.authenticate('jwt'), utils.checkAllowedToAccess('collection'), async (req, res) => { - let id = req.params.id; - let { name, description, imageLink, authors, relatedObjects, publicflag, keywords, previousPublicFlag, collectionCreator } = req.body; - imageLink = urlValidator.validateURL(imageLink); - let updatedon = Date.now(); - - let collectionId = parseInt(id); - - await Collections.findOneAndUpdate( - { id: { $eq: collectionId } }, - { - name: inputSanitizer.removeNonBreakingSpaces(name), - description: inputSanitizer.removeNonBreakingSpaces(description), - imageLink, - authors, - relatedObjects, - publicflag, - keywords, - updatedon, - }, - err => { - if (err) { - return res.json({ success: false, error: err }); - } - } - ).then(() => { - filtersService.optimiseFilters('collection'); - return res.json({ success: true }); - }); - - await Collections.find({ id: collectionId }, { publicflag: 1, id: 1, activeflag: 1, authors: 1, name: 1 }).then(async res => { - if (previousPublicFlag === false && publicflag === true) { - await sendEmailNotifications(res[0], res[0].activeflag, collectionCreator, true); - - if (res[0].authors) { - res[0].authors.forEach(async authorId => { - await createMessage(authorId, res[0], res[0].activeflag, collectionCreator, true); - }); - } - - await createMessage(0, res[0], res[0].activeflag, collectionCreator, true); - } - }); -}); +router.put('/edit/:id', passport.authenticate('jwt'), utils.checkAllowedToAccess('collection'), (req, res) => + collectionsController.editCollection(req, res) +); // @router POST /api/v1/collections/add // @desc Add Collection // @access Private -router.post('/add', passport.authenticate('jwt'), async (req, res) => { - let collections = new Collections(); - - const collectionCreator = req.body.collectionCreator; - - const { name, description, imageLink, authors, relatedObjects, publicflag, keywords } = req.body; - - collections.id = parseInt(Math.random().toString().replace('0.', '')); - collections.name = inputSanitizer.removeNonBreakingSpaces(name); - collections.description = inputSanitizer.removeNonBreakingSpaces(description); - collections.imageLink = imageLink; - collections.authors = authors; - collections.relatedObjects = relatedObjects; - collections.activeflag = 'active'; - collections.publicflag = publicflag; - collections.keywords = keywords; - collections.updatedon = Date.now(); - - if (collections.authors) { - collections.authors.forEach(async authorId => { - await createMessage(authorId, collections, collections.activeflag, collectionCreator); - }); - } - - await createMessage(0, collections, collections.activeflag, collectionCreator); - - await sendEmailNotifications(collections, collections.activeflag, collectionCreator); - - collections.save(err => { - if (err) { - return res.json({ success: false, error: err }); - } else { - return res.json({ success: true, id: collections.id }); - } - }); -}); +router.post('/add', passport.authenticate('jwt'), (req, res) => collectionsController.addCollection(req, res)); // @router PUT /api/v1/collections/status/{id} // @desc Edit Collection // @access Private -router.put('/status/:id', passport.authenticate('jwt'), utils.checkAllowedToAccess('collection'), async (req, res) => { - const collectionId = parseInt(req.params.id); - let { activeflag } = req.body; - activeflag = activeflag.toString(); - - Collections.findOneAndUpdate({ id: collectionId }, { activeflag }, err => { - if (err) { - return res.json({ success: false, error: err }); - } - }).then(() => { - filtersService.optimiseFilters('collection'); - return res.json({ success: true }); - }); -}); +router.put('/status/:id', passport.authenticate('jwt'), utils.checkAllowedToAccess('collection'), (req, res) => + collectionsController.changeStatus(req, res) +); // @router DELETE /api/v1/collections/delete/{id} // @desc Delete Collection // @access Private -router.delete('/delete/:id', passport.authenticate('jwt'), utils.checkAllowedToAccess('collection'), async (req, res) => { - const id = parseInt(req.params.id); - Collections.findOneAndRemove({ id }, err => { - if (err) return res.send(err); - return res.json({ success: true }); - }); -}); +router.delete('/delete/:id', passport.authenticate('jwt'), utils.checkAllowedToAccess('collection'), (req, res) => + collectionsController.deleteCollection(req, res) +); // eslint-disable-next-line no-undef module.exports = router; - -async function createMessage(authorId, collections, activeflag, collectionCreator, isEdit) { - let message = new MessagesModel(); - - const messageRecipients = await UserModel.find({ $or: [{ role: 'Admin' }, { id: { $in: collections.authors } }] }); - async function saveMessage() { - message.messageID = parseInt(Math.random().toString().replace('0.', '')); - message.messageTo = authorId; - message.messageObjectID = collections.id; - message.messageSent = Date.now(); - message.isRead = false; - await message.save(); - } - - if (authorId === 0) { - message.messageType = 'added collection'; - message.messageDescription = generateCollectionEmailSubject('Admin', collections.publicflag, collections.name, false, isEdit); - saveMessage(); - } - - for (let messageRecipient of messageRecipients) { - if (activeflag === 'active' && authorId === messageRecipient.id) { - message.messageType = 'added collection'; - message.messageDescription = generateCollectionEmailSubject( - 'Creator', - collections.publicflag, - collections.name, - authorId === collectionCreator.id ? true : false, - isEdit - ); - saveMessage(); - } - } -} diff --git a/src/resources/collections/collections.service.js b/src/resources/collections/collections.service.js new file mode 100644 index 00000000..506a4e60 --- /dev/null +++ b/src/resources/collections/collections.service.js @@ -0,0 +1,551 @@ +import { Data } from '../tool/data.model'; +import { Course } from '../course/course.model'; +import { Collections } from '../collections/collections.model'; +import { UserModel } from '../user/user.model'; +import emailGenerator from '../utilities/emailGenerator.util'; +import inputSanitizer from '../utilities/inputSanitizer'; +import _ from 'lodash'; + +export default class CollectionsService { + constructor(collectionsRepository) { + this.collectionsRepository = collectionsRepository; + this.hdrukEmail = 'enquiry@healthdatagateway.org'; + } + + async getCollectionObjects(collectionID) { + let relatedObjects = []; + await this.collectionsRepository + .getCollections( + { id: collectionID }, + { + 'relatedObjects._id': 1, + 'relatedObjects.objectId': 1, + 'relatedObjects.objectType': 1, + 'relatedObjects.pid': 1, + 'relatedObjects.updated': 1, + } + ) + .then(async res => { + await new Promise(async (resolve, reject) => { + if (_.isEmpty(res)) { + reject(`Collection not found for ID: ${collectionID}.`); + } else { + for (let object of res[0].relatedObjects) { + let relatedObject = await this.getCollectionObject(object.objectId, object.objectType, object.pid, object.updated); + if (!_.isUndefined(relatedObject)) { + relatedObjects.push(relatedObject); + } else { + await this.collectionsRepository.updateCollection({ id: collectionID }, { $pull: { relatedObjects: { _id: object._id } } }); + } + } + resolve(relatedObjects); + } + }); + }); + + return relatedObjects.sort((a, b) => b.updated - a.updated); + } + + getCollectionObject(objectId, objectType, pid, updated) { + let id = pid && pid.length > 0 ? pid : objectId; + + return new Promise(async resolve => { + let data; + if (objectType !== 'dataset' && objectType !== 'course') { + data = await Data.find( + { id: parseInt(id) }, + { + id: 1, + type: 1, + activeflag: 1, + tags: 1, + description: 1, + name: 1, + persons: 1, + categories: 1, + programmingLanguage: 1, + firstname: 1, + lastname: 1, + bio: 1, + authors: 1, + counter: { $ifNull: ['$counter', 0] }, + relatedresources: { $cond: { if: { $isArray: '$relatedObjects' }, then: { $size: '$relatedObjects' }, else: 0 } }, + } + ) + .populate([{ path: 'persons', options: { select: { id: 1, firstname: 1, lastname: 1 } } }]) + .lean(); + } else if (!isNaN(id) && objectType === 'course') { + data = await Course.find( + { id: parseInt(id) }, + { + id: 1, + type: 1, + activeflag: 1, + title: 1, + provider: 1, + courseOptions: 1, + award: 1, + domains: 1, + tags: 1, + description: 1, + counter: { $ifNull: ['$counter', 0] }, + relatedresources: { $cond: { if: { $isArray: '$relatedObjects' }, then: { $size: '$relatedObjects' }, else: 0 } }, + } + ).lean(); + } else { + const datasetRelatedResources = { + $lookup: { + from: 'tools', + let: { + pid: '$pid', + }, + pipeline: [ + { $unwind: '$relatedObjects' }, + { + $match: { + $expr: { + $and: [ + { + $eq: ['$relatedObjects.pid', '$$pid'], + }, + { + $eq: ['$activeflag', 'active'], + }, + ], + }, + }, + }, + { $group: { _id: null, count: { $sum: 1 } } }, + ], + as: 'relatedResourcesTools', + }, + }; + + const datasetRelatedCourses = { + $lookup: { + from: 'course', + let: { + pid: '$pid', + }, + pipeline: [ + { $unwind: '$relatedObjects' }, + { + $match: { + $expr: { + $and: [ + { + $eq: ['$relatedObjects.pid', '$$pid'], + }, + { + $eq: ['$activeflag', 'active'], + }, + ], + }, + }, + }, + { $group: { _id: null, count: { $sum: 1 } } }, + ], + as: 'relatedResourcesCourses', + }, + }; + + const datasetProjectFields = { + $project: { + id: 1, + datasetid: 1, + pid: 1, + type: 1, + activeflag: 1, + name: 1, + datasetv2: 1, + datasetfields: 1, + tags: 1, + description: 1, + counter: { $ifNull: ['$counter', 0] }, + relatedresources: { + $add: [ + { + $cond: { + if: { $eq: [{ $size: '$relatedResourcesTools' }, 0] }, + then: 0, + else: { $first: '$relatedResourcesTools.count' }, + }, + }, + { + $cond: { + if: { $eq: [{ $size: '$relatedResourcesCourses' }, 0] }, + then: 0, + else: { $first: '$relatedResourcesCourses.count' }, + }, + }, + ], + }, + }, + }; + + // 1. Search for a dataset based on pid + data = await Data.aggregate([ + { $match: { $and: [{ pid: id }, { activeflag: 'active' }] } }, + datasetRelatedResources, + datasetRelatedCourses, + datasetProjectFields, + ]); + + // 2. If dataset not found search for a dataset based on datasetID + if (!data || data.length <= 0) { + data = await Data.find({ datasetid: objectId }, { datasetid: 1, pid: 1 }).lean(); + // 3. Use retrieved dataset's pid to search by pid again + data = await Data.aggregate([ + { $match: { $and: [{ pid: data[0].pid }, { activeflag: 'active' }] } }, + datasetRelatedResources, + datasetRelatedCourses, + datasetProjectFields, + ]); + } + + // 4. If dataset still not found search for deleted dataset by pid + if (!data || data.length <= 0) { + data = await Data.aggregate([ + { $match: { $and: [{ pid: id }, { activeflag: 'archive' }] } }, + datasetRelatedResources, + datasetRelatedCourses, + datasetProjectFields, + ]); + } + } + + let relatedObject = { ...data[0], updated: Date.parse(updated) }; + resolve(relatedObject); + }); + } + + getCollectionByEntity(entityID, dataVersionsArray) { + var q = Collections.aggregate([ + { + $match: { + $and: [ + { + relatedObjects: { + $elemMatch: { + $or: [ + { + objectId: { $in: dataVersionsArray }, + }, + { + pid: entityID, + }, + ], + }, + }, + }, + { publicflag: true }, + { activeflag: 'active' }, + ], + }, + }, + { $lookup: { from: 'tools', localField: 'authors', foreignField: 'id', as: 'persons' } }, + { + $project: { + _id: 1, + id: 1, + name: 1, + description: 1, + imageLink: 1, + relatedObjects: 1, + 'persons.firstname': 1, + 'persons.lastname': 1, + }, + }, + ]); + + return new Promise((resolve, reject) => { + q.exec((err, data) => { + if (err) { + return reject(err); + } else { + return resolve(data); + } + }); + }); + } + + async editCollection(collectionID, updatedCollection) { + let { name, description, imageLink, authors, relatedObjects, publicflag, keywords } = updatedCollection; + let updatedon = Date.now(); + + return new Promise(async (resolve, reject) => { + await Collections.findOneAndUpdate( + { id: { $eq: collectionID } }, + { + name: inputSanitizer.removeNonBreakingSpaces(name), + description: inputSanitizer.removeNonBreakingSpaces(description), + imageLink, + authors, + relatedObjects, + publicflag, + keywords, + updatedon, + }, + err => { + err ? reject(err) : resolve(); + } + ); + }); + } + + addCollection(collections) { + return new Promise(async (resolve, reject) => { + try { + await collections.save(); + resolve(); + } catch (err) { + reject({ success: false, error: err }); + } + }); + } + + changeStatus(collectionID, activeflag) { + return new Promise(async (resolve, reject) => { + Collections.findOneAndUpdate({ id: collectionID }, { activeflag }, err => { + err ? reject(err) : resolve(); + }); + }); + } + + deleteCollection(id) { + return new Promise(async (resolve, reject) => { + await Collections.findOneAndRemove({ id }, err => { + err ? reject(err) : resolve(); + }); + }); + } + + async sendEmailNotifications(collections, activeflag, collectionCreator, isEdit) { + // Generate URL for linking collection in email + const collectionLink = process.env.homeURL + '/collection/' + collections.id; + + // Query Db for all admins or authors of the collection + var q = UserModel.aggregate([ + { $match: { $or: [{ role: 'Admin' }, { id: { $in: collections.authors } }] } }, + { $lookup: { from: 'tools', localField: 'id', foreignField: 'id', as: 'tool' } }, + { + $project: { + _id: 1, + firstname: 1, + lastname: 1, + email: 1, + role: 1, + id: 1, + }, + }, + ]); + + // Use the returned array of email recipients to generate and send emails with SendGrid + q.exec((err, emailRecipients) => { + if (err) { + return new Error({ success: false, error: err }); + } else { + let subject; + let html; + + emailRecipients.map(emailRecipient => { + if (collections.authors.includes(emailRecipient.id)) { + let author = Number(collections.authors.filter(author => author === emailRecipient.id)); + + if (activeflag === 'active') { + subject = this.generateCollectionEmailSubject( + 'Creator', + collections.publicflag, + collections.name, + author === collectionCreator.id ? true : false, + isEdit + ); + html = this.generateCollectionEmailContent( + 'Creator', + collections.publicflag, + collections.name, + collectionLink, + author === collectionCreator.id ? true : false, + isEdit + ); + } + } else if (activeflag === 'active' && emailRecipient.role === 'Admin') { + subject = this.generateCollectionEmailSubject('Admin', collections.publicflag, collections.name, false, isEdit); + html = this.generateCollectionEmailContent('Admin', collections.publicflag, collections.name, collectionLink, false, isEdit); + } + + emailGenerator.sendEmail([emailRecipient], `${this.hdrukEmail}`, subject, html, false); + }); + } + }); + } + + generateCollectionEmailSubject(role, publicflag, collectionName, isCreator, isEdit) { + let emailSubject; + + if (role !== 'Admin' && isCreator !== true) { + if (isEdit === true) { + emailSubject = `The ${ + publicflag === true ? 'public' : 'private' + } collection ${collectionName} that you are a collaborator on has been edited and is now live`; + } else { + emailSubject = `You have been added as a collaborator on the ${ + publicflag === true ? 'public' : 'private' + } collection ${collectionName}`; + } + } else { + emailSubject = `${role === 'Admin' ? 'A' : 'Your'} ${ + publicflag === true ? 'public' : 'private' + } collection ${collectionName} has been ${isEdit === true ? 'edited' : 'published'} and is now live`; + } + + return emailSubject; + } + + generateCollectionEmailContent(role, publicflag, collectionName, collectionLink, isCreator, isEdit) { + return `
+
+ + + + + + + + + + + + + + +
+ ${this.generateCollectionEmailSubject(role, publicflag, collectionName, isCreator, isEdit)} +
+ ${ + publicflag === true + ? `${role === 'Admin' ? 'A' : 'Your'} public collection has been ${ + isEdit === true ? 'edited on' : 'published to' + } the Gateway. The collection is searchable on the Gateway and can be viewed by all users.` + : `${role === 'Admin' ? 'A' : 'Your'} private collection has been ${ + isEdit === true ? 'edited on' : 'published to' + } the Gateway. Only those who you share the collection link with will be able to view the collection.` + } +
+ View Collection +
+
+
`; + } + + async getCollectionsAdmin(searchString, status, startIndex, limit) { + return new Promise(async resolve => { + let searchQuery; + if (status === 'all') { + searchQuery = {}; + } else { + searchQuery = { $and: [{ activeflag: status }] }; + } + + let searchAll = false; + + if (searchString.length > 0) { + searchQuery['$and'].push({ $text: { $search: searchString } }); + } else { + searchAll = true; + } + + await Promise.all([this.getObjectResult(searchAll, searchQuery, startIndex, limit), this.getCountsByStatus()]).then(values => { + resolve(values); + }); + }); + } + + async getCollections(idString, status, startIndex, limit) { + return new Promise(async resolve => { + let searchQuery; + if (status === 'all') { + searchQuery = [{ authors: parseInt(idString) }]; + } else { + searchQuery = [{ authors: parseInt(idString) }, { activeflag: status }]; + } + + let query = Collections.aggregate([ + { $match: { $and: searchQuery } }, + { $lookup: { from: 'tools', localField: 'authors', foreignField: 'id', as: 'persons' } }, + { $sort: { updatedAt: -1, _id: 1 } }, + ]) + .skip(parseInt(startIndex)) + .limit(parseInt(limit)); + + await Promise.all([this.collectionsRepository.searchCollections(query), this.getCountsByStatus(idString)]).then(values => { + resolve(values); + }); + }); + } + + async getCollection(collectionID) { + var q = Collections.aggregate([ + { $match: { $and: [{ id: collectionID }] } }, + + { $lookup: { from: 'tools', localField: 'authors', foreignField: 'id', as: 'persons' } }, + ]); + return new Promise((resolve, reject) => { + q.exec((err, data) => { + err ? reject(err) : resolve(data); + }); + }); + } + + getObjectResult(searchAll, searchQuery, startIndex, limit) { + let newSearchQuery = JSON.parse(JSON.stringify(searchQuery)); + let q = ''; + + if (searchAll) { + q = Collections.aggregate([ + { $match: newSearchQuery }, + { $lookup: { from: 'tools', localField: 'authors', foreignField: 'id', as: 'persons' } }, + ]) + .sort({ updatedAt: -1, _id: 1 }) + .skip(parseInt(startIndex)) + .limit(parseInt(limit)); + } else { + q = Collections.aggregate([ + { $match: newSearchQuery }, + { $lookup: { from: 'tools', localField: 'authors', foreignField: 'id', as: 'persons' } }, + ]) + .sort({ score: { $meta: 'textScore' } }) + .skip(parseInt(startIndex)) + .limit(parseInt(limit)); + } + return this.collectionsRepository.searchCollections(q); + } + + getCountsByStatus(idString) { + let q; + + if (_.isUndefined(idString)) { + q = Collections.find({}, { id: 1, name: 1, activeflag: 1 }); + } else { + q = Collections.find({ authors: parseInt(idString) }, { id: 1, name: 1, activeflag: 1 }); + } + + return new Promise(resolve => { + q.exec((err, data) => { + const activeCount = data.filter(dat => dat.activeflag === 'active').length; + const archiveCount = data.filter(dat => dat.activeflag === 'archive').length; + + let countSummary = { activeCount: activeCount, archiveCount: archiveCount }; + + resolve(countSummary); + }); + }); + } +} diff --git a/src/resources/collections/dependency.js b/src/resources/collections/dependency.js new file mode 100644 index 00000000..ef9d18e4 --- /dev/null +++ b/src/resources/collections/dependency.js @@ -0,0 +1,6 @@ +import CollectionsService from './collections.service'; +import CollectionsRepository from './collections.repository'; + +const collectionsRepository = new CollectionsRepository(); + +export const collectionsService = new CollectionsService(collectionsRepository); diff --git a/src/resources/course/__tests__/course.repository.it.test.js b/src/resources/course/__tests__/course.repository.it.test.js index d8156ff3..ed57a444 100644 --- a/src/resources/course/__tests__/course.repository.it.test.js +++ b/src/resources/course/__tests__/course.repository.it.test.js @@ -35,7 +35,7 @@ describe('CourseRepository', function () { describe('getCourses', () => { it('should return an array of courses', async function () { const courseRepository = new CourseRepository(); - const courses = await courseRepository.getCourses(); + const courses = await courseRepository.getCourses({}, {}); expect(courses.length).toBeGreaterThan(0); }); }); diff --git a/src/resources/course/__tests__/course.repository.test.js b/src/resources/course/__tests__/course.repository.test.js index c07f07ec..1d38fbd7 100644 --- a/src/resources/course/__tests__/course.repository.test.js +++ b/src/resources/course/__tests__/course.repository.test.js @@ -41,7 +41,7 @@ describe('CourseRepository', function () { it('should return an array of courses', async function () { const courseRepository = new CourseRepository(); const stub = sinon.stub(courseRepository, 'find').returns(coursesStub); - const courses = await courseRepository.getCourses(); + const courses = await courseRepository.getCourses({}, {}); expect(stub.calledOnce).toBe(true); @@ -54,10 +54,10 @@ describe('CourseRepository', function () { const courseRepository = new CourseRepository(); const stub = sinon.stub(courseRepository, 'findCountOf').returns(1); const courseCount = await courseRepository.findCountOf({ name: 'Admitted Patient Care Course' }); - + expect(stub.calledOnce).toBe(true); expect(courseCount).toEqual(1); }); }); -}); \ No newline at end of file +}); From a9e0148b95bf2ed72f4da5de85fa1b54b064f323 Mon Sep 17 00:00:00 2001 From: Pritesh Bhole Date: Fri, 15 Oct 2021 12:03:34 +0100 Subject: [PATCH 050/116] added N/A values --- migrations/1627566998386-add_globals.js | 36 +++++++++++++++++++++++++ migrations/README.md | 5 ++-- 2 files changed, 39 insertions(+), 2 deletions(-) diff --git a/migrations/1627566998386-add_globals.js b/migrations/1627566998386-add_globals.js index 69129778..a701310d 100644 --- a/migrations/1627566998386-add_globals.js +++ b/migrations/1627566998386-add_globals.js @@ -101,6 +101,13 @@ const globalData = { label: '1 year', impliedValues: ['platinum', 'gold', 'silver', 'bronze'], }, + { + id: mongoose.Types.ObjectId(), + displayOrder: 5, + definition: 'N/A', + label: 'N/A', + impliedValues: [], + }, ], }, { @@ -232,6 +239,13 @@ const globalData = { label: 'More than 10 years', impliedValues: ['platinum'], }, + { + id: mongoose.Types.ObjectId(), + displayOrder: 5, + definition: 'N/A', + label: 'N/A', + impliedValues: [], + }, ], }, { @@ -306,6 +320,13 @@ const globalData = { label: 'Model conforms to national standard and key fields coded to national/internal standard', impliedValues: ['platinum'], }, + { + id: mongoose.Types.ObjectId(), + displayOrder: 5, + definition: 'N/A', + label: 'N/A', + impliedValues: [], + }, ], }, { @@ -380,6 +401,21 @@ const globalData = { label: "Earlier and 'raw' versions and the impact of each stage of data cleaning", impliedValues: ['platinum'], }, + { + id: mongoose.Types.ObjectId(), + displayOrder: 4, + definition: + 'Ability to view earlier versions, including versions before any transformations have been applied data (in line with deidentification and IG approval) and review the impact of each stage of data cleaning', + label: "Earlier and 'raw' versions and the impact of each stage of data cleaning", + impliedValues: ['platinum'], + }, + { + id: mongoose.Types.ObjectId(), + displayOrder: 4, + definition:'N/A', + label: "N/A", + impliedValues: [], + }, ], }, { diff --git a/migrations/README.md b/migrations/README.md index c31a180b..bbd6eeb6 100644 --- a/migrations/README.md +++ b/migrations/README.md @@ -27,11 +27,12 @@ Complete the scripts required for the UP process, and if possible, the DOWN proc #### Step 4 -With the scripts written, the functions can be tested by running the following command, replacing 'my_new_migration_script' with the name of the script you want to execute. +With the scripts written, the functions can be tested by running the following command, replacing 'my_new_migration_script' with the name of the script you want to execute without the time stamp so for example +node -r esm migrations/migrate.js up my_new_migration_script node -r esm migrations/migrate.js up my_new_migration_script -When this process is completed, the connected database will have a new document representing your migration scripts inside the 'migrations' collection, which tracks the state of the migration. If you need to run your scripts multiple times for test purposes, you can change the state of the migration to 'Down'. +When this process is completed, the connected database will have a new document representing your migration scripts inside the 'migrations' collection, which tracks the state of the migration. If you need to run your scripts multiple times for test purposes, you can change the state of the migration to 'Down'. During this process, please ensure you are using a personal database. From 655838c400818ce78121f2e6208669758e86bdc6 Mon Sep 17 00:00:00 2001 From: CiaraWardPA Date: Fri, 15 Oct 2021 14:53:20 +0100 Subject: [PATCH 051/116] IG-2281 fix for courses in top searches --- src/resources/stats/stats.repository.js | 50 +++++++++++++++++-------- 1 file changed, 35 insertions(+), 15 deletions(-) diff --git a/src/resources/stats/stats.repository.js b/src/resources/stats/stats.repository.js index eeeeb993..a347d383 100644 --- a/src/resources/stats/stats.repository.js +++ b/src/resources/stats/stats.repository.js @@ -331,7 +331,7 @@ export default class StatsRepository extends Repository { topSearch.tools = resources[1][0] !== undefined && resources[1][0].count !== undefined ? resources[1][0].count : 0; topSearch.projects = resources[2][0] !== undefined && resources[2][0].count !== undefined ? resources[2][0].count : 0; topSearch.papers = resources[3][0] !== undefined && resources[3][0].count !== undefined ? resources[3][0].count : 0; - topSearch.course = resources[4][0] !== undefined && resources[4][0].count !== undefined ? resources[4][0].count : 0; + topSearch.courses = resources[4][0] !== undefined && resources[4][0].count !== undefined ? resources[4][0].count : 0; }); return topSearch; }) @@ -346,23 +346,43 @@ export default class StatsRepository extends Repository { newSearchQuery['$and'].push({ type }); var q = ''; - q = Data.aggregate([ - { $match: newSearchQuery }, - { - $group: { - _id: {}, - count: { - $sum: 1, + if (type === 'course') { + q = Course.aggregate([ + { $match: newSearchQuery }, + { + $group: { + _id: {}, + count: { + $sum: 1, + }, }, }, - }, - { - $project: { - count: '$count', - _id: 0, + { + $project: { + count: '$count', + _id: 0, + }, }, - }, - ]); + ]); + } else { + q = Data.aggregate([ + { $match: newSearchQuery }, + { + $group: { + _id: {}, + count: { + $sum: 1, + }, + }, + }, + { + $project: { + count: '$count', + _id: 0, + }, + }, + ]); + } return new Promise((resolve, reject) => { q.exec((err, data) => { From 6d58d34cd5224f8abb3c858ef6ae7b2d200a3216 Mon Sep 17 00:00:00 2001 From: Pritesh Bhole Date: Fri, 15 Oct 2021 15:28:40 +0100 Subject: [PATCH 052/116] updated as per comment --- src/resources/auth/utils.js | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/src/resources/auth/utils.js b/src/resources/auth/utils.js index f1e236ce..53f342d0 100644 --- a/src/resources/auth/utils.js +++ b/src/resources/auth/utils.js @@ -129,21 +129,7 @@ const getTeams = async () => { const catchLoginErrorAndRedirect = (req, res, next) => { if (req.auth.err || !req.auth.user) { - if (req.auth.err === 'loginError' || req.auth.user === undefined) { - return res.status(200).redirect(process.env.homeURL + '/loginerror'); - } - - let redirect = '/'; - let returnPage = null; - if (req.param.returnpage) { - returnPage = Url.parse(req.param.returnpage); - redirect = returnPage.path; - delete req.param.returnpage; - } - - let redirectUrl = process.env.homeURL + redirect; - - return res.status(200).redirect(redirectUrl); + return res.status(200).redirect(process.env.homeURL + '/loginerror'); } next(); }; From 44436bd210c9e657b4f8035d35e3abeb11e6f097 Mon Sep 17 00:00:00 2001 From: CiaraWardPA Date: Fri, 15 Oct 2021 16:29:50 +0100 Subject: [PATCH 053/116] IG-2281 data uses added to top searches stats api, projects removed --- src/resources/stats/stats.repository.js | 66 +++++++++++-------------- 1 file changed, 28 insertions(+), 38 deletions(-) diff --git a/src/resources/stats/stats.repository.js b/src/resources/stats/stats.repository.js index a347d383..ecc075f8 100644 --- a/src/resources/stats/stats.repository.js +++ b/src/resources/stats/stats.repository.js @@ -323,15 +323,15 @@ export default class StatsRepository extends Repository { await Promise.all([ this.getObjectResult('dataset', searchQuery), this.getObjectResult('tool', searchQuery), - this.getObjectResult('project', searchQuery), this.getObjectResult('paper', searchQuery), this.getObjectResult('course', searchQuery), + this.getObjectResult('dataUseRegister', searchQuery), ]).then(resources => { topSearch.datasets = resources[0][0] !== undefined && resources[0][0].count !== undefined ? resources[0][0].count : 0; topSearch.tools = resources[1][0] !== undefined && resources[1][0].count !== undefined ? resources[1][0].count : 0; - topSearch.projects = resources[2][0] !== undefined && resources[2][0].count !== undefined ? resources[2][0].count : 0; - topSearch.papers = resources[3][0] !== undefined && resources[3][0].count !== undefined ? resources[3][0].count : 0; - topSearch.courses = resources[4][0] !== undefined && resources[4][0].count !== undefined ? resources[4][0].count : 0; + topSearch.papers = resources[2][0] !== undefined && resources[2][0].count !== undefined ? resources[2][0].count : 0; + topSearch.courses = resources[3][0] !== undefined && resources[3][0].count !== undefined ? resources[3][0].count : 0; + topSearch.dataUseRegisters = resources[4][0] !== undefined && resources[4][0].count !== undefined ? resources[4][0].count : 0; }); return topSearch; }) @@ -346,43 +346,33 @@ export default class StatsRepository extends Repository { newSearchQuery['$and'].push({ type }); var q = ''; - if (type === 'course') { - q = Course.aggregate([ - { $match: newSearchQuery }, - { - $group: { - _id: {}, - count: { - $sum: 1, - }, - }, - }, - { - $project: { - count: '$count', - _id: 0, - }, - }, - ]); - } else { - q = Data.aggregate([ - { $match: newSearchQuery }, - { - $group: { - _id: {}, - count: { - $sum: 1, - }, + const typeMapper = { + dataset: Data, + tool: Data, + paper: Data, + course: Course, + dataUseRegister: DataUseRegister, + }; + + const model = typeMapper[type]; + + q = model.aggregate([ + { $match: newSearchQuery }, + { + $group: { + _id: {}, + count: { + $sum: 1, }, }, - { - $project: { - count: '$count', - _id: 0, - }, + }, + { + $project: { + count: '$count', + _id: 0, }, - ]); - } + }, + ]); return new Promise((resolve, reject) => { q.exec((err, data) => { From 7f9247852ecabf8e6c0c9a10da3f6fc962821436 Mon Sep 17 00:00:00 2001 From: CiaraWardPA Date: Mon, 18 Oct 2021 11:57:25 +0100 Subject: [PATCH 054/116] IG-2281 data use added to recorded search model and and returned with unmet demand stats --- src/resources/search/record.search.model.js | 1 + src/resources/stats/stats.repository.js | 1 + src/resources/stats/stats.service.js | 1 + 3 files changed, 3 insertions(+) diff --git a/src/resources/search/record.search.model.js b/src/resources/search/record.search.model.js index c2d62137..da1ce476 100644 --- a/src/resources/search/record.search.model.js +++ b/src/resources/search/record.search.model.js @@ -9,6 +9,7 @@ const RecordSearchSchema = new Schema( project: Number, paper: Number, person: Number, + datause: Number, }, datesearched: Date, }, diff --git a/src/resources/stats/stats.repository.js b/src/resources/stats/stats.repository.js index ecc075f8..260d253c 100644 --- a/src/resources/stats/stats.repository.js +++ b/src/resources/stats/stats.repository.js @@ -402,6 +402,7 @@ export default class StatsRepository extends Repository { maxPapers: { $max: '$returned.paper' }, maxCourses: { $max: '$returned.course' }, maxPeople: { $max: '$returned.people' }, + maxDataUses: { $max: '$returned.datause' }, entity: { $max: entityType }, }, }, diff --git a/src/resources/stats/stats.service.js b/src/resources/stats/stats.service.js index 2aec5aec..415ac5b5 100644 --- a/src/resources/stats/stats.service.js +++ b/src/resources/stats/stats.service.js @@ -172,4 +172,5 @@ const entityTypeMap = { Courses: 'course', Papers: 'papers', People: 'person', + DataUses: 'datause', }; From 0763e40815ed3755e66688f36f7b268937511ef3 Mon Sep 17 00:00:00 2001 From: CiaraWardPA Date: Mon, 18 Oct 2021 16:10:39 +0100 Subject: [PATCH 055/116] IG-2281 course added to record search model --- src/resources/search/record.search.model.js | 1 + 1 file changed, 1 insertion(+) diff --git a/src/resources/search/record.search.model.js b/src/resources/search/record.search.model.js index da1ce476..a166a32d 100644 --- a/src/resources/search/record.search.model.js +++ b/src/resources/search/record.search.model.js @@ -10,6 +10,7 @@ const RecordSearchSchema = new Schema( paper: Number, person: Number, datause: Number, + course: Number, }, datesearched: Date, }, From 9350d6c942d65b15a2f1b174fab7c03480126770 Mon Sep 17 00:00:00 2001 From: Paul McCafferty <> Date: Tue, 19 Oct 2021 14:59:03 +0100 Subject: [PATCH 056/116] Fix for is5Safes flag not getting correctly set and also changes to allow for more than one error to be returned for bulk upload and an extra check for drafts that are in review --- .../dataset/datasetonboarding.controller.js | 2 +- .../dataset/utils/datasetonboarding.util.js | 25 +++++++++++-------- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/src/resources/dataset/datasetonboarding.controller.js b/src/resources/dataset/datasetonboarding.controller.js index aa078500..76d2f38c 100644 --- a/src/resources/dataset/datasetonboarding.controller.js +++ b/src/resources/dataset/datasetonboarding.controller.js @@ -841,7 +841,7 @@ module.exports = { data.type = 'dataset'; data.activeflag = 'draft'; data.source = 'HDRUK MDC'; - data.is5Safes = dataset.publisher.allowAccessRequestManagement; + data.is5Safes = dataset.publisher.uses5Safes; data.timestamps.created = Date.now(); data.timestamps.updated = Date.now(); data.questionAnswers = JSON.stringify(dataset.questionAnswers); diff --git a/src/resources/dataset/utils/datasetonboarding.util.js b/src/resources/dataset/utils/datasetonboarding.util.js index 98d2c183..296a727c 100644 --- a/src/resources/dataset/utils/datasetonboarding.util.js +++ b/src/resources/dataset/utils/datasetonboarding.util.js @@ -1022,9 +1022,8 @@ const buildBulkUploadObject = async arrayOfDraftDatasets => { //Check to see that publisher exists publisher = await PublisherModel.findOne({ _id: { $eq: dataset.summary.publisher } }).lean(); if (isEmpty(publisher)) { - resultObject.error = `${dataset.summary.title} failed because publisher was no found`; + resultObject.error.push(`${dataset.summary.title} failed because publisher was no found`); resultObject.result = false; - break; } //Check to see if this is a new entry or a new version @@ -1054,17 +1053,24 @@ const buildBulkUploadObject = async arrayOfDraftDatasets => { //If no pid then all the datasets in the revision history do not exist on the Gateway if (isEmpty(pid)) { - resultObject.error = `${dataset.summary.title} failed because there was revision history but did not match an existing dataset on the Gateway`; + resultObject.error.push( + `${dataset.summary.title} failed because there was revision history but did not match an existing dataset on the Gateway` + ); resultObject.result = false; - break; } //Check there is not already a draft let isDraft = await Data.findOne({ pid, activeflag: 'draft' }, { pid: 1 }).lean(); if (!isEmpty(isDraft)) { - resultObject.error = `${dataset.summary.title} failed because there was already a draft for this dataset`; + resultObject.error.push(`${dataset.summary.title} failed because there was already a draft for this dataset`); + resultObject.result = false; + } + + //Check there is not already a draft in review + let isDraftInReview = await Data.findOne({ pid, activeflag: 'inReview' }, { pid: 1 }).lean(); + if (!isEmpty(isDraftInReview)) { + resultObject.error.push(`${dataset.summary.title} failed because there was already a draft in review for this dataset`); resultObject.result = false; - break; } } @@ -1077,19 +1083,18 @@ const buildBulkUploadObject = async arrayOfDraftDatasets => { title: dataset.summary.title, }); } else { - resultObject.error = `${dataset.summary.title} failed because there was no publisher`; + resultObject.error.push(`${dataset.summary.title} failed because there was no publisher`); resultObject.result = false; - break; } } catch (err) { - resultObject.error = `${dataset.summary.title} failed because ${err}`; + resultObject.error.push(`${dataset.summary.title} failed because ${err}`); resultObject.result = false; } } return resultObject; } catch (err) { - resultObject.error = `Failed because ${err}`; + resultObject.error.push(`Failed because ${err}`); resultObject.result = false; } }; From 79aa78a577a4277b2e807beee442101a31a2b9f0 Mon Sep 17 00:00:00 2001 From: Callum Reekie Date: Tue, 19 Oct 2021 17:58:18 +0100 Subject: [PATCH 057/116] CR - swagger docs for collections and JSON schema --- docs/index.docs.js | 9 +- docs/resources/collections.docs.js | 227 +++++++++++++++++++++++++++++ docs/schemas/collections.schema.js | 58 ++++++++ 3 files changed, 293 insertions(+), 1 deletion(-) create mode 100644 docs/resources/collections.docs.js create mode 100644 docs/schemas/collections.schema.js diff --git a/docs/index.docs.js b/docs/index.docs.js index 6dfa4f65..ea890f34 100644 --- a/docs/index.docs.js +++ b/docs/index.docs.js @@ -11,6 +11,9 @@ import project from './resources/project.docs'; import paper from './resources/paper.docs'; import tool from './resources/tool.docs'; import course from './resources/course.docs'; +import collection from './resources/collections.docs'; + +import collectionsSchema from './schemas/collections.schema'; module.exports = { openapi: '3.0.1', @@ -59,6 +62,7 @@ module.exports = { ...paper, ...tool, ...course, + ...collection, }, components: { securitySchemes: { @@ -73,8 +77,11 @@ module.exports = { }, cookieAuth: { type: 'http', - scheme: 'jwt', + scheme: 'bearer', }, }, + schemas: { + Collections: { ...collectionsSchema }, + }, }, }; diff --git a/docs/resources/collections.docs.js b/docs/resources/collections.docs.js new file mode 100644 index 00000000..2cff0111 --- /dev/null +++ b/docs/resources/collections.docs.js @@ -0,0 +1,227 @@ +module.exports = { + '/api/v1/collections/getList': { + get: { + summary: 'Returns a list of collections', + security: [ + { + cookieAuth: [], + }, + ], + parameters: [], + description: 'Returns a list of collections', + tags: ['Collections'], + responses: { + 200: { + description: 'Successful response containing a list of collections', + }, + 401: { + description: 'Unauthorized', + }, + }, + }, + }, + '/api/v1/collections/{id}': { + get: { + summary: 'Returns a specific collection', + parameters: [ + { + in: 'path', + name: 'id', + required: true, + description: 'The ID of the collection', + schema: { + type: 'integer', + format: 'int64', + minimum: 1, + example: 2181307729084665, + }, + }, + ], + description: 'Returns a single, public collection including its related resource(s)', + tags: ['Collections'], + responses: { + 200: { + description: 'Successful response containing a single collection object', + }, + 404: { + description: 'Collection not found for ID: {id}', + }, + }, + }, + }, + '/api/v1/collections/relatedobjects/{id}': { + get: { + summary: 'Returns related resource(s) of a collection', + parameters: [ + { + in: 'path', + name: 'id', + required: true, + description: 'The ID of the collection', + schema: { + type: 'integer', + format: 'int64', + minimum: 1, + example: 5968326934600661, + }, + }, + ], + description: 'Returns an array of the related resource(s) of a given collection', + tags: ['Collections'], + responses: { + 200: { + description: 'Successful response containing the related resource(s)', + }, + }, + }, + }, + '/api/v1/collections/entityid/{id}': { + get: { + summary: 'Returns collection array for a given entity', + parameters: [ + { + in: 'path', + name: 'id', + required: true, + description: 'The ID of the entity', + schema: { + type: 'string', + format: 'uuid', + example: 'c1f4b16c-9dfa-48e5-94ee-f0aa58c270e4', + }, + }, + ], + description: 'Returns an array of the collection(s) in which a given entity (e.g., dataset or paper) can be found', + tags: ['Collections'], + responses: { + 200: { + description: 'Successful response containing the collection(s)', + }, + }, + }, + }, + '/api/v1/collections/edit/{id}': { + put: { + summary: 'Edit a collection', + parameters: [ + { + in: 'path', + name: 'id', + required: true, + description: 'The ID of the collection', + schema: { + type: 'integer', + format: 'int64', + minimum: 1, + example: 5968326934600661, + }, + }, + ], + description: + 'Edit a collection by posting the updated collection object. This JSON body is validated server-side for structure and field type', + tags: ['Collections'], + responses: { + 200: { + description: 'Successful response detailing whether the update was successful or not', + }, + }, + }, + }, + '/api/v1/collections/status/{id}': { + put: { + summary: 'Change the status of a collection', + security: [ + { + cookieAuth: [], + }, + ], + parameters: [], + requestBody: { + required: true, + content: { + 'application/json': { + schema: { + type: 'object', + properties: { + activeflag: { + type: 'string', + enum: ['active', 'archive'], + }, + }, + }, + }, + }, + }, + description: 'Change the status of a collection', + tags: ['Collections'], + responses: { + 200: { + description: 'Successful response detailing whether the change of status was successful or not', + }, + }, + }, + }, + '/api/v1/collections/add': { + post: { + summary: 'Add a new collection', + security: [ + { + cookieAuth: [], + }, + ], + parameters: [], + requestBody: { + required: true, + content: { + 'application/json': { + schema: { + $ref: '#/components/schemas/Collections', + }, + }, + }, + }, + description: + 'Add a collection by posting a new collection object conforming to the schema. This JSON body is validated server-side for structure and field type', + tags: ['Collections'], + responses: { + 200: { + description: 'Successful response detailing whether the new collection addition was successful or not', + }, + }, + }, + }, + '/api/v1/collections/delete/{id}': { + delete: { + summary: 'Delete a collection', + security: [ + { + cookieAuth: [], + }, + ], + parameters: [ + { + in: 'path', + name: 'id', + required: true, + description: 'The ID of the collection', + schema: { + type: 'integer', + format: 'int64', + minimum: 1, + example: 5968326934600661, + }, + }, + ], + description: 'Delete a collection', + tags: ['Collections'], + responses: { + 200: { + description: 'Successful response detailing whether deleting the collection was a success', + }, + 401: { + description: 'Unauthorized', + }, + }, + }, + }, +}; diff --git a/docs/schemas/collections.schema.js b/docs/schemas/collections.schema.js new file mode 100644 index 00000000..13294cdc --- /dev/null +++ b/docs/schemas/collections.schema.js @@ -0,0 +1,58 @@ +module.exports = { + $schema: 'http://json-schema.org/draft-07/schema', + title: 'Collections schema', + type: 'object', + properties: { + name: { + type: 'string', + }, + description: { + type: 'string', + }, + imageLink: { + type: 'string', + }, + authors: { + type: 'array', + minItems: 0, + items: { + type: 'integer', + }, + }, + relatedObjects: { + type: 'array', + minItems: 0, + items: { + type: 'object', + properties: { + reason: { + type: 'string', + }, + objectType: { + type: 'string', + }, + pid: { + type: 'string', + }, + user: { + type: 'string', + }, + updated: { + type: 'string', + }, + }, + }, + }, + publicflag: { + type: 'boolean', + }, + keywords: { + type: 'array', + minItems: 0, + items: { + type: 'integer', + }, + }, + }, + required: ['name', 'description', 'publicflag', 'authors'], +}; From ac4bccf1dcd68220d9f08057cb92e2281c1ef339 Mon Sep 17 00:00:00 2001 From: massimocianfroccaPA Date: Wed, 20 Oct 2021 10:24:09 +0100 Subject: [PATCH 058/116] Add notifications when updating DUR --- .../dataUseRegister.controller.js | 77 ++++++++++- .../dataUseRegister/dataUseRegister.model.js | 1 + .../dataUseRegister.repository.js | 3 + .../dataUseRegister/dataUseRegister.route.js | 3 +- .../dataUseRegister.service.js | 2 +- src/resources/utilities/constants.util.js | 6 + .../utilities/emailGenerator.util.js | 125 ++++++++++++++---- 7 files changed, 183 insertions(+), 34 deletions(-) diff --git a/src/resources/dataUseRegister/dataUseRegister.controller.js b/src/resources/dataUseRegister/dataUseRegister.controller.js index fa2f381c..038f3ef5 100644 --- a/src/resources/dataUseRegister/dataUseRegister.controller.js +++ b/src/resources/dataUseRegister/dataUseRegister.controller.js @@ -1,10 +1,10 @@ import Controller from '../base/controller'; import { logger } from '../utilities/logger'; -import _ from 'lodash'; import constants from './../utilities/constants.util'; -import dataUseRegisterUtil from './dataUseRegister.util'; import { Data } from '../tool/data.model'; - +import { TeamModel } from '../team/team.model'; +import teamController from '../team/team.controller'; +import emailGenerator from '../utilities/emailGenerator.util'; const logCategory = 'dataUseRegister'; export default class DataUseRegisterController extends Controller { @@ -108,7 +108,7 @@ export default class DataUseRegisterController extends Controller { query = { user: requestingUser._id }; break; case 'admin': - query = { status: constants.dataUseRegisterStatus.INREVIEW }; + query = { activeflag: constants.dataUseRegisterStatus.INREVIEW }; break; default: query = { publisher: team }; @@ -135,11 +135,28 @@ export default class DataUseRegisterController extends Controller { async updateDataUseRegister(req, res) { try { const id = req.params.id; - const body = req.body; + const { activeflag, rejectionReason } = req.body; + const requestingUser = req.user; + + const options = { lean: true, populate: 'user' }; + const dataUseRegister = await this.dataUseRegisterService.getDataUseRegister(id, {}, options); - this.dataUseRegisterService.updateDataUseRegister(id, body).catch(err => { + this.dataUseRegisterService.updateDataUseRegister(id, req.body).catch(err => { logger.logError(err, logCategory); }); + + // Send notifications + if (activeflag === 'active') { + this.createNotifications(constants.dataUseRegisterNotifications.DATAUSEAPPROVED, {}, dataUseRegister, requestingUser); + } else if (activeflag === 'rejected') { + this.createNotifications( + constants.dataUseRegisterNotifications.DATAUSEREJECTED, + { rejectionReason }, + dataUseRegister, + requestingUser + ); + } + // Return success return res.status(200).json({ success: true, @@ -190,4 +207,52 @@ export default class DataUseRegisterController extends Controller { }); } } + + async createNotifications(type, context, dataUseRegister, requestingUser) { + const { teams } = requestingUser; + const { rejectionReason } = context; + const { id, projectTitle, user: uploader } = dataUseRegister; + + switch (type) { + case constants.dataUseRegisterNotifications.DATAUSEAPPROVED: { + const adminTeam = await TeamModel.findOne({ type: 'admin' }) + .populate({ + path: 'users', + }) + .lean(); + const dataUseTeamMembers = teamController.getTeamMembersByRole(adminTeam, constants.roleTypes.ADMIN_DATA_USE); + const emailRecipients = [...dataUseTeamMembers, uploader]; + + const options = { + id, + projectTitle, + }; + + const html = emailGenerator.generateDataUseRegisterApproved(options); + emailGenerator.sendEmail(emailRecipients, constants.hdrukEmail, `A data use has been approved by HDR UK`, html, false); + break; + } + + case constants.dataUseRegisterNotifications.DATAUSEREJECTED: { + const adminTeam = await TeamModel.findOne({ type: 'admin' }) + .populate({ + path: 'users', + }) + .lean(); + + const dataUseTeamMembers = teamController.getTeamMembersByRole(adminTeam, constants.roleTypes.ADMIN_DATA_USE); + const emailRecipients = [...dataUseTeamMembers, uploader]; + + const options = { + id, + projectTitle, + rejectionReason, + }; + + const html = emailGenerator.generateDataUseRegisterRejected(options); + emailGenerator.sendEmail(emailRecipients, constants.hdrukEmail, `A data use has been rejected by HDR UK`, html, false); + break; + } + } + } } diff --git a/src/resources/dataUseRegister/dataUseRegister.model.js b/src/resources/dataUseRegister/dataUseRegister.model.js index b27b5bd4..bcac029c 100644 --- a/src/resources/dataUseRegister/dataUseRegister.model.js +++ b/src/resources/dataUseRegister/dataUseRegister.model.js @@ -66,6 +66,7 @@ const dataUseRegisterSchema = new Schema( dataLocation: String, //TRE Or Any Other Specified Location privacyEnhancements: String, //How Has Data Been Processed To Enhance Privacy researchOutputs: [{ type: String }], //Link To Research Outputs + rejectionReason: String, //Reason For Rejecting A Data Use Register }, { timestamps: true, diff --git a/src/resources/dataUseRegister/dataUseRegister.repository.js b/src/resources/dataUseRegister/dataUseRegister.repository.js index bf20282a..3f0927b0 100644 --- a/src/resources/dataUseRegister/dataUseRegister.repository.js +++ b/src/resources/dataUseRegister/dataUseRegister.repository.js @@ -21,6 +21,9 @@ export default class DataUseRegisterRepository extends Repository { } updateDataUseRegister(id, body) { + body.updatedon = Date.now(); + body.lastActivity = Date.now(); + return this.update(id, body); } diff --git a/src/resources/dataUseRegister/dataUseRegister.route.js b/src/resources/dataUseRegister/dataUseRegister.route.js index c8b950e1..18d51ba6 100644 --- a/src/resources/dataUseRegister/dataUseRegister.route.js +++ b/src/resources/dataUseRegister/dataUseRegister.route.js @@ -87,8 +87,7 @@ const authorizeView = async (req, res, next) => { const requestingUser = req.user; const { team } = req.query; - const authorised = - team === 'user' || (team === 'admin' && isUserDataUseAdmin(requestingUser)) || isUserMemberOfTeam(requestingUser, team); + const authorised = team === 'user' || isUserDataUseAdmin(requestingUser) || isUserMemberOfTeam(requestingUser, team); if (!authorised) { return res.status(401).json({ diff --git a/src/resources/dataUseRegister/dataUseRegister.service.js b/src/resources/dataUseRegister/dataUseRegister.service.js index b2a2f24f..2e1710f3 100644 --- a/src/resources/dataUseRegister/dataUseRegister.service.js +++ b/src/resources/dataUseRegister/dataUseRegister.service.js @@ -13,7 +13,7 @@ export default class DataUseRegisterService { // Protect for no id passed if (!id) return; - query = { ...query, id: id }; + query = { ...query, _id: id }; return this.dataUseRegisterRepository.getDataUseRegister(query, options); } diff --git a/src/resources/utilities/constants.util.js b/src/resources/utilities/constants.util.js index 97793d90..0c995edd 100644 --- a/src/resources/utilities/constants.util.js +++ b/src/resources/utilities/constants.util.js @@ -15,6 +15,11 @@ const _activityLogNotifications = Object.freeze({ MANUALEVENTREMOVED: 'manualEventRemoved', }); +const _dataUseRegisterNotifications = Object.freeze({ + DATAUSEAPPROVED: 'dataUseApproved', + DATAUSEREJECTED: 'dataUseRejected', +}); + const _teamNotificationTypes = Object.freeze({ DATAACCESSREQUEST: 'dataAccessRequest', METADATAONBOARDING: 'metaDataOnboarding', @@ -320,4 +325,5 @@ export default { activityLogNotifications: _activityLogNotifications, DARMessageTypes: _DARMessageTypes, dataUseRegisterStatus: _dataUseRegisterStatus, + dataUseRegisterNotifications: _dataUseRegisterNotifications, }; diff --git a/src/resources/utilities/emailGenerator.util.js b/src/resources/utilities/emailGenerator.util.js index 0606c545..87460a69 100644 --- a/src/resources/utilities/emailGenerator.util.js +++ b/src/resources/utilities/emailGenerator.util.js @@ -200,7 +200,9 @@ const _getSubmissionDetails = ( let body = ` - + @@ -228,7 +230,9 @@ const _getSubmissionDetails = ( const amendBody = `
Project${projectName || 'No project name set'}${ + projectName || 'No project name set' + }
Related NCS project
- + @@ -319,16 +323,8 @@ const _getSubmissionDetails = ( * @return {String} Questions Answered */ const _buildEmail = (aboutApplication, fullQuestions, questionAnswers, options) => { - const { - userType, - userName, - userEmail, - datasetTitles, - initialDatasetTitles, - submissionType, - submissionDescription, - applicationId, - } = options; + const { userType, userName, userEmail, datasetTitles, initialDatasetTitles, submissionType, submissionDescription, applicationId } = + options; const dateSubmitted = moment().format('D MMM YYYY'); const year = moment().year(); const { projectName, isNationalCoreStudies = false, nationalCoreStudiesProjectId = '' } = aboutApplication; @@ -356,7 +352,13 @@ const _buildEmail = (aboutApplication, fullQuestions, questionAnswers, options) // Create json content payload for attaching to email const jsonContent = { - applicationDetails: { projectName: projectName || 'No project name set', linkNationalCoreStudies, datasetTitles, dateSubmitted, applicantName: userName }, + applicationDetails: { + projectName: projectName || 'No project name set', + linkNationalCoreStudies, + datasetTitles, + dateSubmitted, + applicantName: userName, + }, questions: { ...fullQuestions }, answers: { ...questionAnswers }, }; @@ -632,18 +634,16 @@ const _displayActivityLogLink = (accessId, publisher) => { return `View activity log`; }; +const _displayDataUseRegisterLink = dataUseId => { + if (!dataUseId) return ''; + + const dataUseLink = `${process.env.homeURL}/datause/${dataUseId}`; + return `View data use`; +}; + const _generateDARStatusChangedEmail = options => { - let { - id, - applicationStatus, - applicationStatusDesc, - projectId, - projectName, - publisher, - datasetTitles, - dateSubmitted, - applicants, - } = options; + let { id, applicationStatus, applicationStatusDesc, projectId, projectName, publisher, datasetTitles, dateSubmitted, applicants } = + options; let body = `
Project${projectName || 'No project name set'}${ + projectName || 'No project name set' + }
Date of amendment submission
{ return body; }; +const _generateDataUseRegisterApproved = options => { + const { id, projectTitle } = options; + const body = `
+
+ + + + + + + + +
+ New active data use +
+ A data use for ${projectTitle} has been approved by HDR UK and is now public and searchable on the Gateway. You can now edit and archive this data use directly in the Gateway. +
+
+ ${_displayDataUseRegisterLink(id)} +
+ `; + return body; +}; + +const _generateDataUseRegisterRejected = options => { + const { id, projectTitle, rejectionReason } = options; + const body = `
+ + + + + + + + + + + + + + +
+ A data use has been rejected +
+ A data use for ${projectTitle} has been rejected by HDR UK team. +
+ + + + + +
Reason for rejection${rejectionReason}
+
+
+ ${_displayDataUseRegisterLink(id)} +
+
`; + return body; +}; + /** * [_sendEmail] * @@ -2549,7 +2621,7 @@ export default { generateMetadataOnboardingApproved: _generateMetadataOnboardingApproved, generateMetadataOnboardingRejected: _generateMetadataOnboardingRejected, generateMetadataOnboardingDraftDeleted: _generateMetadataOnboardingDraftDeleted, - generateMetadataOnboardingDuplicated: _generateMetadataOnboardingDuplicated, + generateMetadataOnboardingDuplicated: _generateMetadataOnboardingDuplicated, //generateMetadataOnboardingArchived: _generateMetadataOnboardingArchived, //generateMetadataOnboardingUnArchived: _generateMetadataOnboardingUnArchived, //Messages @@ -2558,4 +2630,7 @@ export default { //ActivityLog generateActivityLogManualEventCreated: _generateActivityLogManualEventCreated, generateActivityLogManualEventDeleted: _generateActivityLogManualEventDeleted, + //DataUseRegister + generateDataUseRegisterApproved: _generateDataUseRegisterApproved, + generateDataUseRegisterRejected: _generateDataUseRegisterRejected, }; From 81aa62d1bd183c0cd6600623761f54e3e23f0420 Mon Sep 17 00:00:00 2001 From: CiaraWardPA Date: Wed, 20 Oct 2021 11:20:50 +0100 Subject: [PATCH 059/116] IG-2281 fix for people and papers in unmet demand stats --- src/resources/stats/stats.repository.js | 2 +- src/resources/stats/stats.service.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/resources/stats/stats.repository.js b/src/resources/stats/stats.repository.js index 260d253c..be62bebd 100644 --- a/src/resources/stats/stats.repository.js +++ b/src/resources/stats/stats.repository.js @@ -401,7 +401,7 @@ export default class StatsRepository extends Repository { maxTools: { $max: '$returned.tool' }, maxPapers: { $max: '$returned.paper' }, maxCourses: { $max: '$returned.course' }, - maxPeople: { $max: '$returned.people' }, + maxPeople: { $max: '$returned.person' }, maxDataUses: { $max: '$returned.datause' }, entity: { $max: entityType }, }, diff --git a/src/resources/stats/stats.service.js b/src/resources/stats/stats.service.js index 415ac5b5..a3346cf9 100644 --- a/src/resources/stats/stats.service.js +++ b/src/resources/stats/stats.service.js @@ -170,7 +170,7 @@ const entityTypeMap = { Tools: 'tool', Projects: 'project', Courses: 'course', - Papers: 'papers', + Papers: 'paper', People: 'person', DataUses: 'datause', }; From 81c0181f41586604d9d38546a04c7e9e37b3a59f Mon Sep 17 00:00:00 2001 From: Callum Reekie Date: Wed, 20 Oct 2021 12:51:49 +0100 Subject: [PATCH 060/116] CR - specify format URI for imageLink in collections schema --- docs/schemas/collections.schema.js | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/schemas/collections.schema.js b/docs/schemas/collections.schema.js index 13294cdc..ebdd5c3f 100644 --- a/docs/schemas/collections.schema.js +++ b/docs/schemas/collections.schema.js @@ -11,6 +11,7 @@ module.exports = { }, imageLink: { type: 'string', + format: 'uri', }, authors: { type: 'array', From eb18d5ab3df7c50ab8a7614ac951ee6c5596d68f Mon Sep 17 00:00:00 2001 From: massimocianfroccaPA Date: Wed, 20 Oct 2021 14:25:28 +0100 Subject: [PATCH 061/116] Prevent notifications to be sent on Archive Unarchive --- .../dataUseRegister/dataUseRegister.controller.js | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/resources/dataUseRegister/dataUseRegister.controller.js b/src/resources/dataUseRegister/dataUseRegister.controller.js index 038f3ef5..97a42528 100644 --- a/src/resources/dataUseRegister/dataUseRegister.controller.js +++ b/src/resources/dataUseRegister/dataUseRegister.controller.js @@ -146,9 +146,15 @@ export default class DataUseRegisterController extends Controller { }); // Send notifications - if (activeflag === 'active') { + if ( + activeflag === constants.dataUseRegisterStatus.ACTIVE && + dataUseRegister.activeflag === constants.dataUseRegisterStatus.INREVIEW + ) { this.createNotifications(constants.dataUseRegisterNotifications.DATAUSEAPPROVED, {}, dataUseRegister, requestingUser); - } else if (activeflag === 'rejected') { + } else if ( + activeflag === constants.dataUseRegisterStatus.REJECTED && + dataUseRegister.activeflag === constants.dataUseRegisterStatus.INREVIEW + ) { this.createNotifications( constants.dataUseRegisterNotifications.DATAUSEREJECTED, { rejectionReason }, From 77c133e61b9f40bdb27a07de2b8ed3f5997f1818 Mon Sep 17 00:00:00 2001 From: massimocianfroccaPA Date: Fri, 22 Oct 2021 09:55:09 +0100 Subject: [PATCH 062/116] fix update data use API --- .../dataUseRegister.controller.js | 40 ++++++++++--------- .../dataUseRegister.service.js | 2 +- 2 files changed, 23 insertions(+), 19 deletions(-) diff --git a/src/resources/dataUseRegister/dataUseRegister.controller.js b/src/resources/dataUseRegister/dataUseRegister.controller.js index 97a42528..2178bbb4 100644 --- a/src/resources/dataUseRegister/dataUseRegister.controller.js +++ b/src/resources/dataUseRegister/dataUseRegister.controller.js @@ -1,3 +1,4 @@ +import { isUndefined } from 'lodash'; import Controller from '../base/controller'; import { logger } from '../utilities/logger'; import constants from './../utilities/constants.util'; @@ -103,15 +104,18 @@ export default class DataUseRegisterController extends Controller { const requestingUser = req.user; let query = ''; - switch (team) { - case 'user': - query = { user: requestingUser._id }; - break; - case 'admin': - query = { activeflag: constants.dataUseRegisterStatus.INREVIEW }; - break; - default: - query = { publisher: team }; + + if (team === 'user') { + delete req.query.team; + query = { ...req.query, user: requestingUser._id }; + } else if (team === 'admin') { + delete req.query.team; + query = { ...req.query, activeflag: constants.dataUseRegisterStatus.INREVIEW }; + } else if (team !== 'user' && team !== 'admin') { + delete req.query.team; + query = { ...req.query, publisher: team }; + } else { + query = req.query; } const dataUseRegisters = await this.dataUseRegisterService.getDataUseRegisters(query).catch(err => { @@ -141,20 +145,20 @@ export default class DataUseRegisterController extends Controller { const options = { lean: true, populate: 'user' }; const dataUseRegister = await this.dataUseRegisterService.getDataUseRegister(id, {}, options); - this.dataUseRegisterService.updateDataUseRegister(id, req.body).catch(err => { + this.dataUseRegisterService.updateDataUseRegister(dataUseRegister._id, req.body).catch(err => { logger.logError(err, logCategory); }); + const isDataUseRegisterApproved = + activeflag === constants.dataUseRegisterStatus.ACTIVE && dataUseRegister.activeflag === constants.dataUseRegisterStatus.INREVIEW; + + const isDataUseRegisterRejected = + activeflag === constants.dataUseRegisterStatus.REJECTED && dataUseRegister.activeflag === constants.dataUseRegisterStatus.INREVIEW; + // Send notifications - if ( - activeflag === constants.dataUseRegisterStatus.ACTIVE && - dataUseRegister.activeflag === constants.dataUseRegisterStatus.INREVIEW - ) { + if (isDataUseRegisterApproved) { this.createNotifications(constants.dataUseRegisterNotifications.DATAUSEAPPROVED, {}, dataUseRegister, requestingUser); - } else if ( - activeflag === constants.dataUseRegisterStatus.REJECTED && - dataUseRegister.activeflag === constants.dataUseRegisterStatus.INREVIEW - ) { + } else if (isDataUseRegisterRejected) { this.createNotifications( constants.dataUseRegisterNotifications.DATAUSEREJECTED, { rejectionReason }, diff --git a/src/resources/dataUseRegister/dataUseRegister.service.js b/src/resources/dataUseRegister/dataUseRegister.service.js index 2e1710f3..1c58309d 100644 --- a/src/resources/dataUseRegister/dataUseRegister.service.js +++ b/src/resources/dataUseRegister/dataUseRegister.service.js @@ -13,7 +13,7 @@ export default class DataUseRegisterService { // Protect for no id passed if (!id) return; - query = { ...query, _id: id }; + query = { ...query, id }; return this.dataUseRegisterRepository.getDataUseRegister(query, options); } From 30ff8c0e29038383983dc1921a9320f5538f21ea Mon Sep 17 00:00:00 2001 From: massimocianfroccaPA Date: Fri, 22 Oct 2021 15:45:55 +0100 Subject: [PATCH 063/116] Add search API --- .../dataUseRegister.controller.js | 65 +++++++++++++++++++ .../dataUseRegister/dataUseRegister.route.js | 33 ++++++---- 2 files changed, 85 insertions(+), 13 deletions(-) diff --git a/src/resources/dataUseRegister/dataUseRegister.controller.js b/src/resources/dataUseRegister/dataUseRegister.controller.js index 2178bbb4..19458a68 100644 --- a/src/resources/dataUseRegister/dataUseRegister.controller.js +++ b/src/resources/dataUseRegister/dataUseRegister.controller.js @@ -6,6 +6,10 @@ import { Data } from '../tool/data.model'; import { TeamModel } from '../team/team.model'; import teamController from '../team/team.controller'; import emailGenerator from '../utilities/emailGenerator.util'; + +import { DataUseRegister } from '../dataUseRegister/dataUseRegister.model'; +import { getObjectFilters } from '../search/search.repository'; + const logCategory = 'dataUseRegister'; export default class DataUseRegisterController extends Controller { @@ -218,6 +222,67 @@ export default class DataUseRegisterController extends Controller { } } + async searchDataUseRegisters(req, res) { + try { + // getObjectFilters; + + const searchTerm = (newSearchQuery && newSearchQuery['$and'] && newSearchQuery['$and'].find(exp => !_.isNil(exp['$text']))) || {}; + + if (searchTerm) { + newSearchQuery['$and'] = newSearchQuery['$and'].filter(exp => !exp['$text']); + } + + queryObject = [ + { $match: searchTerm }, + { $lookup: { from: 'tools', localField: 'authors', foreignField: 'id', as: 'persons' } }, + { + $addFields: { + persons: { + $map: { + input: '$persons', + as: 'row', + in: { + id: '$$row.id', + firstname: '$$row.firstname', + lastname: '$$row.lastname', + fullName: { $concat: ['$$row.firstname', ' ', '$$row.lastname'] }, + }, + }, + }, + }, + }, + { $match: newSearchQuery }, + { + $project: { + _id: 0, + id: 1, + projectTitle: 1, + organisationName: 1, + keywords: 1, + datasetTitles: 1, + activeflag: 1, + counter: 1, + type: 1, + }, + }, + ]; + + const result = await DataUseRegister.aggregate(queryObject).catch(err => { + console.log(err); + }); + + // Return data + return res.status(200).json({ success: true, result }); + } catch (err) { + //Return error response if something goes wrong + logger.logError(err, logCategory); + return res.status(500).json({ + success: false, + message: 'A server error occurred, please try again', + }); + } + } + async createNotifications(type, context, dataUseRegister, requestingUser) { const { teams } = requestingUser; const { rejectionReason } = context; diff --git a/src/resources/dataUseRegister/dataUseRegister.route.js b/src/resources/dataUseRegister/dataUseRegister.route.js index 18d51ba6..48dd9637 100644 --- a/src/resources/dataUseRegister/dataUseRegister.route.js +++ b/src/resources/dataUseRegister/dataUseRegister.route.js @@ -5,7 +5,7 @@ import { dataUseRegisterService } from './dependency'; import { logger } from '../utilities/logger'; import passport from 'passport'; import constants from './../utilities/constants.util'; -import { isEmpty, isNull } from 'lodash'; +import { isEmpty, isNull, isUndefined } from 'lodash'; const router = express.Router(); const dataUseRegisterController = new DataUseRegisterController(dataUseRegisterService); @@ -48,18 +48,18 @@ const validateUpdateRequest = (req, res, next) => { next(); }; -const validateViewRequest = (req, res, next) => { - const { team } = req.query; +// const validateViewRequest = (req, res, next) => { +// const { team } = req.query; - if (!team) { - return res.status(400).json({ - success: false, - message: 'You must provide a team parameter', - }); - } +// if (!team) { +// return res.status(400).json({ +// success: false, +// message: 'You must provide a team parameter', +// }); +// } - next(); -}; +// next(); +// }; const validateUploadRequest = (req, res, next) => { const { teamId, dataUses } = req.body; @@ -87,7 +87,7 @@ const authorizeView = async (req, res, next) => { const requestingUser = req.user; const { team } = req.query; - const authorised = team === 'user' || isUserDataUseAdmin(requestingUser) || isUserMemberOfTeam(requestingUser, team); + const authorised = isUndefined(team) || isUserMemberOfTeam(requestingUser, team) || team === 'user' || isUserDataUseAdmin(requestingUser); if (!authorised) { return res.status(401).json({ @@ -156,7 +156,7 @@ router.get( router.get( '/', passport.authenticate('jwt'), - validateViewRequest, + // validateViewRequest, authorizeView, logger.logRequestMiddleware({ logCategory, action: 'Viewed dataUseRegisters data' }), (req, res) => dataUseRegisterController.getDataUseRegisters(req, res) @@ -193,4 +193,11 @@ router.post( (req, res) => dataUseRegisterController.uploadDataUseRegisters(req, res) ); +router.get( + '/search', + passport.authenticate('jwt'), + logger.logRequestMiddleware({ logCategory, action: 'Search uploaded data uses' }), + (req, res) => dataUseRegisterController.searchDataUseRegisters(req, res) +); + module.exports = router; From 0ee5317a7681a64b83bb64d01926f0be3b5f5132 Mon Sep 17 00:00:00 2001 From: Dan Nita Date: Wed, 27 Oct 2021 13:11:10 +0100 Subject: [PATCH 064/116] checking id middleware --- src/resources/dataset/dataset.controller.js | 8 +------- src/resources/dataset/v2/dataset.route.js | 3 ++- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/src/resources/dataset/dataset.controller.js b/src/resources/dataset/dataset.controller.js index f6829c03..77d62065 100644 --- a/src/resources/dataset/dataset.controller.js +++ b/src/resources/dataset/dataset.controller.js @@ -10,13 +10,7 @@ export default class DatasetController extends Controller { try { // Extract id parameter from query string const { id } = req.params; - // If no id provided, it is a bad request - if (!id) { - return res.status(400).json({ - success: false, - message: 'You must provide a dataset identifier', - }); - } + // Find the dataset const options = { lean: false, populate: { path: 'submittedDataAccessRequests' } }; let dataset = await this.datasetService.getDataset(id, req.query, options); diff --git a/src/resources/dataset/v2/dataset.route.js b/src/resources/dataset/v2/dataset.route.js index 6acac939..a6f90244 100644 --- a/src/resources/dataset/v2/dataset.route.js +++ b/src/resources/dataset/v2/dataset.route.js @@ -2,6 +2,7 @@ import express from 'express'; import DatasetController from '../dataset.controller'; import { datasetService } from '../dependency'; import { resultLimit } from '../../../config/middleware'; +import { checkIDMiddleware } from './../../../middlewares'; const router = express.Router(); const datasetController = new DatasetController(datasetService); @@ -9,7 +10,7 @@ const datasetController = new DatasetController(datasetService); // @route GET /api/v2/datasets/id // @desc Returns a dataset based on dataset ID provided // @access Public -router.get('/:id', (req, res) => datasetController.getDataset(req, res)); +router.get('/:id', checkIDMiddleware, (req, res) => datasetController.getDataset(req, res)); // @route GET /api/v2/datasets // @desc Returns a collection of datasets based on supplied query parameters From 87dc33c8eeef1ee0490024f3e90757acae7e344b Mon Sep 17 00:00:00 2001 From: Dan Nita Date: Wed, 27 Oct 2021 13:12:57 +0100 Subject: [PATCH 065/116] checking id middleware --- .../__tests__/checkIDMiddleware.test.js | 56 +++++++++++++++++++ src/middlewares/checkIDMiddleware.js | 15 +++++ src/middlewares/index.js | 5 ++ 3 files changed, 76 insertions(+) create mode 100644 src/middlewares/__tests__/checkIDMiddleware.test.js create mode 100644 src/middlewares/checkIDMiddleware.js create mode 100644 src/middlewares/index.js diff --git a/src/middlewares/__tests__/checkIDMiddleware.test.js b/src/middlewares/__tests__/checkIDMiddleware.test.js new file mode 100644 index 00000000..b35d4a67 --- /dev/null +++ b/src/middlewares/__tests__/checkIDMiddleware.test.js @@ -0,0 +1,56 @@ +import { checkIDMiddleware } from '../index'; + +describe('checkIDMiddleware', () => { + + const nextFunction = jest.fn(); + + const mockedResponse = () => { + const res = {}; + res.status = jest.fn().mockReturnValue(res); + res.json = jest.fn().mockReturnValue(res); + return res; + }; + + it('should return 400 response code when we dont have id value into list of paramteres', () => { + const expectedResponse = { + success: false, + message: 'You must provide a dataset identifier' + }; + + const mockedRequest = () => { + const req = {}; + req.params = {}; + return req; + }; + + const mockedReq = mockedRequest(); + const mockedRes = mockedResponse(); + + checkIDMiddleware(mockedReq, mockedRes, nextFunction); + + expect(mockedRes.status).toHaveBeenCalledWith(400); + expect(mockedRes.json).toHaveBeenCalledWith(expectedResponse); + }); + + it('should pass the middleware when we have id value into list of paramteres', () => { + const expectedResponse = {}; + + const mockedRequest = () => { + const req = {}; + req.params = { id: 1 }; + return req; + }; + + const mockedReq = mockedRequest(); + const mockedRes = mockedResponse(); + + nextFunction.mockReturnValue(expectedResponse); + + checkIDMiddleware(mockedReq, mockedRes, nextFunction); + + expect(mockedRes.status.mock.calls.length).toBe(0); + expect(mockedRes.json.mock.calls.length).toBe(0); + expect(nextFunction.mock.calls.length).toBe(1); + }); + +}); \ No newline at end of file diff --git a/src/middlewares/checkIDMiddleware.js b/src/middlewares/checkIDMiddleware.js new file mode 100644 index 00000000..ed9db761 --- /dev/null +++ b/src/middlewares/checkIDMiddleware.js @@ -0,0 +1,15 @@ +const checkIDMiddleware = (req, res, next) => { + + const { id } = req.params; + + if (!id) { + return res.status(400).json({ + success: false, + message: 'You must provide a dataset identifier', + }); + } + + next(); +} + +export { checkIDMiddleware } \ No newline at end of file diff --git a/src/middlewares/index.js b/src/middlewares/index.js new file mode 100644 index 00000000..9158c22f --- /dev/null +++ b/src/middlewares/index.js @@ -0,0 +1,5 @@ +import { checkIDMiddleware } from './checkIDMiddleware' + +export { + checkIDMiddleware +} \ No newline at end of file From 01ae30a66d2b3c4e7e700f254042cf99b01d3d8e Mon Sep 17 00:00:00 2001 From: massimocianfroccaPA Date: Fri, 29 Oct 2021 01:19:40 +0100 Subject: [PATCH 066/116] Add search API --- .../dataUseRegister.controller.js | 60 ++++------------ .../dataUseRegister.repository.js | 5 +- .../dataUseRegister/dataUseRegister.route.js | 11 ++- .../dataUseRegister.service.js | 35 ++++++++-- .../dataUseRegister/dataUseRegister.util.js | 1 + src/resources/search/search.repository.js | 1 + .../utilities/emailGenerator.util.js | 70 ++++--------------- 7 files changed, 63 insertions(+), 120 deletions(-) diff --git a/src/resources/dataUseRegister/dataUseRegister.controller.js b/src/resources/dataUseRegister/dataUseRegister.controller.js index 19458a68..8ee9da85 100644 --- a/src/resources/dataUseRegister/dataUseRegister.controller.js +++ b/src/resources/dataUseRegister/dataUseRegister.controller.js @@ -1,4 +1,4 @@ -import { isUndefined } from 'lodash'; +import { isNil } from 'lodash'; import Controller from '../base/controller'; import { logger } from '../utilities/logger'; import constants from './../utilities/constants.util'; @@ -6,9 +6,9 @@ import { Data } from '../tool/data.model'; import { TeamModel } from '../team/team.model'; import teamController from '../team/team.controller'; import emailGenerator from '../utilities/emailGenerator.util'; +import { getObjectFilters } from '../search/search.repository'; import { DataUseRegister } from '../dataUseRegister/dataUseRegister.model'; -import { getObjectFilters } from '../search/search.repository'; const logCategory = 'dataUseRegister'; @@ -161,9 +161,9 @@ export default class DataUseRegisterController extends Controller { // Send notifications if (isDataUseRegisterApproved) { - this.createNotifications(constants.dataUseRegisterNotifications.DATAUSEAPPROVED, {}, dataUseRegister, requestingUser); + await this.createNotifications(constants.dataUseRegisterNotifications.DATAUSEAPPROVED, {}, dataUseRegister, requestingUser); } else if (isDataUseRegisterRejected) { - this.createNotifications( + await this.createNotifications( constants.dataUseRegisterNotifications.DATAUSEREJECTED, { rejectionReason }, dataUseRegister, @@ -224,54 +224,20 @@ export default class DataUseRegisterController extends Controller { async searchDataUseRegisters(req, res) { try { - // getObjectFilters; + let searchString = req.query.search || ''; - const searchTerm = (newSearchQuery && newSearchQuery['$and'] && newSearchQuery['$and'].find(exp => !_.isNil(exp['$text']))) || {}; - - if (searchTerm) { - newSearchQuery['$and'] = newSearchQuery['$and'].filter(exp => !exp['$text']); + if (searchString.includes('-') && !searchString.includes('"')) { + const regex = /(?=\S*[-])([a-zA-Z'-]+)/g; + searchString = searchString.replace(regex, '"$1"'); } + let searchQuery = { $and: [{ activeflag: 'active' }] }; - queryObject = [ - { $match: searchTerm }, - { $lookup: { from: 'tools', localField: 'authors', foreignField: 'id', as: 'persons' } }, - { - $addFields: { - persons: { - $map: { - input: '$persons', - as: 'row', - in: { - id: '$$row.id', - firstname: '$$row.firstname', - lastname: '$$row.lastname', - fullName: { $concat: ['$$row.firstname', ' ', '$$row.lastname'] }, - }, - }, - }, - }, - }, - { $match: newSearchQuery }, - { - $project: { - _id: 0, - id: 1, - projectTitle: 1, - organisationName: 1, - keywords: 1, - datasetTitles: 1, - activeflag: 1, - counter: 1, - type: 1, - }, - }, - ]; + if (searchString.length > 0) searchQuery['$and'].push({ $text: { $search: searchString } }); - const result = await DataUseRegister.aggregate(queryObject).catch(err => { - console.log(err); - }); + searchQuery = getObjectFilters(searchQuery, req, 'dataUseRegister'); + + const result = await DataUseRegister.aggregate([{ $match: searchQuery }]); - // Return data return res.status(200).json({ success: true, result }); } catch (err) { //Return error response if something goes wrong diff --git a/src/resources/dataUseRegister/dataUseRegister.repository.js b/src/resources/dataUseRegister/dataUseRegister.repository.js index 3f0927b0..4a66b079 100644 --- a/src/resources/dataUseRegister/dataUseRegister.repository.js +++ b/src/resources/dataUseRegister/dataUseRegister.repository.js @@ -58,17 +58,14 @@ export default class DataUseRegisterRepository extends Repository { ); } - async checkDataUseRegisterExists(dataUseRegister) { - const { projectIdText, projectTitle, laySummary, organisationName, datasetTitles, latestApprovalDate } = dataUseRegister; + async checkDataUseRegisterExists(projectIdText, projectTitle, organisationName, datasetTitles) { const duplicatesFound = await this.dataUseRegister.countDocuments({ $or: [ { projectIdText }, { projectTitle, - laySummary, organisationName, datasetTitles, - latestApprovalDate, }, ], }); diff --git a/src/resources/dataUseRegister/dataUseRegister.route.js b/src/resources/dataUseRegister/dataUseRegister.route.js index 48dd9637..d4a3024b 100644 --- a/src/resources/dataUseRegister/dataUseRegister.route.js +++ b/src/resources/dataUseRegister/dataUseRegister.route.js @@ -140,6 +140,10 @@ const authorizeUpload = async (req, res, next) => { next(); }; +router.get('/search', logger.logRequestMiddleware({ logCategory, action: 'Search uploaded data uses' }), (req, res) => + dataUseRegisterController.searchDataUseRegisters(req, res) +); + // @route GET /api/v2/data-use-registers/id // @desc Returns a dataUseRegister based on dataUseRegister ID provided // @access Public @@ -193,11 +197,4 @@ router.post( (req, res) => dataUseRegisterController.uploadDataUseRegisters(req, res) ); -router.get( - '/search', - passport.authenticate('jwt'), - logger.logRequestMiddleware({ logCategory, action: 'Search uploaded data uses' }), - (req, res) => dataUseRegisterController.searchDataUseRegisters(req, res) -); - module.exports = router; diff --git a/src/resources/dataUseRegister/dataUseRegister.service.js b/src/resources/dataUseRegister/dataUseRegister.service.js index 1c58309d..095311f5 100644 --- a/src/resources/dataUseRegister/dataUseRegister.service.js +++ b/src/resources/dataUseRegister/dataUseRegister.service.js @@ -68,10 +68,8 @@ export default class DataUseRegisterService { el => el.projectIdText === dataUse.projectIdText || (el.projectTitle === dataUse.projectTitle && - el.laySummary === dataUse.laySummary && el.organisationName === dataUse.organisationName && - el.datasetTitles === dataUse.datasetTitles && - el.latestApprovalDate === dataUse.latestApprovalDate) + el.datasetTitles === dataUse.datasetTitles) ); if (!isDuplicate) arr = [...arr, dataUse]; return arr; @@ -91,7 +89,26 @@ export default class DataUseRegisterService { const newDataUses = []; for (const dataUse of dataUses) { - const exists = await this.dataUseRegisterRepository.checkDataUseRegisterExists(dataUse); + const { linkedDatasets = [], namedDatasets = [] } = await dataUseRegisterUtil.getLinkedDatasets( + dataUse.datasetNames && + dataUse.datasetNames + .toString() + .split(',') + .map(el => { + if (!isEmpty(el)) return el.trim(); + }) + ); + + const datasetTitles = [...linkedDatasets.map(dataset => dataset.name), ...namedDatasets]; + + const { projectIdText, projectTitle, organisationName } = dataUse; + + const exists = await this.dataUseRegisterRepository.checkDataUseRegisterExists( + projectIdText, + projectTitle, + organisationName, + datasetTitles + ); if (exists === false) newDataUses.push(dataUse); } @@ -131,7 +148,15 @@ export default class DataUseRegisterService { }) ); - const exists = await this.dataUseRegisterRepository.checkDataUseRegisterExists(obj); + const { projectIdText, projectTitle, organisationName } = obj; + const datasetTitles = [...linkedDatasets.map(dataset => dataset.name), ...namedDatasets]; + + const exists = await this.dataUseRegisterRepository.checkDataUseRegisterExists( + projectIdText, + projectTitle, + organisationName, + datasetTitles + ); //Add new data use with linked entities dataUsesChecks.push({ diff --git a/src/resources/dataUseRegister/dataUseRegister.util.js b/src/resources/dataUseRegister/dataUseRegister.util.js index 98c4c1d6..513174fa 100644 --- a/src/resources/dataUseRegister/dataUseRegister.util.js +++ b/src/resources/dataUseRegister/dataUseRegister.util.js @@ -101,6 +101,7 @@ const buildDataUseRegisters = async (creatorUser, teamId, dataUses = []) => { ...(obj.confidentialDataDescription && { confidentialDataDescription: obj.confidentialDataDescription.toString().trim() }), ...(obj.dataLocation && { dataLocation: obj.dataLocation.toString().trim() }), ...(obj.privacyEnhancements && { privacyEnhancements: obj.privacyEnhancements.toString().trim() }), + ...(obj.dutyOfConfidentiality && { dutyOfConfidentiality: obj.dutyOfConfidentiality.toString().trim() }), ...(projectStartDate.isValid() && { projectStartDate }), ...(projectEndDate.isValid() && { projectEndDate }), ...(latestApprovalDate.isValid() && { latestApprovalDate }), diff --git a/src/resources/search/search.repository.js b/src/resources/search/search.repository.js index e6cb0990..82e7b09f 100644 --- a/src/resources/search/search.repository.js +++ b/src/resources/search/search.repository.js @@ -167,6 +167,7 @@ export async function getObjectResult(type, searchAll, searchQuery, startIndex, activeflag: 1, counter: 1, type: 1, + latestUpdate: '$lastActivity', }, }, ]; diff --git a/src/resources/utilities/emailGenerator.util.js b/src/resources/utilities/emailGenerator.util.js index 87460a69..f678cb78 100644 --- a/src/resources/utilities/emailGenerator.util.js +++ b/src/resources/utilities/emailGenerator.util.js @@ -2402,69 +2402,25 @@ const _generateActivityLogManualEventDeleted = options => { const _generateDataUseRegisterApproved = options => { const { id, projectTitle } = options; const body = `
- - - - - - - - - -
- New active data use -
- A data use for ${projectTitle} has been approved by HDR UK and is now public and searchable on the Gateway. You can now edit and archive this data use directly in the Gateway. -
-
- ${_displayDataUseRegisterLink(id)} -
-
`; +
+

New active data use

+

A data use for ${projectTitle} has been approved by HDR UK and is now public and searchable on the Gateway. You can now edit and archive this data use directly in the Gateway.

+ ${_displayDataUseRegisterLink(id)} +
+ `; + return body; }; const _generateDataUseRegisterRejected = options => { const { id, projectTitle, rejectionReason } = options; const body = `
- - - - - - - - - - - - - - -
- A data use has been rejected -
- A data use for ${projectTitle} has been rejected by HDR UK team. -
- - - - - -
Reason for rejection${rejectionReason}
-
-
+ +
+

A data use has been rejected

+

A data use for ${projectTitle} has been rejected by HDR UK team. +

Reason for rejection:

+

${rejectionReason}

${_displayDataUseRegisterLink(id)}
`; From 865428db95642a67448765b56cc67c0b0cf7d7dc Mon Sep 17 00:00:00 2001 From: massimocianfroccaPA Date: Fri, 29 Oct 2021 10:16:13 +0100 Subject: [PATCH 067/116] Remove unused middleware --- .../dataUseRegister/dataUseRegister.route.js | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/src/resources/dataUseRegister/dataUseRegister.route.js b/src/resources/dataUseRegister/dataUseRegister.route.js index d4a3024b..70eb8d53 100644 --- a/src/resources/dataUseRegister/dataUseRegister.route.js +++ b/src/resources/dataUseRegister/dataUseRegister.route.js @@ -48,19 +48,6 @@ const validateUpdateRequest = (req, res, next) => { next(); }; -// const validateViewRequest = (req, res, next) => { -// const { team } = req.query; - -// if (!team) { -// return res.status(400).json({ -// success: false, -// message: 'You must provide a team parameter', -// }); -// } - -// next(); -// }; - const validateUploadRequest = (req, res, next) => { const { teamId, dataUses } = req.body; let errors = []; From bcd22b59f292baa717d9dda8da8fe542836ce44b Mon Sep 17 00:00:00 2001 From: massimocianfroccaPA Date: Fri, 29 Oct 2021 11:27:19 +0100 Subject: [PATCH 068/116] rollback changes on get API --- .../dataUseRegister/dataUseRegister.route.js | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/src/resources/dataUseRegister/dataUseRegister.route.js b/src/resources/dataUseRegister/dataUseRegister.route.js index 70eb8d53..a6fa9b3f 100644 --- a/src/resources/dataUseRegister/dataUseRegister.route.js +++ b/src/resources/dataUseRegister/dataUseRegister.route.js @@ -5,7 +5,7 @@ import { dataUseRegisterService } from './dependency'; import { logger } from '../utilities/logger'; import passport from 'passport'; import constants from './../utilities/constants.util'; -import { isEmpty, isNull, isUndefined } from 'lodash'; +import { isEmpty, isNull } from 'lodash'; const router = express.Router(); const dataUseRegisterController = new DataUseRegisterController(dataUseRegisterService); @@ -70,11 +70,24 @@ const validateUploadRequest = (req, res, next) => { next(); }; +const validateViewRequest = (req, res, next) => { + const { team } = req.query; + + if (!team) { + return res.status(400).json({ + success: false, + message: 'You must provide a team parameter', + }); + } + + next(); +}; + const authorizeView = async (req, res, next) => { const requestingUser = req.user; const { team } = req.query; - const authorised = isUndefined(team) || isUserMemberOfTeam(requestingUser, team) || team === 'user' || isUserDataUseAdmin(requestingUser); + const authorised = team === 'user' || isUserDataUseAdmin(requestingUser) || isUserMemberOfTeam(requestingUser, team); if (!authorised) { return res.status(401).json({ @@ -147,7 +160,7 @@ router.get( router.get( '/', passport.authenticate('jwt'), - // validateViewRequest, + validateViewRequest, authorizeView, logger.logRequestMiddleware({ logCategory, action: 'Viewed dataUseRegisters data' }), (req, res) => dataUseRegisterController.getDataUseRegisters(req, res) From f235aa2bcae5b340c8bbbcc7a6020bc2cf7b2272 Mon Sep 17 00:00:00 2001 From: Paul McCafferty <> Date: Mon, 1 Nov 2021 17:50:40 +0000 Subject: [PATCH 069/116] Updates for DUR --- .../dataUseRegister.controller.js | 8 ++-- .../dataUseRegister/dataUseRegister.model.js | 7 --- .../dataUseRegister/dataUseRegister.route.js | 7 +-- src/resources/filters/filters.mapper.js | 2 +- src/resources/search/search.repository.js | 48 +++++++++++++++++++ src/resources/search/search.router.js | 2 +- 6 files changed, 56 insertions(+), 18 deletions(-) diff --git a/src/resources/dataUseRegister/dataUseRegister.controller.js b/src/resources/dataUseRegister/dataUseRegister.controller.js index 2178bbb4..8ece1d0d 100644 --- a/src/resources/dataUseRegister/dataUseRegister.controller.js +++ b/src/resources/dataUseRegister/dataUseRegister.controller.js @@ -26,7 +26,7 @@ export default class DataUseRegisterController extends Controller { }); } // Find the dataUseRegister - const options = { lean: true }; + const options = { lean: true, populate: { path: 'gatewayApplicants', select: 'id firstname lastname' } }; const dataUseRegister = await this.dataUseRegisterService.getDataUseRegister(id, req.query, options); // Reverse look up var query = Data.aggregate([ @@ -42,7 +42,7 @@ export default class DataUseRegisterController extends Controller { ]); query.exec((err, data) => { if (data.length > 0) { - var p = Data.aggregate([ + /* var p = Data.aggregate([ { $match: { $and: [{ relatedObjects: { $elemMatch: { objectId: req.params.id } } }], @@ -71,9 +71,9 @@ export default class DataUseRegisterController extends Controller { success: true, data: data, }); - }); + }); */ } else { - return res.status(404).send(`Data Use Register not found for Id: ${escape(id)}`); + //return res.status(404).send(`Data Use Register not found for Id: ${escape(id)}`); } }); // Return if no dataUseRegister found diff --git a/src/resources/dataUseRegister/dataUseRegister.model.js b/src/resources/dataUseRegister/dataUseRegister.model.js index 36935ec8..bcac029c 100644 --- a/src/resources/dataUseRegister/dataUseRegister.model.js +++ b/src/resources/dataUseRegister/dataUseRegister.model.js @@ -76,13 +76,6 @@ const dataUseRegisterSchema = new Schema( } ); -dataUseRegisterSchema.virtual('gatewayApplicantsNames', { - ref: 'User', - foreignField: '_id', - localField: 'gatewayApplicants', - justOne: false, -}); - // Load entity class dataUseRegisterSchema.loadClass(DataUseRegisterClass); diff --git a/src/resources/dataUseRegister/dataUseRegister.route.js b/src/resources/dataUseRegister/dataUseRegister.route.js index 18d51ba6..0f7e2051 100644 --- a/src/resources/dataUseRegister/dataUseRegister.route.js +++ b/src/resources/dataUseRegister/dataUseRegister.route.js @@ -143,11 +143,8 @@ const authorizeUpload = async (req, res, next) => { // @route GET /api/v2/data-use-registers/id // @desc Returns a dataUseRegister based on dataUseRegister ID provided // @access Public -router.get( - '/:id', - passport.authenticate('jwt'), - logger.logRequestMiddleware({ logCategory, action: 'Viewed dataUseRegister data' }), - (req, res) => dataUseRegisterController.getDataUseRegister(req, res) +router.get('/:id', logger.logRequestMiddleware({ logCategory, action: 'Viewed dataUseRegister data' }), (req, res) => + dataUseRegisterController.getDataUseRegister(req, res) ); // @route GET /api/v2/data-use-registers diff --git a/src/resources/filters/filters.mapper.js b/src/resources/filters/filters.mapper.js index 31b24f68..478672ff 100644 --- a/src/resources/filters/filters.mapper.js +++ b/src/resources/filters/filters.mapper.js @@ -861,7 +861,7 @@ export const dataUseRegisterFilters = [ label: 'Data custodian', key: 'publisher', alias: 'datausedatacustodian', - dataPath: 'publisher', + dataPath: 'publisherDetails.name', type: 'contains', tooltip: null, closed: true, diff --git a/src/resources/search/search.repository.js b/src/resources/search/search.repository.js index fd972d07..94507fbf 100644 --- a/src/resources/search/search.repository.js +++ b/src/resources/search/search.repository.js @@ -137,6 +137,7 @@ export async function getObjectResult(type, searchAll, searchQuery, startIndex, } queryObject = [ + { $match: searchTerm }, { $lookup: { from: 'publishers', @@ -158,6 +159,7 @@ export async function getObjectResult(type, searchAll, searchQuery, startIndex, }, }, }, + { $match: newSearchQuery }, { $project: { _id: 0, @@ -592,6 +594,52 @@ export function getObjectCount(type, searchAll, searchQuery) { ]) .sort({ score: { $meta: 'textScore' } }); } + } else if (type === 'dataUseRegister') { + const searchTerm = (newSearchQuery && newSearchQuery['$and'] && newSearchQuery['$and'].find(exp => !_.isNil(exp['$text']))) || {}; + + if (searchTerm) { + newSearchQuery['$and'] = newSearchQuery['$and'].filter(exp => !exp['$text']); + } + + q = collection.aggregate([ + { $match: searchTerm }, + { + $lookup: { + from: 'publishers', + localField: 'publisher', + foreignField: '_id', + as: 'publisherDetails', + }, + }, + { + $addFields: { + publisherDetails: { + $map: { + input: '$publisherDetails', + as: 'row', + in: { + name: '$$row.name', + }, + }, + }, + }, + }, + { $match: newSearchQuery }, + { + $group: { + _id: {}, + count: { + $sum: 1, + }, + }, + }, + { + $project: { + count: '$count', + _id: 0, + }, + }, + ]); } else { if (searchAll) { q = collection.aggregate([ diff --git a/src/resources/search/search.router.js b/src/resources/search/search.router.js index d8268d25..cd8a461d 100644 --- a/src/resources/search/search.router.js +++ b/src/resources/search/search.router.js @@ -155,7 +155,7 @@ router.get('/', async (req, res) => { getObjectCount('person', searchAll, searchQuery), getObjectCount('course', searchAll, getObjectFilters(searchQuery, req, 'course')), getObjectCount('collection', searchAll, getObjectFilters(searchQuery, req, 'collection')), - getObjectCount('dataUseRegister', searchAll, getObjectFilters(searchQuery, req, ' dataUseRegister')), + getObjectCount('dataUseRegister', searchAll, getObjectFilters(searchQuery, req, 'dataUseRegister')), ]); const summary = { From 28cc881c43acfba52544edb7154112d3724f6d9b Mon Sep 17 00:00:00 2001 From: Pritesh Bhole Date: Tue, 2 Nov 2021 10:49:59 +0000 Subject: [PATCH 070/116] done --- migrations/1627566998386-add_globals.js | 8 +++++++- migrations/README.md | 2 +- src/resources/auth/utils.js | 16 +++++++++++++++- 3 files changed, 23 insertions(+), 3 deletions(-) diff --git a/migrations/1627566998386-add_globals.js b/migrations/1627566998386-add_globals.js index a701310d..9988aaec 100644 --- a/migrations/1627566998386-add_globals.js +++ b/migrations/1627566998386-add_globals.js @@ -59,6 +59,12 @@ const globalData = { label: 'Commercial project', impliedValues: ['platinum', 'gold'], }, + { + id: mongoose.Types.ObjectId(), + displayOrder: 5, + label: 'N/A', + impliedValues: [], + }, ], }, { @@ -411,7 +417,7 @@ const globalData = { }, { id: mongoose.Types.ObjectId(), - displayOrder: 4, + displayOrder: 5, definition:'N/A', label: "N/A", impliedValues: [], diff --git a/migrations/README.md b/migrations/README.md index bbd6eeb6..af518e21 100644 --- a/migrations/README.md +++ b/migrations/README.md @@ -28,7 +28,7 @@ Complete the scripts required for the UP process, and if possible, the DOWN proc #### Step 4 With the scripts written, the functions can be tested by running the following command, replacing 'my_new_migration_script' with the name of the script you want to execute without the time stamp so for example -node -r esm migrations/migrate.js up my_new_migration_script +node -r esm migrations/migrate.js up add_globals node -r esm migrations/migrate.js up my_new_migration_script diff --git a/src/resources/auth/utils.js b/src/resources/auth/utils.js index 53f342d0..97cb301b 100644 --- a/src/resources/auth/utils.js +++ b/src/resources/auth/utils.js @@ -129,7 +129,21 @@ const getTeams = async () => { const catchLoginErrorAndRedirect = (req, res, next) => { if (req.auth.err || !req.auth.user) { - return res.status(200).redirect(process.env.homeURL + '/loginerror'); + if (req.auth.err === 'loginError') { + return res.status(200).redirect(process.env.homeURL + '/loginerror'); + } + + let redirect = '/'; + let returnPage = null; + if (req.param.returnpage) { + returnPage = Url.parse(req.param.returnpage); + redirect = returnPage.path; + delete req.param.returnpage; + } + + let redirectUrl = process.env.homeURL + redirect; + + return res.status(200).redirect(redirectUrl); } next(); }; From 55028a3d5bbc953f82bae55be0f50b0b38e2f092 Mon Sep 17 00:00:00 2001 From: Pritesh Bhole Date: Tue, 2 Nov 2021 10:52:46 +0000 Subject: [PATCH 071/116] done --- migrations/README.md | 2 +- .../auth/__tests__/auth.utilities.test.js | 24 ------------------- 2 files changed, 1 insertion(+), 25 deletions(-) diff --git a/migrations/README.md b/migrations/README.md index af518e21..8c6f645e 100644 --- a/migrations/README.md +++ b/migrations/README.md @@ -1,4 +1,4 @@ - +src/resources/auth/utils.js # HDR UK GATEWAY - Data Migrations The primary data source used by the Gateway Project is the noSQL solution provided by MongoDb. Data migration strategy is a fundamental part of software development and release cycles for a data intensive web application. The project team have chosen the NPM package Migrate-Mongoose - https://www.npmjs.com/package/migrate-mongoose to assist in the management of data migration scripts. This package allows developers to write versioned, reversible data migration scripts using the Mongoose library. diff --git a/src/resources/auth/__tests__/auth.utilities.test.js b/src/resources/auth/__tests__/auth.utilities.test.js index 10ebd8f0..29c7e1a1 100644 --- a/src/resources/auth/__tests__/auth.utilities.test.js +++ b/src/resources/auth/__tests__/auth.utilities.test.js @@ -45,30 +45,6 @@ describe('Utilities', () => { expect(res.redirect.mock.calls.length).toBe(1); }); - it('should not call next when (req.auth.err === loginError || req.auth.user === undefined) == true', () => { - let res = {}; - res.status = jest.fn().mockReturnValue(res); - res.redirect = jest.fn().mockReturnValue(res); - let req = { - auth: { - user: undefined, - err: 'loginError', - }, - param: { - returnpage: 'somePage', - }, - }; - const next = jest.fn(); - - catchLoginErrorAndRedirect(req, res, next); - - // assert - expect(next.mock.calls.length).toBe(0); - expect(res.status.mock.calls.length).toBe(1); - expect(res.redirect.mock.calls.length).toBe(1); - }); - }); - describe('loginAndSignToken middleware', () => { it('should be a function', () => { expect(typeof loginAndSignToken).toBe('function'); From 33c1768b34ecd9a4b0c4f55cee48c7f8f076723b Mon Sep 17 00:00:00 2001 From: Pritesh Bhole Date: Tue, 2 Nov 2021 10:55:22 +0000 Subject: [PATCH 072/116] done --- src/resources/auth/__tests__/auth.utilities.test.js | 1 + 1 file changed, 1 insertion(+) diff --git a/src/resources/auth/__tests__/auth.utilities.test.js b/src/resources/auth/__tests__/auth.utilities.test.js index 29c7e1a1..f7b2ccc7 100644 --- a/src/resources/auth/__tests__/auth.utilities.test.js +++ b/src/resources/auth/__tests__/auth.utilities.test.js @@ -68,4 +68,5 @@ describe('Utilities', () => { expect(req.login.mock.calls.length).toBe(1); }); }); + }); }); From e34d51d9fe278abcf4cff464ed10c83e140f6f25 Mon Sep 17 00:00:00 2001 From: Pritesh Bhole Date: Tue, 2 Nov 2021 10:58:05 +0000 Subject: [PATCH 073/116] done --- migrations/README.md | 1 - src/resources/auth/__tests__/auth.utilities.test.js | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/migrations/README.md b/migrations/README.md index 8c6f645e..ce9cf4bb 100644 --- a/migrations/README.md +++ b/migrations/README.md @@ -1,4 +1,3 @@ -src/resources/auth/utils.js # HDR UK GATEWAY - Data Migrations The primary data source used by the Gateway Project is the noSQL solution provided by MongoDb. Data migration strategy is a fundamental part of software development and release cycles for a data intensive web application. The project team have chosen the NPM package Migrate-Mongoose - https://www.npmjs.com/package/migrate-mongoose to assist in the management of data migration scripts. This package allows developers to write versioned, reversible data migration scripts using the Mongoose library. diff --git a/src/resources/auth/__tests__/auth.utilities.test.js b/src/resources/auth/__tests__/auth.utilities.test.js index f7b2ccc7..e943b174 100644 --- a/src/resources/auth/__tests__/auth.utilities.test.js +++ b/src/resources/auth/__tests__/auth.utilities.test.js @@ -44,6 +44,7 @@ describe('Utilities', () => { expect(res.status.mock.calls.length).toBe(1); expect(res.redirect.mock.calls.length).toBe(1); }); + }); describe('loginAndSignToken middleware', () => { it('should be a function', () => { @@ -68,5 +69,4 @@ describe('Utilities', () => { expect(req.login.mock.calls.length).toBe(1); }); }); - }); -}); +}); \ No newline at end of file From 30541733ec30062ea60bb193f283d717b1f11f32 Mon Sep 17 00:00:00 2001 From: Pritesh Bhole Date: Tue, 2 Nov 2021 10:58:58 +0000 Subject: [PATCH 074/116] lint error --- src/resources/auth/__tests__/auth.utilities.test.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/resources/auth/__tests__/auth.utilities.test.js b/src/resources/auth/__tests__/auth.utilities.test.js index e943b174..75da1a4f 100644 --- a/src/resources/auth/__tests__/auth.utilities.test.js +++ b/src/resources/auth/__tests__/auth.utilities.test.js @@ -69,4 +69,4 @@ describe('Utilities', () => { expect(req.login.mock.calls.length).toBe(1); }); }); -}); \ No newline at end of file +}); From 6f1268a5fb4e3ff1dc60598d83cf494503bdeb2d Mon Sep 17 00:00:00 2001 From: pritesh Date: Tue, 2 Nov 2021 15:27:49 +0000 Subject: [PATCH 075/116] Update 1627566998386-add_globals.js removed duplicate --- migrations/1627566998386-add_globals.js | 8 -------- 1 file changed, 8 deletions(-) diff --git a/migrations/1627566998386-add_globals.js b/migrations/1627566998386-add_globals.js index 9988aaec..85bca95e 100644 --- a/migrations/1627566998386-add_globals.js +++ b/migrations/1627566998386-add_globals.js @@ -407,14 +407,6 @@ const globalData = { label: "Earlier and 'raw' versions and the impact of each stage of data cleaning", impliedValues: ['platinum'], }, - { - id: mongoose.Types.ObjectId(), - displayOrder: 4, - definition: - 'Ability to view earlier versions, including versions before any transformations have been applied data (in line with deidentification and IG approval) and review the impact of each stage of data cleaning', - label: "Earlier and 'raw' versions and the impact of each stage of data cleaning", - impliedValues: ['platinum'], - }, { id: mongoose.Types.ObjectId(), displayOrder: 5, From 43cf367db82e50e54082735027df3d3eeb5ffe0b Mon Sep 17 00:00:00 2001 From: Dan Nita Date: Tue, 2 Nov 2021 16:25:33 +0000 Subject: [PATCH 076/116] filter sentry message based on environment --- src/config/server.js | 43 ++--- .../dataset/datasetonboarding.controller.js | 6 +- src/resources/dataset/v1/dataset.route.js | 10 +- src/resources/dataset/v1/dataset.service.js | 158 ++++++++++-------- .../linkchecker/linkchecker.router.js | 3 +- .../utilities/emailGenerator.util.js | 5 +- src/resources/utilities/logger.js | 38 +++-- src/services/hubspot/hubspot.js | 13 +- src/services/hubspot/hubspot.route.js | 5 +- src/services/mailchimp/mailchimp.js | 53 +++--- src/services/mailchimp/mailchimp.route.js | 5 +- 11 files changed, 204 insertions(+), 135 deletions(-) diff --git a/src/config/server.js b/src/config/server.js index 39fbad97..3fc08979 100644 --- a/src/config/server.js +++ b/src/config/server.js @@ -18,26 +18,29 @@ require('dotenv').config(); var app = express(); -Sentry.init({ - dsn: 'https://b6ea46f0fbe048c9974718d2c72e261b@o444579.ingest.sentry.io/5653683', - environment: helper.getEnvironment(), - integrations: [ - // enable HTTP calls tracing - new Sentry.Integrations.Http({ tracing: true }), - // enable Express.js middleware tracing - new Tracing.Integrations.Express({ - // trace all requests to the default router - app, - }), - ], - tracesSampleRate: 1.0, -}); -// RequestHandler creates a separate execution context using domains, so that every -// transaction/span/breadcrumb is attached to its own Hub instance -app.use(Sentry.Handlers.requestHandler()); -// TracingHandler creates a trace for every incoming request -app.use(Sentry.Handlers.tracingHandler()); -app.use(Sentry.Handlers.errorHandler()); +const readEnv = process.env.ENV || 'prod'; +if (readEnv === 'test' || readEnv === 'prod') { + Sentry.init({ + dsn: 'https://b6ea46f0fbe048c9974718d2c72e261b@o444579.ingest.sentry.io/5653683', + environment: helper.getEnvironment(), + integrations: [ + // enable HTTP calls tracing + new Sentry.Integrations.Http({ tracing: true }), + // enable Express.js middleware tracing + new Tracing.Integrations.Express({ + // trace all requests to the default router + app, + }), + ], + tracesSampleRate: 1.0, + }); + // RequestHandler creates a separate execution context using domains, so that every + // transaction/span/breadcrumb is attached to its own Hub instance + app.use(Sentry.Handlers.requestHandler()); + // TracingHandler creates a trace for every incoming request + app.use(Sentry.Handlers.tracingHandler()); + app.use(Sentry.Handlers.errorHandler()); +} const Account = require('./account'); const configuration = require('./configuration'); diff --git a/src/resources/dataset/datasetonboarding.controller.js b/src/resources/dataset/datasetonboarding.controller.js index 76d2f38c..70196cfb 100644 --- a/src/resources/dataset/datasetonboarding.controller.js +++ b/src/resources/dataset/datasetonboarding.controller.js @@ -11,6 +11,8 @@ import moment from 'moment'; import * as Sentry from '@sentry/node'; var fs = require('fs'); +const readEnv = process.env.ENV || 'prod'; + module.exports = { //GET api/v1/dataset-onboarding getDatasetsByPublisher: async (req, res) => { @@ -856,7 +858,9 @@ module.exports = { return res.status(400).json({ success: false, message: 'No metadata found' }); } } catch (err) { - Sentry.captureException(err); + if (readEnv === 'test' || readEnv === 'prod') { + Sentry.captureException(err); + } console.error(err.message); return res.status(500).json({ success: false, message: 'Bulk upload of metadata failed', error: err.message }); } diff --git a/src/resources/dataset/v1/dataset.route.js b/src/resources/dataset/v1/dataset.route.js index 16fd6086..2760d41c 100644 --- a/src/resources/dataset/v1/dataset.route.js +++ b/src/resources/dataset/v1/dataset.route.js @@ -16,6 +16,8 @@ const datasetLimiter = rateLimit({ message: 'Too many calls have been made to this api from this IP, please try again after an hour', }); +const readEnv = process.env.ENV || 'prod'; + router.post('/', async (req, res) => { try { // Check to see if header is in json format @@ -43,7 +45,9 @@ router.post('/', async (req, res) => { // Return response indicating job has started (do not await async import) return res.status(200).json({ success: true, message: 'Caching started' }); } catch (err) { - Sentry.captureException(err); + if (readEnv === 'test' || readEnv === 'prod') { + Sentry.captureException(err); + } console.error(err.message); return res.status(500).json({ success: false, message: 'Caching failed' }); } @@ -73,7 +77,9 @@ router.post('/updateServices', async (req, res) => { return res.status(200).json({ success: true, message: 'Services Update started' }); } catch (err) { - Sentry.captureException(err); + if (readEnv === 'test' || readEnv === 'prod') { + Sentry.captureException(err); + } console.error(err.message); return res.status(500).json({ success: false, message: 'Services update failed' }); } diff --git a/src/resources/dataset/v1/dataset.service.js b/src/resources/dataset/v1/dataset.service.js index 9714c29c..de60f40b 100644 --- a/src/resources/dataset/v1/dataset.service.js +++ b/src/resources/dataset/v1/dataset.service.js @@ -16,6 +16,8 @@ let metadataQualityList = [], datasetsMDCIDs = [], counter = 0; +const readEnv = process.env.ENV || 'prod'; + export async function updateExternalDatasetServices(services) { for (let service of services) { if (service === 'phenotype') { @@ -24,12 +26,14 @@ export async function updateExternalDatasetServices(services) { timeout: 10000, }) .catch(err => { - Sentry.addBreadcrumb({ - category: 'Caching', - message: 'Unable to get metadata quality value ' + err.message, - level: Sentry.Severity.Error, - }); - Sentry.captureException(err); + if (readEnv === 'test' || readEnv === 'prod') { + Sentry.addBreadcrumb({ + category: 'Caching', + message: 'Unable to get metadata quality value ' + err.message, + level: Sentry.Severity.Error, + }); + Sentry.captureException(err); + } console.error('Unable to get metadata quality value ' + err.message); }); @@ -41,12 +45,14 @@ export async function updateExternalDatasetServices(services) { const dataUtilityList = await axios .get('https://raw.githubusercontent.com/HDRUK/datasets/master/reports/data_utility.json', { timeout: 10000 }) .catch(err => { - Sentry.addBreadcrumb({ - category: 'Caching', - message: 'Unable to get data utility ' + err.message, - level: Sentry.Severity.Error, - }); - Sentry.captureException(err); + if (readEnv === 'test' || readEnv === 'prod') { + Sentry.addBreadcrumb({ + category: 'Caching', + message: 'Unable to get data utility ' + err.message, + level: Sentry.Severity.Error, + }); + Sentry.captureException(err); + } console.error('Unable to get data utility ' + err.message); }); @@ -193,12 +199,14 @@ async function importMetadataFromCatalogue(baseUri, dataModelExportRoute, source await logoutCatalogue(baseUri); await loginCatalogue(baseUri, credentials); await loadDatasets(baseUri, dataModelExportRoute, datasetsMDCList.items, datasetsMDCList.count, source, limit).catch(err => { - Sentry.addBreadcrumb({ - category: 'Caching', - message: `Unable to complete the metadata import for ${source} ${err.message}`, - level: Sentry.Severity.Error, - }); - Sentry.captureException(err); + if (readEnv === 'test' || readEnv === 'prod') { + Sentry.addBreadcrumb({ + category: 'Caching', + message: `Unable to complete the metadata import for ${source} ${err.message}`, + level: Sentry.Severity.Error, + }); + Sentry.captureException(err); + } console.error(`Unable to complete the metadata import for ${source} ${err.message}`); }); await logoutCatalogue(baseUri); @@ -232,12 +240,14 @@ async function loadDatasets(baseUri, dataModelExportRoute, datasetsToImport, dat timeout: 60000, }) .catch(err => { - Sentry.addBreadcrumb({ - category: 'Caching', - message: 'Unable to get dataset JSON ' + err.message, - level: Sentry.Severity.Error, - }); - Sentry.captureException(err); + if (readEnv === 'test' || readEnv === 'prod') { + Sentry.addBreadcrumb({ + category: 'Caching', + message: 'Unable to get dataset JSON ' + err.message, + level: Sentry.Severity.Error, + }); + Sentry.captureException(err); + } console.error('Unable to get metadata JSON ' + err.message); }); @@ -249,22 +259,26 @@ async function loadDatasets(baseUri, dataModelExportRoute, datasetsToImport, dat timeout: 10000, }) .catch(err => { + if (readEnv === 'test' || readEnv === 'prod') { + Sentry.addBreadcrumb({ + category: 'Caching', + message: 'Unable to get metadata schema ' + err.message, + level: Sentry.Severity.Error, + }); + Sentry.captureException(err); + } + console.error('Unable to get metadata schema ' + err.message); + }); + + const versionLinksCall = axios.get(`${baseUri}/api/catalogueItems/${datasetMDC.id}/semanticLinks`, { timeout: 10000 }).catch(err => { + if (readEnv === 'test' || readEnv === 'prod') { Sentry.addBreadcrumb({ category: 'Caching', - message: 'Unable to get metadata schema ' + err.message, + message: 'Unable to get version links ' + err.message, level: Sentry.Severity.Error, }); Sentry.captureException(err); - console.error('Unable to get metadata schema ' + err.message); - }); - - const versionLinksCall = axios.get(`${baseUri}/api/catalogueItems/${datasetMDC.id}/semanticLinks`, { timeout: 10000 }).catch(err => { - Sentry.addBreadcrumb({ - category: 'Caching', - message: 'Unable to get version links ' + err.message, - level: Sentry.Severity.Error, - }); - Sentry.captureException(err); + } console.error('Unable to get version links ' + err.message); }); @@ -491,12 +505,14 @@ async function getDataUtilityExport() { return await axios .get('https://raw.githubusercontent.com/HDRUK/datasets/master/reports/data_utility.json', { timeout: 10000 }) .catch(err => { - Sentry.addBreadcrumb({ - category: 'Caching', - message: 'Unable to get data utility ' + err.message, - level: Sentry.Severity.Error, - }); - Sentry.captureException(err); + if (readEnv === 'test' || readEnv === 'prod') { + Sentry.addBreadcrumb({ + category: 'Caching', + message: 'Unable to get data utility ' + err.message, + level: Sentry.Severity.Error, + }); + Sentry.captureException(err); + } console.error('Unable to get data utility ' + err.message); }); } @@ -511,12 +527,14 @@ async function getPhenotypesExport() { return await axios .get('https://raw.githubusercontent.com/spiros/hdr-caliber-phenome-portal/master/_data/dataset2phenotypes.json', { timeout: 10000 }) .catch(err => { - Sentry.addBreadcrumb({ - category: 'Caching', - message: 'Unable to get metadata quality value ' + err.message, - level: Sentry.Severity.Error, - }); - Sentry.captureException(err); + if (readEnv === 'test' || readEnv === 'prod') { + Sentry.addBreadcrumb({ + category: 'Caching', + message: 'Unable to get metadata quality value ' + err.message, + level: Sentry.Severity.Error, + }); + Sentry.captureException(err); + } console.error('Unable to get metadata quality value ' + err.message); }); } @@ -531,12 +549,14 @@ async function getMetadataQualityExport() { return await axios .get('https://raw.githubusercontent.com/HDRUK/datasets/master/reports/metadata_quality.json', { timeout: 10000 }) .catch(err => { - Sentry.addBreadcrumb({ - category: 'Caching', - message: 'Unable to get metadata quality value ' + err.message, - level: Sentry.Severity.Error, - }); - Sentry.captureException(err); + if (readEnv === 'test' || readEnv === 'prod') { + Sentry.addBreadcrumb({ + category: 'Caching', + message: 'Unable to get metadata quality value ' + err.message, + level: Sentry.Severity.Error, + }); + Sentry.captureException(err); + } console.error('Unable to get metadata quality value ' + err.message); }); } @@ -549,12 +569,14 @@ async function getDataModels(baseUri) { resolve(response.data); }) .catch(err => { - Sentry.addBreadcrumb({ - category: 'Caching', - message: 'The caching run has failed because it was unable to get a count from the MDC', - level: Sentry.Severity.Fatal, - }); - Sentry.captureException(err); + if (readEnv === 'test' || readEnv === 'prod') { + Sentry.addBreadcrumb({ + category: 'Caching', + message: 'The caching run has failed because it was unable to get a count from the MDC', + level: Sentry.Severity.Fatal, + }); + Sentry.captureException(err); + } reject(err); }); }).catch(() => { @@ -567,14 +589,16 @@ async function checkDifferentialValid(incomingMetadataCount, source, override) { const datasetsHDRCount = await Data.countDocuments({ type: 'dataset', activeflag: 'active', source }); if ((incomingMetadataCount / datasetsHDRCount) * 100 < 90 && !override) { - Sentry.addBreadcrumb({ - category: 'Caching', - message: `The caching run has failed because the counts from the MDC (${incomingMetadataCount}) where ${ - 100 - (incomingMetadataCount / datasetsHDRCount) * 100 - }% lower than the number stored in the DB (${datasetsHDRCount})`, - level: Sentry.Severity.Fatal, - }); - Sentry.captureException(); + if (readEnv === 'test' || readEnv === 'prod') { + Sentry.addBreadcrumb({ + category: 'Caching', + message: `The caching run has failed because the counts from the MDC (${incomingMetadataCount}) where ${ + 100 - (incomingMetadataCount / datasetsHDRCount) * 100 + }% lower than the number stored in the DB (${datasetsHDRCount})`, + level: Sentry.Severity.Fatal, + }); + Sentry.captureException(); + } return false; } return true; diff --git a/src/resources/linkchecker/linkchecker.router.js b/src/resources/linkchecker/linkchecker.router.js index 7e0ba433..2cb2f4e9 100644 --- a/src/resources/linkchecker/linkchecker.router.js +++ b/src/resources/linkchecker/linkchecker.router.js @@ -8,6 +8,7 @@ import _ from 'lodash'; const sgMail = require('@sendgrid/mail'); const hdrukEmail = `enquiry@healthdatagateway.org`; +const readEnv = process.env.ENV || 'prod'; const axios = require('axios'); const router = express.Router(); @@ -111,7 +112,7 @@ router.post('/', async (req, res) => { }; await sgMail.send(msg, false, err => { - if (err) { + if (err && (readEnv === 'test' || readEnv === 'prod')) { Sentry.addBreadcrumb({ category: 'SendGrid', message: 'Sending email failed', diff --git a/src/resources/utilities/emailGenerator.util.js b/src/resources/utilities/emailGenerator.util.js index a5b0168e..fffeb795 100644 --- a/src/resources/utilities/emailGenerator.util.js +++ b/src/resources/utilities/emailGenerator.util.js @@ -6,6 +6,7 @@ import constants from '../utilities/constants.util'; import * as Sentry from '@sentry/node'; const sgMail = require('@sendgrid/mail'); +const readEnv = process.env.ENV || 'prod'; let parent, qsId; let questionList = []; let excludedQuestionSetIds = ['addRepeatableSection', 'removeRepeatableSection']; @@ -2423,7 +2424,7 @@ const _sendEmail = async (to, from, subject, html, allowUnsubscribe = true, atta // 4. Send email using SendGrid await sgMail.send(msg, false, err => { - if (err) { + if (err && (readEnv === 'test' || readEnv === 'prod')) { Sentry.addBreadcrumb({ category: 'SendGrid', message: 'Sending email failed', @@ -2446,7 +2447,7 @@ const _sendIntroEmail = msg => { sgMail.setApiKey(process.env.SENDGRID_API_KEY); // 2. Send email using SendGrid sgMail.send(msg, false, err => { - if (err) { + if (err && (readEnv === 'test' || readEnv === 'prod')) { Sentry.addBreadcrumb({ category: 'SendGrid', message: 'Sending email failed - Intro', diff --git a/src/resources/utilities/logger.js b/src/resources/utilities/logger.js index ad6a579f..3b937197 100644 --- a/src/resources/utilities/logger.js +++ b/src/resources/utilities/logger.js @@ -1,6 +1,8 @@ import * as Sentry from '@sentry/node'; import constants from './constants.util'; +const readEnv = process.env.ENV || 'prod'; + const logRequestMiddleware = options => { return (req, res, next) => { const { logCategory, action } = options; @@ -11,21 +13,25 @@ const logRequestMiddleware = options => { const logSystemActivity = options => { const { category = 'Action not categorised', action = 'Action not described' } = options; - Sentry.addBreadcrumb({ - category, - message: action, - level: Sentry.Severity.Info, - }); + if (readEnv === 'test' || readEnv === 'prod') { + Sentry.addBreadcrumb({ + category, + message: action, + level: Sentry.Severity.Info, + }); + } // Save to database }; const logUserActivity = (user, category, type, context) => { const { action } = context; - Sentry.addBreadcrumb({ - category, - message: action, - level: Sentry.Severity.Info, - }); + if (readEnv === 'test' || readEnv === 'prod') { + Sentry.addBreadcrumb({ + category, + message: action, + level: Sentry.Severity.Info, + }); + } console.log(`${action}`); // Log date/time // Log action @@ -35,11 +41,13 @@ const logUserActivity = (user, category, type, context) => { }; const logError = (err, category) => { - Sentry.captureException(err, { - tags: { - area: category, - }, - }); + if (readEnv === 'test' || readEnv === 'prod') { + Sentry.captureException(err, { + tags: { + area: category, + }, + }); + } console.error(`The following error occurred: ${err.message}`); }; diff --git a/src/services/hubspot/hubspot.js b/src/services/hubspot/hubspot.js index e91c462d..d1709e95 100644 --- a/src/services/hubspot/hubspot.js +++ b/src/services/hubspot/hubspot.js @@ -10,6 +10,7 @@ import { logger } from '../../resources/utilities/logger'; // Default service params const apiKey = process.env.HUBSPOT_API_KEY; const logCategory = 'Hubspot Integration'; +const readEnv = process.env.ENV || 'prod'; let hubspotClient; if (apiKey) hubspotClient = new Client({ apiKey, numberOfApiCallRetries: NumberOfRetries.Three }); @@ -140,11 +141,13 @@ const syncAllContacts = async () => { if (apiKey) { try { // Track attempted sync in Sentry using log - Sentry.addBreadcrumb({ - category: 'Hubspot', - message: `Syncing Gateway users with Hubspot contacts`, - level: Sentry.Severity.Log, - }); + if (readEnv === 'test' || readEnv === 'prod') { + Sentry.addBreadcrumb({ + category: 'Hubspot', + message: `Syncing Gateway users with Hubspot contacts`, + level: Sentry.Severity.Log, + }); + } // Batch import subscription changes from Hubspot await batchImportFromHubspot(); diff --git a/src/services/hubspot/hubspot.route.js b/src/services/hubspot/hubspot.route.js index 78998511..ef9f92c2 100644 --- a/src/services/hubspot/hubspot.route.js +++ b/src/services/hubspot/hubspot.route.js @@ -2,6 +2,7 @@ import express from 'express'; import * as Sentry from '@sentry/node'; import hubspotConnector from './hubspot'; const router = express.Router(); +const readEnv = process.env.ENV || 'prod'; // @router POST /api/v1/hubspot/sync // @desc Performs a two-way sync of contact details including communication opt in preferences between HubSpot and the Gateway database @@ -28,7 +29,9 @@ router.post('/sync', async (req, res) => { // Return response indicating job has started (do not await async import) return res.status(200).json({ success: true, message: 'Sync started' }); } catch (err) { - Sentry.captureException(err); + if (readEnv === 'test' || readEnv === 'prod') { + Sentry.captureException(err); + } console.error(err.message); return res.status(500).json({ success: false, message: 'Sync failed' }); } diff --git a/src/services/mailchimp/mailchimp.js b/src/services/mailchimp/mailchimp.js index c745642f..cc5b1d5b 100644 --- a/src/services/mailchimp/mailchimp.js +++ b/src/services/mailchimp/mailchimp.js @@ -13,6 +13,7 @@ let mailchimp; if (apiKey) mailchimp = new Mailchimp(apiKey); const tags = ['Gateway User']; const defaultSubscriptionStatus = constants.mailchimpSubscriptionStatuses.SUBSCRIBED; +const readEnv = process.env.ENV || 'prod'; /** * Create MailChimp Subscription Subscriber @@ -37,15 +38,19 @@ const addSubscriptionMember = async (subscriptionId, user, status) => { }, }; // 2. Track attempted update in Sentry using log - Sentry.addBreadcrumb({ - category: 'MailChimp', - message: `Adding subscription for user: ${id} to subscription: ${subscriptionId}`, - level: Sentry.Severity.Log, - }); + if (readEnv === 'test' || readEnv === 'prod') { + Sentry.addBreadcrumb({ + category: 'MailChimp', + message: `Adding subscription for user: ${id} to subscription: ${subscriptionId}`, + level: Sentry.Severity.Log, + }); + } // 3. POST to MailChimp Marketing API to add the Gateway user to the MailChimp subscription members const md5email = Crypto.createHash('md5').update(email).digest('hex'); await mailchimp.put(`lists/${subscriptionId}/members/${md5email}`, body).catch(err => { - Sentry.captureException(err); + if (readEnv === 'test' || readEnv === 'prod') { + Sentry.captureException(err); + } console.error(`Message: ${err.message} Errors: ${JSON.stringify(err.errors)}`); }); } @@ -100,16 +105,20 @@ const updateSubscriptionMembers = async (subscriptionId, members) => { update_existing: true, }; // 4. Track attempted updates in Sentry using log - Sentry.addBreadcrumb({ - category: 'MailChimp', - message: `Updating subscribed for members: ${members.map( - member => `${member.userId} to ${member.status}` - )} against subscription: ${subscriptionId}`, - level: Sentry.Severity.Log, - }); + if (readEnv === 'test' || readEnv === 'prod') { + Sentry.addBreadcrumb({ + category: 'MailChimp', + message: `Updating subscribed for members: ${members.map( + member => `${member.userId} to ${member.status}` + )} against subscription: ${subscriptionId}`, + level: Sentry.Severity.Log, + }); + } // 5. POST to MailChimp Marketing API to update member statuses await mailchimp.post(`lists/${subscriptionId}`, body).catch(err => { - Sentry.captureException(err); + if (readEnv === 'test' || readEnv === 'prod') { + Sentry.captureException(err); + } console.error(`Message: ${err.message} Errors: ${JSON.stringify(err.errors)}`); }); } @@ -126,16 +135,20 @@ const updateSubscriptionMembers = async (subscriptionId, members) => { const syncSubscriptionMembers = async subscriptionId => { if (apiKey) { // 1. Track attempted sync in Sentry using log - Sentry.addBreadcrumb({ - category: 'MailChimp', - message: `Syncing users for subscription: ${subscriptionId}`, - level: Sentry.Severity.Log, - }); + if (readEnv === 'test' || readEnv === 'prod') { + Sentry.addBreadcrumb({ + category: 'MailChimp', + message: `Syncing users for subscription: ${subscriptionId}`, + level: Sentry.Severity.Log, + }); + } // 2. Get total member count to anticipate chunking required to process all contacts const { stats: { member_count: subscribedCount, unsubscribe_count: unsubscribedCount }, } = await mailchimp.get(`lists/${subscriptionId}?fields=stats.member_count,stats.unsubscribe_count`).catch(err => { - Sentry.captureException(err); + if (readEnv === 'test' || readEnv === 'prod') { + Sentry.captureException(err); + } console.error(`Message: ${err.message} Errors: ${JSON.stringify(err.errors)}`); }); const memberCount = subscribedCount + unsubscribedCount; diff --git a/src/services/mailchimp/mailchimp.route.js b/src/services/mailchimp/mailchimp.route.js index cd023d3a..7d3e451e 100644 --- a/src/services/mailchimp/mailchimp.route.js +++ b/src/services/mailchimp/mailchimp.route.js @@ -2,6 +2,7 @@ import express from 'express'; import * as Sentry from '@sentry/node'; import mailchimpConnector from './mailchimp'; const router = express.Router(); +const readEnv = process.env.ENV || 'prod'; // @router GET /api/v1/mailchimp/:subscriptionId/sync // @desc Performs a two-way sync of opt in preferences between MailChimp and the Gateway database @@ -31,7 +32,9 @@ router.post('/sync', async (req, res) => { // Return response indicating job has started (do not await async import) return res.status(200).json({ success: true, message: 'Sync started' }); } catch (err) { - Sentry.captureException(err); + if (readEnv === 'test' || readEnv === 'prod') { + Sentry.captureException(err); + } console.error(err.message); return res.status(500).json({ success: false, message: 'Sync failed' }); } From e74c3365fd7cfa02e7fec8d397b415bde009f70a Mon Sep 17 00:00:00 2001 From: Callum Reekie Date: Tue, 2 Nov 2021 17:06:39 +0000 Subject: [PATCH 077/116] CR - move activitylog middleware --- src/middlewares/activitylog.middleware.js | 139 ++++++++++++++++ .../activitylog/activitylog.route.js | 156 +++--------------- 2 files changed, 161 insertions(+), 134 deletions(-) create mode 100644 src/middlewares/activitylog.middleware.js diff --git a/src/middlewares/activitylog.middleware.js b/src/middlewares/activitylog.middleware.js new file mode 100644 index 00000000..9636e5b4 --- /dev/null +++ b/src/middlewares/activitylog.middleware.js @@ -0,0 +1,139 @@ +import { isEmpty } from 'lodash'; + +import { activityLogService } from '../resources/activitylog/dependency'; +import { dataRequestService } from '../resources//datarequest/dependency'; +import constants from '../resources/utilities/constants.util'; + +const validateViewRequest = (req, res, next) => { + const { versionIds = [], type = '' } = req.body; + + if (isEmpty(versionIds) || !Object.values(constants.activityLogTypes).includes(type)) { + return res.status(400).json({ + success: false, + message: 'You must provide a valid log category and array of version identifiers to retrieve corresponding logs', + }); + } + + next(); +}; + +const authoriseView = async (req, res, next) => { + const requestingUser = req.user; + const { versionIds = [] } = req.body; + + const { authorised, userType, accessRecords } = await dataRequestService.checkUserAuthForVersions(versionIds, requestingUser); + if (!authorised) { + return res.status(401).json({ + success: false, + message: 'You are not authorised to perform this action', + }); + } + req.body.userType = userType; + req.body.accessRecords = accessRecords; + + next(); +}; + +const validateCreateRequest = (req, res, next) => { + const { versionId, description, timestamp } = req.body; + const { type } = req.params; + + if (!versionId || !description || !timestamp || !Object.values(constants.activityLogTypes).includes(type)) { + return res.status(400).json({ + success: false, + message: 'You must provide a valid log category and the following event details: associated version, description and timestamp', + }); + } + + next(); +}; + +const authoriseCreate = async (req, res, next) => { + const requestingUser = req.user; + const { versionId } = req.body; + const { type } = req.params; + + const { authorised, userType, accessRecords } = await dataRequestService.checkUserAuthForVersions([versionId], requestingUser); + if (isEmpty(accessRecords)) { + return res.status(404).json({ + success: false, + message: 'The requested application version could not be found', + }); + } + if (!authorised || userType !== constants.userTypes.CUSTODIAN) { + return res.status(401).json({ + success: false, + message: 'You are not authorised to perform this action', + }); + } + + req.body.userType = userType; + req.body.accessRecord = accessRecords[0]; + req.body.versionTitle = accessRecords[0].getVersionById(versionId).detailedTitle; + req.body.type = type; + + next(); +}; + +const validateDeleteRequest = (req, res, next) => { + const { id } = req.params; + + if (!id) { + return res.status(400).json({ + success: false, + message: 'You must provide a log event identifier', + }); + } + + next(); +}; + +const authoriseDelete = async (req, res, next) => { + const requestingUser = req.user; + const { id, type } = req.params; + + const log = await activityLogService.getLog(id, type); + + if (!log) { + return res.status(404).json({ + success: false, + message: 'The requested application log entry could not be found', + }); + } + + const { authorised, userType, accessRecords } = await dataRequestService.checkUserAuthForVersions([log.versionId], requestingUser); + if (isEmpty(accessRecords)) { + return res.status(404).json({ + success: false, + message: 'The requested application version could not be found', + }); + } + if (!authorised || userType !== constants.userTypes.CUSTODIAN) { + return res.status(401).json({ + success: false, + message: 'You are not authorised to perform this action', + }); + } + if (log.eventType !== constants.activityLogEvents.MANUAL_EVENT) { + return res.status(400).json({ + success: false, + message: 'You cannot delete a system generated log entry', + }); + } + + req.body.userType = userType; + req.body.accessRecord = accessRecords[0]; + req.body.versionId = log.versionId; + req.body.type = type; + + next(); +}; + +export default { + validateViewRequest, + authoriseView, + authoriseCreate, + validateCreateRequest, + validateDeleteRequest, + authoriseDelete, +}; diff --git a/src/resources/activitylog/activitylog.route.js b/src/resources/activitylog/activitylog.route.js index f13d02a3..b70a6bbe 100644 --- a/src/resources/activitylog/activitylog.route.js +++ b/src/resources/activitylog/activitylog.route.js @@ -1,161 +1,49 @@ import express from 'express'; import passport from 'passport'; +import ActivityLogMiddleware from '../../middlewares/activitylog.middleware'; import ActivityLogController from './activitylog.controller'; import { activityLogService } from './dependency'; -import { dataRequestService } from '../datarequest/dependency'; import { logger } from '../utilities/logger'; -import { isEmpty } from 'lodash'; -import constants from '../utilities/constants.util'; const router = express.Router(); const activityLogController = new ActivityLogController(activityLogService); const logCategory = 'Activity Log'; -const validateViewRequest = (req, res, next) => { - const { versionIds = [], type = '' } = req.body; - - if (isEmpty(versionIds) || !Object.values(constants.activityLogTypes).includes(type)) { - return res.status(400).json({ - success: false, - message: 'You must provide a valid log category and array of version identifiers to retrieve corresponding logs', - }); - } - - next(); -}; - -const authoriseView = async (req, res, next) => { - const requestingUser = req.user; - const { versionIds = [] } = req.body; - - const { authorised, userType, accessRecords } = await dataRequestService.checkUserAuthForVersions(versionIds, requestingUser); - if (!authorised) { - return res.status(401).json({ - success: false, - message: 'You are not authorised to perform this action', - }); - } - req.body.userType = userType; - req.body.accessRecords = accessRecords; - - next(); -}; - -const validateCreateRequest = (req, res, next) => { - const { versionId, description, timestamp } = req.body; - const { type } = req.params; - - if (!versionId || !description || !timestamp || !Object.values(constants.activityLogTypes).includes(type)) { - return res.status(400).json({ - success: false, - message: 'You must provide a valid log category and the following event details: associated version, description and timestamp', - }); - } - - next(); -}; - -const authoriseCreate = async (req, res, next) => { - const requestingUser = req.user; - const { versionId } = req.body; - const { type } = req.params; - - const { authorised, userType, accessRecords } = await dataRequestService.checkUserAuthForVersions([versionId], requestingUser); - if(isEmpty(accessRecords)) { - return res.status(404).json({ - success: false, - message: 'The requested application version could not be found', - }); - } - if (!authorised || userType !== constants.userTypes.CUSTODIAN) { - return res.status(401).json({ - success: false, - message: 'You are not authorised to perform this action', - }); - } - - req.body.userType = userType; - req.body.accessRecord = accessRecords[0]; - req.body.versionTitle = accessRecords[0].getVersionById(versionId).detailedTitle; - req.body.type = type; - - next(); -}; - -const validateDeleteRequest = (req, res, next) => { - const { id } = req.params; - - if (!id) { - return res.status(400).json({ - success: false, - message: 'You must provide a log event identifier', - }); - } - - next(); -}; - -const authoriseDelete = async (req, res, next) => { - const requestingUser = req.user; - const { id, type } = req.params; - - const log = await activityLogService.getLog(id, type); - - if(!log) { - return res.status(404).json({ - success: false, - message: 'The requested application log entry could not be found', - }); - } - - const { authorised, userType, accessRecords } = await dataRequestService.checkUserAuthForVersions([log.versionId], requestingUser); - if(isEmpty(accessRecords)) { - return res.status(404).json({ - success: false, - message: 'The requested application version could not be found', - }); - } - if (!authorised || userType !== constants.userTypes.CUSTODIAN) { - return res.status(401).json({ - success: false, - message: 'You are not authorised to perform this action', - }); - } - if (log.eventType !== constants.activityLogEvents.MANUAL_EVENT) { - return res.status(400).json({ - success: false, - message: 'You cannot delete a system generated log entry', - }); - } - - req.body.userType = userType; - req.body.accessRecord = accessRecords[0]; - req.body.versionId = log.versionId; - req.body.type = type; - - next(); -}; - // @route POST /api/v2/activitylog // @desc Returns a collection of logs based on supplied query parameters // @access Private -router.post('/', passport.authenticate('jwt'), validateViewRequest, authoriseView, logger.logRequestMiddleware({ logCategory, action: 'Viewed activity logs' }), (req, res) => - activityLogController.searchLogs(req, res) +router.post( + '/', + passport.authenticate('jwt'), + ActivityLogMiddleware.validateViewRequest, + ActivityLogMiddleware.authoriseView, + logger.logRequestMiddleware({ logCategory, action: 'Viewed activity logs' }), + (req, res) => activityLogController.searchLogs(req, res) ); // @route POST /api/v2/activitylog/event // @desc Creates a new manual event in the activity log identified in the payload // @access Private -router.post('/:type', passport.authenticate('jwt'), validateCreateRequest, authoriseCreate, logger.logRequestMiddleware({ logCategory, action: 'Created manual event' }), (req, res) => - activityLogController.createLog(req, res) +router.post( + '/:type', + passport.authenticate('jwt'), + ActivityLogMiddleware.validateCreateRequest, + ActivityLogMiddleware.authoriseCreate, + logger.logRequestMiddleware({ logCategory, action: 'Created manual event' }), + (req, res) => activityLogController.createLog(req, res) ); // @route DELETE /api/v2/activitylog/id // @desc Delete a manual event from the activity log // @access Private -router.delete('/:type/:id', passport.authenticate('jwt'), validateDeleteRequest, authoriseDelete, logger.logRequestMiddleware({ logCategory, action: 'Deleted manual event' }), (req, res) => - activityLogController.deleteLog(req, res) +router.delete( + '/:type/:id', + passport.authenticate('jwt'), + ActivityLogMiddleware.validateDeleteRequest, + ActivityLogMiddleware.authoriseDelete, + logger.logRequestMiddleware({ logCategory, action: 'Deleted manual event' }), + (req, res) => activityLogController.deleteLog(req, res) ); module.exports = router; From 4cd65b450b9378906adc5c5d9be36b5084fa947d Mon Sep 17 00:00:00 2001 From: Paul McCafferty <> Date: Thu, 4 Nov 2021 15:59:41 +0000 Subject: [PATCH 078/116] Updates to DUR --- .../dataUseRegister.controller.js | 52 +++++++++++++++++-- .../dataUseRegister/dataUseRegister.model.js | 5 +- .../dataUseRegister.repository.js | 32 ++++++++++-- .../dataUseRegister/dataUseRegister.route.js | 8 +-- .../dataUseRegister.service.js | 4 +- .../dataUseRegister/dataUseRegister.util.js | 15 ++++-- src/resources/dataset/dataset.service.js | 7 ++- src/resources/dataset/v1/dataset.route.js | 5 +- src/resources/search/search.repository.js | 28 +++++++++- 9 files changed, 133 insertions(+), 23 deletions(-) diff --git a/src/resources/dataUseRegister/dataUseRegister.controller.js b/src/resources/dataUseRegister/dataUseRegister.controller.js index f0d960d9..f4aa930a 100644 --- a/src/resources/dataUseRegister/dataUseRegister.controller.js +++ b/src/resources/dataUseRegister/dataUseRegister.controller.js @@ -1,4 +1,4 @@ -import { isNil } from 'lodash'; +import Mongoose from 'mongoose'; import Controller from '../base/controller'; import { logger } from '../utilities/logger'; import constants from './../utilities/constants.util'; @@ -43,6 +43,52 @@ export default class DataUseRegisterController extends Controller { as: 'creator', }, }, + { + $lookup: { + from: 'tools', + let: { + pid: '$pid', + }, + pipeline: [ + //{ $match: { $expr: { $in: ['$gatewayDatasets', '$$pid'] } } }, + { + $match: { + $expr: { + $and: [{ $eq: ['$gatewayDatasets', '$$pid'] }], + }, + }, + }, + { $project: { pid: 1, name: 1 } }, + + /* { + $match: { + $expr: { + $and: [ + { + $eq: ['$relatedObjects.pid', '$$pid'], + }, + { + $eq: ['$activeflag', 'active'], + }, + ], + }, + }, + }, + { $group: { _id: null, count: { $sum: 1 } } }, + */ + ], + as: 'gatewayDatasets2', + }, + }, + + /* { + $lookup: { + from: 'tools', + localField: 'gatewayDatasets', + foreignField: 'pid', + as: 'gatewayDatasets', + }, + }, */ ]); query.exec((err, data) => { if (data.length > 0) { @@ -117,12 +163,12 @@ export default class DataUseRegisterController extends Controller { query = { ...req.query, activeflag: constants.dataUseRegisterStatus.INREVIEW }; } else if (team !== 'user' && team !== 'admin') { delete req.query.team; - query = { ...req.query, publisher: team }; + query = { publisher: new Mongoose.Types.ObjectId(team) }; } else { query = req.query; } - const dataUseRegisters = await this.dataUseRegisterService.getDataUseRegisters(query).catch(err => { + const dataUseRegisters = await this.dataUseRegisterService.getDataUseRegisters({ $and: [query] }, { aggregate: true }).catch(err => { logger.logError(err, logCategory); }); // Return the dataUseRegisters diff --git a/src/resources/dataUseRegister/dataUseRegister.model.js b/src/resources/dataUseRegister/dataUseRegister.model.js index bcac029c..eb7c7044 100644 --- a/src/resources/dataUseRegister/dataUseRegister.model.js +++ b/src/resources/dataUseRegister/dataUseRegister.model.js @@ -29,11 +29,12 @@ const dataUseRegisterSchema = new Schema( projectId: { type: Schema.Types.ObjectId, ref: 'data_request' }, projectIdText: String, //Project ID datasetTitles: [{ type: String }], //Dataset Name(s) - datasetIds: [{ type: String }], - datasetPids: [{ type: String }], + gatewayDatasets: [{ type: String }], //Datasets on the Gateway + nonGatewayDatasets: [{ type: String }], //Dataset Name(s) publisher: { type: Schema.Types.ObjectId, ref: 'Publisher', required: true }, user: { type: Schema.Types.ObjectId, ref: 'User', required: true }, organisationName: { type: String }, //Organisation Name + organisationId: { type: String }, //Organisation ID organisationSector: String, //Organisation Sector gatewayApplicants: [ { diff --git a/src/resources/dataUseRegister/dataUseRegister.repository.js b/src/resources/dataUseRegister/dataUseRegister.repository.js index 90792bce..62fc7720 100644 --- a/src/resources/dataUseRegister/dataUseRegister.repository.js +++ b/src/resources/dataUseRegister/dataUseRegister.repository.js @@ -1,6 +1,7 @@ import Repository from '../base/repository'; import { DataUseRegister } from './dataUseRegister.model'; import { isNil } from 'lodash'; +import { filtersService } from '../filters/dependency'; export default class DataUseRegisterRepository extends Repository { constructor() { @@ -30,6 +31,30 @@ export default class DataUseRegisterRepository extends Repository { as: 'publisherDetails', }, }, + { + $lookup: { + from: 'tools', + let: { + listOfGatewayDatasets: '$gatewayDatasets', + }, + pipeline: [ + { + $match: { + $expr: { + $and: [ + { $in: ['$pid', '$$listOfGatewayDatasets'] }, + { + $eq: ['$activeflag', 'active'], + }, + ], + }, + }, + }, + { $project: { pid: 1, name: 1 } }, + ], + as: 'gatewayDatasets', + }, + }, { $addFields: { publisherDetails: { @@ -64,11 +89,12 @@ export default class DataUseRegisterRepository extends Repository { return this.dataUseRegister.findOne({ projectId: applicationId }, 'id').lean(); } - updateDataUseRegister(id, body) { + async updateDataUseRegister(id, body) { body.updatedon = Date.now(); body.lastActivity = Date.now(); - - return this.update(id, body); + const updatedBody = await this.update(id, body); + filtersService.optimiseFilters('dataUseRegister'); + return updatedBody; } uploadDataUseRegisters(dataUseRegisters) { diff --git a/src/resources/dataUseRegister/dataUseRegister.route.js b/src/resources/dataUseRegister/dataUseRegister.route.js index cb072804..b778e3e1 100644 --- a/src/resources/dataUseRegister/dataUseRegister.route.js +++ b/src/resources/dataUseRegister/dataUseRegister.route.js @@ -70,7 +70,7 @@ const validateUploadRequest = (req, res, next) => { next(); }; -const validateViewRequest = (req, res, next) => { +/* const validateViewRequest = (req, res, next) => { const { team } = req.query; if (!team) { @@ -81,7 +81,7 @@ const validateViewRequest = (req, res, next) => { } next(); -}; +}; */ const authorizeView = async (req, res, next) => { const requestingUser = req.user; @@ -157,13 +157,13 @@ router.get('/:id', logger.logRequestMiddleware({ logCategory, action: 'Viewed da router.get( '/', passport.authenticate('jwt'), - validateViewRequest, + /* validateViewRequest, */ authorizeView, logger.logRequestMiddleware({ logCategory, action: 'Viewed dataUseRegisters data' }), (req, res) => dataUseRegisterController.getDataUseRegisters(req, res) ); -// @route PUT /api/v2/data-use-registers/id +// @route PATCH /api/v2/data-use-registers/id // @desc Update the content of the data user register based on dataUseRegister ID provided // @access Public router.patch( diff --git a/src/resources/dataUseRegister/dataUseRegister.service.js b/src/resources/dataUseRegister/dataUseRegister.service.js index 095311f5..271d0bca 100644 --- a/src/resources/dataUseRegister/dataUseRegister.service.js +++ b/src/resources/dataUseRegister/dataUseRegister.service.js @@ -17,8 +17,8 @@ export default class DataUseRegisterService { return this.dataUseRegisterRepository.getDataUseRegister(query, options); } - getDataUseRegisters(query = {}) { - return this.dataUseRegisterRepository.getDataUseRegisters(query); + getDataUseRegisters(query = {}, options = {}) { + return this.dataUseRegisterRepository.getDataUseRegisters(query, options); } updateDataUseRegister(id, body = {}) { diff --git a/src/resources/dataUseRegister/dataUseRegister.util.js b/src/resources/dataUseRegister/dataUseRegister.util.js index 513174fa..349e5cf0 100644 --- a/src/resources/dataUseRegister/dataUseRegister.util.js +++ b/src/resources/dataUseRegister/dataUseRegister.util.js @@ -29,8 +29,6 @@ const buildDataUseRegisters = async (creatorUser, teamId, dataUses = []) => { }) ); const datasetTitles = [...linkedDatasets.map(dataset => dataset.name), ...namedDatasets]; - const datasetIds = [...linkedDatasets.map(dataset => dataset.datasetid)]; - const datasetPids = [...linkedDatasets.map(dataset => dataset.pid)]; // Handle applicant linkages const { gatewayApplicants, nonGatewayApplicants } = await getLinkedApplicants( @@ -84,6 +82,7 @@ const buildDataUseRegisters = async (creatorUser, teamId, dataUses = []) => { ...(obj.projectTitle && { projectTitle: obj.projectTitle.toString().trim() }), ...(obj.projectIdText && { projectIdText: obj.projectIdText.toString().trim() }), ...(obj.organisationName && { organisationName: obj.organisationName.toString().trim() }), + ...(obj.organisationId && { organisationId: obj.organisationId.toString().trim() }), ...(obj.organisationSector && { organisationSector: obj.organisationSector.toString().trim() }), ...(obj.applicantId && { applicantId: obj.applicantId.toString().trim() }), ...(obj.accreditedResearcherStatus && { accreditedResearcherStatus: obj.accreditedResearcherStatus.toString().trim() }), @@ -107,8 +106,9 @@ const buildDataUseRegisters = async (creatorUser, teamId, dataUses = []) => { ...(latestApprovalDate.isValid() && { latestApprovalDate }), ...(accessDate.isValid() && { accessDate }), ...(!isEmpty(datasetTitles) && { datasetTitles }), - ...(!isEmpty(datasetIds) && { datasetIds }), - ...(!isEmpty(datasetPids) && { datasetPids }), + ...(!isEmpty(linkedDatasets) && { gatewayDatasets: linkedDatasets.map(dataset => dataset.pid) }), + ...(!isEmpty(namedDatasets) && { nonGatewayDatasets: namedDatasets }), + ...(!isEmpty(gatewayApplicants) && { gatewayApplicants: gatewayApplicants.map(gatewayApplicant => gatewayApplicant._id) }), ...(!isEmpty(nonGatewayApplicants) && { nonGatewayApplicants }), ...(!isEmpty(fundersAndSponsors) && { fundersAndSponsors }), @@ -147,7 +147,12 @@ const getLinkedDatasets = async (datasetNames = []) => { if (datasetPid) { unverifiedDatasetPids.push(datasetPid); } else { - namedDatasets.push(datasetName); + let foundDataset = await datasetService.getDatasetsByName(datasetName); + if (foundDataset) { + unverifiedDatasetPids.push(foundDataset.pid); + } else { + namedDatasets.push(datasetName); + } } } diff --git a/src/resources/dataset/dataset.service.js b/src/resources/dataset/dataset.service.js index 6b09bfd7..c0968217 100644 --- a/src/resources/dataset/dataset.service.js +++ b/src/resources/dataset/dataset.service.js @@ -37,7 +37,7 @@ export default class DatasetService { return dataset; } - async getDatasets(query = {}, options = {} ) { + async getDatasets(query = {}, options = {}) { return this.datasetRepository.getDatasets(query, options); } @@ -114,4 +114,9 @@ export default class DatasetService { getDatasetsByPids(pids) { return this.datasetRepository.getDatasetsByPids(pids); } + + getDatasetsByName(name) { + let query = {}; + return this.datasetRepository.getDataset({ name, fields: 'pid' }, { lean: true }); + } } diff --git a/src/resources/dataset/v1/dataset.route.js b/src/resources/dataset/v1/dataset.route.js index 16fd6086..e5ea7db3 100644 --- a/src/resources/dataset/v1/dataset.route.js +++ b/src/resources/dataset/v1/dataset.route.js @@ -18,8 +18,9 @@ const datasetLimiter = rateLimit({ router.post('/', async (req, res) => { try { + filtersService.optimiseFilters('dataUseRegister'); // Check to see if header is in json format - let parsedBody = {}; + /* let parsedBody = {}; if (req.header('content-type') === 'application/json') { parsedBody = req.body; } else { @@ -39,7 +40,7 @@ router.post('/', async (req, res) => { importCatalogues(catalogues, override, limit).then(() => { filtersService.optimiseFilters('dataset'); saveUptime(); - }); + }); */ // Return response indicating job has started (do not await async import) return res.status(200).json({ success: true, message: 'Caching started' }); } catch (err) { diff --git a/src/resources/search/search.repository.js b/src/resources/search/search.repository.js index 6bf4d36e..f51b23d8 100644 --- a/src/resources/search/search.repository.js +++ b/src/resources/search/search.repository.js @@ -146,6 +146,30 @@ export async function getObjectResult(type, searchAll, searchQuery, startIndex, as: 'publisherDetails', }, }, + { + $lookup: { + from: 'tools', + let: { + listOfGatewayDatasets: '$gatewayDatasets', + }, + pipeline: [ + { + $match: { + $expr: { + $and: [ + { $in: ['$pid', '$$listOfGatewayDatasets'] }, + { + $eq: ['$activeflag', 'active'], + }, + ], + }, + }, + }, + { $project: { pid: 1, name: 1 } }, + ], + as: 'gatewayDatasets', + }, + }, { $addFields: { publisherDetails: { @@ -153,7 +177,7 @@ export async function getObjectResult(type, searchAll, searchQuery, startIndex, input: '$publisherDetails', as: 'row', in: { - name: '$$row.name', + name: '$$row.publisherDetails.name', }, }, }, @@ -169,6 +193,8 @@ export async function getObjectResult(type, searchAll, searchQuery, startIndex, keywords: 1, datasetTitles: 1, publisherDetails: 1, + gatewayDatasets: 1, + nonGatewayDatasets: 1, activeflag: 1, counter: 1, type: 1, From fae684640fe6b1591fc228d396aa11a0a74249ff Mon Sep 17 00:00:00 2001 From: Paul McCafferty <> Date: Fri, 5 Nov 2021 14:43:06 +0000 Subject: [PATCH 079/116] Updates --- .../__mocks__/dataUseRegisters.js | 16 +++---- .../dataUseRegister/dataUseRegister.model.js | 2 +- .../dataUseRegister.service.js | 4 +- .../dataUseRegister/dataUseRegister.util.js | 2 +- src/resources/dataset/dataset.controller.js | 46 +++++++++---------- 5 files changed, 35 insertions(+), 35 deletions(-) diff --git a/src/resources/dataUseRegister/__mocks__/dataUseRegisters.js b/src/resources/dataUseRegister/__mocks__/dataUseRegisters.js index 7bb54d15..72ad8be5 100644 --- a/src/resources/dataUseRegister/__mocks__/dataUseRegisters.js +++ b/src/resources/dataUseRegister/__mocks__/dataUseRegisters.js @@ -32,7 +32,7 @@ export const dataUseRegisterUploads = [ datasetLinkageDescription: 'data Processing Description', confidentialDataDescription: 'confidential Data Description', accessDate: '2021-09-26', - dataLocation: 'data Location', + accessType: 'data Location', privacyEnhancements: 'privacy Enhancements', researchOutputs: 'research Outputs', }, @@ -69,7 +69,7 @@ export const dataUseRegisterUploads = [ datasetLinkageDescription: 'data Processing Description', confidentialDataDescription: 'confidential Data Description', accessDate: '2021-09-26', - dataLocation: 'data Location', + accessType: 'data Location', privacyEnhancements: 'privacy Enhancements', researchOutputs: 'research Outputs', }, @@ -109,7 +109,7 @@ export const dataUseRegisterUploadsWithDuplicates = [ datasetLinkageDescription: 'data Processing Description', confidentialDataDescription: 'confidential Data Description', accessDate: '2021-09-26', - dataLocation: 'data Location', + accessType: 'data Location', privacyEnhancements: 'privacy Enhancements', researchOutputs: 'research Outputs', }, @@ -146,7 +146,7 @@ export const dataUseRegisterUploadsWithDuplicates = [ datasetLinkageDescription: 'data Processing Description', confidentialDataDescription: 'confidential Data Description', accessDate: '2021-09-26', - dataLocation: 'data Location', + accessType: 'data Location', privacyEnhancements: 'privacy Enhancements', researchOutputs: 'research Outputs', }, @@ -183,7 +183,7 @@ export const dataUseRegisterUploadsWithDuplicates = [ datasetLinkageDescription: 'data Processing Description', confidentialDataDescription: 'confidential Data Description', accessDate: '2021-09-26', - dataLocation: 'data Location', + accessType: 'data Location', privacyEnhancements: 'privacy Enhancements', researchOutputs: 'research Outputs', }, @@ -220,7 +220,7 @@ export const dataUseRegisterUploadsWithDuplicates = [ datasetLinkageDescription: 'data Processing Description', confidentialDataDescription: 'confidential Data Description', accessDate: '2021-09-26', - dataLocation: 'data Location', + accessType: 'data Location', privacyEnhancements: 'privacy Enhancements', researchOutputs: 'research Outputs', }, @@ -257,7 +257,7 @@ export const dataUseRegisterUploadsWithDuplicates = [ datasetLinkageDescription: 'data Processing Description', confidentialDataDescription: 'confidential Data Description', accessDate: '2021-09-26', - dataLocation: 'data Location', + accessType: 'data Location', privacyEnhancements: 'privacy Enhancements', researchOutputs: 'research Outputs', }, @@ -294,7 +294,7 @@ export const dataUseRegisterUploadsWithDuplicates = [ datasetLinkageDescription: 'data Processing Description', confidentialDataDescription: 'confidential Data Description', accessDate: '2021-09-26', - dataLocation: 'data Location', + accessType: 'data Location', privacyEnhancements: 'privacy Enhancements', researchOutputs: 'research Outputs', }, diff --git a/src/resources/dataUseRegister/dataUseRegister.model.js b/src/resources/dataUseRegister/dataUseRegister.model.js index eb7c7044..c9740942 100644 --- a/src/resources/dataUseRegister/dataUseRegister.model.js +++ b/src/resources/dataUseRegister/dataUseRegister.model.js @@ -64,7 +64,7 @@ const dataUseRegisterSchema = new Schema( datasetLinkageDescription: String, //Description Of How The Data Will Be Processed (changed to 'For linked datasets, specify how the linkage will take place') confidentialDataDescription: String, //Description Of The Confidential Data Being Used accessDate: Date, //Release/Access Date - dataLocation: String, //TRE Or Any Other Specified Location + accessType: String, //TRE Or Any Other Specified Location privacyEnhancements: String, //How Has Data Been Processed To Enhance Privacy researchOutputs: [{ type: String }], //Link To Research Outputs rejectionReason: String, //Reason For Rejecting A Data Use Register diff --git a/src/resources/dataUseRegister/dataUseRegister.service.js b/src/resources/dataUseRegister/dataUseRegister.service.js index 271d0bca..44060ecc 100644 --- a/src/resources/dataUseRegister/dataUseRegister.service.js +++ b/src/resources/dataUseRegister/dataUseRegister.service.js @@ -205,7 +205,7 @@ export default class DataUseRegisterService { safeprojectprojectdetailsresearchprojectsummarykeywords: keywords, ['safeproject-projectdetails-startdate']: startDate, ['safeproject-projectdetails-enddate']: endDate, - safedatastorageandprocessingaccessmethodtrustedresearchenvironment: dataLocation, + safedatastorageandprocessingaccessmethodtrustedresearchenvironment: accessType, safedataconfidentialityavenuelegalbasisconfidentialinformation: dutyOfConfidentiality, safedataotherdatasetslinkadditionaldatasetslinkagedetails: datasetLinkageDetails = '', safedataotherdatasetsrisksmitigations: datasetLinkageRiskMitigation = '', @@ -240,7 +240,7 @@ export default class DataUseRegisterService { ...(organisationName && { organisationName: organisationName.toString().trim() }), ...(laySummary && { laySummary: laySummary.toString().trim() }), ...(publicBenefitStatement && { publicBenefitStatement: publicBenefitStatement.toString().trim() }), - ...(dataLocation && { dataLocation: dataLocation.toString().trim() }), + ...(accessType && { accessType: accessType.toString().trim() }), ...(dutyOfConfidentiality && { dutyOfConfidentiality: dutyOfConfidentiality.toString().trim() }), ...(!isEmpty(datasetLinkageDescription) && { datasetLinkageDescription: datasetLinkageDescription.trim() }), ...(!isEmpty(requestFrequency) && { requestFrequency }), diff --git a/src/resources/dataUseRegister/dataUseRegister.util.js b/src/resources/dataUseRegister/dataUseRegister.util.js index 349e5cf0..a68711bc 100644 --- a/src/resources/dataUseRegister/dataUseRegister.util.js +++ b/src/resources/dataUseRegister/dataUseRegister.util.js @@ -98,7 +98,7 @@ const buildDataUseRegisters = async (creatorUser, teamId, dataUses = []) => { ...(obj.requestFrequency && { requestFrequency: obj.requestFrequency.toString().trim() }), ...(obj.datasetLinkageDescription && { datasetLinkageDescription: obj.datasetLinkageDescription.toString().trim() }), ...(obj.confidentialDataDescription && { confidentialDataDescription: obj.confidentialDataDescription.toString().trim() }), - ...(obj.dataLocation && { dataLocation: obj.dataLocation.toString().trim() }), + ...(obj.accessType && { accessType: obj.accessType.toString().trim() }), ...(obj.privacyEnhancements && { privacyEnhancements: obj.privacyEnhancements.toString().trim() }), ...(obj.dutyOfConfidentiality && { dutyOfConfidentiality: obj.dutyOfConfidentiality.toString().trim() }), ...(projectStartDate.isValid() && { projectStartDate }), diff --git a/src/resources/dataset/dataset.controller.js b/src/resources/dataset/dataset.controller.js index f6829c03..fd0bf4f5 100644 --- a/src/resources/dataset/dataset.controller.js +++ b/src/resources/dataset/dataset.controller.js @@ -2,63 +2,63 @@ import Controller from '../base/controller'; export default class DatasetController extends Controller { constructor(datasetService) { - super(datasetService); + super(datasetService); this.datasetService = datasetService; } async getDataset(req, res) { try { - // Extract id parameter from query string + // Extract id parameter from query string const { id } = req.params; - // If no id provided, it is a bad request + // If no id provided, it is a bad request if (!id) { return res.status(400).json({ success: false, message: 'You must provide a dataset identifier', }); } - // Find the dataset + // Find the dataset const options = { lean: false, populate: { path: 'submittedDataAccessRequests' } }; let dataset = await this.datasetService.getDataset(id, req.query, options); - // Return if no dataset found + // Return if no dataset found if (!dataset) { return res.status(404).json({ success: false, message: 'A dataset could not be found with the provided id', }); - } - // Return the dataset + } + // Return the dataset return res.status(200).json({ success: true, - ...dataset + ...dataset, }); } catch (err) { - // Return error response if something goes wrong - console.error(err.message); - return res.status(500).json({ + // Return error response if something goes wrong + console.error(err.message); + return res.status(500).json({ success: false, message: 'A server error occurred, please try again', }); - } - } - - async getDatasets(req, res) { + } + } + + async getDatasets(req, res) { try { - // Find the datasets + // Find the datasets const options = { lean: false, populate: { path: 'submittedDataAccessRequests' } }; - let datasets = await this.datasetService.getDatasets(req.query, options); - // Return the datasets + let datasets = await this.datasetService.getDatasets(req.query, options); + // Return the datasets return res.status(200).json({ success: true, - datasets + datasets, }); } catch (err) { - // Return error response if something goes wrong - console.error(err.message); - return res.status(500).json({ + // Return error response if something goes wrong + console.error(err.message); + return res.status(500).json({ success: false, message: 'A server error occurred, please try again', }); - } + } } } From cf4aca3d3e89e61e9d5664ba43b9145fed215aa2 Mon Sep 17 00:00:00 2001 From: Callum Reekie Date: Sat, 6 Nov 2021 10:33:26 +0000 Subject: [PATCH 080/116] CR - added dataset activity logs --- package.json | 1 + src/middlewares/activitylog.middleware.js | 41 +- .../activitylog/activitylog.controller.js | 28 +- .../activitylog/activitylog.model.js | 21 +- .../activitylog/activitylog.service.js | 378 +++++++++++------- .../datarequest/datarequest.controller.js | 58 ++- .../dataset/datasetonboarding.controller.js | 187 ++++----- .../dataset/utils/datasetonboarding.util.js | 132 +++++- src/resources/message/message.controller.js | 2 +- src/resources/utilities/constants.util.js | 53 ++- 10 files changed, 572 insertions(+), 329 deletions(-) diff --git a/package.json b/package.json index 1cc23db3..7773f154 100644 --- a/package.json +++ b/package.json @@ -38,6 +38,7 @@ "express-session": "^1.17.1", "express-validator": "^6.6.1", "faker": "^5.3.1", + "flat": "^5.0.2", "form-data": "^3.0.0", "googleapis": "^55.0.0", "jose": "^2.0.2", diff --git a/src/middlewares/activitylog.middleware.js b/src/middlewares/activitylog.middleware.js index 9636e5b4..41d8fb8d 100644 --- a/src/middlewares/activitylog.middleware.js +++ b/src/middlewares/activitylog.middleware.js @@ -2,6 +2,8 @@ import { isEmpty } from 'lodash'; import { activityLogService } from '../resources/activitylog/dependency'; import { dataRequestService } from '../resources//datarequest/dependency'; +import { datasetService } from '../resources/dataset/dependency'; +import datasetonboardingUtil from '../resources/dataset/utils/datasetonboarding.util'; import constants from '../resources/utilities/constants.util'; const validateViewRequest = (req, res, next) => { @@ -20,16 +22,39 @@ const validateViewRequest = (req, res, next) => { const authoriseView = async (req, res, next) => { const requestingUser = req.user; const { versionIds = [] } = req.body; - - const { authorised, userType, accessRecords } = await dataRequestService.checkUserAuthForVersions(versionIds, requestingUser); - if (!authorised) { - return res.status(401).json({ - success: false, - message: 'You are not authorised to perform this action', + let authorised, userType, accessRecords; + + if (req.body.type === constants.activityLogTypes.DATA_ACCESS_REQUEST) { + ({ authorised, userType, accessRecords } = await dataRequestService.checkUserAuthForVersions(versionIds, requestingUser)); + if (!authorised) { + return res.status(401).json({ + success: false, + message: 'You are not authorised to perform this action', + }); + } + + req.body.userType = userType; + req.body.versions = accessRecords; + } else if (req.body.type === constants.activityLogTypes.DATASET) { + const datasetVersions = await datasetService.getDatasets({ _id: { $in: versionIds } }, { lean: true }); + await datasetVersions.forEach(async version => { + ({ authorised } = await datasetonboardingUtil.getUserPermissionsForDataset( + version.datasetv2.identifier, + requestingUser, + version.datasetv2.summary.publisher.identifier + )); + if (!authorised) { + return res.status(401).json({ + success: false, + message: 'You are not authorised to perform this action', + }); + } }); + req.body.userType = requestingUser.teams.map(team => team.type).includes(constants.userTypes.ADMIN) + ? constants.userTypes.ADMIN + : constants.userTypes.CUSTODIAN; + req.body.versions = datasetVersions; } - req.body.userType = userType; - req.body.accessRecords = accessRecords; next(); }; diff --git a/src/resources/activitylog/activitylog.controller.js b/src/resources/activitylog/activitylog.controller.js index d08ac4ab..207750a7 100644 --- a/src/resources/activitylog/activitylog.controller.js +++ b/src/resources/activitylog/activitylog.controller.js @@ -16,10 +16,10 @@ export default class ActivityLogController extends Controller { async searchLogs(req, res) { try { // Extract required log params - const { versionIds = [], type = '', userType, accessRecords } = req.body; + const { versionIds = [], type = '', userType, versions } = req.body; // Find the logs - const logs = await this.activityLogService.searchLogs(versionIds, type, userType, accessRecords); + const logs = await this.activityLogService.searchLogs(versionIds, type, userType, versions); // Return the logs return res.status(200).json({ @@ -42,7 +42,7 @@ export default class ActivityLogController extends Controller { const { versionId, description, timestamp, type, userType, accessRecord, versionTitle } = req.body; // Create new event log - await this.activityLogService.logActivity(constants.activityLogEvents.MANUAL_EVENT, { + await this.activityLogService.logActivity(constants.activityLogEvents.data_access_request.MANUAL_EVENT, { versionId, versionTitle, description, @@ -52,7 +52,12 @@ export default class ActivityLogController extends Controller { }); // Send notifications - await this.createNotifications(constants.activityLogNotifications.MANUALEVENTADDED, { description, timestamp }, accessRecord, req.user); + await this.createNotifications( + constants.activityLogNotifications.MANUALEVENTADDED, + { description, timestamp }, + accessRecord, + req.user + ); // Get logs for version that was updated const [affectedVersion] = await this.activityLogService.searchLogs([versionId], type, userType, [accessRecord], false); @@ -85,7 +90,12 @@ export default class ActivityLogController extends Controller { await this.activityLogService.deleteLog(id); // Send notifications - await this.createNotifications(constants.activityLogNotifications.MANUALEVENTREMOVED, { description: log.plainText, timestamp: log.timestamp }, accessRecord, req.user); + await this.createNotifications( + constants.activityLogNotifications.MANUALEVENTREMOVED, + { description: log.plainText, timestamp: log.timestamp }, + accessRecord, + req.user + ); // Get logs for version that was updated const [affectedVersion] = await this.activityLogService.searchLogs([versionId], type, userType, [accessRecord], false); @@ -119,7 +129,9 @@ export default class ActivityLogController extends Controller { // Create in-app notifications await notificationBuilder.triggerNotificationMessage( teamMembersIds, - `${user.firstname} ${user.lastname} (${publisher}) has added an event to the activity log of '${projectName || `No project name set`}' data access request application`, + `${user.firstname} ${user.lastname} (${publisher}) has added an event to the activity log of '${ + projectName || `No project name set` + }' data access request application`, 'data access request log updated', _id, publisher @@ -150,7 +162,9 @@ export default class ActivityLogController extends Controller { // Create in-app notifications await notificationBuilder.triggerNotificationMessage( teamMembersIds, - `${user.firstname} ${user.lastname} (${publisher}) has deleted an event from the activity log of '${projectName || `No project name set`}' data access request application`, + `${user.firstname} ${user.lastname} (${publisher}) has deleted an event from the activity log of '${ + projectName || `No project name set` + }' data access request application`, 'data access request log updated', _id, publisher diff --git a/src/resources/activitylog/activitylog.model.js b/src/resources/activitylog/activitylog.model.js index 95b34797..f50a27a0 100644 --- a/src/resources/activitylog/activitylog.model.js +++ b/src/resources/activitylog/activitylog.model.js @@ -2,18 +2,29 @@ import { model, Schema } from 'mongoose'; import constants from '../utilities/constants.util'; const ActivityLogSchema = new Schema({ - eventType: { type: String, required: true, enum: Object.values(constants.activityLogEvents) }, + eventType: { + type: String, + required: true, + enum: Object.values({ ...constants.activityLogEvents.dataset, ...constants.activityLogEvents.data_access_request }), + }, logType: { type: String, required: true, enum: Object.values(constants.activityLogTypes) }, - userTypes: [], + userTypes: { type: Array, required: false, default: void 0 }, timestamp: { type: Date, required: true }, user: { type: Schema.Types.ObjectId, ref: 'User', required: true }, + userDetails: { + firstName: { type: String }, + lastName: { type: String }, + role: { type: String }, + }, versionId: { type: Schema.Types.ObjectId, required: true }, version: { type: String, required: true }, - plainText: { type: String, required: true }, + plainText: { type: String, required: false }, detailedText: String, - html: { type: String, required: true }, + html: { type: String, required: false }, detailedHtml: String, - isPresubmission: Boolean + isPresubmission: Boolean, + datasetUpdates: {}, + adminComment: String, }); export const ActivityLog = model('ActivityLog', ActivityLogSchema); diff --git a/src/resources/activitylog/activitylog.service.js b/src/resources/activitylog/activitylog.service.js index f3b735b0..3adc204a 100644 --- a/src/resources/activitylog/activitylog.service.js +++ b/src/resources/activitylog/activitylog.service.js @@ -10,7 +10,7 @@ export default class activityLogService { async searchLogs(versionIds, type, userType, versions, includePresubmission) { const logs = await this.activityLogRepository.searchLogs(versionIds, type, userType); - return this.formatLogs(logs, versions, includePresubmission); + return this.formatLogs(logs, type, versions, includePresubmission); } getLog(id, type) { @@ -21,6 +21,207 @@ export default class activityLogService { return this.activityLogRepository.deleteLog(id); } + formatLogs(logs, type, versions, includePresubmission = true) { + let formattedVersionEvents; + switch (type) { + case constants.activityLogTypes.DATA_ACCESS_REQUEST: + let presubmissionEvents = []; + if (includePresubmission) { + presubmissionEvents = this.buildPresubmissionEvents(logs); + } + + formattedVersionEvents = versions.reduce((arr, version) => { + const { + majorVersion: majorVersionNumber, + dateSubmitted, + dateCreated, + applicationType, + applicationStatus, + _id: majorVersionId, + amendmentIterations = [], + } = version; + + const partyDurations = this.getPartyTimeDistribution(version); + + const majorVersion = this.buildVersionEvents( + `${majorVersionNumber}`, + dateSubmitted, + dateCreated, + null, + applicationType, + applicationStatus, + () => this.getEventsForVersion(logs, majorVersionId), + () => this.calculateTimeWithParty(partyDurations, constants.userTypes.APPLICANT) + ); + + if (majorVersion.events.length > 0) { + arr.push(majorVersion); + } + + amendmentIterations.forEach((iterationMinorVersion, index) => { + const { + dateSubmitted: minorVersionDateSubmitted, + dateCreated: minorVersionDateCreated, + dateReturned: minorVersionDateReturned, + _id: minorVersionId, + } = iterationMinorVersion; + const partyDurations = this.getPartyTimeDistribution(iterationMinorVersion); + const minorVersion = this.buildVersionEvents( + `${majorVersionNumber}.${index + 1}`, + minorVersionDateSubmitted, + minorVersionDateCreated, + minorVersionDateReturned, + 'Update', + applicationStatus, + () => this.getEventsForVersion(logs, minorVersionId), + () => this.calculateTimeWithParty(partyDurations, constants.userTypes.APPLICANT) + ); + if (minorVersion.events.length > 0) { + arr.push(minorVersion); + } + }); + + return arr; + }, []); + + if (!isEmpty(presubmissionEvents)) { + formattedVersionEvents.push(presubmissionEvents); + } + break; + + case constants.activityLogTypes.DATASET: + formattedVersionEvents = versions.reduce((arr, version) => { + const { + datasetVersion, + timestamps: { submitted: dateSubmitted, created: dateCreated }, + activeflag, + _id, + } = version; + + const formattedVersion = { + version: `Version ${datasetVersion}`, + versionNumber: parseFloat(datasetVersion), + meta: { + ...(dateSubmitted && { dateSubmitted }), + ...(dateCreated && { dateCreated }), + applicationStatus: activeflag, + }, + events: this.getEventsForVersion(logs, _id), + }; + + arr.push(formattedVersion); + + return arr; + }, []); + break; + } + const orderedVersionEvents = orderBy(formattedVersionEvents, ['versionNumber'], ['desc']); + return orderedVersionEvents; + } + + logActivity(eventType, context) { + const logType = context.type; + switch (logType) { + case constants.activityLogTypes.DATA_ACCESS_REQUEST: + this.logDataAccessRequestActivity(eventType, context); + break; + case constants.activityLogTypes.DATASET: + this.logDatasetActivity(eventType, context); + break; + } + } + + async logDatasetActivity(eventType, context) { + const { updatedDataset, user, differences } = context; + const userRole = user.teams.map(team => team.type).includes(constants.userTypes.ADMIN) + ? constants.userTypes.ADMIN + : constants.userTypes.CUSTODIAN; + let log = { + eventType: eventType, + logType: constants.activityLogTypes.DATASET, + timestamp: Date.now(), + user: user._id, + userDetails: { firstName: user.firstname, lastName: user.lastname, role: userRole }, + version: updatedDataset.datasetVersion, + versionId: updatedDataset._id, + userTypes: [constants.userTypes.ADMIN, constants.userTypes.CUSTODIAN], + }; + + if ( + eventType === constants.activityLogEvents.dataset.DATASET_VERSION_REJECTED || + eventType === constants.activityLogEvents.dataset.DATASET_VERSION_APPROVED + ) + log['adminComment'] = updatedDataset.applicationStatusDesc; + + if (eventType === constants.activityLogEvents.dataset.DATASET_UPDATES_SUBMITTED && differences) log['datasetUpdates'] = differences; + + await this.activityLogRepository.createActivityLog(log); + } + + logDataAccessRequestActivity(eventType, context) { + switch (eventType) { + case constants.activityLogEvents.data_access_request.APPLICATION_SUBMITTED: + this.logApplicationSubmittedEvent(context); + break; + case constants.activityLogEvents.data_access_request.REVIEW_PROCESS_STARTED: + this.logReviewProcessStartedEvent(context); + break; + case constants.activityLogEvents.data_access_request.UPDATES_SUBMITTED: + this.logUpdatesSubmittedEvent(context); + break; + case constants.activityLogEvents.data_access_request.AMENDMENT_SUBMITTED: + this.logAmendmentSubmittedEvent(context); + break; + case constants.activityLogEvents.data_access_request.APPLICATION_APPROVED: + this.logApplicationApprovedEvent(context); + break; + case constants.activityLogEvents.data_access_request.APPLICATION_APPROVED_WITH_CONDITIONS: + this.logApplicationApprovedWithConditionsEvent(context); + break; + case constants.activityLogEvents.data_access_request.APPLICATION_REJECTED: + this.logApplicationRejectedEvent(context); + break; + case constants.activityLogEvents.data_access_request.COLLABORATOR_ADDEDD: + this.logCollaboratorAddedEvent(context); + break; + case constants.activityLogEvents.data_access_request.COLLABORATOR_REMOVED: + this.logCollaboratorRemovedEvent(context); + break; + case constants.activityLogEvents.data_access_request.PRESUBMISSION_MESSAGE: + this.logPresubmissionMessages(context); + break; + case constants.activityLogEvents.data_access_request.CONTEXTUAL_MESSAGE: + this.logContextualMessage(context); + break; + case constants.activityLogEvents.data_access_request.NOTE: + this.logNote(context); + break; + case constants.activityLogEvents.data_access_request.UPDATE_REQUESTED: + this.logUpdateRequestedEvent(context); + break; + case constants.activityLogEvents.data_access_request.WORKFLOW_ASSIGNED: + this.logWorkflowAssignedEvent(context); + break; + case constants.activityLogEvents.data_access_request.REVIEW_PHASE_STARTED: + this.logReviewPhaseStartedEvent(context); + break; + case constants.activityLogEvents.data_access_request.RECOMMENDATION_WITH_ISSUE: + this.logReccomendationWithIssueEvent(context); + break; + case constants.activityLogEvents.data_access_request.RECOMMENDATION_WITH_NO_ISSUE: + this.logReccomendationWithNoIssueEvent(context); + break; + case constants.activityLogEvents.data_access_request.DEADLINE_PASSED: + this.logDeadlinePassedEvent(context); + break; + case constants.activityLogEvents.data_access_request.FINAL_DECISION_REQUIRED: + this.logFinalDecisionRequiredEvent(context); + break; + case constants.activityLogEvents.data_access_request.MANUAL_EVENT: + this.logManualEvent(context); + } + } + getActiveQuestion(questionsArr, questionId) { let child; @@ -45,75 +246,6 @@ export default class activityLogService { } } - formatLogs(logs, versions, includePresubmission = true) { - let presubmissionEvents = []; - if (includePresubmission) { - presubmissionEvents = this.buildPresubmissionEvents(logs); - } - - const formattedVersionEvents = versions.reduce((arr, version) => { - const { - majorVersion: majorVersionNumber, - dateSubmitted, - dateCreated, - applicationType, - applicationStatus, - _id: majorVersionId, - amendmentIterations = [], - } = version; - - const partyDurations = this.getPartyTimeDistribution(version); - - const majorVersion = this.buildVersionEvents( - `${majorVersionNumber}`, - dateSubmitted, - dateCreated, - null, - applicationType, - applicationStatus, - () => this.getEventsForVersion(logs, majorVersionId), - () => this.calculateTimeWithParty(partyDurations, constants.userTypes.APPLICANT) - ); - - if (majorVersion.events.length > 0) { - arr.push(majorVersion); - } - - amendmentIterations.forEach((iterationMinorVersion, index) => { - const { - dateSubmitted: minorVersionDateSubmitted, - dateCreated: minorVersionDateCreated, - dateReturned: minorVersionDateReturned, - _id: minorVersionId, - } = iterationMinorVersion; - const partyDurations = this.getPartyTimeDistribution(iterationMinorVersion); - const minorVersion = this.buildVersionEvents( - `${majorVersionNumber}.${index + 1}`, - minorVersionDateSubmitted, - minorVersionDateCreated, - minorVersionDateReturned, - 'Update', - applicationStatus, - () => this.getEventsForVersion(logs, minorVersionId), - () => this.calculateTimeWithParty(partyDurations, constants.userTypes.APPLICANT) - ); - if (minorVersion.events.length > 0) { - arr.push(minorVersion); - } - }); - - return arr; - }, []); - - if (!isEmpty(presubmissionEvents)) { - formattedVersionEvents.push(presubmissionEvents); - } - - const orderedVersionEvents = orderBy(formattedVersionEvents, ['versionNumber'], ['desc']); - - return orderedVersionEvents; - } - buildPresubmissionEvents(logs) { const presubmissionEvents = this.getEventsForVersion(logs); @@ -293,76 +425,12 @@ export default class activityLogService { return partyDurations; } - async logActivity(eventType, context) { - switch (eventType) { - case constants.activityLogEvents.APPLICATION_SUBMITTED: - this.logApplicationSubmittedEvent(context); - break; - case constants.activityLogEvents.REVIEW_PROCESS_STARTED: - this.logReviewProcessStartedEvent(context); - break; - case constants.activityLogEvents.UPDATES_SUBMITTED: - this.logUpdatesSubmittedEvent(context); - break; - case constants.activityLogEvents.AMENDMENT_SUBMITTED: - this.logAmendmentSubmittedEvent(context); - break; - case constants.activityLogEvents.APPLICATION_APPROVED: - this.logApplicationApprovedEvent(context); - break; - case constants.activityLogEvents.APPLICATION_APPROVED_WITH_CONDITIONS: - this.logApplicationApprovedWithConditionsEvent(context); - break; - case constants.activityLogEvents.APPLICATION_REJECTED: - this.logApplicationRejectedEvent(context); - break; - case constants.activityLogEvents.COLLABORATOR_ADDEDD: - this.logCollaboratorAddedEvent(context); - break; - case constants.activityLogEvents.COLLABORATOR_REMOVED: - this.logCollaboratorRemovedEvent(context); - break; - case constants.activityLogEvents.PRESUBMISSION_MESSAGE: - this.logPresubmissionMessages(context); - break; - case constants.activityLogEvents.CONTEXTUAL_MESSAGE: - this.logContextualMessage(context); - break; - case constants.activityLogEvents.NOTE: - this.logNote(context); - break; - case constants.activityLogEvents.UPDATE_REQUESTED: - this.logUpdateRequestedEvent(context); - break; - case constants.activityLogEvents.WORKFLOW_ASSIGNED: - this.logWorkflowAssignedEvent(context); - break; - case constants.activityLogEvents.REVIEW_PHASE_STARTED: - this.logReviewPhaseStartedEvent(context); - break; - case constants.activityLogEvents.RECOMMENDATION_WITH_ISSUE: - this.logReccomendationWithIssueEvent(context); - break; - case constants.activityLogEvents.RECOMMENDATION_WITH_NO_ISSUE: - this.logReccomendationWithNoIssueEvent(context); - break; - case constants.activityLogEvents.DEADLINE_PASSED: - this.logDeadlinePassedEvent(context); - break; - case constants.activityLogEvents.FINAL_DECISION_REQUIRED: - this.logFinalDecisionRequiredEvent(context); - break; - case constants.activityLogEvents.MANUAL_EVENT: - this.logManualEvent(context); - } - } - async logReviewProcessStartedEvent(context) { const { accessRequest, user } = context; const version = accessRequest.versionTree[`${accessRequest.majorVersion}.0`]; const log = { - eventType: constants.activityLogEvents.REVIEW_PROCESS_STARTED, + eventType: constants.activityLogEvents.data_access_request.REVIEW_PROCESS_STARTED, logType: constants.activityLogTypes.DATA_ACCESS_REQUEST, timestamp: Date.now(), plainText: `Review process started by custodian manager ${user.firstname} ${user.lastname}`, @@ -381,7 +449,7 @@ export default class activityLogService { const version = accessRequest.versionTree[`${accessRequest.majorVersion}.${accessRequest.amendmentIterations.length}`]; const log = { - eventType: constants.activityLogEvents.APPLICATION_APPROVED, + eventType: constants.activityLogEvents.data_access_request.APPLICATION_APPROVED, logType: constants.activityLogTypes.DATA_ACCESS_REQUEST, timestamp: Date.now(), plainText: `Application approved by custodian manager ${user.firstname} ${user.lastname}`, @@ -406,7 +474,7 @@ export default class activityLogService { `
`; const log = { - eventType: constants.activityLogEvents.APPLICATION_APPROVED_WITH_CONDITIONS, + eventType: constants.activityLogEvents.data_access_request.APPLICATION_APPROVED_WITH_CONDITIONS, logType: constants.activityLogTypes.DATA_ACCESS_REQUEST, timestamp: Date.now(), plainText: `Application approved with conditions by custodian manager ${user.firstname} ${user.lastname}`, @@ -433,7 +501,7 @@ export default class activityLogService { ``; const log = { - eventType: constants.activityLogEvents.APPLICATION_REJECTED, + eventType: constants.activityLogEvents.data_access_request.APPLICATION_REJECTED, logType: constants.activityLogTypes.DATA_ACCESS_REQUEST, timestamp: Date.now(), plainText: `Application rejected by custodian manager ${user.firstname} ${user.lastname}`, @@ -454,7 +522,7 @@ export default class activityLogService { const version = accessRequest.versionTree[`${accessRequest.majorVersion}.0`]; const log = { - eventType: constants.activityLogEvents.APPLICATION_SUBMITTED, + eventType: constants.activityLogEvents.data_access_request.APPLICATION_SUBMITTED, logType: constants.activityLogTypes.DATA_ACCESS_REQUEST, timestamp: Date.now(), plainText: `Version 1 application has been submitted by applicant ${user.firstname} ${user.lastname}`, @@ -479,7 +547,7 @@ export default class activityLogService { ``; const log = { - eventType: constants.activityLogEvents.AMENDMENT_SUBMITTED, + eventType: constants.activityLogEvents.data_access_request.AMENDMENT_SUBMITTED, logType: constants.activityLogTypes.DATA_ACCESS_REQUEST, timestamp: Date.now(), plainText: `Amendment submitted by applicant ${user.firstname} ${user.lastname}. ${version.displayTitle} of this application has been created.`, @@ -545,7 +613,7 @@ export default class activityLogService { }); const logUpdate = { - eventType: constants.activityLogEvents.UPDATE_SUBMITTED, + eventType: constants.activityLogEvents.data_access_request.UPDATE_SUBMITTED, logType: constants.activityLogTypes.DATA_ACCESS_REQUEST, timestamp: Date.now(), detailedText: detText, @@ -565,7 +633,7 @@ export default class activityLogService { await this.activityLogRepository.createActivityLog(logUpdate); const logUpdates = { - eventType: constants.activityLogEvents.UPDATES_SUBMITTED, + eventType: constants.activityLogEvents.data_access_request.UPDATES_SUBMITTED, logType: constants.activityLogTypes.DATA_ACCESS_REQUEST, timestamp: Date.now(), plainText: `Updates submitted by applicant ${user.firstname} ${user.lastname}. ${currentVersion.displayTitle} of this application has been created.`, @@ -619,7 +687,7 @@ export default class activityLogService { }); const log = { - eventType: constants.activityLogEvents.UPDATE_REQUESTED, + eventType: constants.activityLogEvents.data_access_request.UPDATE_REQUESTED, logType: constants.activityLogTypes.DATA_ACCESS_REQUEST, timestamp: Date.now(), @@ -649,7 +717,7 @@ export default class activityLogService { }); const log = { - eventType: constants.activityLogEvents.COLLABORATOR_ADDEDD, + eventType: constants.activityLogEvents.data_access_request.COLLABORATOR_ADDEDD, logType: constants.activityLogTypes.DATA_ACCESS_REQUEST, timestamp: Date.now(), plainText: `Applicant ${user.firstname} ${user.lastname} added ${collaborator.firstname} ${collaborator.lastname} as a collaborator`, @@ -672,7 +740,7 @@ export default class activityLogService { }); const log = { - eventType: constants.activityLogEvents.COLLABORATOR_REMOVED, + eventType: constants.activityLogEvents.data_access_request.COLLABORATOR_REMOVED, logType: constants.activityLogTypes.DATA_ACCESS_REQUEST, timestamp: Date.now(), plainText: `Applicant ${user.firstname} ${user.lastname} removed ${collaborator.firstname} ${collaborator.lastname} as a collaborator`, @@ -715,7 +783,7 @@ export default class activityLogService { .join('')}`; const log = { - eventType: constants.activityLogEvents.WORKFLOW_ASSIGNED, + eventType: constants.activityLogEvents.data_access_request.WORKFLOW_ASSIGNED, logType: constants.activityLogTypes.DATA_ACCESS_REQUEST, timestamp: Date.now(), plainText: `${workflow.workflowName} has been assigned by custodian manager ${user.firstname} ${user.lastname}`, @@ -740,7 +808,7 @@ export default class activityLogService { const step = workflow.steps.find(step => step.active); const log = { - eventType: constants.activityLogEvents.REVIEW_PHASE_STARTED, + eventType: constants.activityLogEvents.data_access_request.REVIEW_PHASE_STARTED, logType: constants.activityLogTypes.DATA_ACCESS_REQUEST, timestamp: Date.now(), plainText: `${step.stepName} has started. ${workflow.steps.findIndex(step => step.active) + 1} out of ${ @@ -771,7 +839,7 @@ export default class activityLogService { const detText = `Recommendation: Issues found\n${comments}`; const log = { - eventType: constants.activityLogEvents.RECOMMENDATION_WITH_ISSUE, + eventType: constants.activityLogEvents.data_access_request.RECOMMENDATION_WITH_ISSUE, logType: constants.activityLogTypes.DATA_ACCESS_REQUEST, timestamp: Date.now(), plainText: `Recommendation with issues found sent by reviewer ${user.firstname} ${user.lastname}`, @@ -800,7 +868,7 @@ export default class activityLogService { const detText = `Recommendation: No issues found\n${comments}`; const log = { - eventType: constants.activityLogEvents.RECOMMENDATION_WITH_NO_ISSUE, + eventType: constants.activityLogEvents.data_access_request.RECOMMENDATION_WITH_NO_ISSUE, logType: constants.activityLogTypes.DATA_ACCESS_REQUEST, timestamp: Date.now(), plainText: `Recommendation with no issues found sent by reviewer ${user.firstname} ${user.lastname}`, @@ -821,7 +889,7 @@ export default class activityLogService { const version = accessRequest.versionTree[`${accessRequest.majorVersion}.${accessRequest.amendmentIterations.length}`]; const log = { - eventType: constants.activityLogEvents.FINAL_DECISION_REQUIRED, + eventType: constants.activityLogEvents.data_access_request.FINAL_DECISION_REQUIRED, logType: constants.activityLogTypes.DATA_ACCESS_REQUEST, timestamp: Date.now(), plainText: `Final decision required by custodian by custodian manager ${user.firstname} ${user.lastname}. All review phases completed`, @@ -846,7 +914,7 @@ export default class activityLogService { if (!userType) return; const log = { - eventType: constants.activityLogEvents.PRESUBMISSION_MESSAGE, + eventType: constants.activityLogEvents.data_access_request.PRESUBMISSION_MESSAGE, logType: constants.activityLogTypes.DATA_ACCESS_REQUEST, timestamp: createdDate, user: createdBy._id, @@ -905,7 +973,7 @@ export default class activityLogService { .join(''); const log = { - eventType: constants.activityLogEvents.DEADLINE_PASSED, + eventType: constants.activityLogEvents.data_access_request.DEADLINE_PASSED, logType: constants.activityLogTypes.DATA_ACCESS_REQUEST, timestamp: Date.now(), plainText: `Deadline was ${daysSinceDeadlinePassed} ${daysSinceDeadlinePassed > 1 ? 'days' : 'day'} ago for ${step.stepName} ${ @@ -931,7 +999,7 @@ export default class activityLogService { const { versionId, versionTitle, description, timestamp, user = {} } = context; const log = { - eventType: constants.activityLogEvents.MANUAL_EVENT, + eventType: constants.activityLogEvents.data_access_request.MANUAL_EVENT, logType: constants.activityLogTypes.DATA_ACCESS_REQUEST, timestamp, user: user._id, @@ -984,7 +1052,7 @@ export default class activityLogService { : `Message sent from applicant ${user.firstname} ${user.lastname}`; const log = { - eventType: constants.activityLogEvents.CONTEXTUAL_MESSAGE, + eventType: constants.activityLogEvents.data_access_request.CONTEXTUAL_MESSAGE, logType: constants.activityLogTypes.DATA_ACCESS_REQUEST, timestamp: Date.now(), user: user._id, @@ -1039,7 +1107,7 @@ export default class activityLogService { : `Note added by applicant ${user.firstname} ${user.lastname}`; const log = { - eventType: constants.activityLogEvents.NOTE, + eventType: constants.activityLogEvents.data_access_request.NOTE, logType: constants.activityLogTypes.DATA_ACCESS_REQUEST, timestamp: Date.now(), user: user._id, diff --git a/src/resources/datarequest/datarequest.controller.js b/src/resources/datarequest/datarequest.controller.js index fa558f58..887c2f63 100644 --- a/src/resources/datarequest/datarequest.controller.js +++ b/src/resources/datarequest/datarequest.controller.js @@ -130,13 +130,8 @@ export default class DataRequestController extends Controller { const countAmendments = this.amendmentService.countAmendments(accessRecord, userType, isLatestMinorVersion); // 8. Get the workflow status for the requested application version for the requesting user - const { - inReviewMode, - reviewSections, - hasRecommended, - isManager, - workflow, - } = this.workflowService.getApplicationWorkflowStatusForUser(accessRecord, requestingUserObjectId); + const { inReviewMode, reviewSections, hasRecommended, isManager, workflow } = + this.workflowService.getApplicationWorkflowStatusForUser(accessRecord, requestingUserObjectId); // 9. Get role type for requesting user, applicable for only Custodian users i.e. Manager/Reviewer role const userRole = @@ -362,7 +357,7 @@ export default class DataRequestController extends Controller { switch (accessRecord.applicationType) { case constants.submissionTypes.AMENDED: accessRecord = await this.dataRequestService.doAmendSubmission(accessRecord, description); - await this.activityLogService.logActivity(constants.activityLogEvents.AMENDMENT_SUBMITTED, { + await this.activityLogService.logActivity(constants.activityLogEvents.data_access_request.AMENDMENT_SUBMITTED, { accessRequest: accessRecord, user: requestingUser, }); @@ -371,7 +366,7 @@ export default class DataRequestController extends Controller { case constants.submissionTypes.INITIAL: default: accessRecord = await this.dataRequestService.doInitialSubmission(accessRecord); - await this.activityLogService.logActivity(constants.activityLogEvents.APPLICATION_SUBMITTED, { + await this.activityLogService.logActivity(constants.activityLogEvents.data_access_request.APPLICATION_SUBMITTED, { accessRequest: accessRecord, user: requestingUser, }); @@ -384,7 +379,7 @@ export default class DataRequestController extends Controller { ) { accessRecord = await this.amendmentService.doResubmission(accessRecord, requestingUserObjectId.toString()); await this.dataRequestService.syncRelatedVersions(accessRecord.versionTree); - await this.activityLogService.logActivity(constants.activityLogEvents.UPDATES_SUBMITTED, { + await this.activityLogService.logActivity(constants.activityLogEvents.data_access_request.UPDATES_SUBMITTED, { accessRequest: accessRecord, user: requestingUser, }); @@ -651,7 +646,7 @@ export default class DataRequestController extends Controller { let addedAuthors = [...newAuthors].filter(author => !currentAuthors.includes(author)); await addedAuthors.forEach(addedAuthor => - this.activityLogService.logActivity(constants.activityLogEvents.COLLABORATOR_ADDEDD, { + this.activityLogService.logActivity(constants.activityLogEvents.data_access_request.COLLABORATOR_ADDEDD, { accessRequest: accessRecord, user: req.user, collaboratorId: addedAuthor, @@ -660,7 +655,7 @@ export default class DataRequestController extends Controller { let removedAuthors = [...currentAuthors].filter(author => !newAuthors.includes(author)); await removedAuthors.forEach(removedAuthor => - this.activityLogService.logActivity(constants.activityLogEvents.COLLABORATOR_REMOVED, { + this.activityLogService.logActivity(constants.activityLogEvents.data_access_request.COLLABORATOR_REMOVED, { accessRequest: accessRecord, user: req.user, collaboratorId: removedAuthor, @@ -672,17 +667,20 @@ export default class DataRequestController extends Controller { this.dataRequestService.updateVersionStatus(accessRecord, accessRecord.applicationStatus); if (accessRecord.applicationStatus === constants.applicationStatuses.APPROVED) - await this.activityLogService.logActivity(constants.activityLogEvents.APPLICATION_APPROVED, { + await this.activityLogService.logActivity(constants.activityLogEvents.data_access_request.APPLICATION_APPROVED, { accessRequest: accessRecord, user: req.user, }); else if (accessRecord.applicationStatus === constants.applicationStatuses.APPROVEDWITHCONDITIONS) { - await this.activityLogService.logActivity(constants.activityLogEvents.APPLICATION_APPROVED_WITH_CONDITIONS, { - accessRequest: accessRecord, - user: req.user, - }); + await this.activityLogService.logActivity( + constants.activityLogEvents.data_access_request.APPLICATION_APPROVED_WITH_CONDITIONS, + { + accessRequest: accessRecord, + user: req.user, + } + ); } else if (accessRecord.applicationStatus === constants.applicationStatuses.REJECTED) { - await this.activityLogService.logActivity(constants.activityLogEvents.APPLICATION_REJECTED, { + await this.activityLogService.logActivity(constants.activityLogEvents.data_access_request.APPLICATION_REJECTED, { accessRequest: accessRecord, user: req.user, }); @@ -1385,13 +1383,13 @@ export default class DataRequestController extends Controller { this.createNotifications(constants.notificationTypes.WORKFLOWASSIGNED, emailContext, accessRecord, requestingUser); //Create activity log - this.activityLogService.logActivity(constants.activityLogEvents.WORKFLOW_ASSIGNED, { + this.activityLogService.logActivity(constants.activityLogEvents.data_access_request.WORKFLOW_ASSIGNED, { accessRequest: accessRecord, user: req.user, }); //Create activity log - this.activityLogService.logActivity(constants.activityLogEvents.REVIEW_PHASE_STARTED, { + this.activityLogService.logActivity(constants.activityLogEvents.data_access_request.REVIEW_PHASE_STARTED, { accessRequest: accessRecord, user: req.user, }); @@ -1498,7 +1496,7 @@ export default class DataRequestController extends Controller { // Create notifications to managers that the application is awaiting final approval relevantStepIndex = activeStepIndex; relevantNotificationType = constants.notificationTypes.FINALDECISIONREQUIRED; - this.activityLogService.logActivity(constants.activityLogEvents.FINAL_DECISION_REQUIRED, { + this.activityLogService.logActivity(constants.activityLogEvents.data_access_request.FINAL_DECISION_REQUIRED, { accessRequest: accessRecord, user: requestingUser, }); @@ -1506,7 +1504,7 @@ export default class DataRequestController extends Controller { // Create notifications to reviewers of the next step that has been activated relevantStepIndex = activeStepIndex + 1; relevantNotificationType = constants.notificationTypes.REVIEWSTEPSTART; - this.activityLogService.logActivity(constants.activityLogEvents.REVIEW_PHASE_STARTED, { + this.activityLogService.logActivity(constants.activityLogEvents.data_access_request.REVIEW_PHASE_STARTED, { accessRequest: accessRecord, user: requestingUser, }); @@ -1647,13 +1645,13 @@ export default class DataRequestController extends Controller { }); if (approved) { - this.activityLogService.logActivity(constants.activityLogEvents.RECOMMENDATION_WITH_NO_ISSUE, { + this.activityLogService.logActivity(constants.activityLogEvents.data_access_request.RECOMMENDATION_WITH_NO_ISSUE, { comments, accessRequest: accessRecord, user: requestingUser, }); } else { - this.activityLogService.logActivity(constants.activityLogEvents.RECOMMENDATION_WITH_ISSUE, { + this.activityLogService.logActivity(constants.activityLogEvents.data_access_request.RECOMMENDATION_WITH_ISSUE, { comments, accessRequest: accessRecord, user: requestingUser, @@ -1667,7 +1665,7 @@ export default class DataRequestController extends Controller { // Create notifications to reviewers of the next step that has been activated relevantStepIndex = activeStepIndex + 1; relevantNotificationType = constants.notificationTypes.REVIEWSTEPSTART; - this.activityLogService.logActivity(constants.activityLogEvents.REVIEW_PHASE_STARTED, { + this.activityLogService.logActivity(constants.activityLogEvents.data_access_request.REVIEW_PHASE_STARTED, { accessRequest: accessRecord, user: requestingUser, }); @@ -1675,7 +1673,7 @@ export default class DataRequestController extends Controller { // Create notifications to managers that the application is awaiting final approval relevantStepIndex = activeStepIndex; relevantNotificationType = constants.notificationTypes.FINALDECISIONREQUIRED; - this.activityLogService.logActivity(constants.activityLogEvents.FINAL_DECISION_REQUIRED, { + this.activityLogService.logActivity(constants.activityLogEvents.data_access_request.FINAL_DECISION_REQUIRED, { accessRequest: accessRecord, user: requestingUser, }); @@ -1771,7 +1769,7 @@ export default class DataRequestController extends Controller { } // 11. Log event in the activity log - await this.activityLogService.logActivity(constants.activityLogEvents.REVIEW_PROCESS_STARTED, { + await this.activityLogService.logActivity(constants.activityLogEvents.data_access_request.REVIEW_PROCESS_STARTED, { accessRequest: accessRecord, user: req.user, }); @@ -1816,7 +1814,7 @@ export default class DataRequestController extends Controller { // 4. Send emails based on deadline elapsed or approaching if (emailContext.deadlineElapsed) { this.createNotifications(constants.notificationTypes.DEADLINEPASSED, emailContext, accessRecord, requestingUser); - await this.activityLogService.logActivity(constants.activityLogEvents.DEADLINE_PASSED, { + await this.activityLogService.logActivity(constants.activityLogEvents.data_access_request.DEADLINE_PASSED, { accessRequest: accessRecord, }); } else { @@ -2907,8 +2905,8 @@ export default class DataRequestController extends Controller { this.activityLogService.logActivity( messageType === constants.DARMessageTypes.DARMESSAGE - ? constants.activityLogEvents.CONTEXTUAL_MESSAGE - : constants.activityLogEvents.NOTE, + ? constants.activityLogEvents.data_access_request.CONTEXTUAL_MESSAGE + : constants.activityLogEvents.data_access_request.NOTE, { accessRequest: accessRecord, user: req.user, diff --git a/src/resources/dataset/datasetonboarding.controller.js b/src/resources/dataset/datasetonboarding.controller.js index 76d2f38c..77633255 100644 --- a/src/resources/dataset/datasetonboarding.controller.js +++ b/src/resources/dataset/datasetonboarding.controller.js @@ -1,3 +1,10 @@ +import axios from 'axios'; +import FormData from 'form-data'; +import moment from 'moment'; +import * as Sentry from '@sentry/node'; +var fs = require('fs'); +import _ from 'lodash'; + import { Data } from '../tool/data.model'; import { PublisherModel } from '../publisher/publisher.model'; import { filtersService } from '../filters/dependency'; @@ -5,11 +12,7 @@ import constants from '../utilities/constants.util'; import datasetonboardingUtil from './utils/datasetonboarding.util'; import { v4 as uuidv4 } from 'uuid'; import { isEmpty, isNil, escapeRegExp } from 'lodash'; -import axios from 'axios'; -import FormData from 'form-data'; -import moment from 'moment'; -import * as Sentry from '@sentry/node'; -var fs = require('fs'); +import { activityLogService } from '../activitylog/dependency'; module.exports = { //GET api/v1/dataset-onboarding @@ -203,7 +206,9 @@ module.exports = { data.id = uniqueID; data.datasetid = 'New dataset version'; data.name = datasetToCopy.name; - data.datasetv2 = publisherObject; + data.datasetv2 = datasetToCopy.datasetv2; + data.datasetv2.identifier = ''; + data.datasetv2.version = ''; data.type = 'dataset'; data.activeflag = 'draft'; data.source = 'HDRUK MDC'; @@ -316,14 +321,43 @@ module.exports = { return res.status(401).json({ status: 'failure', message: 'Unauthorised' }); } + let dataset = await Data.findOne({ _id: id }); + dataset.questionAnswers = JSON.parse(dataset.questionAnswers); + + let datasetv2Object = await datasetonboardingUtil.buildv2Object(dataset); + //update dataset to inreview - constants.datatsetStatuses.INREVIEW let updatedDataset = await Data.findOneAndUpdate( { _id: id }, - { activeflag: constants.datatsetStatuses.INREVIEW, 'timestamps.updated': Date.now(), 'timestamps.submitted': Date.now() } + { + datasetv2: datasetv2Object, + activeflag: constants.datatsetStatuses.INREVIEW, + 'timestamps.updated': Date.now(), + 'timestamps.submitted': Date.now(), + } ); - //emails / notifications - await datasetonboardingUtil.createNotifications(constants.notificationTypes.DATASETSUBMITTED, updatedDataset); + // emails / notifications + //await datasetonboardingUtil.createNotifications(constants.notificationTypes.DATASETSUBMITTED, updatedDataset); + + await activityLogService.logActivity(constants.activityLogEvents.dataset.DATASET_VERSION_SUBMITTED, { + type: constants.activityLogTypes.DATASET, + updatedDataset, + user: req.user, + }); + + if (updatedDataset.datasetVersion !== '1.0.0') { + let datasetv2DifferenceObject = datasetonboardingUtil.datasetv2ObjectComparison(datasetv2Object, dataset.datasetv2); + + if (!_.isEmpty(datasetv2DifferenceObject)) { + await activityLogService.logActivity(constants.activityLogEvents.dataset.DATASET_UPDATES_SUBMITTED, { + type: constants.activityLogTypes.DATASET, + updatedDataset, + user: req.user, + differences: datasetv2DifferenceObject, + }); + } + } return res.status(200).json({ status: 'success' }); } catch (err) { @@ -355,6 +389,7 @@ module.exports = { } let dataset = await Data.findOne({ _id: id }); + if (!dataset) return res.status(404).json({ status: 'error', message: 'Dataset could not be found.' }); dataset.questionAnswers = JSON.parse(dataset.questionAnswers); @@ -423,94 +458,7 @@ module.exports = { }); // Adding to DB - let observations = await datasetonboardingUtil.buildObservations(dataset.questionAnswers); - - let datasetv2Object = { - identifier: newDatasetVersionId, - version: dataset.datasetVersion, - issued: moment(Date.now()).format('DD/MM/YYYY'), - modified: moment(Date.now()).format('DD/MM/YYYY'), - revisions: [], - summary: { - title: dataset.questionAnswers['properties/summary/title'] || '', - abstract: dataset.questionAnswers['properties/summary/abstract'] || '', - publisher: { - identifier: publisherData[0]._id.toString(), - name: publisherData[0].publisherDetails.name, - logo: publisherData[0].publisherDetails.logo || '', - description: publisherData[0].publisherDetails.description || '', - contactPoint: publisherData[0].publisherDetails.contactPoint || [], - memberOf: publisherData[0].publisherDetails.memberOf, - accessRights: publisherData[0].publisherDetails.accessRights || [], - deliveryLeadTime: publisherData[0].publisherDetails.deliveryLeadTime || '', - accessService: publisherData[0].publisherDetails.accessService || '', - accessRequestCost: publisherData[0].publisherDetails.accessRequestCost || '', - dataUseLimitation: publisherData[0].publisherDetails.dataUseLimitation || [], - dataUseRequirements: publisherData[0].publisherDetails.dataUseRequirements || [], - }, - contactPoint: dataset.questionAnswers['properties/summary/contactPoint'] || '', - keywords: dataset.questionAnswers['properties/summary/keywords'] || [], - alternateIdentifiers: dataset.questionAnswers['properties/summary/alternateIdentifiers'] || [], - doiName: dataset.questionAnswers['properties/summary/doiName'] || '', - }, - documentation: { - description: dataset.questionAnswers['properties/documentation/description'] || '', - associatedMedia: dataset.questionAnswers['properties/documentation/associatedMedia'] || [], - isPartOf: dataset.questionAnswers['properties/documentation/isPartOf'] || [], - }, - coverage: { - spatial: dataset.questionAnswers['properties/coverage/spatial'] || [], - typicalAgeRange: dataset.questionAnswers['properties/coverage/typicalAgeRange'] || '', - physicalSampleAvailability: dataset.questionAnswers['properties/coverage/physicalSampleAvailability'] || [], - followup: dataset.questionAnswers['properties/coverage/followup'] || '', - pathway: dataset.questionAnswers['properties/coverage/pathway'] || '', - }, - provenance: { - origin: { - purpose: dataset.questionAnswers['properties/provenance/origin/purpose'] || [], - source: dataset.questionAnswers['properties/provenance/origin/source'] || [], - collectionSituation: dataset.questionAnswers['properties/provenance/origin/collectionSituation'] || [], - }, - temporal: { - accrualPeriodicity: dataset.questionAnswers['properties/provenance/temporal/accrualPeriodicity'] || '', - distributionReleaseDate: dataset.questionAnswers['properties/provenance/temporal/distributionReleaseDate'] || '', - startDate: dataset.questionAnswers['properties/provenance/temporal/startDate'] || '', - endDate: dataset.questionAnswers['properties/provenance/temporal/endDate'] || '', - timeLag: dataset.questionAnswers['properties/provenance/temporal/timeLag'] || '', - }, - }, - accessibility: { - usage: { - dataUseLimitation: dataset.questionAnswers['properties/accessibility/usage/dataUseLimitation'] || [], - dataUseRequirements: dataset.questionAnswers['properties/accessibility/usage/dataUseRequirements'] || [], - resourceCreator: dataset.questionAnswers['properties/accessibility/usage/resourceCreator'] || '', - investigations: dataset.questionAnswers['properties/accessibility/usage/investigations'] || [], - isReferencedBy: dataset.questionAnswers['properties/accessibility/usage/isReferencedBy'] || [], - }, - access: { - accessRights: dataset.questionAnswers['properties/accessibility/access/accessRights'] || [], - accessService: dataset.questionAnswers['properties/accessibility/access/accessService'] || '', - accessRequestCost: dataset.questionAnswers['properties/accessibility/access/accessRequestCost'] || '', - deliveryLeadTime: dataset.questionAnswers['properties/accessibility/access/deliveryLeadTime'] || '', - jurisdiction: dataset.questionAnswers['properties/accessibility/access/jurisdiction'] || [], - dataProcessor: dataset.questionAnswers['properties/accessibility/access/dataProcessor'] || '', - dataController: dataset.questionAnswers['properties/accessibility/access/dataController'] || '', - }, - formatAndStandards: { - vocabularyEncodingScheme: - dataset.questionAnswers['properties/accessibility/formatAndStandards/vocabularyEncodingScheme'] || [], - conformsTo: dataset.questionAnswers['properties/accessibility/formatAndStandards/conformsTo'] || [], - language: dataset.questionAnswers['properties/accessibility/formatAndStandards/language'] || [], - format: dataset.questionAnswers['properties/accessibility/formatAndStandards/format'] || [], - }, - }, - enrichmentAndLinkage: { - qualifiedRelation: dataset.questionAnswers['properties/enrichmentAndLinkage/qualifiedRelation'] || [], - derivation: dataset.questionAnswers['properties/enrichmentAndLinkage/derivation'] || [], - tools: dataset.questionAnswers['properties/enrichmentAndLinkage/tools'] || [], - }, - observations: observations, - }; + let datasetv2Object = await datasetonboardingUtil.buildv2Object(dataset, newDatasetVersionId); let previousDataset = await Data.findOneAndUpdate({ pid: dataset.pid, activeflag: 'active' }, { activeflag: 'archive' }); let previousCounter = 0; @@ -574,8 +522,25 @@ module.exports = { filtersService.optimiseFilters('dataset'); + let datasetv2DifferenceObject = datasetonboardingUtil.datasetv2ObjectComparison(datasetv2Object, dataset.datasetv2); + + if (!_.isEmpty(datasetv2DifferenceObject)) { + await activityLogService.logActivity(constants.activityLogEvents.dataset.DATASET_UPDATES_SUBMITTED, { + type: constants.activityLogTypes.DATASET, + updatedDataset, + user: req.user, + differences: datasetv2DifferenceObject, + }); + } + //emails / notifications await datasetonboardingUtil.createNotifications(constants.notificationTypes.DATASETAPPROVED, updatedDataset); + + await activityLogService.logActivity(constants.activityLogEvents.dataset.DATASET_VERSION_APPROVED, { + type: constants.activityLogTypes.DATASET, + updatedDataset, + user: req.user, + }); }) .catch(err => { console.error('Error when trying to create new dataset on the MDC - ' + err.message); @@ -608,7 +573,13 @@ module.exports = { ); //emails / notifications - await datasetonboardingUtil.createNotifications(constants.notificationTypes.DATASETREJECTED, updatedDataset); + //await datasetonboardingUtil.createNotifications(constants.notificationTypes.DATASETREJECTED, updatedDataset); + + await activityLogService.logActivity(constants.activityLogEvents.dataset.DATASET_VERSION_REJECTED, { + type: constants.activityLogTypes.DATASET, + updatedDataset, + user: req.user, + }); return res.status(200).json({ status: 'success' }); } else if (applicationStatus === 'archive') { @@ -648,10 +619,17 @@ module.exports = { console.error('Error when trying to logout of the MDC - ' + err.message); }); } - await Data.findOneAndUpdate( + let updatedDataset = await Data.findOneAndUpdate( { _id: id }, { activeflag: constants.datatsetStatuses.ARCHIVE, 'timestamps.updated': Date.now(), 'timestamps.archived': Date.now() } ); + + await activityLogService.logActivity(constants.activityLogEvents.dataset.DATASET_VERSION_ARCHIVED, { + type: constants.activityLogTypes.DATASET, + updatedDataset, + user: req.user, + }); + return res.status(200).json({ status: 'success' }); } else if (applicationStatus === 'unarchive') { let dataset = await Data.findOne({ _id: id }).lean(); @@ -697,7 +675,14 @@ module.exports = { flagIs = 'active'; } - await Data.findOneAndUpdate({ _id: id }, { activeflag: flagIs }); //active or draft + const updatedDataset = await Data.findOneAndUpdate({ _id: id }, { activeflag: flagIs }); //active or draft + + await activityLogService.logActivity(constants.activityLogEvents.dataset.DATASET_VERSION_UNARCHIVED, { + type: constants.activityLogTypes.DATASET, + updatedDataset, + user: req.user, + }); + return res.status(200).json({ status: 'success' }); } } catch (err) { @@ -893,7 +878,7 @@ module.exports = { await Data.create(datasetCopy); - await datasetonboardingUtil.createNotifications(constants.notificationTypes.DATASETDUPLICATED, dataset); + //await datasetonboardingUtil.createNotifications(constants.notificationTypes.DATASETDUPLICATED, dataset); return res.status(200).json({ success: true, diff --git a/src/resources/dataset/utils/datasetonboarding.util.js b/src/resources/dataset/utils/datasetonboarding.util.js index 296a727c..9e08a356 100644 --- a/src/resources/dataset/utils/datasetonboarding.util.js +++ b/src/resources/dataset/utils/datasetonboarding.util.js @@ -4,13 +4,14 @@ import { PublisherModel } from '../../publisher/publisher.model'; import { UserModel } from '../../user/user.model'; import notificationBuilder from '../../utilities/notificationBuilder'; import emailGenerator from '../../utilities/emailGenerator.util'; -import { isEmpty, isNil, cloneDeep, isString, map, groupBy, orderBy } from 'lodash'; +import _, { isEmpty, isNil, cloneDeep, isString, map, groupBy, orderBy } from 'lodash'; import constants from '../../utilities/constants.util'; import moment from 'moment'; import randomstring from 'randomstring'; import Ajv from 'ajv'; import addFormats from 'ajv-formats'; var fs = require('fs'); +import { flatten, unflatten } from 'flat'; /** * Checks to see if the user has the correct permissions to access the dataset @@ -1099,6 +1100,133 @@ const buildBulkUploadObject = async arrayOfDraftDatasets => { } }; +/** + * Build the datasetV2 object from dataset.questionAnswers + * + * @param {Object} dataset [dataset.questionAnswers object] + * + * @return {Object} [return datasetv2 object] + */ +const buildv2Object = async (dataset, newDatasetVersionId = '') => { + const publisherData = await PublisherModel.find({ _id: dataset.datasetv2.summary.publisher.identifier }).lean(); + const questionAnswers = dataset.questionAnswers; + const observations = await buildObservations(dataset.questionAnswers); + + let datasetv2Object = { + identifier: newDatasetVersionId || '', + version: dataset.datasetVersion, + issued: moment(Date.now()).format('DD/MM/YYYY'), + modified: moment(Date.now()).format('DD/MM/YYYY'), + revisions: [], + summary: { + title: questionAnswers['properties/summary/title'] || '', + abstract: questionAnswers['properties/summary/abstract'] || '', + publisher: { + identifier: publisherData[0]._id.toString(), + name: publisherData[0].publisherDetails.name, + logo: publisherData[0].publisherDetails.logo || '', + description: publisherData[0].publisherDetails.description || '', + contactPoint: publisherData[0].publisherDetails.contactPoint || [], + memberOf: publisherData[0].publisherDetails.memberOf, + accessRights: publisherData[0].publisherDetails.accessRights || [], + deliveryLeadTime: publisherData[0].publisherDetails.deliveryLeadTime || '', + accessService: publisherData[0].publisherDetails.accessService || '', + accessRequestCost: publisherData[0].publisherDetails.accessRequestCost || '', + dataUseLimitation: publisherData[0].publisherDetails.dataUseLimitation || [], + dataUseRequirements: publisherData[0].publisherDetails.dataUseRequirements || [], + }, + contactPoint: questionAnswers['properties/summary/contactPoint'] || '', + keywords: questionAnswers['properties/summary/keywords'] || [], + alternateIdentifiers: questionAnswers['properties/summary/alternateIdentifiers'] || [], + doiName: questionAnswers['properties/summary/doiName'] || '', + }, + documentation: { + description: questionAnswers['properties/documentation/description'] || '', + associatedMedia: questionAnswers['properties/documentation/associatedMedia'] || [], + isPartOf: questionAnswers['properties/documentation/isPartOf'] || [], + }, + coverage: { + spatial: questionAnswers['properties/coverage/spatial'] || [], + typicalAgeRange: questionAnswers['properties/coverage/typicalAgeRange'] || '', + physicalSampleAvailability: questionAnswers['properties/coverage/physicalSampleAvailability'] || [], + followup: questionAnswers['properties/coverage/followup'] || '', + pathway: questionAnswers['properties/coverage/pathway'] || '', + }, + provenance: { + origin: { + purpose: questionAnswers['properties/provenance/origin/purpose'] || [], + source: questionAnswers['properties/provenance/origin/source'] || [], + collectionSituation: questionAnswers['properties/provenance/origin/collectionSituation'] || [], + }, + temporal: { + accrualPeriodicity: questionAnswers['properties/provenance/temporal/accrualPeriodicity'] || '', + distributionReleaseDate: questionAnswers['properties/provenance/temporal/distributionReleaseDate'] || '', + startDate: questionAnswers['properties/provenance/temporal/startDate'] || '', + endDate: questionAnswers['properties/provenance/temporal/endDate'] || '', + timeLag: questionAnswers['properties/provenance/temporal/timeLag'] || '', + }, + }, + accessibility: { + usage: { + dataUseLimitation: questionAnswers['properties/accessibility/usage/dataUseLimitation'] || [], + dataUseRequirements: questionAnswers['properties/accessibility/usage/dataUseRequirements'] || [], + resourceCreator: questionAnswers['properties/accessibility/usage/resourceCreator'] || '', + investigations: questionAnswers['properties/accessibility/usage/investigations'] || [], + isReferencedBy: questionAnswers['properties/accessibility/usage/isReferencedBy'] || [], + }, + access: { + accessRights: questionAnswers['properties/accessibility/access/accessRights'] || [], + accessService: questionAnswers['properties/accessibility/access/accessService'] || '', + accessRequestCost: questionAnswers['properties/accessibility/access/accessRequestCost'] || '', + deliveryLeadTime: questionAnswers['properties/accessibility/access/deliveryLeadTime'] || '', + jurisdiction: questionAnswers['properties/accessibility/access/jurisdiction'] || [], + dataProcessor: questionAnswers['properties/accessibility/access/dataProcessor'] || '', + dataController: questionAnswers['properties/accessibility/access/dataController'] || '', + }, + formatAndStandards: { + vocabularyEncodingScheme: questionAnswers['properties/accessibility/formatAndStandards/vocabularyEncodingScheme'] || [], + conformsTo: questionAnswers['properties/accessibility/formatAndStandards/conformsTo'] || [], + language: questionAnswers['properties/accessibility/formatAndStandards/language'] || [], + format: questionAnswers['properties/accessibility/formatAndStandards/format'] || [], + }, + }, + enrichmentAndLinkage: { + qualifiedRelation: questionAnswers['properties/enrichmentAndLinkage/qualifiedRelation'] || [], + derivation: questionAnswers['properties/enrichmentAndLinkage/derivation'] || [], + tools: questionAnswers['properties/enrichmentAndLinkage/tools'] || [], + }, + observations: observations, + }; + return datasetv2Object; +}; + +const datasetv2ObjectComparison = (updatedJSON, previousJSON) => { + updatedJSON = flatten(updatedJSON, { safe: true }); + previousJSON = flatten(previousJSON, { safe: true }); + let result = {}; + Object.keys(updatedJSON) + .concat(Object.keys(previousJSON)) + .forEach(key => { + if ( + previousJSON[key] !== updatedJSON[key] && + !_.isArray(updatedJSON[key], previousJSON[key]) && + !_.isObject(updatedJSON[key], previousJSON[key]) + ) { + result[key] = { previousAnswer: previousJSON[key], updatedAnswer: updatedJSON[key] }; + } + if (_.isArray(previousJSON[key]) || _.isArray(updatedJSON[key])) { + if (!_.isEqual(updatedJSON[key], previousJSON[key])) { + result[key] = { previousAnswer: previousJSON[key], updatedAnswer: updatedJSON[key] }; + } + } + }); + delete result['identifier']; + delete result['version']; + delete result['issued']; + delete result['modified']; + return unflatten(result); +}; + export default { getUserPermissionsForDataset, populateQuestionAnswers, @@ -1112,4 +1240,6 @@ export default { buildMetadataQuality, createNotifications, buildBulkUploadObject, + buildv2Object, + datasetv2ObjectComparison, }; diff --git a/src/resources/message/message.controller.js b/src/resources/message/message.controller.js index bf9db3d9..f45a32c3 100644 --- a/src/resources/message/message.controller.js +++ b/src/resources/message/message.controller.js @@ -163,7 +163,7 @@ module.exports = { // 20. Update activity log if there is a linked DAR if (topicObj.linkedDataAccessApplication) { - activityLogService.logActivity(constants.activityLogEvents.PRESUBMISSION_MESSAGE, { + activityLogService.logActivity(constants.activityLogEvents.data_access_request.PRESUBMISSION_MESSAGE, { messages: [messageObj], applicationId: topicObj.linkedDataAccessApplication, publisher: publisher.name, diff --git a/src/resources/utilities/constants.util.js b/src/resources/utilities/constants.util.js index 15303db6..acd01ba3 100644 --- a/src/resources/utilities/constants.util.js +++ b/src/resources/utilities/constants.util.js @@ -251,31 +251,42 @@ const _logTypes = { // Activity log related enums const _activityLogEvents = { - APPLICATION_SUBMITTED: 'applicationSubmitted', - REVIEW_PROCESS_STARTED: 'reviewProcessStarted', - UPDATES_SUBMITTED: 'updatesSubmitted', - AMENDMENT_SUBMITTED: 'amendmentSubmitted', - APPLICATION_APPROVED: 'applicationApproved', - APPLICATION_APPROVED_WITH_CONDITIONS: 'applicationApprovedWithConditions', - APPLICATION_REJECTED: 'applicationRejected', - COLLABORATOR_ADDEDD: 'collaboratorAdded', - COLLABORATOR_REMOVED: 'collaboratorRemoved', - PRESUBMISSION_MESSAGE: 'presubmissionMessage', - UPDATE_REQUESTED: 'updateRequested', - UPDATE_SUBMITTED: 'updateSubmitted', - WORKFLOW_ASSIGNED: 'workflowAssigned', - REVIEW_PHASE_STARTED: 'reviewPhaseStarted', - RECOMMENDATION_WITH_ISSUE: 'reccomendationWithIssue', - RECOMMENDATION_WITH_NO_ISSUE: 'reccomendationWithNoIssue', - FINAL_DECISION_REQUIRED: 'finalDecisionRequired', - DEADLINE_PASSED: 'deadlinePassed', - MANUAL_EVENT: 'manualEvent', - CONTEXTUAL_MESSAGE: 'contextualMessage', - NOTE: 'note', + data_access_request: { + APPLICATION_SUBMITTED: 'applicationSubmitted', + REVIEW_PROCESS_STARTED: 'reviewProcessStarted', + UPDATES_SUBMITTED: 'updatesSubmitted', + AMENDMENT_SUBMITTED: 'amendmentSubmitted', + APPLICATION_APPROVED: 'applicationApproved', + APPLICATION_APPROVED_WITH_CONDITIONS: 'applicationApprovedWithConditions', + APPLICATION_REJECTED: 'applicationRejected', + COLLABORATOR_ADDEDD: 'collaboratorAdded', + COLLABORATOR_REMOVED: 'collaboratorRemoved', + PRESUBMISSION_MESSAGE: 'presubmissionMessage', + UPDATE_REQUESTED: 'updateRequested', + UPDATE_SUBMITTED: 'updateSubmitted', + WORKFLOW_ASSIGNED: 'workflowAssigned', + REVIEW_PHASE_STARTED: 'reviewPhaseStarted', + RECOMMENDATION_WITH_ISSUE: 'reccomendationWithIssue', + RECOMMENDATION_WITH_NO_ISSUE: 'reccomendationWithNoIssue', + FINAL_DECISION_REQUIRED: 'finalDecisionRequired', + DEADLINE_PASSED: 'deadlinePassed', + MANUAL_EVENT: 'manualEvent', + CONTEXTUAL_MESSAGE: 'contextualMessage', + NOTE: 'note', + }, + dataset: { + DATASET_VERSION_SUBMITTED: 'newDatasetVersionSubmitted', + DATASET_VERSION_APPROVED: 'datasetVersionApproved', + DATASET_VERSION_REJECTED: 'datasetVersionRejected', + DATASET_VERSION_ARCHIVED: 'datasetVersionArchived', + DATASET_VERSION_UNARCHIVED: 'datasetVersionUnarchived', + DATASET_UPDATES_SUBMITTED: 'datasetUpdatesSubmitted', + }, }; const _activityLogTypes = { DATA_ACCESS_REQUEST: 'data_request', + DATASET: 'dataset', }; const _systemGeneratedUser = { From 4012b23e48f07a5a24abea688619d54387c68e0e Mon Sep 17 00:00:00 2001 From: Callum Reekie Date: Sat, 6 Nov 2021 10:34:08 +0000 Subject: [PATCH 081/116] CR - added dataset activity logs - update --- src/resources/datarequest/amendment/amendment.controller.js | 2 +- src/resources/datarequest/datarequest.service.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/resources/datarequest/amendment/amendment.controller.js b/src/resources/datarequest/amendment/amendment.controller.js index fc4b09f7..2347b26a 100644 --- a/src/resources/datarequest/amendment/amendment.controller.js +++ b/src/resources/datarequest/amendment/amendment.controller.js @@ -252,7 +252,7 @@ export default class AmendmentController extends Controller { } else { // 10. Send update request notifications let fullAccessRecord = await this.dataRequestService.getApplicationById(id); - await this.activityLogService.logActivity(constants.activityLogEvents.UPDATE_REQUESTED, { + await this.activityLogService.logActivity(constants.activityLogEvents.data_access_request.UPDATE_REQUESTED, { accessRequest: fullAccessRecord, user: req.user, }); diff --git a/src/resources/datarequest/datarequest.service.js b/src/resources/datarequest/datarequest.service.js index d9ea1246..bc3ae68b 100644 --- a/src/resources/datarequest/datarequest.service.js +++ b/src/resources/datarequest/datarequest.service.js @@ -141,7 +141,7 @@ export default class DataRequestService { topic.save(err => { if (!err) { // Create activity log entries based on existing messages in topic - activityLogService.logActivity(constants.activityLogEvents.PRESUBMISSION_MESSAGE, { + activityLogService.logActivity(constants.activityLogEvents.data_access_request.PRESUBMISSION_MESSAGE, { messages: topic.topicMessages, applicationId, publisher, From 1254fa426bb2d3f7fae1dee231c87956010e1c43 Mon Sep 17 00:00:00 2001 From: Callum Reekie Date: Mon, 8 Nov 2021 08:36:09 +0000 Subject: [PATCH 082/116] CR - added dataset activity logs - update --- src/resources/dataset/datasetonboarding.controller.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/resources/dataset/datasetonboarding.controller.js b/src/resources/dataset/datasetonboarding.controller.js index 77633255..09326d1e 100644 --- a/src/resources/dataset/datasetonboarding.controller.js +++ b/src/resources/dataset/datasetonboarding.controller.js @@ -338,7 +338,7 @@ module.exports = { ); // emails / notifications - //await datasetonboardingUtil.createNotifications(constants.notificationTypes.DATASETSUBMITTED, updatedDataset); + await datasetonboardingUtil.createNotifications(constants.notificationTypes.DATASETSUBMITTED, updatedDataset); await activityLogService.logActivity(constants.activityLogEvents.dataset.DATASET_VERSION_SUBMITTED, { type: constants.activityLogTypes.DATASET, From 406f50bb877104cf938d5d88789f7b20864220b4 Mon Sep 17 00:00:00 2001 From: Callum Reekie Date: Mon, 8 Nov 2021 11:14:50 +0000 Subject: [PATCH 083/116] CR - added dataset activity logs - update --- src/middlewares/activitylog.middleware.js | 44 +++++++++++++---------- 1 file changed, 26 insertions(+), 18 deletions(-) diff --git a/src/middlewares/activitylog.middleware.js b/src/middlewares/activitylog.middleware.js index 41d8fb8d..224a1ac1 100644 --- a/src/middlewares/activitylog.middleware.js +++ b/src/middlewares/activitylog.middleware.js @@ -36,24 +36,32 @@ const authoriseView = async (req, res, next) => { req.body.userType = userType; req.body.versions = accessRecords; } else if (req.body.type === constants.activityLogTypes.DATASET) { - const datasetVersions = await datasetService.getDatasets({ _id: { $in: versionIds } }, { lean: true }); - await datasetVersions.forEach(async version => { - ({ authorised } = await datasetonboardingUtil.getUserPermissionsForDataset( - version.datasetv2.identifier, - requestingUser, - version.datasetv2.summary.publisher.identifier - )); - if (!authorised) { - return res.status(401).json({ - success: false, - message: 'You are not authorised to perform this action', - }); - } - }); - req.body.userType = requestingUser.teams.map(team => team.type).includes(constants.userTypes.ADMIN) - ? constants.userTypes.ADMIN - : constants.userTypes.CUSTODIAN; - req.body.versions = datasetVersions; + try { + const datasetVersions = await datasetService.getDatasets({ _id: { $in: versionIds } }, { lean: true }); + await datasetVersions.forEach(async version => { + ({ authorised } = await datasetonboardingUtil.getUserPermissionsForDataset( + version.datasetv2.identifier, + requestingUser, + version.datasetv2.summary.publisher.identifier + )); + if (!authorised) { + return res.status(401).json({ + success: false, + message: 'You are not authorised to perform this action', + }); + } + }); + req.body.userType = requestingUser.teams.map(team => team.type).includes(constants.userTypes.ADMIN) + ? constants.userTypes.ADMIN + : constants.userTypes.CUSTODIAN; + req.body.versions = datasetVersions; + } catch (error) { + console.log('Error authenticating the user against submitted dataset version IDs - ', error); + return res.status(401).json({ + success: false, + message: 'Error authenticating the user against submitted dataset version IDs. Please check the submitted dataset versionIds', + }); + } } next(); From db7108008917818bfcd76736061f0f0db4d6a5c9 Mon Sep 17 00:00:00 2001 From: Callum Reekie Date: Mon, 8 Nov 2021 11:23:00 +0000 Subject: [PATCH 084/116] CR - remove redundant test --- .../dataset/__tests__/dataset.controller.test.js | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/src/resources/dataset/__tests__/dataset.controller.test.js b/src/resources/dataset/__tests__/dataset.controller.test.js index 4f9e6eb1..779fb458 100644 --- a/src/resources/dataset/__tests__/dataset.controller.test.js +++ b/src/resources/dataset/__tests__/dataset.controller.test.js @@ -35,18 +35,6 @@ describe('DatasetController', function () { expect(json.calledWith({ success: true, ...stubValue })).toBe(true); }); - it('should return a bad request response if no dataset id is provided', async function () { - req = { params: {} }; - - const serviceStub = sinon.stub(datasetService, 'getDataset').returns({}); - datasetController = new DatasetController(datasetService); - await datasetController.getDataset(req, res); - - expect(serviceStub.notCalled).toBe(true); - expect(status.calledWith(400)).toBe(true); - expect(json.calledWith({ success: false, message: 'You must provide a dataset identifier' })).toBe(true); - }); - it('should return a not found response if no dataset could be found for the id provided', async function () { req = { params: { id: faker.random.number({ min: 1, max: 999999999 }) } }; @@ -75,7 +63,7 @@ describe('DatasetController', function () { describe('getDatasets', function () { let req, res, status, json, datasetService, datasetController; - req = { params: {} }; + req = { params: {} }; beforeEach(() => { status = sinon.stub(); From d83ca6d7558905dfaade6d13cd5bf2b6ed4e510d Mon Sep 17 00:00:00 2001 From: Callum Reekie Date: Tue, 9 Nov 2021 13:29:50 +0000 Subject: [PATCH 085/116] CR - unit tests for activity log middleware --- .../__tests__/activitylog.middleware.test.js | 175 ++++++++++++++++++ src/middlewares/activitylog.middleware.js | 16 +- .../dataset/datasetonboarding.controller.js | 14 +- .../dataset/utils/datasetonboarding.util.js | 87 ++++++--- 4 files changed, 257 insertions(+), 35 deletions(-) create mode 100644 src/middlewares/__tests__/activitylog.middleware.test.js diff --git a/src/middlewares/__tests__/activitylog.middleware.test.js b/src/middlewares/__tests__/activitylog.middleware.test.js new file mode 100644 index 00000000..f3696e4c --- /dev/null +++ b/src/middlewares/__tests__/activitylog.middleware.test.js @@ -0,0 +1,175 @@ +import sinon from 'sinon'; + +import activitylogMiddleware from '../activitylog.middleware'; +import { datasetService } from '../../resources/dataset/dependency'; +import { UserModel } from '../../resources/user/user.model'; + +afterEach(function () { + sinon.restore(); +}); + +describe('Testing the ActivityLog middleware', () => { + const mockedRequest = () => { + const req = {}; + return req; + }; + + const mockedResponse = () => { + const res = {}; + res.status = jest.fn().mockReturnValue(res); + res.json = jest.fn().mockReturnValue(res); + return res; + }; + + describe('Testing the validateViewRequest middleware', () => { + const expectedResponse = { + success: false, + message: 'You must provide a valid log category and array of version identifiers to retrieve corresponding logs', + }; + + it('Should return 400 when no versionIds are passed in request', () => { + let req = mockedRequest(); + let res = mockedResponse(); + req.body = { versionIds: [], type: 'dataset' }; + const nextFunction = jest.fn(); + + activitylogMiddleware.validateViewRequest(req, res, nextFunction); + + expect(res.status).toHaveBeenCalledWith(400); + expect(res.json).toHaveBeenCalledWith(expectedResponse); + expect(nextFunction.mock.calls.length).toBe(0); + }); + + it('Should return 400 if activity log type "data_request" or "dataset"', () => { + let req = mockedRequest(); + let res = mockedResponse(); + req.body = { versionIds: [123, 456], type: 'notARealType' }; + const nextFunction = jest.fn(); + + activitylogMiddleware.validateViewRequest(req, res, nextFunction); + + expect(res.status).toHaveBeenCalledWith(400); + expect(res.json).toHaveBeenCalledWith(expectedResponse); + expect(nextFunction.mock.calls.length).toBe(0); + }); + + it('Should invoke next() if conditions are satisfied', () => { + let req = mockedRequest(); + let res = mockedResponse(); + req.body = { versionIds: [123, 456], type: 'dataset' }; + const nextFunction = jest.fn(); + + activitylogMiddleware.validateViewRequest(req, res, nextFunction); + + expect(nextFunction.mock.calls.length).toBe(1); + }); + }); + describe('Testing the authoriseView middleware', () => { + const expectedResponse = { + success: false, + message: 'You are not authorised to perform this action', + }; + it('Should return a 401 error if the user is not authorised', async () => { + let req = mockedRequest(); + let res = mockedResponse(); + req.body = { versionIds: ['xyz', 'abc'], type: 'dataset' }; + req.user = undefined; + const nextFunction = jest.fn(); + + let versionsStub = sinon.stub(datasetService, 'getDatasets').returns([ + { + datasetv2: { + identifier: 'abc', + summary: { + publisher: { + identifier: 'pub1', + }, + }, + }, + }, + { + datasetv2: { + identifier: 'xyz', + summary: { + publisher: { + identifier: 'pub2', + }, + }, + }, + }, + ]); + + await activitylogMiddleware.authoriseView(req, res, nextFunction); + + expect(versionsStub.calledOnce).toBe(true); + expect(res.status).toHaveBeenCalledWith(401); + expect(res.json).toHaveBeenCalledWith(expectedResponse); + expect(nextFunction.mock.calls.length).toBe(0); + }); + it('Should invoke next() if the user is authorised against dataset(s)', async () => { + let req = mockedRequest(); + let res = mockedResponse(); + req.body = { versionIds: ['xyz', 'abc'], type: 'dataset' }; + req.user = new UserModel({ + _id: '618a72fd5ec8f54772b7a17b', + firstname: 'John', + lastname: 'Smith', + teams: [ + { + publisher: { _id: 'fakeTeam', name: 'fakeTeam' }, + type: 'admin', + members: [{ memberid: '618a72fd5ec8f54772b7a17b', roles: ['admin_dataset'] }], + }, + ], + }); + const nextFunction = jest.fn(); + + let versionsStub = sinon.stub(datasetService, 'getDatasets').returns([ + { + datasetv2: { + identifier: 'abc', + summary: { + publisher: { + identifier: 'pub1', + }, + }, + }, + }, + { + datasetv2: { + identifier: 'xyz', + summary: { + publisher: { + identifier: 'pub2', + }, + }, + }, + }, + ]); + + await activitylogMiddleware.authoriseView(req, res, nextFunction); + + expect(versionsStub.calledOnce).toBe(true); + expect(nextFunction.mock.calls.length).toBe(1); + }); + + it('Should respond 401 if an error is thrown', async () => { + let req = mockedRequest(); + let res = mockedResponse(); + req.body = { versionIds: ['xyz', 'abc'], type: 'dataset' }; + const nextFunction = jest.fn(); + + let versionsStub = sinon.stub(datasetService, 'getDatasets').throws(); + + let badCall = await activitylogMiddleware.authoriseView(req, res, nextFunction); + + try { + badCall(); + } catch { + expect(versionsStub.calledOnce).toBe(true); + expect(nextFunction.mock.calls.length).toBe(0); + expect(res.status).toHaveBeenCalledWith(401); + } + }); + }); +}); diff --git a/src/middlewares/activitylog.middleware.js b/src/middlewares/activitylog.middleware.js index 224a1ac1..7af0ae6b 100644 --- a/src/middlewares/activitylog.middleware.js +++ b/src/middlewares/activitylog.middleware.js @@ -38,25 +38,27 @@ const authoriseView = async (req, res, next) => { } else if (req.body.type === constants.activityLogTypes.DATASET) { try { const datasetVersions = await datasetService.getDatasets({ _id: { $in: versionIds } }, { lean: true }); + let permissionsArray = []; await datasetVersions.forEach(async version => { ({ authorised } = await datasetonboardingUtil.getUserPermissionsForDataset( version.datasetv2.identifier, requestingUser, version.datasetv2.summary.publisher.identifier )); - if (!authorised) { - return res.status(401).json({ - success: false, - message: 'You are not authorised to perform this action', - }); - } + permissionsArray.push(authorised); }); + + if (!permissionsArray.includes(true)) { + return res.status(401).json({ + success: false, + message: 'You are not authorised to perform this action', + }); + } req.body.userType = requestingUser.teams.map(team => team.type).includes(constants.userTypes.ADMIN) ? constants.userTypes.ADMIN : constants.userTypes.CUSTODIAN; req.body.versions = datasetVersions; } catch (error) { - console.log('Error authenticating the user against submitted dataset version IDs - ', error); return res.status(401).json({ success: false, message: 'Error authenticating the user against submitted dataset version IDs. Please check the submitted dataset versionIds', diff --git a/src/resources/dataset/datasetonboarding.controller.js b/src/resources/dataset/datasetonboarding.controller.js index 09326d1e..5c14a866 100644 --- a/src/resources/dataset/datasetonboarding.controller.js +++ b/src/resources/dataset/datasetonboarding.controller.js @@ -331,20 +331,20 @@ module.exports = { { _id: id }, { datasetv2: datasetv2Object, - activeflag: constants.datatsetStatuses.INREVIEW, + activeflag: constants.datatsetStatuses.DRAFT, 'timestamps.updated': Date.now(), 'timestamps.submitted': Date.now(), } ); // emails / notifications - await datasetonboardingUtil.createNotifications(constants.notificationTypes.DATASETSUBMITTED, updatedDataset); + //await datasetonboardingUtil.createNotifications(constants.notificationTypes.DATASETSUBMITTED, updatedDataset); - await activityLogService.logActivity(constants.activityLogEvents.dataset.DATASET_VERSION_SUBMITTED, { - type: constants.activityLogTypes.DATASET, - updatedDataset, - user: req.user, - }); + // await activityLogService.logActivity(constants.activityLogEvents.dataset.DATASET_VERSION_SUBMITTED, { + // type: constants.activityLogTypes.DATASET, + // updatedDataset, + // user: req.user, + // }); if (updatedDataset.datasetVersion !== '1.0.0') { let datasetv2DifferenceObject = datasetonboardingUtil.datasetv2ObjectComparison(datasetv2Object, dataset.datasetv2); diff --git a/src/resources/dataset/utils/datasetonboarding.util.js b/src/resources/dataset/utils/datasetonboarding.util.js index 9e08a356..28e75977 100644 --- a/src/resources/dataset/utils/datasetonboarding.util.js +++ b/src/resources/dataset/utils/datasetonboarding.util.js @@ -1201,30 +1201,75 @@ const buildv2Object = async (dataset, newDatasetVersionId = '') => { }; const datasetv2ObjectComparison = (updatedJSON, previousJSON) => { - updatedJSON = flatten(updatedJSON, { safe: true }); - previousJSON = flatten(previousJSON, { safe: true }); - let result = {}; - Object.keys(updatedJSON) - .concat(Object.keys(previousJSON)) - .forEach(key => { - if ( - previousJSON[key] !== updatedJSON[key] && - !_.isArray(updatedJSON[key], previousJSON[key]) && - !_.isObject(updatedJSON[key], previousJSON[key]) - ) { - result[key] = { previousAnswer: previousJSON[key], updatedAnswer: updatedJSON[key] }; + updatedJSON = flatten(updatedJSON, { safe: true, delimiter: '/' }); + previousJSON = flatten(previousJSON, { safe: true, delimiter: '/' }); + + const unusedKeys = ['identifier', 'version', 'issued', 'modified']; + unusedKeys.forEach(key => { + delete updatedJSON[key]; + delete previousJSON[key]; + }); + + let result = []; + const datasetv2Keys = [...new Set(Object.keys(updatedJSON).concat(Object.keys(previousJSON)))]; + datasetv2Keys.forEach(key => { + if ( + previousJSON[key] !== updatedJSON[key] && + !_.isArray(updatedJSON[key], previousJSON[key]) && + !_.isObject(updatedJSON[key], previousJSON[key]) && + key !== 'observations' + ) { + let arrayObject = {}; + arrayObject[key] = { previousAnswer: previousJSON[key], updatedAnswer: updatedJSON[key] }; + result.push(arrayObject); + } + if ((_.isArray(previousJSON[key]) || _.isArray(updatedJSON[key])) && key !== 'observations') { + if (!_.isEqual(updatedJSON[key], previousJSON[key])) { + let arrayObject = {}; + arrayObject[key] = { previousAnswer: previousJSON[key].join(', '), updatedAnswer: updatedJSON[key].join(', ') }; + result.push(arrayObject); } - if (_.isArray(previousJSON[key]) || _.isArray(updatedJSON[key])) { - if (!_.isEqual(updatedJSON[key], previousJSON[key])) { - result[key] = { previousAnswer: previousJSON[key], updatedAnswer: updatedJSON[key] }; - } + } + }); + + const observationKeys = ['observedNode', 'measuredValue', 'disambiguatingDescription', 'observationDate', 'measuredProperty']; + + const maxObservationLength = Math.max(previousJSON['observations'].length, updatedJSON['observations'].length); + let resultObservations = {}; + for (let i = 0; i < maxObservationLength; i++) { + let newKeyName = 'observations/' + (i + 1).toString() + '/'; + resultObservations[newKeyName] = {}; + if (updatedJSON['observations'][i] === undefined) { + updatedJSON['observations'][i] = {}; + observationKeys.forEach(key => { + updatedJSON['observations'][i][key] = ''; + }); + } + + if (previousJSON['observations'][i] === undefined) { + previousJSON['observations'][i] = {}; + observationKeys.forEach(key => { + previousJSON['observations'][i][key] = ''; + }); + } + + observationKeys.forEach(key => { + if (!_.isEqual(updatedJSON['observations'][i][key], previousJSON['observations'][i][key])) { + resultObservations[newKeyName + key] = { + previousAnswer: previousJSON['observations'][i][key], + updatedAnswer: updatedJSON['observations'][i][key], + }; } }); - delete result['identifier']; - delete result['version']; - delete result['issued']; - delete result['modified']; - return unflatten(result); + if (_.isEmpty(resultObservations[newKeyName])) delete resultObservations[newKeyName]; + } + + Object.keys(resultObservations).forEach(key => { + let arrayObject = {}; + arrayObject[key] = resultObservations[key]; + result.push(arrayObject); + }); + return result; }; export default { From e325102704e89d97021b27960101f7a919d283ac Mon Sep 17 00:00:00 2001 From: Callum Reekie Date: Tue, 9 Nov 2021 13:56:27 +0000 Subject: [PATCH 086/116] CR - unit tests for activity log middleware - update --- .../dataset/datasetonboarding.controller.js | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/resources/dataset/datasetonboarding.controller.js b/src/resources/dataset/datasetonboarding.controller.js index 5c14a866..09326d1e 100644 --- a/src/resources/dataset/datasetonboarding.controller.js +++ b/src/resources/dataset/datasetonboarding.controller.js @@ -331,20 +331,20 @@ module.exports = { { _id: id }, { datasetv2: datasetv2Object, - activeflag: constants.datatsetStatuses.DRAFT, + activeflag: constants.datatsetStatuses.INREVIEW, 'timestamps.updated': Date.now(), 'timestamps.submitted': Date.now(), } ); // emails / notifications - //await datasetonboardingUtil.createNotifications(constants.notificationTypes.DATASETSUBMITTED, updatedDataset); + await datasetonboardingUtil.createNotifications(constants.notificationTypes.DATASETSUBMITTED, updatedDataset); - // await activityLogService.logActivity(constants.activityLogEvents.dataset.DATASET_VERSION_SUBMITTED, { - // type: constants.activityLogTypes.DATASET, - // updatedDataset, - // user: req.user, - // }); + await activityLogService.logActivity(constants.activityLogEvents.dataset.DATASET_VERSION_SUBMITTED, { + type: constants.activityLogTypes.DATASET, + updatedDataset, + user: req.user, + }); if (updatedDataset.datasetVersion !== '1.0.0') { let datasetv2DifferenceObject = datasetonboardingUtil.datasetv2ObjectComparison(datasetv2Object, dataset.datasetv2); From 2db9bfa574cb3753eee16d19a9ec9e3d6d7afe83 Mon Sep 17 00:00:00 2001 From: Callum Reekie Date: Tue, 9 Nov 2021 17:17:43 +0000 Subject: [PATCH 087/116] CR - dataset activity logs service tests --- .../{activitylogs.js => activitylogs.dar.js} | 0 .../__mocks__/activitylogs.dataset.js | 104 ++++++++++++++++++ .../__tests__/activitylog.service.test.js | 85 +++++++++++++- 3 files changed, 187 insertions(+), 2 deletions(-) rename src/resources/activitylog/__mocks__/{activitylogs.js => activitylogs.dar.js} (100%) create mode 100644 src/resources/activitylog/__mocks__/activitylogs.dataset.js diff --git a/src/resources/activitylog/__mocks__/activitylogs.js b/src/resources/activitylog/__mocks__/activitylogs.dar.js similarity index 100% rename from src/resources/activitylog/__mocks__/activitylogs.js rename to src/resources/activitylog/__mocks__/activitylogs.dar.js diff --git a/src/resources/activitylog/__mocks__/activitylogs.dataset.js b/src/resources/activitylog/__mocks__/activitylogs.dataset.js new file mode 100644 index 00000000..7446eff2 --- /dev/null +++ b/src/resources/activitylog/__mocks__/activitylogs.dataset.js @@ -0,0 +1,104 @@ +export const datasetActivityLogMocks = [ + { + _id: '6189679675a82a0867ce55b9', + eventType: 'newDatasetVersionSubmitted', + logType: 'dataset', + timestamp: '12345', + user: '616993c3034a7d773064e208', + userDetails: { firstName: 'John', lastName: 'Smith', role: 'custodian' }, + version: '1.0.0', + versionId: '6189673475a82a0867ce54fa', + userTypes: ['admin', 'custodian'], + __v: 0, + }, + { + _id: '618967d475a82a0867ce56b0', + eventType: 'newDatasetVersionSubmitted', + logType: 'dataset', + timestamp: '12345', + user: '616993c3034a7d773064e208', + userDetails: { firstName: 'John', lastName: 'Smith', role: 'custodian' }, + version: '2.0.0', + versionId: '618967b075a82a0867ce5650', + userTypes: ['admin', 'custodian'], + __v: 0, + }, +]; + +export const formattedJSONResponseMock = [ + { + version: 'Version 2.0.0', + versionNumber: 2, + meta: { + dateSubmitted: '12345', + dateCreated: '12345', + applicationStatus: 'active', + }, + events: [ + { + _id: '618967d475a82a0867ce56b0', + eventType: 'newDatasetVersionSubmitted', + logType: 'dataset', + timestamp: '12345', + user: '616993c3034a7d773064e208', + userDetails: { + firstName: 'John', + lastName: 'Smith', + role: 'custodian', + }, + version: '2.0.0', + versionId: '618967b075a82a0867ce5650', + userTypes: ['admin', 'custodian'], + __v: 0, + }, + ], + }, + { + version: 'Version 1.0.0', + versionNumber: 1, + meta: { + dateSubmitted: '12345', + dateCreated: '12345', + applicationStatus: 'rejected', + }, + events: [ + { + _id: '6189679675a82a0867ce55b9', + eventType: 'newDatasetVersionSubmitted', + logType: 'dataset', + timestamp: '12345', + user: '616993c3034a7d773064e208', + userDetails: { + firstName: 'John', + lastName: 'Smith', + role: 'custodian', + }, + version: '1.0.0', + versionId: '6189673475a82a0867ce54fa', + userTypes: ['admin', 'custodian'], + __v: 0, + }, + ], + }, +]; + +export const datasetVersionsMock = [ + { + _id: '6189673475a82a0867ce54fa', + timestamps: { + created: '12345', + submitted: '12345', + }, + datasetVersion: '1.0.0', + activeflag: 'rejected', + }, + { + _id: '618967b075a82a0867ce5650', + timestamps: { + created: '12345', + submitted: '12345', + }, + datasetVersion: '2.0.0', + activeflag: 'active', + }, +]; diff --git a/src/resources/activitylog/__tests__/activitylog.service.test.js b/src/resources/activitylog/__tests__/activitylog.service.test.js index 735eea81..394ef82d 100644 --- a/src/resources/activitylog/__tests__/activitylog.service.test.js +++ b/src/resources/activitylog/__tests__/activitylog.service.test.js @@ -1,8 +1,15 @@ -import { activityLogService } from '../dependency'; -import { partyTimeRanges } from '../__mocks__/activitylogs'; import { cloneDeep } from 'lodash'; +import sinon from 'sinon'; + +import { activityLogService, activityLogRepository } from '../dependency'; +import { partyTimeRanges } from '../__mocks__/activitylogs.dar'; +import { datasetActivityLogMocks, formattedJSONResponseMock, datasetVersionsMock } from '../__mocks__/activitylogs.dataset'; import constants from '../../utilities/constants.util'; +afterEach(function () { + sinon.restore(); +}); + describe('ActivityLogService', function () { describe('calculateTimeWithParty', function () { // Arrange @@ -36,4 +43,78 @@ describe('ActivityLogService', function () { } ); }); + describe('logActivity', () => { + it('Should invoke the logDatasetActivity function when eventype is "dataset', async () => { + let serviceStub = sinon.stub(activityLogService, 'logDatasetActivity'); + + await activityLogService.logActivity('mockEventType', { type: constants.activityLogTypes.DATASET }); + + expect(serviceStub.calledOnce).toBe(true); + }); + }); + + describe('logDatasetActivity', () => { + const datasetLoggingActivities = Object.keys(constants.activityLogEvents.dataset); + const context = { + updatedDataset: { datasetVersion: '1.0.0', _id: '618a72fd5ec8f54772b7a17a', applicationStatusDesc: 'Some admin comment!' }, + user: { + firstname: 'John', + lastname: 'Smith', + _id: '618a72fd5ec8f54772b7a17b', + teams: [ + { + publisher: { _id: 'fakeTeam', name: 'fakeTeam' }, + type: 'admin', + members: [{ memberid: '618a72fd5ec8f54772b7a17b', roles: ['admin_dataset'] }], + }, + ], + }, + differences: [{ 'summary/title': 'VERSION 2' }], + }; + + test.each(datasetLoggingActivities)('Each event type creates a valid log', async event => { + let createActivityStub = sinon.stub(activityLogRepository, 'createActivityLog'); + sinon.stub(Date, 'now').returns('123456'); + + let log = { + eventType: constants.activityLogEvents.dataset[event], + logType: constants.activityLogTypes.DATASET, + timestamp: '123456', + user: context.user._id, + userDetails: { firstName: context.user.firstname, lastName: context.user.lastname, role: 'admin' }, + version: context.updatedDataset.datasetVersion, + versionId: context.updatedDataset._id, + userTypes: [constants.userTypes.ADMIN, constants.userTypes.CUSTODIAN], + }; + + await activityLogService.logDatasetActivity(constants.activityLogEvents.dataset[event], context); + + expect(createActivityStub.calledOnce).toBe(true); + if (event === 'DATASET_VERSION_SUBMITTED' || event === 'DATASET_VERSION_ARCHIVED' || event === 'DATASET_VERSION_UNARCHIVED') { + expect(createActivityStub.calledWith(log)).toBe(true); + } + if (event === 'DATASET_VERSION_APPROVED' || event === 'DATASET_VERSION_REJECTED') { + log.adminComment = 'Some admin comment!'; + expect(createActivityStub.calledWith(log)).toBe(true); + } + if (event === 'DATASET_UPDATES_SUBMITTED') { + log.datasetUpdates = [{ 'summary/title': 'VERSION 2' }]; + expect(createActivityStub.calledWith(log)).toBe(true); + } + }); + }); + describe('searchLogs and formatLogs', () => { + it('Should returned correctly formatted logs', async () => { + const formatLogsStub = sinon.stub(activityLogRepository, 'searchLogs').returns(datasetActivityLogMocks); + const versionIds = [datasetVersionsMock[0]._id, datasetVersionsMock[1]._id]; + const type = 'dataset'; + const userType = 'admin'; + const versions = datasetVersionsMock; + + const formattedResponse = await activityLogService.searchLogs(versionIds, type, userType, versions); + + expect(formatLogsStub.calledOnce).toBe(true); + expect(formattedResponse).toStrictEqual(formattedJSONResponseMock); + }); + }); }); From 142fb8a062d2ff00a5b7ad4d8ff8fe3cc407c63a Mon Sep 17 00:00:00 2001 From: Paul McCafferty <> Date: Wed, 10 Nov 2021 09:47:37 +0000 Subject: [PATCH 088/116] Updates --- .../__mocks__/dataUseRegisters.js | 16 +-- .../dataUseRegister.controller.js | 39 ++----- .../dataUseRegister/dataUseRegister.model.js | 23 +++- .../dataUseRegister.service.js | 13 +++ .../dataUseRegister/dataUseRegister.util.js | 106 +++++++++++++----- src/resources/dataset/v1/dataset.route.js | 5 +- src/resources/paper/paper.repository.js | 5 + src/resources/paper/paper.service.js | 4 + src/resources/tool/v2/tool.repository.js | 5 + src/resources/tool/v2/tool.service.js | 4 + 10 files changed, 155 insertions(+), 65 deletions(-) diff --git a/src/resources/dataUseRegister/__mocks__/dataUseRegisters.js b/src/resources/dataUseRegister/__mocks__/dataUseRegisters.js index 72ad8be5..2c8f4b00 100644 --- a/src/resources/dataUseRegister/__mocks__/dataUseRegisters.js +++ b/src/resources/dataUseRegister/__mocks__/dataUseRegisters.js @@ -32,7 +32,7 @@ export const dataUseRegisterUploads = [ datasetLinkageDescription: 'data Processing Description', confidentialDataDescription: 'confidential Data Description', accessDate: '2021-09-26', - accessType: 'data Location', + accessType: 'accessType', privacyEnhancements: 'privacy Enhancements', researchOutputs: 'research Outputs', }, @@ -69,7 +69,7 @@ export const dataUseRegisterUploads = [ datasetLinkageDescription: 'data Processing Description', confidentialDataDescription: 'confidential Data Description', accessDate: '2021-09-26', - accessType: 'data Location', + accessType: 'accessType', privacyEnhancements: 'privacy Enhancements', researchOutputs: 'research Outputs', }, @@ -109,7 +109,7 @@ export const dataUseRegisterUploadsWithDuplicates = [ datasetLinkageDescription: 'data Processing Description', confidentialDataDescription: 'confidential Data Description', accessDate: '2021-09-26', - accessType: 'data Location', + accessType: 'accessType', privacyEnhancements: 'privacy Enhancements', researchOutputs: 'research Outputs', }, @@ -146,7 +146,7 @@ export const dataUseRegisterUploadsWithDuplicates = [ datasetLinkageDescription: 'data Processing Description', confidentialDataDescription: 'confidential Data Description', accessDate: '2021-09-26', - accessType: 'data Location', + accessType: 'accessType', privacyEnhancements: 'privacy Enhancements', researchOutputs: 'research Outputs', }, @@ -183,7 +183,7 @@ export const dataUseRegisterUploadsWithDuplicates = [ datasetLinkageDescription: 'data Processing Description', confidentialDataDescription: 'confidential Data Description', accessDate: '2021-09-26', - accessType: 'data Location', + accessType: 'accessType', privacyEnhancements: 'privacy Enhancements', researchOutputs: 'research Outputs', }, @@ -220,7 +220,7 @@ export const dataUseRegisterUploadsWithDuplicates = [ datasetLinkageDescription: 'data Processing Description', confidentialDataDescription: 'confidential Data Description', accessDate: '2021-09-26', - accessType: 'data Location', + accessType: 'accessType', privacyEnhancements: 'privacy Enhancements', researchOutputs: 'research Outputs', }, @@ -257,7 +257,7 @@ export const dataUseRegisterUploadsWithDuplicates = [ datasetLinkageDescription: 'data Processing Description', confidentialDataDescription: 'confidential Data Description', accessDate: '2021-09-26', - accessType: 'data Location', + accessType: 'accessType', privacyEnhancements: 'privacy Enhancements', researchOutputs: 'research Outputs', }, @@ -294,7 +294,7 @@ export const dataUseRegisterUploadsWithDuplicates = [ datasetLinkageDescription: 'data Processing Description', confidentialDataDescription: 'confidential Data Description', accessDate: '2021-09-26', - accessType: 'data Location', + accessType: 'accessType', privacyEnhancements: 'privacy Enhancements', researchOutputs: 'research Outputs', }, diff --git a/src/resources/dataUseRegister/dataUseRegister.controller.js b/src/resources/dataUseRegister/dataUseRegister.controller.js index f4aa930a..ce12e35e 100644 --- a/src/resources/dataUseRegister/dataUseRegister.controller.js +++ b/src/resources/dataUseRegister/dataUseRegister.controller.js @@ -29,9 +29,19 @@ export default class DataUseRegisterController extends Controller { message: 'You must provide a dataUseRegister identifier', }); } + // Find the dataUseRegister - const options = { lean: true, populate: { path: 'gatewayApplicants', select: 'id firstname lastname' } }; + const options = { + lean: true, + populate: [ + { path: 'gatewayApplicants', select: 'id firstname lastname' }, + { path: 'gatewayDatasetsInfo', select: 'name pid' }, + { path: 'gatewayOutputsToolsInfo', select: 'name id' }, + { path: 'gatewayOutputsPapersInfo', select: 'name id' }, + ], + }; const dataUseRegister = await this.dataUseRegisterService.getDataUseRegister(id, req.query, options); + // Reverse look up var query = Data.aggregate([ { $match: { id: parseInt(req.params.id) } }, @@ -50,7 +60,6 @@ export default class DataUseRegisterController extends Controller { pid: '$pid', }, pipeline: [ - //{ $match: { $expr: { $in: ['$gatewayDatasets', '$$pid'] } } }, { $match: { $expr: { @@ -59,36 +68,10 @@ export default class DataUseRegisterController extends Controller { }, }, { $project: { pid: 1, name: 1 } }, - - /* { - $match: { - $expr: { - $and: [ - { - $eq: ['$relatedObjects.pid', '$$pid'], - }, - { - $eq: ['$activeflag', 'active'], - }, - ], - }, - }, - }, - { $group: { _id: null, count: { $sum: 1 } } }, - */ ], as: 'gatewayDatasets2', }, }, - - /* { - $lookup: { - from: 'tools', - localField: 'gatewayDatasets', - foreignField: 'pid', - as: 'gatewayDatasets', - }, - }, */ ]); query.exec((err, data) => { if (data.length > 0) { diff --git a/src/resources/dataUseRegister/dataUseRegister.model.js b/src/resources/dataUseRegister/dataUseRegister.model.js index c9740942..35086ff8 100644 --- a/src/resources/dataUseRegister/dataUseRegister.model.js +++ b/src/resources/dataUseRegister/dataUseRegister.model.js @@ -66,7 +66,9 @@ const dataUseRegisterSchema = new Schema( accessDate: Date, //Release/Access Date accessType: String, //TRE Or Any Other Specified Location privacyEnhancements: String, //How Has Data Been Processed To Enhance Privacy - researchOutputs: [{ type: String }], //Link To Research Outputs + gatewayOutputsTools: [{ type: String }], //Link To Gateway Tool Research Outputs + gatewayOutputsPapers: [{ type: String }], //Link To Gateway Paper Research Outputs + nonGatewayOutputs: [{ type: String }], //Link To NonGateway Research Outputs rejectionReason: String, //Reason For Rejecting A Data Use Register }, { @@ -87,4 +89,23 @@ dataUseRegisterSchema.virtual('publisherInfo', { justOne: true, }); +dataUseRegisterSchema.virtual('gatewayDatasetsInfo', { + ref: 'Data', + foreignField: 'pid', + localField: 'gatewayDatasets', + options: { sort: { createdAt: -1 }, limit: 1 }, +}); + +dataUseRegisterSchema.virtual('gatewayOutputsToolsInfo', { + ref: 'Data', + foreignField: 'id', + localField: 'gatewayOutputsTools', +}); + +dataUseRegisterSchema.virtual('gatewayOutputsPapersInfo', { + ref: 'Data', + foreignField: 'id', + localField: 'gatewayOutputsPapers', +}); + export const DataUseRegister = model('DataUseRegister', dataUseRegisterSchema); diff --git a/src/resources/dataUseRegister/dataUseRegister.service.js b/src/resources/dataUseRegister/dataUseRegister.service.js index 44060ecc..f28ba646 100644 --- a/src/resources/dataUseRegister/dataUseRegister.service.js +++ b/src/resources/dataUseRegister/dataUseRegister.service.js @@ -148,6 +148,16 @@ export default class DataUseRegisterService { }) ); + const { gatewayOutputsTools, gatewayOutputsPapers, nonGatewayOutputs } = await dataUseRegisterUtil.getLinkedOutputs( + obj.researchOutputs && + obj.researchOutputs + .toString() + .split(',') + .map(el => { + if (!isEmpty(el)) return el.trim(); + }) + ); + const { projectIdText, projectTitle, organisationName } = obj; const datasetTitles = [...linkedDatasets.map(dataset => dataset.name), ...namedDatasets]; @@ -170,6 +180,9 @@ export default class DataUseRegisterService { namedDatasets, gatewayApplicants, nonGatewayApplicants, + gatewayOutputsTools, + gatewayOutputsPapers, + nonGatewayOutputs, isDuplicated: exists, }); } diff --git a/src/resources/dataUseRegister/dataUseRegister.util.js b/src/resources/dataUseRegister/dataUseRegister.util.js index a68711bc..b6254540 100644 --- a/src/resources/dataUseRegister/dataUseRegister.util.js +++ b/src/resources/dataUseRegister/dataUseRegister.util.js @@ -2,6 +2,8 @@ import moment from 'moment'; import { isEmpty } from 'lodash'; import DataUseRegister from './dataUseRegister.entity'; import { getUsersByIds } from '../user/user.repository'; +import { toolService } from '../tool/v2/dependency'; +import { paperService } from '../paper/dependency'; import { datasetService } from '../dataset/dependency'; /** @@ -41,8 +43,22 @@ const buildDataUseRegisters = async (creatorUser, teamId, dataUses = []) => { }) ); + const { gatewayOutputsTools, gatewayOutputsPapers, nonGatewayOutputs } = await getLinkedOutputs( + obj.researchOutputs && + obj.researchOutputs + .toString() + .split(',') + .map(el => { + if (!isEmpty(el)) return el.trim(); + }) + ); + // Create related objects - const relatedObjects = buildRelatedDatasets(creatorUser, linkedDatasets); + let relatedObjects = [ + ...buildRelatedObjects(creatorUser, 'dataset', linkedDatasets), + ...buildRelatedObjects(creatorUser, 'tool', gatewayOutputsTools), + ...buildRelatedObjects(creatorUser, 'paper', gatewayOutputsPapers), + ]; // Handle comma separated fields const fundersAndSponsors = @@ -53,14 +69,6 @@ const buildDataUseRegisters = async (creatorUser, teamId, dataUses = []) => { .map(el => { if (!isEmpty(el)) return el.trim(); }); - const researchOutputs = - obj.researchOutputs && - obj.researchOutputs - .toString() - .split(',') - .map(el => { - if (!isEmpty(el)) return el.trim(); - }); const otherApprovalCommittees = obj.otherApprovalCommittees && obj.otherApprovalCommittees @@ -108,11 +116,12 @@ const buildDataUseRegisters = async (creatorUser, teamId, dataUses = []) => { ...(!isEmpty(datasetTitles) && { datasetTitles }), ...(!isEmpty(linkedDatasets) && { gatewayDatasets: linkedDatasets.map(dataset => dataset.pid) }), ...(!isEmpty(namedDatasets) && { nonGatewayDatasets: namedDatasets }), - ...(!isEmpty(gatewayApplicants) && { gatewayApplicants: gatewayApplicants.map(gatewayApplicant => gatewayApplicant._id) }), ...(!isEmpty(nonGatewayApplicants) && { nonGatewayApplicants }), ...(!isEmpty(fundersAndSponsors) && { fundersAndSponsors }), - ...(!isEmpty(researchOutputs) && { researchOutputs }), + ...(!isEmpty(gatewayOutputsTools) && { gatewayOutputsTools: gatewayOutputsTools.map(tool => tool.id) }), + ...(!isEmpty(gatewayOutputsPapers) && { gatewayOutputsPapers: gatewayOutputsPapers.map(paper => paper.id) }), + ...(!isEmpty(nonGatewayOutputs) && { nonGatewayOutputs: nonGatewayOutputs }), ...(!isEmpty(otherApprovalCommittees) && { otherApprovalCommittees }), ...(!isEmpty(relatedObjects) && { relatedObjects }), activeflag: 'inReview', @@ -159,7 +168,7 @@ const getLinkedDatasets = async (datasetNames = []) => { const linkedDatasets = isEmpty(unverifiedDatasetPids) ? [] : (await datasetService.getDatasetsByPids(unverifiedDatasetPids)).map(dataset => { - return { datasetid: dataset.datasetid, name: dataset.name, pid: dataset.pid }; + return { id: dataset.datasetid, name: dataset.name, pid: dataset.pid }; }); return { linkedDatasets, namedDatasets }; @@ -198,28 +207,74 @@ const getLinkedApplicants = async (applicantNames = []) => { }; /** - * Build Related Datasets + * Get Linked Outputs + * + * @desc Accepts a comma separated string containing tools or papers which can be in the form of text based names or URLs belonging to the Gateway which resolve to a users profile page, or a mix of both. + * The function separates URLs and uses regex to locate a suspected user ID to use in a search against the Gateway database. If a match is found, the entry is considered a Gateway tool or paper. + * Entries which cannot be matched are returned as non Gateway tools or papers. Failed attempts at adding URLs which do not resolve are excluded. + * @param {String} outputs A comma separated string representation of the tools or papers names to attempt to find and link to existing Gateway tools or papers + * @returns {Object} An object containing Gateway tools or papers and non Gateway tools or papers in separate arrays + */ +const getLinkedOutputs = async (outputs = []) => { + const unverifiedOutputsToolIds = [], + unverifiedOutputsPaperIds = [], + nonGatewayOutputs = []; + const validLinkRegexpTool = new RegExp(`^${process.env.homeURL}\/tool\/(\\d+)\/?$`, 'i'); + const validLinkRegexpPaper = new RegExp(`^${process.env.homeURL}\/paper\/(\\d+)\/?$`, 'i'); + + for (const output of outputs) { + const [, toolId] = validLinkRegexpTool.exec(output) || []; + if (toolId) { + unverifiedOutputsToolIds.push(toolId); + } else { + const [, paperId] = validLinkRegexpPaper.exec(output) || []; + if (paperId) { + unverifiedOutputsPaperIds.push(paperId); + } else { + nonGatewayOutputs.push(output); + } + } + } + + const gatewayOutputsTools = isEmpty(unverifiedOutputsToolIds) + ? [] + : (await toolService.getToolsByIds(unverifiedOutputsToolIds)).map(tool => { + return { id: tool.id, name: tool.name }; + }); + + const gatewayOutputsPapers = isEmpty(unverifiedOutputsPaperIds) + ? [] + : (await paperService.getPapersByIds(unverifiedOutputsPaperIds)).map(paper => { + return { id: paper.id, name: paper.name }; + }); + + return { gatewayOutputsTools, gatewayOutputsPapers, nonGatewayOutputs }; +}; + +/** + * Build Related Objects for datause * - * @desc Accepts an array of datasets and outputs an array of related objects which can be assigned to an entity to show the relationship to the datasets. - * Related objects contain the 'objectId' (dataset version identifier), 'pid', 'objectType' (dataset), 'updated' date and 'user' that created the linkage. + * @desc Accepts an array of objects to relate and outputs an array of related objects which can be assigned to an entity to show the relationship to the object. + * Related objects contain the 'objectId' (object identifier), 'pid', 'objectType' (dataset), 'updated' date and 'user' that created the linkage. * @param {Object} creatorUser A user object to allow the assignment of their name to the creator of the linkage - * @param {Array} datasets An array of dataset objects containing the necessary properties to assemble a related object record reference - * @returns {Array} An array containing the assembled related objects relative to the datasets provided + * @param {String} type The type of object that is being passed in + * @param {Array} objects An array of objects containing the necessary properties to assemble a related object record reference + */ -const buildRelatedDatasets = (creatorUser, datasets = [], manualUpload = true) => { +const buildRelatedObjects = (creatorUser, type, objects = [], manualUpload = true) => { const { firstname, lastname } = creatorUser; - return datasets.map(dataset => { - const { datasetid: objectId, pid } = dataset; + return objects.map(object => { + const { id: objectId, pid } = object; return { objectId, pid, - objectType: 'dataset', + objectType: type, user: `${firstname} ${lastname}`, - updated: Date.now(), + updated: moment().format('DD MMM YYYY'), isLocked: true, reason: manualUpload - ? 'This dataset was added automatically during the manual upload of this data use register' - : 'This dataset was added automatically from an approved data access request', + ? `This ${type} was added automatically during the manual upload of this data use register` + : `This ${type} was added automatically from an approved data access request`, }; }); }; @@ -270,7 +325,8 @@ export default { buildDataUseRegisters, getLinkedDatasets, getLinkedApplicants, - buildRelatedDatasets, + getLinkedOutputs, + buildRelatedObjects, extractFormApplicants, extractFundersAndSponsors, }; diff --git a/src/resources/dataset/v1/dataset.route.js b/src/resources/dataset/v1/dataset.route.js index e5ea7db3..16fd6086 100644 --- a/src/resources/dataset/v1/dataset.route.js +++ b/src/resources/dataset/v1/dataset.route.js @@ -18,9 +18,8 @@ const datasetLimiter = rateLimit({ router.post('/', async (req, res) => { try { - filtersService.optimiseFilters('dataUseRegister'); // Check to see if header is in json format - /* let parsedBody = {}; + let parsedBody = {}; if (req.header('content-type') === 'application/json') { parsedBody = req.body; } else { @@ -40,7 +39,7 @@ router.post('/', async (req, res) => { importCatalogues(catalogues, override, limit).then(() => { filtersService.optimiseFilters('dataset'); saveUptime(); - }); */ + }); // Return response indicating job has started (do not await async import) return res.status(200).json({ success: true, message: 'Caching started' }); } catch (err) { diff --git a/src/resources/paper/paper.repository.js b/src/resources/paper/paper.repository.js index 346529b1..8a108366 100644 --- a/src/resources/paper/paper.repository.js +++ b/src/resources/paper/paper.repository.js @@ -17,4 +17,9 @@ export default class PaperRepository extends Repository { const options = { lean: true }; return this.find(query, options); } + + async getPapersByIds(paperIds) { + const options = { lean: true }; + return this.find({ id: { $in: paperIds } }, options); + } } diff --git a/src/resources/paper/paper.service.js b/src/resources/paper/paper.service.js index 33d2cce5..6193adaf 100644 --- a/src/resources/paper/paper.service.js +++ b/src/resources/paper/paper.service.js @@ -10,4 +10,8 @@ export default class PaperService { getPapers(query = {}) { return this.paperRepository.getPapers(query); } + + getPapersByIds(paperIds) { + return this.paperRepository.getPapersByIds(paperIds); + } } diff --git a/src/resources/tool/v2/tool.repository.js b/src/resources/tool/v2/tool.repository.js index c224e376..7304a131 100644 --- a/src/resources/tool/v2/tool.repository.js +++ b/src/resources/tool/v2/tool.repository.js @@ -17,4 +17,9 @@ export default class ToolRepository extends Repository { const options = { lean: true }; return this.find(query, options); } + + async getToolsByIds(toolIds) { + const options = { lean: true }; + return this.find({ id: { $in: toolIds } }, options); + } } diff --git a/src/resources/tool/v2/tool.service.js b/src/resources/tool/v2/tool.service.js index 86e1477c..76e19225 100644 --- a/src/resources/tool/v2/tool.service.js +++ b/src/resources/tool/v2/tool.service.js @@ -10,4 +10,8 @@ export default class ToolService { getTools(query = {}) { return this.toolRepository.getTools(query); } + + getToolsByIds(toolIds) { + return this.toolRepository.getToolsByIds(toolIds); + } } From b8b150eabc5cfbdf721ebe78026d1852a42c1335 Mon Sep 17 00:00:00 2001 From: Paul McCafferty <> Date: Wed, 10 Nov 2021 12:34:48 +0000 Subject: [PATCH 089/116] LGTM Fixes --- src/resources/dataset/dataset.service.js | 1 - 1 file changed, 1 deletion(-) diff --git a/src/resources/dataset/dataset.service.js b/src/resources/dataset/dataset.service.js index c0968217..d5063e4b 100644 --- a/src/resources/dataset/dataset.service.js +++ b/src/resources/dataset/dataset.service.js @@ -116,7 +116,6 @@ export default class DatasetService { } getDatasetsByName(name) { - let query = {}; return this.datasetRepository.getDataset({ name, fields: 'pid' }, { lean: true }); } } From 2256907b2d768e2353164a6b9785143ed8a446cc Mon Sep 17 00:00:00 2001 From: Callum Reekie Date: Thu, 11 Nov 2021 08:42:49 +0000 Subject: [PATCH 090/116] CR - datasetonboardiing util tests --- .../__tests__/activitylog.middleware.test.js | 14 +- src/middlewares/activitylog.middleware.js | 43 +++--- src/middlewares/index.js | 20 ++- .../activitylog/activitylog.route.js | 21 ++- .../dataset/utils/__mocks__/datasetobjects.js | 138 ++++++++++++++++++ .../__tests__/datasetonboarding.util.test.js | 66 +++++++++ .../dataset/utils/datasetonboarding.util.js | 16 +- 7 files changed, 271 insertions(+), 47 deletions(-) create mode 100644 src/resources/dataset/utils/__mocks__/datasetobjects.js create mode 100644 src/resources/dataset/utils/__tests__/datasetonboarding.util.test.js diff --git a/src/middlewares/__tests__/activitylog.middleware.test.js b/src/middlewares/__tests__/activitylog.middleware.test.js index f3696e4c..71d9cd3d 100644 --- a/src/middlewares/__tests__/activitylog.middleware.test.js +++ b/src/middlewares/__tests__/activitylog.middleware.test.js @@ -1,6 +1,6 @@ import sinon from 'sinon'; -import activitylogMiddleware from '../activitylog.middleware'; +import { validateViewRequest, authoriseView } from '../activitylog.middleware'; import { datasetService } from '../../resources/dataset/dependency'; import { UserModel } from '../../resources/user/user.model'; @@ -33,7 +33,7 @@ describe('Testing the ActivityLog middleware', () => { req.body = { versionIds: [], type: 'dataset' }; const nextFunction = jest.fn(); - activitylogMiddleware.validateViewRequest(req, res, nextFunction); + validateViewRequest(req, res, nextFunction); expect(res.status).toHaveBeenCalledWith(400); expect(res.json).toHaveBeenCalledWith(expectedResponse); @@ -46,7 +46,7 @@ describe('Testing the ActivityLog middleware', () => { req.body = { versionIds: [123, 456], type: 'notARealType' }; const nextFunction = jest.fn(); - activitylogMiddleware.validateViewRequest(req, res, nextFunction); + validateViewRequest(req, res, nextFunction); expect(res.status).toHaveBeenCalledWith(400); expect(res.json).toHaveBeenCalledWith(expectedResponse); @@ -59,7 +59,7 @@ describe('Testing the ActivityLog middleware', () => { req.body = { versionIds: [123, 456], type: 'dataset' }; const nextFunction = jest.fn(); - activitylogMiddleware.validateViewRequest(req, res, nextFunction); + validateViewRequest(req, res, nextFunction); expect(nextFunction.mock.calls.length).toBe(1); }); @@ -99,7 +99,7 @@ describe('Testing the ActivityLog middleware', () => { }, ]); - await activitylogMiddleware.authoriseView(req, res, nextFunction); + await authoriseView(req, res, nextFunction); expect(versionsStub.calledOnce).toBe(true); expect(res.status).toHaveBeenCalledWith(401); @@ -147,7 +147,7 @@ describe('Testing the ActivityLog middleware', () => { }, ]); - await activitylogMiddleware.authoriseView(req, res, nextFunction); + await authoriseView(req, res, nextFunction); expect(versionsStub.calledOnce).toBe(true); expect(nextFunction.mock.calls.length).toBe(1); @@ -161,7 +161,7 @@ describe('Testing the ActivityLog middleware', () => { let versionsStub = sinon.stub(datasetService, 'getDatasets').throws(); - let badCall = await activitylogMiddleware.authoriseView(req, res, nextFunction); + let badCall = await authoriseView(req, res, nextFunction); try { badCall(); diff --git a/src/middlewares/activitylog.middleware.js b/src/middlewares/activitylog.middleware.js index 7af0ae6b..2e26a0b6 100644 --- a/src/middlewares/activitylog.middleware.js +++ b/src/middlewares/activitylog.middleware.js @@ -24,19 +24,19 @@ const authoriseView = async (req, res, next) => { const { versionIds = [] } = req.body; let authorised, userType, accessRecords; - if (req.body.type === constants.activityLogTypes.DATA_ACCESS_REQUEST) { - ({ authorised, userType, accessRecords } = await dataRequestService.checkUserAuthForVersions(versionIds, requestingUser)); - if (!authorised) { - return res.status(401).json({ - success: false, - message: 'You are not authorised to perform this action', - }); - } + try { + if (req.body.type === constants.activityLogTypes.DATA_ACCESS_REQUEST) { + ({ authorised, userType, accessRecords } = await dataRequestService.checkUserAuthForVersions(versionIds, requestingUser)); + if (!authorised) { + return res.status(401).json({ + success: false, + message: 'You are not authorised to perform this action', + }); + } - req.body.userType = userType; - req.body.versions = accessRecords; - } else if (req.body.type === constants.activityLogTypes.DATASET) { - try { + req.body.userType = userType; + req.body.versions = accessRecords; + } else if (req.body.type === constants.activityLogTypes.DATASET) { const datasetVersions = await datasetService.getDatasets({ _id: { $in: versionIds } }, { lean: true }); let permissionsArray = []; await datasetVersions.forEach(async version => { @@ -58,12 +58,12 @@ const authoriseView = async (req, res, next) => { ? constants.userTypes.ADMIN : constants.userTypes.CUSTODIAN; req.body.versions = datasetVersions; - } catch (error) { - return res.status(401).json({ - success: false, - message: 'Error authenticating the user against submitted dataset version IDs. Please check the submitted dataset versionIds', - }); } + } catch (error) { + return res.status(401).json({ + success: false, + message: 'Error authenticating the user against submitted versionIds. Please check the submitted dataset versionIds', + }); } next(); @@ -164,11 +164,4 @@ const authoriseDelete = async (req, res, next) => { next(); }; -export default { - validateViewRequest, - authoriseView, - authoriseCreate, - validateCreateRequest, - validateDeleteRequest, - authoriseDelete, -}; +export { validateViewRequest, authoriseView, authoriseCreate, validateCreateRequest, validateDeleteRequest, authoriseDelete }; diff --git a/src/middlewares/index.js b/src/middlewares/index.js index 9158c22f..f8d6fc4b 100644 --- a/src/middlewares/index.js +++ b/src/middlewares/index.js @@ -1,5 +1,19 @@ -import { checkIDMiddleware } from './checkIDMiddleware' +import { checkIDMiddleware } from './checkIDMiddleware'; +import { + validateViewRequest, + authoriseView, + authoriseCreate, + validateCreateRequest, + validateDeleteRequest, + authoriseDelete, +} from './activitylog.middleware'; export { - checkIDMiddleware -} \ No newline at end of file + checkIDMiddleware, + validateViewRequest, + authoriseView, + authoriseCreate, + validateCreateRequest, + validateDeleteRequest, + authoriseDelete, +}; diff --git a/src/resources/activitylog/activitylog.route.js b/src/resources/activitylog/activitylog.route.js index b70a6bbe..00f048cf 100644 --- a/src/resources/activitylog/activitylog.route.js +++ b/src/resources/activitylog/activitylog.route.js @@ -1,7 +1,14 @@ import express from 'express'; import passport from 'passport'; -import ActivityLogMiddleware from '../../middlewares/activitylog.middleware'; +import { + validateViewRequest, + authoriseView, + authoriseCreate, + validateCreateRequest, + validateDeleteRequest, + authoriseDelete, +} from '../../middlewares/index'; import ActivityLogController from './activitylog.controller'; import { activityLogService } from './dependency'; import { logger } from '../utilities/logger'; @@ -16,8 +23,8 @@ const logCategory = 'Activity Log'; router.post( '/', passport.authenticate('jwt'), - ActivityLogMiddleware.validateViewRequest, - ActivityLogMiddleware.authoriseView, + validateViewRequest, + authoriseView, logger.logRequestMiddleware({ logCategory, action: 'Viewed activity logs' }), (req, res) => activityLogController.searchLogs(req, res) ); @@ -28,8 +35,8 @@ router.post( router.post( '/:type', passport.authenticate('jwt'), - ActivityLogMiddleware.validateCreateRequest, - ActivityLogMiddleware.authoriseCreate, + validateCreateRequest, + authoriseCreate, logger.logRequestMiddleware({ logCategory, action: 'Created manual event' }), (req, res) => activityLogController.createLog(req, res) ); @@ -40,8 +47,8 @@ router.post( router.delete( '/:type/:id', passport.authenticate('jwt'), - ActivityLogMiddleware.validateDeleteRequest, - ActivityLogMiddleware.authoriseDelete, + validateDeleteRequest, + authoriseDelete, logger.logRequestMiddleware({ logCategory, action: 'Deleted manual event' }), (req, res) => activityLogController.deleteLog(req, res) ); diff --git a/src/resources/dataset/utils/__mocks__/datasetobjects.js b/src/resources/dataset/utils/__mocks__/datasetobjects.js new file mode 100644 index 00000000..59a2c924 --- /dev/null +++ b/src/resources/dataset/utils/__mocks__/datasetobjects.js @@ -0,0 +1,138 @@ +import { ObjectID } from 'mongodb'; + +export const datasetQuestionAnswersMocks = { + 'properties/summary/abstract': 'test', + 'properties/summary/contactPoint': 'test@test.com', + 'properties/summary/keywords': ['testKeywordBowel', 'testKeywordCancer'], + 'properties/provenance/temporal/accrualPeriodicity': 'DAILY', + 'properties/provenance/temporal/startDate': '25/12/2021', + 'properties/provenance/temporal/timeLag': 'NOT APPLICABLE', + 'properties/accessibility/access/accessRights': ['http://www.google.com'], + 'properties/accessibility/access/jurisdiction': ['GB-GB'], + 'properties/accessibility/access/dataController': 'testtesttesttesttesttest', + 'properties/accessibility/formatAndStandards/vocabularyEncodingScheme': ['LOCAL'], + 'properties/accessibility/formatAndStandards/conformsTo': ['NHS SCOTLAND DATA DICTIONARY'], + 'properties/accessibility/formatAndStandards/language': ['ab'], + 'properties/accessibility/formatAndStandards/format': ['testtesttest'], + 'properties/observation/observedNode': 'PERSONS', + 'properties/observation/measuredValue': '25', + 'properties/observation/disambiguatingDescription': 'testtesttest', + 'properties/observation/observationDate': '03/09/2021', + 'properties/observation/measuredProperty': 'Count', + 'properties/summary/title': 'Test title', + 'properties/provenance/origin/purpose': ['STUDY'], + 'properties/coverage/physicalSampleAvailability': ['NOT AVAILABLE'], + 'properties/enrichmentAndLinkage/qualifiedRelation': ['https://google.com', 'https://google.com'], + 'properties/observation/observedNode_1xguo': 'EVENTS', + 'properties/observation/measuredValue_1xguo': '100', + 'properties/observation/disambiguatingDescription_1xguo': 'testtesttest', + 'properties/observation/observationDate_1xguo': '03/11/2021', + 'properties/observation/measuredProperty_1xguo': 'Count', +}; + +export const datasetv2ObjectMock = { + identifier: '', + version: '2.0.0', + revisions: [], + summary: { + title: 'Test title', + abstract: 'test', + publisher: { + identifier: '5f3f98068af2ef61552e1d75', + name: 'SAIL', + logo: '', + description: '', + contactPoint: [], + memberOf: 'ALLIANCE', + accessRights: [], + deliveryLeadTime: '', + accessService: '', + accessRequestCost: '', + dataUseLimitation: [], + dataUseRequirements: [], + }, + contactPoint: 'test@test.com', + keywords: ['testKeywordBowel', 'testKeywordCancer'], + alternateIdentifiers: [], + doiName: '', + }, + documentation: { description: '', associatedMedia: [], isPartOf: [] }, + coverage: { + spatial: [], + typicalAgeRange: '', + physicalSampleAvailability: ['NOT AVAILABLE'], + followup: '', + pathway: '', + }, + provenance: { + origin: { purpose: ['STUDY'], source: [], collectionSituation: [] }, + temporal: { + accrualPeriodicity: 'DAILY', + distributionReleaseDate: '', + startDate: '25/12/2021', + endDate: '', + timeLag: 'NOT APPLICABLE', + }, + }, + accessibility: { + usage: { + dataUseLimitation: [], + dataUseRequirements: [], + resourceCreator: '', + investigations: [], + isReferencedBy: [], + }, + access: { + accessRights: ['http://www.google.com'], + accessService: '', + accessRequestCost: '', + deliveryLeadTime: '', + jurisdiction: ['GB-GB'], + dataProcessor: '', + dataController: 'testtesttesttesttesttest', + }, + formatAndStandards: { + vocabularyEncodingScheme: ['LOCAL'], + conformsTo: ['NHS SCOTLAND DATA DICTIONARY'], + language: ['ab'], + format: ['testtesttest'], + }, + }, + enrichmentAndLinkage: { + qualifiedRelation: ['https://google.com', 'https://google.com'], + derivation: [], + tools: [], + }, + observations: [ + { + observedNode: 'PERSONS', + measuredValue: '25', + disambiguatingDescription: 'testtesttest', + observationDate: '03/09/2021', + measuredProperty: 'Count', + }, + { + observedNode: 'EVENTS', + measuredValue: '100', + disambiguatingDescription: 'testtesttest', + observationDate: '03/11/2021', + measuredProperty: 'Count', + }, + ], +}; + +export const publisherDetailsMock = [ + { + _id: ObjectID('5f3f98068af2ef61552e1d75'), + name: 'ALLIANCE > SAIL', + active: true, + imageURL: '', + dataRequestModalContent: {}, + allowsMessaging: true, + workflowEnabled: true, + allowAccessRequestManagement: true, + publisherDetails: { name: 'SAIL', memberOf: 'ALLIANCE' }, + uses5Safes: true, + mdcFolderId: 'c4f50de0-2188-426b-a6cd-6b11a8d6c3cb', + }, +]; diff --git a/src/resources/dataset/utils/__tests__/datasetonboarding.util.test.js b/src/resources/dataset/utils/__tests__/datasetonboarding.util.test.js new file mode 100644 index 00000000..84b04da9 --- /dev/null +++ b/src/resources/dataset/utils/__tests__/datasetonboarding.util.test.js @@ -0,0 +1,66 @@ +import dbHandler from '../../../../config/in-memory-db'; +import datasetonboardingUtil from '../datasetonboarding.util'; +import { datasetQuestionAnswersMocks, datasetv2ObjectMock, publisherDetailsMock } from '../__mocks__/datasetobjects'; + +beforeAll(async () => { + await dbHandler.connect(); + await dbHandler.loadData({ publishers: publisherDetailsMock }); +}); + +afterAll(async () => await dbHandler.closeDatabase()); + +describe('Dataset onboarding utility', () => { + describe('buildv2Object', () => { + it('Should return a correctly formatted V2 object when supplied with questionAnswers', async () => { + let datasetv2Object = await datasetonboardingUtil.buildv2Object({ + questionAnswers: datasetQuestionAnswersMocks, + datasetVersion: '2.0.0', + datasetv2: { + summary: { + publisher: { + identifier: '5f3f98068af2ef61552e1d75', + }, + }, + }, + }); + + delete datasetv2Object.issued; + delete datasetv2Object.modified; + + expect(datasetv2Object).toStrictEqual(datasetv2ObjectMock); + }); + }); + + describe('datasetv2ObjectComparison', () => { + it('Should return a correctly formatted diff array', async () => { + let datasetv2DiffObject = await datasetonboardingUtil.datasetv2ObjectComparison( + { + summary: { title: 'Title 2' }, + provenance: { temporal: { updated: 'ONCE WEEKLY', updatedDates: ['1/1/1'] } }, + observations: [ + { observedNode: 'Obs2', observationDate: '3/3/3', measuredValue: '', disambiguatingDescription: '', measuredProperty: '' }, + { observedNode: 'Obs3', observationDate: '4/4/4', measuredValue: '', disambiguatingDescription: '', measuredProperty: '' }, + ], + }, + { + summary: { title: 'Title 1' }, + provenance: { temporal: { updated: 'TWICE WEEKLY', updatedDates: ['1/1/1', '2/2/2'] } }, + observations: [ + { observedNode: 'Obs1', observationDate: '3/3/3', measuredValue: '', disambiguatingDescription: '', measuredProperty: '' }, + ], + } + ); + + const diffArray = [ + { 'summary/title': { updatedAnswer: 'Title 2', previousAnswer: 'Title 1' } }, + { 'provenance/temporal/updated': { updatedAnswer: 'ONCE WEEKLY', previousAnswer: 'TWICE WEEKLY' } }, + { 'provenance/temporal/updatedDates': { updatedAnswer: '1/1/1', previousAnswer: '1/1/1, 2/2/2' } }, + { 'observations/1/observedNode': { updatedAnswer: 'Obs2', previousAnswer: 'Obs1' } }, + { 'observations/2/observedNode': { updatedAnswer: 'Obs3', previousAnswer: '' } }, + { 'observations/2/observationDate': { updatedAnswer: '4/4/4', previousAnswer: '' } }, + ]; + + expect(datasetv2DiffObject).toStrictEqual(diffArray); + }); + }); +}); diff --git a/src/resources/dataset/utils/datasetonboarding.util.js b/src/resources/dataset/utils/datasetonboarding.util.js index 28e75977..c3b4be5c 100644 --- a/src/resources/dataset/utils/datasetonboarding.util.js +++ b/src/resources/dataset/utils/datasetonboarding.util.js @@ -11,7 +11,7 @@ import randomstring from 'randomstring'; import Ajv from 'ajv'; import addFormats from 'ajv-formats'; var fs = require('fs'); -import { flatten, unflatten } from 'flat'; +import { flatten } from 'flat'; /** * Checks to see if the user has the correct permissions to access the dataset @@ -1204,6 +1204,7 @@ const datasetv2ObjectComparison = (updatedJSON, previousJSON) => { updatedJSON = flatten(updatedJSON, { safe: true, delimiter: '/' }); previousJSON = flatten(previousJSON, { safe: true, delimiter: '/' }); + // Remove fields which change automatically between datasets const unusedKeys = ['identifier', 'version', 'issued', 'modified']; unusedKeys.forEach(key => { delete updatedJSON[key]; @@ -1232,13 +1233,14 @@ const datasetv2ObjectComparison = (updatedJSON, previousJSON) => { } }); + // Compute diff of 'observations' separately, which can be an array of objects const observationKeys = ['observedNode', 'measuredValue', 'disambiguatingDescription', 'observationDate', 'measuredProperty']; const maxObservationLength = Math.max(previousJSON['observations'].length, updatedJSON['observations'].length); let resultObservations = {}; for (let i = 0; i < maxObservationLength; i++) { - let newKeyName = 'observations/' + (i + 1).toString() + '/'; - resultObservations[newKeyName] = {}; + let observationNumberKey = 'observations/' + (i + 1).toString() + '/'; + resultObservations[observationNumberKey] = {}; if (updatedJSON['observations'][i] === undefined) { updatedJSON['observations'][i] = {}; observationKeys.forEach(key => { @@ -1254,21 +1256,25 @@ const datasetv2ObjectComparison = (updatedJSON, previousJSON) => { } observationKeys.forEach(key => { + if (updatedJSON['observations'][i][key] === undefined) updatedJSON['observations'][i][key] = ''; + if (previousJSON['observations'][i][key] === undefined) previousJSON['observations'][i][key] = ''; if (!_.isEqual(updatedJSON['observations'][i][key], previousJSON['observations'][i][key])) { - resultObservations[newKeyName + key] = { + resultObservations[observationNumberKey + key] = { previousAnswer: previousJSON['observations'][i][key], updatedAnswer: updatedJSON['observations'][i][key], }; } }); - if (_.isEmpty(resultObservations[newKeyName])) delete resultObservations[newKeyName]; + if (_.isEmpty(resultObservations[observationNumberKey])) delete resultObservations[observationNumberKey]; } + // Append observation diff to previous result array Object.keys(resultObservations).forEach(key => { let arrayObject = {}; arrayObject[key] = resultObservations[key]; result.push(arrayObject); }); + return result; }; From bbff31d09fad5d5e5e013bf12d3b5b8248d83cbb Mon Sep 17 00:00:00 2001 From: Callum Reekie Date: Thu, 11 Nov 2021 10:36:29 +0000 Subject: [PATCH 091/116] CR - fixed bad constant --- src/middlewares/activitylog.middleware.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/middlewares/activitylog.middleware.js b/src/middlewares/activitylog.middleware.js index 2e26a0b6..60141579 100644 --- a/src/middlewares/activitylog.middleware.js +++ b/src/middlewares/activitylog.middleware.js @@ -149,7 +149,7 @@ const authoriseDelete = async (req, res, next) => { message: 'You are not authorised to perform this action', }); } - if (log.eventType !== constants.activityLogEvents.MANUAL_EVENT) { + if (log.eventType !== constants.activityLogEvents.data_access_request.MANUAL_EVENT) { return res.status(400).json({ success: false, message: 'You cannot delete a system generated log entry', From 73ef53c6b3d471faeec5e02cb6ae5411475640c5 Mon Sep 17 00:00:00 2001 From: Callum Reekie Date: Thu, 11 Nov 2021 12:57:50 +0000 Subject: [PATCH 092/116] CR - updated Swagger docs for activity logs --- docs/index.docs.js | 2 + docs/resources/activitylog.docs.js | 160 +++++++++++++++++++++++++++++ 2 files changed, 162 insertions(+) create mode 100644 docs/resources/activitylog.docs.js diff --git a/docs/index.docs.js b/docs/index.docs.js index ea890f34..40ded82a 100644 --- a/docs/index.docs.js +++ b/docs/index.docs.js @@ -12,6 +12,7 @@ import paper from './resources/paper.docs'; import tool from './resources/tool.docs'; import course from './resources/course.docs'; import collection from './resources/collections.docs'; +import activitylog from './resources/activitylog.docs'; import collectionsSchema from './schemas/collections.schema'; @@ -63,6 +64,7 @@ module.exports = { ...tool, ...course, ...collection, + ...activitylog, }, components: { securitySchemes: { diff --git a/docs/resources/activitylog.docs.js b/docs/resources/activitylog.docs.js new file mode 100644 index 00000000..af427301 --- /dev/null +++ b/docs/resources/activitylog.docs.js @@ -0,0 +1,160 @@ +module.exports = { + '/api/v2/activitylog': { + post: { + summary: 'Search activity logs for a given dataset or data access request', + security: [ + { + cookieAuth: [], + }, + ], + requestBody: { + required: true, + content: { + 'application/json': { + schema: { + type: 'object', + required: ['versionIds', 'type'], + properties: { + versionIds: { + type: 'array', + }, + type: { + type: 'array', + }, + }, + example: { + versionIds: ['618cd6170d111006c0550fa3', '618cd556f19753063504a492'], + type: 'dataset', + }, + }, + }, + }, + }, + description: + 'Returns a list of activity logs for a given set of versionIds sorted into thier respective versions. Activity logs can either be for datasets or data access requests. The requesting user must be an admin user or a member of the custodian team to which the version IDs relate.', + tags: ['Activity Logs'], + responses: { + 200: { + description: 'Successful response including the JSON payload.', + }, + 401: { + description: 'Unauthorised.', + }, + }, + }, + }, + '/api/v2/activitylog/{type}': { + post: { + summary: 'Create a manual activity log for a data access requesr', + security: [ + { + cookieAuth: [], + }, + ], + parameters: [ + { + in: 'path', + name: 'type', + required: true, + description: 'The type of activity log. Functionality only exists in current API for data access requests.', + schema: { + type: 'string', + example: 'data_request', + }, + }, + ], + requestBody: { + required: true, + content: { + 'application/json': { + schema: { + type: 'object', + required: ['description', 'timestamp', 'versionId'], + properties: { + description: { + type: 'string', + description: 'The text associated with the manual log.', + }, + timestamp: { + type: 'string', + format: 'date-time', + description: 'Timestamp of when the log was created.', + }, + versionId: { + type: 'string', + description: 'The versionId of the data access request version the activity log relates to.', + }, + }, + example: { description: 'Test', timestamp: '2021-11-11T12:03:49.714Z', versionId: '615b2ba0e33a38453bcf306b' }, + }, + }, + }, + }, + description: + 'Creates a manual activity log for a data access request version. The user must be an admin user or a member of the custodian team to which the log relates.', + tags: ['Activity Logs'], + responses: { + 200: { + description: 'Successful response including the updated JSON payload for the associated data access request version.', + }, + 400: { + description: 'Bad request, including missing information in request body.', + }, + 401: { + description: 'Unauthorised.', + }, + 401: { + description: 'Data access request for submitted version I', + }, + }, + }, + }, + '/api/v2/activitylog/{type}/{id}': { + delete: { + summary: 'Delete a manually created activity log for a data access request', + security: [ + { + cookieAuth: [], + }, + ], + parameters: [ + { + in: 'path', + name: 'type', + required: true, + description: 'The type of activity log. Functionality only exists in current API for data access requests.', + schema: { + type: 'string', + example: 'data_request', + }, + }, + { + in: 'path', + name: 'id', + required: true, + description: 'The id of the manually created activity log.', + schema: { + type: 'string', + }, + }, + ], + description: + 'Deletes a manually created activity log for a data access request version. The user must be a member of the relevant custodian team or an admin user.', + tags: ['Activity Logs'], + responses: { + 200: { + description: 'Successful deletion, including payload for updated version.', + }, + 400: { + description: 'Bad request - only manually created logs can be deleted.', + }, + 401: { + description: 'Unauthorised.', + }, + 404: { + description: 'Log not found for submitted version ID.', + }, + }, + }, + }, +}; From 3fc0d21e4f02bdd367660daca7e3b0ac238515cb Mon Sep 17 00:00:00 2001 From: Callum Reekie Date: Thu, 11 Nov 2021 14:09:36 +0000 Subject: [PATCH 093/116] CR - fixed notifications that had mistakenly been commented out --- src/resources/dataset/datasetonboarding.controller.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/resources/dataset/datasetonboarding.controller.js b/src/resources/dataset/datasetonboarding.controller.js index 09326d1e..645e45c9 100644 --- a/src/resources/dataset/datasetonboarding.controller.js +++ b/src/resources/dataset/datasetonboarding.controller.js @@ -573,7 +573,7 @@ module.exports = { ); //emails / notifications - //await datasetonboardingUtil.createNotifications(constants.notificationTypes.DATASETREJECTED, updatedDataset); + await datasetonboardingUtil.createNotifications(constants.notificationTypes.DATASETREJECTED, updatedDataset); await activityLogService.logActivity(constants.activityLogEvents.dataset.DATASET_VERSION_REJECTED, { type: constants.activityLogTypes.DATASET, @@ -878,7 +878,7 @@ module.exports = { await Data.create(datasetCopy); - //await datasetonboardingUtil.createNotifications(constants.notificationTypes.DATASETDUPLICATED, dataset); + await datasetonboardingUtil.createNotifications(constants.notificationTypes.DATASETDUPLICATED, dataset); return res.status(200).json({ success: true, From 98f8d981a6afa8d6e689c8f5efc68ff0a4e5a3b6 Mon Sep 17 00:00:00 2001 From: Callum Reekie Date: Fri, 12 Nov 2021 09:26:26 +0000 Subject: [PATCH 094/116] CR - admin publisher to return all versions --- src/resources/dataset/datasetonboarding.controller.js | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/resources/dataset/datasetonboarding.controller.js b/src/resources/dataset/datasetonboarding.controller.js index 645e45c9..f2a5280b 100644 --- a/src/resources/dataset/datasetonboarding.controller.js +++ b/src/resources/dataset/datasetonboarding.controller.js @@ -30,7 +30,7 @@ module.exports = { if (publisherID === 'admin') { // get all datasets in review for admin query = { - activeflag: 'inReview', + activeflag: { $in: ['active', 'inReview', 'draft', 'rejected', 'archive'] }, type: 'dataset', }; } else { @@ -50,7 +50,7 @@ module.exports = { .lean(); //Loop through the list of datasets and attach the list of versions to them - const listOfDatasets = datasets.reduce((arr, dataset) => { + let listOfDatasets = datasets.reduce((arr, dataset) => { dataset.listOfVersions = []; const datasetIdx = arr.findIndex(item => item.pid === dataset.pid); if (datasetIdx === -1) { @@ -63,6 +63,10 @@ module.exports = { return arr; }, []); + if (publisherID === 'admin') { + listOfDatasets = listOfDatasets.filter(dataset => dataset.activeflag === 'inReview'); + } + return res.status(200).json({ success: true, data: { listOfDatasets }, From 3afd4afc130fe7f106aeba3efb8571919fd529a0 Mon Sep 17 00:00:00 2001 From: Callum Reekie Date: Fri, 12 Nov 2021 09:52:35 +0000 Subject: [PATCH 095/116] CR - change admin string to constant --- src/resources/dataset/datasetonboarding.controller.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/resources/dataset/datasetonboarding.controller.js b/src/resources/dataset/datasetonboarding.controller.js index f2a5280b..83132bdb 100644 --- a/src/resources/dataset/datasetonboarding.controller.js +++ b/src/resources/dataset/datasetonboarding.controller.js @@ -27,7 +27,7 @@ module.exports = { //Build query, if the publisherId is admin then only return the inReview datasets let query = {}; - if (publisherID === 'admin') { + if (publisherID === constants.userTypes.ADMIN) { // get all datasets in review for admin query = { activeflag: { $in: ['active', 'inReview', 'draft', 'rejected', 'archive'] }, @@ -63,7 +63,7 @@ module.exports = { return arr; }, []); - if (publisherID === 'admin') { + if (publisherID === constants.userTypes.ADMIN) { listOfDatasets = listOfDatasets.filter(dataset => dataset.activeflag === 'inReview'); } From 707042088b8ed7d9a2e1e413ced3434582ca0739 Mon Sep 17 00:00:00 2001 From: Callum Reekie Date: Fri, 12 Nov 2021 10:13:32 +0000 Subject: [PATCH 096/116] CR - change admin string to constant - update --- src/resources/dataset/datasetonboarding.controller.js | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/resources/dataset/datasetonboarding.controller.js b/src/resources/dataset/datasetonboarding.controller.js index 83132bdb..5ba6a6b8 100644 --- a/src/resources/dataset/datasetonboarding.controller.js +++ b/src/resources/dataset/datasetonboarding.controller.js @@ -60,13 +60,12 @@ module.exports = { const versionDetails = { _id, datasetVersion, activeflag }; arr[datasetIdx].listOfVersions = [...arr[datasetIdx].listOfVersions, versionDetails]; } + if (publisherID === constants.userTypes.ADMIN) { + arr = arr.filter(dataset => dataset.activeflag === constants.applicationStatuses.INREVIEW); + } return arr; }, []); - if (publisherID === constants.userTypes.ADMIN) { - listOfDatasets = listOfDatasets.filter(dataset => dataset.activeflag === 'inReview'); - } - return res.status(200).json({ success: true, data: { listOfDatasets }, From f8a4c73ea9d890afbf85532524a3473871427a25 Mon Sep 17 00:00:00 2001 From: Callum Reekie Date: Fri, 12 Nov 2021 10:25:07 +0000 Subject: [PATCH 097/116] CR - change admin string to constant - update --- src/resources/dataset/datasetonboarding.controller.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/resources/dataset/datasetonboarding.controller.js b/src/resources/dataset/datasetonboarding.controller.js index 5ba6a6b8..beac33c6 100644 --- a/src/resources/dataset/datasetonboarding.controller.js +++ b/src/resources/dataset/datasetonboarding.controller.js @@ -50,7 +50,7 @@ module.exports = { .lean(); //Loop through the list of datasets and attach the list of versions to them - let listOfDatasets = datasets.reduce((arr, dataset) => { + const listOfDatasets = datasets.reduce((arr, dataset) => { dataset.listOfVersions = []; const datasetIdx = arr.findIndex(item => item.pid === dataset.pid); if (datasetIdx === -1) { From ae163527c972ab0509408f4ff493cee9285933c9 Mon Sep 17 00:00:00 2001 From: Callum Reekie Date: Fri, 12 Nov 2021 15:46:43 +0000 Subject: [PATCH 098/116] Updated Axios version for security vulnerability --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 7773f154..c4853af8 100644 --- a/package.json +++ b/package.json @@ -18,7 +18,7 @@ "ajv-formats": "^2.0.2", "async": "^3.2.0", "await-to-js": "^2.1.1", - "axios": "0.21.1", + "axios": "0.21.3", "axios-retry": "^3.1.9", "base64url": "^3.0.1", "bcrypt": "^5.0.0", From c43c9a1c4567888451febf548468ac0352e9342b Mon Sep 17 00:00:00 2001 From: massimocianfroccaPA Date: Fri, 12 Nov 2021 17:35:02 +0000 Subject: [PATCH 099/116] Extend data use register patch API and add activity log for DUR --- .../activitylog/activitylog.model.js | 6 +- .../activitylog/activitylog.service.js | 52 ++++++++ .../dataUseRegister.controller.js | 27 +++- .../dataUseRegister.repository.js | 5 +- .../dataUseRegister/dataUseRegister.route.js | 24 +++- .../dataUseRegister.service.js | 126 +++++++++++++++++- src/resources/utilities/constants.util.js | 2 + 7 files changed, 226 insertions(+), 16 deletions(-) diff --git a/src/resources/activitylog/activitylog.model.js b/src/resources/activitylog/activitylog.model.js index 95b34797..6f4a1948 100644 --- a/src/resources/activitylog/activitylog.model.js +++ b/src/resources/activitylog/activitylog.model.js @@ -7,13 +7,13 @@ const ActivityLogSchema = new Schema({ userTypes: [], timestamp: { type: Date, required: true }, user: { type: Schema.Types.ObjectId, ref: 'User', required: true }, - versionId: { type: Schema.Types.ObjectId, required: true }, - version: { type: String, required: true }, + versionId: { type: Schema.Types.ObjectId }, + version: { type: String }, plainText: { type: String, required: true }, detailedText: String, html: { type: String, required: true }, detailedHtml: String, - isPresubmission: Boolean + isPresubmission: Boolean, }); export const ActivityLog = model('ActivityLog', ActivityLogSchema); diff --git a/src/resources/activitylog/activitylog.service.js b/src/resources/activitylog/activitylog.service.js index f3b735b0..ba3bfc5f 100644 --- a/src/resources/activitylog/activitylog.service.js +++ b/src/resources/activitylog/activitylog.service.js @@ -354,6 +354,10 @@ export default class activityLogService { break; case constants.activityLogEvents.MANUAL_EVENT: this.logManualEvent(context); + break; + case constants.activityLogEvents.DATA_USE_REGISTER_UPDATED: + this.logDataUseRegisterUpdated(context); + break; } } @@ -1056,6 +1060,54 @@ export default class activityLogService { await this.activityLogRepository.createActivityLog(log); } + async logDataUseRegisterUpdated(context) { + const { dataUseRegister, updateObj, user } = context; + + let detHtml = ''; + let detText = ''; + + Object.keys(updateObj).forEach(updatedField => { + const oldValue = dataUseRegister[updatedField]; + const newValue = updateObj[updatedField]; + + detHtml = detHtml.concat( + `
` + + `
${dataUseRegister.projectTitle}
` + + `
` + + `
Field
` + + `
${updatedField}
` + + `
` + + `
` + + `
Previous Value
` + + `
${oldValue ? oldValue : ''}
` + + `
` + + `
` + + `
Updated Value
` + + `
${newValue ? newValue : ''}
` + + `
` + + `
` + ); + + detText = detText.concat( + `${dataUseRegister.projectTitle}\nField: ${updatedField}\nPrevious Value: ${oldValue}\nUpdated Value: ${newValue}\n\n` + ); + }); + + const logUpdate = { + eventType: constants.activityLogEvents.DATA_USE_REGISTER_UPDATED, + logType: constants.activityLogTypes.DATA_USE_REGISTER, + timestamp: Date.now(), + detailedText: detText, + plainText: `updates submitted by custodian ${user.firstname} ${user.lastname}.`, + html: `updates submitted by custodian ${user.firstname} ${user.lastname}.`, + detailedHtml: detHtml, + user: user._id, + userTypes: [constants.userTypes.APPLICANT, constants.userTypes.CUSTODIAN], + }; + + await this.activityLogRepository.createActivityLog(logUpdate); + } + getQuestionInfo(accessRequest, questionId) { const questionSet = accessRequest.jsonSchema.questionSets.find(qs => qs.questions.find(question => question.questionId === questionId)); diff --git a/src/resources/dataUseRegister/dataUseRegister.controller.js b/src/resources/dataUseRegister/dataUseRegister.controller.js index ce12e35e..32f99878 100644 --- a/src/resources/dataUseRegister/dataUseRegister.controller.js +++ b/src/resources/dataUseRegister/dataUseRegister.controller.js @@ -9,13 +9,15 @@ import emailGenerator from '../utilities/emailGenerator.util'; import { getObjectFilters } from '../search/search.repository'; import { DataUseRegister } from '../dataUseRegister/dataUseRegister.model'; +import { isEmpty } from 'lodash'; const logCategory = 'dataUseRegister'; export default class DataUseRegisterController extends Controller { - constructor(dataUseRegisterService) { + constructor(dataUseRegisterService, activityLogService) { super(dataUseRegisterService); this.dataUseRegisterService = dataUseRegisterService; + this.activityLogService = activityLogService; } async getDataUseRegister(req, res) { @@ -172,21 +174,24 @@ export default class DataUseRegisterController extends Controller { async updateDataUseRegister(req, res) { try { const id = req.params.id; - const { activeflag, rejectionReason } = req.body; const requestingUser = req.user; - const options = { lean: true, populate: 'user' }; const dataUseRegister = await this.dataUseRegisterService.getDataUseRegister(id, {}, options); + const updateObj = this.dataUseRegisterService.buildUpdateObject(dataUseRegister, req.body); - this.dataUseRegisterService.updateDataUseRegister(dataUseRegister._id, req.body).catch(err => { + this.dataUseRegisterService.updateDataUseRegister(dataUseRegister._id, updateObj).catch(err => { logger.logError(err, logCategory); }); const isDataUseRegisterApproved = - activeflag === constants.dataUseRegisterStatus.ACTIVE && dataUseRegister.activeflag === constants.dataUseRegisterStatus.INREVIEW; + updateObj.activeflag && + updateObj.activeflag === constants.dataUseRegisterStatus.ACTIVE && + dataUseRegister.activeflag === constants.dataUseRegisterStatus.INREVIEW; const isDataUseRegisterRejected = - activeflag === constants.dataUseRegisterStatus.REJECTED && dataUseRegister.activeflag === constants.dataUseRegisterStatus.INREVIEW; + updateObj.activeflag && + updateObj.activeflag === constants.dataUseRegisterStatus.REJECTED && + dataUseRegister.activeflag === constants.dataUseRegisterStatus.INREVIEW; // Send notifications if (isDataUseRegisterApproved) { @@ -194,12 +199,20 @@ export default class DataUseRegisterController extends Controller { } else if (isDataUseRegisterRejected) { await this.createNotifications( constants.dataUseRegisterNotifications.DATAUSEREJECTED, - { rejectionReason }, + { rejectoionReason: updateObj.rejectionReason }, dataUseRegister, requestingUser ); } + if (!isEmpty(updateObj)) { + await this.activityLogService.logActivity(constants.activityLogEvents.DATA_USE_REGISTER_UPDATED, { + dataUseRegister, + updateObj, + user: requestingUser, + }); + } + // Return success return res.status(200).json({ success: true, diff --git a/src/resources/dataUseRegister/dataUseRegister.repository.js b/src/resources/dataUseRegister/dataUseRegister.repository.js index 62fc7720..cccff4b2 100644 --- a/src/resources/dataUseRegister/dataUseRegister.repository.js +++ b/src/resources/dataUseRegister/dataUseRegister.repository.js @@ -90,9 +90,8 @@ export default class DataUseRegisterRepository extends Repository { } async updateDataUseRegister(id, body) { - body.updatedon = Date.now(); - body.lastActivity = Date.now(); - const updatedBody = await this.update(id, body); + const updateObject = { ...body, updatedon: Date.now(), lastActivity: Date.now() }; + const updatedBody = await this.update(id, updateObject); filtersService.optimiseFilters('dataUseRegister'); return updatedBody; } diff --git a/src/resources/dataUseRegister/dataUseRegister.route.js b/src/resources/dataUseRegister/dataUseRegister.route.js index b778e3e1..c413e326 100644 --- a/src/resources/dataUseRegister/dataUseRegister.route.js +++ b/src/resources/dataUseRegister/dataUseRegister.route.js @@ -2,13 +2,14 @@ import express from 'express'; import DataUseRegisterController from './dataUseRegister.controller'; import { dataUseRegisterService } from './dependency'; +import { activityLogService } from '../activitylog/dependency'; import { logger } from '../utilities/logger'; import passport from 'passport'; import constants from './../utilities/constants.util'; -import { isEmpty, isNull } from 'lodash'; +import { isEmpty, isNull, isEqual } from 'lodash'; const router = express.Router(); -const dataUseRegisterController = new DataUseRegisterController(dataUseRegisterService); +const dataUseRegisterController = new DataUseRegisterController(dataUseRegisterService, activityLogService); const logCategory = 'dataUseRegister'; function isUserMemberOfTeam(user, teamId) { @@ -102,6 +103,7 @@ const authorizeView = async (req, res, next) => { const authorizeUpdate = async (req, res, next) => { const requestingUser = req.user; const { id } = req.params; + const { projectId, projectIdText, datasetTitles, datasetIds, datasetPids } = req.body; const dataUseRegister = await dataUseRegisterService.getDataUseRegister(id); @@ -121,6 +123,24 @@ const authorizeUpdate = async (req, res, next) => { }); } + if (!dataUseRegister.manualUpload) { + if (!isEqual(projectId, dataUseRegister.projectId) || !isEqual(projectIdText, dataUseRegister.projectId)) + return res.status(401).json({ + success: false, + message: 'You are not authorised to update the project ID of an automatic data use register', + }); + + if ( + !isEqual(datasetTitles, dataUseRegister.datasetTitles) || + !isEqual(datasetIds, dataUseRegister.datasetIds) || + !isEqual(datasetPids, dataUseRegister.datasetPids) + ) + return res.status(401).json({ + success: false, + message: 'You are not authorised to update the datasets of an automatic data use register', + }); + } + next(); }; diff --git a/src/resources/dataUseRegister/dataUseRegister.service.js b/src/resources/dataUseRegister/dataUseRegister.service.js index f28ba646..12026d83 100644 --- a/src/resources/dataUseRegister/dataUseRegister.service.js +++ b/src/resources/dataUseRegister/dataUseRegister.service.js @@ -1,7 +1,7 @@ import dataUseRegisterUtil from './dataUseRegister.util'; import DataUseRegister from './dataUseRegister.entity'; import constants from '../utilities/constants.util'; -import { isEmpty, isNil } from 'lodash'; +import { isEmpty, isNil, isEqual, isUndefined } from 'lodash'; import moment from 'moment'; export default class DataUseRegisterService { @@ -321,4 +321,128 @@ export default class DataUseRegisterService { return relatedDataUseRegisters; } + + buildUpdateObject(dataUseRegister, dataUseRegisterPayload) { + let updateObj = {}; + + const { + activeflag, + rejectionReason, + discourseTopicId, + relatedObjects, + keywords, + projectTitle, + projectId, + projectIdText, + datasetTitles, + gatewayDatasets, + nonGatewayDatasets, + organisationName, + organisationId, + organisationSector, + gatewayApplicants, + nonGatewayApplicants, + applicantId, + fundersAndSponsors, + accreditedResearcherStatus, + sublicenceArrangements, + laySummary, + publicBenefitStatement, + requestCategoryType, + technicalSummary, + otherApprovalCommittees, + projectStartDate, + projectEndDate, + latestApprovalDate, + dataSensitivityLevel, + legalBasisForDataArticle6, + legalBasisForDataArticle9, + dutyOfConfidentiality, + nationalDataOptOut, + requestFrequency, + datasetLinkageDescription, + confidentialDataDescription, + accessDate, + accessType, + privacyEnhancements, + gatewayOutputsTools, + gatewayOutputsPapers, + nonGatewayOutputs, + } = dataUseRegisterPayload; + + if (!isUndefined(activeflag) && !isEqual(activeflag, dataUseRegister.activeflag)) updateObj.activeflag = activeflag; + if (!isUndefined(rejectionReason) && !isEqual(rejectionReason, dataUseRegister.rejectionReason)) + updateObj.rejectionReason = rejectionReason; + if (!isUndefined(discourseTopicId) && !isEqual(discourseTopicId, dataUseRegister.discourseTopicId)) + updateObj.discourseTopicId = discourseTopicId; + if (!isUndefined(relatedObjects) && !isEqual(relatedObjects, dataUseRegister.relatedObjects)) updateObj.relatedObjects = relatedObjects; + if (!isUndefined(keywords) && !isEqual(keywords, dataUseRegister.keywords)) updateObj.keywords = keywords; + if (!isUndefined(projectTitle) && !isEqual(projectTitle, dataUseRegister.projectTitle)) updateObj.projectTitle = projectTitle; + if (!isUndefined(projectId) && !isEqual(projectId, dataUseRegister.projectId)) updateObj.projectId = projectId; + if (!isUndefined(projectIdText) && !isEqual(projectIdText, dataUseRegister.projectIdText)) updateObj.projectIdText = projectIdText; + if (!isUndefined(datasetTitles) && !isEqual(datasetTitles, dataUseRegister.datasetTitles)) updateObj.datasetTitles = datasetTitles; + if (!isUndefined(gatewayDatasets) && !isEqual(gatewayDatasets, dataUseRegister.gatewayDatasets)) + updateObj.gatewayDatasets = gatewayDatasets; + if (!isUndefined(nonGatewayDatasets) && !isEqual(nonGatewayDatasets, dataUseRegister.nonGatewayDatasets)) + updateObj.nonGatewayDatasets = nonGatewayDatasets; + if (!isUndefined(projectTitle) && !isEqual(projectTitle, dataUseRegister.projectTitle)) updateObj.projectTitle = projectTitle; + if (!isUndefined(organisationName) && !isEqual(organisationName, dataUseRegister.organisationName)) + updateObj.organisationName = organisationName; + if (!isUndefined(organisationId) && !isEqual(organisationId, dataUseRegister.organisationId)) updateObj.organisationId = organisationId; + if (!isUndefined(organisationSector) && !isEqual(organisationSector, dataUseRegister.organisationSector)) + updateObj.organisationSector = organisationSector; + if (!isUndefined(gatewayApplicants) && !isEqual(gatewayApplicants, dataUseRegister.gatewayApplicants)) + updateObj.gatewayApplicants = gatewayApplicants; + if (!isUndefined(nonGatewayApplicants) && !isEqual(nonGatewayApplicants, dataUseRegister.nonGatewayApplicants)) + updateObj.nonGatewayApplicants = nonGatewayApplicants; + if (!isUndefined(applicantId) && !isEqual(applicantId, dataUseRegister.applicantId)) updateObj.applicantId = applicantId; + if (!isUndefined(fundersAndSponsors) && !isEqual(fundersAndSponsors, dataUseRegister.fundersAndSponsors)) + updateObj.fundersAndSponsors = fundersAndSponsors; + if (!isUndefined(accreditedResearcherStatus) && !isEqual(accreditedResearcherStatus, dataUseRegister.accreditedResearcherStatus)) + updateObj.accreditedResearcherStatus = accreditedResearcherStatus; + if (!isUndefined(sublicenceArrangements) && !isEqual(sublicenceArrangements, dataUseRegister.sublicenceArrangements)) + updateObj.sublicenceArrangements = sublicenceArrangements; + if (!isUndefined(laySummary) && !isEqual(laySummary, dataUseRegister.laySummary)) updateObj.laySummary = laySummary; + if (!isUndefined(publicBenefitStatement) && !isEqual(publicBenefitStatement, dataUseRegister.publicBenefitStatement)) + updateObj.publicBenefitStatement = publicBenefitStatement; + if (!isUndefined(requestCategoryType) && !isEqual(requestCategoryType, dataUseRegister.requestCategoryType)) + updateObj.requestCategoryType = requestCategoryType; + if (!isUndefined(technicalSummary) && !isEqual(technicalSummary, dataUseRegister.technicalSummary)) + updateObj.technicalSummary = technicalSummary; + if (!isUndefined(otherApprovalCommittees) && !isEqual(otherApprovalCommittees, dataUseRegister.otherApprovalCommittees)) + updateObj.otherApprovalCommittees = otherApprovalCommittees; + if (!isUndefined(projectStartDate) && !isEqual(projectStartDate, dataUseRegister.projectStartDate)) + updateObj.projectStartDate = projectStartDate; + if (!isUndefined(projectEndDate) && !isEqual(projectEndDate, dataUseRegister.projectEndDate)) updateObj.projectEndDate = projectEndDate; + if (!isUndefined(latestApprovalDate) && !isEqual(latestApprovalDate, dataUseRegister.latestApprovalDate)) + updateObj.latestApprovalDate = latestApprovalDate; + if (!isUndefined(dataSensitivityLevel) && !isEqual(dataSensitivityLevel, dataUseRegister.dataSensitivityLevel)) + updateObj.dataSensitivityLevel = dataSensitivityLevel; + if (!isUndefined(legalBasisForDataArticle6) && !isEqual(legalBasisForDataArticle6, dataUseRegister.legalBasisForDataArticle6)) + updateObj.legalBasisForDataArticle6 = legalBasisForDataArticle6; + if (!isUndefined(legalBasisForDataArticle9) && !isEqual(legalBasisForDataArticle9, dataUseRegister.legalBasisForDataArticle9)) + updateObj.legalBasisForDataArticle9 = legalBasisForDataArticle9; + if (!isUndefined(dutyOfConfidentiality) && !isEqual(dutyOfConfidentiality, dataUseRegister.dutyOfConfidentiality)) + updateObj.dutyOfConfidentiality = dutyOfConfidentiality; + if (!isUndefined(nationalDataOptOut) && !isEqual(nationalDataOptOut, dataUseRegister.nationalDataOptOut)) + updateObj.nationalDataOptOut = nationalDataOptOut; + if (!isUndefined(requestFrequency) && !isEqual(requestFrequency, dataUseRegister.requestFrequency)) + updateObj.requestFrequency = requestFrequency; + if (!isUndefined(datasetLinkageDescription) && !isEqual(datasetLinkageDescription, dataUseRegister.datasetLinkageDescription)) + updateObj.datasetLinkageDescription = datasetLinkageDescription; + if (!isUndefined(confidentialDataDescription) && !isEqual(confidentialDataDescription, dataUseRegister.confidentialDataDescription)) + updateObj.confidentialDataDescription = confidentialDataDescription; + if (!isUndefined(accessDate) && !isEqual(accessDate, dataUseRegister.accessDate)) updateObj.accessDate = accessDate; + if (!isUndefined(accessType) && !isEqual(accessType, dataUseRegister.accessType)) updateObj.accessType = accessType; + if (!isUndefined(privacyEnhancements) && !isEqual(privacyEnhancements, dataUseRegister.privacyEnhancements)) + updateObj.privacyEnhancements = privacyEnhancements; + if (!isUndefined(gatewayOutputsTools) && !isEqual(gatewayOutputsTools, dataUseRegister.gatewayOutputsTools)) + updateObj.gatewayOutputsTools = gatewayOutputsTools; + if (!isUndefined(gatewayOutputsPapers) && !isEqual(gatewayOutputsPapers, dataUseRegister.gatewayOutputsPapers)) + updateObj.gatewayOutputsPapers = gatewayOutputsPapers; + if (!isUndefined(nonGatewayOutputs) && !isEqual(nonGatewayOutputs, dataUseRegister.nonGatewayOutputs)) + updateObj.nonGatewayOutputs = nonGatewayOutputs; + + return updateObj; + } } diff --git a/src/resources/utilities/constants.util.js b/src/resources/utilities/constants.util.js index 0c995edd..aada804c 100644 --- a/src/resources/utilities/constants.util.js +++ b/src/resources/utilities/constants.util.js @@ -284,10 +284,12 @@ const _activityLogEvents = { MANUAL_EVENT: 'manualEvent', CONTEXTUAL_MESSAGE: 'contextualMessage', NOTE: 'note', + DATA_USE_REGISTER_UPDATED: 'dataUseRegisterUpdated', }; const _activityLogTypes = { DATA_ACCESS_REQUEST: 'data_request', + DATA_USE_REGISTER: 'data_use_register', }; const _systemGeneratedUser = { From 9c9416820d686e371d718d18e51153b01bd7c7d9 Mon Sep 17 00:00:00 2001 From: Paul McCafferty <> Date: Sun, 14 Nov 2021 14:46:50 +0000 Subject: [PATCH 100/116] Updates --- .../dataUseRegister/dataUseRegister.controller.js | 10 +++++++++- src/resources/dataUseRegister/dataUseRegister.model.js | 7 +++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/src/resources/dataUseRegister/dataUseRegister.controller.js b/src/resources/dataUseRegister/dataUseRegister.controller.js index ce12e35e..5b5f76d5 100644 --- a/src/resources/dataUseRegister/dataUseRegister.controller.js +++ b/src/resources/dataUseRegister/dataUseRegister.controller.js @@ -265,7 +265,15 @@ export default class DataUseRegisterController extends Controller { searchQuery = getObjectFilters(searchQuery, req, 'dataUseRegister'); - const result = await DataUseRegister.aggregate([{ $match: searchQuery }]); + const result = await DataUseRegister.find(searchQuery) + .populate([ + { path: 'publisher' }, + { path: 'gatewayApplicants' }, + { path: 'gatewayOutputsToolsInfo' }, + { path: 'gatewayOutputsPapersInfo' }, + { path: 'publisherInfo' }, + ]) + .lean(); return res.status(200).json({ success: true, result }); } catch (err) { diff --git a/src/resources/dataUseRegister/dataUseRegister.model.js b/src/resources/dataUseRegister/dataUseRegister.model.js index 35086ff8..7e893e2e 100644 --- a/src/resources/dataUseRegister/dataUseRegister.model.js +++ b/src/resources/dataUseRegister/dataUseRegister.model.js @@ -89,6 +89,13 @@ dataUseRegisterSchema.virtual('publisherInfo', { justOne: true, }); +dataUseRegisterSchema.virtual('publisherDetails', { + ref: 'Publisher', + foreignField: '_id', + localField: 'publisher', + justOne: true, +}); + dataUseRegisterSchema.virtual('gatewayDatasetsInfo', { ref: 'Data', foreignField: 'pid', From 44ca9368fe923acde9a14120709f3fbdcab3546d Mon Sep 17 00:00:00 2001 From: Callum Reekie Date: Sun, 14 Nov 2021 17:39:55 +0000 Subject: [PATCH 101/116] CR - fix for diff object arrays bug --- .../dataset/utils/__mocks__/datasetobjects.js | 4 ++-- .../dataset/utils/datasetonboarding.util.js | 13 +++++++++---- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/src/resources/dataset/utils/__mocks__/datasetobjects.js b/src/resources/dataset/utils/__mocks__/datasetobjects.js index 59a2c924..60fe422e 100644 --- a/src/resources/dataset/utils/__mocks__/datasetobjects.js +++ b/src/resources/dataset/utils/__mocks__/datasetobjects.js @@ -78,14 +78,14 @@ export const datasetv2ObjectMock = { usage: { dataUseLimitation: [], dataUseRequirements: [], - resourceCreator: '', + resourceCreator: [], investigations: [], isReferencedBy: [], }, access: { accessRights: ['http://www.google.com'], accessService: '', - accessRequestCost: '', + accessRequestCost: [], deliveryLeadTime: '', jurisdiction: ['GB-GB'], dataProcessor: '', diff --git a/src/resources/dataset/utils/datasetonboarding.util.js b/src/resources/dataset/utils/datasetonboarding.util.js index c3b4be5c..b67c11a7 100644 --- a/src/resources/dataset/utils/datasetonboarding.util.js +++ b/src/resources/dataset/utils/datasetonboarding.util.js @@ -1170,14 +1170,14 @@ const buildv2Object = async (dataset, newDatasetVersionId = '') => { usage: { dataUseLimitation: questionAnswers['properties/accessibility/usage/dataUseLimitation'] || [], dataUseRequirements: questionAnswers['properties/accessibility/usage/dataUseRequirements'] || [], - resourceCreator: questionAnswers['properties/accessibility/usage/resourceCreator'] || '', + resourceCreator: questionAnswers['properties/accessibility/usage/resourceCreator'] || [], investigations: questionAnswers['properties/accessibility/usage/investigations'] || [], isReferencedBy: questionAnswers['properties/accessibility/usage/isReferencedBy'] || [], }, access: { accessRights: questionAnswers['properties/accessibility/access/accessRights'] || [], accessService: questionAnswers['properties/accessibility/access/accessService'] || '', - accessRequestCost: questionAnswers['properties/accessibility/access/accessRequestCost'] || '', + accessRequestCost: questionAnswers['properties/accessibility/access/accessRequestCost'] || [], deliveryLeadTime: questionAnswers['properties/accessibility/access/deliveryLeadTime'] || '', jurisdiction: questionAnswers['properties/accessibility/access/jurisdiction'] || [], dataProcessor: questionAnswers['properties/accessibility/access/dataProcessor'] || '', @@ -1225,9 +1225,14 @@ const datasetv2ObjectComparison = (updatedJSON, previousJSON) => { result.push(arrayObject); } if ((_.isArray(previousJSON[key]) || _.isArray(updatedJSON[key])) && key !== 'observations') { - if (!_.isEqual(updatedJSON[key], previousJSON[key])) { + let previousAnswer = _.isArray(previousJSON[key]) ? previousJSON[key].join(', ') : previousJSON[key]; + let updatedAnswer = _.isArray(updatedJSON[key]) ? updatedJSON[key].join(', ') : updatedJSON[key]; + if (!_.isEqual(updatedAnswer, previousAnswer)) { let arrayObject = {}; - arrayObject[key] = { previousAnswer: previousJSON[key].join(', '), updatedAnswer: updatedJSON[key].join(', ') }; + arrayObject[key] = { + previousAnswer: previousAnswer, + updatedAnswer: updatedAnswer, + }; result.push(arrayObject); } } From 1767ad0f24e7f29b75f62adf28a2a1f48f899b3c Mon Sep 17 00:00:00 2001 From: Paul McCafferty <> Date: Sun, 14 Nov 2021 22:22:13 +0000 Subject: [PATCH 102/116] Updates --- .../dataUseRegister.controller.js | 5 ++ .../dataUseRegister.service.js | 52 +++++++++++++++---- 2 files changed, 47 insertions(+), 10 deletions(-) diff --git a/src/resources/dataUseRegister/dataUseRegister.controller.js b/src/resources/dataUseRegister/dataUseRegister.controller.js index ab77853e..41aeec5b 100644 --- a/src/resources/dataUseRegister/dataUseRegister.controller.js +++ b/src/resources/dataUseRegister/dataUseRegister.controller.js @@ -179,6 +179,11 @@ export default class DataUseRegisterController extends Controller { const dataUseRegister = await this.dataUseRegisterService.getDataUseRegister(id, {}, options); const updateObj = this.dataUseRegisterService.buildUpdateObject(dataUseRegister, req.body); + if (isEmpty(updateObj)) { + return res.status(200).json({ + success: true, + }); + } this.dataUseRegisterService.updateDataUseRegister(dataUseRegister._id, updateObj).catch(err => { logger.logError(err, logCategory); }); diff --git a/src/resources/dataUseRegister/dataUseRegister.service.js b/src/resources/dataUseRegister/dataUseRegister.service.js index 12026d83..d44cd492 100644 --- a/src/resources/dataUseRegister/dataUseRegister.service.js +++ b/src/resources/dataUseRegister/dataUseRegister.service.js @@ -370,6 +370,24 @@ export default class DataUseRegisterService { nonGatewayOutputs, } = dataUseRegisterPayload; + const fundersAndSponsorsList = + fundersAndSponsors && + fundersAndSponsors + .toString() + .split(',') + .map(el => { + if (!isEmpty(el)) return el.trim(); + }); + + const otherApprovalCommitteesList = + otherApprovalCommittees && + otherApprovalCommittees + .toString() + .split(',') + .map(el => { + if (!isEmpty(el)) return el.trim(); + }); + if (!isUndefined(activeflag) && !isEqual(activeflag, dataUseRegister.activeflag)) updateObj.activeflag = activeflag; if (!isUndefined(rejectionReason) && !isEqual(rejectionReason, dataUseRegister.rejectionReason)) updateObj.rejectionReason = rejectionReason; @@ -396,8 +414,8 @@ export default class DataUseRegisterService { if (!isUndefined(nonGatewayApplicants) && !isEqual(nonGatewayApplicants, dataUseRegister.nonGatewayApplicants)) updateObj.nonGatewayApplicants = nonGatewayApplicants; if (!isUndefined(applicantId) && !isEqual(applicantId, dataUseRegister.applicantId)) updateObj.applicantId = applicantId; - if (!isUndefined(fundersAndSponsors) && !isEqual(fundersAndSponsors, dataUseRegister.fundersAndSponsors)) - updateObj.fundersAndSponsors = fundersAndSponsors; + if (!isUndefined(fundersAndSponsorsList) && !isEqual(fundersAndSponsorsList, dataUseRegister.fundersAndSponsors)) + updateObj.fundersAndSponsors = fundersAndSponsorsList; if (!isUndefined(accreditedResearcherStatus) && !isEqual(accreditedResearcherStatus, dataUseRegister.accreditedResearcherStatus)) updateObj.accreditedResearcherStatus = accreditedResearcherStatus; if (!isUndefined(sublicenceArrangements) && !isEqual(sublicenceArrangements, dataUseRegister.sublicenceArrangements)) @@ -409,13 +427,23 @@ export default class DataUseRegisterService { updateObj.requestCategoryType = requestCategoryType; if (!isUndefined(technicalSummary) && !isEqual(technicalSummary, dataUseRegister.technicalSummary)) updateObj.technicalSummary = technicalSummary; - if (!isUndefined(otherApprovalCommittees) && !isEqual(otherApprovalCommittees, dataUseRegister.otherApprovalCommittees)) - updateObj.otherApprovalCommittees = otherApprovalCommittees; - if (!isUndefined(projectStartDate) && !isEqual(projectStartDate, dataUseRegister.projectStartDate)) - updateObj.projectStartDate = projectStartDate; - if (!isUndefined(projectEndDate) && !isEqual(projectEndDate, dataUseRegister.projectEndDate)) updateObj.projectEndDate = projectEndDate; - if (!isUndefined(latestApprovalDate) && !isEqual(latestApprovalDate, dataUseRegister.latestApprovalDate)) - updateObj.latestApprovalDate = latestApprovalDate; + if (!isUndefined(otherApprovalCommitteesList) && !isEqual(otherApprovalCommitteesList, dataUseRegister.otherApprovalCommittees)) + updateObj.otherApprovalCommittees = otherApprovalCommitteesList; + if ( + !isUndefined(projectStartDate) && + !isEqual(moment(projectStartDate).format('YYYY-MM-DD'), moment(dataUseRegister.projectStartDate).format('YYYY-MM-DD')) + ) + updateObj.projectStartDate = moment(projectStartDate, 'YYYY-MM-DD'); + if ( + !isUndefined(projectEndDate) && + !isEqual(moment(projectEndDate).format('YYYY-MM-DD'), moment(dataUseRegister.projectEndDate).format('YYYY-MM-DD')) + ) + updateObj.projectEndDate = moment(projectEndDate, 'YYYY-MM-DD'); + if ( + !isUndefined(latestApprovalDate) && + !isEqual(moment(latestApprovalDate).format('YYYY-MM-DD'), moment(dataUseRegister.latestApprovalDate).format('YYYY-MM-DD')) + ) + updateObj.projectStartDate = moment(latestApprovalDate, 'YYYY-MM-DD'); if (!isUndefined(dataSensitivityLevel) && !isEqual(dataSensitivityLevel, dataUseRegister.dataSensitivityLevel)) updateObj.dataSensitivityLevel = dataSensitivityLevel; if (!isUndefined(legalBasisForDataArticle6) && !isEqual(legalBasisForDataArticle6, dataUseRegister.legalBasisForDataArticle6)) @@ -432,7 +460,11 @@ export default class DataUseRegisterService { updateObj.datasetLinkageDescription = datasetLinkageDescription; if (!isUndefined(confidentialDataDescription) && !isEqual(confidentialDataDescription, dataUseRegister.confidentialDataDescription)) updateObj.confidentialDataDescription = confidentialDataDescription; - if (!isUndefined(accessDate) && !isEqual(accessDate, dataUseRegister.accessDate)) updateObj.accessDate = accessDate; + if ( + !isUndefined(accessDate) && + !isEqual(moment(accessDate).format('YYYY-MM-DD'), moment(dataUseRegister.accessDate).format('YYYY-MM-DD')) + ) + updateObj.accessDate = moment(accessDate, 'YYYY-MM-DD'); if (!isUndefined(accessType) && !isEqual(accessType, dataUseRegister.accessType)) updateObj.accessType = accessType; if (!isUndefined(privacyEnhancements) && !isEqual(privacyEnhancements, dataUseRegister.privacyEnhancements)) updateObj.privacyEnhancements = privacyEnhancements; From 33f7e9f3549589295b59981dfee9f2a117232c8a Mon Sep 17 00:00:00 2001 From: Paul McCafferty <> Date: Mon, 15 Nov 2021 21:13:51 +0000 Subject: [PATCH 103/116] Updates to access --- src/resources/auth/auth.route.js | 21 +++++++++++++++++++ .../dataUseRegister.controller.js | 2 +- 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/src/resources/auth/auth.route.js b/src/resources/auth/auth.route.js index cfb72157..fdc342f3 100644 --- a/src/resources/auth/auth.route.js +++ b/src/resources/auth/auth.route.js @@ -64,6 +64,27 @@ router.get('/status', function (req, res, next) { } }); } + if (adminArray[0].roles.includes(constants.roleTypes.ADMIN_DATA_USE)) { + const allTeams = await getTeams(); + allTeams.forEach(newTeam => { + const foundTeam = teams.find(team => team._id && team._id.toString() === newTeam._id.toString()); + if (!isEmpty(foundTeam)) { + const foundRole = foundTeam.roles.find(role => role === constants.roleTypes.REVIEWER); + if (isEmpty(foundRole)) { + foundTeam.roles.push(constants.roleTypes.REVIEWER); + } + foundTeam.isAdmin = true; + } else { + teams.push({ + _id: newTeam._id, + name: newTeam.publisher.name, + roles: [constants.roleTypes.REVIEWER], + type: newTeam.type, + isAdmin: true, + }); + } + }); + } } //Remove admin team and then sort teams alphabetically diff --git a/src/resources/dataUseRegister/dataUseRegister.controller.js b/src/resources/dataUseRegister/dataUseRegister.controller.js index 5b5f76d5..9121bf77 100644 --- a/src/resources/dataUseRegister/dataUseRegister.controller.js +++ b/src/resources/dataUseRegister/dataUseRegister.controller.js @@ -140,7 +140,7 @@ export default class DataUseRegisterController extends Controller { if (team === 'user') { delete req.query.team; - query = { ...req.query, user: requestingUser._id }; + query = { ...req.query, gatewayApplicants: requestingUser._id }; } else if (team === 'admin') { delete req.query.team; query = { ...req.query, activeflag: constants.dataUseRegisterStatus.INREVIEW }; From 5d33edc5985da9592a6198e225ebc1e72248e47f Mon Sep 17 00:00:00 2001 From: Paul McCafferty <> Date: Mon, 15 Nov 2021 21:31:40 +0000 Subject: [PATCH 104/116] Fixing LGTM issues --- .../dataUseRegister/dataUseRegister.controller.js | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/src/resources/dataUseRegister/dataUseRegister.controller.js b/src/resources/dataUseRegister/dataUseRegister.controller.js index 9121bf77..952f1178 100644 --- a/src/resources/dataUseRegister/dataUseRegister.controller.js +++ b/src/resources/dataUseRegister/dataUseRegister.controller.js @@ -173,7 +173,6 @@ export default class DataUseRegisterController extends Controller { try { const id = req.params.id; const { activeflag, rejectionReason } = req.body; - const requestingUser = req.user; const options = { lean: true, populate: 'user' }; const dataUseRegister = await this.dataUseRegisterService.getDataUseRegister(id, {}, options); @@ -190,14 +189,9 @@ export default class DataUseRegisterController extends Controller { // Send notifications if (isDataUseRegisterApproved) { - await this.createNotifications(constants.dataUseRegisterNotifications.DATAUSEAPPROVED, {}, dataUseRegister, requestingUser); + await this.createNotifications(constants.dataUseRegisterNotifications.DATAUSEAPPROVED, {}, dataUseRegister); } else if (isDataUseRegisterRejected) { - await this.createNotifications( - constants.dataUseRegisterNotifications.DATAUSEREJECTED, - { rejectionReason }, - dataUseRegister, - requestingUser - ); + await this.createNotifications(constants.dataUseRegisterNotifications.DATAUSEREJECTED, { rejectionReason }, dataUseRegister); } // Return success @@ -255,7 +249,7 @@ export default class DataUseRegisterController extends Controller { try { let searchString = req.query.search || ''; - if (searchString.includes('-') && !searchString.includes('"')) { + if (typeof searchString === 'string' && searchString.includes('-') && !searchString.includes('"')) { const regex = /(?=\S*[-])([a-zA-Z'-]+)/g; searchString = searchString.replace(regex, '"$1"'); } @@ -286,8 +280,7 @@ export default class DataUseRegisterController extends Controller { } } - async createNotifications(type, context, dataUseRegister, requestingUser) { - const { teams } = requestingUser; + async createNotifications(type, context, dataUseRegister) { const { rejectionReason } = context; const { id, projectTitle, user: uploader } = dataUseRegister; From ecf944f37c201a5bf2db7b70367a61e6c10ad722 Mon Sep 17 00:00:00 2001 From: Paul McCafferty <> Date: Tue, 16 Nov 2021 10:52:46 +0000 Subject: [PATCH 105/116] Updates --- src/resources/auth/auth.route.js | 21 +++++++++++++ .../dataUseRegister/dataUseRegister.route.js | 12 +++---- .../dataUseRegister.service.js | 31 ++++++++++--------- 3 files changed, 41 insertions(+), 23 deletions(-) diff --git a/src/resources/auth/auth.route.js b/src/resources/auth/auth.route.js index cfb72157..fdc342f3 100644 --- a/src/resources/auth/auth.route.js +++ b/src/resources/auth/auth.route.js @@ -64,6 +64,27 @@ router.get('/status', function (req, res, next) { } }); } + if (adminArray[0].roles.includes(constants.roleTypes.ADMIN_DATA_USE)) { + const allTeams = await getTeams(); + allTeams.forEach(newTeam => { + const foundTeam = teams.find(team => team._id && team._id.toString() === newTeam._id.toString()); + if (!isEmpty(foundTeam)) { + const foundRole = foundTeam.roles.find(role => role === constants.roleTypes.REVIEWER); + if (isEmpty(foundRole)) { + foundTeam.roles.push(constants.roleTypes.REVIEWER); + } + foundTeam.isAdmin = true; + } else { + teams.push({ + _id: newTeam._id, + name: newTeam.publisher.name, + roles: [constants.roleTypes.REVIEWER], + type: newTeam.type, + isAdmin: true, + }); + } + }); + } } //Remove admin team and then sort teams alphabetically diff --git a/src/resources/dataUseRegister/dataUseRegister.route.js b/src/resources/dataUseRegister/dataUseRegister.route.js index c413e326..641cb3a7 100644 --- a/src/resources/dataUseRegister/dataUseRegister.route.js +++ b/src/resources/dataUseRegister/dataUseRegister.route.js @@ -103,7 +103,7 @@ const authorizeView = async (req, res, next) => { const authorizeUpdate = async (req, res, next) => { const requestingUser = req.user; const { id } = req.params; - const { projectId, projectIdText, datasetTitles, datasetIds, datasetPids } = req.body; + const { projectIdText, datasetTitles } = req.body; const dataUseRegister = await dataUseRegisterService.getDataUseRegister(id); @@ -115,7 +115,7 @@ const authorizeUpdate = async (req, res, next) => { } const { publisher } = dataUseRegister; - const authorised = isUserMemberOfTeam(requestingUser, publisher._id) || isUserDataUseAdmin(requestingUser); + const authorised = isUserDataUseAdmin(requestingUser) || isUserMemberOfTeam(requestingUser, publisher._id); if (!authorised) { return res.status(401).json({ success: false, @@ -124,17 +124,13 @@ const authorizeUpdate = async (req, res, next) => { } if (!dataUseRegister.manualUpload) { - if (!isEqual(projectId, dataUseRegister.projectId) || !isEqual(projectIdText, dataUseRegister.projectId)) + if (!isEqual(projectIdText, dataUseRegister.projectIdText)) return res.status(401).json({ success: false, message: 'You are not authorised to update the project ID of an automatic data use register', }); - if ( - !isEqual(datasetTitles, dataUseRegister.datasetTitles) || - !isEqual(datasetIds, dataUseRegister.datasetIds) || - !isEqual(datasetPids, dataUseRegister.datasetPids) - ) + if (!isEqual(datasetTitles, dataUseRegister.datasetTitles)) return res.status(401).json({ success: false, message: 'You are not authorised to update the datasets of an automatic data use register', diff --git a/src/resources/dataUseRegister/dataUseRegister.service.js b/src/resources/dataUseRegister/dataUseRegister.service.js index d44cd492..6c4964cf 100644 --- a/src/resources/dataUseRegister/dataUseRegister.service.js +++ b/src/resources/dataUseRegister/dataUseRegister.service.js @@ -234,7 +234,11 @@ export default class DataUseRegisterService { [...authors, mainApplicant], questionAnswers ); - const relatedDatasets = dataUseRegisterUtil.buildRelatedDatasets(creatorUser, datasets, false); + const { linkedDatasets = [], namedDatasets = [] } = await dataUseRegisterUtil.getLinkedDatasets([ + ...datasets.map(dataset => dataset.name), + ]); + const datasetTitles = [...linkedDatasets.map(dataset => dataset.name), ...namedDatasets]; + const relatedDatasets = dataUseRegisterUtil.buildRelatedObjects(creatorUser, 'dataset', datasets, false); const relatedApplications = await this.buildRelatedDataUseRegisters(creatorUser, versionTree, applicationId); const datasetLinkageDescription = `${datasetLinkageDetails.toString().trim()} ${datasetLinkageRiskMitigation.toString().trim()}`; const requestFrequency = dataRefreshRequired === 'Yes' ? 'Recurring' : dataRefreshRequired === 'No' ? 'One-off' : ''; @@ -247,7 +251,7 @@ export default class DataUseRegisterService { publisher, projectIdText: projectId, projectId: applicationId, - applicantId: applicantId.trim(), + applicantId: applicantId ? applicantId.trim() : '', accreditedResearcherStatus: isNil(accreditedResearcherStatus) ? 'Unknown' : accreditedResearcherStatus.toString().trim(), ...(projectTitle && { projectTitle: projectTitle.toString().trim() }), ...(organisationName && { organisationName: organisationName.toString().trim() }), @@ -263,15 +267,15 @@ export default class DataUseRegisterService { ...(projectStartDate.isValid() && { projectStartDate }), ...(projectEndDate.isValid() && { projectEndDate }), ...(latestApprovalDate.isValid() && { latestApprovalDate }), - datasetTitles: [...datasets.map(dataset => dataset.name)], - datasetIds: [...datasets.map(dataset => dataset.datasetid)], - datasetPids: [...datasets.map(dataset => dataset.pid)], + ...(!isEmpty(datasetTitles) && { datasetTitles }), + ...(!isEmpty(linkedDatasets) && { gatewayDatasets: linkedDatasets.map(dataset => dataset.pid) }), + ...(!isEmpty(namedDatasets) && { nonGatewayDatasets: namedDatasets }), keywords: isNil(keywords) || isEmpty(keywords) ? [] : keywords.split(' ').slice(0, 6), fundersAndSponsors, gatewayApplicants, nonGatewayApplicants, relatedObjects: [...relatedDatasets, ...relatedApplications], - activeflag: 'inReview', + activeflag: 'active', user: creatorUser._id, userName: `${creatorUser.firstname} ${creatorUser.lastname}`, updatedon: Date.now(), @@ -414,7 +418,7 @@ export default class DataUseRegisterService { if (!isUndefined(nonGatewayApplicants) && !isEqual(nonGatewayApplicants, dataUseRegister.nonGatewayApplicants)) updateObj.nonGatewayApplicants = nonGatewayApplicants; if (!isUndefined(applicantId) && !isEqual(applicantId, dataUseRegister.applicantId)) updateObj.applicantId = applicantId; - if (!isUndefined(fundersAndSponsorsList) && !isEqual(fundersAndSponsorsList, dataUseRegister.fundersAndSponsors)) + if (!isEmpty(fundersAndSponsorsList) && !isEqual(fundersAndSponsorsList, dataUseRegister.fundersAndSponsors)) updateObj.fundersAndSponsors = fundersAndSponsorsList; if (!isUndefined(accreditedResearcherStatus) && !isEqual(accreditedResearcherStatus, dataUseRegister.accreditedResearcherStatus)) updateObj.accreditedResearcherStatus = accreditedResearcherStatus; @@ -427,20 +431,20 @@ export default class DataUseRegisterService { updateObj.requestCategoryType = requestCategoryType; if (!isUndefined(technicalSummary) && !isEqual(technicalSummary, dataUseRegister.technicalSummary)) updateObj.technicalSummary = technicalSummary; - if (!isUndefined(otherApprovalCommitteesList) && !isEqual(otherApprovalCommitteesList, dataUseRegister.otherApprovalCommittees)) + if (!isEmpty(otherApprovalCommitteesList) && !isEqual(otherApprovalCommitteesList, dataUseRegister.otherApprovalCommittees)) updateObj.otherApprovalCommittees = otherApprovalCommitteesList; if ( - !isUndefined(projectStartDate) && + !isEmpty(projectStartDate) && !isEqual(moment(projectStartDate).format('YYYY-MM-DD'), moment(dataUseRegister.projectStartDate).format('YYYY-MM-DD')) ) updateObj.projectStartDate = moment(projectStartDate, 'YYYY-MM-DD'); if ( - !isUndefined(projectEndDate) && + !isEmpty(projectEndDate) && !isEqual(moment(projectEndDate).format('YYYY-MM-DD'), moment(dataUseRegister.projectEndDate).format('YYYY-MM-DD')) ) updateObj.projectEndDate = moment(projectEndDate, 'YYYY-MM-DD'); if ( - !isUndefined(latestApprovalDate) && + !isEmpty(latestApprovalDate) && !isEqual(moment(latestApprovalDate).format('YYYY-MM-DD'), moment(dataUseRegister.latestApprovalDate).format('YYYY-MM-DD')) ) updateObj.projectStartDate = moment(latestApprovalDate, 'YYYY-MM-DD'); @@ -460,10 +464,7 @@ export default class DataUseRegisterService { updateObj.datasetLinkageDescription = datasetLinkageDescription; if (!isUndefined(confidentialDataDescription) && !isEqual(confidentialDataDescription, dataUseRegister.confidentialDataDescription)) updateObj.confidentialDataDescription = confidentialDataDescription; - if ( - !isUndefined(accessDate) && - !isEqual(moment(accessDate).format('YYYY-MM-DD'), moment(dataUseRegister.accessDate).format('YYYY-MM-DD')) - ) + if (!isEmpty(accessDate) && !isEqual(moment(accessDate).format('YYYY-MM-DD'), moment(dataUseRegister.accessDate).format('YYYY-MM-DD'))) updateObj.accessDate = moment(accessDate, 'YYYY-MM-DD'); if (!isUndefined(accessType) && !isEqual(accessType, dataUseRegister.accessType)) updateObj.accessType = accessType; if (!isUndefined(privacyEnhancements) && !isEqual(privacyEnhancements, dataUseRegister.privacyEnhancements)) From fe8072906b4757644eb48f8efa56c40df0609672 Mon Sep 17 00:00:00 2001 From: Paul McCafferty <> Date: Tue, 16 Nov 2021 11:41:51 +0000 Subject: [PATCH 106/116] Fix for camelcase route --- src/config/server.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/config/server.js b/src/config/server.js index 750cdd91..4cc51275 100644 --- a/src/config/server.js +++ b/src/config/server.js @@ -254,7 +254,7 @@ app.use('/api/v1/global', require('../resources/global/global.route')); app.use('/api/v1/search-preferences', require('../resources/searchpreferences/searchpreferences.route')); -app.use('/api/v2/data-use-registers', require('../resources/datauseregister/datauseregister.route')); +app.use('/api/v2/data-use-registers', require('../resources/dataUseRegister/dataUseRegister.route')); initialiseAuthentication(app); From 6e6769e78889cc811cf20be58394fec4a4d4cbbb Mon Sep 17 00:00:00 2001 From: Paul McCafferty <> Date: Fri, 19 Nov 2021 16:10:53 +0000 Subject: [PATCH 107/116] Fix for when sorting by metadata quality score on main search page - updating the field to look at to weighted_quality_score from quality_score --- src/resources/search/search.repository.js | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/resources/search/search.repository.js b/src/resources/search/search.repository.js index cd063f39..1bd8b852 100644 --- a/src/resources/search/search.repository.js +++ b/src/resources/search/search.repository.js @@ -295,7 +295,7 @@ export async function getObjectResult(type, searchAll, searchQuery, startIndex, activeflag: 1, counter: 1, - 'datasetfields.metadataquality.quality_score': 1, + 'datasetfields.metadataquality.weighted_quality_score': 1, latestUpdate: '$timestamps.updated', relatedresources: { @@ -364,7 +364,7 @@ export async function getObjectResult(type, searchAll, searchQuery, startIndex, activeflag: 1, counter: 1, - 'datasetfields.metadataquality.quality_score': 1, + 'datasetfields.metadataquality.weighted_quality_score': 1, latestUpdate: { $cond: { if: { $gte: ['$createdAt', '$updatedon'] }, @@ -391,7 +391,7 @@ export async function getObjectResult(type, searchAll, searchQuery, startIndex, if (sort === '') { if (type === 'dataset') { - if (searchAll) queryObject.push({ $sort: { 'datasetfields.metadataquality.quality_score': -1, name: 1 } }); + if (searchAll) queryObject.push({ $sort: { 'datasetfields.metadataquality.weighted_quality_score': -1, name: 1 } }); else queryObject.push({ $sort: { score: { $meta: 'textScore' } } }); } else if (type === 'paper') { if (searchAll) queryObject.push({ $sort: { journalYear: -1 } }); @@ -424,8 +424,8 @@ export async function getObjectResult(type, searchAll, searchQuery, startIndex, else queryObject.push({ $sort: { counter: -1, score: { $meta: 'textScore' } } }); } } else if (sort === 'metadata') { - if (searchAll) queryObject.push({ $sort: { 'datasetfields.metadataquality.quality_score': -1, name: 1 } }); - else queryObject.push({ $sort: { 'datasetfields.metadataquality.quality_score': -1, score: { $meta: 'textScore' } } }); + if (searchAll) queryObject.push({ $sort: { 'datasetfields.metadataquality.weighted_quality_score': -1, name: 1 } }); + else queryObject.push({ $sort: { 'datasetfields.metadataquality.weighted_quality_score': -1, score: { $meta: 'textScore' } } }); } else if (sort === 'startdate') { if (form === 'true' && searchAll) { queryObject.push({ $sort: { myEntity: -1, 'courseOptions.startDate': 1 } }); From 1c0db67448ebecaa0118f53f0c3fd07d103f3a8b Mon Sep 17 00:00:00 2001 From: Paul McCafferty <> Date: Tue, 23 Nov 2021 23:07:02 +0000 Subject: [PATCH 108/116] Updates --- src/resources/dataUseRegister/dataUseRegister.controller.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/resources/dataUseRegister/dataUseRegister.controller.js b/src/resources/dataUseRegister/dataUseRegister.controller.js index 41aeec5b..429addd7 100644 --- a/src/resources/dataUseRegister/dataUseRegister.controller.js +++ b/src/resources/dataUseRegister/dataUseRegister.controller.js @@ -142,7 +142,7 @@ export default class DataUseRegisterController extends Controller { if (team === 'user') { delete req.query.team; - query = { ...req.query, user: requestingUser._id }; + query = { ...req.query, gatewayApplicants: requestingUser._id }; } else if (team === 'admin') { delete req.query.team; query = { ...req.query, activeflag: constants.dataUseRegisterStatus.INREVIEW }; From 7d3a5077add9521da401805621c2efc4921ea4d4 Mon Sep 17 00:00:00 2001 From: Paul McCafferty <> Date: Wed, 24 Nov 2021 09:51:10 +0000 Subject: [PATCH 109/116] Fixes for DUR --- cloudbuild_dynamic.yaml | 2 +- src/config/server.js | 2 +- .../dataUseRegister.controller.js | 87 +++++++++++++++++-- .../dataUseRegister/dataUseRegister.model.js | 6 +- 4 files changed, 83 insertions(+), 14 deletions(-) diff --git a/cloudbuild_dynamic.yaml b/cloudbuild_dynamic.yaml index 846ee11f..ebb677b2 100644 --- a/cloudbuild_dynamic.yaml +++ b/cloudbuild_dynamic.yaml @@ -41,6 +41,6 @@ steps: images: - gcr.io/$PROJECT_ID/${_APP_NAME}:${_ENVIRONMENT} -timeout: 900s +timeout: 1200s options: machineType: 'E2_HIGHCPU_8' diff --git a/src/config/server.js b/src/config/server.js index 750cdd91..4cc51275 100644 --- a/src/config/server.js +++ b/src/config/server.js @@ -254,7 +254,7 @@ app.use('/api/v1/global', require('../resources/global/global.route')); app.use('/api/v1/search-preferences', require('../resources/searchpreferences/searchpreferences.route')); -app.use('/api/v2/data-use-registers', require('../resources/datauseregister/datauseregister.route')); +app.use('/api/v2/data-use-registers', require('../resources/dataUseRegister/dataUseRegister.route')); initialiseAuthentication(app); diff --git a/src/resources/dataUseRegister/dataUseRegister.controller.js b/src/resources/dataUseRegister/dataUseRegister.controller.js index 952f1178..c2b2105e 100644 --- a/src/resources/dataUseRegister/dataUseRegister.controller.js +++ b/src/resources/dataUseRegister/dataUseRegister.controller.js @@ -259,15 +259,84 @@ export default class DataUseRegisterController extends Controller { searchQuery = getObjectFilters(searchQuery, req, 'dataUseRegister'); - const result = await DataUseRegister.find(searchQuery) - .populate([ - { path: 'publisher' }, - { path: 'gatewayApplicants' }, - { path: 'gatewayOutputsToolsInfo' }, - { path: 'gatewayOutputsPapersInfo' }, - { path: 'publisherInfo' }, - ]) - .lean(); + const aggregateQuery = [ + { + $lookup: { + from: 'publishers', + localField: 'publisher', + foreignField: '_id', + as: 'publisherDetails', + }, + }, + { + $lookup: { + from: 'tools', + localField: 'gatewayOutputsTools', + foreignField: 'id', + as: 'gatewayOutputsToolsInfo', + }, + }, + { + $lookup: { + from: 'tools', + localField: 'gatewayOutputsPapers', + foreignField: 'id', + as: 'gatewayOutputsPapersInfo', + }, + }, + { + $lookup: { + from: 'users', + let: { + listOfGatewayApplicants: '$gatewayApplicants', + }, + pipeline: [ + { + $match: { + $expr: { + $and: [{ $in: ['$_id', '$$listOfGatewayApplicants'] }], + }, + }, + }, + { $project: { firstname: 1, lastname: 1 } }, + ], + + as: 'gatewayApplicantsDetails', + }, + }, + { + $lookup: { + from: 'tools', + let: { + listOfGatewayDatasets: '$gatewayDatasets', + }, + pipeline: [ + { + $match: { + $expr: { + $and: [ + { $in: ['$pid', '$$listOfGatewayDatasets'] }, + { + $eq: ['$activeflag', 'active'], + }, + ], + }, + }, + }, + { $project: { pid: 1, name: 1 } }, + ], + as: 'gatewayDatasetsInfo', + }, + }, + { + $addFields: { + publisherInfo: { name: '$publisherDetails.name' }, + }, + }, + { $match: searchQuery }, + ]; + + const result = await DataUseRegister.aggregate(aggregateQuery); return res.status(200).json({ success: true, result }); } catch (err) { diff --git a/src/resources/dataUseRegister/dataUseRegister.model.js b/src/resources/dataUseRegister/dataUseRegister.model.js index 7e893e2e..d885669f 100644 --- a/src/resources/dataUseRegister/dataUseRegister.model.js +++ b/src/resources/dataUseRegister/dataUseRegister.model.js @@ -66,8 +66,8 @@ const dataUseRegisterSchema = new Schema( accessDate: Date, //Release/Access Date accessType: String, //TRE Or Any Other Specified Location privacyEnhancements: String, //How Has Data Been Processed To Enhance Privacy - gatewayOutputsTools: [{ type: String }], //Link To Gateway Tool Research Outputs - gatewayOutputsPapers: [{ type: String }], //Link To Gateway Paper Research Outputs + gatewayOutputsTools: [{ type: Number }], //Link To Gateway Tool Research Outputs + gatewayOutputsPapers: [{ type: Number }], //Link To Gateway Paper Research Outputs nonGatewayOutputs: [{ type: String }], //Link To NonGateway Research Outputs rejectionReason: String, //Reason For Rejecting A Data Use Register }, @@ -100,7 +100,7 @@ dataUseRegisterSchema.virtual('gatewayDatasetsInfo', { ref: 'Data', foreignField: 'pid', localField: 'gatewayDatasets', - options: { sort: { createdAt: -1 }, limit: 1 }, + options: { sort: { createdAt: -1 } }, }); dataUseRegisterSchema.virtual('gatewayOutputsToolsInfo', { From 6598eaa8619fa97855cbbef5d72118790e82f9a6 Mon Sep 17 00:00:00 2001 From: Paul McCafferty <> Date: Thu, 25 Nov 2021 10:32:23 +0000 Subject: [PATCH 110/116] Updates to DUR --- .../dataUseRegister.controller.js | 34 ++++++++++- src/resources/utilities/constants.util.js | 1 + .../utilities/emailGenerator.util.js | 59 ++++++++++++++++++- 3 files changed, 92 insertions(+), 2 deletions(-) diff --git a/src/resources/dataUseRegister/dataUseRegister.controller.js b/src/resources/dataUseRegister/dataUseRegister.controller.js index c2b2105e..07a0f99d 100644 --- a/src/resources/dataUseRegister/dataUseRegister.controller.js +++ b/src/resources/dataUseRegister/dataUseRegister.controller.js @@ -214,6 +214,7 @@ export default class DataUseRegisterController extends Controller { const requestingUser = req.user; const result = await this.dataUseRegisterService.uploadDataUseRegisters(requestingUser, teamId, dataUses); // Return success + await this.createNotifications(constants.dataUseRegisterNotifications.DATAUSEPENDING, {}, result, teamId); return res.status(result.uploadedCount > 0 ? 201 : 200).json({ success: true, result, @@ -349,7 +350,7 @@ export default class DataUseRegisterController extends Controller { } } - async createNotifications(type, context, dataUseRegister) { + async createNotifications(type, context, dataUseRegister, publisher) { const { rejectionReason } = context; const { id, projectTitle, user: uploader } = dataUseRegister; @@ -393,6 +394,37 @@ export default class DataUseRegisterController extends Controller { emailGenerator.sendEmail(emailRecipients, constants.hdrukEmail, `A data use has been rejected by HDR UK`, html, false); break; } + case constants.dataUseRegisterNotifications.DATAUSEPENDING: { + const adminTeam = await TeamModel.findOne({ type: 'admin' }) + .populate({ + path: 'users', + }) + .lean(); + + const publisherTeam = await TeamModel.findOne({ _id: publisher }) + .populate({ + path: 'publisher', + }) + .lean(); + + const dataUseTeamMembers = teamController.getTeamMembersByRole(adminTeam, constants.roleTypes.ADMIN_DATA_USE); + const emailRecipients = [...dataUseTeamMembers]; + + const { uploaded } = dataUseRegister; + let listOfProjectTitles = []; + uploaded.forEach(dataset => { + listOfProjectTitles.push(dataset.projectTitle); + }); + + const options = { + listOfProjectTitles, + publisher: publisherTeam.publisher.name, + }; + + const html = emailGenerator.generateDataUseRegisterPending(options); + emailGenerator.sendEmail(emailRecipients, constants.hdrukEmail, `New data uses to review`, html, false); + break; + } } } } diff --git a/src/resources/utilities/constants.util.js b/src/resources/utilities/constants.util.js index 8375d628..c58ee12d 100644 --- a/src/resources/utilities/constants.util.js +++ b/src/resources/utilities/constants.util.js @@ -18,6 +18,7 @@ const _activityLogNotifications = Object.freeze({ const _dataUseRegisterNotifications = Object.freeze({ DATAUSEAPPROVED: 'dataUseApproved', DATAUSEREJECTED: 'dataUseRejected', + DATAUSEPENDING: 'dataUsePending', }); const _teamNotificationTypes = Object.freeze({ diff --git a/src/resources/utilities/emailGenerator.util.js b/src/resources/utilities/emailGenerator.util.js index 0777acf3..40d06730 100644 --- a/src/resources/utilities/emailGenerator.util.js +++ b/src/resources/utilities/emailGenerator.util.js @@ -642,6 +642,11 @@ const _displayDataUseRegisterLink = dataUseId => { return `View data use`; }; +const _displayDataUseRegisterDashboardLink = () => { + const dataUseLink = `${process.env.homeURL}/account?tab=datause&team=admin`; + return `View all data uses for review `; +}; + const _generateDARStatusChangedEmail = options => { let { id, applicationStatus, applicationStatusDesc, projectId, projectName, publisher, datasetTitles, dateSubmitted, applicants } = options; @@ -2419,7 +2424,7 @@ const _generateDataUseRegisterRejected = options => {

A data use has been rejected

-

A data use for ${projectTitle} has been rejected by HDR UK team. +

A data use for ${projectTitle} has been rejected by HDR UK team.

Reason for rejection:

${rejectionReason}

${_displayDataUseRegisterLink(id)} @@ -2428,6 +2433,57 @@ const _generateDataUseRegisterRejected = options => { return body; }; +const _generateDataUseRegisterPending = options => { + const { listOfProjectTitles, publisher } = options; + + const body = `
+ + + + + + + + + + + + + + +
+ New data uses to review +
+ ${publisher} has submitted [${listOfProjectTitles.length}] data uses for review including: +
+ + + + + + + + + +
Project title${listOfProjectTitles.join( + ', ' + )}
Date and time submitted${moment().format( + 'DD/MM/YYYY, HH:mmA' + )}
+
+
+${_displayDataUseRegisterDashboardLink()} +
+
`; + return body; +}; + /** * [_sendEmail] * @@ -2590,4 +2646,5 @@ export default { //DataUseRegister generateDataUseRegisterApproved: _generateDataUseRegisterApproved, generateDataUseRegisterRejected: _generateDataUseRegisterRejected, + generateDataUseRegisterPending: _generateDataUseRegisterPending, }; From 9cf640f20107d511f4afe0a049d9a48774373688 Mon Sep 17 00:00:00 2001 From: Paul McCafferty <> Date: Fri, 26 Nov 2021 10:56:06 +0000 Subject: [PATCH 111/116] Disabling keyword filter --- src/resources/filters/filters.mapper.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/resources/filters/filters.mapper.js b/src/resources/filters/filters.mapper.js index 478672ff..0863e777 100644 --- a/src/resources/filters/filters.mapper.js +++ b/src/resources/filters/filters.mapper.js @@ -916,7 +916,7 @@ export const dataUseRegisterFilters = [ highlighted: [], beta: false, }, - { + /* { id: 5, label: 'Keywords', key: 'keywords', @@ -930,5 +930,5 @@ export const dataUseRegisterFilters = [ filters: [], highlighted: [], beta: false, - }, + }, */ ]; From 57ec5f759066e635991c5f6ed4d1dbf6d99ef949 Mon Sep 17 00:00:00 2001 From: Paul McCafferty <> Date: Mon, 29 Nov 2021 23:30:48 +0000 Subject: [PATCH 112/116] Fixing tests --- .../__mocks__/dataUseRegisters.js | 22 +++++++++------ .../__tests__/dataUseRegister.service.test.js | 3 +++ .../__tests__/dataUseRegister.util.test.js | 27 ++++++++++++++----- .../dataUseRegister.repository.js | 2 +- 4 files changed, 39 insertions(+), 15 deletions(-) diff --git a/src/resources/dataUseRegister/__mocks__/dataUseRegisters.js b/src/resources/dataUseRegister/__mocks__/dataUseRegisters.js index 2c8f4b00..680caaa9 100644 --- a/src/resources/dataUseRegister/__mocks__/dataUseRegisters.js +++ b/src/resources/dataUseRegister/__mocks__/dataUseRegisters.js @@ -302,19 +302,19 @@ export const dataUseRegisterUploadsWithDuplicates = [ export const datasets = [ { - datasetid: '70b4d407-288a-4945-a4d5-506d60715110', + id: '70b4d407-288a-4945-a4d5-506d60715110', pid: 'e55df485-5acd-4606-bbb8-668d4c06380a', }, { - datasetid: '82ef7d1a-98d8-48b6-9acd-461bf2a399c3', + id: '82ef7d1a-98d8-48b6-9acd-461bf2a399c3', pid: 'e55df485-5acd-4606-bbb8-668d4c06380a', }, { - datasetid: '673626f3-bdac-4d32-9bb8-c890b727c0d1', + id: '673626f3-bdac-4d32-9bb8-c890b727c0d1', pid: '594d79a4-92b9-4a7f-b991-abf850bf2b67', }, { - datasetid: '89e57932-ac48-48ac-a6e5-29795bc38b94', + id: '89e57932-ac48-48ac-a6e5-29795bc38b94', pid: 'efbd4275-70e2-4887-8499-18b1fb24ce5b', }, ]; @@ -325,7 +325,7 @@ export const relatedObjectDatasets = [ pid: 'e55df485-5acd-4606-bbb8-668d4c06380a', objectType: 'dataset', user: 'James Smith', - updated: '2021-24-09T11:01:58.135Z', + updated: '24 Sept 2021', isLocked: true, reason: 'This dataset was added automatically during the manual upload of this data use register', }, @@ -334,7 +334,7 @@ export const relatedObjectDatasets = [ pid: 'e55df485-5acd-4606-bbb8-668d4c06380a', objectType: 'dataset', user: 'James Smith', - updated: '2021-24-09T11:01:58.135Z', + updated: '24 Sept 2021', isLocked: true, reason: 'This dataset was added automatically during the manual upload of this data use register', }, @@ -343,7 +343,7 @@ export const relatedObjectDatasets = [ pid: '594d79a4-92b9-4a7f-b991-abf850bf2b67', objectType: 'dataset', user: 'James Smith', - updated: '2021-24-09T11:01:58.135Z', + updated: '24 Sept 2021', isLocked: true, reason: 'This dataset was added automatically during the manual upload of this data use register', }, @@ -352,7 +352,7 @@ export const relatedObjectDatasets = [ pid: 'efbd4275-70e2-4887-8499-18b1fb24ce5b', objectType: 'dataset', user: 'James Smith', - updated: '2021-24-09T11:01:58.135Z', + updated: '24 Sept 2021', isLocked: true, reason: 'This dataset was added automatically during the manual upload of this data use register', }, @@ -372,6 +372,12 @@ export const expectedGatewayDatasets = [ { datasetid: '3', name: 'dataset 3', pid: '333' }, ]; +export const expectedGatewayDatasetsReturned = [ + { id: '1', name: 'dataset 1', pid: '111' }, + { id: '2', name: 'dataset 2', pid: '222' }, + { id: '3', name: 'dataset 3', pid: '333' }, +]; + export const nonGatewayApplicantNames = ['applicant one', 'applicant two', 'applicant three', 'applicant four']; export const gatewayApplicantNames = ['http://localhost:3000/person/8495781222000176', 'http://localhost:3000/person/4495285946631793']; diff --git a/src/resources/dataUseRegister/__tests__/dataUseRegister.service.test.js b/src/resources/dataUseRegister/__tests__/dataUseRegister.service.test.js index b1eeb4d2..cc2c236a 100644 --- a/src/resources/dataUseRegister/__tests__/dataUseRegister.service.test.js +++ b/src/resources/dataUseRegister/__tests__/dataUseRegister.service.test.js @@ -2,6 +2,7 @@ import sinon from 'sinon'; import DataUseRegisterService from '../dataUseRegister.service'; import DataUseRegisterRepository from '../dataUseRegister.repository'; +import dataUseRegisterUtil from '../dataUseRegister.util'; import { dataUseRegisterUploadsWithDuplicates, dataUseRegisterUploads } from '../__mocks__/dataUseRegisters'; describe('DataUseRegisterService', function () { @@ -44,6 +45,8 @@ describe('DataUseRegisterService', function () { const checkDataUseRegisterExistsStub = sinon.stub(dataUseRegisterRepository, 'checkDataUseRegisterExists'); checkDataUseRegisterExistsStub.onCall(0).returns(false); checkDataUseRegisterExistsStub.onCall(1).returns(true); + const getLinkedDatasetsStub = sinon.stub(dataUseRegisterUtil, 'getLinkedDatasets'); + getLinkedDatasetsStub.returns({ linkedDatasets: [], namedDatasets: [] }); // Act const result = await dataUseRegisterService.filterExistingDataUseRegisters(dataUseRegisterUploads); diff --git a/src/resources/dataUseRegister/__tests__/dataUseRegister.util.test.js b/src/resources/dataUseRegister/__tests__/dataUseRegister.util.test.js index 49b2aaf1..843ac425 100644 --- a/src/resources/dataUseRegister/__tests__/dataUseRegister.util.test.js +++ b/src/resources/dataUseRegister/__tests__/dataUseRegister.util.test.js @@ -1,4 +1,5 @@ import sinon from 'sinon'; +import { fn as momentProto } from 'moment'; import { cloneDeep } from 'lodash'; import dataUseRegisterUtil from '../dataUseRegister.util'; @@ -8,16 +9,19 @@ import { nonGatewayDatasetNames, gatewayDatasetNames, expectedGatewayDatasets, + expectedGatewayDatasetsReturned, nonGatewayApplicantNames, gatewayApplicantNames, expectedGatewayApplicants, applications, - authors + authors, } from '../__mocks__/dataUseRegisters'; import { uploader } from '../__mocks__/dataUseRegisterUsers'; import * as userRepository from '../../user/user.repository'; import { datasetService } from '../../dataset/dependency'; +const sandbox = sinon.createSandbox(); + describe('DataUseRegisterUtil', function () { beforeAll(function () { process.env.homeURL = 'http://localhost:3000'; @@ -25,6 +29,10 @@ describe('DataUseRegisterUtil', function () { describe('getLinkedDatasets', function () { it('returns the names of the datasets that could not be found on the Gateway as named datasets', async function () { + // Arrange + const getDatasetsByNameStub = sinon.stub(datasetService, 'getDatasetsByName'); + getDatasetsByNameStub.returns(); + // Act const result = await dataUseRegisterUtil.getLinkedDatasets(nonGatewayDatasetNames); @@ -41,7 +49,7 @@ describe('DataUseRegisterUtil', function () { // Assert expect(getDatasetsByPidsStub.calledOnce).toBe(true); - expect(result).toEqual({ linkedDatasets: expectedGatewayDatasets, namedDatasets: [] }); + expect(result).toEqual({ linkedDatasets: expectedGatewayDatasetsReturned, namedDatasets: [] }); }); }); @@ -63,18 +71,24 @@ describe('DataUseRegisterUtil', function () { // Assert expect(getUsersByIdsStub.calledOnce).toBe(true); - expect(result).toEqual({ gatewayApplicants: expectedGatewayApplicants, nonGatewayApplicants: [] }); + expect(result.gatewayApplicants[0]._id).toEqual(expectedGatewayApplicants[0]); + expect(result.gatewayApplicants[1]._id).toEqual(expectedGatewayApplicants[1]); + expect(result.nonGatewayApplicants).toEqual([]); }); }); - describe('buildRelatedDatasets', function () { + describe('buildRelatedObjects', function () { + beforeEach(() => { + sandbox.stub(momentProto, 'format'); + momentProto.format.withArgs('DD MMM YYYY').returns('24 Sept 2021'); + }); + it('filters out data uses that are found to already exist in the database', async function () { // Arrange const data = cloneDeep(datasets); - sinon.stub(Date, 'now').returns('2021-24-09T11:01:58.135Z'); // Act - const result = dataUseRegisterUtil.buildRelatedDatasets(uploader, data); + const result = dataUseRegisterUtil.buildRelatedObjects(uploader, 'dataset', data); // Assert expect(result.length).toBe(data.length); @@ -83,6 +97,7 @@ describe('DataUseRegisterUtil', function () { afterEach(function () { sinon.restore(); + sandbox.restore(); }); }); diff --git a/src/resources/dataUseRegister/dataUseRegister.repository.js b/src/resources/dataUseRegister/dataUseRegister.repository.js index 695211c1..515267f3 100644 --- a/src/resources/dataUseRegister/dataUseRegister.repository.js +++ b/src/resources/dataUseRegister/dataUseRegister.repository.js @@ -1,7 +1,7 @@ import Repository from '../base/repository'; import { DataUseRegister } from './dataUseRegister.model'; import { isNil } from 'lodash'; -import { filtersService } from '../filters/dependency'; +import { filtersService } from '../filters/filters.service'; export default class DataUseRegisterRepository extends Repository { constructor() { From 8c821d6fb355d0943290fd363c63b16861ecef1e Mon Sep 17 00:00:00 2001 From: Paul McCafferty <> Date: Tue, 30 Nov 2021 15:54:50 +0000 Subject: [PATCH 113/116] Update to fix broken import --- src/resources/dataUseRegister/dataUseRegister.controller.js | 2 +- src/resources/dataUseRegister/dataUseRegister.repository.js | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/resources/dataUseRegister/dataUseRegister.controller.js b/src/resources/dataUseRegister/dataUseRegister.controller.js index 07a0f99d..abd31877 100644 --- a/src/resources/dataUseRegister/dataUseRegister.controller.js +++ b/src/resources/dataUseRegister/dataUseRegister.controller.js @@ -401,7 +401,7 @@ export default class DataUseRegisterController extends Controller { }) .lean(); - const publisherTeam = await TeamModel.findOne({ _id: publisher }) + const publisherTeam = await TeamModel.findOne({ _id: { $eq: publisher } }) .populate({ path: 'publisher', }) diff --git a/src/resources/dataUseRegister/dataUseRegister.repository.js b/src/resources/dataUseRegister/dataUseRegister.repository.js index 515267f3..0b46e4a9 100644 --- a/src/resources/dataUseRegister/dataUseRegister.repository.js +++ b/src/resources/dataUseRegister/dataUseRegister.repository.js @@ -1,7 +1,7 @@ import Repository from '../base/repository'; import { DataUseRegister } from './dataUseRegister.model'; import { isNil } from 'lodash'; -import { filtersService } from '../filters/filters.service'; +import FiltersService from '../filters/filters.service'; export default class DataUseRegisterRepository extends Repository { constructor() { @@ -85,7 +85,7 @@ export default class DataUseRegisterRepository extends Repository { body.updatedon = Date.now(); body.lastActivity = Date.now(); const updatedBody = await this.update(id, body); - filtersService.optimiseFilters('dataUseRegister'); + FiltersService.optimiseFilters('dataUseRegister'); return updatedBody; } From e6362302aa1cde3ac7ed6b674f4b42cc309992d5 Mon Sep 17 00:00:00 2001 From: Paul McCafferty <> Date: Mon, 6 Dec 2021 11:10:53 +0000 Subject: [PATCH 114/116] Updates to v2 --- ...-remove_projects_from_related_resources.js | 32 ++++ .../dataUseRegister.controller.js | 173 ++++++++---------- .../dataUseRegister/dataUseRegister.model.js | 6 + .../dataUseRegister.repository.js | 4 +- .../dataUseRegister/dataUseRegister.route.js | 38 +--- .../dataUseRegister.service.js | 96 ++++++++-- .../dataUseRegister/dataUseRegister.util.js | 19 +- src/resources/filters/filters.mapper.js | 4 +- src/resources/search/search.repository.js | 3 + 9 files changed, 232 insertions(+), 143 deletions(-) create mode 100644 migrations/1638716002879-remove_projects_from_related_resources.js diff --git a/migrations/1638716002879-remove_projects_from_related_resources.js b/migrations/1638716002879-remove_projects_from_related_resources.js new file mode 100644 index 00000000..1972a3ea --- /dev/null +++ b/migrations/1638716002879-remove_projects_from_related_resources.js @@ -0,0 +1,32 @@ +import { Data } from '../src/resources/tool/data.model'; +import { Collections } from '../src/resources/collections/collections.model'; +import { Course } from '../src/resources/course/course.model'; + +/** + * Make any changes you need to make to the database here + */ +async function up() { + //Remove projects that are in related resources for tools and papers + await Data.update({ 'relatedObjects.objectType': 'project' }, { $pull: { relatedObjects: { objectType: 'project' } } }, { multi: true }); + //Remove projects that are in related resources for collections + await Collections.update( + { 'relatedObjects.objectType': 'project' }, + { $pull: { relatedObjects: { objectType: 'project' } } }, + { multi: true } + ); + //Remove projects that are in related resources for courses + await Course.update( + { 'relatedObjects.objectType': 'project' }, + { $pull: { relatedObjects: { objectType: 'project' } } }, + { multi: true } + ); +} + +/** + * Make any changes that UNDO the up function side effects here (if possible) + */ +async function down() { + // Write migration here +} + +module.exports = { up, down }; diff --git a/src/resources/dataUseRegister/dataUseRegister.controller.js b/src/resources/dataUseRegister/dataUseRegister.controller.js index f852cdb1..daa8082e 100644 --- a/src/resources/dataUseRegister/dataUseRegister.controller.js +++ b/src/resources/dataUseRegister/dataUseRegister.controller.js @@ -1,3 +1,4 @@ +/* eslint-disable class-methods-use-this */ import Mongoose from 'mongoose'; import Controller from '../base/controller'; import { logger } from '../utilities/logger'; @@ -9,7 +10,7 @@ import emailGenerator from '../utilities/emailGenerator.util'; import { getObjectFilters } from '../search/search.repository'; import { DataUseRegister } from '../dataUseRegister/dataUseRegister.model'; -import { isEmpty } from 'lodash'; +import { isEmpty, isUndefined } from 'lodash'; const logCategory = 'dataUseRegister'; @@ -24,6 +25,9 @@ export default class DataUseRegisterController extends Controller { try { // Extract id parameter from query string const { id } = req.params; + const isEdit = req.query.isEdit || false; + if (req.query.isEdit) delete req.query.isEdit; + // If no id provided, it is a bad request if (!id) { return res.status(400).json({ @@ -44,73 +48,6 @@ export default class DataUseRegisterController extends Controller { }; const dataUseRegister = await this.dataUseRegisterService.getDataUseRegister(id, req.query, options); - // Reverse look up - var query = Data.aggregate([ - { $match: { id: parseInt(req.params.id) } }, - { - $lookup: { - from: 'tools', - localField: 'creator', - foreignField: 'id', - as: 'creator', - }, - }, - { - $lookup: { - from: 'tools', - let: { - pid: '$pid', - }, - pipeline: [ - { - $match: { - $expr: { - $and: [{ $eq: ['$gatewayDatasets', '$$pid'] }], - }, - }, - }, - { $project: { pid: 1, name: 1 } }, - ], - as: 'gatewayDatasets2', - }, - }, - ]); - query.exec((err, data) => { - if (data.length > 0) { - /* var p = Data.aggregate([ - { - $match: { - $and: [{ relatedObjects: { $elemMatch: { objectId: req.params.id } } }], - }, - }, - ]); - p.exec((err, relatedData) => { - relatedData.forEach(dat => { - dat.relatedObjects.forEach(x => { - if (x.objectId === req.params.id && dat.id !== req.params.id) { - let relatedObject = { - objectId: dat.id, - reason: x.reason, - objectType: dat.type, - user: x.user, - updated: x.updated, - }; - data[0].relatedObjects = [relatedObject, ...(data[0].relatedObjects || [])]; - } - }); - }); - - if (err) return res.json({ success: false, error: err }); - - return res.json({ - success: true, - data: data, - }); - }); */ - } else { - //return res.status(404).send(`Data Use Register not found for Id: ${escape(id)}`); - } - }); // Return if no dataUseRegister found if (!dataUseRegister) { return res.status(404).json({ @@ -118,10 +55,33 @@ export default class DataUseRegisterController extends Controller { message: 'A dataUseRegister could not be found with the provided id', }); } - // Return the dataUseRegister - return res.status(200).json({ - success: true, - ...dataUseRegister, + + // Reverse look up + var p = Data.aggregate([{ $match: { $and: [{ relatedObjects: { $elemMatch: { objectId: id } } }] } }]); + p.exec((err, relatedData) => { + if (!isEdit) { + relatedData.forEach(dat => { + dat.relatedObjects.forEach(x => { + if (x.objectId === id && dat.id !== id) { + if (typeof dataUseRegister.relatedObjects === 'undefined') dataUseRegister.relatedObjects = []; + dataUseRegister.relatedObjects.push({ + objectId: dat.id, + reason: x.reason, + objectType: dat.type, + user: x.user, + updated: x.updated, + }); + } + }); + }); + } + if (err) return res.json({ success: false, error: err }); + + // Return the dataUseRegister + return res.status(200).json({ + success: true, + ...dataUseRegister, + }); }); } catch (err) { // Return error response if something goes wrong @@ -140,27 +100,38 @@ export default class DataUseRegisterController extends Controller { let query = ''; - if (team === 'user') { - delete req.query.team; - query = { ...req.query, gatewayApplicants: requestingUser._id }; - } else if (team === 'admin') { - delete req.query.team; - query = { ...req.query, activeflag: constants.dataUseRegisterStatus.INREVIEW }; - } else if (team !== 'user' && team !== 'admin') { - delete req.query.team; - query = { publisher: new Mongoose.Types.ObjectId(team) }; + if (!isUndefined(team)) { + if (team === 'user') { + delete req.query.team; + query = { ...req.query, gatewayApplicants: requestingUser._id }; + } else if (team === 'admin') { + delete req.query.team; + query = { ...req.query, activeflag: constants.dataUseRegisterStatus.INREVIEW }; + } else if (team !== 'user' && team !== 'admin') { + delete req.query.team; + query = { publisher: new Mongoose.Types.ObjectId(team) }; + } + + const dataUseRegisters = await this.dataUseRegisterService + .getDataUseRegisters({ $and: [query] }, { aggregate: true }) + .catch(err => { + logger.logError(err, logCategory); + }); + // Return the dataUseRegisters + return res.status(200).json({ + success: true, + data: dataUseRegisters, + }); } else { - query = req.query; + const dataUseRegisters = await this.dataUseRegisterService.getDataUseRegisters(req.query).catch(err => { + logger.logError(err, logCategory); + }); + // Return the dataUseRegisters + return res.status(200).json({ + success: true, + data: dataUseRegisters, + }); } - - const dataUseRegisters = await this.dataUseRegisterService.getDataUseRegisters({ $and: [query] }, { aggregate: true }).catch(err => { - logger.logError(err, logCategory); - }); - // Return the dataUseRegisters - return res.status(200).json({ - success: true, - data: dataUseRegisters, - }); } catch (err) { // Return error response if something goes wrong logger.logError(err, logCategory); @@ -177,15 +148,16 @@ export default class DataUseRegisterController extends Controller { const requestingUser = req.user; const { rejectionReason } = req.body; - const options = { lean: true, populate: 'user' }; + const options = { lean: true, populate: 'applicantDetails' }; const dataUseRegister = await this.dataUseRegisterService.getDataUseRegister(id, {}, options); - const updateObj = this.dataUseRegisterService.buildUpdateObject(dataUseRegister, req.body); + const updateObj = await this.dataUseRegisterService.buildUpdateObject(dataUseRegister, req.body, requestingUser); if (isEmpty(updateObj)) { return res.status(200).json({ success: true, }); } + this.dataUseRegisterService.updateDataUseRegister(dataUseRegister._id, updateObj).catch(err => { logger.logError(err, logCategory); }); @@ -448,4 +420,19 @@ export default class DataUseRegisterController extends Controller { } } } + + updateDataUseRegisterCounter(req, res) { + try { + const { id, counter } = req.body; + this.dataUseRegisterService.updateDataUseRegister(id, { counter }); + return res.status(200).json({ success: true }); + } catch (err) { + // Return error response if something goes wrong + logger.logError(err, logCategory); + return res.status(500).json({ + success: false, + message: 'A server error occurred, please try again', + }); + } + } } diff --git a/src/resources/dataUseRegister/dataUseRegister.model.js b/src/resources/dataUseRegister/dataUseRegister.model.js index d885669f..0db2f02e 100644 --- a/src/resources/dataUseRegister/dataUseRegister.model.js +++ b/src/resources/dataUseRegister/dataUseRegister.model.js @@ -96,6 +96,12 @@ dataUseRegisterSchema.virtual('publisherDetails', { justOne: true, }); +dataUseRegisterSchema.virtual('applicantDetails', { + ref: 'User', + foreignField: '_id', + localField: 'gatewayApplicants', +}); + dataUseRegisterSchema.virtual('gatewayDatasetsInfo', { ref: 'Data', foreignField: 'pid', diff --git a/src/resources/dataUseRegister/dataUseRegister.repository.js b/src/resources/dataUseRegister/dataUseRegister.repository.js index 0b46e4a9..695211c1 100644 --- a/src/resources/dataUseRegister/dataUseRegister.repository.js +++ b/src/resources/dataUseRegister/dataUseRegister.repository.js @@ -1,7 +1,7 @@ import Repository from '../base/repository'; import { DataUseRegister } from './dataUseRegister.model'; import { isNil } from 'lodash'; -import FiltersService from '../filters/filters.service'; +import { filtersService } from '../filters/dependency'; export default class DataUseRegisterRepository extends Repository { constructor() { @@ -85,7 +85,7 @@ export default class DataUseRegisterRepository extends Repository { body.updatedon = Date.now(); body.lastActivity = Date.now(); const updatedBody = await this.update(id, body); - FiltersService.optimiseFilters('dataUseRegister'); + filtersService.optimiseFilters('dataUseRegister'); return updatedBody; } diff --git a/src/resources/dataUseRegister/dataUseRegister.route.js b/src/resources/dataUseRegister/dataUseRegister.route.js index 641cb3a7..5b63ad54 100644 --- a/src/resources/dataUseRegister/dataUseRegister.route.js +++ b/src/resources/dataUseRegister/dataUseRegister.route.js @@ -71,35 +71,6 @@ const validateUploadRequest = (req, res, next) => { next(); }; -/* const validateViewRequest = (req, res, next) => { - const { team } = req.query; - - if (!team) { - return res.status(400).json({ - success: false, - message: 'You must provide a team parameter', - }); - } - - next(); -}; */ - -const authorizeView = async (req, res, next) => { - const requestingUser = req.user; - const { team } = req.query; - - const authorised = team === 'user' || isUserDataUseAdmin(requestingUser) || isUserMemberOfTeam(requestingUser, team); - - if (!authorised) { - return res.status(401).json({ - success: false, - message: 'You are not authorised to perform this action', - }); - } - - next(); -}; - const authorizeUpdate = async (req, res, next) => { const requestingUser = req.user; const { id } = req.params; @@ -173,12 +144,17 @@ router.get('/:id', logger.logRequestMiddleware({ logCategory, action: 'Viewed da router.get( '/', passport.authenticate('jwt'), - /* validateViewRequest, */ - authorizeView, logger.logRequestMiddleware({ logCategory, action: 'Viewed dataUseRegisters data' }), (req, res) => dataUseRegisterController.getDataUseRegisters(req, res) ); +// @route PATCH /api/v2/data-use-registers/counter +// @desc Updates the data use register counter for page views +// @access Public +router.patch('/counter', logger.logRequestMiddleware({ logCategory, action: 'Data use counter update' }), (req, res) => + dataUseRegisterController.updateDataUseRegisterCounter(req, res) +); + // @route PATCH /api/v2/data-use-registers/id // @desc Update the content of the data user register based on dataUseRegister ID provided // @access Public diff --git a/src/resources/dataUseRegister/dataUseRegister.service.js b/src/resources/dataUseRegister/dataUseRegister.service.js index 6c4964cf..982be6c5 100644 --- a/src/resources/dataUseRegister/dataUseRegister.service.js +++ b/src/resources/dataUseRegister/dataUseRegister.service.js @@ -1,3 +1,4 @@ +/* eslint-disable class-methods-use-this */ import dataUseRegisterUtil from './dataUseRegister.util'; import DataUseRegister from './dataUseRegister.entity'; import constants from '../utilities/constants.util'; @@ -326,7 +327,7 @@ export default class DataUseRegisterService { return relatedDataUseRegisters; } - buildUpdateObject(dataUseRegister, dataUseRegisterPayload) { + async buildUpdateObject(dataUseRegister, dataUseRegisterPayload, user) { let updateObj = {}; const { @@ -369,11 +370,85 @@ export default class DataUseRegisterService { accessDate, accessType, privacyEnhancements, - gatewayOutputsTools, - gatewayOutputsPapers, + gatewayOutputs, nonGatewayOutputs, } = dataUseRegisterPayload; + const gatewayDatasetPids = await dataUseRegisterUtil.getDatasetsByPids(gatewayDatasets); + const gatewayApplicantIDs = await dataUseRegisterUtil.getAppplicantByIds(gatewayApplicants); + const { gatewayToolIDs, gatewayPaperIDs } = await dataUseRegisterUtil.getSafeOutputsByIds(gatewayOutputs || []); + + let gatewayApplicantIDsList = []; + gatewayApplicantIDs.forEach(applicant => { + gatewayApplicantIDsList.push(applicant._id); + }); + if (!isUndefined(gatewayApplicants) && !isEqual(gatewayApplicantIDsList, dataUseRegister.gatewayApplicants)) + updateObj.gatewayApplicants = gatewayApplicantIDsList; + + let gatewayOutputsToolIDsList = [], + gatewayOutputsToolIDsListRelatedResource = []; + gatewayToolIDs.forEach(tool => { + gatewayOutputsToolIDsList.push(tool.id); + gatewayOutputsToolIDsListRelatedResource.push({ id: tool.id.toString() }); + }); + if (!isUndefined(gatewayOutputs) && !isEqual(gatewayOutputsToolIDsList, dataUseRegister.gatewayOutputsTools)) + updateObj.gatewayOutputsTools = gatewayOutputsToolIDsList; + + let gatewayOutputsPaperIDsList = [], + gatewayOutputsPaperIDsListRelatedResource = []; + gatewayPaperIDs.forEach(paper => { + gatewayOutputsPaperIDsList.push(paper.id); + gatewayOutputsPaperIDsListRelatedResource.push({ id: paper.id.toString() }); + }); + if (!isUndefined(gatewayOutputs) && !isEqual(gatewayOutputsPaperIDsList, dataUseRegister.gatewayOutputsPapers)) + updateObj.gatewayOutputsPapers = gatewayOutputsPaperIDsList; + + let gatewayDatasetPidsListRelatedResource = []; + gatewayDatasetPids.forEach(dataset => { + gatewayDatasetPidsListRelatedResource.push({ id: dataset.datasetid, pid: dataset.pid }); + }); + + let automaticRelatedResources = [ + ...dataUseRegisterUtil.buildRelatedObjects(user, 'dataset', gatewayDatasetPidsListRelatedResource, false, true), + ...dataUseRegisterUtil.buildRelatedObjects(user, 'tool', gatewayOutputsToolIDsListRelatedResource, false, true), + ...dataUseRegisterUtil.buildRelatedObjects(user, 'paper', gatewayOutputsPaperIDsListRelatedResource, false, true), + ]; + + //dataUseRegister.relatedObjects + + //Loop through automaticRelatedResources to see if it exists, if not add to another array + + let newAutomaticRelatedResources = []; + automaticRelatedResources.forEach(automaticResource => { + if (!dataUseRegister.relatedObjects.find(resource => resource.objectId === automaticResource.objectId)) { + newAutomaticRelatedResources.push(automaticResource); + } + }); + + let newManualRelatedResources = []; + relatedObjects.forEach(manualResource => { + if (!dataUseRegister.relatedObjects.find(resource => resource.objectId === manualResource.objectId)) { + if (!manualResource.isLocked) newManualRelatedResources.push(manualResource); + } + }); + + let relatedResourcesWithRemovedOldAutomaticEntries = []; + dataUseRegister.relatedObjects.forEach(resource => { + if (resource.isLocked && automaticRelatedResources.find(automaticResource => automaticResource.objectId === resource.objectId)) { + relatedResourcesWithRemovedOldAutomaticEntries.push(resource); + } else if (!resource.isLocked) { + relatedResourcesWithRemovedOldAutomaticEntries.push(resource); + } + }); + + //relatedObjects + + updateObj.relatedObjects = [ + ...relatedResourcesWithRemovedOldAutomaticEntries, + ...newAutomaticRelatedResources, + ...newManualRelatedResources, + ]; + const fundersAndSponsorsList = fundersAndSponsors && fundersAndSponsors @@ -382,6 +457,8 @@ export default class DataUseRegisterService { .map(el => { if (!isEmpty(el)) return el.trim(); }); + if (!isEmpty(fundersAndSponsorsList) && !isEqual(fundersAndSponsorsList, dataUseRegister.fundersAndSponsors)) + updateObj.fundersAndSponsors = fundersAndSponsorsList; const otherApprovalCommitteesList = otherApprovalCommittees && @@ -391,13 +468,14 @@ export default class DataUseRegisterService { .map(el => { if (!isEmpty(el)) return el.trim(); }); + if (!isEmpty(otherApprovalCommitteesList) && !isEqual(otherApprovalCommitteesList, dataUseRegister.otherApprovalCommittees)) + updateObj.otherApprovalCommittees = otherApprovalCommitteesList; if (!isUndefined(activeflag) && !isEqual(activeflag, dataUseRegister.activeflag)) updateObj.activeflag = activeflag; if (!isUndefined(rejectionReason) && !isEqual(rejectionReason, dataUseRegister.rejectionReason)) updateObj.rejectionReason = rejectionReason; if (!isUndefined(discourseTopicId) && !isEqual(discourseTopicId, dataUseRegister.discourseTopicId)) updateObj.discourseTopicId = discourseTopicId; - if (!isUndefined(relatedObjects) && !isEqual(relatedObjects, dataUseRegister.relatedObjects)) updateObj.relatedObjects = relatedObjects; if (!isUndefined(keywords) && !isEqual(keywords, dataUseRegister.keywords)) updateObj.keywords = keywords; if (!isUndefined(projectTitle) && !isEqual(projectTitle, dataUseRegister.projectTitle)) updateObj.projectTitle = projectTitle; if (!isUndefined(projectId) && !isEqual(projectId, dataUseRegister.projectId)) updateObj.projectId = projectId; @@ -413,13 +491,9 @@ export default class DataUseRegisterService { if (!isUndefined(organisationId) && !isEqual(organisationId, dataUseRegister.organisationId)) updateObj.organisationId = organisationId; if (!isUndefined(organisationSector) && !isEqual(organisationSector, dataUseRegister.organisationSector)) updateObj.organisationSector = organisationSector; - if (!isUndefined(gatewayApplicants) && !isEqual(gatewayApplicants, dataUseRegister.gatewayApplicants)) - updateObj.gatewayApplicants = gatewayApplicants; if (!isUndefined(nonGatewayApplicants) && !isEqual(nonGatewayApplicants, dataUseRegister.nonGatewayApplicants)) updateObj.nonGatewayApplicants = nonGatewayApplicants; if (!isUndefined(applicantId) && !isEqual(applicantId, dataUseRegister.applicantId)) updateObj.applicantId = applicantId; - if (!isEmpty(fundersAndSponsorsList) && !isEqual(fundersAndSponsorsList, dataUseRegister.fundersAndSponsors)) - updateObj.fundersAndSponsors = fundersAndSponsorsList; if (!isUndefined(accreditedResearcherStatus) && !isEqual(accreditedResearcherStatus, dataUseRegister.accreditedResearcherStatus)) updateObj.accreditedResearcherStatus = accreditedResearcherStatus; if (!isUndefined(sublicenceArrangements) && !isEqual(sublicenceArrangements, dataUseRegister.sublicenceArrangements)) @@ -431,8 +505,6 @@ export default class DataUseRegisterService { updateObj.requestCategoryType = requestCategoryType; if (!isUndefined(technicalSummary) && !isEqual(technicalSummary, dataUseRegister.technicalSummary)) updateObj.technicalSummary = technicalSummary; - if (!isEmpty(otherApprovalCommitteesList) && !isEqual(otherApprovalCommitteesList, dataUseRegister.otherApprovalCommittees)) - updateObj.otherApprovalCommittees = otherApprovalCommitteesList; if ( !isEmpty(projectStartDate) && !isEqual(moment(projectStartDate).format('YYYY-MM-DD'), moment(dataUseRegister.projectStartDate).format('YYYY-MM-DD')) @@ -469,10 +541,6 @@ export default class DataUseRegisterService { if (!isUndefined(accessType) && !isEqual(accessType, dataUseRegister.accessType)) updateObj.accessType = accessType; if (!isUndefined(privacyEnhancements) && !isEqual(privacyEnhancements, dataUseRegister.privacyEnhancements)) updateObj.privacyEnhancements = privacyEnhancements; - if (!isUndefined(gatewayOutputsTools) && !isEqual(gatewayOutputsTools, dataUseRegister.gatewayOutputsTools)) - updateObj.gatewayOutputsTools = gatewayOutputsTools; - if (!isUndefined(gatewayOutputsPapers) && !isEqual(gatewayOutputsPapers, dataUseRegister.gatewayOutputsPapers)) - updateObj.gatewayOutputsPapers = gatewayOutputsPapers; if (!isUndefined(nonGatewayOutputs) && !isEqual(nonGatewayOutputs, dataUseRegister.nonGatewayOutputs)) updateObj.nonGatewayOutputs = nonGatewayOutputs; diff --git a/src/resources/dataUseRegister/dataUseRegister.util.js b/src/resources/dataUseRegister/dataUseRegister.util.js index b6254540..19891239 100644 --- a/src/resources/dataUseRegister/dataUseRegister.util.js +++ b/src/resources/dataUseRegister/dataUseRegister.util.js @@ -261,7 +261,7 @@ const getLinkedOutputs = async (outputs = []) => { * @param {Array} objects An array of objects containing the necessary properties to assemble a related object record reference */ -const buildRelatedObjects = (creatorUser, type, objects = [], manualUpload = true) => { +const buildRelatedObjects = (creatorUser, type, objects = [], manualUpload = true, addedViaEdit = false) => { const { firstname, lastname } = creatorUser; return objects.map(object => { const { id: objectId, pid } = object; @@ -274,6 +274,8 @@ const buildRelatedObjects = (creatorUser, type, objects = [], manualUpload = tru isLocked: true, reason: manualUpload ? `This ${type} was added automatically during the manual upload of this data use register` + : addedViaEdit + ? `This ${type} was added via an edit of this data use register` : `This ${type} was added automatically from an approved data access request`, }; }); @@ -321,6 +323,18 @@ const extractFundersAndSponsors = (applicationQuestionAnswers = {}) => { .map(key => applicationQuestionAnswers[key]); }; +const getDatasetsByPids = async datasetPids => { + return await datasetService.getDatasetsByPids(datasetPids); +}; + +const getAppplicantByIds = async applicantIds => { + return await getUsersByIds(applicantIds); +}; + +const getSafeOutputsByIds = async outputIds => { + return { gatewayToolIDs: await toolService.getToolsByIds(outputIds), gatewayPaperIDs: await paperService.getPapersByIds(outputIds) }; +}; + export default { buildDataUseRegisters, getLinkedDatasets, @@ -329,4 +343,7 @@ export default { buildRelatedObjects, extractFormApplicants, extractFundersAndSponsors, + getDatasetsByPids, + getAppplicantByIds, + getSafeOutputsByIds, }; diff --git a/src/resources/filters/filters.mapper.js b/src/resources/filters/filters.mapper.js index 0863e777..478672ff 100644 --- a/src/resources/filters/filters.mapper.js +++ b/src/resources/filters/filters.mapper.js @@ -916,7 +916,7 @@ export const dataUseRegisterFilters = [ highlighted: [], beta: false, }, - /* { + { id: 5, label: 'Keywords', key: 'keywords', @@ -930,5 +930,5 @@ export const dataUseRegisterFilters = [ filters: [], highlighted: [], beta: false, - }, */ + }, ]; diff --git a/src/resources/search/search.repository.js b/src/resources/search/search.repository.js index cd063f39..028b2cb0 100644 --- a/src/resources/search/search.repository.js +++ b/src/resources/search/search.repository.js @@ -192,6 +192,7 @@ export async function getObjectResult(type, searchAll, searchQuery, startIndex, counter: 1, type: 1, latestUpdate: '$lastActivity', + relatedresources: { $cond: { if: { $isArray: '$relatedObjects' }, then: { $size: '$relatedObjects' }, else: 0 } }, }, }, ]; @@ -907,6 +908,8 @@ export const getFilter = async (searchString, type, field, isArray, activeFilter collection = Course; } else if (type === 'collection') { collection = Collections; + } else if (type === 'datause') { + collection = DataUseRegister; } let q = '', p = ''; From 82198245bcfac8d966a5eb25572922f294c09995 Mon Sep 17 00:00:00 2001 From: Paul McCafferty <> Date: Mon, 6 Dec 2021 13:30:13 +0000 Subject: [PATCH 115/116] Moving filter optimise as its breaking a test --- src/resources/dataUseRegister/dataUseRegister.controller.js | 5 ++++- src/resources/dataUseRegister/dataUseRegister.repository.js | 2 -- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/resources/dataUseRegister/dataUseRegister.controller.js b/src/resources/dataUseRegister/dataUseRegister.controller.js index daa8082e..93701e17 100644 --- a/src/resources/dataUseRegister/dataUseRegister.controller.js +++ b/src/resources/dataUseRegister/dataUseRegister.controller.js @@ -8,6 +8,7 @@ import { TeamModel } from '../team/team.model'; import teamController from '../team/team.controller'; import emailGenerator from '../utilities/emailGenerator.util'; import { getObjectFilters } from '../search/search.repository'; +import { filtersService } from '../filters/dependency'; import { DataUseRegister } from '../dataUseRegister/dataUseRegister.model'; import { isEmpty, isUndefined } from 'lodash'; @@ -158,10 +159,12 @@ export default class DataUseRegisterController extends Controller { }); } - this.dataUseRegisterService.updateDataUseRegister(dataUseRegister._id, updateObj).catch(err => { + await this.dataUseRegisterService.updateDataUseRegister(dataUseRegister._id, updateObj).catch(err => { logger.logError(err, logCategory); }); + filtersService.optimiseFilters('dataUseRegister'); + const isDataUseRegisterApproved = updateObj.activeflag && updateObj.activeflag === constants.dataUseRegisterStatus.ACTIVE && diff --git a/src/resources/dataUseRegister/dataUseRegister.repository.js b/src/resources/dataUseRegister/dataUseRegister.repository.js index 695211c1..ca74ed61 100644 --- a/src/resources/dataUseRegister/dataUseRegister.repository.js +++ b/src/resources/dataUseRegister/dataUseRegister.repository.js @@ -1,7 +1,6 @@ import Repository from '../base/repository'; import { DataUseRegister } from './dataUseRegister.model'; import { isNil } from 'lodash'; -import { filtersService } from '../filters/dependency'; export default class DataUseRegisterRepository extends Repository { constructor() { @@ -85,7 +84,6 @@ export default class DataUseRegisterRepository extends Repository { body.updatedon = Date.now(); body.lastActivity = Date.now(); const updatedBody = await this.update(id, body); - filtersService.optimiseFilters('dataUseRegister'); return updatedBody; } From d4ce9e487678da90aa0997b237fd9b0b4293475f Mon Sep 17 00:00:00 2001 From: Paul McCafferty <> Date: Tue, 7 Dec 2021 12:58:07 +0000 Subject: [PATCH 116/116] Fix for issue were user is unable to assign a workflow --- src/resources/workflow/workflow.repository.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/resources/workflow/workflow.repository.js b/src/resources/workflow/workflow.repository.js index 508a34a1..0c81be16 100644 --- a/src/resources/workflow/workflow.repository.js +++ b/src/resources/workflow/workflow.repository.js @@ -54,7 +54,7 @@ export default class WorkflowRepository extends Repository { async assignWorkflowToApplication(accessRecord, workflowId) { // Retrieve workflow using ID from database - const workflow = await WorkflowRepository.getWorkflowById(workflowId, { lean: false }); + const workflow = await this.getWorkflowById(workflowId, { lean: false }); if (!workflow) { throw new Error('Workflow could not be found'); }