diff --git a/migrations/1631621029553-update_publishers_uses5safes.js b/migrations/1631621029553-update_publishers_uses5safes.js index 32a7742a..fa11e9a7 100644 --- a/migrations/1631621029553-update_publishers_uses5safes.js +++ b/migrations/1631621029553-update_publishers_uses5safes.js @@ -31,3 +31,4 @@ async function up() { async function down() {} module.exports = { up, down }; + diff --git a/src/config/account.js b/src/config/account.js index 01c5be29..78ac99f9 100755 --- a/src/config/account.js +++ b/src/config/account.js @@ -1,7 +1,7 @@ import { getUserByUserId } from '../resources/user/user.repository'; import ga4ghUtils from '../resources/utilities/ga4gh.utils'; import { to } from 'await-to-js'; -import _ from 'lodash'; +import { isNil } from 'lodash'; const store = new Map(); const logins = new Map(); @@ -29,8 +29,8 @@ class Account { sub: this.accountId, // it is essential to always return a sub claim }; - let [, user] = await to(getUserByUserId(parseInt(this.accountId))); - if (!_.isNil(user)) { + let [err, user] = await to(getUserByUserId(parseInt(this.accountId))); + if (!isNil(user)) { if (claimsToSend.includes('profile')) { claim.firstname = user.firstname; claim.lastname = user.lastname; diff --git a/src/config/configuration.js b/src/config/configuration.js index bc83e6fe..a11b3cf0 100755 --- a/src/config/configuration.js +++ b/src/config/configuration.js @@ -27,13 +27,7 @@ export const clients = [ //response_types: ['code id_token'], redirect_uris: process.env.MDWRedirectURI.split(',') || [''], id_token_signed_response_alg: 'HS256', - post_logout_redirect_uris: [ - 'https://hdr.auth.metadata.works/logout', - 'https://hdr.auth.metadata.works/auth/logout', - 'https://hdruk-preprod-auth.metadata.works/auth/logout', - 'http://localhost:8080/logout', - 'http://localhost:8080/auth/logout', - ], + post_logout_redirect_uris: ['https://hdruk-auth.metadata.works/auth/logout'], }, { //BC Platforms diff --git a/src/config/db.js b/src/config/db.js index ccb496d2..f0d67cf1 100644 --- a/src/config/db.js +++ b/src/config/db.js @@ -20,7 +20,7 @@ const connectToDatabase = async () => { autoIndex: false, // Don't build indexes poolSize: 10, // Maintain up to 10 socket connections // If not connected, return errors immediately rather than waiting for reconnect - bufferMaxEntries: 0 + bufferMaxEntries: 0, }); console.log('MongoDB connected...'); diff --git a/src/resources/dataset/dataset.model.js b/src/resources/dataset/dataset.model.js index 84cafd5d..8c00799e 100644 --- a/src/resources/dataset/dataset.model.js +++ b/src/resources/dataset/dataset.model.js @@ -71,7 +71,7 @@ const datasetSchema = new Schema( phenotypes: [], }, datasetv2: {}, - isLatestVersion: Boolean + isLatestVersion: Boolean, }, { timestamps: true, @@ -115,15 +115,15 @@ datasetSchema.virtual('submittedDataAccessRequests', { }); // Pre hook query middleware -datasetSchema.pre('find', function() { - this.where({type: 'dataset'}); +datasetSchema.pre('find', function () { + this.where({ type: 'dataset' }); }); -datasetSchema.pre('findOne', function() { - this.where({type: 'dataset'}); +datasetSchema.pre('findOne', function () { + this.where({ type: 'dataset' }); }); // Load entity class datasetSchema.loadClass(DatasetClass); -export const Dataset = model('Dataset', datasetSchema, 'tools'); \ No newline at end of file +export const Dataset = model('Dataset', datasetSchema, 'tools'); diff --git a/src/resources/dataset/datasetonboarding.controller.js b/src/resources/dataset/datasetonboarding.controller.js index 17c6f2dc..bad8df69 100644 --- a/src/resources/dataset/datasetonboarding.controller.js +++ b/src/resources/dataset/datasetonboarding.controller.js @@ -269,11 +269,7 @@ module.exports = { } else { Data.findByIdAndUpdate( { _id: id }, - { - structuralMetadata, - percentageCompleted, - 'timestamps.updated': Date.now(), - }, + { structuralMetadata, percentageCompleted: data.percentageCompleted, 'timestamps.updated': Date.now() }, { new: true } ).catch(err => { console.error(err); @@ -853,7 +849,7 @@ module.exports = { return res.status(500).json({ success: false, message: 'Bulk upload of metadata failed', error: err.message }); } }, - + //POST api/v1/dataset-onboarding/duplicate/:id duplicateDataset: async (req, res) => { try { @@ -897,3 +893,10 @@ module.exports = { } }, }; + +/* Sentry.addBreadcrumb({ + category: 'Bulk Upload', + message: 'Unable to get metadata quality value ' + err.message, + level: Sentry.Severity.Error, + }); + Sentry.captureException(err); */ diff --git a/src/resources/dataset/v1/dataset.service.js b/src/resources/dataset/v1/dataset.service.js index de60f40b..3d28c2be 100644 --- a/src/resources/dataset/v1/dataset.service.js +++ b/src/resources/dataset/v1/dataset.service.js @@ -6,7 +6,7 @@ import { v4 as uuidv4 } from 'uuid'; import { filtersService } from '../../filters/dependency'; import { PublisherModel } from '../../publisher/publisher.model'; import { metadataCatalogues, validateCatalogueParams } from '../dataset.util'; -import { isEmpty } from 'lodash'; +import { has, isEmpty } from 'lodash'; let metadataQualityList = [], phenotypesList = [], diff --git a/src/resources/stats/stats.router.js b/src/resources/stats/stats.router.js new file mode 100644 index 00000000..fa2430f9 --- /dev/null +++ b/src/resources/stats/stats.router.js @@ -0,0 +1,636 @@ +import express from 'express'; +import { RecordSearchData } from '../search/record.search.model'; +import { Data } from '../tool/data.model'; +import { DataRequestModel } from '../datarequests/datarequests.model'; +import { getHdrDatasetId } from './kpis.router'; +import { Course } from '../course/course.model'; +const router = express.Router(); + +/** + * {get} /stats get some basic high level stats + * + * This will return a JSON document to show high level stats + */ +router.get('', async (req, res) => { + try { + const { query = {} } = req; + + switch (req.query.rank) { + case undefined: + var result; + + //get some dates for query + var lastDay = new Date(); + lastDay.setDate(lastDay.getDate() - 1); + + var lastWeek = new Date(); + lastWeek.setDate(lastWeek.getDate() - 7); + + var lastMonth = new Date(); + lastMonth.setMonth(lastMonth.getMonth() - 1); + + var lastYear = new Date(); + lastYear.setYear(lastYear.getYear() - 1); + + var aggregateQuerySearches = [ + { + $facet: { + lastDay: [ + { $match: { datesearched: { $gt: lastDay } } }, + { + $group: { + _id: 'lastDay', + count: { $sum: 1 }, + }, + }, + ], + lastWeek: [ + { $match: { datesearched: { $gt: lastWeek } } }, + { + $group: { + _id: 'lastWeek', + count: { $sum: 1 }, + }, + }, + ], + lastMonth: [ + { $match: { datesearched: { $gt: lastMonth } } }, + { + $group: { + _id: 'lastMonth', + count: { $sum: 1 }, + }, + }, + ], + lastYear: [ + { $match: { datesearched: { $gt: lastYear } } }, + { + $group: { + _id: 'lastYear', + count: { $sum: 1 }, + }, + }, + ], + }, + }, + ]; + + //set the aggregate queries + var aggregateQueryTypes = [ + { + $match: { + $and: [ + { activeflag: 'active' }, + { 'datasetfields.publisher': { $ne: 'OTHER > HEALTH DATA RESEARCH UK' } }, + { 'datasetfields.publisher': { $ne: 'HDR UK' } }, + ], + }, + }, + { $group: { _id: '$type', count: { $sum: 1 } } }, + ]; + + //set the aggregate queries + const courseQuery = [ + { + $match: { + $and: [{ activeflag: 'active' }], + }, + }, + { $group: { _id: '$type', count: { $sum: 1 } } }, + ]; + + var q = RecordSearchData.aggregate(aggregateQuerySearches); + + var aggregateAccessRequests = [ + { + $match: { + $or: [ + { applicationStatus: 'submitted' }, + { applicationStatus: 'approved' }, + { applicationStatus: 'rejected' }, + { applicationStatus: 'inReview' }, + { applicationStatus: 'approved with conditions' }, + ], + }, + }, + { $project: { datasetIds: 1 } }, + ]; + + var y = DataRequestModel.aggregate(aggregateAccessRequests); + let courseData = Course.aggregate(courseQuery); + + let counts = {}; //hold the type (i.e. tool, person, project, access requests) counts data + await courseData.exec((err, res) => { + if (err) return res.json({ success: false, error: err }); + + let { count = 0 } = res[0]; + counts['course'] = count; + }); + + q.exec((err, dataSearches) => { + if (err) return res.json({ success: false, error: err }); + + var x = Data.aggregate(aggregateQueryTypes); + x.exec((errx, dataTypes) => { + if (errx) return res.json({ success: false, error: errx }); + + for (var i = 0; i < dataTypes.length; i++) { + //format the result in a clear and dynamic way + counts[dataTypes[i]._id] = dataTypes[i].count; + } + + y.exec(async (err, accessRequests) => { + let hdrDatasetID = await getHdrDatasetId(); + let hdrDatasetIds = []; + hdrDatasetID.map(hdrDatasetid => { + hdrDatasetIds.push(hdrDatasetid.datasetid); + }); + let accessRequestsCount = 0; + + if (err) return res.json({ success: false, error: err }); + + accessRequests.map(accessRequest => { + if (accessRequest.datasetIds && accessRequest.datasetIds.length > 0) { + accessRequest.datasetIds.map(datasetid => { + if (!hdrDatasetIds.includes(datasetid)) { + accessRequestsCount++; + } + }); + } + + counts['accessRequests'] = accessRequestsCount; + }); + + if (typeof dataSearches[0].lastDay[0] === 'undefined') { + dataSearches[0].lastDay[0] = { count: 0 }; + } + if (typeof dataSearches[0].lastWeek[0] === 'undefined') { + dataSearches[0].lastWeek[0] = { count: 0 }; + } + if (typeof dataSearches[0].lastMonth[0] === 'undefined') { + dataSearches[0].lastMonth[0] = { count: 0 }; + } + if (typeof dataSearches[0].lastYear[0] === 'undefined') { + dataSearches[0].lastYear[0] = { count: 0 }; + } + + result = res.json({ + success: true, + data: { + typecounts: counts, + daycounts: { + day: dataSearches[0].lastDay[0].count, + week: dataSearches[0].lastWeek[0].count, + month: dataSearches[0].lastMonth[0].count, + year: dataSearches[0].lastYear[0].count, + }, + }, + }); + }); + }); + }); + + return result; + break; + + case 'recent': + var q = RecordSearchData.aggregate([ + { $match: { $or: [{ 'returned.tool': { $gt: 0 } }, { 'returned.project': { $gt: 0 } }, { 'returned.person': { $gt: 0 } }] } }, + { + $group: { + _id: { $toLower: '$searched' }, + count: { $sum: 1 }, + returned: { $first: '$returned' }, + }, + }, + { $sort: { datesearched: 1 } }, + ]).limit(10); + + q.exec((err, data) => { + if (err) return res.json({ success: false, error: err }); + return res.json({ success: true, data: data }); + }); + break; + + case 'popular': + let popularType = {}; + if (query.type) popularType = { type: query.type }; + let popularData; + + if (popularType.type !== 'course') { + popularData = await Data.aggregate([ + { + $match: { + ...popularType, + counter: { + $gt: 0, + }, + name: { + $exists: true, + }, + pid: { + $ne: 'fd8d0743-344a-4758-bb97-f8ad84a37357', //PID for HDR-UK Papers dataset + }, + }, + }, + { $lookup: { from: 'tools', localField: 'authors', foreignField: 'id', as: 'persons' } }, + { + $project: { + _id: 0, + type: 1, + bio: 1, + firstname: 1, + lastname: 1, + name: 1, + categories: 1, + pid: 1, + id: 1, + counter: 1, + programmingLanguage: 1, + tags: 1, + description: 1, + activeflag: 1, + datasetv2: 1, + datasetfields: 1, + 'persons.id': 1, + 'persons.firstname': 1, + 'persons.lastname': 1, + }, + }, + { + $group: { + _id: '$name', + type: { $first: '$type' }, + name: { $first: '$name' }, + pid: { $first: '$pid' }, + bio: { $first: '$bio' }, + firstname: { $first: '$firstname' }, + lastname: { $first: '$lastname' }, + id: { $first: '$id' }, + categories: { $first: '$categories' }, + counter: { $sum: '$counter' }, + programmingLanguage: { $first: '$programmingLanguage' }, + tags: { $first: '$tags' }, + description: { $first: '$description' }, + activeflag: { $first: '$activeflag' }, + datasetv2: { $first: '$datasetv2' }, + datasetfields: { $first: '$datasetfields' }, + persons: { $first: '$persons' }, + }, + }, + { + $sort: { + counter: -1, + name: 1, + }, + }, + { + $limit: 10, + }, + ]); + } else if (popularType.type === 'course') { + popularData = await Course.aggregate([ + { + $match: { + counter: { + $gt: 0, + }, + title: { + $exists: true, + }, + }, + }, + { + $project: { + _id: 0, + type: 1, + title: 1, + provider: 1, + courseOptions: 1, + award: 1, + domains: 1, + description: 1, + id: 1, + counter: 1, + }, + }, + { + $group: { + _id: '$title', + type: { $first: '$type' }, + title: { $first: '$title' }, + provider: { $first: '$provider' }, + courseOptions: { $first: '$courseOptions' }, + award: { $first: '$award' }, + domains: { $first: '$domains' }, + description: { $first: '$description' }, + id: { $first: '$id' }, + counter: { $sum: '$counter' }, + }, + }, + { + $sort: { + counter: -1, + title: 1, + }, + }, + { + $limit: 10, + }, + ]); + } + + return res.json({ success: true, data: popularData }); + + case 'updates': + let recentlyUpdated = Data.find({ activeflag: 'active' }).sort({ updatedon: -1 }).limit(10); + + if (req.query.type && req.query.type === 'course') { + recentlyUpdated = Course.find( + { activeflag: 'active' }, + { + _id: 0, + type: 1, + title: 1, + provider: 1, + courseOptions: 1, + award: 1, + domains: 1, + description: 1, + id: 1, + counter: 1, + updatedon: 1, + } + ) + .sort({ updatedon: -1, title: 1 }) + .limit(10); + } else if (req.query.type && req.query.type === 'dataset') { + recentlyUpdated = Data.find( + { + $and: [ + { + type: req.query.type, + activeflag: 'active', + pid: { + $ne: 'fd8d0743-344a-4758-bb97-f8ad84a37357', //Production PID for HDR-UK Papers dataset + }, + }, + ], + }, + { + _id: 0, + type: 1, + name: 1, + pid: 1, + id: 1, + counter: 1, + activeflag: 1, + datasetv2: 1, + datasetfields: 1, + description: 1, + 'timestamps.updated': 1, + } + ) + .sort({ 'timestamps.updated': -1, name: 1 }) + .limit(10); + } else if (req.query.type && req.query.type !== 'course' && req.query.type !== 'dataset') { + recentlyUpdated = Data.find( + { + $and: [ + { + type: req.query.type, + activeflag: 'active', + }, + ], + }, + { + _id: 0, + type: 1, + bio: 1, + firstname: 1, + lastname: 1, + name: 1, + categories: 1, + id: 1, + counter: 1, + programmingLanguage: 1, + tags: 1, + description: 1, + activeflag: 1, + authors: 1, + updatedon: 1, + } + ) + .populate([{ path: 'persons', options: { select: { id: 1, firstname: 1, lastname: 1 } } }]) + .sort({ updatedon: -1, name: 1 }) + .limit(10); + } + + recentlyUpdated.exec((err, data) => { + if (err) return res.json({ success: false, error: err }); + return res.json({ success: true, data: data }); + }); + break; + + case 'unmet': + switch (req.query.type) { + case 'Datasets': + req.entity = 'dataset'; + await getUnmetSearches(req) + .then(data => { + return res.json({ success: true, data: data }); + }) + .catch(err => { + return res.json({ success: false, error: err }); + }); + break; + + case 'Tools': + req.entity = 'tool'; + await getUnmetSearches(req) + .then(data => { + return res.json({ success: true, data: data }); + }) + .catch(err => { + return res.json({ success: false, error: err }); + }); + break; + + case 'Projects': + req.entity = 'project'; + await getUnmetSearches(req) + .then(data => { + return res.json({ success: true, data: data }); + }) + .catch(err => { + return res.json({ success: false, error: err }); + }); + break; + + case 'Courses': + req.entity = 'course'; + await getUnmetSearches(req) + .then(data => { + return res.json({ success: true, data: data }); + }) + .catch(err => { + return res.json({ success: false, error: err }); + }); + break; + + case 'Papers': + req.entity = 'paper'; + await getUnmetSearches(req) + .then(data => { + return res.json({ success: true, data: data }); + }) + .catch(err => { + return res.json({ success: false, error: err }); + }); + break; + + case 'People': + req.entity = 'person'; + await getUnmetSearches(req) + .then(data => { + return res.json({ success: true, data: data }); + }) + .catch(err => { + return res.json({ success: false, error: err }); + }); + break; + } + } + } catch (err) { + console.error(err.message); + return res.json({ success: false, error: err.message }); + } +}); + +router.get('/topSearches', async (req, res) => { + await getTopSearches(req) + .then(data => { + return res.json({ success: true, data: data }); + }) + .catch(err => { + return res.json({ success: false, error: err }); + }); +}); + +module.exports = router; + +const getTopSearches = async (req, res) => { + return new Promise(async (resolve, reject) => { + let searchMonth = parseInt(req.query.month); + let searchYear = parseInt(req.query.year); + + let q = RecordSearchData.aggregate([ + { $addFields: { month: { $month: '$createdAt' }, year: { $year: '$createdAt' } } }, + { + $match: { + $and: [{ month: searchMonth }, { year: searchYear }, { searched: { $ne: '' } }], + }, + }, + { + $group: { + _id: { $toLower: '$searched' }, + count: { $sum: 1 }, + }, + }, + { $sort: { count: -1 } }, + ]).limit(10); + + q.exec(async (err, topSearches) => { + if (err) reject(err); + + let resolvedArray = await Promise.all( + topSearches.map(async topSearch => { + let searchQuery = { $and: [{ activeflag: 'active' }] }; + searchQuery['$and'].push({ $text: { $search: topSearch._id } }); + + await Promise.all([ + getObjectResult('dataset', searchQuery), + getObjectResult('tool', searchQuery), + getObjectResult('project', searchQuery), + getObjectResult('paper', searchQuery), + getObjectResult('course', searchQuery), + ]).then(resources => { + topSearch.datasets = resources[0][0] !== undefined && resources[0][0].count !== undefined ? resources[0][0].count : 0; + topSearch.tools = resources[1][0] !== undefined && resources[1][0].count !== undefined ? resources[1][0].count : 0; + topSearch.projects = resources[2][0] !== undefined && resources[2][0].count !== undefined ? resources[2][0].count : 0; + topSearch.papers = resources[3][0] !== undefined && resources[3][0].count !== undefined ? resources[3][0].count : 0; + topSearch.course = resources[4][0] !== undefined && resources[4][0].count !== undefined ? resources[4][0].count : 0; + }); + return topSearch; + }) + ); + resolve(resolvedArray); + }); + }); +}; + +function getObjectResult(type, searchQuery) { + var newSearchQuery = JSON.parse(JSON.stringify(searchQuery)); + newSearchQuery['$and'].push({ type: type }); + var q = ''; + + q = Data.aggregate([ + { $match: newSearchQuery }, + { + $group: { + _id: {}, + count: { + $sum: 1, + }, + }, + }, + { + $project: { + count: '$count', + _id: 0, + }, + }, + ]); + + return new Promise((resolve, reject) => { + q.exec((err, data) => { + if (typeof data === 'undefined') resolve([]); + else resolve(data); + }); + }); +} + +const getUnmetSearches = async (req, res) => { + return new Promise(async (resolve, reject) => { + let searchMonth = parseInt(req.query.month); + let searchYear = parseInt(req.query.year); + let entitySearch = { ['returned.' + req.entity]: { $lte: 0 } }; + let q = RecordSearchData.aggregate([ + { $addFields: { month: { $month: '$createdAt' }, year: { $year: '$createdAt' } } }, + { + $match: { + $and: [{ month: searchMonth }, { year: searchYear }, entitySearch, { searched: { $ne: '' } }], + }, + }, + { + $group: { + _id: { $toLower: '$searched' }, + count: { $sum: 1 }, + maxDatasets: { $max: '$returned.dataset' }, + maxProjects: { $max: '$returned.project' }, + maxTools: { $max: '$returned.tool' }, + maxPapers: { $max: '$returned.paper' }, + maxCourses: { $max: '$returned.course' }, + maxPeople: { $max: '$returned.people' }, + entity: { $max: req.entity }, + }, + }, + { $sort: { count: -1 } }, + ]).limit(10); + + q.exec((err, data) => { + if (err) reject(err); + return resolve(data); + }); + }); +}; diff --git a/src/resources/topic/topic.controller.js b/src/resources/topic/topic.controller.js index 07663d0a..a3a40ab4 100644 --- a/src/resources/topic/topic.controller.js +++ b/src/resources/topic/topic.controller.js @@ -2,7 +2,6 @@ import mongoose from 'mongoose'; import { TopicModel } from './topic.model'; import { Data as ToolModel } from '../tool/data.model'; import _ from 'lodash'; - module.exports = { buildRecipients: async (team, createdBy) => { // 1. Cause error if no members found diff --git a/src/resources/utilities/constants.util.js b/src/resources/utilities/constants.util.js index 952b8c57..6f389cad 100644 --- a/src/resources/utilities/constants.util.js +++ b/src/resources/utilities/constants.util.js @@ -119,7 +119,7 @@ const _navigationFlags = { displayOrder: 1, }, ], - text: '#NAME# made this change on on #DATE#', + text: '#NAME# made this change on #DATE#', }, incomplete: { status: 'DANGER', options: [], text: '#NAME# requested an update on #DATE#' }, },