From 3604537f83588f6a79cf7bc5d89bbe1e051c7713 Mon Sep 17 00:00:00 2001 From: Robin Kavanagh Date: Wed, 3 Feb 2021 20:10:39 +0000 Subject: [PATCH 01/42] Fixed swagger documentation errors and updated to follow Open API 3.0.1 spec --- swagger.yaml | 694 +++++++++++++++------------------------------------ 1 file changed, 198 insertions(+), 496 deletions(-) diff --git a/swagger.yaml b/swagger.yaml index 7b737210..2fa8dad7 100644 --- a/swagger.yaml +++ b/swagger.yaml @@ -1,24 +1,102 @@ -openapi: 3.0.0 +openapi: 3.0.1 info: - title: HDR UK API - description: API for Tools and artefacts repository. - version: 0.0.1 - + title: defaultTitle + description: defaultDescription + version: '0.1' servers: - - url: https://api.www.healthdatagateway.org/api - - url: http://localhost:3001/api - - - url: https://api.{environment}.healthdatagateway.org:{port}/api - variables: - environment: - default: latest - description: The Environment name. - port: - enum: - - '443' - default: '443' - + - url: 'https://api.www.healthdatagateway.org' +security: +- oauth2: [] paths: + /oauth/token: + post: + tags: + - Authorization + description: Auto generated using Swagger Inspector + requestBody: + content: + application/json: + schema: + type: object + properties: + grant_type: + type: string + client_secret: + type: string + client_id: + type: string + examples: + '0': + value: |- + { + "grant_type":"client_credentials", + "client_id":"2ca1f61a90e3547", + "client_secret":"3f80fecbf781b6da280a8d17aa1a22066fb66daa415d8befc1" + } + responses: + '200': + description: Auto generated using Swagger Inspector + content: + application/json; charset=utf-8: + schema: + type: string + examples: {} + + /api/v1/data-access-request/5fbb9c464705566a6ddfcced: + get: + tags: + - Data Access Request + description: Auto generated using Swagger Inspector + responses: + '200': + description: Auto generated using Swagger Inspector + servers: + - url: 'https://api.uatbeta.healthdatagateway.org' + servers: + - url: 'https://api.uatbeta.healthdatagateway.org' + + /api/v1/data-access-request/60142c5b4316a0e0fcd47c56: + put: + tags: + - Data Access Request + description: Auto generated using Swagger Inspector + requestBody: + content: + application/json: + schema: + type: object + properties: + applicationStatus: + type: string + applicationStatusDesc: + type: string + examples: + '0': + value: |- + { + "applicationStatus": "approved", + "applicationStatusDesc": "" + } + responses: + '200': + description: Auto generated using Swagger Inspector + servers: + - url: 'https://api.uatbeta.healthdatagateway.org' + servers: + - url: 'https://api.uatbeta.healthdatagateway.org' + + /api/v1/publishers/OTHER > HEALTH DATA RESEARCH UK/dataaccessrequests: + get: + tags: + - Publishers + description: Auto generated using Swagger Inspector + responses: + '200': + description: Auto generated using Swagger Inspector + servers: + - url: 'https://api.uatbeta.healthdatagateway.org' + servers: + - url: 'https://api.uatbeta.healthdatagateway.org' /v1/datasets/{datasetID}: get: summary: Returns Dataset object. @@ -95,25 +173,30 @@ paths: - Data Access Request parameters: - in: path - name: datasetID + name: id required: true description: The ID of the datset schema: type : string example: 5ee249426136805fbf094eef - - in: body - name: questionAnswers - description: Answers object - schema: - type: object - properties: - questionAnswers: - type: string - example: '{firstName: Roger}' + requestBody: + content: + application/json: + schema: + type: object + properties: + questionAnswers: + type: object + examples: + '0': + value: |- + { + "firstName": "Roger" + } responses: '200': description: OK - + /v1/person/{id}: get: summary: Returns details for a person. @@ -227,8 +310,6 @@ paths: tags: - Search summary: Search for HDRUK /search?search - produces: - - application/json parameters: - in: query name: params @@ -264,65 +345,22 @@ paths: responses: '200': description: OK - - /v1/datasets/?search={searchTerm}: - get: - summary: Returns dataset search results by searchTerm - tags: - - Datasets - parameters: - - name: searchTerm - in: path - required: true - description: A search term. - schema: - type : string - example: epilepsy - responses: - '200': - description: OK - - /v1/stats/: - get: - summary: This will return a JSON document to show high level stats. - tags: - - Stats - responses: - '200': - description: OK - - /v1/stats?rank={rank}: - get: - summary: Returns the details on recent searches, popular objects or recently updated objects based on the rank query parameter. - tags: - - Stats - parameters: - - name: rank - in: path - required: true - description: The type of stat. - schema: - type : string - example: recent - responses: - '200': - description: OK - /v1/stats/topSearches?month={month}&year={year}: + /v1/stats/topSearches: get: summary: Returns top searches for a given month and year. tags: - Stats parameters: - name: month - in: path + in: query required: true description: Month number. schema: type : string example: 7 - name: year - in: path + in: query required: true description: Year. schema: @@ -332,35 +370,35 @@ paths: '200': description: OK - /v1/stats?rank={rank}&type={type}&month={month}&year={year}: + /v1/stats/: get: - summary: Returns unmet demand information for the resource type specified, based on the month and year query params. + summary: Returns the details on recent searches, popular objects, unmet demands or recently updated objects based on the rank query parameter. tags: - Stats parameters: - name: rank - in: path + in: query required: true description: The type of stat. schema: type : string example: unmet - name: type - in: path + in: query required: true description: Resource type. schema: type : string example: Tools - name: month - in: path + in: query required: true description: Month number. schema: type : string example: 7 - name: year - in: path + in: query required: true description: Year. schema: @@ -370,38 +408,21 @@ paths: '200': description: OK - /v1/kpis?kpi={type}: - get: - summary: Returns information for KPIs, based on the KPI type parameter. - tags: - - KPIs - parameters: - - name: type - in: path - required: true - description: The type of KPI. - schema: - type : string - example: technicalmetadata - responses: - '200': - description: OK - - /v1/kpis?kpi={type}&selectedDate={selectedDate}: + /v1/kpis/: get: summary: Returns information for KPIs, based on the KPI type and selectedDate parameters. tags: - KPIs parameters: - name: type - in: path + in: query required: true description: The type of KPI. schema: type : string example: uptime - name: selectedDate - in: path + in: query required: true description: Full date time string. schema: @@ -412,7 +433,6 @@ paths: description: OK /v1/messages/{id}: - delete: summary: Delete a Message security: @@ -430,7 +450,6 @@ paths: responses: '204': description: Ok - put: summary: Update a single Message security: @@ -445,25 +464,25 @@ paths: schema: type: string example: "5ee249426136805fbf094eef" - - in: body - name: payload - description: Payload for updating existing Message - schema: - type: object - properties: - isRead: - type: boolean - messageDescription: - type: string - messageType: - type: string + requestBody: + content: + application/json: + schema: + type: object + properties: + isRead: + type: boolean + examples: + 'Update message to read': + value: |- + { + "isRead": true + } responses: '204': description: OK - /v1/messages/unread/count: - get: summary: Returns the number of unread messages for the authenticated user security: @@ -475,26 +494,37 @@ paths: description: OK /v1/messages/: - post: summary: Returns a new Message object and creates an associated parent Topic if a Topic is not specified in request body security: - cookieAuth: [] tags: - Messages - parameters: - - in: body - name: payload - description: Payload for updating existing Message - schema: - type: object - properties: - isRead: - type: boolean - messageDescription: - type: string - messageType: - type: string + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + isRead: + type: boolean + messageDescription: + type: string + messageType: + type: string + required: + - isRead + - messageDescription + - messageType + examples: + 'Create new message': + value: |- + { + "isRead": false, + "messageDescription": "this is an example", + "messageType": "message" + } responses: '201': description: OK @@ -506,16 +536,23 @@ paths: - cookieAuth: [] tags: - Topics - parameters: - - in: body - name: payload - description: Payload for posting new message Topic - schema: - type: object - properties: - relatedObjectIds: - type: array - example: "['1','2','3']" + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + relatedObjectIds: + type: array + items: + type: string + examples: + 'Create a new topic': + value: |- + { + "relatedObjectIds": "['1','2','3']" + } responses: '201': description: A new Topic @@ -535,18 +572,24 @@ paths: description: Subtitle of message relatedObjectIds: type: array + items: + type: string description: Object ID this Topic is related to createdBy: type: object description: User that created the topic createdDate: - type: date + type: string description: Date the topic was created recipients: type: array + items: + type: string description: Collection of user IDs tags: type: array + items: + type: string description: Collection of tags to describe topic get: summary: Returns a list of all topics that the authenticated user is a recipient or member of @@ -557,7 +600,6 @@ paths: responses: '200': description: Ok - /v1/topics/{id}: get: @@ -577,7 +619,6 @@ paths: responses: '200': description: Ok - delete: summary: Soft deletes a message Topic but does not affect associated messages security: @@ -659,7 +700,6 @@ paths: responses: '200': description: OK - get: summary: Returns List of Project objects. tags: @@ -1119,37 +1159,6 @@ paths: responses: '200': description: OK - - /v1/tools/: - get: - summary: Return List of Tool objects. - tags: - - Tools - parameters: - - in: query - name: limit - required: false - description: Limit the number of results - schema: - type : integer - example: 3 - - in: query - name: offset - required: false - description: Index to offset the search results - schema: - type : integer - example: 1 - - in: query - name: q - required: false - description: Filter using search query - schema: - type : string - example: epilepsy - responses: - '200': - description: OK /v1/tools/{id}: get: @@ -1279,322 +1288,15 @@ paths: responses: '200': description: OK - # Delete requires validation to be implemented first - # delete: - # summary: Delete a tool. - # security: - # - cookieAuth: [] - # tags: - # - Tools - # parameters: - # - in: path - # name: id - # required: true - # schema: - # type : integer - # example: 123 - # requestBody: - # content: - # application/json: - # schema: - # type: object - # required: - # - id - # properties: - # id: - # type: number - # id: 26542005388306332 - # example: # Sample object - # id: 26542005388306332 - # responses: - # '200': - # description: OK - # '204': - # description: No Content - - - - - - - # /v1/tools/get: - # get: - # summary: Tool search. - # tags: - # - Tool - # responses: - # '200': - # description: OK - - # /v1/tools/get/admin: - # get: - # summary: Tool search admins. - # tags: - # - Tool - # responses: - # '200': - # description: OK - - # /v1/tools/status: - # put: - # summary: Tool update status. - # tags: - # - Tool - # responses: - # '200': - # description: OK - - # /v1/project/get: - # get: - # summary: Project search. - # tags: - # - Project - # responses: - # '200': - # description: OK - - # /v1/project/get/admin: - # get: - # summary: Project search admins. - # tags: - # - Project - # responses: - # '200': - # description: OK - - # /v1/project/status: - # put: - # summary: Project update status. - # tags: - # - Project - # responses: - # '200': - # description: OK - - # /v1/paper/get: - # get: - # summary: Paper search. - # tags: - # - Paper - # responses: - # '200': - # description: OK - - # /v1/paper/get/admin: - # get: - # summary: Paper search admins. - # tags: - # - Paper - # responses: - # '200': - # description: OK - - # /v1/paper/status: - # put: - # summary: Paper update status. - # tags: - # - Paper - # responses: - # '200': - # description: OK - - # /v1/users/{userId}: - # get: - # summary: Find user by id. - # tags: - # - Users - # parameters: - # - name: userId - # in: path - # required: true - # description: The ID of the user - # schema: - # type : string - # example: b2475d19-75fe-422b-9d0c-c53cdfff8230 - # responses: - # '200': - # description: OK - - # /v1/users: - # get: - # summary: Find all users - # tags: - # - users - # responses: - # '200': - # description: OK - - # /v1/auth/register: - # post: - # summary: Register User. - # tags: - # - Users - # responses: - # '200': - # description: OK - - # /v1/auth/login: - # post: - # summary: Authenticate User. - # tags: - # - Auth - # responses: - # '200': - # description: OK - - # /v1/auth/logout: - # get: - # summary: Logout. - # tags: - # - Auth - # responses: - # '200': - # description: OK - - # /v1/auth/status: - # get: - # summary: Auth status. - # tags: - # - Auth - # responses: - # '200': - # description: OK - - # /v1/tools/add: - # post: - # summary: Add a tool. - # tags: - # - Tool - # responses: - # '200': - # description: OK - - # /v1/tools/edit: - # put: - # summary: Edit a tool. - # tags: - # - Tool - # responses: - # '200': - # description: OK - - # /v1/tools/delete: - # delete: - # summary: delete a tool. - # tags: - # - Tool - # responses: - # '200': - # description: OK - - # /v1/project/add: - # post: - # summary: Add a project. - # tags: - # - Project - # responses: - # '200': - # description: OK - - # /v1/project/edit: - # put: - # summary: Edit a project. - # tags: - # - Project - # responses: - # '200': - # description: OK - - # /v1/project/delete: - # delete: - # summary: delete a project. - # tags: - # - Project - # responses: - # '200': - # description: OK - - # /v1/paper/add: - # post: - # summary: Add a paper. - # tags: - # - Paper - # responses: - # '200': - # description: OK - - # /v1/paper/edit: - # put: - # summary: Edit a paper. - # tags: - # - Paper - # responses: - # '200': - # description: OK - - # /v1/paper/delete: - # delete: - # summary: delete a paper. - # tags: - # - Paper - # responses: - # '200': - # description: OK - - # /v1/messages/{personId}: - # get: - # summary: Find messages by person. - # tags: - # - Messages - # parameters: - # - name: personId - # in: path - # required: true - # description: The ID of the person - # schema: - # type : string - # example: b2475d19-75fe-422b-9d0c-c53cdfff8230 - - # /v1/messages/admin/{personId}: - # get: - # summary: find messages for admin by person id - # tags: - # - Messages - # parameters: - # - name: personId - # in: path - # required: true - # description: The ID of the person - # schema: - # type : string - # example: b2475d19-75fe-422b-9d0c-c53cdfff8230 - - # /v1/reviews: - # get: - # summary: find all reviews. - # tags: - # - Reviews - - # /v1/reviews/pending: - # get: - # summary: find all pending reviews. - # tags: - # - Reviews - - # /v1/reviews/admin/pending: - # get: - # summary: find all pending reviews for admins. - # tags: - # - Reviews - - # /v1/discourse/topic/tool/{toolId}: - # put: - # summary: This route creates a new topic in Discourse if the tool exists and is active. Returns an object with the link to the new Discourse topic. - # tags: - # - Discourse - # parameters: - # - name: toolId - # in: path - # required: true - # description: Tool id +components: + securitySchemes: + oauth2: + type: oauth2 + flows: + clientCredentials: + tokenUrl: 'https://api.www.healthdatagateway.org/oauth/token' + cookieAuth: + type: http + scheme: jwt + \ No newline at end of file From bb2f1c95a2c2c34d5856fef0a1433b76d3df3d4b Mon Sep 17 00:00:00 2001 From: Robin Kavanagh Date: Fri, 5 Feb 2021 17:06:36 +0000 Subject: [PATCH 02/42] Added documentation for DAR endpoints for external usage --- swagger.yaml | 1458 +++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 1370 insertions(+), 88 deletions(-) diff --git a/swagger.yaml b/swagger.yaml index 2fa8dad7..d2983a37 100644 --- a/swagger.yaml +++ b/swagger.yaml @@ -12,8 +12,9 @@ paths: post: tags: - Authorization - description: Auto generated using Swagger Inspector + description: OAuth2.0 token endpoint responsible for issuing short-lived json web tokens (JWT) for access to secure Gateway APIs. For client credentials grant flow, a valid client id and secret must be provided to identify your application and provide the expected permissions. This type of authentication is reserved for team based connectivity through client applications and is not provided for human user access. For more information, contact the HDR-UK team. requestBody: + required: true content: application/json: schema: @@ -21,13 +22,20 @@ paths: properties: grant_type: type: string - client_secret: - type: string + description: The OAuth2.0 grant type that will be used to provide authentication. client_id: type: string + description: A unique identifer provided to your team by the HDR-UK team at the time of onboarding to the Gateway. Contact the HDR-UK team for issue of new credentials. + client_secret: + type: string + description: A long (50 character) string provided by the HDR-UK team at the time of onboarding to the Gateway. Contact the HDR-UK team for issue of new credentials. + required: + - grant_type + - client_secret + - client_id examples: - '0': - value: |- + 'Client Credentials Grant Flow': + value: { "grant_type":"client_credentials", "client_id":"2ca1f61a90e3547", @@ -35,31 +43,382 @@ paths: } responses: '200': - description: Auto generated using Swagger Inspector + description: Successful response containing json web token (JWT) that will authorize an HTTP request against secured resources. content: - application/json; charset=utf-8: + application/json: schema: - type: string - examples: {} + type: object + properties: + access_token: + type: string + description: The encoded json web token (JWT) that must be appended to the Authorization of subsequent API HTTP requests in order to access secured resources. + token_type: + type: string + description: The type of token issued, in this case, a json web token (JWT). + expires_in: + type: integer + description: The length of time in seconds before the issued JWT expires, defaulted to 900 seconds (15 minutes). + examples: + 'Client Credentials Grant Flow': + value: + { + "access_token":"Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJkYXRhIjp7Il9pZCI6IjYwMGJmYzk5YzhiZjcwMGYyYzdkNWMzNiIsInRpbWVTdGFtcCI2MTYxMjM4MzkwMzE5Nn0sImlhdCI6MTYxMjM4MzkwMywiZXhwIjoxNjEyMzg0ODAzfQ.-YvUBdjtJvdrRacz6E8-cYPQlum4TrEmiCFl8jO5a-M", + "token_type":"jwt", + "expires_in":900 + } + '400': + description: Failure response caused by incomplete or invalid client credentials being passed to the endpoint. + content: + application/json: + schema: + type: object + properties: + success: + type: boolean + description: A field that indicates the API request failed. + message: + type: string + description: A message indicating that the request failed for a given reason. + examples: + 'Invalid Client Credentials': + value: + { + "success":false, + "message":"Invalid client credentials were provided for the authorisation attempt" + } + 'Incomplete Client Credentials': + value: + { + "success":false, + "message":"Incomplete client credentials were provided for the authorisation attempt" + } + 'Invalid Grant Type': + value: + { + "success":false, + "message":"An invalid grant type has been specified" + } - /api/v1/data-access-request/5fbb9c464705566a6ddfcced: + /api/v1/data-access-request/{id}: get: tags: - Data Access Request - description: Auto generated using Swagger Inspector + parameters: + - in: path + name: id + required: true + description: The unique identifier for a single data access request application. + schema: + type : string + example: 5ee249426136805fbf094eef + description: Retrieve a single Data Access Request application using a supplied identifer responses: '200': - description: Auto generated using Swagger Inspector - servers: - - url: 'https://api.uatbeta.healthdatagateway.org' - servers: - - url: 'https://api.uatbeta.healthdatagateway.org' - - /api/v1/data-access-request/60142c5b4316a0e0fcd47c56: + description: Successful response containing a full data access request application. + content: + application/json: + schema: + type: object + properties: + status: + type: string + data: + type: object + properties: + id: + type: string + description: The unique identifier for the application. + aboutApplication: + description: An object which holds data relating to the 'about application' section of the application form including details of whether the project is an NCS project or not. + type: object + properties: + isNationalCoreStudies: + type: boolean + description: A flag to indicate if this application is in relation to a National Core Studies Project. + nationalCoreStudiesProjectId: + type: integer + description: The unique identifier correlating to a Gateway Project entity indicating that this application is relating to a National Core Studies project. + projectName: + type: string + description: The project name that has been assigned to the application by the applicant(s). + authorIds: + type: array + items: + type: integer + description: An array of values correlating to specific user's via their numeric identifiers. An author is also known as a contributor to an application and can view, edit or submit. + datasetIds: + type: array + items: + type: string + description: An array of values correlating to datasets selected for the application via their identifier, which is unique per version. + datasetTitles: + type: array + items: + type: string + description: An array of strings correlating to the dataset titles that have been selected for the application. + applicationStatus: + type: string + enum: + - inProgress + - submitted + - inReview + - approved + - rejected + - approved with conditions + description: The current status of the application. + jsonSchema: + type: object + description: The object containing the json definition that renders the application form using the Winterfell library. This contains the details of questions, questions sets, question panels, headings and navigation items that appear. + questionAnswers: + type: object + description: The object containing the answers provided on the application form. This consists of a series of key pairs, where the key is the unqiue question Id, and the value the is the answer provided to the question. In the case of a multi select on the form, the value may be an array. + publisher: + type: string + description: The name of the Custodian that holds the dataset and is processing the application. + publisherObj: + type: object + description: The object containing details regarding the Custodian/publisher relating to the application. + userId: + type: integer + description: The unique identifier that correlates to the user account of the main applicant. This is always the user that started the application. + schemaId: + type: string + description: The unique identifier that correlates to the schema from which the application form was generated. + files: + type: array + items: + type: object + description: An array containing the links to files that have been uploaded to the application form and are held within the Gateway ecosystem. + amendmentIterations: + type: array + items: + type: object + description: An array containing an object with details for each iteration the application has passed through. An iteration is defined as an application which has been returned by the Custodian for correction, corrected by the applicant(s) and resubmitted. The object contains dates that the application was returned, and resubmitted as well as reference to any questions that were highlighted for amendment. + createdAt: + type: string + description: The date and time that the application was started. + updatedAt: + type: string + description: The date and time that the application was last updated by any party. + projectId: + type: string + description: The unique identifier for the application converted to a more human friendly format in uppercase and hypenated. + dateSubmitted: + type: string + description: The date and time that the application was originally submitted by the applicant(s) to the Custodian for review. + dateReviewStart: + type: string + description: The date and time that the review process was commenced by a Custodian manager. The review starts from the moment the manager opens the application to triage it. + dateFinalStatus: + type: string + description: The date and time that the Custodian triggered a status change to the application once a final decision was made. E.g. when application was approved. This date can be used in conjunction with the dateReviewStart date to calculate the length of time the Custodian took to make a decision through their review process. + datasets: + type: array + items: + type: object + description: An array containing the full metadata for each of the datasets that have been applied for through this application. + mainApplicant: + type: object + description: An object containing the details of the main applicant of the application as referenced by the userId field. + authors: + type: array + items: + type: object + description: An array containing the details of the contributors of the application as referenced by the authorIds field. + readOnly: + type: boolean + description: A value to indicate if the requesting party is able to modify the application in its present state. For example, this will be false for a Custodian, but true for applicants if the applicant(s) are working on resubmitting the application following a request for amendments. + unansweredAmendments: + type: integer + description: The number of amendments that have been requested by the Custodian in the current amendment iteration. + answeredAmendments: + type: integer + description: The number of requested amendments that the applicant(s) have fixed in the current amendment iteration. + userType: + type: string + enum: + - custodian + - applicant + description: The type of user that has requested the Data Access Request application based on their permissions. It is either an applicant or a Custodian user. + activeParty: + type: string + enum: + - custodian + - applicant + description: The party that is currently handling the application. This is the applicant during presubmission, then the Custodian following submission. The active party then fluctuates between parties during amendment iterations. + inReviewMode: + type: boolean + description: A flag to indicate if the current user is a reviewer of the application. This value will be false unless the requesting user is an assigned reviewer to a currently active workflow step. When this value is true, the requesting user is able to recommend approval or rejection of the application. + reviewSections: + type: array + items: + type: string + description: An array containing the sections of the application form that the current user is required to review if they are a reviewer of the current workflow step that the application is in. E.g. ['Safe People','Safe Data'] + hasRecommended: + type: boolean + description: A flag to indicate if the current user as a reviewer of the current workflow phase has submitted their recommendation for approval or rejection based on their review of the review sections assigned to them. + workflow: + type: object + description: The full details of the workflow that has been assigned to the Data Access Request application. This includes information such as the review phases that the application will pass through and associated metadata. + examples: + 'Approved Application': + value: + { + "status": "success", + "data": { + "aboutApplication": { + "selectedDatasets": [ + { + "_id": "5fc31a18d98e4f4cff7e9315", + "datasetId": "d5faf9c6-6c34-46d7-93c4-7706a5436ed9", + "name": "HDR UK Papers & Preprints", + "description": "Publications that mention HDR-UK (or any variant thereof) in Acknowledgements or Author Affiliations\n\nThis will include:\n- Papers\n- COVID-19 Papers\n- COVID-19 Preprint", + "abstract": "Publications that mention HDR-UK (or any variant thereof) in Acknowledgements or Author Affiliations", + "publisher": "OTHER > HEALTH DATA RESEARCH UK", + "contactPoint": "hdr.hdr@hdruk.ac.uk", + "publisherObj": { + "dataRequestModalContent": { + "header": " ", + "body": "{omitted for brevity...}", + "footer": "" + }, + "active": true, + "allowsMessaging": true, + "workflowEnabled": true, + "_id": "5f7b1a2bce9f65e6ed83e7da", + "name": "OTHER > HEALTH DATA RESEARCH UK", + "imageURL": "", + "team": { + "active": true, + "_id": "5f7b1a2bce9f65e6ed83e7da", + "members": [ + { + "roles": [ + "manager" + ], + "memberid": "5f1a98861a821b4a53e44d15" + }, + { + "roles": [ + "manager" + ], + "memberid": "600bfc99c8bf700f2c7d5c36" + } + ], + "type": "publisher", + "__v": 3, + "createdAt": "2020-11-30T21:12:40.855Z", + "updatedAt": "2020-12-02T13:33:45.232Z" + } + } + } + ], + "isNationalCoreStudies": true, + "nationalCoreStudiesProjectId": "4324836585275824", + "projectName": "Test application title", + "completedDatasetSelection": true, + "completedInviteCollaborators": true, + "completedReadAdvice": true, + "completedCommunicateAdvice": true, + "completedApprovalsAdvice": true, + "completedSubmitAdvice": true + }, + "authorIds": [], + "datasetIds": [ + "d5faf9c6-6c34-46d7-93c4-7706a5436ed9" + ], + "datasetTitles": [], + "applicationStatus": "approved", + "jsonSchema": "{omitted for brevity...}", + "questionAnswers": { + "fullname-892140ec730145dc5a28b8fe139c2876": "James Smith", + "jobtitle-ff1d692a04b4bb9a2babe9093339136f": "Consultant", + "organisation-65c06905b8319ffa29919732a197d581": "Consulting Inc." + }, + "publisher": "OTHER > HEALTH DATA RESEARCH UK", + "_id": "60142c5b4316a0e0fcd47c56", + "version": 1, + "userId": 9190228196797084, + "schemaId": "5f55e87e780ba204b0a98eb8", + "files": [], + "amendmentIterations": [], + "createdAt": "2021-01-29T15:40:11.943Z", + "updatedAt": "2021-02-03T14:38:22.688Z", + "__v": 0, + "projectId": "6014-2C5B-4316-A0E0-FCD4-7C56", + "dateSubmitted": "2021-01-29T16:30:27.351Z", + "dateReviewStart": "2021-02-03T14:36:22.341Z", + "dateFinalStatus": "2021-02-03T14:38:22.680Z", + "datasets": [ + "{omitted for brevity...}" + ], + "dataset": null, + "mainApplicant": { + "_id": "5f1a98861a821b4a53e44d15", + "firstname": "James", + "lastname": "Smith" + }, + "authors": [], + "id": "60142c5b4316a0e0fcd47c56", + "readOnly": true, + "unansweredAmendments": 0, + "answeredAmendments": 0, + "userType": "custodian", + "activeParty": "custodian", + "inReviewMode": false, + "reviewSections": [], + "hasRecommended": false, + "workflow": {} + } + } + '404': + description: Failed to find the application requested. + content: + application/json: + schema: + type: object + properties: + status: + type: string + message: + type: string + examples: + 'Not Found': + value: + { + "status":"error", + "message":"Application not found." + } + '401': + description: Unauthorised attempt to access an application. + content: + application/json: + schema: + type: object + properties: + status: + type: string + message: + type: string + examples: + 'Unauthorised': + value: + { + "status":"failure", + "message":"Unauthorised" + } put: tags: - Data Access Request - description: Auto generated using Swagger Inspector + parameters: + - in: path + name: id + required: true + description: The unique identifier for a single Data Access Request application. + schema: + type : string + example: 5ee249426136805fbf094eef + description: Update a single Data Access Request application. requestBody: content: application/json: @@ -71,33 +430,989 @@ paths: applicationStatusDesc: type: string examples: - '0': - value: |- + 'Update Application Status': + value: { "applicationStatus": "approved", - "applicationStatusDesc": "" + "applicationStatusDesc": "This application meets all the requirements." } responses: '200': - description: Auto generated using Swagger Inspector - servers: - - url: 'https://api.uatbeta.healthdatagateway.org' - servers: - - url: 'https://api.uatbeta.healthdatagateway.org' - - /api/v1/publishers/OTHER > HEALTH DATA RESEARCH UK/dataaccessrequests: + description: Successful response containing the full, updated data access request application. + content: + application/json: + schema: + type: object + properties: + status: + type: string + data: + type: object + properties: + id: + type: string + description: The unique identifier for the application. + aboutApplication: + description: An object which holds data relating to the 'about application' section of the application form including details of whether the project is an NCS project or not. + type: object + properties: + isNationalCoreStudies: + type: boolean + description: A flag to indicate if this application is in relation to a National Core Studies Project. + nationalCoreStudiesProjectId: + type: integer + description: The unique identifier correlating to a Gateway Project entity indicating that this application is relating to a National Core Studies project. + projectName: + type: string + description: The project name that has been assigned to the application by the applicant(s). + authorIds: + type: array + items: + type: integer + description: An array of values correlating to specific user's via their numeric identifiers. An author is also known as a contributor to an application and can view, edit or submit. + datasetIds: + type: array + items: + type: string + description: An array of values correlating to datasets selected for the application via their identifier, which is unique per version. + datasetTitles: + type: array + items: + type: string + description: An array of strings correlating to the dataset titles that have been selected for the application. + applicationStatus: + type: string + enum: + - inProgress + - submitted + - inReview + - approved + - rejected + - approved with conditions + description: The current status of the application. + jsonSchema: + type: object + description: The object containing the json definition that renders the application form using the Winterfell library. This contains the details of questions, questions sets, question panels, headings and navigation items that appear. + questionAnswers: + type: object + description: The object containing the answers provided on the application form. This consists of a series of key pairs, where the key is the unqiue question Id, and the value the is the answer provided to the question. In the case of a multi select on the form, the value may be an array. + publisher: + type: string + description: The name of the Custodian that holds the dataset and is processing the application. + publisherObj: + type: object + description: The object containing details regarding the Custodian/publisher relating to the application. + userId: + type: integer + description: The unique identifier that correlates to the user account of the main applicant. This is always the user that started the application. + schemaId: + type: string + description: The unique identifier that correlates to the schema from which the application form was generated. + files: + type: array + items: + type: object + description: An array containing the links to files that have been uploaded to the application form and are held within the Gateway ecosystem. + amendmentIterations: + type: array + items: + type: object + description: An array containing an object with details for each iteration the application has passed through. An iteration is defined as an application which has been returned by the Custodian for correction, corrected by the applicant(s) and resubmitted. The object contains dates that the application was returned, and resubmitted as well as reference to any questions that were highlighted for amendment. + createdAt: + type: string + description: The date and time that the application was started. + updatedAt: + type: string + description: The date and time that the application was last updated by any party. + projectId: + type: string + description: The unique identifier for the application converted to a more human friendly format in uppercase and hypenated. + dateSubmitted: + type: string + description: The date and time that the application was originally submitted by the applicant(s) to the Custodian for review. + dateReviewStart: + type: string + description: The date and time that the review process was commenced by a Custodian manager. The review starts from the moment the manager opens the application to triage it. + dateFinalStatus: + type: string + description: The date and time that the Custodian triggered a status change to the application once a final decision was made. E.g. when application was approved. This date can be used in conjunction with the dateReviewStart date to calculate the length of time the Custodian took to make a decision through their review process. + datasets: + type: array + items: + type: object + description: An array containing the full metadata for each of the datasets that have been applied for through this application. + mainApplicant: + type: object + description: An object containing the details of the main applicant of the application as referenced by the userId field. + authors: + type: array + items: + type: object + description: An array containing the details of the contributors of the application as referenced by the authorIds field. + readOnly: + type: boolean + description: A value to indicate if the requesting party is able to modify the application in its present state. For example, this will be false for a Custodian, but true for applicants if the applicant(s) are working on resubmitting the application following a request for amendments. + unansweredAmendments: + type: integer + description: The number of amendments that have been requested by the Custodian in the current amendment iteration. + answeredAmendments: + type: integer + description: The number of requested amendments that the applicant(s) have fixed in the current amendment iteration. + userType: + type: string + enum: + - custodian + - applicant + description: The type of user that has requested the Data Access Request application based on their permissions. It is either an applicant or a Custodian user. + activeParty: + type: string + enum: + - custodian + - applicant + description: The party that is currently handling the application. This is the applicant during presubmission, then the Custodian following submission. The active party then fluctuates between parties during amendment iterations. + inReviewMode: + type: boolean + description: A flag to indicate if the current user is a reviewer of the application. This value will be false unless the requesting user is an assigned reviewer to a currently active workflow step. When this value is true, the requesting user is able to recommend approval or rejection of the application. + reviewSections: + type: array + items: + type: string + description: An array containing the sections of the application form that the current user is required to review if they are a reviewer of the current workflow step that the application is in. E.g. ['Safe People','Safe Data'] + hasRecommended: + type: boolean + description: A flag to indicate if the current user as a reviewer of the current workflow phase has submitted their recommendation for approval or rejection based on their review of the review sections assigned to them. + workflow: + type: object + description: The full details of the workflow that has been assigned to the Data Access Request application. This includes information such as the review phases that the application will pass through and associated metadata. + examples: + 'Approved Application': + value: + { + "status": "success", + "data": { + "aboutApplication": { + "selectedDatasets": [ + { + "_id": "5fc31a18d98e4f4cff7e9315", + "datasetId": "d5faf9c6-6c34-46d7-93c4-7706a5436ed9", + "name": "HDR UK Papers & Preprints", + "description": "Publications that mention HDR-UK (or any variant thereof) in Acknowledgements or Author Affiliations\n\nThis will include:\n- Papers\n- COVID-19 Papers\n- COVID-19 Preprint", + "abstract": "Publications that mention HDR-UK (or any variant thereof) in Acknowledgements or Author Affiliations", + "publisher": "OTHER > HEALTH DATA RESEARCH UK", + "contactPoint": "hdr.hdr@hdruk.ac.uk", + "publisherObj": { + "dataRequestModalContent": { + "header": " ", + "body": "{omitted for brevity...}", + "footer": "" + }, + "active": true, + "allowsMessaging": true, + "workflowEnabled": true, + "_id": "5f7b1a2bce9f65e6ed83e7da", + "name": "OTHER > HEALTH DATA RESEARCH UK", + "imageURL": "", + "team": { + "active": true, + "_id": "5f7b1a2bce9f65e6ed83e7da", + "members": [ + { + "roles": [ + "manager" + ], + "memberid": "5f1a98861a821b4a53e44d15" + }, + { + "roles": [ + "manager" + ], + "memberid": "600bfc99c8bf700f2c7d5c36" + } + ], + "type": "publisher", + "__v": 3, + "createdAt": "2020-11-30T21:12:40.855Z", + "updatedAt": "2020-12-02T13:33:45.232Z" + } + } + } + ], + "isNationalCoreStudies": true, + "nationalCoreStudiesProjectId": "4324836585275824", + "projectName": "Test application title", + "completedDatasetSelection": true, + "completedInviteCollaborators": true, + "completedReadAdvice": true, + "completedCommunicateAdvice": true, + "completedApprovalsAdvice": true, + "completedSubmitAdvice": true + }, + "authorIds": [], + "datasetIds": [ + "d5faf9c6-6c34-46d7-93c4-7706a5436ed9" + ], + "datasetTitles": [], + "applicationStatus": "approved", + "jsonSchema": "{omitted for brevity...}", + "questionAnswers": { + "fullname-892140ec730145dc5a28b8fe139c2876": "James Smith", + "jobtitle-ff1d692a04b4bb9a2babe9093339136f": "Consultant", + "organisation-65c06905b8319ffa29919732a197d581": "Consulting Inc." + }, + "publisher": "OTHER > HEALTH DATA RESEARCH UK", + "_id": "60142c5b4316a0e0fcd47c56", + "version": 1, + "userId": 9190228196797084, + "schemaId": "5f55e87e780ba204b0a98eb8", + "files": [], + "amendmentIterations": [], + "createdAt": "2021-01-29T15:40:11.943Z", + "updatedAt": "2021-02-03T14:38:22.688Z", + "__v": 0, + "projectId": "6014-2C5B-4316-A0E0-FCD4-7C56", + "dateSubmitted": "2021-01-29T16:30:27.351Z", + "dateReviewStart": "2021-02-03T14:36:22.341Z", + "dateFinalStatus": "2021-02-03T14:38:22.680Z", + "datasets": [ + "{omitted for brevity...}" + ], + "dataset": null, + "mainApplicant": { + "_id": "5f1a98861a821b4a53e44d15", + "firstname": "James", + "lastname": "Smith" + }, + "authors": [], + "id": "60142c5b4316a0e0fcd47c56", + "readOnly": true, + "unansweredAmendments": 0, + "answeredAmendments": 0, + "userType": "custodian", + "activeParty": "custodian", + "inReviewMode": false, + "reviewSections": [], + "hasRecommended": false, + "workflow": {} + } + } + '404': + description: Failed to find the application requested. + content: + application/json: + schema: + type: object + properties: + status: + type: string + message: + type: string + examples: + 'Not Found': + value: + { + "status": "error", + "message": "Application not found." + } + '401': + description: Unauthorised attempt to update an application. + content: + application/json: + schema: + type: object + properties: + status: + type: string + message: + type: string + examples: + 'Unauthorised': + value: + { + "status":"error", + "message":"Unauthorised to perform this update." + } + patch: + summary: Update a users question answers for access request. + security: + - cookieAuth: [] + tags: + - Data Access Request + parameters: + - in: path + name: id + required: true + description: The ID of the datset + schema: + type : string + example: 5ee249426136805fbf094eef + requestBody: + content: + application/json: + schema: + type: object + properties: + questionAnswers: + type: object + examples: + '0': + value: |- + { + "firstName": "Roger" + } + responses: + '200': + description: OK + /api/v1/publishers/{publisher}/dataaccessrequests: get: tags: - Publishers - description: Auto generated using Swagger Inspector + parameters: + - in: path + name: publisher + required: true + description: The full name of the Custodian/Publisher, as registered on the Gateway. + schema: + type : string + example: OTHER > HEALTH DATA RESEARCH UK + description: Returns a collection of all Data Access Requests that have been submitted to the Custodian team for review. responses: '200': - description: Auto generated using Swagger Inspector - servers: - - url: 'https://api.uatbeta.healthdatagateway.org' - servers: - - url: 'https://api.uatbeta.healthdatagateway.org' - /v1/datasets/{datasetID}: + description: Successful response containing a collection of Data Access Request applications. + content: + application/json: + schema: + type: object + properties: + avgDecisionTime: + type: string + description: The average number of days the Custodian has taken to process applications from submission to decision. + canViewSubmitted: + type: boolean + description: A flag to indicate if the requesting user has permissions to view submitted applications, which are visible only to managers of the Custodian team. Using OAuth2.0 client credentials will return this value as true. + status: + type: string + data: + type: array + items: + type: object + properties: + aboutApplication: + description: An object which holds data relating to the 'about application' section of the application form including details of whether the project is an NCS project or not. + type: object + properties: + isNationalCoreStudies: + type: boolean + description: A flag to indicate if this application is in relation to a National Core Studies Project. + nationalCoreStudiesProjectId: + type: integer + description: The unique identifier correlating to a Gateway Project entity indicating that this application is relating to a National Core Studies project. + projectName: + type: string + description: The project name that has been assigned to the application by the applicant(s). + amendmentIterations: + type: array + items: + type: object + description: An array containing an object with details for each iteration the application has passed through. An iteration is defined as an application which has been returned by the Custodian for correction, corrected by the applicant(s) and resubmitted. The object contains dates that the application was returned, and resubmitted as well as reference to any questions that were highlighted for amendment. + amendmentStatus: + type: string + description: A textual indicator of what state the application is in relating to updates made by the Custodian e.g. if it is awaiting updates from the applicant or if new updates have been submitted by the applicant(s). + applicants: + type: string + description: Concatenated list of applicants names who are contributing to the application. + applicationStatus: + type: string + enum: + - inProgress + - submitted + - inReview + - approved + - rejected + - approved with conditions + description: The current status of the application. + authorIds: + type: array + items: + type: integer + description: An array of values correlating to specific user's via their numeric identifiers. An author is also known as a contributor to an application and can view, edit or submit. + createdAt: + type: string + description: The date and time that the application was started. + datasetIds: + type: array + items: + type: string + description: An array of values correlating to datasets selected for the application via their identifier, which is unique per version. + datasetTitles: + type: array + items: + type: string + description: An array of strings correlating to the dataset titles that have been selected for the application. + datasets: + type: array + items: + type: object + description: An array containing the full metadata for each of the datasets that have been applied for through this application. + dateSubmitted: + type: string + description: The date and time that the application was originally submitted by the applicant(s) to the Custodian for review. + files: + type: array + items: + type: object + description: An array containing the links to files that have been uploaded to the application form and are held within the Gateway ecosystem. + id: + type: string + description: The unique identifier for the application. + + jsonSchema: + type: object + description: The object containing the json definition that renders the application form using the Winterfell library. This contains the details of questions, questions sets, question panels, headings and navigation items that appear. + questionAnswers: + type: object + description: The object containing the answers provided on the application form. This consists of a series of key pairs, where the key is the unqiue question Id, and the value the is the answer provided to the question. In the case of a multi select on the form, the value may be an array. + mainApplicant: + type: object + description: An object containing the details of the main applicant of the application as referenced by the userId field. + projectId: + type: string + description: The unique identifier for the application converted to a more human friendly format in uppercase and hypenated. + projectName: + type: string + description: The project name that has been assigned to the application by the applicant(s). + publisher: + type: string + description: The name of the Custodian that holds the dataset and is processing the application. + publisherObj: + type: object + description: The object containing details regarding the Custodian/publisher relating to the application. + reviewPanels: + type: array + items: + type: string + description: An array containing the sections of the application form that the current user is required to review if they are a reviewer of the current workflow step that the application is in. E.g. ['Safe People','Safe Data'] + schemaId: + type: string + description: The unique identifier that correlates to the schema from which the application form was generated. + updatedAt: + type: string + description: The date and time that the application was last updated by any party. + userId: + type: integer + description: The unique identifier that correlates to the user account of the main applicant. This is always the user that started the application. + deadlinePassed: + type: boolean + description: A flag to indicate if the deadline has passed for the current review phase for this application. + decisionApproved: + type: boolean + description: A flag to indicate if the request users decision as a reviewer of the current workflow phase was positive or negative. i.e. correlating to approval or rejection recommendation. + decisionComments: + type: string + description: A supporting note or comment made by the requesting user as context to their decision as a reviewer of the current workflow phase. + decisionDate: + type: string + description: The date that the requesting user made their decision as a reviewer of the current workflow phase. + decisionDuration: + type: integer + description: The number of days from submission until a final decision was made on the application. i.e. the application status was changed to a final status e.g. 'Approved'. + decisionMade: + type: boolean + description: A flag to indicate if the requesting user has made an expected decision as a reviewer of the current workflow phase. + decisionStatus: + type: string + description: A message indicating if the requesting user as a reviewer of the application has made a decision or is still required to make a decision for the current work flow. + isReviewer: + type: boolean + description: A flag to indicate if the requesting user is a reviewer of the current workflow step for the application. + remainingActioners: + type: array + items: + type: string + description: An array containing the names of Custodian team reviewers expected to complete a review for the current workflow phase, or a list of managers if the application is awaiting a final decision. + reviewStatus: + type: string + description: A message indicating the current status of the application review in relation to the assigned workflow. E.g. 'Final decision required' or 'Deadline is today'. This message changes based on the requesting user's relationship to the application. E.g. if they are a reviewer or manager. + stepName: + type: string + description: The name of the current workflow step that the application is in. + workflowCompleted: + type: boolean + description: A flag to indicate if the assigned workflow for the review process has been completed. + workflowName: + type: string + description: The name of the workflow the Custodian team have assigned to the application for the review process. + examples: + 'Single Request Received': + value: + { + "success": true, + "data": [ + { + "authorIds": [], + "datasetIds": [ + "d5faf9c6-6c34-46d7-93c4-7706a5436ed9" + ], + "datasetTitles": [], + "applicationStatus": "submitted", + "jsonSchema": "{omitted for brevity...}", + "questionAnswers": "{omitted for brevity...}", + "publisher": "OTHER > HEALTH DATA RESEARCH UK", + "_id": "601853db22dc004f9adfaa24", + "version": 1, + "userId": 7584453789581072, + "schemaId": "5f55e87e780ba204b0a98eb8", + "files": [ + { + "error": "", + "_id": "601aacf8ecdfa66e5cbc2742", + "status": "UPLOADED", + "description": "QuestionAnswers", + "fileId": "9e76ee1a676f423b9b5c7aabf59c69db", + "size": 509984, + "name": "QuestionAnswersFlags.png", + "owner": "5ec7f1b39219d627e5cafae3" + }, + { + "error": "", + "_id": "601aadbcecdfa6c532bc2743", + "status": "UPLOADED", + "description": "Notifications", + "fileId": "adb1718dcc094b9cb4b0ab347ad2ee94", + "size": 54346, + "name": "HQIP-Workflow-Assigned-Notification.png", + "owner": "5ec7f1b39219d627e5cafae3" + } + ], + "amendmentIterations": [], + "createdAt": "2021-02-01T19:17:47.470Z", + "updatedAt": "2021-02-03T16:36:36.720Z", + "__v": 2, + "projectId": "6018-53DB-22DC-004F-9ADF-AA24", + "aboutApplication": { + "selectedDatasets": [ + { + "_id": "5fc31a18d98e4f4cff7e9315", + "datasetId": "d5faf9c6-6c34-46d7-93c4-7706a5436ed9", + "name": "HDR UK Papers & Preprints", + "description": "Publications that mention HDR-UK (or any variant thereof) in Acknowledgements or Author Affiliations\n\nThis will include:\n- Papers\n- COVID-19 Papers\n- COVID-19 Preprint", + "abstract": "Publications that mention HDR-UK (or any variant thereof) in Acknowledgements or Author Affiliations", + "publisher": "OTHER > HEALTH DATA RESEARCH UK", + "contactPoint": "hdr.hdr@hdruk.ac.uk", + "publisherObj": { + "dataRequestModalContent": { + "header": " ", + "body": "{omitted for brevity...}", + "footer": "" + }, + "active": true, + "allowsMessaging": true, + "workflowEnabled": true, + "_id": "5f7b1a2bce9f65e6ed83e7da", + "name": "OTHER > HEALTH DATA RESEARCH UK", + "imageURL": "", + "team": { + "active": true, + "_id": "5f7b1a2bce9f65e6ed83e7da", + "members": [ + { + "roles": [ + "manager" + ], + "memberid": "5f1a98861a821b4a53e44d15" + }, + { + "roles": [ + "manager" + ], + "memberid": "600bfc99c8bf700f2c7d5c36" + } + ], + "type": "publisher", + "__v": 3, + "createdAt": "2020-11-30T21:12:40.855Z", + "updatedAt": "2020-12-02T13:33:45.232Z" + } + } + } + ], + "isNationalCoreStudies": true, + "nationalCoreStudiesProjectId": "4324836585275824", + "projectName": "Test application title", + "completedDatasetSelection": true, + "completedInviteCollaborators": true, + "completedReadAdvice": true, + "completedCommunicateAdvice": true, + "completedApprovalsAdvice": true, + "completedSubmitAdvice": true + }, + "dateSubmitted": "2021-02-03T16:37:36.081Z", + "datasets": [ + { + "categories": { + "programmingLanguage": [] + }, + "tags": { + "features": [ + "Preprints", + "Papers", + "HDR UK" + ], + "topics": [] + }, + "datasetfields": { + "geographicCoverage": [ + "https://www.geonames.org/countries/GB/united-kingdom.html" + ], + "physicalSampleAvailability": [ + "Not Available" + ], + "technicaldetails": "{omitted for brevity...}", + "versionLinks": [ + { + "id": "142b1618-2691-4019-97b4-16b1e27c5f95", + "linkType": "Superseded By", + "domainType": "CatalogueSemanticLink", + "source": { + "id": "9e798632-442a-427b-8d0e-456f754d28dc", + "domainType": "DataModel", + "label": "HDR UK Papers & Preprints", + "documentationVersion": "0.0.1" + }, + "target": { + "id": "d5faf9c6-6c34-46d7-93c4-7706a5436ed9", + "domainType": "DataModel", + "label": "HDR UK Papers & Preprints", + "documentationVersion": "1.0.0" + } + } + ], + "phenotypes": [], + "publisher": "OTHER > HEALTH DATA RESEARCH UK", + "abstract": "Publications that mention HDR-UK (or any variant thereof) in Acknowledgements or Author Affiliations", + "releaseDate": "2020-11-27T00:00:00Z", + "accessRequestDuration": "Other", + "conformsTo": "OTHER", + "accessRights": "https://github.com/HDRUK/papers/blob/master/LICENSE", + "jurisdiction": "GB-ENG", + "datasetStartDate": "2020-03-31", + "datasetEndDate": "2022-04-30", + "statisticalPopulation": "0", + "ageBand": "0-0", + "contactPoint": "hdr.hdr@hdruk.ac.uk", + "periodicity": "Daily", + "metadataquality": { + "id": "d5faf9c6-6c34-46d7-93c4-7706a5436ed9", + "publisher": "OTHER > HEALTH DATA RESEARCH UK", + "title": "HDR UK Papers & Preprints", + "completeness_percent": 95.24, + "weighted_completeness_percent": 100, + "error_percent": 11.63, + "weighted_error_percent": 19.05, + "quality_score": 91.81, + "quality_rating": "Gold", + "weighted_quality_score": 90.47, + "weighted_quality_rating": "Gold" + }, + "datautility": { + "id": "d5faf9c6-6c34-46d7-93c4-7706a5436ed9", + "publisher": "OTHER > HEALTH DATA RESEARCH UK", + "title": "HDR UK Papers & Preprints", + "metadata_richness": "Gold", + "availability_of_additional_documentation_and_support": "", + "data_model": "", + "data_dictionary": "", + "provenance": "", + "data_quality_management_process": "", + "dama_quality_dimensions": "", + "pathway_coverage": "", + "length_of_follow_up": "", + "allowable_uses": "", + "research_environment": "", + "time_lag": "", + "timeliness": "", + "linkages": "", + "data_enrichments": "" + }, + "metadataschema": { + "@context": "http://schema.org/", + "@type": "Dataset", + "identifier": "d5faf9c6-6c34-46d7-93c4-7706a5436ed9", + "url": "https://healthdatagateway.org/detail/d5faf9c6-6c34-46d7-93c4-7706a5436ed9", + "name": "HDR UK Papers & Preprints", + "description": "Publications that mention HDR-UK (or any variant thereof) in Acknowledgements or Author Affiliations\n\nThis will include:\n- Papers\n- COVID-19 Papers\n- COVID-19 Preprint", + "license": "Open Access", + "keywords": [ + "Preprints,Papers,HDR UK", + "OTHER > HEALTH DATA RESEARCH UK", + "NOT APPLICABLE", + "GB-ENG", + "https://www.geonames.org/countries/GB/united-kingdom.html" + ], + "includedinDataCatalog": [ + { + "@type": "DataCatalog", + "name": "OTHER > HEALTH DATA RESEARCH UK", + "url": "hdr.hdr@hdruk.ac.uk" + }, + { + "@type": "DataCatalog", + "name": "HDR UK Health Data Gateway", + "url": "http://healthdatagateway.org" + } + ] + } + }, + "authors": [], + "showOrganisation": false, + "toolids": [], + "datasetids": [], + "_id": "5fc31a18d98e4f4cff7e9315", + "relatedObjects": [], + "programmingLanguage": [], + "pid": "b7a62c6d-ed00-4423-ad27-e90b71222d8e", + "datasetVersion": "1.0.0", + "id": 9816147066244124, + "datasetid": "d5faf9c6-6c34-46d7-93c4-7706a5436ed9", + "type": "dataset", + "activeflag": "active", + "name": "HDR UK Papers & Preprints", + "description": "Publications that mention HDR-UK (or any variant thereof) in Acknowledgements or Author Affiliations\n\nThis will include:\n- Papers\n- COVID-19 Papers\n- COVID-19 Preprint", + "license": "Open Access", + "datasetv2": { + "identifier": "", + "version": "", + "issued": "", + "modified": "", + "revisions": [], + "summary": { + "title": "", + "abstract": "Publications that mention HDR-UK (or any variant thereof) in Acknowledgements or Author Affiliations", + "publisher": { + "identifier": "", + "name": "HEALTH DATA RESEARCH UK", + "logo": "", + "description": "", + "contactPoint": "hdr.hdr@hdruk.ac.uk", + "memberOf": "OTHER", + "accessRights": [], + "deliveryLeadTime": "", + "accessService": "", + "accessRequestCost": "", + "dataUseLimitation": [], + "dataUseRequirements": [] + }, + "contactPoint": "hdr.hdr@hdruk.ac.uk", + "keywords": [ + "Preprints", + "Papers", + "HDR UK" + ], + "alternateIdentifiers": [], + "doiName": "https://doi.org/10.5281/zenodo.326615" + }, + "documentation": { + "description": "", + "associatedMedia": [ + "https://github.com/HDRUK/papers" + ], + "isPartOf": "NOT APPLICABLE" + }, + "coverage": { + "spatial": "GB", + "typicalAgeRange": "0-0", + "physicalSampleAvailability": [ + "NOT AVAILABLE" + ], + "followup": "UNKNOWN", + "pathway": "NOT APPLICABLE" + }, + "provenance": { + "origin": { + "purpose": "OTHER", + "source": "MACHINE GENERATED", + "collectionSituation": "OTHER" + }, + "temporal": { + "accrualPeriodicity": "DAILY", + "distributionReleaseDate": "2020-11-27", + "startDate": "2020-03-31", + "endDate": "2022-04-30", + "timeLag": "NO TIMELAG" + } + }, + "accessibility": { + "usage": { + "dataUseLimitation": "GENERAL RESEARCH USE", + "dataUseRequirements": "RETURN TO DATABASE OR RESOURCE", + "resourceCreator": "HDR UK Using Team", + "investigations": [ + "https://github.com/HDRUK/papers" + ], + "isReferencedBy": [ + "Not Available" + ] + }, + "access": { + "accessRights": [ + "Open Access" + ], + "accessService": "https://github.com/HDRUK/papers", + "accessRequestCost": "Free", + "deliveryLeadTime": "OTHER", + "jurisdiction": "GB-ENG", + "dataProcessor": "HDR UK", + "dataController": "HDR UK" + }, + "formatAndStandards": { + "vocabularyEncodingScheme": "OTHER", + "conformsTo": "OTHER", + "language": "en", + "format": [ + "csv", + "JSON" + ] + } + }, + "enrichmentAndLinkage": { + "qualifiedRelation": [ + "Not Available" + ], + "derivation": [ + "Not Available" + ], + "tools": [ + "https://github.com/HDRUK/papers" + ] + }, + "observations": [] + }, + "createdAt": "2020-11-29T03:48:41.794Z", + "updatedAt": "2021-02-02T10:09:57.030Z", + "__v": 0, + "counter": 20 + } + ], + "dataset": null, + "mainApplicant": { + "isServiceAccount": false, + "_id": "5ec7f1b39219d627e5cafae3", + "id": 7584453789581072, + "providerId": "112563375053074694443", + "provider": "google", + "firstname": "Chris", + "lastname": "Marks", + "email": "chris.marks@paconsulting.com", + "role": "Admin", + "__v": 0, + "redirectURL": "/tool/100000012", + "discourseKey": "2f52ecaa21a0d0223a119da5a09f8f8b09459e7b69ec3f981102d09f66488d99", + "discourseUsername": "chris.marks", + "updatedAt": "2021-02-01T12:39:56.372Z" + }, + "publisherObj": { + "dataRequestModalContent": { + "header": "", + "body": "", + "footer": "" + }, + "active": true, + "allowsMessaging": true, + "workflowEnabled": true, + "_id": "5f7b1a2bce9f65e6ed83e7da", + "name": "OTHER > HEALTH DATA RESEARCH UK", + "imageURL": "", + "team": { + "active": true, + "_id": "5f7b1a2bce9f65e6ed83e7da", + "members": [ + { + "roles": [ + "manager" + ], + "memberid": "5f1a98861a821b4a53e44d15" + }, + { + "roles": [ + "manager" + ], + "memberid": "600bfc99c8bf700f2c7d5c36" + } + ], + "type": "publisher", + "__v": 3, + "createdAt": "2020-11-30T21:12:40.855Z", + "updatedAt": "2020-12-02T13:33:45.232Z", + "users": [ + { + "_id": "5f1a98861a821b4a53e44d15", + "firstname": "Robin", + "lastname": "Kavanagh" + }, + { + "_id": "600bfc99c8bf700f2c7d5c36", + "firstname": "HDR-UK", + "lastname": "Service Account" + } + ] + } + }, + "id": "601853db22dc004f9adfaa24", + "projectName": "PA Paper", + "applicants": "Chris Marks", + "workflowName": "", + "workflowCompleted": false, + "decisionDuration": "", + "decisionMade": false, + "decisionStatus": "", + "decisionComments": "", + "decisionDate": "", + "decisionApproved": false, + "remainingActioners": "Robin Kavanagh (you), HDR-UK Service Account", + "stepName": "", + "deadlinePassed": "", + "reviewStatus": "", + "isReviewer": false, + "reviewPanels": [], + "amendmentStatus": "" + } + ], + "avgDecisionTime": 1, + "canViewSubmitted": true + } + '404': + description: Failed to find the application requested. + content: + application/json: + schema: + type: object + properties: + success: + type: boolean + examples: + 'Not Found': + value: + { + "success":false + } + '401': + description: Unauthorised attempt to access an application. + content: + application/json: + schema: + type: object + properties: + status: + type: string + message: + type: string + examples: + 'Unauthorised': + value: + { + "status":"failure", + "message":"Unauthorised" + } + /api/v1/datasets/{datasetID}: get: summary: Returns Dataset object. tags: @@ -114,7 +1429,7 @@ paths: '200': description: OK - /v1/datasets/: + /api/v1/datasets: get: summary: Returns List of Dataset objects. tags: @@ -145,7 +1460,7 @@ paths: '200': description: OK - /v1/data-access-request/{datasetID}: + /api/v1/data-access-request/{datasetID}: get: summary: Returns access request template. security: @@ -164,40 +1479,7 @@ paths: '200': description: OK - /v1/data-access-request/{id}: - patch: - summary: Update a users question answers for access request. - security: - - cookieAuth: [] - tags: - - Data Access Request - parameters: - - in: path - name: id - required: true - description: The ID of the datset - schema: - type : string - example: 5ee249426136805fbf094eef - requestBody: - content: - application/json: - schema: - type: object - properties: - questionAnswers: - type: object - examples: - '0': - value: |- - { - "firstName": "Roger" - } - responses: - '200': - description: OK - - /v1/person/{id}: + /api/v1/person/{id}: get: summary: Returns details for a person. tags: @@ -214,7 +1496,7 @@ paths: '200': description: OK - /v1/person/: + /api/v1/person: get: summary: Returns an array of person objects. tags: @@ -305,7 +1587,7 @@ paths: emailNotifications: false terms: true - /v1/search/: + /api/v1/search: get: tags: - Search @@ -346,7 +1628,7 @@ paths: '200': description: OK - /v1/stats/topSearches: + /api/v1/stats/topSearches: get: summary: Returns top searches for a given month and year. tags: @@ -370,7 +1652,7 @@ paths: '200': description: OK - /v1/stats/: + /api/v1/stats: get: summary: Returns the details on recent searches, popular objects, unmet demands or recently updated objects based on the rank query parameter. tags: @@ -408,7 +1690,7 @@ paths: '200': description: OK - /v1/kpis/: + /api/v1/kpis: get: summary: Returns information for KPIs, based on the KPI type and selectedDate parameters. tags: @@ -432,7 +1714,7 @@ paths: '200': description: OK - /v1/messages/{id}: + /api/v1/messages/{id}: delete: summary: Delete a Message security: @@ -482,7 +1764,7 @@ paths: '204': description: OK - /v1/messages/unread/count: + /api/v1/messages/unread/count: get: summary: Returns the number of unread messages for the authenticated user security: @@ -493,7 +1775,7 @@ paths: '200': description: OK - /v1/messages/: + /api/v1/messages: post: summary: Returns a new Message object and creates an associated parent Topic if a Topic is not specified in request body security: @@ -529,7 +1811,7 @@ paths: '201': description: OK - /v1/topics/: + /api/v1/topics: post: summary: Returns a new Topic object with ID (Does not create any associated messages) security: @@ -601,7 +1883,7 @@ paths: '200': description: Ok - /v1/topics/{id}: + /api/v1/topics/{id}: get: summary: Returns Topic object by ID security: @@ -637,7 +1919,7 @@ paths: '204': description: Ok - /v1/projects/: + /api/v1/projects: post: summary: Returns a Project object with ID. security: @@ -730,7 +2012,7 @@ paths: '200': description: OK - /v1/projects/{id}: + /api/v1/projects/{id}: get: summary: Returns Project object. tags: @@ -853,7 +2135,7 @@ paths: '200': description: OK - /v1/papers/: + /api/v1/papers: post: summary: Returns a Paper object with ID. security: @@ -946,7 +2228,7 @@ paths: '200': description: OK - /v1/papers/{id}: + /api/v1/papers/{id}: get: summary: Returns Paper object. tags: @@ -1074,7 +2356,7 @@ paths: '200': description: OK - /v1/tools/: + /api/v1/tools: get: summary: Return List of Tool objects. tags: @@ -1160,7 +2442,7 @@ paths: '200': description: OK - /v1/tools/{id}: + /api/v1/tools/{id}: get: summary: Returns Tool object. tags: From dd1893550d200c4d0d3a72c7f24c555c6e3d7009 Mon Sep 17 00:00:00 2001 From: Robin Kavanagh Date: Fri, 5 Feb 2021 17:21:12 +0000 Subject: [PATCH 03/42] Fixed version, servers and title --- swagger.yaml | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/swagger.yaml b/swagger.yaml index d2983a37..4390de0d 100644 --- a/swagger.yaml +++ b/swagger.yaml @@ -1,10 +1,20 @@ openapi: 3.0.1 info: - title: defaultTitle - description: defaultDescription - version: '0.1' + title: HDR UK API + description: API for Tools and artefacts repository. + version: 1.0.0 servers: - - url: 'https://api.www.healthdatagateway.org' + - url: https://api.www.healthdatagateway.org/api + - url: http://localhost:3001/api + - url: https://api.{environment}.healthdatagateway.org:{port}/api + variables: + environment: + default: latest + description: The Environment name. + port: + enum: + - '443' + default: '443' security: - oauth2: [] paths: From 13fd4601d7f7754514d7954bed6131a5e709312a Mon Sep 17 00:00:00 2001 From: CiaraWardPA Date: Mon, 8 Feb 2021 14:09:53 +0000 Subject: [PATCH 04/42] IG-1243 start of searchable collections - all public active collections returned from search with count --- src/resources/search/search.repository.js | 73 +++++++++++++++++++++-- src/resources/search/search.router.js | 36 ++++++++++- 2 files changed, 104 insertions(+), 5 deletions(-) diff --git a/src/resources/search/search.repository.js b/src/resources/search/search.repository.js index 2473217c..0ba42b8b 100644 --- a/src/resources/search/search.repository.js +++ b/src/resources/search/search.repository.js @@ -1,13 +1,21 @@ import { Data } from '../tool/data.model'; import { Course } from '../course/course.model'; +import { Collections } from '../collections/collections.model'; import _ from 'lodash'; import moment from 'moment'; export function getObjectResult(type, searchAll, searchQuery, startIndex, maxResults, sort) { let collection = Data; - if (type === 'course') collection = Course; + if (type === 'course') { + collection = Course; + } else if (type === 'collection') { + collection = Collections; + } + console.log(`searchQuery: ${JSON.stringify(searchQuery, null, 2)}`); var newSearchQuery = JSON.parse(JSON.stringify(searchQuery)); - newSearchQuery['$and'].push({ type: type }); + if (type !== 'collection') { + newSearchQuery['$and'].push({ type: type }); + } if (type === 'course') { newSearchQuery['$and'].forEach(x => { @@ -43,6 +51,8 @@ export function getObjectResult(type, searchAll, searchQuery, startIndex, maxRes }, }, ]; + } else if (type === 'collection') { + queryObject = [{ $match: newSearchQuery }, { $lookup: { from: 'tools', localField: 'authors', foreignField: 'id', as: 'persons' } }]; } else { queryObject = [ { $match: newSearchQuery }, @@ -92,6 +102,8 @@ export function getObjectResult(type, searchAll, searchQuery, startIndex, maxRes ]; } + console.log(`queryObject: ${JSON.stringify(queryObject, null, 2)}`); + if (sort === '' || sort === 'relevance') { if (type === 'person') { if (searchAll) queryObject.push({ $sort: { lastname: 1 } }); @@ -127,9 +139,15 @@ export function getObjectResult(type, searchAll, searchQuery, startIndex, maxRes export function getObjectCount(type, searchAll, searchQuery) { let collection = Data; - if (type === 'course') collection = Course; + if (type === 'course') { + collection = Course; + } else if (type === 'collection') { + collection = Collections; + } var newSearchQuery = JSON.parse(JSON.stringify(searchQuery)); - newSearchQuery['$and'].push({ type: type }); + if (type !== 'collection') { + newSearchQuery['$and'].push({ type: type }); + } if (type === 'course') { newSearchQuery['$and'].forEach(x => { if (x.$or) { @@ -186,6 +204,47 @@ export function getObjectCount(type, searchAll, searchQuery) { ]) .sort({ score: { $meta: 'textScore' } }); } + } + // TODO - get count for collections + else if (type === 'collection') { + if (searchAll) { + q = collection.aggregate([ + { $match: newSearchQuery }, + { + $group: { + _id: {}, + count: { + $sum: 1, + }, + }, + }, + { + $project: { + count: '$count', + _id: 0, + }, + }, + ]); + } else { + q = collection.aggregate([ + { $match: newSearchQuery }, + { + $group: { + _id: {}, + count: { + $sum: 1, + }, + }, + }, + { + $project: { + count: '$count', + _id: 0, + }, + }, + ]); + // .sort({ score: { $meta: 'textScore' } }); + } } else { if (searchAll) { q = collection.aggregate([ @@ -266,6 +325,7 @@ export function getObjectFilters(searchQueryStart, req, type) { courseentrylevel = '', courseframework = '', coursepriority = '', + //TODO - add collection filter options in here } = req.query; if (type === 'dataset') { @@ -479,6 +539,11 @@ export function getObjectFilters(searchQueryStart, req, type) { searchQuery['$and'].push({ $or: filterTermArray }); } } + // TODO - add in collection filters + else if (type === 'collection') { + searchQuery['$and'].push({ publicflag: true }); + console.log(`collection searchQuery: ${JSON.stringify(searchQuery, null, 2)}`); + } return searchQuery; } diff --git a/src/resources/search/search.router.js b/src/resources/search/search.router.js index 4037b10a..96c32900 100644 --- a/src/resources/search/search.router.js +++ b/src/resources/search/search.router.js @@ -52,7 +52,9 @@ router.get('/', async (req, res) => { projectResults = [], paperResults = [], personResults = [], - courseResults = []; + courseResults = [], + // TODO + collectionResults = []; if (tab === '') { allResults = await Promise.all([ @@ -97,6 +99,15 @@ router.get('/', async (req, res) => { req.query.maxResults || 40, 'startdate' ), + // TODO + getObjectResult( + 'collection', + searchAll, + getObjectFilters(searchQuery, req, 'collection'), + req.query.collectionIndex || 0, + req.query.maxResults || 40, + 'startdate' + ), ]); } else if (tab === 'Datasets') { datasetResults = await Promise.all([ @@ -158,6 +169,19 @@ router.get('/', async (req, res) => { ), ]); } + // TODO + else if (tab === 'Collections') { + collectionResults = await Promise.all([ + getObjectResult( + 'collection', + searchAll, + getObjectFilters(searchQuery, req, 'collection'), + req.query.collectionIndex || 0, + req.query.maxResults || 40, + 'startdate' + ), + ]); + } var summaryCounts = await Promise.all([ getObjectCount('dataset', searchAll, getObjectFilters(searchQuery, req, 'dataset')), @@ -166,6 +190,8 @@ router.get('/', async (req, res) => { getObjectCount('paper', searchAll, getObjectFilters(searchQuery, req, 'paper')), getObjectCount('person', searchAll, searchQuery), getObjectCount('course', searchAll, getObjectFilters(searchQuery, req, 'course')), + //TODO + getObjectCount('collection', searchAll, getObjectFilters(searchQuery, req, 'collection')), ]); var summary = { @@ -175,6 +201,8 @@ router.get('/', async (req, res) => { papers: summaryCounts[3][0] !== undefined ? summaryCounts[3][0].count : 0, persons: summaryCounts[4][0] !== undefined ? summaryCounts[4][0].count : 0, courses: summaryCounts[5][0] !== undefined ? summaryCounts[5][0].count : 0, + //TODO + collections: summaryCounts[5][0] !== undefined ? summaryCounts[6][0].count : 0, }; let recordSearchData = new RecordSearchData(); @@ -185,6 +213,8 @@ router.get('/', async (req, res) => { recordSearchData.returned.paper = summaryCounts[3][0] !== undefined ? summaryCounts[3][0].count : 0; recordSearchData.returned.person = summaryCounts[4][0] !== undefined ? summaryCounts[4][0].count : 0; recordSearchData.returned.course = summaryCounts[5][0] !== undefined ? summaryCounts[5][0].count : 0; + //TODO + recordSearchData.returned.collection = summaryCounts[6][0] !== undefined ? summaryCounts[6][0].count : 0; recordSearchData.datesearched = Date.now(); recordSearchData.save(err => {}); @@ -197,6 +227,8 @@ router.get('/', async (req, res) => { paperResults: allResults[3], personResults: allResults[4], courseResults: allResults[5], + // TODO + collectionResults: allResults[6], summary: summary, }); } @@ -208,6 +240,8 @@ router.get('/', async (req, res) => { paperResults: paperResults[0], personResults: personResults[0], courseResults: courseResults[0], + // TODO + collectionResults: collectionResults[0], summary: summary, }); }); From 10a4f00f70b60eb6647937663b808dc5ebbb0ffd Mon Sep 17 00:00:00 2001 From: Richard Date: Tue, 9 Feb 2021 15:12:29 +0000 Subject: [PATCH 05/42] Backend logic for Cohort Discovery user validation --- src/config/account.js | 24 ++++++++++++++---------- src/config/configuration.js | 1 + src/resources/auth/auth.route.js | 3 +++ src/resources/user/user.model.js | 4 +++- src/resources/user/user.route.js | 28 ++++++++++++++++++++++++++++ 5 files changed, 49 insertions(+), 11 deletions(-) diff --git a/src/config/account.js b/src/config/account.js index 65c82c8c..ace1d1ab 100755 --- a/src/config/account.js +++ b/src/config/account.js @@ -21,19 +21,23 @@ class Account { * or not return them in id tokens but only userinfo and so on. */ async claims(use, scope) { + let claimsToSend = scope.split(' '); // eslint-disable-line no-unused-vars - if (this.profile) { - return { - sub: this.accountId, // it is essential to always return a sub claim - email: this.profile.email, - firstname: this.profile.firstname, - lastname: this.profile.lastname, - }; - } - - return { + let claim = { sub: this.accountId, // it is essential to always return a sub claim }; + if (claimsToSend.includes('profile')) { + claim.firstname = this.profile.firstname; + claim.lastname = this.profile.lastname; + } + if (claimsToSend.includes('email')) { + claim.email = this.profile.email; + } + if (claimsToSend.includes('rquestroles')) { + claim.rquestroles = this.profile.advancedSearchRoles; + } + + return claim; } static async findByFederated(provider, claims) { diff --git a/src/config/configuration.js b/src/config/configuration.js index e9464305..23657073 100755 --- a/src/config/configuration.js +++ b/src/config/configuration.js @@ -63,6 +63,7 @@ export const cookies = { export const claims = { email: ['email'], profile: ['firstname', 'lastname'], + rquestroles: ['rquestroles'], }; export const features = { diff --git a/src/resources/auth/auth.route.js b/src/resources/auth/auth.route.js index 79eb6df8..f3d54b39 100644 --- a/src/resources/auth/auth.route.js +++ b/src/resources/auth/auth.route.js @@ -89,6 +89,9 @@ router.get('/status', function (req, res, next) { name: req.user.firstname + ' ' + req.user.lastname, loggedIn: true, teams, + provider: req.user.provider, + advancedSearchRoles: req.user.advancedSearchRoles, + acceptedAdvancedSearchTerms: req.user.acceptedAdvancedSearchTerms, }, ], }); diff --git a/src/resources/user/user.model.js b/src/resources/user/user.model.js index e68db2fe..9ec59574 100644 --- a/src/resources/user/user.model.js +++ b/src/resources/user/user.model.js @@ -20,7 +20,9 @@ const UserSchema = new Schema( discourseKey: String, isServiceAccount: { type: Boolean, default: false }, clientId: String, - clientSecret: String + clientSecret: String, + advancedSearchRoles: [], + acceptedAdvancedSearchTerms: Boolean, }, { timestamps: true, diff --git a/src/resources/user/user.route.js b/src/resources/user/user.route.js index 178ced9e..13967afe 100644 --- a/src/resources/user/user.route.js +++ b/src/resources/user/user.route.js @@ -86,6 +86,34 @@ router.get('/', passport.authenticate('jwt'), utils.checkIsInRole(ROLES.Admin, R }); }); +// @router PATCH /api/v1/users/advancedSearch/terms/:id +// @desc Accept the advanced search T&Cs for a user +// @access Private +router.patch('/advancedSearch/terms/:id', passport.authenticate('jwt'), async (req, res) => { + if (parseInt(req.params.id) !== req.user.id) { + return res.status(400).json({ + status: 'error', + message: "Can't accept terms of a different user", + }); + } + let user = await UserModel.findOneAndUpdate( + { id: req.params.id }, + { acceptedAdvancedSearchTerms: req.body.acceptedAdvancedSearchTerms }, + { new: true } + ); + if (!user) return res.status(500).json({ status: 'error', message: 'Request failed' }); + return res.status(200).json({ status: 'success', response: user }); +}); + +// @router PATCH /api/v1/users/advancedSearch/roles/:id +// @desc Set advanced search roles for a user +// @access Private +router.patch('/advancedSearch/roles/:id', passport.authenticate('jwt'), async (req, res) => { + let user = await UserModel.findOneAndUpdate({ id: req.params.id }, { advancedSearchRoles: req.body.advancedSearchRoles }, { new: true }); + if (!user) return res.status(500).json({ status: 'error', message: 'Request failed' }); + return res.status(200).json({ status: 'success', response: user }); +}); + // @router POST /api/v1/users/serviceaccount // @desc create service account // @access Private From 564ff1bf51f63fbd7436ffcbb6078e55ca058ee2 Mon Sep 17 00:00:00 2001 From: Richard Date: Tue, 9 Feb 2021 15:47:23 +0000 Subject: [PATCH 06/42] Preventing SQL injection by checking user supplied query variables --- src/resources/user/user.route.js | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/resources/user/user.route.js b/src/resources/user/user.route.js index 13967afe..b9457c9e 100644 --- a/src/resources/user/user.route.js +++ b/src/resources/user/user.route.js @@ -96,11 +96,8 @@ router.patch('/advancedSearch/terms/:id', passport.authenticate('jwt'), async (r message: "Can't accept terms of a different user", }); } - let user = await UserModel.findOneAndUpdate( - { id: req.params.id }, - { acceptedAdvancedSearchTerms: req.body.acceptedAdvancedSearchTerms }, - { new: true } - ); + const { acceptedAdvancedSearchTerms } = req.body; + let user = await UserModel.findOneAndUpdate({ id: req.params.id }, { acceptedAdvancedSearchTerms }, { new: true }); if (!user) return res.status(500).json({ status: 'error', message: 'Request failed' }); return res.status(200).json({ status: 'success', response: user }); }); @@ -109,7 +106,8 @@ router.patch('/advancedSearch/terms/:id', passport.authenticate('jwt'), async (r // @desc Set advanced search roles for a user // @access Private router.patch('/advancedSearch/roles/:id', passport.authenticate('jwt'), async (req, res) => { - let user = await UserModel.findOneAndUpdate({ id: req.params.id }, { advancedSearchRoles: req.body.advancedSearchRoles }, { new: true }); + const { advancedSearchRoles } = req.body; + let user = await UserModel.findOneAndUpdate({ id: req.params.id }, { advancedSearchRoles }, { new: true }); if (!user) return res.status(500).json({ status: 'error', message: 'Request failed' }); return res.status(200).json({ status: 'success', response: user }); }); From f0ca2f707ec8253d24f1bb4a222f2b3db8084fba Mon Sep 17 00:00:00 2001 From: Robin Kavanagh Date: Tue, 9 Feb 2021 17:17:43 +0000 Subject: [PATCH 07/42] Started dataset v2 API creation --- jest.config.js | 14 +-- package.json | 2 + src/config/server.js | 3 +- src/resources/base/entity.js | 29 +++++ src/resources/base/repository.js | 85 +++++++++++++ .../dataset/__mocks__/dataaccessreequests.js | 49 ++++++++ src/resources/dataset/__mocks__/dataset.js | 99 +++++++++++++++ src/resources/dataset/__mocks__/datasets.js | 100 +++++++++++++++ .../__tests__/dataset.controller.test.js | 69 +++++++++++ .../dataset/__tests__/dataset.entity.test.js | 111 +++++++++++++++++ .../__tests__/dataset.repository.it.test.js | 38 ++++++ .../__tests__/dataset.repository.test.js | 36 ++++++ .../dataset/__tests__/dataset.service.test.js | 38 ++++++ src/resources/dataset/dataset.controller.js | 70 +++++++++++ src/resources/dataset/dataset.entity.js | 52 ++++++++ src/resources/dataset/dataset.model.js | 117 ++++++++++++++++++ src/resources/dataset/dataset.repository.js | 25 ++++ src/resources/dataset/dataset.service.js | 27 +++- src/resources/dataset/dependency.js | 5 + .../dataset/{ => v1}/dataset.route.js | 6 +- src/resources/dataset/v2/dataset.route.js | 18 +++ 21 files changed, 980 insertions(+), 13 deletions(-) create mode 100644 src/resources/base/entity.js create mode 100644 src/resources/base/repository.js create mode 100644 src/resources/dataset/__mocks__/dataaccessreequests.js create mode 100644 src/resources/dataset/__mocks__/dataset.js create mode 100644 src/resources/dataset/__mocks__/datasets.js create mode 100644 src/resources/dataset/__tests__/dataset.controller.test.js create mode 100644 src/resources/dataset/__tests__/dataset.entity.test.js create mode 100644 src/resources/dataset/__tests__/dataset.repository.it.test.js create mode 100644 src/resources/dataset/__tests__/dataset.repository.test.js create mode 100644 src/resources/dataset/__tests__/dataset.service.test.js create mode 100644 src/resources/dataset/dataset.controller.js create mode 100644 src/resources/dataset/dataset.entity.js create mode 100644 src/resources/dataset/dataset.model.js create mode 100644 src/resources/dataset/dataset.repository.js create mode 100644 src/resources/dataset/dependency.js rename src/resources/dataset/{ => v1}/dataset.route.js (95%) create mode 100644 src/resources/dataset/v2/dataset.route.js diff --git a/jest.config.js b/jest.config.js index 1afbce39..fb9a85a8 100644 --- a/jest.config.js +++ b/jest.config.js @@ -20,13 +20,13 @@ module.exports = { collectCoverage: false, // An array of glob patterns indicating a set of files for which coverage information should be collected - collectCoverageFrom: [ - "/src/**/*.js", - "!/src/config/**", - "!/test/*.js", - "!**/node_modules/**", - "!**/vendor/**" - ], + // collectCoverageFrom: [ + // //"/src/**/*.js", + // "!/src/config/**", + // "!/test/*.js", + // "!**/node_modules/**", + // "!**/vendor/**" + // ], // The directory where Jest should output its coverage files coverageDirectory: "coverage", diff --git a/package.json b/package.json index 5cf03eec..898eae12 100644 --- a/package.json +++ b/package.json @@ -27,6 +27,7 @@ "express-rate-limit": "^5.1.3", "express-session": "^1.17.1", "express-validator": "^6.6.1", + "faker": "^5.3.1", "googleapis": "^55.0.0", "jose": "^2.0.2", "jsonwebtoken": "^8.5.1", @@ -45,6 +46,7 @@ "prettier": "^2.2.1", "query-string": "^6.12.1", "randomstring": "^1.1.5", + "sinon": "^9.2.4", "snyk": "^1.334.0", "swagger-ui-express": "^4.1.4", "test": "^0.6.0", diff --git a/src/config/server.js b/src/config/server.js index 5c5583a9..ae69102d 100644 --- a/src/config/server.js +++ b/src/config/server.js @@ -207,7 +207,8 @@ app.use('/api/v1/coursecounter', require('../resources/course/coursecounter.rout app.use('/api/v1/discourse', require('../resources/discourse/discourse.route')); -app.use('/api/v1/datasets', require('../resources/dataset/dataset.route')); +app.use('/api/v1/datasets', require('../resources/dataset/v1/dataset.route')); +app.use('/api/v2/datasets', require('../resources/dataset/v2/dataset.route')); app.use('/api/v1/data-access-request/schema', require('../resources/datarequest/datarequest.schemas.route')); app.use('/api/v1/data-access-request', require('../resources/datarequest/datarequest.route')); diff --git a/src/resources/base/entity.js b/src/resources/base/entity.js new file mode 100644 index 00000000..17c0fcb7 --- /dev/null +++ b/src/resources/base/entity.js @@ -0,0 +1,29 @@ +class Entity { + constructor () { + this.id = ''; + } + + equals (other) { + if (other instanceof Entity === false) { + return false; + } + + return other.id ? this.referenceEquals(other.id) : this === other; + } + + referenceEquals (id) { + if (!this.id) { + return this.equals(id); + } + + const reference = typeof id !== 'string' ? id.toString() : id; + + return this.id === reference; + } + + toString () { + return this.id; + } +} + +module.exports = Entity; \ No newline at end of file diff --git a/src/resources/base/repository.js b/src/resources/base/repository.js new file mode 100644 index 00000000..305086a0 --- /dev/null +++ b/src/resources/base/repository.js @@ -0,0 +1,85 @@ +class Repository { + constructor(Model) { + this.collection = Model; + } + + // @desc Allows us to query a collection via the model inheriting this class with various options + async find(query = {}, { multiple = true, count, page = 1, pageSize = 20, select, populate, sort = {}, lean } = {}) { + const results = multiple ? + this.collection.find(query).sort(sort).limit(pageSize) : + this.collection.findOne(query); + + if(populate) { + results.populate(populate); + } + + if(select) { + results.select(select); + } + + if(multiple && page > 1) { + results.skip(parseInt(page - 1) * parseInt(pageSize)); + } + + if (count) { + return results.countDocuments().exec(); + } else if (lean) { + return results.lean().exec(); + } else { + return results.exec(); + } + } + + // @desc Allows us to count to total number of documents within this collection via the model inheriting this class + async count() { + return this.collection.estimatedDocumentCount(); + } + + // @desc Allows us to create a new Mongoose document within the collection via the model inheriting this class + async create(body) { + const document = new this.collection(body); + return document.save(); + } + + // @desc Allows us to update an existing Mongoose document within the collection via the model inheriting this class + async update(document, body = {}) { + const id = typeof document._id !== 'undefined' ? document._id : document; + return this.collection.findByIdAndUpdate(id, body, { new: true }); + } + + // @desc Allows us to delete an existing Mongoose document within the collection via the model inheriting this class + async remove(document) { + const reloadedDocument = await this.reload(document); + return reloadedDocument.remove(); + } + + // @desc Allows us to convert identifiers to Mongoose documents, plain entities to Mongoose documents, + // or to simply reload Mongoose documents with different query parameters (selected fields, populated fields, + // or a lean version) + async reload(document, { select, populate, lean } = {}) { + if(!select && !populate && !lean && document instanceof this.collection) { + return document; + } + + return (typeof document._id !== 'undefined') + ? this.findById(document._id, { select, populate, lean }) + : this.findById(document, { select, populate, lean }); + } + + // @desc A helper function to find all documents with a given query + async findAll({ count, select, populate, lean, sort } = {}) { + return this.find({}, { multiple: true, count, select, populate, lean, sort }); + } + + // @desc A helper function to find a single document by unique identifier + async findById(id, { select, populate, lean } = {}) { + return this.find({ _id: id }, { multiple: false, count: false, select, populate, lean }); + } + + // @desc A helper function to find the first document returned by a given query + async findOne(query = {}, { select, populate, lean } = {}) { + return this.find(query, { multiple: false, count: false, select, populate, lean }); + } +} + +module.exports = Repository; diff --git a/src/resources/dataset/__mocks__/dataaccessreequests.js b/src/resources/dataset/__mocks__/dataaccessreequests.js new file mode 100644 index 00000000..4420b036 --- /dev/null +++ b/src/resources/dataset/__mocks__/dataaccessreequests.js @@ -0,0 +1,49 @@ +export const dataAccessRequests = [{ + "_id" : "6021b437da9a2332004cde73", + "authorIds" : [ + + ], + "datasetIds" : [ + "dfb21b3b-7fd9-40c4-892e-810edd6dfc25" + ], + "datasetTitles" : [ + "Admitted Patient Care Dataset" + ], + "applicationStatus" : "submitted", + "jsonSchema" : " {\"classes\":{\"form\":\"login-form\",\"select\":\"form-control\",\"typeaheadCustom\":\"form-control\",\"datePickerCustom\":\"form-control\",\"question\":\"form-group\",\"input\":\"form-control\",\"radioListItem\":\"dar__radio--item\",\"radioList\":\"dar__radio--list list-group\",\"checkboxInput\":\"checkbox list-group\",\"checkboxListItem\":\"dar__check--item\",\"checkboxList\":\"dar__check list-group\",\"controlButton\":\"btn btn-primary pull-right\",\"backButton\":\"btn btn-default pull-left\",\"errorMessage\":\"alert alert-danger\",\"buttonBar\":\"button-bar hidden\"},\"pages\":[{\"pageId\":\"applicant\",\"title\":\"Make an enquiry\",\"description\":\"Give details about your project and the data you're interested\",\"active\":true}],\"formPanels\":[{\"index\":1,\"panelId\":\"applicant\",\"pageId\":\"applicant\"}],\"questionPanels\":[{\"panelId\":\"applicant\",\"panelHeader\":\"Project Details\",\"navHeader\":\"Applicant\",\"questionPanelHeaderText\":\"Test\",\"pageId\":\"applicant\",\"questionSets\":[{\"index\":1,\"questionSetId\":\"applicant\"}]}],\"questionSets\":[{\"questionSetId\":\"applicant\",\"questionSetHeader\":\"\",\"questions\":[{\"questionId\":\"applicantName\",\"question\":\"Applicant name\",\"input\":{\"type\":\"textInput\"},\"validations\":[{\"type\":\"isLength\",\"params\":[1,90]}],\"guidance\":\"Guidance information for applicant name, please insert your fullname.\"},{\"questionId\":\"researchAim\",\"question\":\"Research Aim\",\"input\":{\"type\":\"textareaInput\"},\"validations\":[{\"type\":\"isLength\",\"params\":[5]}],\"guidance\":\"Please briefly explain the purpose of your research and why you require this dataset.\"},{\"questionId\":\"linkedDatasets\",\"question\":\"Do you have any datasets you would like to link with this one?\",\"input\":{\"type\":\"radioOptionsInput\",\"options\":[{\"text\":\"Yes\",\"value\":\"true\",\"conditionalQuestions\":[{\"questionId\":\"linkeDatasetsParts\",\"question\":\"Please identify the names of the datasets.\",\"input\":{\"type\":\"textareaInput\"},\"validations\":[{\"type\":\"isLength\",\"params\":[5]}]}]},{\"text\":\"No\",\"value\":\"false\"}]}},{\"questionId\":\"dataRequirements\",\"question\":\"Do you know which parts of the dataset you are interested in?\",\"input\":{\"type\":\"radioOptionsInput\",\"options\":[{\"text\":\"Yes\",\"value\":\"true\",\"conditionalQuestions\":[{\"questionId\":\"dataRequirementsReason\",\"question\":\"Please explain which parts of the dataset.\",\"input\":{\"type\":\"textareaInput\"},\"validations\":[{\"type\":\"isLength\",\"params\":[5]}]}]},{\"text\":\"No\",\"value\":\"false\"}]}},{\"questionId\":\"projectStartDate\",\"question\":\"Proposed project start date (optional)\",\"input\":{\"type\":\"datePickerCustom\",\"value\":\"02/12/2020\"},\"validations\":[{\"type\":\"isCustomDate\",\"format\":\"dd/MM/yyyy\"}],\"guidance\":\"Please select a date for the proposed start date.\"},{\"questionId\":\"regICONumber\",\"question\":\"ICO number\",\"input\":{\"type\":\"textInput\"},\"validations\":[{\"type\":\"isLength\",\"params\":[1,8]}],\"guidance\":\"ICO registration number.\"},{\"questionId\":\"researchBenefits\",\"question\":\"Research benefits(optional)\",\"input\":{\"type\":\"textareaInput\"},\"guidance\":\"Please provide evidence of how your research will benefit the health and social care system.\"},{\"questionId\":\"ethicalProcessingEvidence\",\"question\":\"Ethical processing evidence (optional)\",\"input\":{\"type\":\"textareaInput\"},\"guidance\":\"Please provide a link(s) to relevant sources that showcase evidence of thee fair processing of data by your organisation.\"},{\"questionId\":\"contactNumber\",\"question\":\"Contact Number (optional)\",\"input\":{\"type\":\"textInput\"},\"guidance\":\"Please provide a telephone or mobile contact point.\"}]}]}", + "questionAnswers" : "{\"applicantName\":\"Robin Kavanagh\",\"researchAim\":\"Testing\",\"linkedDatasets\":\"false\",\"dataRequirements\":\"false\",\"projectStartDate\":\"22/02/2021\",\"regICONumber\":\"fsssfdf\"}", + "publisher" : "Oxford University Hospitals NHS Foundation Trust", + "version" : 1, + "userId" : 6689395059831886.0, + "dataSetId" : "dfb21b3b-7fd9-40c4-892e-810edd6dfc25", + "schemaId" : "5f0346914eaf3eba7e4f9237", + "files" : [ + + ], + "amendmentIterations" : [ + { + "_id" : "6021b46cda9a2332004cde7a", + "dateReturned" : "2021-02-08T22:00:12.386+0000", + "returnedBy" : "5f03530178e28143d7af2eb1", + "dateCreated" : "2021-02-08T22:00:12.386+0000", + "createdBy" : "5f03530178e28143d7af2eb1", + "questionAnswers" : { + "projectStartDate" : { + "_id" : "6021b46cda9a2332004cde79", + "questionSetId" : "", + "requested" : false, + "reason" : "", + "answer" : "", + "updatedBy" : "Alistair Kavanagh", + "updatedByUser" : "5f03530178e28143d7af2eb1", + "dateUpdated" : "2021-02-08T22:00:12.383+0000" + } + } + } + ], + "createdAt" : "2021-02-08T21:59:19.776+0000", + "updatedAt" : "2021-02-08T21:59:43.143+0000", + "__v" : 0, + "projectId" : "6021-B437-DA9A-2332-004C-DE73", + "dateSubmitted" : "2021-02-08T21:59:43.842+0000" +}] diff --git a/src/resources/dataset/__mocks__/dataset.js b/src/resources/dataset/__mocks__/dataset.js new file mode 100644 index 00000000..4e9e404e --- /dev/null +++ b/src/resources/dataset/__mocks__/dataset.js @@ -0,0 +1,99 @@ +export const datasetStub = { + id: '675584862177848', + name: 'Admitted Patient Care Dataset', + description: 'This is a dataset about admitted patient care', + resultsInsights: null, + link: null, + type: 'dataset', + datasetid: 'dfb21b3b-7fd9-40c4-892e-810edd6dfc25', + categories: {}, + license: null, + authors: [], + tags: {}, + activeflag: 'active', + counter: 15, + discourseTopicId: null, + relatedObjects: [], + uploader: null, + datasetid: 'dfb21b3b-7fd9-40c4-892e-810edd6dfc25', + pid: '4ef841d3-5e86-4f92-883f-1015ffd4b979', + datasetVersion: '0.0.1', + datasetfields: { + publisher: "Oxford University Hospitals NHS Foundation Trust", + geographicCoverage: [], + physicalSampleAvailability: [], + abstract: "Nationally defined dataset which ontaining administrative details for inpatient admissions (elective, emergency and maternity) and good coverage of clinical coding of diagnosis (ICD10) and procedures (OPCS4). Includes home birth and delivery spells.", + releaseDate: null, + accessRequestDuration: null, + conformsTo: null, + accessRights: "Available locally within Trust Clinical Data Warehouse System and authorised access by Trust staff only.\nDataset is also available via SUS/HES for government statistical purposes.", + jurisdiction: null, + datasetStartDate: null, + datasetEndDate: null, + statisticalPopulation: null, + ageBand: null, + contactPoint: "kinga.varnai@ouh.nhs.uk", + periodicity: null, + metadataquality: { + id: "dfb21b3b-7fd9-40c4-892e-810edd6dfc25", + publisher: "Oxford University Hospitals NHS Foundation Trust", + title: "Admitted Patient Care Dataset", + completeness_percent: 16.67, + weighted_completeness_percent: 14.29, + error_percent: 39.53, + weighted_error_percent: 39.68, + quality_score: 38.57, + quality_rating: "Not Rated", + weighted_quality_score: 37.3, + weighted_quality_rating: "Not Rated" + }, + datautility: { + id: "dfb21b3b-7fd9-40c4-892e-810edd6dfc25", + publisher: "Oxford University Hospitals NHS Foundation Trust", + title: "Admitted Patient Care Dataset", + metadata_richness: "Not Rated", + availability_of_additional_documentation_and_support: "", + data_model: "", + data_dictionary: "", + provenance: "", + data_quality_management_process: "", + dama_quality_dimensions: "", + pathway_coverage: "", + length_of_follow_up: "", + allowable_uses: "", + research_environment: "", + time_lag: "", + timeliness: "", + linkages: "", + data_enrichments: "" + }, + metadataschema: { + context: "http://schema.org/", + type: "Dataset", + identifier: "dfb21b3b-7fd9-40c4-892e-810edd6dfc25", + url: "https://healthdatagateway.org/detail/dfb21b3b-7fd9-40c4-892e-810edd6dfc25", + name: "Admitted Patient Care Dataset", + description: "", + keywords: [ + "Oxford University Hospitals NHS Foundation Trust", + "CDS" + ], + includedinDataCatalog: [ + { + type: "DataCatalog", + name: "Oxford University Hospitals NHS Foundation Trust", + url: "kinga.varnai@ouh.nhs.uk" + }, + { + type: "DataCatalog", + name: "HDR UK Health Data Gateway", + url: "http://healthdatagateway.org" + } + ] + }, + technicaldetails: [], + versionLinks: [], + phenotypes: [] + }, + datasetv2: {} + }; \ No newline at end of file diff --git a/src/resources/dataset/__mocks__/datasets.js b/src/resources/dataset/__mocks__/datasets.js new file mode 100644 index 00000000..41eae0ad --- /dev/null +++ b/src/resources/dataset/__mocks__/datasets.js @@ -0,0 +1,100 @@ +export const datasets = [{ + id: '675584862177848', + submittedDataAccessRequests: 1, + name: 'Admitted Patient Care Dataset', + description: 'This is a dataset about admitted patient care', + resultsInsights: null, + link: null, + type: 'dataset', + datasetid: 'dfb21b3b-7fd9-40c4-892e-810edd6dfc25', + categories: {}, + license: null, + authors: [], + tags: {}, + activeflag: 'active', + counter: 15, + discourseTopicId: null, + relatedObjects: [], + uploader: null, + datasetid: 'dfb21b3b-7fd9-40c4-892e-810edd6dfc25', + pid: '4ef841d3-5e86-4f92-883f-1015ffd4b979', + datasetVersion: '0.0.1', + datasetfields: { + publisher: "Oxford University Hospitals NHS Foundation Trust", + geographicCoverage: [], + physicalSampleAvailability: [], + abstract: "Nationally defined dataset which ontaining administrative details for inpatient admissions (elective, emergency and maternity) and good coverage of clinical coding of diagnosis (ICD10) and procedures (OPCS4). Includes home birth and delivery spells.", + releaseDate: null, + accessRequestDuration: null, + conformsTo: null, + accessRights: "Available locally within Trust Clinical Data Warehouse System and authorised access by Trust staff only.\nDataset is also available via SUS/HES for government statistical purposes.", + jurisdiction: null, + datasetStartDate: null, + datasetEndDate: null, + statisticalPopulation: null, + ageBand: null, + contactPoint: "kinga.varnai@ouh.nhs.uk", + periodicity: null, + metadataquality: { + id: "dfb21b3b-7fd9-40c4-892e-810edd6dfc25", + publisher: "Oxford University Hospitals NHS Foundation Trust", + title: "Admitted Patient Care Dataset", + completeness_percent: 16.67, + weighted_completeness_percent: 14.29, + error_percent: 39.53, + weighted_error_percent: 39.68, + quality_score: 38.57, + quality_rating: "Not Rated", + weighted_quality_score: 37.3, + weighted_quality_rating: "Not Rated" + }, + datautility: { + id: "dfb21b3b-7fd9-40c4-892e-810edd6dfc25", + publisher: "Oxford University Hospitals NHS Foundation Trust", + title: "Admitted Patient Care Dataset", + metadata_richness: "Not Rated", + availability_of_additional_documentation_and_support: "", + data_model: "", + data_dictionary: "", + provenance: "", + data_quality_management_process: "", + dama_quality_dimensions: "", + pathway_coverage: "", + length_of_follow_up: "", + allowable_uses: "", + research_environment: "", + time_lag: "", + timeliness: "", + linkages: "", + data_enrichments: "" + }, + metadataschema: { + context: "http://schema.org/", + type: "Dataset", + identifier: "dfb21b3b-7fd9-40c4-892e-810edd6dfc25", + url: "https://healthdatagateway.org/detail/dfb21b3b-7fd9-40c4-892e-810edd6dfc25", + name: "Admitted Patient Care Dataset", + description: "", + keywords: [ + "Oxford University Hospitals NHS Foundation Trust", + "CDS" + ], + includedinDataCatalog: [ + { + type: "DataCatalog", + name: "Oxford University Hospitals NHS Foundation Trust", + url: "kinga.varnai@ouh.nhs.uk" + }, + { + type: "DataCatalog", + name: "HDR UK Health Data Gateway", + url: "http://healthdatagateway.org" + } + ] + }, + technicaldetails: [], + versionLinks: [], + phenotypes: [] + }, + datasetv2: {} + }]; \ No newline at end of file diff --git a/src/resources/dataset/__tests__/dataset.controller.test.js b/src/resources/dataset/__tests__/dataset.controller.test.js new file mode 100644 index 00000000..51ef150a --- /dev/null +++ b/src/resources/dataset/__tests__/dataset.controller.test.js @@ -0,0 +1,69 @@ +import sinon from 'sinon'; +import faker from 'faker'; + +import DatasetController from '../dataset.controller'; +import DatasetService from '../dataset.service'; + +describe('DatasetController', function () { + describe('getDataset', function () { + let req, res, status, json, datasetService, datasetController; + beforeEach(() => { + status = sinon.stub(); + json = sinon.spy(); + res = { json, status }; + status.returns(res); + datasetService = new DatasetService(); + }); + + it('should return a dataset that matches the id param', async function () { + req = { params: { id: faker.random.number({'min':1, 'max':999999999}) }}; + const stubValue = { + id: req.params.id + }; + const serviceStub = sinon.stub(datasetService, 'getDataset').returns(stubValue); + datasetController = new DatasetController(datasetService); + await datasetController.getDataset(req, res); + + expect(serviceStub.calledOnce).toBe(true); + expect(status.calledWith(200)).toBe(true); + expect(json.calledWith({ success: true, data: stubValue })).toBe(true); + }); + + it('should return a bad request response if no dataset id is provided', async function () { + req = { params: {} }; + + const serviceStub = sinon.stub(datasetService, 'getDataset').returns({}); + datasetController = new DatasetController(datasetService); + await datasetController.getDataset(req, res); + + expect(serviceStub.notCalled).toBe(true); + expect(status.calledWith(400)).toBe(true); + expect(json.calledWith({ success: false, message: 'You must provide a dataset version id or a dataset persistent id' })).toBe(true); + }); + + it('should return a not found response if no dataset could be found for the id provided', async function () { + req = { params: { id: faker.random.number({'min':1, 'max':999999999}) }}; + + const serviceStub = sinon.stub(datasetService, 'getDataset').returns(null); + datasetController = new DatasetController(datasetService); + await datasetController.getDataset(req, res); + + expect(serviceStub.calledOnce).toBe(true); + expect(status.calledWith(404)).toBe(true); + expect(json.calledWith({ success: false, message: 'A dataset could not be found with the provided id' })).toBe(true); + }); + + it('should return a server error if an unexpected exception occurs', async function () { + req = { params: { id: faker.random.number({'min':1, 'max':999999999}) }}; + + const error = new Error("A server error occurred"); + const serviceStub = sinon.stub(datasetService, 'getDataset').throws(error); + datasetController = new DatasetController(datasetService); + await datasetController.getDataset(req, res); + + expect(serviceStub.calledOnce).toBe(true); + expect(status.calledWith(500)).toBe(true); + expect(json.calledWith({ success: false, message: 'A server error occurred, please try again' })).toBe(true); + }); + }); +}); diff --git a/src/resources/dataset/__tests__/dataset.entity.test.js b/src/resources/dataset/__tests__/dataset.entity.test.js new file mode 100644 index 00000000..0aefe434 --- /dev/null +++ b/src/resources/dataset/__tests__/dataset.entity.test.js @@ -0,0 +1,111 @@ +import DatasetClass from '../dataset.entity'; + +describe('DatasetEntity', function () { + + describe('constructor', function () { + it('should create an instance of a dataset entity with the expected properties', async function () { + const dataset = new DatasetClass( + 675584862177848, + "Admitted Patient Care Dataset", + "This is a dataset about admitted patient care", + null, + null, + "dataset", + {}, + null, + [], + {}, + "active", + 15, + null, + [], + null, + "dfb21b3b-7fd9-40c4-892e-810edd6dfc25", + "4ef841d3-5e86-4f92-883f-1015ffd4b979", + "0.0.1", + { publisher: "Oxford University Hospitals NHS Foundation Trust" }, + {} + ); + + expect(dataset.datasetid).toEqual("dfb21b3b-7fd9-40c4-892e-810edd6dfc25"); + expect(dataset.type).toEqual("dataset"); + expect(dataset.id).toEqual(675584862177848); + expect(dataset.name).toEqual("Admitted Patient Care Dataset"); + expect(dataset.description).toEqual("This is a dataset about admitted patient care"); + expect(dataset.resultsInsights).toEqual(null); + expect(dataset.datasetid).toEqual("dfb21b3b-7fd9-40c4-892e-810edd6dfc25"); + expect(dataset.categories).toEqual({}); + expect(dataset.license).toEqual(null); + expect(dataset.authors).toEqual([]); + expect(dataset.activeflag).toEqual("active"); + expect(dataset.counter).toEqual(15); + expect(dataset.discourseTopicId).toEqual(null); + expect(dataset.relatedObjects).toEqual([]); + expect(dataset.uploader).toEqual(null); + expect(dataset.pid).toEqual("4ef841d3-5e86-4f92-883f-1015ffd4b979"); + expect(dataset.datasetVersion).toEqual("0.0.1"); + expect(dataset.datasetfields).toEqual({ publisher: "Oxford University Hospitals NHS Foundation Trust" }); + expect(dataset.datasetv2).toEqual({}); + }); + }); + + describe('isLatestVersion', function () { + it('should return a boolean indicating this is the latest version of the dataset', async function () { + const dataset = new DatasetClass( + 675584862177848, + "Admitted Patient Care Dataset", + "This is a dataset about admitted patient care", + null, + null, + "dataset", + {}, + null, + [], + {}, + "active", + 15, + null, + [], + null, + "dfb21b3b-7fd9-40c4-892e-810edd6dfc25", + "4ef841d3-5e86-4f92-883f-1015ffd4b979", + "0.0.1", + { publisher: "Oxford University Hospitals NHS Foundation Trust" }, + {} + ); + + const result = dataset.isLatestVersion(); + + expect(result).toBe(true); + }); + + it('should return a boolean indicating this is not the latest version of the dataset', async function () { + const dataset = new DatasetClass( + 675584862177848, + "Admitted Patient Care Dataset", + "This is a dataset about admitted patient care", + null, + null, + "dataset", + {}, + null, + [], + {}, + "archive", + 15, + null, + [], + null, + "dfb21b3b-7fd9-40c4-892e-810edd6dfc25", + "4ef841d3-5e86-4f92-883f-1015ffd4b979", + "0.0.1", + { publisher: "Oxford University Hospitals NHS Foundation Trust" }, + {} + ); + + const result = dataset.isLatestVersion(); + + expect(result).toBe(false); + }); + }); +}); diff --git a/src/resources/dataset/__tests__/dataset.repository.it.test.js b/src/resources/dataset/__tests__/dataset.repository.it.test.js new file mode 100644 index 00000000..69ad55b8 --- /dev/null +++ b/src/resources/dataset/__tests__/dataset.repository.it.test.js @@ -0,0 +1,38 @@ +import dbHandler from '../../../config/in-memory-db'; +import DatasetRepository from '../dataset.repository'; +import { datasets } from '../__mocks__/datasets'; +import { dataAccessRequests } from '../__mocks__/dataaccessreequests'; + +//const amendmentController = require('../amendment.controller'); +//const amendmentModel = require('../amendment.model'); + +/** + * Connect to a new in-memory database before running any tests. + */ +beforeAll(async () => { + await dbHandler.connect(); + await dbHandler.loadData({ tools: datasets, data_requests: dataAccessRequests }); +}); + +/** + * Revert to initial test data after every test. + */ +afterEach(async () => { + await dbHandler.clearDatabase(); + await dbHandler.loadData({ tools: datasets }); +}); + +/** + * Remove and close the db and server. + */ +afterAll(async () => await dbHandler.closeDatabase()); + +describe('DatasetRepository', function () { + describe('getUser', () => { + it('should return a dataset by a specified id', async function () { + const datasetRepository = new DatasetRepository(); + const dataset = await datasetRepository.getDataset("dfb21b3b-7fd9-40c4-892e-810edd6dfc25"); + expect(dataset).toEqual(datasets[0]); + }); + }); +}); diff --git a/src/resources/dataset/__tests__/dataset.repository.test.js b/src/resources/dataset/__tests__/dataset.repository.test.js new file mode 100644 index 00000000..9998e9e9 --- /dev/null +++ b/src/resources/dataset/__tests__/dataset.repository.test.js @@ -0,0 +1,36 @@ +import sinon from 'sinon'; + +import DatasetRepository from '../dataset.repository'; +import { datasetStub } from '../__mocks__/dataset'; + +describe('DatasetRepository', function () { + describe('getDataset', function () { + it('should return a dataset by a specified id', async function () { + const datasetRepository = new DatasetRepository(); + const stub = sinon.stub(datasetRepository, 'findOne').returns(datasetStub); + const dataset = await datasetRepository.getDataset(datasetStub.id); + + expect(stub.calledOnce).toBe(true); + + expect(dataset.datasetid).toEqual(datasetStub.datasetid); + expect(dataset.type).toEqual(datasetStub.type); + expect(dataset.id).toEqual(datasetStub.id); + expect(dataset.name).toEqual(datasetStub.name); + expect(dataset.description).toEqual(datasetStub.description); + expect(dataset.resultsInsights).toEqual(datasetStub.resultsInsights); + expect(dataset.datasetid).toEqual(datasetStub.datasetid); + expect(dataset.categories).toEqual(datasetStub.categories); + expect(dataset.license).toEqual(datasetStub.license); + expect(dataset.authors).toEqual(datasetStub.authors); + expect(dataset.activeflag).toEqual(datasetStub.activeflag); + expect(dataset.counter).toEqual(datasetStub.counter); + expect(dataset.discourseTopicId).toEqual(datasetStub.discourseTopicId); + expect(dataset.relatedObjects).toEqual(datasetStub.relatedObjects); + expect(dataset.uploader).toEqual(datasetStub.uploader); + expect(dataset.pid).toEqual(datasetStub.pid); + expect(dataset.datasetVersion).toEqual(datasetStub.datasetVersion); + expect(dataset.datasetfields).toEqual(datasetStub.datasetfields); + expect(dataset.datasetv2).toEqual(datasetStub.datasetv2); + }); + }); +}); diff --git a/src/resources/dataset/__tests__/dataset.service.test.js b/src/resources/dataset/__tests__/dataset.service.test.js new file mode 100644 index 00000000..75a0d18f --- /dev/null +++ b/src/resources/dataset/__tests__/dataset.service.test.js @@ -0,0 +1,38 @@ +import sinon from 'sinon'; + +import DatasetRepository from '../dataset.repository'; +import DatasetService from '../dataset.service'; +import { datasetStub } from '../__mocks__/dataset'; + +describe('DatasetService', function () { + describe('getDataset', function () { + it('should return a dataset by a specified id', async function () { + const datasetRepository = new DatasetRepository(); + const stub = sinon.stub(datasetRepository, 'getDataset').returns(datasetStub); + const datasetService = new DatasetService(datasetRepository); + const dataset = await datasetService.getDataset(datasetStub.id); + + expect(stub.calledOnce).toBe(true); + + expect(dataset.datasetid).toEqual(datasetStub.datasetid); + expect(dataset.type).toEqual(datasetStub.type); + expect(dataset.id).toEqual(datasetStub.id); + expect(dataset.name).toEqual(datasetStub.name); + expect(dataset.description).toEqual(datasetStub.description); + expect(dataset.resultsInsights).toEqual(datasetStub.resultsInsights); + expect(dataset.datasetid).toEqual(datasetStub.datasetid); + expect(dataset.categories).toEqual(datasetStub.categories); + expect(dataset.license).toEqual(datasetStub.license); + expect(dataset.authors).toEqual(datasetStub.authors); + expect(dataset.activeflag).toEqual(datasetStub.activeflag); + expect(dataset.counter).toEqual(datasetStub.counter); + expect(dataset.discourseTopicId).toEqual(datasetStub.discourseTopicId); + expect(dataset.relatedObjects).toEqual(datasetStub.relatedObjects); + expect(dataset.uploader).toEqual(datasetStub.uploader); + expect(dataset.pid).toEqual(datasetStub.pid); + expect(dataset.datasetVersion).toEqual(datasetStub.datasetVersion); + expect(dataset.datasetfields).toEqual(datasetStub.datasetfields); + expect(dataset.datasetv2).toEqual(datasetStub.datasetv2); + }); + }); +}); diff --git a/src/resources/dataset/dataset.controller.js b/src/resources/dataset/dataset.controller.js new file mode 100644 index 00000000..e824ccf0 --- /dev/null +++ b/src/resources/dataset/dataset.controller.js @@ -0,0 +1,70 @@ +export default class DatasetController { + constructor(datasetService) { + this.datasetService = datasetService; + } + + async getDataset(req, res) { + try { + // Extract id parameter from query string + const { id } = req.params; + // If no id provided, it is a bad request + if (!id) { + return res.status(400).json({ + success: false, + message: 'You must provide a dataset version id or a dataset persistent id', + }); + } + // Find the dataset + let dataset = {}; + if(req.params.expanded) { + dataset = await this.datasetService.getDatasetExpanded(); + } else { + dataset = await this.datasetService.getDataset(); + } + // Return if no dataset found + if (!dataset) { + return res.status(404).json({ + success: false, + message: 'A dataset could not be found with the provided id', + }); + } + // Return the dataset + return res.status(200).json({ + success: true, + data: dataset, + }); + } catch (err) { + // Return error response if something goes wrong + return res.status(500).json({ + success: false, + message: 'A server error occurred, please try again', + }); + } + } + + async getDatasets(req, res) { + try { + // Parse filter options from query params + // TODO + + // Find the datasets + let datasets = []; + if(req.params.expanded) { + datasets = await this.datasetService.getDatasetsExpanded(); + } else { + datasets = await this.datasetService.getDatasets(); + } + // Return the datasets + return res.status(200).json({ + success: true, + data: datasets + }); + } catch (err) { + // Return error response if something goes wrong + return res.status(500).json({ + success: false, + message: 'A server error occurred, please try again', + }); + } + } +} diff --git a/src/resources/dataset/dataset.entity.js b/src/resources/dataset/dataset.entity.js new file mode 100644 index 00000000..b623696c --- /dev/null +++ b/src/resources/dataset/dataset.entity.js @@ -0,0 +1,52 @@ +import Entity from '../base/entity'; + +export default class DatasetClass extends Entity { + constructor( + id, + name, + description, + resultsInsights, + link, + type, + categories, + license, + authors, + tags, + activeflag, + counter, + discourseTopicId, + relatedObjects, + uploader, + datasetid, + pid, + datasetVersion, + datasetfields, + datasetv2 + ) { + super(); + this.id = id; + this.name = name; + this.description = description; + this.resultsInsights = resultsInsights; + this.link = link; + this.type = type; + this.categories = categories; + this.license = license; + this.authors = authors; + this.tags = tags; + this.activeflag = activeflag; + this.counter = counter; + this.discourseTopicId = discourseTopicId; + this.relatedObjects = relatedObjects; + this.uploader = uploader; + this.datasetid = datasetid; + this.pid = pid; + this.datasetVersion = datasetVersion; + this.datasetfields = datasetfields; + this.datasetv2 = datasetv2; + } + + isLatestVersion() { + return this.activeflag === 'active'; + } +} diff --git a/src/resources/dataset/dataset.model.js b/src/resources/dataset/dataset.model.js new file mode 100644 index 00000000..de274f2d --- /dev/null +++ b/src/resources/dataset/dataset.model.js @@ -0,0 +1,117 @@ +import { model, Schema } from 'mongoose'; + +import DatasetClass from './dataset.entity'; + +//DO NOT DELETE publisher and team model below +import { PublisherModel } from '../publisher/publisher.model'; +import { TeamModel } from '../team/team.model'; +import { DataRequestModel } from '../datarequest/datarequest.model'; + +const datasetSchema = new Schema( + { + id: Number, + name: String, + description: String, + resultsInsights: String, + link: String, + type: String, + categories: { + category: { type: String }, + }, + license: String, + authors: [Number], + tags: { + features: [String], + topics: [String], + }, + activeflag: String, + updatedon: Date, + counter: Number, + discourseTopicId: Number, + relatedObjects: [ + { + objectId: String, + reason: String, + pid: String, + objectType: String, + user: String, + updated: String, + }, + ], + uploader: Number, + datasetid: String, + pid: String, + datasetVersion: String, + datasetfields: { + publisher: String, + geographicCoverage: [String], + physicalSampleAvailability: [String], + abstract: String, + releaseDate: String, + accessRequestDuration: String, + conformsTo: String, + accessRights: String, + jurisdiction: String, + datasetStartDate: String, + datasetEndDate: String, + statisticalPopulation: String, + ageBand: String, + contactPoint: String, + periodicity: String, + populationSize: String, + metadataquality: {}, + datautility: {}, + metadataschema: {}, + technicaldetails: [], + versionLinks: [], + phenotypes: [], + }, + datasetv2: {}, + }, + { + timestamps: true, + collection: 'tools', + toJSON: { virtuals: true }, + toObject: { virtuals: true }, + } +); + +datasetSchema.virtual('publisher', { + ref: 'Publisher', + foreignField: 'name', + localField: 'datasetfields.publisher', + justOne: true, +}); + +datasetSchema.virtual('reviews', { + ref: 'Reviews', + foreignField: 'reviewerID', + localField: 'id', + justOne: false, +}); + +datasetSchema.virtual('tools', { + ref: 'Data', + foreignField: 'authors', + localField: 'id', + justOne: false, +}); + +datasetSchema.virtual('submittedDataAccessRequests', { + ref: 'data_request', + foreignField: 'datasetIds', + localField: 'datasetid', + count: true, + match: { + applicationStatus: { $in: ['submitted', 'approved', 'inReview', 'rejected', 'approved with conditions'] }, + }, + justOne: false, +}); + + +// TODO Add virtual for Related Objects connected to this dataset + +datasetSchema.loadClass(DatasetClass); + +export const Dataset = model('Dataset', datasetSchema, 'tools'); +export const type = 'dataset'; \ No newline at end of file diff --git a/src/resources/dataset/dataset.repository.js b/src/resources/dataset/dataset.repository.js new file mode 100644 index 00000000..0d5bd780 --- /dev/null +++ b/src/resources/dataset/dataset.repository.js @@ -0,0 +1,25 @@ +import Repository from '../base/repository'; +import { Dataset, type } from './dataset.model'; + +export default class DatasetRepository extends Repository { + constructor() { + super(Dataset); + this.dataset = Dataset; + } + + async getDataset(id) { + return this.findOne({ type, datasetid: id }, { lean: true, populate: { path: 'submittedDataAccessRequests' } }); + } + + async getDatasetExpanded(id) { + return this.findOne({ type, datasetid: id }, { lean: true, populate: { path: 'submittedDataAccessRequests' } }); + } + + async getDatasets() { + return this.find({ type }, { lean: true, populate: { path: 'submittedDataAccessRequests' } }); + } + + async getDatasetsExpanded() { + return this.find({ type }, { lean: true, populate: { path: 'submittedDataAccessRequests' } }); + } +} diff --git a/src/resources/dataset/dataset.service.js b/src/resources/dataset/dataset.service.js index 68e30b5d..928f3d4e 100644 --- a/src/resources/dataset/dataset.service.js +++ b/src/resources/dataset/dataset.service.js @@ -1,9 +1,10 @@ -import { Data } from '../tool/data.model'; -import { MetricsData } from '../stats/metrics.model'; import axios from 'axios'; import * as Sentry from '@sentry/node'; import { v4 as uuidv4 } from 'uuid'; +import { Data } from '../tool/data.model'; +import { MetricsData } from '../stats/metrics.model'; + export async function loadDataset(datasetID) { var metadataCatalogueLink = process.env.metadataURL || 'https://metadata-catalogue.org/hdruk'; const datasetCall = axios @@ -956,3 +957,25 @@ async function saveUptime() { metricsData.uptime = averageUptime; await metricsData.save(); } + +export default class DatasetService { + constructor(datasetRepository) { + this.datasetRepository = datasetRepository; + } + + getDataset(id) { + return this.datasetRepository.getDataset(id); + } + + getDatasetExpanded(id) { + return this.datasetRepository.getDatasetExpanded(id); + } + + getDatasets() { + return this.datasetRepository.getDatasets(); + } + + getDatasetsExpanded() { + return this.datasetRepository.getDatasetsExpanded(); + } +} diff --git a/src/resources/dataset/dependency.js b/src/resources/dataset/dependency.js new file mode 100644 index 00000000..06a5be78 --- /dev/null +++ b/src/resources/dataset/dependency.js @@ -0,0 +1,5 @@ +import DatasetRepository from './dataset.repository'; +import DatasetService from './dataset.service'; + +export const datasetRepository = new DatasetRepository(); +export const datasetService = new DatasetService(datasetRepository); diff --git a/src/resources/dataset/dataset.route.js b/src/resources/dataset/v1/dataset.route.js similarity index 95% rename from src/resources/dataset/dataset.route.js rename to src/resources/dataset/v1/dataset.route.js index 4a032d6e..095e7f5f 100644 --- a/src/resources/dataset/dataset.route.js +++ b/src/resources/dataset/v1/dataset.route.js @@ -1,7 +1,7 @@ import express from 'express'; -import { Data } from '../tool/data.model'; -import { loadDataset, loadDatasets } from './dataset.service'; -import { getAllTools } from '../tool/data.repository'; +import { Data } from '../../tool/data.model'; +import { loadDataset, loadDatasets } from '../dataset.service'; +import { getAllTools } from '../../tool/data.repository'; import _ from 'lodash'; import escape from 'escape-html'; const router = express.Router(); diff --git a/src/resources/dataset/v2/dataset.route.js b/src/resources/dataset/v2/dataset.route.js new file mode 100644 index 00000000..b8966616 --- /dev/null +++ b/src/resources/dataset/v2/dataset.route.js @@ -0,0 +1,18 @@ +import express from 'express'; +import DatasetController from '../dataset.controller'; +import { datasetService } from '../dependency'; + +const router = express.Router(); +const datasetController = new DatasetController(datasetService); + +// @route GET /api/v2/datasets/id +// @desc Returns a dataset based on either dataset ID or PID (persistent identifier) provided +// @access Public +router.get('/:id', (req, res) => datasetController.getDataset(req, res)); + +// @route GET /api/v2/datasets +// @desc Returns a collection of datasets based on supplied query parameters +// @access Public +//router.get('/', datasetController.getDatasets); + +module.exports = router; From 1f70127f06c6da99cc4cc6e0f6417e6b99c7cf21 Mon Sep 17 00:00:00 2001 From: Richard Date: Wed, 10 Feb 2021 11:41:31 +0000 Subject: [PATCH 08/42] Added callback to findOneAndUpdate for advanced search PATCHes --- src/resources/user/user.route.js | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/src/resources/user/user.route.js b/src/resources/user/user.route.js index b9457c9e..f08b19a5 100644 --- a/src/resources/user/user.route.js +++ b/src/resources/user/user.route.js @@ -93,12 +93,16 @@ router.patch('/advancedSearch/terms/:id', passport.authenticate('jwt'), async (r if (parseInt(req.params.id) !== req.user.id) { return res.status(400).json({ status: 'error', - message: "Can't accept terms of a different user", + message: 'Invalid user id supplied', }); } const { acceptedAdvancedSearchTerms } = req.body; - let user = await UserModel.findOneAndUpdate({ id: req.params.id }, { acceptedAdvancedSearchTerms }, { new: true }); - if (!user) return res.status(500).json({ status: 'error', message: 'Request failed' }); + if (typeof acceptedAdvancedSearchTerms !== 'boolean') { + return res.status(400).json({ status: 'error', message: 'Invalid input supplied.' }); + } + let user = await UserModel.findOneAndUpdate({ id: req.params.id }, { acceptedAdvancedSearchTerms }, { new: true }, err => { + if (err) return res.json({ success: false, error: err }); + }); return res.status(200).json({ status: 'success', response: user }); }); @@ -107,8 +111,13 @@ router.patch('/advancedSearch/terms/:id', passport.authenticate('jwt'), async (r // @access Private router.patch('/advancedSearch/roles/:id', passport.authenticate('jwt'), async (req, res) => { const { advancedSearchRoles } = req.body; - let user = await UserModel.findOneAndUpdate({ id: req.params.id }, { advancedSearchRoles }, { new: true }); - if (!user) return res.status(500).json({ status: 'error', message: 'Request failed' }); + if (typeof advancedSearchRoles !== 'object') { + return res.status(400).json({ status: 'error', message: 'Invalid role(s) supplied.' }); + } + + let user = await UserModel.findOneAndUpdate({ id: req.params.id }, { advancedSearchRoles }, { new: true }, err => { + if (err) return res.json({ success: false, error: err }); + }); return res.status(200).json({ status: 'success', response: user }); }); From fa22567de15bdbfceb40d10ab7594ba22d8d1705 Mon Sep 17 00:00:00 2001 From: Richard Date: Wed, 10 Feb 2021 12:31:44 +0000 Subject: [PATCH 09/42] Safe guarding user supplied value for query --- src/resources/user/user.route.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/resources/user/user.route.js b/src/resources/user/user.route.js index f08b19a5..3a4cb5e7 100644 --- a/src/resources/user/user.route.js +++ b/src/resources/user/user.route.js @@ -114,8 +114,8 @@ router.patch('/advancedSearch/roles/:id', passport.authenticate('jwt'), async (r if (typeof advancedSearchRoles !== 'object') { return res.status(400).json({ status: 'error', message: 'Invalid role(s) supplied.' }); } - - let user = await UserModel.findOneAndUpdate({ id: req.params.id }, { advancedSearchRoles }, { new: true }, err => { + let roles = advancedSearchRoles.map(role => role.toString()); + let user = await UserModel.findOneAndUpdate({ id: req.params.id }, { advancedSearchRoles: roles }, { new: true }, err => { if (err) return res.json({ success: false, error: err }); }); return res.status(200).json({ status: 'success', response: user }); From ac540f2873650c467e632fc9820670510f92f223 Mon Sep 17 00:00:00 2001 From: massimocianfroccaPA Date: Thu, 11 Feb 2021 11:05:47 +0100 Subject: [PATCH 10/42] As a user I want to be able to print a hard copy of my presubmission DAR --- .../datarequest/datarequest.controller.js | 98 ++++++++++++++++++- .../datarequest/datarequest.route.js | 5 + src/resources/utilities/constants.util.js | 4 +- .../utilities/emailGenerator.util.js | 8 +- 4 files changed, 110 insertions(+), 5 deletions(-) diff --git a/src/resources/datarequest/datarequest.controller.js b/src/resources/datarequest/datarequest.controller.js index f3033cd4..42cb9593 100644 --- a/src/resources/datarequest/datarequest.controller.js +++ b/src/resources/datarequest/datarequest.controller.js @@ -1447,6 +1447,65 @@ module.exports = { return accessRecord; }, + //POST api/v1/data-access-request/:id/email + mailDataAccessRequestInfoById: async (req, res) => { + + try{ + + // 1. Get the required request params + const { + params: { id }, + } = req; + + // 2. Retrieve DAR from database + let accessRecord = await DataRequestModel.findOne({ _id: id }).populate([ + { + path: 'datasets dataset', + }, + { + path: 'mainApplicant', + } + + ]); + + if (!accessRecord) { + return res.status(404).json({ status: 'error', message: 'Application not found.' }); + } + + // 3. Ensure single datasets are mapped correctly into array + if (_.isEmpty(accessRecord.datasets)) { + accessRecord.datasets = [accessRecord.dataset]; + } + + // 4. If application is not in progress, actions cannot be performed + if (accessRecord.applicationStatus !== constants.applicationStatuses.INPROGRESS) { + return res.status(400).json({ + success: false, + message: 'This application is no longer in pre-submission status and therefore this action cannot be performed', + }); + } + + // 5. Get the requesting users permission levels + let { authorised, userType } = datarequestUtil.getUserPermissionsForApplication(accessRecord.toObject(), req.user.id, req.user._id); + // 6. Return unauthorised message if the requesting user is not an applicant + if (!authorised || userType !== constants.userTypes.APPLICANT) { + return res.status(401).json({ status: 'failure', message: 'Unauthorised' }); + } + + // 7. Send notification to the authorised user + module.exports.createNotifications(constants.notificationTypes.INPROGRESS, {}, accessRecord, req.user); + + return res.status(200).json({ status: 'success' }); + + } catch (err) { + console.error(err.message); + return res.status(500).json({ + success: false, + message: 'An error occurred', + }); + } + }, + //POST api/v1/data-access-request/:id/notify notifyAccessRequestById: async (req, res) => { // 1. Get the required request params @@ -1689,7 +1748,44 @@ module.exports = { } = context; switch (type) { - case constants.notificationTypes.STATUSCHANGE: + case constants.notificationTypes.INPROGRESS: + await notificationBuilder.triggerNotificationMessage( + [user.id], + `An email with the data access request info for ${datasetTitles} has been sent to you`, + 'data access request', + accessRecord._id + ); + + options = { + userType: '', + userEmail: appEmail, + publisher, + datasetTitles, + userName: `${appFirstName} ${appLastName}`, + userType: 'applicant', + submissionType: constants.submissionTypes.INPROGRESS, + }; + + + // Build email template + ({ html, jsonContent } = await emailGenerator.generateEmail( + aboutApplication, + questions, + pages, + questionPanels, + questionAnswers, + options + )); + await emailGenerator.sendEmail( + [user], + constants.hdrukEmail, + `Data Access Request in progress for ${datasetTitles}`, + html, + false, + attachments + ); + break; + case constants.notificationTypes.STATUSCHANGE: // 1. Create notifications // Custodian manager and current step reviewer notifications if (_.has(accessRecord.datasets[0].toObject(), 'publisher.team.users')) { diff --git a/src/resources/datarequest/datarequest.route.js b/src/resources/datarequest/datarequest.route.js index 8b9bd659..72ce6379 100644 --- a/src/resources/datarequest/datarequest.route.js +++ b/src/resources/datarequest/datarequest.route.js @@ -109,4 +109,9 @@ router.post('/:id', passport.authenticate('jwt'), datarequestController.submitAc // @access Private router.post('/:id/notify', passport.authenticate('jwt'), datarequestController.notifyAccessRequestById); +// @route POST api/v1/data-access-request/:id/email +// @desc Mail a Data Access Request information in presubmission +// @access Private - Applicant +router.post('/:id/email', passport.authenticate('jwt'), datarequestController.mailDataAccessRequestInfoById); + module.exports = router; \ No newline at end of file diff --git a/src/resources/utilities/constants.util.js b/src/resources/utilities/constants.util.js index dbfb7fbb..98edb917 100644 --- a/src/resources/utilities/constants.util.js +++ b/src/resources/utilities/constants.util.js @@ -271,7 +271,8 @@ const _notificationTypes = { MEMBERREMOVED: 'MemberRemoved', MEMBERROLECHANGED: 'MemberRoleChanged', WORKFLOWASSIGNED: 'WorkflowAssigned', - WORKFLOWCREATED: 'WorkflowCreated' + WORKFLOWCREATED: 'WorkflowCreated', + INPROGRESS: 'InProgress' }; const _applicationStatuses = { @@ -291,6 +292,7 @@ const _amendmentModes = { }; const _submissionTypes = { + INPROGRESS: 'inProgress', INITIAL: 'initial', RESUBMISSION: 'resubmission', }; diff --git a/src/resources/utilities/emailGenerator.util.js b/src/resources/utilities/emailGenerator.util.js index be6224e1..289644d0 100644 --- a/src/resources/utilities/emailGenerator.util.js +++ b/src/resources/utilities/emailGenerator.util.js @@ -189,7 +189,9 @@ const _buildSubjectTitle = (user, title, submissionType) => { if (user.toUpperCase() === 'DATACUSTODIAN') { subject = `Someone has submitted an application to access ${title} dataset. Please let the applicant know as soon as there is progress in the review of their submission.`; } else { - if (submissionType === constants.submissionTypes.INITIAL) { + if ( submissionType === constants.submissionTypes.INPROGRESS){ + subject = `You are in progress with a request access to ${title}. The custodian will be in contact after you submit the application.`; + } else if (submissionType === constants.submissionTypes.INITIAL) { subject = `You have requested access to ${title}. The custodian will be in contact about the application.`; } else { subject = `You have made updates to your Data Access Request for ${title}. The custodian will be in contact about the application.`; @@ -214,9 +216,9 @@ const _buildEmail = (aboutApplication, fullQuestions, questionAnswers, options) let { projectName = 'No project name set', isNationalCoreStudies = false, nationalCoreStudiesProjectId = '' } = aboutApplication; let linkNationalCoreStudies = nationalCoreStudiesProjectId === '' ? '' : `${process.env.homeURL}/project/${nationalCoreStudiesProjectId}`; let heading = - submissionType === constants.submissionTypes.INITIAL + submissionType === constants.submissionTypes.INPROGRESS ? 'Data access request application in progress' : (constants.submissionTypes.INITIAL ? `New data access request application` - : `Existing data access request application with new updates`; + : `Existing data access request application with new updates`); let subject = _buildSubjectTitle(userType, datasetTitles, submissionType); let questionTree = { ...fullQuestions }; let answers = { ...questionAnswers }; From a4ed0c4364d5a5f03d3e75c1fdb1c1e3eb8858d2 Mon Sep 17 00:00:00 2001 From: Paul McCafferty <> Date: Thu, 11 Feb 2021 11:50:23 +0000 Subject: [PATCH 11/42] Formatting the swagger.yaml file to fix issue to builds --- swagger.yaml | 1734 +++++++++++++++++++++++--------------------------- 1 file changed, 810 insertions(+), 924 deletions(-) diff --git a/swagger.yaml b/swagger.yaml index 4390de0d..7eda6efb 100644 --- a/swagger.yaml +++ b/swagger.yaml @@ -1,6 +1,6 @@ openapi: 3.0.1 info: - title: HDR UK API + title: HDR UK API description: API for Tools and artefacts repository. version: 1.0.0 servers: @@ -15,12 +15,12 @@ servers: enum: - '443' default: '443' -security: -- oauth2: [] +security: + - oauth2: [] paths: /oauth/token: post: - tags: + tags: - Authorization description: OAuth2.0 token endpoint responsible for issuing short-lived json web tokens (JWT) for access to secure Gateway APIs. For client credentials grant flow, a valid client id and secret must be provided to identify your application and provide the expected permissions. This type of authentication is reserved for team based connectivity through client applications and is not provided for human user access. For more information, contact the HDR-UK team. requestBody: @@ -40,16 +40,16 @@ paths: type: string description: A long (50 character) string provided by the HDR-UK team at the time of onboarding to the Gateway. Contact the HDR-UK team for issue of new credentials. required: - - grant_type - - client_secret - - client_id + - grant_type + - client_secret + - client_id examples: 'Client Credentials Grant Flow': value: { - "grant_type":"client_credentials", - "client_id":"2ca1f61a90e3547", - "client_secret":"3f80fecbf781b6da280a8d17aa1a22066fb66daa415d8befc1" + 'grant_type': 'client_credentials', + 'client_id': '2ca1f61a90e3547', + 'client_secret': '3f80fecbf781b6da280a8d17aa1a22066fb66daa415d8befc1', } responses: '200': @@ -72,9 +72,9 @@ paths: 'Client Credentials Grant Flow': value: { - "access_token":"Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJkYXRhIjp7Il9pZCI6IjYwMGJmYzk5YzhiZjcwMGYyYzdkNWMzNiIsInRpbWVTdGFtcCI2MTYxMjM4MzkwMzE5Nn0sImlhdCI6MTYxMjM4MzkwMywiZXhwIjoxNjEyMzg0ODAzfQ.-YvUBdjtJvdrRacz6E8-cYPQlum4TrEmiCFl8jO5a-M", - "token_type":"jwt", - "expires_in":900 + 'access_token': 'Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJkYXRhIjp7Il9pZCI6IjYwMGJmYzk5YzhiZjcwMGYyYzdkNWMzNiIsInRpbWVTdGFtcCI2MTYxMjM4MzkwMzE5Nn0sImlhdCI6MTYxMjM4MzkwMywiZXhwIjoxNjEyMzg0ODAzfQ.-YvUBdjtJvdrRacz6E8-cYPQlum4TrEmiCFl8jO5a-M', + 'token_type': 'jwt', + 'expires_in': 900, } '400': description: Failure response caused by incomplete or invalid client credentials being passed to the endpoint. @@ -91,27 +91,15 @@ paths: description: A message indicating that the request failed for a given reason. examples: 'Invalid Client Credentials': - value: - { - "success":false, - "message":"Invalid client credentials were provided for the authorisation attempt" - } + value: { 'success': false, 'message': 'Invalid client credentials were provided for the authorisation attempt' } 'Incomplete Client Credentials': - value: - { - "success":false, - "message":"Incomplete client credentials were provided for the authorisation attempt" - } + value: { 'success': false, 'message': 'Incomplete client credentials were provided for the authorisation attempt' } 'Invalid Grant Type': - value: - { - "success":false, - "message":"An invalid grant type has been specified" - } - + value: { 'success': false, 'message': 'An invalid grant type has been specified' } + /api/v1/data-access-request/{id}: get: - tags: + tags: - Data Access Request parameters: - in: path @@ -119,7 +107,7 @@ paths: required: true description: The unique identifier for a single data access request application. schema: - type : string + type: string example: 5ee249426136805fbf094eef description: Retrieve a single Data Access Request application using a supplied identifer responses: @@ -168,7 +156,7 @@ paths: description: An array of strings correlating to the dataset titles that have been selected for the application. applicationStatus: type: string - enum: + enum: - inProgress - submitted - inReview @@ -240,19 +228,19 @@ paths: description: A value to indicate if the requesting party is able to modify the application in its present state. For example, this will be false for a Custodian, but true for applicants if the applicant(s) are working on resubmitting the application following a request for amendments. unansweredAmendments: type: integer - description: The number of amendments that have been requested by the Custodian in the current amendment iteration. + description: The number of amendments that have been requested by the Custodian in the current amendment iteration. answeredAmendments: type: integer - description: The number of requested amendments that the applicant(s) have fixed in the current amendment iteration. + description: The number of requested amendments that the applicant(s) have fixed in the current amendment iteration. userType: type: string - enum: + enum: - custodian - applicant description: The type of user that has requested the Data Access Request application based on their permissions. It is either an applicant or a Custodian user. activeParty: type: string - enum: + enum: - custodian - applicant description: The party that is currently handling the application. This is the applicant during presubmission, then the Custodian following submission. The active party then fluctuates between parties during amendment iterations. @@ -274,112 +262,97 @@ paths: 'Approved Application': value: { - "status": "success", - "data": { - "aboutApplication": { - "selectedDatasets": [ + 'status': 'success', + 'data': + { + 'aboutApplication': { - "_id": "5fc31a18d98e4f4cff7e9315", - "datasetId": "d5faf9c6-6c34-46d7-93c4-7706a5436ed9", - "name": "HDR UK Papers & Preprints", - "description": "Publications that mention HDR-UK (or any variant thereof) in Acknowledgements or Author Affiliations\n\nThis will include:\n- Papers\n- COVID-19 Papers\n- COVID-19 Preprint", - "abstract": "Publications that mention HDR-UK (or any variant thereof) in Acknowledgements or Author Affiliations", - "publisher": "OTHER > HEALTH DATA RESEARCH UK", - "contactPoint": "hdr.hdr@hdruk.ac.uk", - "publisherObj": { - "dataRequestModalContent": { - "header": " ", - "body": "{omitted for brevity...}", - "footer": "" - }, - "active": true, - "allowsMessaging": true, - "workflowEnabled": true, - "_id": "5f7b1a2bce9f65e6ed83e7da", - "name": "OTHER > HEALTH DATA RESEARCH UK", - "imageURL": "", - "team": { - "active": true, - "_id": "5f7b1a2bce9f65e6ed83e7da", - "members": [ - { - "roles": [ - "manager" - ], - "memberid": "5f1a98861a821b4a53e44d15" - }, - { - "roles": [ - "manager" - ], - "memberid": "600bfc99c8bf700f2c7d5c36" - } - ], - "type": "publisher", - "__v": 3, - "createdAt": "2020-11-30T21:12:40.855Z", - "updatedAt": "2020-12-02T13:33:45.232Z" - } - } - } - ], - "isNationalCoreStudies": true, - "nationalCoreStudiesProjectId": "4324836585275824", - "projectName": "Test application title", - "completedDatasetSelection": true, - "completedInviteCollaborators": true, - "completedReadAdvice": true, - "completedCommunicateAdvice": true, - "completedApprovalsAdvice": true, - "completedSubmitAdvice": true - }, - "authorIds": [], - "datasetIds": [ - "d5faf9c6-6c34-46d7-93c4-7706a5436ed9" - ], - "datasetTitles": [], - "applicationStatus": "approved", - "jsonSchema": "{omitted for brevity...}", - "questionAnswers": { - "fullname-892140ec730145dc5a28b8fe139c2876": "James Smith", - "jobtitle-ff1d692a04b4bb9a2babe9093339136f": "Consultant", - "organisation-65c06905b8319ffa29919732a197d581": "Consulting Inc." - }, - "publisher": "OTHER > HEALTH DATA RESEARCH UK", - "_id": "60142c5b4316a0e0fcd47c56", - "version": 1, - "userId": 9190228196797084, - "schemaId": "5f55e87e780ba204b0a98eb8", - "files": [], - "amendmentIterations": [], - "createdAt": "2021-01-29T15:40:11.943Z", - "updatedAt": "2021-02-03T14:38:22.688Z", - "__v": 0, - "projectId": "6014-2C5B-4316-A0E0-FCD4-7C56", - "dateSubmitted": "2021-01-29T16:30:27.351Z", - "dateReviewStart": "2021-02-03T14:36:22.341Z", - "dateFinalStatus": "2021-02-03T14:38:22.680Z", - "datasets": [ - "{omitted for brevity...}" - ], - "dataset": null, - "mainApplicant": { - "_id": "5f1a98861a821b4a53e44d15", - "firstname": "James", - "lastname": "Smith" + 'selectedDatasets': + [ + { + '_id': '5fc31a18d98e4f4cff7e9315', + 'datasetId': 'd5faf9c6-6c34-46d7-93c4-7706a5436ed9', + 'name': 'HDR UK Papers & Preprints', + 'description': "Publications that mention HDR-UK (or any variant thereof) in Acknowledgements or Author Affiliations\n\nThis will include:\n- Papers\n- COVID-19 Papers\n- COVID-19 Preprint", + 'abstract': 'Publications that mention HDR-UK (or any variant thereof) in Acknowledgements or Author Affiliations', + 'publisher': 'OTHER > HEALTH DATA RESEARCH UK', + 'contactPoint': 'hdr.hdr@hdruk.ac.uk', + 'publisherObj': + { + 'dataRequestModalContent': { 'header': ' ', 'body': '{omitted for brevity...}', 'footer': '' }, + 'active': true, + 'allowsMessaging': true, + 'workflowEnabled': true, + '_id': '5f7b1a2bce9f65e6ed83e7da', + 'name': 'OTHER > HEALTH DATA RESEARCH UK', + 'imageURL': '', + 'team': + { + 'active': true, + '_id': '5f7b1a2bce9f65e6ed83e7da', + 'members': + [ + { 'roles': ['manager'], 'memberid': '5f1a98861a821b4a53e44d15' }, + { 'roles': ['manager'], 'memberid': '600bfc99c8bf700f2c7d5c36' }, + ], + 'type': 'publisher', + '__v': 3, + 'createdAt': '2020-11-30T21:12:40.855Z', + 'updatedAt': '2020-12-02T13:33:45.232Z', + }, + }, + }, + ], + 'isNationalCoreStudies': true, + 'nationalCoreStudiesProjectId': '4324836585275824', + 'projectName': 'Test application title', + 'completedDatasetSelection': true, + 'completedInviteCollaborators': true, + 'completedReadAdvice': true, + 'completedCommunicateAdvice': true, + 'completedApprovalsAdvice': true, + 'completedSubmitAdvice': true, + }, + 'authorIds': [], + 'datasetIds': ['d5faf9c6-6c34-46d7-93c4-7706a5436ed9'], + 'datasetTitles': [], + 'applicationStatus': 'approved', + 'jsonSchema': '{omitted for brevity...}', + 'questionAnswers': + { + 'fullname-892140ec730145dc5a28b8fe139c2876': 'James Smith', + 'jobtitle-ff1d692a04b4bb9a2babe9093339136f': 'Consultant', + 'organisation-65c06905b8319ffa29919732a197d581': 'Consulting Inc.', + }, + 'publisher': 'OTHER > HEALTH DATA RESEARCH UK', + '_id': '60142c5b4316a0e0fcd47c56', + 'version': 1, + 'userId': 9190228196797084, + 'schemaId': '5f55e87e780ba204b0a98eb8', + 'files': [], + 'amendmentIterations': [], + 'createdAt': '2021-01-29T15:40:11.943Z', + 'updatedAt': '2021-02-03T14:38:22.688Z', + '__v': 0, + 'projectId': '6014-2C5B-4316-A0E0-FCD4-7C56', + 'dateSubmitted': '2021-01-29T16:30:27.351Z', + 'dateReviewStart': '2021-02-03T14:36:22.341Z', + 'dateFinalStatus': '2021-02-03T14:38:22.680Z', + 'datasets': ['{omitted for brevity...}'], + 'dataset': null, + 'mainApplicant': { '_id': '5f1a98861a821b4a53e44d15', 'firstname': 'James', 'lastname': 'Smith' }, + 'authors': [], + 'id': '60142c5b4316a0e0fcd47c56', + 'readOnly': true, + 'unansweredAmendments': 0, + 'answeredAmendments': 0, + 'userType': 'custodian', + 'activeParty': 'custodian', + 'inReviewMode': false, + 'reviewSections': [], + 'hasRecommended': false, + 'workflow': {}, }, - "authors": [], - "id": "60142c5b4316a0e0fcd47c56", - "readOnly": true, - "unansweredAmendments": 0, - "answeredAmendments": 0, - "userType": "custodian", - "activeParty": "custodian", - "inReviewMode": false, - "reviewSections": [], - "hasRecommended": false, - "workflow": {} - } } '404': description: Failed to find the application requested. @@ -394,11 +367,7 @@ paths: type: string examples: 'Not Found': - value: - { - "status":"error", - "message":"Application not found." - } + value: { 'status': 'error', 'message': 'Application not found.' } '401': description: Unauthorised attempt to access an application. content: @@ -412,13 +381,9 @@ paths: type: string examples: 'Unauthorised': - value: - { - "status":"failure", - "message":"Unauthorised" - } + value: { 'status': 'failure', 'message': 'Unauthorised' } put: - tags: + tags: - Data Access Request parameters: - in: path @@ -426,7 +391,7 @@ paths: required: true description: The unique identifier for a single Data Access Request application. schema: - type : string + type: string example: 5ee249426136805fbf094eef description: Update a single Data Access Request application. requestBody: @@ -441,11 +406,7 @@ paths: type: string examples: 'Update Application Status': - value: - { - "applicationStatus": "approved", - "applicationStatusDesc": "This application meets all the requirements." - } + value: { 'applicationStatus': 'approved', 'applicationStatusDesc': 'This application meets all the requirements.' } responses: '200': description: Successful response containing the full, updated data access request application. @@ -492,7 +453,7 @@ paths: description: An array of strings correlating to the dataset titles that have been selected for the application. applicationStatus: type: string - enum: + enum: - inProgress - submitted - inReview @@ -564,19 +525,19 @@ paths: description: A value to indicate if the requesting party is able to modify the application in its present state. For example, this will be false for a Custodian, but true for applicants if the applicant(s) are working on resubmitting the application following a request for amendments. unansweredAmendments: type: integer - description: The number of amendments that have been requested by the Custodian in the current amendment iteration. + description: The number of amendments that have been requested by the Custodian in the current amendment iteration. answeredAmendments: type: integer - description: The number of requested amendments that the applicant(s) have fixed in the current amendment iteration. + description: The number of requested amendments that the applicant(s) have fixed in the current amendment iteration. userType: type: string - enum: + enum: - custodian - applicant description: The type of user that has requested the Data Access Request application based on their permissions. It is either an applicant or a Custodian user. activeParty: type: string - enum: + enum: - custodian - applicant description: The party that is currently handling the application. This is the applicant during presubmission, then the Custodian following submission. The active party then fluctuates between parties during amendment iterations. @@ -598,112 +559,97 @@ paths: 'Approved Application': value: { - "status": "success", - "data": { - "aboutApplication": { - "selectedDatasets": [ + 'status': 'success', + 'data': + { + 'aboutApplication': { - "_id": "5fc31a18d98e4f4cff7e9315", - "datasetId": "d5faf9c6-6c34-46d7-93c4-7706a5436ed9", - "name": "HDR UK Papers & Preprints", - "description": "Publications that mention HDR-UK (or any variant thereof) in Acknowledgements or Author Affiliations\n\nThis will include:\n- Papers\n- COVID-19 Papers\n- COVID-19 Preprint", - "abstract": "Publications that mention HDR-UK (or any variant thereof) in Acknowledgements or Author Affiliations", - "publisher": "OTHER > HEALTH DATA RESEARCH UK", - "contactPoint": "hdr.hdr@hdruk.ac.uk", - "publisherObj": { - "dataRequestModalContent": { - "header": " ", - "body": "{omitted for brevity...}", - "footer": "" - }, - "active": true, - "allowsMessaging": true, - "workflowEnabled": true, - "_id": "5f7b1a2bce9f65e6ed83e7da", - "name": "OTHER > HEALTH DATA RESEARCH UK", - "imageURL": "", - "team": { - "active": true, - "_id": "5f7b1a2bce9f65e6ed83e7da", - "members": [ - { - "roles": [ - "manager" - ], - "memberid": "5f1a98861a821b4a53e44d15" - }, - { - "roles": [ - "manager" - ], - "memberid": "600bfc99c8bf700f2c7d5c36" - } - ], - "type": "publisher", - "__v": 3, - "createdAt": "2020-11-30T21:12:40.855Z", - "updatedAt": "2020-12-02T13:33:45.232Z" - } - } - } - ], - "isNationalCoreStudies": true, - "nationalCoreStudiesProjectId": "4324836585275824", - "projectName": "Test application title", - "completedDatasetSelection": true, - "completedInviteCollaborators": true, - "completedReadAdvice": true, - "completedCommunicateAdvice": true, - "completedApprovalsAdvice": true, - "completedSubmitAdvice": true - }, - "authorIds": [], - "datasetIds": [ - "d5faf9c6-6c34-46d7-93c4-7706a5436ed9" - ], - "datasetTitles": [], - "applicationStatus": "approved", - "jsonSchema": "{omitted for brevity...}", - "questionAnswers": { - "fullname-892140ec730145dc5a28b8fe139c2876": "James Smith", - "jobtitle-ff1d692a04b4bb9a2babe9093339136f": "Consultant", - "organisation-65c06905b8319ffa29919732a197d581": "Consulting Inc." - }, - "publisher": "OTHER > HEALTH DATA RESEARCH UK", - "_id": "60142c5b4316a0e0fcd47c56", - "version": 1, - "userId": 9190228196797084, - "schemaId": "5f55e87e780ba204b0a98eb8", - "files": [], - "amendmentIterations": [], - "createdAt": "2021-01-29T15:40:11.943Z", - "updatedAt": "2021-02-03T14:38:22.688Z", - "__v": 0, - "projectId": "6014-2C5B-4316-A0E0-FCD4-7C56", - "dateSubmitted": "2021-01-29T16:30:27.351Z", - "dateReviewStart": "2021-02-03T14:36:22.341Z", - "dateFinalStatus": "2021-02-03T14:38:22.680Z", - "datasets": [ - "{omitted for brevity...}" - ], - "dataset": null, - "mainApplicant": { - "_id": "5f1a98861a821b4a53e44d15", - "firstname": "James", - "lastname": "Smith" + 'selectedDatasets': + [ + { + '_id': '5fc31a18d98e4f4cff7e9315', + 'datasetId': 'd5faf9c6-6c34-46d7-93c4-7706a5436ed9', + 'name': 'HDR UK Papers & Preprints', + 'description': "Publications that mention HDR-UK (or any variant thereof) in Acknowledgements or Author Affiliations\n\nThis will include:\n- Papers\n- COVID-19 Papers\n- COVID-19 Preprint", + 'abstract': 'Publications that mention HDR-UK (or any variant thereof) in Acknowledgements or Author Affiliations', + 'publisher': 'OTHER > HEALTH DATA RESEARCH UK', + 'contactPoint': 'hdr.hdr@hdruk.ac.uk', + 'publisherObj': + { + 'dataRequestModalContent': { 'header': ' ', 'body': '{omitted for brevity...}', 'footer': '' }, + 'active': true, + 'allowsMessaging': true, + 'workflowEnabled': true, + '_id': '5f7b1a2bce9f65e6ed83e7da', + 'name': 'OTHER > HEALTH DATA RESEARCH UK', + 'imageURL': '', + 'team': + { + 'active': true, + '_id': '5f7b1a2bce9f65e6ed83e7da', + 'members': + [ + { 'roles': ['manager'], 'memberid': '5f1a98861a821b4a53e44d15' }, + { 'roles': ['manager'], 'memberid': '600bfc99c8bf700f2c7d5c36' }, + ], + 'type': 'publisher', + '__v': 3, + 'createdAt': '2020-11-30T21:12:40.855Z', + 'updatedAt': '2020-12-02T13:33:45.232Z', + }, + }, + }, + ], + 'isNationalCoreStudies': true, + 'nationalCoreStudiesProjectId': '4324836585275824', + 'projectName': 'Test application title', + 'completedDatasetSelection': true, + 'completedInviteCollaborators': true, + 'completedReadAdvice': true, + 'completedCommunicateAdvice': true, + 'completedApprovalsAdvice': true, + 'completedSubmitAdvice': true, + }, + 'authorIds': [], + 'datasetIds': ['d5faf9c6-6c34-46d7-93c4-7706a5436ed9'], + 'datasetTitles': [], + 'applicationStatus': 'approved', + 'jsonSchema': '{omitted for brevity...}', + 'questionAnswers': + { + 'fullname-892140ec730145dc5a28b8fe139c2876': 'James Smith', + 'jobtitle-ff1d692a04b4bb9a2babe9093339136f': 'Consultant', + 'organisation-65c06905b8319ffa29919732a197d581': 'Consulting Inc.', + }, + 'publisher': 'OTHER > HEALTH DATA RESEARCH UK', + '_id': '60142c5b4316a0e0fcd47c56', + 'version': 1, + 'userId': 9190228196797084, + 'schemaId': '5f55e87e780ba204b0a98eb8', + 'files': [], + 'amendmentIterations': [], + 'createdAt': '2021-01-29T15:40:11.943Z', + 'updatedAt': '2021-02-03T14:38:22.688Z', + '__v': 0, + 'projectId': '6014-2C5B-4316-A0E0-FCD4-7C56', + 'dateSubmitted': '2021-01-29T16:30:27.351Z', + 'dateReviewStart': '2021-02-03T14:36:22.341Z', + 'dateFinalStatus': '2021-02-03T14:38:22.680Z', + 'datasets': ['{omitted for brevity...}'], + 'dataset': null, + 'mainApplicant': { '_id': '5f1a98861a821b4a53e44d15', 'firstname': 'James', 'lastname': 'Smith' }, + 'authors': [], + 'id': '60142c5b4316a0e0fcd47c56', + 'readOnly': true, + 'unansweredAmendments': 0, + 'answeredAmendments': 0, + 'userType': 'custodian', + 'activeParty': 'custodian', + 'inReviewMode': false, + 'reviewSections': [], + 'hasRecommended': false, + 'workflow': {}, }, - "authors": [], - "id": "60142c5b4316a0e0fcd47c56", - "readOnly": true, - "unansweredAmendments": 0, - "answeredAmendments": 0, - "userType": "custodian", - "activeParty": "custodian", - "inReviewMode": false, - "reviewSections": [], - "hasRecommended": false, - "workflow": {} - } } '404': description: Failed to find the application requested. @@ -718,11 +664,7 @@ paths: type: string examples: 'Not Found': - value: - { - "status": "error", - "message": "Application not found." - } + value: { 'status': 'error', 'message': 'Application not found.' } '401': description: Unauthorised attempt to update an application. content: @@ -736,11 +678,7 @@ paths: type: string examples: 'Unauthorised': - value: - { - "status":"error", - "message":"Unauthorised to perform this update." - } + value: { 'status': 'error', 'message': 'Unauthorised to perform this update.' } patch: summary: Update a users question answers for access request. security: @@ -753,7 +691,7 @@ paths: required: true description: The ID of the datset schema: - type : string + type: string example: 5ee249426136805fbf094eef requestBody: content: @@ -769,9 +707,9 @@ paths: { "firstName": "Roger" } - responses: + responses: '200': - description: OK + description: OK /api/v1/publishers/{publisher}/dataaccessrequests: get: tags: @@ -782,7 +720,7 @@ paths: required: true description: The full name of the Custodian/Publisher, as registered on the Gateway. schema: - type : string + type: string example: OTHER > HEALTH DATA RESEARCH UK description: Returns a collection of all Data Access Requests that have been submitted to the Custodian team for review. responses: @@ -796,7 +734,7 @@ paths: avgDecisionTime: type: string description: The average number of days the Custodian has taken to process applications from submission to decision. - canViewSubmitted: + canViewSubmitted: type: boolean description: A flag to indicate if the requesting user has permissions to view submitted applications, which are visible only to managers of the Custodian team. Using OAuth2.0 client credentials will return this value as true. status: @@ -824,7 +762,7 @@ paths: items: type: object description: An array containing an object with details for each iteration the application has passed through. An iteration is defined as an application which has been returned by the Custodian for correction, corrected by the applicant(s) and resubmitted. The object contains dates that the application was returned, and resubmitted as well as reference to any questions that were highlighted for amendment. - amendmentStatus: + amendmentStatus: type: string description: A textual indicator of what state the application is in relating to updates made by the Custodian e.g. if it is awaiting updates from the applicant or if new updates have been submitted by the applicant(s). applicants: @@ -832,7 +770,7 @@ paths: description: Concatenated list of applicants names who are contributing to the application. applicationStatus: type: string - enum: + enum: - inProgress - submitted - inReview @@ -874,7 +812,7 @@ paths: id: type: string description: The unique identifier for the application. - + jsonSchema: type: object description: The object containing the json definition that renders the application form using the Winterfell library. This contains the details of questions, questions sets, question panels, headings and navigation items that appear. @@ -938,13 +876,13 @@ paths: type: array items: type: string - description: An array containing the names of Custodian team reviewers expected to complete a review for the current workflow phase, or a list of managers if the application is awaiting a final decision. + description: An array containing the names of Custodian team reviewers expected to complete a review for the current workflow phase, or a list of managers if the application is awaiting a final decision. reviewStatus: type: string description: A message indicating the current status of the application review in relation to the assigned workflow. E.g. 'Final decision required' or 'Deadline is today'. This message changes based on the requesting user's relationship to the application. E.g. if they are a reviewer or manager. stepName: type: string - description: The name of the current workflow step that the application is in. + description: The name of the current workflow step that the application is in. workflowCompleted: type: boolean description: A flag to indicate if the assigned workflow for the review process has been completed. @@ -955,439 +893,395 @@ paths: 'Single Request Received': value: { - "success": true, - "data": [ - { - "authorIds": [], - "datasetIds": [ - "d5faf9c6-6c34-46d7-93c4-7706a5436ed9" - ], - "datasetTitles": [], - "applicationStatus": "submitted", - "jsonSchema": "{omitted for brevity...}", - "questionAnswers": "{omitted for brevity...}", - "publisher": "OTHER > HEALTH DATA RESEARCH UK", - "_id": "601853db22dc004f9adfaa24", - "version": 1, - "userId": 7584453789581072, - "schemaId": "5f55e87e780ba204b0a98eb8", - "files": [ - { - "error": "", - "_id": "601aacf8ecdfa66e5cbc2742", - "status": "UPLOADED", - "description": "QuestionAnswers", - "fileId": "9e76ee1a676f423b9b5c7aabf59c69db", - "size": 509984, - "name": "QuestionAnswersFlags.png", - "owner": "5ec7f1b39219d627e5cafae3" - }, - { - "error": "", - "_id": "601aadbcecdfa6c532bc2743", - "status": "UPLOADED", - "description": "Notifications", - "fileId": "adb1718dcc094b9cb4b0ab347ad2ee94", - "size": 54346, - "name": "HQIP-Workflow-Assigned-Notification.png", - "owner": "5ec7f1b39219d627e5cafae3" - } - ], - "amendmentIterations": [], - "createdAt": "2021-02-01T19:17:47.470Z", - "updatedAt": "2021-02-03T16:36:36.720Z", - "__v": 2, - "projectId": "6018-53DB-22DC-004F-9ADF-AA24", - "aboutApplication": { - "selectedDatasets": [ - { - "_id": "5fc31a18d98e4f4cff7e9315", - "datasetId": "d5faf9c6-6c34-46d7-93c4-7706a5436ed9", - "name": "HDR UK Papers & Preprints", - "description": "Publications that mention HDR-UK (or any variant thereof) in Acknowledgements or Author Affiliations\n\nThis will include:\n- Papers\n- COVID-19 Papers\n- COVID-19 Preprint", - "abstract": "Publications that mention HDR-UK (or any variant thereof) in Acknowledgements or Author Affiliations", - "publisher": "OTHER > HEALTH DATA RESEARCH UK", - "contactPoint": "hdr.hdr@hdruk.ac.uk", - "publisherObj": { - "dataRequestModalContent": { - "header": " ", - "body": "{omitted for brevity...}", - "footer": "" - }, - "active": true, - "allowsMessaging": true, - "workflowEnabled": true, - "_id": "5f7b1a2bce9f65e6ed83e7da", - "name": "OTHER > HEALTH DATA RESEARCH UK", - "imageURL": "", - "team": { - "active": true, - "_id": "5f7b1a2bce9f65e6ed83e7da", - "members": [ - { - "roles": [ - "manager" - ], - "memberid": "5f1a98861a821b4a53e44d15" - }, - { - "roles": [ - "manager" - ], - "memberid": "600bfc99c8bf700f2c7d5c36" - } - ], - "type": "publisher", - "__v": 3, - "createdAt": "2020-11-30T21:12:40.855Z", - "updatedAt": "2020-12-02T13:33:45.232Z" - } - } - } - ], - "isNationalCoreStudies": true, - "nationalCoreStudiesProjectId": "4324836585275824", - "projectName": "Test application title", - "completedDatasetSelection": true, - "completedInviteCollaborators": true, - "completedReadAdvice": true, - "completedCommunicateAdvice": true, - "completedApprovalsAdvice": true, - "completedSubmitAdvice": true - }, - "dateSubmitted": "2021-02-03T16:37:36.081Z", - "datasets": [ - { - "categories": { - "programmingLanguage": [] - }, - "tags": { - "features": [ - "Preprints", - "Papers", - "HDR UK" - ], - "topics": [] - }, - "datasetfields": { - "geographicCoverage": [ - "https://www.geonames.org/countries/GB/united-kingdom.html" - ], - "physicalSampleAvailability": [ - "Not Available" - ], - "technicaldetails": "{omitted for brevity...}", - "versionLinks": [ - { - "id": "142b1618-2691-4019-97b4-16b1e27c5f95", - "linkType": "Superseded By", - "domainType": "CatalogueSemanticLink", - "source": { - "id": "9e798632-442a-427b-8d0e-456f754d28dc", - "domainType": "DataModel", - "label": "HDR UK Papers & Preprints", - "documentationVersion": "0.0.1" - }, - "target": { - "id": "d5faf9c6-6c34-46d7-93c4-7706a5436ed9", - "domainType": "DataModel", - "label": "HDR UK Papers & Preprints", - "documentationVersion": "1.0.0" - } - } - ], - "phenotypes": [], - "publisher": "OTHER > HEALTH DATA RESEARCH UK", - "abstract": "Publications that mention HDR-UK (or any variant thereof) in Acknowledgements or Author Affiliations", - "releaseDate": "2020-11-27T00:00:00Z", - "accessRequestDuration": "Other", - "conformsTo": "OTHER", - "accessRights": "https://github.com/HDRUK/papers/blob/master/LICENSE", - "jurisdiction": "GB-ENG", - "datasetStartDate": "2020-03-31", - "datasetEndDate": "2022-04-30", - "statisticalPopulation": "0", - "ageBand": "0-0", - "contactPoint": "hdr.hdr@hdruk.ac.uk", - "periodicity": "Daily", - "metadataquality": { - "id": "d5faf9c6-6c34-46d7-93c4-7706a5436ed9", - "publisher": "OTHER > HEALTH DATA RESEARCH UK", - "title": "HDR UK Papers & Preprints", - "completeness_percent": 95.24, - "weighted_completeness_percent": 100, - "error_percent": 11.63, - "weighted_error_percent": 19.05, - "quality_score": 91.81, - "quality_rating": "Gold", - "weighted_quality_score": 90.47, - "weighted_quality_rating": "Gold" + 'success': true, + 'data': + [ + { + 'authorIds': [], + 'datasetIds': ['d5faf9c6-6c34-46d7-93c4-7706a5436ed9'], + 'datasetTitles': [], + 'applicationStatus': 'submitted', + 'jsonSchema': '{omitted for brevity...}', + 'questionAnswers': '{omitted for brevity...}', + 'publisher': 'OTHER > HEALTH DATA RESEARCH UK', + '_id': '601853db22dc004f9adfaa24', + 'version': 1, + 'userId': 7584453789581072, + 'schemaId': '5f55e87e780ba204b0a98eb8', + 'files': + [ + { + 'error': '', + '_id': '601aacf8ecdfa66e5cbc2742', + 'status': 'UPLOADED', + 'description': 'QuestionAnswers', + 'fileId': '9e76ee1a676f423b9b5c7aabf59c69db', + 'size': 509984, + 'name': 'QuestionAnswersFlags.png', + 'owner': '5ec7f1b39219d627e5cafae3', }, - "datautility": { - "id": "d5faf9c6-6c34-46d7-93c4-7706a5436ed9", - "publisher": "OTHER > HEALTH DATA RESEARCH UK", - "title": "HDR UK Papers & Preprints", - "metadata_richness": "Gold", - "availability_of_additional_documentation_and_support": "", - "data_model": "", - "data_dictionary": "", - "provenance": "", - "data_quality_management_process": "", - "dama_quality_dimensions": "", - "pathway_coverage": "", - "length_of_follow_up": "", - "allowable_uses": "", - "research_environment": "", - "time_lag": "", - "timeliness": "", - "linkages": "", - "data_enrichments": "" + { + 'error': '', + '_id': '601aadbcecdfa6c532bc2743', + 'status': 'UPLOADED', + 'description': 'Notifications', + 'fileId': 'adb1718dcc094b9cb4b0ab347ad2ee94', + 'size': 54346, + 'name': 'HQIP-Workflow-Assigned-Notification.png', + 'owner': '5ec7f1b39219d627e5cafae3', }, - "metadataschema": { - "@context": "http://schema.org/", - "@type": "Dataset", - "identifier": "d5faf9c6-6c34-46d7-93c4-7706a5436ed9", - "url": "https://healthdatagateway.org/detail/d5faf9c6-6c34-46d7-93c4-7706a5436ed9", - "name": "HDR UK Papers & Preprints", - "description": "Publications that mention HDR-UK (or any variant thereof) in Acknowledgements or Author Affiliations\n\nThis will include:\n- Papers\n- COVID-19 Papers\n- COVID-19 Preprint", - "license": "Open Access", - "keywords": [ - "Preprints,Papers,HDR UK", - "OTHER > HEALTH DATA RESEARCH UK", - "NOT APPLICABLE", - "GB-ENG", - "https://www.geonames.org/countries/GB/united-kingdom.html" - ], - "includedinDataCatalog": [ + ], + 'amendmentIterations': [], + 'createdAt': '2021-02-01T19:17:47.470Z', + 'updatedAt': '2021-02-03T16:36:36.720Z', + '__v': 2, + 'projectId': '6018-53DB-22DC-004F-9ADF-AA24', + 'aboutApplication': + { + 'selectedDatasets': + [ { - "@type": "DataCatalog", - "name": "OTHER > HEALTH DATA RESEARCH UK", - "url": "hdr.hdr@hdruk.ac.uk" + '_id': '5fc31a18d98e4f4cff7e9315', + 'datasetId': 'd5faf9c6-6c34-46d7-93c4-7706a5436ed9', + 'name': 'HDR UK Papers & Preprints', + 'description': "Publications that mention HDR-UK (or any variant thereof) in Acknowledgements or Author Affiliations\n\nThis will include:\n- Papers\n- COVID-19 Papers\n- COVID-19 Preprint", + 'abstract': 'Publications that mention HDR-UK (or any variant thereof) in Acknowledgements or Author Affiliations', + 'publisher': 'OTHER > HEALTH DATA RESEARCH UK', + 'contactPoint': 'hdr.hdr@hdruk.ac.uk', + 'publisherObj': + { + 'dataRequestModalContent': { 'header': ' ', 'body': '{omitted for brevity...}', 'footer': '' }, + 'active': true, + 'allowsMessaging': true, + 'workflowEnabled': true, + '_id': '5f7b1a2bce9f65e6ed83e7da', + 'name': 'OTHER > HEALTH DATA RESEARCH UK', + 'imageURL': '', + 'team': + { + 'active': true, + '_id': '5f7b1a2bce9f65e6ed83e7da', + 'members': + [ + { 'roles': ['manager'], 'memberid': '5f1a98861a821b4a53e44d15' }, + { 'roles': ['manager'], 'memberid': '600bfc99c8bf700f2c7d5c36' }, + ], + 'type': 'publisher', + '__v': 3, + 'createdAt': '2020-11-30T21:12:40.855Z', + 'updatedAt': '2020-12-02T13:33:45.232Z', + }, + }, }, - { - "@type": "DataCatalog", - "name": "HDR UK Health Data Gateway", - "url": "http://healthdatagateway.org" - } - ] - } - }, - "authors": [], - "showOrganisation": false, - "toolids": [], - "datasetids": [], - "_id": "5fc31a18d98e4f4cff7e9315", - "relatedObjects": [], - "programmingLanguage": [], - "pid": "b7a62c6d-ed00-4423-ad27-e90b71222d8e", - "datasetVersion": "1.0.0", - "id": 9816147066244124, - "datasetid": "d5faf9c6-6c34-46d7-93c4-7706a5436ed9", - "type": "dataset", - "activeflag": "active", - "name": "HDR UK Papers & Preprints", - "description": "Publications that mention HDR-UK (or any variant thereof) in Acknowledgements or Author Affiliations\n\nThis will include:\n- Papers\n- COVID-19 Papers\n- COVID-19 Preprint", - "license": "Open Access", - "datasetv2": { - "identifier": "", - "version": "", - "issued": "", - "modified": "", - "revisions": [], - "summary": { - "title": "", - "abstract": "Publications that mention HDR-UK (or any variant thereof) in Acknowledgements or Author Affiliations", - "publisher": { - "identifier": "", - "name": "HEALTH DATA RESEARCH UK", - "logo": "", - "description": "", - "contactPoint": "hdr.hdr@hdruk.ac.uk", - "memberOf": "OTHER", - "accessRights": [], - "deliveryLeadTime": "", - "accessService": "", - "accessRequestCost": "", - "dataUseLimitation": [], - "dataUseRequirements": [] - }, - "contactPoint": "hdr.hdr@hdruk.ac.uk", - "keywords": [ - "Preprints", - "Papers", - "HDR UK" ], - "alternateIdentifiers": [], - "doiName": "https://doi.org/10.5281/zenodo.326615" - }, - "documentation": { - "description": "", - "associatedMedia": [ - "https://github.com/HDRUK/papers" - ], - "isPartOf": "NOT APPLICABLE" - }, - "coverage": { - "spatial": "GB", - "typicalAgeRange": "0-0", - "physicalSampleAvailability": [ - "NOT AVAILABLE" - ], - "followup": "UNKNOWN", - "pathway": "NOT APPLICABLE" - }, - "provenance": { - "origin": { - "purpose": "OTHER", - "source": "MACHINE GENERATED", - "collectionSituation": "OTHER" - }, - "temporal": { - "accrualPeriodicity": "DAILY", - "distributionReleaseDate": "2020-11-27", - "startDate": "2020-03-31", - "endDate": "2022-04-30", - "timeLag": "NO TIMELAG" - } - }, - "accessibility": { - "usage": { - "dataUseLimitation": "GENERAL RESEARCH USE", - "dataUseRequirements": "RETURN TO DATABASE OR RESOURCE", - "resourceCreator": "HDR UK Using Team", - "investigations": [ - "https://github.com/HDRUK/papers" - ], - "isReferencedBy": [ - "Not Available" - ] - }, - "access": { - "accessRights": [ - "Open Access" - ], - "accessService": "https://github.com/HDRUK/papers", - "accessRequestCost": "Free", - "deliveryLeadTime": "OTHER", - "jurisdiction": "GB-ENG", - "dataProcessor": "HDR UK", - "dataController": "HDR UK" - }, - "formatAndStandards": { - "vocabularyEncodingScheme": "OTHER", - "conformsTo": "OTHER", - "language": "en", - "format": [ - "csv", - "JSON" - ] - } - }, - "enrichmentAndLinkage": { - "qualifiedRelation": [ - "Not Available" - ], - "derivation": [ - "Not Available" - ], - "tools": [ - "https://github.com/HDRUK/papers" - ] - }, - "observations": [] + 'isNationalCoreStudies': true, + 'nationalCoreStudiesProjectId': '4324836585275824', + 'projectName': 'Test application title', + 'completedDatasetSelection': true, + 'completedInviteCollaborators': true, + 'completedReadAdvice': true, + 'completedCommunicateAdvice': true, + 'completedApprovalsAdvice': true, + 'completedSubmitAdvice': true, }, - "createdAt": "2020-11-29T03:48:41.794Z", - "updatedAt": "2021-02-02T10:09:57.030Z", - "__v": 0, - "counter": 20 - } - ], - "dataset": null, - "mainApplicant": { - "isServiceAccount": false, - "_id": "5ec7f1b39219d627e5cafae3", - "id": 7584453789581072, - "providerId": "112563375053074694443", - "provider": "google", - "firstname": "Chris", - "lastname": "Marks", - "email": "chris.marks@paconsulting.com", - "role": "Admin", - "__v": 0, - "redirectURL": "/tool/100000012", - "discourseKey": "2f52ecaa21a0d0223a119da5a09f8f8b09459e7b69ec3f981102d09f66488d99", - "discourseUsername": "chris.marks", - "updatedAt": "2021-02-01T12:39:56.372Z" - }, - "publisherObj": { - "dataRequestModalContent": { - "header": "", - "body": "", - "footer": "" - }, - "active": true, - "allowsMessaging": true, - "workflowEnabled": true, - "_id": "5f7b1a2bce9f65e6ed83e7da", - "name": "OTHER > HEALTH DATA RESEARCH UK", - "imageURL": "", - "team": { - "active": true, - "_id": "5f7b1a2bce9f65e6ed83e7da", - "members": [ + 'dateSubmitted': '2021-02-03T16:37:36.081Z', + 'datasets': + [ { - "roles": [ - "manager" - ], - "memberid": "5f1a98861a821b4a53e44d15" + 'categories': { 'programmingLanguage': [] }, + 'tags': { 'features': ['Preprints', 'Papers', 'HDR UK'], 'topics': [] }, + 'datasetfields': + { + 'geographicCoverage': ['https://www.geonames.org/countries/GB/united-kingdom.html'], + 'physicalSampleAvailability': ['Not Available'], + 'technicaldetails': '{omitted for brevity...}', + 'versionLinks': + [ + { + 'id': '142b1618-2691-4019-97b4-16b1e27c5f95', + 'linkType': 'Superseded By', + 'domainType': 'CatalogueSemanticLink', + 'source': + { + 'id': '9e798632-442a-427b-8d0e-456f754d28dc', + 'domainType': 'DataModel', + 'label': 'HDR UK Papers & Preprints', + 'documentationVersion': '0.0.1', + }, + 'target': + { + 'id': 'd5faf9c6-6c34-46d7-93c4-7706a5436ed9', + 'domainType': 'DataModel', + 'label': 'HDR UK Papers & Preprints', + 'documentationVersion': '1.0.0', + }, + }, + ], + 'phenotypes': [], + 'publisher': 'OTHER > HEALTH DATA RESEARCH UK', + 'abstract': 'Publications that mention HDR-UK (or any variant thereof) in Acknowledgements or Author Affiliations', + 'releaseDate': '2020-11-27T00:00:00Z', + 'accessRequestDuration': 'Other', + 'conformsTo': 'OTHER', + 'accessRights': 'https://github.com/HDRUK/papers/blob/master/LICENSE', + 'jurisdiction': 'GB-ENG', + 'datasetStartDate': '2020-03-31', + 'datasetEndDate': '2022-04-30', + 'statisticalPopulation': '0', + 'ageBand': '0-0', + 'contactPoint': 'hdr.hdr@hdruk.ac.uk', + 'periodicity': 'Daily', + 'metadataquality': + { + 'id': 'd5faf9c6-6c34-46d7-93c4-7706a5436ed9', + 'publisher': 'OTHER > HEALTH DATA RESEARCH UK', + 'title': 'HDR UK Papers & Preprints', + 'completeness_percent': 95.24, + 'weighted_completeness_percent': 100, + 'error_percent': 11.63, + 'weighted_error_percent': 19.05, + 'quality_score': 91.81, + 'quality_rating': 'Gold', + 'weighted_quality_score': 90.47, + 'weighted_quality_rating': 'Gold', + }, + 'datautility': + { + 'id': 'd5faf9c6-6c34-46d7-93c4-7706a5436ed9', + 'publisher': 'OTHER > HEALTH DATA RESEARCH UK', + 'title': 'HDR UK Papers & Preprints', + 'metadata_richness': 'Gold', + 'availability_of_additional_documentation_and_support': '', + 'data_model': '', + 'data_dictionary': '', + 'provenance': '', + 'data_quality_management_process': '', + 'dama_quality_dimensions': '', + 'pathway_coverage': '', + 'length_of_follow_up': '', + 'allowable_uses': '', + 'research_environment': '', + 'time_lag': '', + 'timeliness': '', + 'linkages': '', + 'data_enrichments': '', + }, + 'metadataschema': + { + '@context': 'http://schema.org/', + '@type': 'Dataset', + 'identifier': 'd5faf9c6-6c34-46d7-93c4-7706a5436ed9', + 'url': 'https://healthdatagateway.org/detail/d5faf9c6-6c34-46d7-93c4-7706a5436ed9', + 'name': 'HDR UK Papers & Preprints', + 'description': "Publications that mention HDR-UK (or any variant thereof) in Acknowledgements or Author Affiliations\n\nThis will include:\n- Papers\n- COVID-19 Papers\n- COVID-19 Preprint", + 'license': 'Open Access', + 'keywords': + [ + 'Preprints,Papers,HDR UK', + 'OTHER > HEALTH DATA RESEARCH UK', + 'NOT APPLICABLE', + 'GB-ENG', + 'https://www.geonames.org/countries/GB/united-kingdom.html', + ], + 'includedinDataCatalog': + [ + { + '@type': 'DataCatalog', + 'name': 'OTHER > HEALTH DATA RESEARCH UK', + 'url': 'hdr.hdr@hdruk.ac.uk', + }, + { + '@type': 'DataCatalog', + 'name': 'HDR UK Health Data Gateway', + 'url': 'http://healthdatagateway.org', + }, + ], + }, + }, + 'authors': [], + 'showOrganisation': false, + 'toolids': [], + 'datasetids': [], + '_id': '5fc31a18d98e4f4cff7e9315', + 'relatedObjects': [], + 'programmingLanguage': [], + 'pid': 'b7a62c6d-ed00-4423-ad27-e90b71222d8e', + 'datasetVersion': '1.0.0', + 'id': 9816147066244124, + 'datasetid': 'd5faf9c6-6c34-46d7-93c4-7706a5436ed9', + 'type': 'dataset', + 'activeflag': 'active', + 'name': 'HDR UK Papers & Preprints', + 'description': "Publications that mention HDR-UK (or any variant thereof) in Acknowledgements or Author Affiliations\n\nThis will include:\n- Papers\n- COVID-19 Papers\n- COVID-19 Preprint", + 'license': 'Open Access', + 'datasetv2': + { + 'identifier': '', + 'version': '', + 'issued': '', + 'modified': '', + 'revisions': [], + 'summary': + { + 'title': '', + 'abstract': 'Publications that mention HDR-UK (or any variant thereof) in Acknowledgements or Author Affiliations', + 'publisher': + { + 'identifier': '', + 'name': 'HEALTH DATA RESEARCH UK', + 'logo': '', + 'description': '', + 'contactPoint': 'hdr.hdr@hdruk.ac.uk', + 'memberOf': 'OTHER', + 'accessRights': [], + 'deliveryLeadTime': '', + 'accessService': '', + 'accessRequestCost': '', + 'dataUseLimitation': [], + 'dataUseRequirements': [], + }, + 'contactPoint': 'hdr.hdr@hdruk.ac.uk', + 'keywords': ['Preprints', 'Papers', 'HDR UK'], + 'alternateIdentifiers': [], + 'doiName': 'https://doi.org/10.5281/zenodo.326615', + }, + 'documentation': + { + 'description': '', + 'associatedMedia': ['https://github.com/HDRUK/papers'], + 'isPartOf': 'NOT APPLICABLE', + }, + 'coverage': + { + 'spatial': 'GB', + 'typicalAgeRange': '0-0', + 'physicalSampleAvailability': ['NOT AVAILABLE'], + 'followup': 'UNKNOWN', + 'pathway': 'NOT APPLICABLE', + }, + 'provenance': + { + 'origin': { 'purpose': 'OTHER', 'source': 'MACHINE GENERATED', 'collectionSituation': 'OTHER' }, + 'temporal': + { + 'accrualPeriodicity': 'DAILY', + 'distributionReleaseDate': '2020-11-27', + 'startDate': '2020-03-31', + 'endDate': '2022-04-30', + 'timeLag': 'NO TIMELAG', + }, + }, + 'accessibility': + { + 'usage': + { + 'dataUseLimitation': 'GENERAL RESEARCH USE', + 'dataUseRequirements': 'RETURN TO DATABASE OR RESOURCE', + 'resourceCreator': 'HDR UK Using Team', + 'investigations': ['https://github.com/HDRUK/papers'], + 'isReferencedBy': ['Not Available'], + }, + 'access': + { + 'accessRights': ['Open Access'], + 'accessService': 'https://github.com/HDRUK/papers', + 'accessRequestCost': 'Free', + 'deliveryLeadTime': 'OTHER', + 'jurisdiction': 'GB-ENG', + 'dataProcessor': 'HDR UK', + 'dataController': 'HDR UK', + }, + 'formatAndStandards': + { + 'vocabularyEncodingScheme': 'OTHER', + 'conformsTo': 'OTHER', + 'language': 'en', + 'format': ['csv', 'JSON'], + }, + }, + 'enrichmentAndLinkage': + { + 'qualifiedRelation': ['Not Available'], + 'derivation': ['Not Available'], + 'tools': ['https://github.com/HDRUK/papers'], + }, + 'observations': [], + }, + 'createdAt': '2020-11-29T03:48:41.794Z', + 'updatedAt': '2021-02-02T10:09:57.030Z', + '__v': 0, + 'counter': 20, }, - { - "roles": [ - "manager" - ], - "memberid": "600bfc99c8bf700f2c7d5c36" - } ], - "type": "publisher", - "__v": 3, - "createdAt": "2020-11-30T21:12:40.855Z", - "updatedAt": "2020-12-02T13:33:45.232Z", - "users": [ - { - "_id": "5f1a98861a821b4a53e44d15", - "firstname": "Robin", - "lastname": "Kavanagh" - }, - { - "_id": "600bfc99c8bf700f2c7d5c36", - "firstname": "HDR-UK", - "lastname": "Service Account" - } - ] - } + 'dataset': null, + 'mainApplicant': + { + 'isServiceAccount': false, + '_id': '5ec7f1b39219d627e5cafae3', + 'id': 7584453789581072, + 'providerId': '112563375053074694443', + 'provider': 'google', + 'firstname': 'Chris', + 'lastname': 'Marks', + 'email': 'chris.marks@paconsulting.com', + 'role': 'Admin', + '__v': 0, + 'redirectURL': '/tool/100000012', + 'discourseKey': '2f52ecaa21a0d0223a119da5a09f8f8b09459e7b69ec3f981102d09f66488d99', + 'discourseUsername': 'chris.marks', + 'updatedAt': '2021-02-01T12:39:56.372Z', + }, + 'publisherObj': + { + 'dataRequestModalContent': { 'header': '', 'body': '', 'footer': '' }, + 'active': true, + 'allowsMessaging': true, + 'workflowEnabled': true, + '_id': '5f7b1a2bce9f65e6ed83e7da', + 'name': 'OTHER > HEALTH DATA RESEARCH UK', + 'imageURL': '', + 'team': + { + 'active': true, + '_id': '5f7b1a2bce9f65e6ed83e7da', + 'members': + [ + { 'roles': ['manager'], 'memberid': '5f1a98861a821b4a53e44d15' }, + { 'roles': ['manager'], 'memberid': '600bfc99c8bf700f2c7d5c36' }, + ], + 'type': 'publisher', + '__v': 3, + 'createdAt': '2020-11-30T21:12:40.855Z', + 'updatedAt': '2020-12-02T13:33:45.232Z', + 'users': + [ + { '_id': '5f1a98861a821b4a53e44d15', 'firstname': 'Robin', 'lastname': 'Kavanagh' }, + { '_id': '600bfc99c8bf700f2c7d5c36', 'firstname': 'HDR-UK', 'lastname': 'Service Account' }, + ], + }, + }, + 'id': '601853db22dc004f9adfaa24', + 'projectName': 'PA Paper', + 'applicants': 'Chris Marks', + 'workflowName': '', + 'workflowCompleted': false, + 'decisionDuration': '', + 'decisionMade': false, + 'decisionStatus': '', + 'decisionComments': '', + 'decisionDate': '', + 'decisionApproved': false, + 'remainingActioners': 'Robin Kavanagh (you), HDR-UK Service Account', + 'stepName': '', + 'deadlinePassed': '', + 'reviewStatus': '', + 'isReviewer': false, + 'reviewPanels': [], + 'amendmentStatus': '', }, - "id": "601853db22dc004f9adfaa24", - "projectName": "PA Paper", - "applicants": "Chris Marks", - "workflowName": "", - "workflowCompleted": false, - "decisionDuration": "", - "decisionMade": false, - "decisionStatus": "", - "decisionComments": "", - "decisionDate": "", - "decisionApproved": false, - "remainingActioners": "Robin Kavanagh (you), HDR-UK Service Account", - "stepName": "", - "deadlinePassed": "", - "reviewStatus": "", - "isReviewer": false, - "reviewPanels": [], - "amendmentStatus": "" - } - ], - "avgDecisionTime": 1, - "canViewSubmitted": true + ], + 'avgDecisionTime': 1, + 'canViewSubmitted': true, } '404': description: Failed to find the application requested. @@ -1400,10 +1294,7 @@ paths: type: boolean examples: 'Not Found': - value: - { - "success":false - } + value: { 'success': false } '401': description: Unauthorised attempt to access an application. content: @@ -1417,11 +1308,7 @@ paths: type: string examples: 'Unauthorised': - value: - { - "status":"failure", - "message":"Unauthorised" - } + value: { 'status': 'failure', 'message': 'Unauthorised' } /api/v1/datasets/{datasetID}: get: summary: Returns Dataset object. @@ -1433,11 +1320,11 @@ paths: required: true description: The ID of the datset schema: - type : string + type: string example: '756daeaa-6e47-4269-9df5-477c01cdd271' - responses: + responses: '200': - description: OK + description: OK /api/v1/datasets: get: @@ -1445,28 +1332,28 @@ paths: tags: - Datasets parameters: - - in: query - name: limit - required: false - description: Limit the number of results - schema: - type : integer - example: 3 - - in: query - name: offset - required: false - description: Index to offset the search results - schema: - type : integer - example: 1 - - in: query - name: q - required: false - description: Filter using search query - schema: - type : string - example: epilepsy - responses: + - in: query + name: limit + required: false + description: Limit the number of results + schema: + type: integer + example: 3 + - in: query + name: offset + required: false + description: Index to offset the search results + schema: + type: integer + example: 1 + - in: query + name: q + required: false + description: Filter using search query + schema: + type: string + example: epilepsy + responses: '200': description: OK @@ -1483,11 +1370,11 @@ paths: required: true description: The ID of the datset schema: - type : string + type: string example: 6efbc62f-6ebb-4f18-959b-1ec6fd0cc6fb - responses: + responses: '200': - description: OK + description: OK /api/v1/person/{id}: get: @@ -1500,12 +1387,12 @@ paths: required: true description: The ID of the person schema: - type : string + type: string example: 900000014 responses: '200': description: OK - + /api/v1/person: get: summary: Returns an array of person objects. @@ -1521,7 +1408,7 @@ paths: requestBody: content: application/json: - schema: # Request body contents + schema: # Request body contents type: object required: - firstname @@ -1567,7 +1454,7 @@ paths: requestBody: content: application/json: - schema: # Request body contents + schema: # Request body contents type: object required: - id @@ -1596,7 +1483,7 @@ paths: orcid: 'https://orcid.org/123456789' emailNotifications: false terms: true - + /api/v1/search: get: tags: @@ -1634,10 +1521,10 @@ paths: example: 10 style: form explode: true - responses: + responses: '200': description: OK - + /api/v1/stats/topSearches: get: summary: Returns top searches for a given month and year. @@ -1649,16 +1536,16 @@ paths: required: true description: Month number. schema: - type : string + type: string example: 7 - name: year in: query required: true description: Year. schema: - type : string + type: string example: 2020 - responses: + responses: '200': description: OK @@ -1673,30 +1560,30 @@ paths: required: true description: The type of stat. schema: - type : string + type: string example: unmet - name: type in: query required: true description: Resource type. schema: - type : string + type: string example: Tools - name: month in: query required: true description: Month number. schema: - type : string + type: string example: 7 - name: year in: query required: true description: Year. schema: - type : string + type: string example: 2020 - responses: + responses: '200': description: OK @@ -1711,19 +1598,19 @@ paths: required: true description: The type of KPI. schema: - type : string + type: string example: uptime - name: selectedDate in: query required: true description: Full date time string. schema: - type : string + type: string example: Wed Jul 01 2020 01:00:00 GMT 0100 (British Summer Time) - responses: + responses: '200': description: OK - + /api/v1/messages/{id}: delete: summary: Delete a Message @@ -1738,7 +1625,7 @@ paths: description: The ID of the Message schema: type: string - example: "5ee249426136805fbf094eef" + example: '5ee249426136805fbf094eef' responses: '204': description: Ok @@ -1755,7 +1642,7 @@ paths: description: The ID of the Message schema: type: string - example: "5ee249426136805fbf094eef" + example: '5ee249426136805fbf094eef' requestBody: content: application/json: @@ -1770,10 +1657,10 @@ paths: { "isRead": true } - responses: + responses: '204': - description: OK - + description: OK + /api/v1/messages/unread/count: get: summary: Returns the number of unread messages for the authenticated user @@ -1781,10 +1668,10 @@ paths: - cookieAuth: [] tags: - Messages - responses: + responses: '200': - description: OK - + description: OK + /api/v1/messages: post: summary: Returns a new Message object and creates an associated parent Topic if a Topic is not specified in request body @@ -1817,10 +1704,10 @@ paths: "messageDescription": "this is an example", "messageType": "message" } - responses: + responses: '201': description: OK - + /api/v1/topics: post: summary: Returns a new Topic object with ID (Does not create any associated messages) @@ -1891,8 +1778,8 @@ paths: - Topics responses: '200': - description: Ok - + description: Ok + /api/v1/topics/{id}: get: summary: Returns Topic object by ID @@ -1907,7 +1794,7 @@ paths: description: The ID of the topic schema: type: string - example: "5ee249426136805fbf094eef" + example: '5ee249426136805fbf094eef' responses: '200': description: Ok @@ -1924,11 +1811,11 @@ paths: description: The ID of the Topic schema: type: string - example: "5ee249426136805fbf094eef" + example: '5ee249426136805fbf094eef' responses: '204': description: Ok - + /api/v1/projects: post: summary: Returns a Project object with ID. @@ -1939,7 +1826,7 @@ paths: requestBody: content: application/json: - schema: # Request body contents + schema: # Request body contents type: object required: - name @@ -1979,17 +1866,17 @@ paths: topics: type: array items: - type: string - example: # Sample object + type: string + example: # Sample object type: 'project' name: 'Epilepsy data research' link: 'http://epilepsy.org' description: 'Epilespy data research description' - categories: { category: 'API', programmingLanguage: ['Javascript'], programmingLanguageVersion: '0.0.0'} + categories: { category: 'API', programmingLanguage: ['Javascript'], programmingLanguageVersion: '0.0.0' } licence: 'MIT licence' authors: [4495285946631793] - tags: {features: ['Arbitrage'], topics: ['Epilepsy']} - responses: + tags: { features: ['Arbitrage'], topics: ['Epilepsy'] } + responses: '200': description: OK get: @@ -1997,31 +1884,31 @@ paths: tags: - Projects parameters: - - in: query - name: limit - required: false - description: Limit the number of results - schema: - type : integer - example: 3 - - in: query - name: offset - required: false - description: Index to offset the search results - schema: - type : integer - example: 1 - - in: query - name: q - required: false - description: Filter using search query - schema: - type : string - example: epilepsy - responses: + - in: query + name: limit + required: false + description: Limit the number of results + schema: + type: integer + example: 3 + - in: query + name: offset + required: false + description: Index to offset the search results + schema: + type: integer + example: 1 + - in: query + name: q + required: false + description: Filter using search query + schema: + type: string + example: epilepsy + responses: '200': description: OK - + /api/v1/projects/{id}: get: summary: Returns Project object. @@ -2032,9 +1919,9 @@ paths: name: id required: true schema: - type : integer + type: integer example: 441788967946948 - responses: + responses: '200': description: OK patch: @@ -2053,16 +1940,16 @@ paths: requestBody: content: application/json: - schema: # Request body contents + schema: # Request body contents type: object required: - name properties: activeflag: type: string - example: # Sample object + example: # Sample object activeflag: 'active' - responses: + responses: '200': description: OK put: @@ -2072,18 +1959,18 @@ paths: tags: - Projects parameters: - - in: path - name: id - required: true - description: The ID of the project - schema: - type : integer - format: int64 - example: 26542005388306332 + - in: path + name: id + required: true + description: The ID of the project + schema: + type: integer + format: int64 + example: 26542005388306332 requestBody: content: application/json: - schema: # Request body contents + schema: # Request body contents type: object required: - name @@ -2125,23 +2012,23 @@ paths: topics: type: array items: - type: string + type: string toolids: type: array items: type: string - example: # Sample object + example: # Sample object id: 26542005388306332 type: 'project' name: 'Research Data TEST EPILEPSY' link: 'http://localhost:8080/epilepsy' description: 'Epilespy data research description' - categories: { category: 'API', programmingLanguage: ['Javascript'], programmingLanguageVersion: '1.0.0'} + categories: { category: 'API', programmingLanguage: ['Javascript'], programmingLanguageVersion: '1.0.0' } licence: 'MIT licence' authors: [4495285946631793] - tags: {features: ['Arbitrage'], topics: ['Epilepsy']} + tags: { features: ['Arbitrage'], topics: ['Epilepsy'] } toolids: [] - responses: + responses: '200': description: OK @@ -2155,7 +2042,7 @@ paths: requestBody: content: application/json: - schema: # Request body contents + schema: # Request body contents type: object required: - name @@ -2195,17 +2082,17 @@ paths: topics: type: array items: - type: string - example: # Sample object + type: string + example: # Sample object type: 'paper' name: 'Epilepsy data research' link: 'http://epilepsy.org' description: 'Epilespy data research description' - categories: { category: 'API', programmingLanguage: ['Javascript'], programmingLanguageVersion: '0.0.0'} + categories: { category: 'API', programmingLanguage: ['Javascript'], programmingLanguageVersion: '0.0.0' } licence: 'MIT licence' authors: [4495285946631793] - tags: {features: ['Arbitrage'], topics: ['Epilepsy']} - responses: + tags: { features: ['Arbitrage'], topics: ['Epilepsy'] } + responses: '200': description: OK get: @@ -2213,31 +2100,31 @@ paths: tags: - Papers parameters: - - in: query - name: limit - required: false - description: Limit the number of results - schema: - type : integer - example: 3 - - in: query - name: offset - required: false - description: Index to offset the search results - schema: - type : integer - example: 1 - - in: query - name: q - required: false - description: Filter using search query - schema: - type : string - example: epilepsy - responses: + - in: query + name: limit + required: false + description: Limit the number of results + schema: + type: integer + example: 3 + - in: query + name: offset + required: false + description: Index to offset the search results + schema: + type: integer + example: 1 + - in: query + name: q + required: false + description: Filter using search query + schema: + type: string + example: epilepsy + responses: '200': description: OK - + /api/v1/papers/{id}: get: summary: Returns Paper object. @@ -2249,11 +2136,11 @@ paths: required: true description: The ID of the user schema: - type : integer + type: integer format: int64 minimum: 1 example: 8370396016757367 - responses: + responses: '200': description: OK patch: @@ -2263,16 +2150,16 @@ paths: tags: - Papers parameters: - - in: path - name: id - required: true - schema: - type: integer - example: 7485531672584456 + - in: path + name: id + required: true + schema: + type: integer + example: 7485531672584456 requestBody: content: application/json: - schema: # Request body contents + schema: # Request body contents type: object required: - name @@ -2281,9 +2168,9 @@ paths: type: number activeflag: type: string - example: # Sample object + example: # Sample object activeflag: 'active' - responses: + responses: '200': description: OK put: @@ -2293,18 +2180,18 @@ paths: tags: - Papers parameters: - - in: path - name: id - required: true - description: The ID of the paper - schema: - type : integer - format: int64 - example: 7485531672584456 + - in: path + name: id + required: true + description: The ID of the paper + schema: + type: integer + format: int64 + example: 7485531672584456 requestBody: content: application/json: - schema: # Request body contents + schema: # Request body contents type: object required: - name @@ -2346,23 +2233,23 @@ paths: topics: type: array items: - type: string + type: string toolids: type: array items: type: string - example: # Sample object + example: # Sample object id: 7485531672584456 type: 'paper' name: 'Test Paper Title 2' link: 'http://localhost:8080/epilepsy' description: 'Test abstract 2' - categories: { category: 'API', programmingLanguage: ['Javascript'], programmingLanguageVersion: '1.0.0'} + categories: { category: 'API', programmingLanguage: ['Javascript'], programmingLanguageVersion: '1.0.0' } licence: 'MIT licence' authors: [4495285946631793] - tags: {features: ['Arbitrage'], topics: ['Epilepsy']} + tags: { features: ['Arbitrage'], topics: ['Epilepsy'] } toolids: [] - responses: + responses: '200': description: OK @@ -2372,28 +2259,28 @@ paths: tags: - Tools parameters: - - in: query - name: limit - required: false - description: Limit the number of results - schema: - type : integer - example: 3 - - in: query - name: offset - required: false - description: Index to offset the search results - schema: - type : integer - example: 1 - - in: query - name: q - required: false - description: Filter using search query - schema: - type : string - example: epilepsy - responses: + - in: query + name: limit + required: false + description: Limit the number of results + schema: + type: integer + example: 3 + - in: query + name: offset + required: false + description: Index to offset the search results + schema: + type: integer + example: 1 + - in: query + name: q + required: false + description: Filter using search query + schema: + type: string + example: epilepsy + responses: '200': description: OK post: @@ -2405,7 +2292,7 @@ paths: requestBody: content: application/json: - schema: # Request body contents + schema: # Request body contents type: object required: - name @@ -2445,13 +2332,13 @@ paths: topics: type: array items: - type: string - example: # Sample object + type: string + example: # Sample object id: 26542005388306332 - responses: + responses: '200': description: OK - + /api/v1/tools/{id}: get: summary: Returns Tool object. @@ -2463,11 +2350,11 @@ paths: required: true description: The ID of the tool schema: - type : integer + type: integer format: int64 minimum: 1 example: 19009 - responses: + responses: '200': description: OK put: @@ -2481,12 +2368,12 @@ paths: name: id required: true schema: - type : integer + type: integer example: 123 requestBody: content: application/json: - schema: # Request body contents + schema: # Request body contents type: object required: - name @@ -2528,23 +2415,23 @@ paths: topics: type: array items: - type: string + type: string toolids: type: array items: type: string - example: # Sample object + example: # Sample object id: 26542005388306332 type: 'tool' name: 'Research Data TEST EPILEPSY' link: 'http://localhost:8080/epilepsy' description: 'Epilespy data research description' - categories: { category: 'API', programmingLanguage: ['Javascript'], programmingLanguageVersion: '1.0.0'} + categories: { category: 'API', programmingLanguage: ['Javascript'], programmingLanguageVersion: '1.0.0' } licence: 'MIT licence' authors: [4495285946631793] - tags: {features: ['Arbitrage'], topics: ['Epilepsy']} + tags: { features: ['Arbitrage'], topics: ['Epilepsy'] } toolids: [] - responses: + responses: '200': description: OK patch: @@ -2559,13 +2446,13 @@ paths: required: true description: The ID of the tool schema: - type : integer + type: integer format: int64 example: 5032687830560181 requestBody: content: application/json: - schema: # Request body contents + schema: # Request body contents type: object required: - name @@ -2574,13 +2461,13 @@ paths: type: number activeflag: type: string - example: # Sample object + example: # Sample object id: 662346984100503 activeflag: 'active' - responses: + responses: '200': description: OK - + components: securitySchemes: oauth2: @@ -2591,4 +2478,3 @@ components: cookieAuth: type: http scheme: jwt - \ No newline at end of file From 9a5f1e221822763cea85a61bd177e63dc0b61f26 Mon Sep 17 00:00:00 2001 From: Paul McCafferty <> Date: Thu, 11 Feb 2021 15:58:07 +0000 Subject: [PATCH 12/42] Fix for courses - switching the match to the beginning of the pipeline. Commenting out code for requiring secure cookies when calling the api application. --- src/config/server.js | 187 +++++++++++----------- src/resources/search/search.repository.js | 20 +-- 2 files changed, 103 insertions(+), 104 deletions(-) diff --git a/src/config/server.js b/src/config/server.js index 5f941083..e8431cd3 100644 --- a/src/config/server.js +++ b/src/config/server.js @@ -18,10 +18,10 @@ import helper from '../resources/utilities/helper.util'; require('dotenv').config(); if (helper.getEnvironment() !== 'local') { - Sentry.init({ - dsn: 'https://c7c564a153884dc0a6b676943b172121@o444579.ingest.sentry.io/5419637', - environment: helper.getEnvironment(), - }); + Sentry.init({ + dsn: 'https://c7c564a153884dc0a6b676943b172121@o444579.ingest.sentry.io/5419637', + environment: helper.getEnvironment(), + }); } const Account = require('./account'); @@ -42,14 +42,14 @@ var rx = /^([http|https]+:\/\/[a-z]+)\.([^/]*)/; var arr = rx.exec(process.env.homeURL); if (Array.isArray(arr) && arr.length > 0) { - domains.push('https://' + arr[2]); + domains.push('https://' + arr[2]); } app.use( - cors({ - origin: domains, - credentials: true, - }) + cors({ + origin: domains, + credentials: true, + }) ); // apply rate limiter of 100 requests per minute @@ -71,101 +71,101 @@ app.use(passport.initialize()); app.use(passport.session()); app.use( - session({ - secret: process.env.JWTSecret, - resave: false, - saveUninitialized: true, - name: 'sessionId', - cookie: { + session({ + secret: process.env.JWTSecret, + resave: false, + saveUninitialized: true, + name: 'sessionId', + /* cookie: { secure: process.env.api_url ? true : false, httpOnly: true - } - }) + } */ + }) ); function setNoCache(req, res, next) { - res.set('Pragma', 'no-cache'); - res.set('Cache-Control', 'no-cache, no-store'); - next(); + res.set('Pragma', 'no-cache'); + res.set('Cache-Control', 'no-cache, no-store'); + next(); } app.get('/api/v1/openid/endsession', setNoCache, (req, res, next) => { - passport.authenticate('jwt', async function (err, user, info) { - if (err || !user) { - return res.status(200).redirect(process.env.homeURL + '/search?search='); - } - oidc.Session.destory; - req.logout(); - res.clearCookie('jwt'); - - return res.status(200).redirect(process.env.homeURL + '/search?search='); - })(req, res, next); + passport.authenticate('jwt', async function (err, user, info) { + if (err || !user) { + return res.status(200).redirect(process.env.homeURL + '/search?search='); + } + oidc.Session.destory; + req.logout(); + res.clearCookie('jwt'); + + return res.status(200).redirect(process.env.homeURL + '/search?search='); + })(req, res, next); }); app.get('/api/v1/openid/interaction/:uid', setNoCache, (req, res, next) => { - passport.authenticate('jwt', async function (err, user, info) { - if (err || !user) { - //login in user - go to login screen - var apiURL = process.env.api_url || 'http://localhost:3001'; - return res.status(200).redirect(process.env.homeURL + '/search?search=&showLogin=true&loginReferrer=' + apiURL + req.url); - } else { - try { - const { uid, prompt, params, session } = await oidc.interactionDetails(req, res); - - const client = await oidc.Client.find(params.client_id); - - switch (prompt.name) { - case 'select_account': { - } - case 'login': { - const result = { - select_account: {}, // make sure its skipped by the interaction policy since we just logged in - login: { - account: user.id.toString(), - }, - }; - - return await oidc.interactionFinished(req, res, result, { mergeWithLastSubmission: false }); - } - case 'consent': { - if (!session) { - return oidc.interactionFinished(req, res, { select_account: {} }, { mergeWithLastSubmission: false }); - } - - const account = await oidc.Account.findAccount(undefined, session.accountId); - const { email } = await account.claims('prompt', 'email', { email: null }, []); - - const { - prompt: { name, details }, - } = await oidc.interactionDetails(req, res); - //assert.equal(name, 'consent'); - - const consent = {}; - - // any scopes you do not wish to grant go in here - // otherwise details.scopes.new.concat(details.scopes.accepted) will be granted - consent.rejectedScopes = []; - - // any claims you do not wish to grant go in here - // otherwise all claims mapped to granted scopes - // and details.claims.new.concat(details.claims.accepted) will be granted - consent.rejectedClaims = []; - - // replace = false means previously rejected scopes and claims remain rejected - // changing this to true will remove those rejections in favour of just what you rejected above - consent.replace = false; - - const result = { consent }; - return await oidc.interactionFinished(req, res, result, { mergeWithLastSubmission: true }); - } - default: - return undefined; - } - } catch (err) { - return next(err); - } - } - })(req, res, next); + passport.authenticate('jwt', async function (err, user, info) { + if (err || !user) { + //login in user - go to login screen + var apiURL = process.env.api_url || 'http://localhost:3001'; + return res.status(200).redirect(process.env.homeURL + '/search?search=&showLogin=true&loginReferrer=' + apiURL + req.url); + } else { + try { + const { uid, prompt, params, session } = await oidc.interactionDetails(req, res); + + const client = await oidc.Client.find(params.client_id); + + switch (prompt.name) { + case 'select_account': { + } + case 'login': { + const result = { + select_account: {}, // make sure its skipped by the interaction policy since we just logged in + login: { + account: user.id.toString(), + }, + }; + + return await oidc.interactionFinished(req, res, result, { mergeWithLastSubmission: false }); + } + case 'consent': { + if (!session) { + return oidc.interactionFinished(req, res, { select_account: {} }, { mergeWithLastSubmission: false }); + } + + const account = await oidc.Account.findAccount(undefined, session.accountId); + const { email } = await account.claims('prompt', 'email', { email: null }, []); + + const { + prompt: { name, details }, + } = await oidc.interactionDetails(req, res); + //assert.equal(name, 'consent'); + + const consent = {}; + + // any scopes you do not wish to grant go in here + // otherwise details.scopes.new.concat(details.scopes.accepted) will be granted + consent.rejectedScopes = []; + + // any claims you do not wish to grant go in here + // otherwise all claims mapped to granted scopes + // and details.claims.new.concat(details.claims.accepted) will be granted + consent.rejectedClaims = []; + + // replace = false means previously rejected scopes and claims remain rejected + // changing this to true will remove those rejections in favour of just what you rejected above + consent.replace = false; + + const result = { consent }; + return await oidc.interactionFinished(req, res, result, { mergeWithLastSubmission: true }); + } + default: + return undefined; + } + } catch (err) { + return next(err); + } + } + })(req, res, next); }); app.use('/api/v1/openid', oidc.callback); @@ -177,7 +177,6 @@ app.use('/api/v1/auth/sso/discourse', require('../resources/auth/sso/sso.discour app.use('/api/v1/auth', require('../resources/auth/auth.route')); app.use('/api/v1/auth/register', require('../resources/user/user.register.route')); - app.use('/api/v1/users', require('../resources/user/user.route')); app.use('/api/v1/topics', require('../resources/topic/topic.route')); app.use('/api/v1/publishers', require('../resources/publisher/publisher.route')); diff --git a/src/resources/search/search.repository.js b/src/resources/search/search.repository.js index 2473217c..cdfbe1c8 100644 --- a/src/resources/search/search.repository.js +++ b/src/resources/search/search.repository.js @@ -25,8 +25,8 @@ export function getObjectResult(type, searchAll, searchQuery, startIndex, maxRes var queryObject; if (type === 'course') { queryObject = [ - { $unwind: '$courseOptions' }, { $match: newSearchQuery }, + { $unwind: '$courseOptions' }, { $project: { _id: 0, @@ -147,8 +147,8 @@ export function getObjectCount(type, searchAll, searchQuery) { if (type === 'course') { if (searchAll) { q = collection.aggregate([ - { $unwind: '$courseOptions' }, { $match: newSearchQuery }, + { $unwind: '$courseOptions' }, { $group: { _id: {}, @@ -167,8 +167,8 @@ export function getObjectCount(type, searchAll, searchQuery) { } else { q = collection .aggregate([ - { $unwind: '$courseOptions' }, { $match: newSearchQuery }, + { $unwind: '$courseOptions' }, { $group: { _id: {}, @@ -532,13 +532,6 @@ export const getFilter = async (searchString, type, field, isArray, activeFilter export function filterQueryGenerator(filter, searchString, type, isArray, activeFiltersQuery) { var queryArray = []; - if (type === 'course') { - queryArray.push({ $unwind: '$courseOptions' }); - queryArray.push({ - $match: { $or: [{ 'courseOptions.startDate': { $gte: new Date(Date.now()) } }, { 'courseOptions.flexibleDates': true }] }, - }); - } - if (!_.isEmpty(activeFiltersQuery)) { queryArray.push({ $match: activeFiltersQuery }); } else { @@ -547,6 +540,13 @@ export function filterQueryGenerator(filter, searchString, type, isArray, active else queryArray.push({ $match: { $and: [{ type: type }, { activeflag: 'active' }] } }); } + if (type === 'course') { + queryArray.push({ + $match: { $or: [{ 'courseOptions.startDate': { $gte: new Date(Date.now()) } }, { 'courseOptions.flexibleDates': true }] }, + }); + queryArray.push({ $unwind: '$courseOptions' }); + } + queryArray.push({ $project: { result: '$' + filter, From b28afa42dac4258d26864b9e0f0cfa0202c09767 Mon Sep 17 00:00:00 2001 From: Robin Kavanagh Date: Mon, 15 Feb 2021 15:55:46 +0000 Subject: [PATCH 13/42] Completed API V2 entities GET operations --- src/config/server.js | 16 +- src/resources/base/__tests__/entity.test.js | 74 ++ .../base/__tests__/repository.test.js | 31 + src/resources/base/controller.js | 19 + src/resources/base/entity.js | 10 +- src/resources/base/repository.js | 134 ++- src/resources/course/__mocks__/courses.js | 198 ++++ .../__tests__/course.controller.test.js | 117 +++ .../course/__tests__/course.entity.test.js | 95 ++ .../__tests__/course.repository.it.test.js | 42 + .../__tests__/course.repository.test.js | 63 ++ .../course/__tests__/course.service.test.js | 53 + src/resources/course/course.model.js | 9 +- src/resources/course/{ => v1}/course.route.js | 10 +- src/resources/course/v2/course.controller.js | 62 ++ src/resources/course/v2/course.entity.js | 50 + src/resources/course/v2/course.repository.js | 20 + src/resources/course/v2/course.route.js | 18 + src/resources/course/v2/course.service.js | 13 + src/resources/course/v2/dependency.js | 5 + .../datarequest/datarequest.controller.js | 2 +- src/resources/dataset/__mocks__/dataset.js | 99 -- src/resources/dataset/__mocks__/datasets.js | 2 +- .../__tests__/dataset.controller.test.js | 130 ++- .../dataset/__tests__/dataset.entity.test.js | 7 +- .../__tests__/dataset.repository.it.test.js | 21 +- .../__tests__/dataset.repository.test.js | 29 +- .../dataset/__tests__/dataset.service.test.js | 75 +- src/resources/dataset/dataset.controller.js | 26 +- src/resources/dataset/dataset.entity.js | 4 +- src/resources/dataset/dataset.model.js | 16 +- src/resources/dataset/dataset.repository.js | 21 +- src/resources/dataset/dataset.service.js | 980 +----------------- src/resources/dataset/v1/dataset.route.js | 2 +- src/resources/dataset/v1/dataset.service.js | 959 +++++++++++++++++ src/resources/dataset/v2/dataset.route.js | 4 +- src/resources/paper/__mocks__/papers.js | 269 +++++ .../paper/__tests__/paper.controller.test.js | 117 +++ .../paper/__tests__/paper.entity.test.js | 54 + .../__tests__/paper.repository.it.test.js | 42 + .../paper/__tests__/paper.repository.test.js | 59 ++ .../paper/__tests__/paper.service.test.js | 49 + src/resources/paper/dependency.js | 5 + src/resources/paper/paper.controller.js | 62 ++ src/resources/paper/paper.entity.js | 44 + src/resources/paper/paper.model.js | 63 ++ src/resources/paper/paper.repository.js | 20 + src/resources/paper/paper.service.js | 13 + src/resources/paper/{ => v1}/paper.route.js | 10 +- src/resources/paper/v2/paper.route.js | 18 + src/resources/project/__mocks__/projects.js | 207 ++++ .../__tests__/project.controller.test.js | 117 +++ .../project/__tests__/project.entity.test.js | 50 + .../__tests__/project.repository.it.test.js | 42 + .../__tests__/project.repository.test.js | 56 + .../project/__tests__/project.service.test.js | 46 + src/resources/project/dependency.js | 5 + src/resources/project/project.controller.js | 62 ++ src/resources/project/project.entity.js | 38 + src/resources/project/project.model.js | 60 ++ src/resources/project/project.repository.js | 20 + src/resources/project/project.service.js | 13 + .../project/{ => v1}/project.route.js | 13 +- src/resources/project/v2/project.route.js | 18 + src/resources/tool/__mocks__/tools.js | 147 +++ .../tool/__tests__/tool.controller.test.js | 117 +++ .../tool/__tests__/tool.entity.test.js | 61 ++ .../tool/__tests__/tool.repository.it.test.js | 42 + .../tool/__tests__/tool.repository.test.js | 57 + .../tool/__tests__/tool.service.test.js | 47 + src/resources/tool/{ => v1}/tool.route.js | 20 +- src/resources/tool/v2/dependency.js | 5 + src/resources/tool/v2/tool.controller.js | 62 ++ src/resources/tool/v2/tool.entity.js | 40 + src/resources/tool/v2/tool.model.js | 68 ++ src/resources/tool/v2/tool.repository.js | 20 + src/resources/tool/v2/tool.route.js | 18 + src/resources/tool/v2/tool.service.js | 13 + src/resources/utilities/helper.util.js | 2 +- 79 files changed, 4451 insertions(+), 1256 deletions(-) create mode 100644 src/resources/base/__tests__/entity.test.js create mode 100644 src/resources/base/__tests__/repository.test.js create mode 100644 src/resources/base/controller.js create mode 100644 src/resources/course/__mocks__/courses.js create mode 100644 src/resources/course/__tests__/course.controller.test.js create mode 100644 src/resources/course/__tests__/course.entity.test.js create mode 100644 src/resources/course/__tests__/course.repository.it.test.js create mode 100644 src/resources/course/__tests__/course.repository.test.js create mode 100644 src/resources/course/__tests__/course.service.test.js rename src/resources/course/{ => v1}/course.route.js (95%) create mode 100644 src/resources/course/v2/course.controller.js create mode 100644 src/resources/course/v2/course.entity.js create mode 100644 src/resources/course/v2/course.repository.js create mode 100644 src/resources/course/v2/course.route.js create mode 100644 src/resources/course/v2/course.service.js create mode 100644 src/resources/course/v2/dependency.js delete mode 100644 src/resources/dataset/__mocks__/dataset.js create mode 100644 src/resources/dataset/v1/dataset.service.js create mode 100644 src/resources/paper/__mocks__/papers.js create mode 100644 src/resources/paper/__tests__/paper.controller.test.js create mode 100644 src/resources/paper/__tests__/paper.entity.test.js create mode 100644 src/resources/paper/__tests__/paper.repository.it.test.js create mode 100644 src/resources/paper/__tests__/paper.repository.test.js create mode 100644 src/resources/paper/__tests__/paper.service.test.js create mode 100644 src/resources/paper/dependency.js create mode 100644 src/resources/paper/paper.controller.js create mode 100644 src/resources/paper/paper.entity.js create mode 100644 src/resources/paper/paper.model.js create mode 100644 src/resources/paper/paper.repository.js create mode 100644 src/resources/paper/paper.service.js rename src/resources/paper/{ => v1}/paper.route.js (96%) create mode 100644 src/resources/paper/v2/paper.route.js create mode 100644 src/resources/project/__mocks__/projects.js create mode 100644 src/resources/project/__tests__/project.controller.test.js create mode 100644 src/resources/project/__tests__/project.entity.test.js create mode 100644 src/resources/project/__tests__/project.repository.it.test.js create mode 100644 src/resources/project/__tests__/project.repository.test.js create mode 100644 src/resources/project/__tests__/project.service.test.js create mode 100644 src/resources/project/dependency.js create mode 100644 src/resources/project/project.controller.js create mode 100644 src/resources/project/project.entity.js create mode 100644 src/resources/project/project.model.js create mode 100644 src/resources/project/project.repository.js create mode 100644 src/resources/project/project.service.js rename src/resources/project/{ => v1}/project.route.js (94%) create mode 100644 src/resources/project/v2/project.route.js create mode 100644 src/resources/tool/__mocks__/tools.js create mode 100644 src/resources/tool/__tests__/tool.controller.test.js create mode 100644 src/resources/tool/__tests__/tool.entity.test.js create mode 100644 src/resources/tool/__tests__/tool.repository.it.test.js create mode 100644 src/resources/tool/__tests__/tool.repository.test.js create mode 100644 src/resources/tool/__tests__/tool.service.test.js rename src/resources/tool/{ => v1}/tool.route.js (96%) create mode 100644 src/resources/tool/v2/dependency.js create mode 100644 src/resources/tool/v2/tool.controller.js create mode 100644 src/resources/tool/v2/tool.entity.js create mode 100644 src/resources/tool/v2/tool.model.js create mode 100644 src/resources/tool/v2/tool.repository.js create mode 100644 src/resources/tool/v2/tool.route.js create mode 100644 src/resources/tool/v2/tool.service.js diff --git a/src/config/server.js b/src/config/server.js index ae69102d..8a57943e 100644 --- a/src/config/server.js +++ b/src/config/server.js @@ -186,7 +186,7 @@ app.use('/api/v1/workflows', require('../resources/workflow/workflow.route')); app.use('/api/v1/messages', require('../resources/message/message.route')); app.use('/api/v1/reviews', require('../resources/tool/review.route')); app.use('/api/v1/relatedobject/', require('../resources/relatedobjects/relatedobjects.route')); -app.use('/api/v1/tools', require('../resources/tool/tool.route')); + app.use('/api/v1/accounts', require('../resources/account/account.route')); app.use('/api/v1/search/filter', require('../resources/search/filter.route')); app.use('/api/v1/search', require('../resources/search/search.router')); // tools projects people @@ -196,12 +196,20 @@ app.use('/api/v1/linkchecker', require('../resources/linkchecker/linkchecker.rou app.use('/api/v1/stats', require('../resources/stats/stats.router')); app.use('/api/v1/kpis', require('../resources/stats/kpis.router')); -app.use('/api/v1/course', require('../resources/course/course.route')); +app.use('/api/v1/course', require('../resources/course/v1/course.route')); +app.use('/api/v2/courses', require('../resources/course/v2/course.route')); app.use('/api/v1/person', require('../resources/person/person.route')); -app.use('/api/v1/projects', require('../resources/project/project.route')); -app.use('/api/v1/papers', require('../resources/paper/paper.route')); +app.use('/api/v1/tools', require('../resources/tool/v1/tool.route')); +app.use('/api/v2/tools', require('../resources/tool/v2/tool.route')); + +app.use('/api/v1/projects', require('../resources/project/v1/project.route')); +app.use('/api/v2/projects', require('../resources/project/v2/project.route')); + +app.use('/api/v1/papers', require('../resources/paper/v1/paper.route')); +app.use('/api/v2/papers', require('../resources/paper/v2/paper.route')); + app.use('/api/v1/counter', require('../resources/tool/counter.route')); app.use('/api/v1/coursecounter', require('../resources/course/coursecounter.route')); diff --git a/src/resources/base/__tests__/entity.test.js b/src/resources/base/__tests__/entity.test.js new file mode 100644 index 00000000..eb27a803 --- /dev/null +++ b/src/resources/base/__tests__/entity.test.js @@ -0,0 +1,74 @@ +import DatasetClass from '../../dataset/dataset.entity'; + +describe('Entity', function () { + describe('equals', function () { + it('should return true if equality exists where both objects are of the same instance', async function () { + const dataset = new DatasetClass(675584862177848, 'Admitted Patient Care Dataset'); + + const result = dataset.equals(dataset); + + expect(result).toBe(true); + }); + it('should return false if both objects are of the same type with differing properties', async function () { + const dataset = new DatasetClass(675584862177848, 'Admitted Patient Care Dataset'); + const referenceDataset = new DatasetClass(null, 'Admitted Patient Care Dataset'); + + const result = dataset.equals(referenceDataset); + + expect(result).toBe(false); + }); + it('should return false if equality does not exist', async function () { + const dataset = new DatasetClass(675584862177848, 'Admitted Patient Care Dataset'); + const referenceDataset = new DatasetClass(246523922611217, 'Reference Dataset'); + + const result = dataset.equals(referenceDataset); + + expect(result).toBe(false); + }); + it('should return immediate false result if reference object is not an entity', async function () { + const dataset = new DatasetClass(675584862177848, 'Admitted Patient Care Dataset'); + const referenceObject = { id: 675584862177848 }; + + const result = dataset.equals(referenceObject); + + expect(result).toBe(false); + }); + }); + + describe('referenceEquals', function () { + it('should return false if entity does not have an identifier assigned', async function () { + const dataset = new DatasetClass(null, 'Admitted Patient Care Dataset'); + const referenceId = 675584862177848; + + const result = dataset.referenceEquals(referenceId); + + expect(result).toBe(false); + }); + it('should return true if reference equality exists', async function () { + const dataset = new DatasetClass(675584862177848, 'Admitted Patient Care Dataset'); + const referenceId = 675584862177848; + + const result = dataset.referenceEquals(referenceId); + + expect(result).toBe(true); + }); + it('should return false if reference equality does not exist', async function () { + const dataset = new DatasetClass(675584862177848, 'Admitted Patient Care Dataset'); + const referenceId = 246523922611217; + + const result = dataset.referenceEquals(referenceId); + + expect(result).toBe(false); + }); + }); + + describe('toString', function () { + it('should return a string representation of an object identifier', async function () { + const dataset = new DatasetClass(675584862177848, 'Admitted Patient Care Dataset'); + + const result = dataset.toString(); + + expect(result).toEqual(dataset.id); + }); + }); +}); diff --git a/src/resources/base/__tests__/repository.test.js b/src/resources/base/__tests__/repository.test.js new file mode 100644 index 00000000..6167984f --- /dev/null +++ b/src/resources/base/__tests__/repository.test.js @@ -0,0 +1,31 @@ +import { processQueryParamOperators } from '../repository'; + +describe('Repository', function () { + describe('processQueryParamOperators', function () { + it('should find any user specified greater than or equals to operator and convert it to the expected query language', async function () { + const queryStr = '{"$text":{"$search":"West"},"datasetfields.metadataquality.completeness_percent":{"gte":"90.01"}}'; + const result = processQueryParamOperators(queryStr); + expect(result).toEqual('{"$text":{"$search":"West"},"datasetfields.metadataquality.completeness_percent":{"$gte":90.01}}'); + }); + it('should find any user specified greater than operator and convert it to the expected query language', async function () { + const queryStr = '{"$text":{"$search":"West"},"datasetfields.metadataquality.completeness_percent":{"gt":"90.01"}}'; + const result = processQueryParamOperators(queryStr); + expect(result).toEqual('{"$text":{"$search":"West"},"datasetfields.metadataquality.completeness_percent":{"$gt":90.01}}'); + }); + it('should find any user specified less than or equal to operator and convert it to the expected query language', async function () { + const queryStr = '{"$text":{"$search":"West"},"datasetfields.metadataquality.completeness_percent":{"lte":"90.01"}}'; + const result = processQueryParamOperators(queryStr); + expect(result).toEqual('{"$text":{"$search":"West"},"datasetfields.metadataquality.completeness_percent":{"$lte":90.01}}'); + }); + it('should find any user specified less than operator and convert it to the expected query language', async function () { + const queryStr = '{"$text":{"$search":"West"},"datasetfields.metadataquality.completeness_percent":{"lt":"90.01"}}'; + const result = processQueryParamOperators(queryStr); + expect(result).toEqual('{"$text":{"$search":"West"},"datasetfields.metadataquality.completeness_percent":{"$lt":90.01}}'); + }); + it('should find any user specified equal to operator and convert it to the expected query language', async function () { + const queryStr = '{"$text":{"$search":"West"},"datasetfields.metadataquality.completeness_percent":{"eq":"90.01"}}'; + const result = processQueryParamOperators(queryStr); + expect(result).toEqual('{"$text":{"$search":"West"},"datasetfields.metadataquality.completeness_percent":{"$eq":90.01}}'); + }); + }); +}); diff --git a/src/resources/base/controller.js b/src/resources/base/controller.js new file mode 100644 index 00000000..eb635656 --- /dev/null +++ b/src/resources/base/controller.js @@ -0,0 +1,19 @@ +export default class Controller { + constructor(service) { + this.service = service; + } + + // processQueryParametersForService(query) { + // const { allowedFilters, allowedOptions } = this.service; + // const allowedQueryParameters = [...allowedFilters, ...allowedOptions]; + // let opts = {}; + + // Object.keys(query).forEach(key => { + // if (query[key] && allowedQueryParameters.indexOf(key)) { + // opts[key] = query[key]; + // } + // }); + + // return opts; + // } +} diff --git a/src/resources/base/entity.js b/src/resources/base/entity.js index 17c0fcb7..8587ce68 100644 --- a/src/resources/base/entity.js +++ b/src/resources/base/entity.js @@ -1,13 +1,9 @@ class Entity { - constructor () { - this.id = ''; - } equals (other) { if (other instanceof Entity === false) { return false; } - return other.id ? this.referenceEquals(other.id) : this === other; } @@ -15,10 +11,8 @@ class Entity { if (!this.id) { return this.equals(id); } - - const reference = typeof id !== 'string' ? id.toString() : id; - - return this.id === reference; + const reference = id.toString(); + return this.id.toString() === reference; } toString () { diff --git a/src/resources/base/repository.js b/src/resources/base/repository.js index 305086a0..c529c150 100644 --- a/src/resources/base/repository.js +++ b/src/resources/base/repository.js @@ -1,26 +1,52 @@ -class Repository { +export default class Repository { constructor(Model) { this.collection = Model; - } - - // @desc Allows us to query a collection via the model inheriting this class with various options - async find(query = {}, { multiple = true, count, page = 1, pageSize = 20, select, populate, sort = {}, lean } = {}) { - const results = multiple ? - this.collection.find(query).sort(sort).limit(pageSize) : - this.collection.findOne(query); - - if(populate) { - results.populate(populate); - } - - if(select) { - results.select(select); - } - - if(multiple && page > 1) { - results.skip(parseInt(page - 1) * parseInt(pageSize)); - } + } + + // @desc Allows us to query a collection via the model inheriting this class with various options + async find(query = {}, { multiple = true, count, lean, populate } = {}) { + //Build query + let queryObj = { ...query }; + + // Filtering + const excludedFields = ['page', 'sort', 'limit', 'fields', 'count', 'search', 'expanded']; + excludedFields.forEach(el => delete queryObj[el]); + + // Keyword search + queryObj = query.search ? { $text: { $search: query['search'] }, ...queryObj } : queryObj; + + let queryStr = JSON.stringify(queryObj); + + // Advanced filtering + queryStr = processQueryParamOperators(queryStr); + + let results = multiple + ? query.search + ? this.collection.find(JSON.parse(queryStr), { searchRelevance: { $meta: 'textScore' } }) + : this.collection.find(JSON.parse(queryStr)) + : this.collection.findOne(JSON.parse(queryStr)); + + // Sorting + const sortBy = query.sort ? query.sort.split(',').join(' ') : query.search ? { searchRelevance: { $meta: 'textScore' } } : null; + results = sortBy ? results.sort(sortBy) : results; + + // Field limiting + const fields = query.fields ? query.fields.split(',').join(' ') : null; + results = fields ? results.select(fields) : results.select('-__v'); + + // Pagination + const page = query.page * 1 || 1; + const limit = query.limit * 1 || null; + const skip = (page - 1) * limit; + results = results.skip(skip).limit(limit); + // Population + results = populate ? results.populate(populate) : results; + + // Count user option + count = query.count || count; + + // Execute query if (count) { return results.countDocuments().exec(); } else if (lean) { @@ -30,56 +56,66 @@ class Repository { } } - // @desc Allows us to count to total number of documents within this collection via the model inheriting this class + // @desc Allows us to count to total number of documents within this collection via the model inheriting this class async count() { return this.collection.estimatedDocumentCount(); } - // @desc Allows us to create a new Mongoose document within the collection via the model inheriting this class + // @desc Allows us to create a new Mongoose document within the collection via the model inheriting this class async create(body) { const document = new this.collection(body); return document.save(); } - // @desc Allows us to update an existing Mongoose document within the collection via the model inheriting this class + // @desc Allows us to update an existing Mongoose document within the collection via the model inheriting this class async update(document, body = {}) { const id = typeof document._id !== 'undefined' ? document._id : document; return this.collection.findByIdAndUpdate(id, body, { new: true }); } - // @desc Allows us to delete an existing Mongoose document within the collection via the model inheriting this class + // @desc Allows us to delete an existing Mongoose document within the collection via the model inheriting this class async remove(document) { const reloadedDocument = await this.reload(document); return reloadedDocument.remove(); - } - - // @desc Allows us to convert identifiers to Mongoose documents, plain entities to Mongoose documents, - // or to simply reload Mongoose documents with different query parameters (selected fields, populated fields, - // or a lean version) - async reload(document, { select, populate, lean } = {}) { - if(!select && !populate && !lean && document instanceof this.collection) { - return document; - } - - return (typeof document._id !== 'undefined') - ? this.findById(document._id, { select, populate, lean }) - : this.findById(document, { select, populate, lean }); - } - - // @desc A helper function to find all documents with a given query - async findAll({ count, select, populate, lean, sort } = {}) { - return this.find({}, { multiple: true, count, select, populate, lean, sort }); } - // @desc A helper function to find a single document by unique identifier - async findById(id, { select, populate, lean } = {}) { - return this.find({ _id: id }, { multiple: false, count: false, select, populate, lean }); + // @desc Allows us to convert identifiers to Mongoose documents, plain entities to Mongoose documents, + // or to simply reload Mongoose documents with different query parameters (selected fields, populated fields, + // or a lean version) + async reload(document, { select, populate, lean } = {}) { + if (!select && !populate && !lean && document instanceof this.collection) { + return document; + } + + return typeof document._id !== 'undefined' + ? this.findById(document._id, { select, populate, lean }) + : this.findById(document, { select, populate, lean }); + } + + // @desc A helper function to find all documents with given options + async findAll({ count, lean, populate } = {}) { + return this.find({}, { multiple: true, count, lean, populate }); + } + + // @desc A helper function to find a single document by unique identifier + async findById(id, { lean, populate } = {}) { + return this.find({ _id: id }, { multiple: false, count: false, lean, populate }); + } + + // @desc A helper function to find the first document returned by a given query + async findOne(query = {}, { lean, populate } = {}) { + return this.find(query, { multiple: false, count: false, lean, populate }); } - // @desc A helper function to find the first document returned by a given query - async findOne(query = {}, { select, populate, lean } = {}) { - return this.find(query, { multiple: false, count: false, select, populate, lean }); + // @desc A helper function to count the number of documents returned by a given query + async findCountOf(query = {}) { + return this.find(query, { multiple: true, count: true }); } } -module.exports = Repository; +export const processQueryParamOperators = (queryStr) => { + return queryStr.replace(/\"\b(gte|gt|lte|lt|eq)\b\":\"\b(\-?\d*\.?\d+)\b\"/g, (match, operator, value) => { + const parsedValue = parseFloat(value); + return `"$${operator}":${parsedValue}`; + }); +} \ No newline at end of file diff --git a/src/resources/course/__mocks__/courses.js b/src/resources/course/__mocks__/courses.js new file mode 100644 index 00000000..6fcb4e57 --- /dev/null +++ b/src/resources/course/__mocks__/courses.js @@ -0,0 +1,198 @@ +export const coursesStub = [ + { + _id: '602427a273d8765aeb89f87b', + id: 50033, + activeflag: 'active', + award: [], + competencyFramework: '', + counter: 5, + courseDelivery: 'campus', + courseOptions: [ + { + flexibleDates: true, + startDate: null, + studyMode: '', + studyDurationNumber: null, + studyDurationMeasure: '', + fees: [ + { + feeDescription: '', + feeAmount: null, + feePer: '', + }, + ], + }, + ], + createdAt: '2021-02-10T18:36:18.463Z', + creator: 50000, + description: + 'Health data analytics involves extracting insights from health data, either to shape national policy, manage local organisations or inform the care of an individual. As more and more data becomes available electronically, the demand for skilled and trained individuals to take advantage of it becomes increasingly urgent.', + domains: ['Course'], + entries: [ + { + level: '', + subject: '', + }, + ], + keywords: [], + link: 'https://www.ucl.ac.uk/health-informatics/study/postgraduate-taught-programmes/health-data-analytics-mscpg-dippg-cert', + location: 'London', + nationalPriority: '', + provider: 'UCL Institute of Health Informatics', + relatedObjects: [], + restrictions: 'other:Relevant first degree or equivalent experience required', + title: 'Health Data Analytics', + type: 'course', + updatedAt: '2021-02-12T17:33:14.501Z', + persons: [], + objects: [], + reviews: [], + }, + { + _id: '602427a273d8765aeb89f7cb', + id: 50017, + activeflag: 'active', + award: [], + competencyFramework: '', + counter: 4, + courseDelivery: 'campus', + courseOptions: [ + { + flexibleDates: true, + startDate: null, + studyMode: '', + studyDurationNumber: null, + studyDurationMeasure: '', + fees: [ + { + feeDescription: '', + feeAmount: null, + feePer: '', + }, + ], + }, + ], + createdAt: '2021-02-10T18:36:18.463Z', + creator: 50000, + description: + 'The aim of this course is to familiarize the participants with the primary analysis of RNA-seq data.\n\nThis course starts with a brief introduction to RNA-seq and discusses quality control issues. Next, we will present the alignment step, quantification of expression and differential expression analysis. For downstream analysis we will focus on tools available through the Bioconductor project for manipulating and analysing bulk RNA-seq.', + domains: ['Course'], + entries: [ + { + level: '', + subject: '', + }, + ], + keywords: [], + link: 'https://training.csx.cam.ac.uk/bioinformatics/course/bioinfo-RNAseq3', + location: 'Cambridge', + nationalPriority: '', + provider: 'University of Cambridge', + relatedObjects: [], + restrictions: 'Course prerequisites apply', + title: 'Introduction to RNA-seq data analysis', + type: 'course', + updatedAt: '2021-02-12T15:20:08.869Z', + persons: [], + objects: [], + reviews: [], + }, + { + _id: '602427a373d8765aeb89fa5f', + id: 50077, + activeflag: 'active', + award: [], + competencyFramework: '', + counter: 2, + courseDelivery: 'campus', + courseOptions: [ + { + flexibleDates: true, + startDate: null, + studyMode: '', + studyDurationNumber: null, + studyDurationMeasure: '', + fees: [ + { + feeDescription: '', + feeAmount: null, + feePer: '', + }, + ], + }, + ], + createdAt: '2021-02-10T18:36:18.463Z', + creator: 50000, + description: + 'The course delivers core components of identifying, combining and analysing routinely collected health data. It covers aspects of information governance and disclosure control, as well as focus on data management, manipulation and advanced methods of data analysis. The course module is aimed at health, social and clinical researchers, who wish to learn techniques and skills to analyse linked health data. It aims to equip participants with the necessary analytical skills to analyse these types of data and to be aware of issues around clinical and information governance.', + domains: ['Course'], + entries: [ + { + level: '', + subject: '', + }, + ], + keywords: [], + link: 'https://www.gla.ac.uk/researchinstitutes/healthwellbeing/research/hehta/continuingprofessionaldevelopment/datascience/', + location: 'Scotland', + nationalPriority: '', + provider: 'University of Glasgow', + relatedObjects: [], + restrictions: 'Open', + title: 'Data Science - Identifying, combining and analysing health data sets ', + type: 'course', + updatedAt: '2021-02-11T17:25:22.752Z', + persons: [], + objects: [], + reviews: [], + }, + { + _id: '602427a273d8765aeb89f886', + id: 50034, + activeflag: 'active', + award: [], + competencyFramework: '', + counter: 2, + courseDelivery: 'campus', + courseOptions: [ + { + flexibleDates: true, + startDate: null, + studyMode: '', + studyDurationNumber: null, + studyDurationMeasure: '', + fees: [ + { + feeDescription: '', + feeAmount: null, + feePer: '', + }, + ], + }, + ], + createdAt: '2021-02-10T18:36:18.463Z', + creator: 50000, + description: + 'Health data analytics involves extracting insights from health data, either to shape national policy, manage local organisations or inform the care of an individual. As more and more data becomes available electronically, the demand for skilled and trained individuals to take advantage of it becomes increasingly urgent.', + domains: ['Course'], + entries: [ + { + level: '', + subject: '', + }, + ], + keywords: [], + link: 'https://www.ucl.ac.uk/health-informatics/study/postgraduate-taught-programmes/health-data-analytics-mscpg-dippg-cert', + location: 'London', + nationalPriority: '', + provider: 'UCL Institute of Health Informatics', + relatedObjects: [], + restrictions: 'other:Relevant first degree or equivalent experience required', + title: 'Health Data Analytics ', + type: 'course', + updatedAt: '2021-02-11T17:22:50.815Z', + persons: [], + objects: [], + reviews: [], + }, +]; diff --git a/src/resources/course/__tests__/course.controller.test.js b/src/resources/course/__tests__/course.controller.test.js new file mode 100644 index 00000000..02705f2c --- /dev/null +++ b/src/resources/course/__tests__/course.controller.test.js @@ -0,0 +1,117 @@ +import sinon from 'sinon'; +import faker from 'faker'; + +import CourseController from '../v2/course.controller'; +import CourseService from '../v2/course.service'; + +describe('CourseController', function () { + beforeAll(() => { + console.log = sinon.stub(); + console.error = sinon.stub(); + }); + + describe('getCourse', function () { + let req, res, status, json, courseService, courseController; + + beforeEach(() => { + status = sinon.stub(); + json = sinon.spy(); + res = { json, status }; + status.returns(res); + courseService = new CourseService(); + }); + + it('should return a course that matches the id param', async function () { + req = { params: { id: faker.random.number({ min: 1, max: 999999999 }) } }; + const stubValue = { + id: req.params.id, + }; + const serviceStub = sinon.stub(courseService, 'getCourse').returns(stubValue); + courseController = new CourseController(courseService); + await courseController.getCourse(req, res); + + expect(serviceStub.calledOnce).toBe(true); + expect(status.calledWith(200)).toBe(true); + expect(json.calledWith({ success: true, data: stubValue })).toBe(true); + }); + + it('should return a bad request response if no course id is provided', async function () { + req = { params: {} }; + + const serviceStub = sinon.stub(courseService, 'getCourse').returns({}); + courseController = new CourseController(courseService); + await courseController.getCourse(req, res); + + expect(serviceStub.notCalled).toBe(true); + expect(status.calledWith(400)).toBe(true); + expect(json.calledWith({ success: false, message: 'You must provide a course identifier' })).toBe(true); + }); + + it('should return a not found response if no course could be found for the id provided', async function () { + req = { params: { id: faker.random.number({ min: 1, max: 999999999 }) } }; + + const serviceStub = sinon.stub(courseService, 'getCourse').returns(null); + courseController = new CourseController(courseService); + await courseController.getCourse(req, res); + + expect(serviceStub.calledOnce).toBe(true); + expect(status.calledWith(404)).toBe(true); + expect(json.calledWith({ success: false, message: 'A course could not be found with the provided id' })).toBe(true); + }); + + it('should return a server error if an unexpected exception occurs', async function () { + req = { params: { id: faker.random.number({ min: 1, max: 999999999 }) } }; + + const error = new Error('A server error occurred'); + const serviceStub = sinon.stub(courseService, 'getCourse').throws(error); + courseController = new CourseController(courseService); + await courseController.getCourse(req, res); + + expect(serviceStub.calledOnce).toBe(true); + expect(status.calledWith(500)).toBe(true); + expect(json.calledWith({ success: false, message: 'A server error occurred, please try again' })).toBe(true); + }); + }); + + describe('getCourses', function () { + let req, res, status, json, courseService, courseController; + req = { params: {} }; + + beforeEach(() => { + status = sinon.stub(); + json = sinon.spy(); + res = { json, status }; + status.returns(res); + courseService = new CourseService(); + }); + + it('should return an array of courses', async function () { + const stubValue = [ + { + id: faker.random.number({ min: 1, max: 999999999 }), + }, + { + id: faker.random.number({ min: 1, max: 999999999 }), + }, + ]; + const serviceStub = sinon.stub(courseService, 'getCourses').returns(stubValue); + courseController = new CourseController(courseService); + await courseController.getCourses(req, res); + + expect(serviceStub.calledOnce).toBe(true); + expect(status.calledWith(200)).toBe(true); + expect(json.calledWith({ success: true, data: stubValue })).toBe(true); + }); + + it('should return a server error if an unexpected exception occurs', async function () { + const error = new Error('A server error occurred'); + const serviceStub = sinon.stub(courseService, 'getCourses').throws(error); + courseController = new CourseController(courseService); + await courseController.getCourses(req, res); + + expect(serviceStub.calledOnce).toBe(true); + expect(status.calledWith(500)).toBe(true); + expect(json.calledWith({ success: false, message: 'A server error occurred, please try again' })).toBe(true); + }); + }); +}); diff --git a/src/resources/course/__tests__/course.entity.test.js b/src/resources/course/__tests__/course.entity.test.js new file mode 100644 index 00000000..427a0575 --- /dev/null +++ b/src/resources/course/__tests__/course.entity.test.js @@ -0,0 +1,95 @@ +import CourseClass from '../v2/course.entity'; + +describe('CourseEntity', function () { + describe('constructor', function () { + it('should create an instance of a course entity with the expected properties', async function () { + const course = new CourseClass( + 50033, + 'course', + 50000, + 'active', + 5, + null, + [], + 'Health Data Analytics', + 'https://www.ucl.ac.uk/health-informatics/study/postgraduate-taught-programmes/health-data-analytics-mscpg-dippg-cert', + 'UCL Institute of Health Informatics', + 'Health data analytics involves extracting insights from health data, either to shape national policy, manage local organisations or inform the care of an individual. As more and more data becomes available electronically, the demand for skilled and trained individuals to take advantage of it becomes increasingly urgent.', + 'campus', + 'London', + [], + ['Course'], + [ + { + flexibleDates: true, + startDate: null, + studyMode: '', + studyDurationNumber: null, + studyDurationMeasure: '', + fees: [ + { + feeDescription: '', + feeAmount: null, + feePer: '', + }, + ], + }, + ], + [ + { + level: '', + subject: '', + }, + ], + 'other:Relevant first degree or equivalent experience required', + [], + '', + '' + ); + + expect(course.id).toEqual(50033); + expect(course.type).toEqual('course'); + expect(course.creator).toEqual(50000); + expect(course.activeflag).toEqual('active'); + expect(course.counter).toEqual(5); + expect(course.discourseTopicId).toEqual(null); + expect(course.relatedObjects).toEqual([]); + expect(course.title).toEqual('Health Data Analytics'); + expect(course.link).toEqual('https://www.ucl.ac.uk/health-informatics/study/postgraduate-taught-programmes/health-data-analytics-mscpg-dippg-cert'); + expect(course.provider).toEqual('UCL Institute of Health Informatics'); + expect(course.description).toEqual( + 'Health data analytics involves extracting insights from health data, either to shape national policy, manage local organisations or inform the care of an individual. As more and more data becomes available electronically, the demand for skilled and trained individuals to take advantage of it becomes increasingly urgent.' + ); + expect(course.courseDelivery).toEqual('campus'); + expect(course.location).toEqual('London'); + expect(course.keywords).toEqual([]); + expect(course.domains).toEqual(['Course']); + expect(course.courseOptions).toEqual([ + { + flexibleDates: true, + startDate: null, + studyMode: '', + studyDurationNumber: null, + studyDurationMeasure: '', + fees: [ + { + feeDescription: '', + feeAmount: null, + feePer: '', + }, + ], + }, + ]); + expect(course.entries).toEqual([ + { + level: '', + subject: '', + }, + ]); + expect(course.restrictions).toEqual('other:Relevant first degree or equivalent experience required'); + expect(course.award).toEqual([]); + expect(course.competencyFramework).toEqual(''); + expect(course.nationalPriority).toEqual(''); + }); + }); +}); diff --git a/src/resources/course/__tests__/course.repository.it.test.js b/src/resources/course/__tests__/course.repository.it.test.js new file mode 100644 index 00000000..d8156ff3 --- /dev/null +++ b/src/resources/course/__tests__/course.repository.it.test.js @@ -0,0 +1,42 @@ +import dbHandler from '../../../config/in-memory-db'; +import CourseRepository from '../v2/course.repository'; +import { coursesStub } from '../__mocks__/courses'; + +/** + * Connect to a new in-memory database before running any tests. + */ +beforeAll(async () => { + await dbHandler.connect(); + await dbHandler.loadData({ course: coursesStub }); +}); + +/** + * Revert to initial test data after every test. + */ +afterEach(async () => { + await dbHandler.clearDatabase(); + await dbHandler.loadData({ course: coursesStub }); +}); + +/** + * Remove and close the db and server. + */ +afterAll(async () => await dbHandler.closeDatabase()); + +describe('CourseRepository', function () { + describe('getCourse', () => { + it('should return a course by a specified id', async function () { + const courseRepository = new CourseRepository(); + const course = await courseRepository.getCourse(50033); + expect(course).toEqual(coursesStub[0]); + }); + }); + + describe('getCourses', () => { + it('should return an array of courses', async function () { + const courseRepository = new CourseRepository(); + const courses = await courseRepository.getCourses(); + expect(courses.length).toBeGreaterThan(0); + }); + }); +}); diff --git a/src/resources/course/__tests__/course.repository.test.js b/src/resources/course/__tests__/course.repository.test.js new file mode 100644 index 00000000..c07f07ec --- /dev/null +++ b/src/resources/course/__tests__/course.repository.test.js @@ -0,0 +1,63 @@ +import sinon from 'sinon'; + +import CourseRepository from '../v2/course.repository'; +import { coursesStub } from '../__mocks__/courses'; + +describe('CourseRepository', function () { + describe('getCourse', function () { + it('should return a course by a specified id', async function () { + const courseStub = coursesStub[0]; + const courseRepository = new CourseRepository(); + const stub = sinon.stub(courseRepository, 'findOne').returns(courseStub); + const course = await courseRepository.getCourse(courseStub.id); + + expect(stub.calledOnce).toBe(true); + + expect(course.type).toEqual(courseStub.type); + expect(course.id).toEqual(courseStub.id); + expect(course.creator).toEqual(courseStub.creator); + expect(course.description).toEqual(courseStub.description); + expect(course.activeflag).toEqual(courseStub.activeflag); + expect(course.counter).toEqual(courseStub.counter); + expect(course.discourseTopicId).toEqual(courseStub.discourseTopicId); + expect(course.relatedObjects).toEqual(courseStub.relatedObjects); + expect(course.title).toEqual(courseStub.title); + expect(course.link).toEqual(courseStub.link); + expect(course.provider).toEqual(courseStub.provider); + expect(course.courseDelivery).toEqual(courseStub.courseDelivery); + expect(course.location).toEqual(courseStub.location); + expect(course.keywords).toEqual(courseStub.keywords); + expect(course.domains).toEqual(courseStub.domains); + expect(course.courseOptions).toEqual(courseStub.courseOptions); + expect(course.entries).toEqual(courseStub.entries); + expect(course.restrictions).toEqual(courseStub.restrictions); + expect(course.award).toEqual(courseStub.award); + expect(course.competencyFramework).toEqual(courseStub.competencyFramework); + expect(course.nationalPriority).toEqual(courseStub.nationalPriority); + }); + }); + + describe('getCourses', function () { + it('should return an array of courses', async function () { + const courseRepository = new CourseRepository(); + const stub = sinon.stub(courseRepository, 'find').returns(coursesStub); + const courses = await courseRepository.getCourses(); + + expect(stub.calledOnce).toBe(true); + + expect(courses.length).toBeGreaterThan(0); + }); + }); + + describe('findCountOf', function () { + it('should return the number of documents found by a given query', async function () { + const courseRepository = new CourseRepository(); + const stub = sinon.stub(courseRepository, 'findCountOf').returns(1); + const courseCount = await courseRepository.findCountOf({ name: 'Admitted Patient Care Course' }); + + expect(stub.calledOnce).toBe(true); + + expect(courseCount).toEqual(1); + }); + }); +}); \ No newline at end of file diff --git a/src/resources/course/__tests__/course.service.test.js b/src/resources/course/__tests__/course.service.test.js new file mode 100644 index 00000000..b7be80af --- /dev/null +++ b/src/resources/course/__tests__/course.service.test.js @@ -0,0 +1,53 @@ +import sinon from 'sinon'; + +import CourseRepository from '../v2/course.repository'; +import CourseService from '../v2/course.service'; +import { coursesStub } from '../__mocks__/courses'; + +describe('CourseService', function () { + describe('getCourse', function () { + it('should return a course by a specified id', async function () { + const courseStub = coursesStub[0]; + const courseRepository = new CourseRepository(); + const stub = sinon.stub(courseRepository, 'getCourse').returns(courseStub); + const courseService = new CourseService(courseRepository); + const course = await courseService.getCourse(courseStub.id); + + expect(stub.calledOnce).toBe(true); + + expect(course.type).toEqual(courseStub.type); + expect(course.id).toEqual(courseStub.id); + expect(course.creator).toEqual(courseStub.creator); + expect(course.description).toEqual(courseStub.description); + expect(course.activeflag).toEqual(courseStub.activeflag); + expect(course.counter).toEqual(courseStub.counter); + expect(course.discourseTopicId).toEqual(courseStub.discourseTopicId); + expect(course.relatedObjects).toEqual(courseStub.relatedObjects); + expect(course.title).toEqual(courseStub.title); + expect(course.link).toEqual(courseStub.link); + expect(course.provider).toEqual(courseStub.provider); + expect(course.courseDelivery).toEqual(courseStub.courseDelivery); + expect(course.location).toEqual(courseStub.location); + expect(course.keywords).toEqual(courseStub.keywords); + expect(course.domains).toEqual(courseStub.domains); + expect(course.courseOptions).toEqual(courseStub.courseOptions); + expect(course.entries).toEqual(courseStub.entries); + expect(course.restrictions).toEqual(courseStub.restrictions); + expect(course.award).toEqual(courseStub.award); + expect(course.competencyFramework).toEqual(courseStub.competencyFramework); + expect(course.nationalPriority).toEqual(courseStub.nationalPriority); + }); + }); + describe('getCourses', function () { + it('should return an array of courses', async function () { + const courseRepository = new CourseRepository(); + const stub = sinon.stub(courseRepository, 'getCourses').returns(coursesStub); + const courseService = new CourseService(courseRepository); + const courses = await courseService.getCourses(); + + expect(stub.calledOnce).toBe(true); + + expect(courses.length).toBeGreaterThan(0); + }); + }); +}); diff --git a/src/resources/course/course.model.js b/src/resources/course/course.model.js index 4a9a2c80..dea9c325 100644 --- a/src/resources/course/course.model.js +++ b/src/resources/course/course.model.js @@ -1,6 +1,8 @@ import { model, Schema } from 'mongoose'; -const CourseSchema = new Schema( +import CourseClass from './v2/course.entity'; + +const courseSchema = new Schema( { id: Number, type: String, @@ -63,4 +65,7 @@ const CourseSchema = new Schema( } ); -export const Course = model('Course', CourseSchema); +// Load entity class +courseSchema.loadClass(CourseClass); + +export const Course = model('Course', courseSchema); diff --git a/src/resources/course/course.route.js b/src/resources/course/v1/course.route.js similarity index 95% rename from src/resources/course/course.route.js rename to src/resources/course/v1/course.route.js index 6c427a4a..ca36750b 100644 --- a/src/resources/course/course.route.js +++ b/src/resources/course/v1/course.route.js @@ -1,10 +1,10 @@ import express from 'express'; -import { ROLES } from '../user/user.roles'; -import { Data } from '../tool/data.model'; -import { Course } from './course.model'; +import { ROLES } from '../../user/user.roles'; +import { Data } from '../../tool/data.model'; +import { Course } from '../course.model'; import passport from 'passport'; -import { utils } from '../auth'; -import { addCourse, editCourse, setStatus, getCourseAdmin, getCourse, getAllCourses } from './course.repository'; +import { utils } from '../../auth'; +import { addCourse, editCourse, setStatus, getCourseAdmin, getCourse, getAllCourses } from '../course.repository'; import escape from 'escape-html'; const router = express.Router(); diff --git a/src/resources/course/v2/course.controller.js b/src/resources/course/v2/course.controller.js new file mode 100644 index 00000000..7b46ddb3 --- /dev/null +++ b/src/resources/course/v2/course.controller.js @@ -0,0 +1,62 @@ +import Controller from '../../base/controller'; + +export default class CourseController extends Controller { + constructor(courseService) { + super(courseService); + this.courseService = courseService; + } + + async getCourse(req, res) { + try { + // Extract id parameter from query string + const { id } = req.params; + // If no id provided, it is a bad request + if (!id) { + return res.status(400).json({ + success: false, + message: 'You must provide a course identifier', + }); + } + // Find the course + let course = await this.courseService.getCourse(id, req.query); + // Return if no course found + if (!course) { + return res.status(404).json({ + success: false, + message: 'A course could not be found with the provided id', + }); + } + // Return the course + return res.status(200).json({ + success: true, + data: course, + }); + } catch (err) { + // Return error response if something goes wrong + console.error(err); + return res.status(500).json({ + success: false, + message: 'A server error occurred, please try again', + }); + } + } + + async getCourses(req, res) { + try { + // Find the courses + let courses = await this.courseService.getCourses(req.query); + // Return the courses + return res.status(200).json({ + success: true, + data: courses + }); + } catch (err) { + // Return error response if something goes wrong + console.error(err); + return res.status(500).json({ + success: false, + message: 'A server error occurred, please try again', + }); + } + } +} diff --git a/src/resources/course/v2/course.entity.js b/src/resources/course/v2/course.entity.js new file mode 100644 index 00000000..8e821026 --- /dev/null +++ b/src/resources/course/v2/course.entity.js @@ -0,0 +1,50 @@ +import Entity from '../../base/entity'; + +export default class CourseClass extends Entity { + constructor( + id, + type, + creator, + activeflag, + counter, + discourseTopicId, + relatedObjects, + title, + link, + provider, + description, + courseDelivery, + location, + keywords, + domains, + courseOptions, + entries, + restrictions, + award, + competencyFramework, + nationalPriority + ) { + super(); + this.id = id; + this.type = type; + this.creator = creator; + this.activeflag = activeflag; + this.counter = counter; + this.discourseTopicId = discourseTopicId; + this.relatedObjects = relatedObjects; + this.title = title; + this.link = link; + this.provider = provider; + this.description = description; + this.courseDelivery = courseDelivery; + this.location = location; + this.keywords = keywords; + this.domains = domains; + this.courseOptions = courseOptions; + this.entries = entries; + this.restrictions = restrictions; + this.award = award; + this.competencyFramework = competencyFramework; + this.nationalPriority = nationalPriority; + } +} diff --git a/src/resources/course/v2/course.repository.js b/src/resources/course/v2/course.repository.js new file mode 100644 index 00000000..5c7cfa27 --- /dev/null +++ b/src/resources/course/v2/course.repository.js @@ -0,0 +1,20 @@ +import Repository from '../../base/repository'; +import { Course } from '../course.model'; + +export default class CourseRepository extends Repository { + constructor() { + super(Course); + this.course = Course; + } + + async getCourse(id, query) { + query = { ...query, id }; + const options = { lean: true }; + return this.findOne(query, options); + } + + async getCourses(query) { + const options = { lean: true }; + return this.find(query, options); + } +} diff --git a/src/resources/course/v2/course.route.js b/src/resources/course/v2/course.route.js new file mode 100644 index 00000000..8c6aab39 --- /dev/null +++ b/src/resources/course/v2/course.route.js @@ -0,0 +1,18 @@ +import express from 'express'; +import CourseController from './course.controller'; +import { courseService } from './dependency'; + +const router = express.Router(); +const courseController = new CourseController(courseService); + +// @route GET /api/v2/courses/id +// @desc Returns a course based on identifier provided +// @access Public +router.get('/:id', (req, res) => courseController.getCourse(req, res)); + +// @route GET /api/v2/courses +// @desc Returns a collection of courses based on supplied query parameters +// @access Public +router.get('/', (req, res) => courseController.getCourses(req, res)); + +module.exports = router; diff --git a/src/resources/course/v2/course.service.js b/src/resources/course/v2/course.service.js new file mode 100644 index 00000000..4eca2e31 --- /dev/null +++ b/src/resources/course/v2/course.service.js @@ -0,0 +1,13 @@ +export default class CourseService { + constructor(courseRepository) { + this.courseRepository = courseRepository; + } + + getCourse(id, query = {}) { + return this.courseRepository.getCourse(id, query); + } + + getCourses(query = {}) { + return this.courseRepository.getCourses(query); + } +} diff --git a/src/resources/course/v2/dependency.js b/src/resources/course/v2/dependency.js new file mode 100644 index 00000000..34c8ef7b --- /dev/null +++ b/src/resources/course/v2/dependency.js @@ -0,0 +1,5 @@ +import CourseRepository from './course.repository'; +import CourseService from './course.service'; + +export const courseRepository = new CourseRepository(); +export const courseService = new CourseService(courseRepository); diff --git a/src/resources/datarequest/datarequest.controller.js b/src/resources/datarequest/datarequest.controller.js index 2799d9f0..aaf31fac 100644 --- a/src/resources/datarequest/datarequest.controller.js +++ b/src/resources/datarequest/datarequest.controller.js @@ -1354,7 +1354,7 @@ module.exports = { // 9. Return aplication and successful response return res.status(200).json({ status: 'success', data: accessRecord._doc }); } catch (err) { - console.log(err.message); + console.error(err.message); res.status(500).json({ status: 'error', message: err.message }); } }, diff --git a/src/resources/dataset/__mocks__/dataset.js b/src/resources/dataset/__mocks__/dataset.js deleted file mode 100644 index 4e9e404e..00000000 --- a/src/resources/dataset/__mocks__/dataset.js +++ /dev/null @@ -1,99 +0,0 @@ -export const datasetStub = { - id: '675584862177848', - name: 'Admitted Patient Care Dataset', - description: 'This is a dataset about admitted patient care', - resultsInsights: null, - link: null, - type: 'dataset', - datasetid: 'dfb21b3b-7fd9-40c4-892e-810edd6dfc25', - categories: {}, - license: null, - authors: [], - tags: {}, - activeflag: 'active', - counter: 15, - discourseTopicId: null, - relatedObjects: [], - uploader: null, - datasetid: 'dfb21b3b-7fd9-40c4-892e-810edd6dfc25', - pid: '4ef841d3-5e86-4f92-883f-1015ffd4b979', - datasetVersion: '0.0.1', - datasetfields: { - publisher: "Oxford University Hospitals NHS Foundation Trust", - geographicCoverage: [], - physicalSampleAvailability: [], - abstract: "Nationally defined dataset which ontaining administrative details for inpatient admissions (elective, emergency and maternity) and good coverage of clinical coding of diagnosis (ICD10) and procedures (OPCS4). Includes home birth and delivery spells.", - releaseDate: null, - accessRequestDuration: null, - conformsTo: null, - accessRights: "Available locally within Trust Clinical Data Warehouse System and authorised access by Trust staff only.\nDataset is also available via SUS/HES for government statistical purposes.", - jurisdiction: null, - datasetStartDate: null, - datasetEndDate: null, - statisticalPopulation: null, - ageBand: null, - contactPoint: "kinga.varnai@ouh.nhs.uk", - periodicity: null, - metadataquality: { - id: "dfb21b3b-7fd9-40c4-892e-810edd6dfc25", - publisher: "Oxford University Hospitals NHS Foundation Trust", - title: "Admitted Patient Care Dataset", - completeness_percent: 16.67, - weighted_completeness_percent: 14.29, - error_percent: 39.53, - weighted_error_percent: 39.68, - quality_score: 38.57, - quality_rating: "Not Rated", - weighted_quality_score: 37.3, - weighted_quality_rating: "Not Rated" - }, - datautility: { - id: "dfb21b3b-7fd9-40c4-892e-810edd6dfc25", - publisher: "Oxford University Hospitals NHS Foundation Trust", - title: "Admitted Patient Care Dataset", - metadata_richness: "Not Rated", - availability_of_additional_documentation_and_support: "", - data_model: "", - data_dictionary: "", - provenance: "", - data_quality_management_process: "", - dama_quality_dimensions: "", - pathway_coverage: "", - length_of_follow_up: "", - allowable_uses: "", - research_environment: "", - time_lag: "", - timeliness: "", - linkages: "", - data_enrichments: "" - }, - metadataschema: { - context: "http://schema.org/", - type: "Dataset", - identifier: "dfb21b3b-7fd9-40c4-892e-810edd6dfc25", - url: "https://healthdatagateway.org/detail/dfb21b3b-7fd9-40c4-892e-810edd6dfc25", - name: "Admitted Patient Care Dataset", - description: "", - keywords: [ - "Oxford University Hospitals NHS Foundation Trust", - "CDS" - ], - includedinDataCatalog: [ - { - type: "DataCatalog", - name: "Oxford University Hospitals NHS Foundation Trust", - url: "kinga.varnai@ouh.nhs.uk" - }, - { - type: "DataCatalog", - name: "HDR UK Health Data Gateway", - url: "http://healthdatagateway.org" - } - ] - }, - technicaldetails: [], - versionLinks: [], - phenotypes: [] - }, - datasetv2: {} - }; \ No newline at end of file diff --git a/src/resources/dataset/__mocks__/datasets.js b/src/resources/dataset/__mocks__/datasets.js index 41eae0ad..cea577ec 100644 --- a/src/resources/dataset/__mocks__/datasets.js +++ b/src/resources/dataset/__mocks__/datasets.js @@ -1,4 +1,4 @@ -export const datasets = [{ +export const datasetsStub = [{ id: '675584862177848', submittedDataAccessRequests: 1, name: 'Admitted Patient Care Dataset', diff --git a/src/resources/dataset/__tests__/dataset.controller.test.js b/src/resources/dataset/__tests__/dataset.controller.test.js index 51ef150a..5d50f07c 100644 --- a/src/resources/dataset/__tests__/dataset.controller.test.js +++ b/src/resources/dataset/__tests__/dataset.controller.test.js @@ -5,65 +5,113 @@ import DatasetController from '../dataset.controller'; import DatasetService from '../dataset.service'; describe('DatasetController', function () { + beforeAll(() => { + console.log = sinon.stub(); + console.error = sinon.stub(); + }); + describe('getDataset', function () { let req, res, status, json, datasetService, datasetController; + beforeEach(() => { - status = sinon.stub(); - json = sinon.spy(); - res = { json, status }; - status.returns(res); + status = sinon.stub(); + json = sinon.spy(); + res = { json, status }; + status.returns(res); datasetService = new DatasetService(); }); it('should return a dataset that matches the id param', async function () { - req = { params: { id: faker.random.number({'min':1, 'max':999999999}) }}; + req = { params: { id: faker.random.number({ min: 1, max: 999999999 }) } }; const stubValue = { - id: req.params.id + id: req.params.id, }; const serviceStub = sinon.stub(datasetService, 'getDataset').returns(stubValue); datasetController = new DatasetController(datasetService); - await datasetController.getDataset(req, res); - - expect(serviceStub.calledOnce).toBe(true); - expect(status.calledWith(200)).toBe(true); - expect(json.calledWith({ success: true, data: stubValue })).toBe(true); - }); - - it('should return a bad request response if no dataset id is provided', async function () { - req = { params: {} }; + await datasetController.getDataset(req, res); + + expect(serviceStub.calledOnce).toBe(true); + expect(status.calledWith(200)).toBe(true); + expect(json.calledWith({ success: true, data: stubValue })).toBe(true); + }); + + it('should return a bad request response if no dataset id is provided', async function () { + req = { params: {} }; const serviceStub = sinon.stub(datasetService, 'getDataset').returns({}); datasetController = new DatasetController(datasetService); - await datasetController.getDataset(req, res); - - expect(serviceStub.notCalled).toBe(true); - expect(status.calledWith(400)).toBe(true); - expect(json.calledWith({ success: false, message: 'You must provide a dataset version id or a dataset persistent id' })).toBe(true); - }); - - it('should return a not found response if no dataset could be found for the id provided', async function () { - req = { params: { id: faker.random.number({'min':1, 'max':999999999}) }}; + await datasetController.getDataset(req, res); + + expect(serviceStub.notCalled).toBe(true); + expect(status.calledWith(400)).toBe(true); + expect(json.calledWith({ success: false, message: 'You must provide a dataset identifier' })).toBe(true); + }); + + it('should return a not found response if no dataset could be found for the id provided', async function () { + req = { params: { id: faker.random.number({ min: 1, max: 999999999 }) } }; const serviceStub = sinon.stub(datasetService, 'getDataset').returns(null); datasetController = new DatasetController(datasetService); - await datasetController.getDataset(req, res); - - expect(serviceStub.calledOnce).toBe(true); - expect(status.calledWith(404)).toBe(true); - expect(json.calledWith({ success: false, message: 'A dataset could not be found with the provided id' })).toBe(true); - }); - - it('should return a server error if an unexpected exception occurs', async function () { - req = { params: { id: faker.random.number({'min':1, 'max':999999999}) }}; - - const error = new Error("A server error occurred"); - const serviceStub = sinon.stub(datasetService, 'getDataset').throws(error); + await datasetController.getDataset(req, res); + + expect(serviceStub.calledOnce).toBe(true); + expect(status.calledWith(404)).toBe(true); + expect(json.calledWith({ success: false, message: 'A dataset could not be found with the provided id' })).toBe(true); + }); + + it('should return a server error if an unexpected exception occurs', async function () { + req = { params: { id: faker.random.number({ min: 1, max: 999999999 }) } }; + + const error = new Error('A server error occurred'); + const serviceStub = sinon.stub(datasetService, 'getDataset').throws(error); datasetController = new DatasetController(datasetService); - await datasetController.getDataset(req, res); - - expect(serviceStub.calledOnce).toBe(true); - expect(status.calledWith(500)).toBe(true); - expect(json.calledWith({ success: false, message: 'A server error occurred, please try again' })).toBe(true); + await datasetController.getDataset(req, res); + + expect(serviceStub.calledOnce).toBe(true); + expect(status.calledWith(500)).toBe(true); + expect(json.calledWith({ success: false, message: 'A server error occurred, please try again' })).toBe(true); + }); + }); + + describe('getDatasets', function () { + let req, res, status, json, datasetService, datasetController; + req = { params: {} }; + + beforeEach(() => { + status = sinon.stub(); + json = sinon.spy(); + res = { json, status }; + status.returns(res); + datasetService = new DatasetService(); + }); + + it('should return an array of datasets', async function () { + const stubValue = [ + { + id: faker.random.number({ min: 1, max: 999999999 }), + }, + { + id: faker.random.number({ min: 1, max: 999999999 }), + }, + ]; + const serviceStub = sinon.stub(datasetService, 'getDatasets').returns(stubValue); + datasetController = new DatasetController(datasetService); + await datasetController.getDatasets(req, res); + + expect(serviceStub.calledOnce).toBe(true); + expect(status.calledWith(200)).toBe(true); + expect(json.calledWith({ success: true, data: stubValue })).toBe(true); + }); + + it('should return a server error if an unexpected exception occurs', async function () { + const error = new Error('A server error occurred'); + const serviceStub = sinon.stub(datasetService, 'getDatasets').throws(error); + datasetController = new DatasetController(datasetService); + await datasetController.getDatasets(req, res); + + expect(serviceStub.calledOnce).toBe(true); + expect(status.calledWith(500)).toBe(true); + expect(json.calledWith({ success: false, message: 'A server error occurred, please try again' })).toBe(true); }); }); }); diff --git a/src/resources/dataset/__tests__/dataset.entity.test.js b/src/resources/dataset/__tests__/dataset.entity.test.js index 0aefe434..5cb85016 100644 --- a/src/resources/dataset/__tests__/dataset.entity.test.js +++ b/src/resources/dataset/__tests__/dataset.entity.test.js @@ -1,7 +1,6 @@ import DatasetClass from '../dataset.entity'; describe('DatasetEntity', function () { - describe('constructor', function () { it('should create an instance of a dataset entity with the expected properties', async function () { const dataset = new DatasetClass( @@ -49,7 +48,7 @@ describe('DatasetEntity', function () { }); }); - describe('isLatestVersion', function () { + describe('checkLatestVersion', function () { it('should return a boolean indicating this is the latest version of the dataset', async function () { const dataset = new DatasetClass( 675584862177848, @@ -74,7 +73,7 @@ describe('DatasetEntity', function () { {} ); - const result = dataset.isLatestVersion(); + const result = dataset.checkLatestVersion(); expect(result).toBe(true); }); @@ -103,7 +102,7 @@ describe('DatasetEntity', function () { {} ); - const result = dataset.isLatestVersion(); + const result = dataset.checkLatestVersion(); expect(result).toBe(false); }); diff --git a/src/resources/dataset/__tests__/dataset.repository.it.test.js b/src/resources/dataset/__tests__/dataset.repository.it.test.js index 69ad55b8..bce45989 100644 --- a/src/resources/dataset/__tests__/dataset.repository.it.test.js +++ b/src/resources/dataset/__tests__/dataset.repository.it.test.js @@ -1,17 +1,14 @@ import dbHandler from '../../../config/in-memory-db'; import DatasetRepository from '../dataset.repository'; -import { datasets } from '../__mocks__/datasets'; +import { datasetsStub } from '../__mocks__/datasets'; import { dataAccessRequests } from '../__mocks__/dataaccessreequests'; -//const amendmentController = require('../amendment.controller'); -//const amendmentModel = require('../amendment.model'); - /** * Connect to a new in-memory database before running any tests. */ beforeAll(async () => { await dbHandler.connect(); - await dbHandler.loadData({ tools: datasets, data_requests: dataAccessRequests }); + await dbHandler.loadData({ tools: datasetsStub, data_requests: dataAccessRequests }); }); /** @@ -19,7 +16,7 @@ beforeAll(async () => { */ afterEach(async () => { await dbHandler.clearDatabase(); - await dbHandler.loadData({ tools: datasets }); + await dbHandler.loadData({ tools: datasetsStub, data_requests: dataAccessRequests }); }); /** @@ -28,11 +25,19 @@ afterEach(async () => { afterAll(async () => await dbHandler.closeDatabase()); describe('DatasetRepository', function () { - describe('getUser', () => { + describe('getDataset', () => { it('should return a dataset by a specified id', async function () { const datasetRepository = new DatasetRepository(); const dataset = await datasetRepository.getDataset("dfb21b3b-7fd9-40c4-892e-810edd6dfc25"); - expect(dataset).toEqual(datasets[0]); + expect(dataset).toEqual(datasetsStub[0]); + }); + }); + + describe('getDatasets', () => { + it('should return an array of datasets', async function () { + const datasetRepository = new DatasetRepository(); + const datasets = await datasetRepository.getDatasets(); + expect(datasets.length).toBeGreaterThan(0); }); }); }); diff --git a/src/resources/dataset/__tests__/dataset.repository.test.js b/src/resources/dataset/__tests__/dataset.repository.test.js index 9998e9e9..8cadab39 100644 --- a/src/resources/dataset/__tests__/dataset.repository.test.js +++ b/src/resources/dataset/__tests__/dataset.repository.test.js @@ -1,11 +1,12 @@ import sinon from 'sinon'; import DatasetRepository from '../dataset.repository'; -import { datasetStub } from '../__mocks__/dataset'; +import { datasetsStub } from '../__mocks__/datasets'; describe('DatasetRepository', function () { describe('getDataset', function () { it('should return a dataset by a specified id', async function () { + const datasetStub = datasetsStub[0]; const datasetRepository = new DatasetRepository(); const stub = sinon.stub(datasetRepository, 'findOne').returns(datasetStub); const dataset = await datasetRepository.getDataset(datasetStub.id); @@ -33,4 +34,28 @@ describe('DatasetRepository', function () { expect(dataset.datasetv2).toEqual(datasetStub.datasetv2); }); }); -}); + + describe('getDatasets', function () { + it('should return an array of datasets', async function () { + const datasetRepository = new DatasetRepository(); + const stub = sinon.stub(datasetRepository, 'find').returns(datasetsStub); + const datasets = await datasetRepository.getDatasets(); + + expect(stub.calledOnce).toBe(true); + + expect(datasets.length).toBeGreaterThan(0); + }); + }); + + describe('findCountOf', function () { + it('should return the number of documents found by a given query', async function () { + const datasetRepository = new DatasetRepository(); + const stub = sinon.stub(datasetRepository, 'findCountOf').returns(1); + const datasetCount = await datasetRepository.findCountOf({ name: 'Admitted Patient Care Dataset' }); + + expect(stub.calledOnce).toBe(true); + + expect(datasetCount).toEqual(1); + }); + }); +}); \ No newline at end of file diff --git a/src/resources/dataset/__tests__/dataset.service.test.js b/src/resources/dataset/__tests__/dataset.service.test.js index 75a0d18f..482a2fc0 100644 --- a/src/resources/dataset/__tests__/dataset.service.test.js +++ b/src/resources/dataset/__tests__/dataset.service.test.js @@ -2,11 +2,12 @@ import sinon from 'sinon'; import DatasetRepository from '../dataset.repository'; import DatasetService from '../dataset.service'; -import { datasetStub } from '../__mocks__/dataset'; +import { datasetsStub } from '../__mocks__/datasets'; describe('DatasetService', function () { describe('getDataset', function () { it('should return a dataset by a specified id', async function () { + const datasetStub = datasetsStub[0]; const datasetRepository = new DatasetRepository(); const stub = sinon.stub(datasetRepository, 'getDataset').returns(datasetStub); const datasetService = new DatasetService(datasetRepository); @@ -34,5 +35,77 @@ describe('DatasetService', function () { expect(dataset.datasetfields).toEqual(datasetStub.datasetfields); expect(dataset.datasetv2).toEqual(datasetStub.datasetv2); }); + + it('should return a dataset by a specified id when a query parameter is passed as the second argument', async function () { + const datasetStub = datasetsStub[0]; + const datasetRepository = new DatasetRepository(); + const stub = sinon.stub(datasetRepository, 'getDataset').returns(datasetStub); + const datasetService = new DatasetService(datasetRepository); + const dataset = await datasetService.getDataset(datasetStub.id, { expanded: false }); + + expect(stub.calledOnce).toBe(true); + + expect(dataset.datasetid).toEqual(datasetStub.datasetid); + expect(dataset.type).toEqual(datasetStub.type); + expect(dataset.id).toEqual(datasetStub.id); + expect(dataset.name).toEqual(datasetStub.name); + expect(dataset.description).toEqual(datasetStub.description); + expect(dataset.resultsInsights).toEqual(datasetStub.resultsInsights); + expect(dataset.datasetid).toEqual(datasetStub.datasetid); + expect(dataset.categories).toEqual(datasetStub.categories); + expect(dataset.license).toEqual(datasetStub.license); + expect(dataset.authors).toEqual(datasetStub.authors); + expect(dataset.activeflag).toEqual(datasetStub.activeflag); + expect(dataset.counter).toEqual(datasetStub.counter); + expect(dataset.discourseTopicId).toEqual(datasetStub.discourseTopicId); + expect(dataset.relatedObjects).toEqual(datasetStub.relatedObjects); + expect(dataset.uploader).toEqual(datasetStub.uploader); + expect(dataset.pid).toEqual(datasetStub.pid); + expect(dataset.datasetVersion).toEqual(datasetStub.datasetVersion); + expect(dataset.datasetfields).toEqual(datasetStub.datasetfields); + expect(dataset.datasetv2).toEqual(datasetStub.datasetv2); + }); + + it('should return a dataset by a specified id when expanded', async function () { + const datasetStub = datasetsStub[0]; + const datasetRepository = new DatasetRepository(); + const stub = sinon.stub(datasetRepository, 'getDataset').returns({ ...datasetStub, checkLatestVersion: () => true}); + const datasetService = new DatasetService(datasetRepository); + const dataset = await datasetService.getDataset(datasetStub.id, { expanded: true }); + + expect(stub.calledOnce).toBe(true); + + expect(dataset.datasetid).toEqual(datasetStub.datasetid); + expect(dataset.type).toEqual(datasetStub.type); + expect(dataset.id).toEqual(datasetStub.id); + expect(dataset.name).toEqual(datasetStub.name); + expect(dataset.description).toEqual(datasetStub.description); + expect(dataset.resultsInsights).toEqual(datasetStub.resultsInsights); + expect(dataset.datasetid).toEqual(datasetStub.datasetid); + expect(dataset.categories).toEqual(datasetStub.categories); + expect(dataset.license).toEqual(datasetStub.license); + expect(dataset.authors).toEqual(datasetStub.authors); + expect(dataset.activeflag).toEqual(datasetStub.activeflag); + expect(dataset.counter).toEqual(datasetStub.counter); + expect(dataset.discourseTopicId).toEqual(datasetStub.discourseTopicId); + expect(dataset.relatedObjects).toEqual(datasetStub.relatedObjects); + expect(dataset.uploader).toEqual(datasetStub.uploader); + expect(dataset.pid).toEqual(datasetStub.pid); + expect(dataset.datasetVersion).toEqual(datasetStub.datasetVersion); + expect(dataset.datasetfields).toEqual(datasetStub.datasetfields); + expect(dataset.datasetv2).toEqual(datasetStub.datasetv2); + }); + }); + describe('getDatasets', function () { + it('should return an array of datasets', async function () { + const datasetRepository = new DatasetRepository(); + const stub = sinon.stub(datasetRepository, 'getDatasets').returns(datasetsStub); + const datasetService = new DatasetService(datasetRepository); + const datasets = await datasetService.getDatasets(); + + expect(stub.calledOnce).toBe(true); + + expect(datasets.length).toBeGreaterThan(0); + }); }); }); diff --git a/src/resources/dataset/dataset.controller.js b/src/resources/dataset/dataset.controller.js index e824ccf0..4bf0b79a 100644 --- a/src/resources/dataset/dataset.controller.js +++ b/src/resources/dataset/dataset.controller.js @@ -1,5 +1,8 @@ -export default class DatasetController { +import Controller from '../base/controller'; + +export default class DatasetController extends Controller { constructor(datasetService) { + super(datasetService); this.datasetService = datasetService; } @@ -11,16 +14,11 @@ export default class DatasetController { if (!id) { return res.status(400).json({ success: false, - message: 'You must provide a dataset version id or a dataset persistent id', + message: 'You must provide a dataset identifier', }); } // Find the dataset - let dataset = {}; - if(req.params.expanded) { - dataset = await this.datasetService.getDatasetExpanded(); - } else { - dataset = await this.datasetService.getDataset(); - } + let dataset = await this.datasetService.getDataset(id, req.query); // Return if no dataset found if (!dataset) { return res.status(404).json({ @@ -35,6 +33,7 @@ export default class DatasetController { }); } catch (err) { // Return error response if something goes wrong + console.error(err); return res.status(500).json({ success: false, message: 'A server error occurred, please try again', @@ -44,16 +43,8 @@ export default class DatasetController { async getDatasets(req, res) { try { - // Parse filter options from query params - // TODO - // Find the datasets - let datasets = []; - if(req.params.expanded) { - datasets = await this.datasetService.getDatasetsExpanded(); - } else { - datasets = await this.datasetService.getDatasets(); - } + let datasets = await this.datasetService.getDatasets(req.query); // Return the datasets return res.status(200).json({ success: true, @@ -61,6 +52,7 @@ export default class DatasetController { }); } catch (err) { // Return error response if something goes wrong + console.error(err); return res.status(500).json({ success: false, message: 'A server error occurred, please try again', diff --git a/src/resources/dataset/dataset.entity.js b/src/resources/dataset/dataset.entity.js index b623696c..30ca0efa 100644 --- a/src/resources/dataset/dataset.entity.js +++ b/src/resources/dataset/dataset.entity.js @@ -46,7 +46,9 @@ export default class DatasetClass extends Entity { this.datasetv2 = datasetv2; } - isLatestVersion() { + checkLatestVersion() { return this.activeflag === 'active'; } + + } diff --git a/src/resources/dataset/dataset.model.js b/src/resources/dataset/dataset.model.js index de274f2d..0affd522 100644 --- a/src/resources/dataset/dataset.model.js +++ b/src/resources/dataset/dataset.model.js @@ -67,6 +67,7 @@ const datasetSchema = new Schema( phenotypes: [], }, datasetv2: {}, + isLatestVersion: Boolean }, { timestamps: true, @@ -76,6 +77,7 @@ const datasetSchema = new Schema( } ); +// Virtuals datasetSchema.virtual('publisher', { ref: 'Publisher', foreignField: 'name', @@ -105,13 +107,19 @@ datasetSchema.virtual('submittedDataAccessRequests', { match: { applicationStatus: { $in: ['submitted', 'approved', 'inReview', 'rejected', 'approved with conditions'] }, }, - justOne: false, + justOne: true, }); +// Pre hook query middleware +datasetSchema.pre('find', function() { + this.where({type: 'dataset'}); +}); -// TODO Add virtual for Related Objects connected to this dataset +datasetSchema.pre('findOne', function() { + this.where({type: 'dataset'}); +}); +// Load entity class datasetSchema.loadClass(DatasetClass); -export const Dataset = model('Dataset', datasetSchema, 'tools'); -export const type = 'dataset'; \ No newline at end of file +export const Dataset = model('Dataset', datasetSchema, 'tools'); \ No newline at end of file diff --git a/src/resources/dataset/dataset.repository.js b/src/resources/dataset/dataset.repository.js index 0d5bd780..74007e22 100644 --- a/src/resources/dataset/dataset.repository.js +++ b/src/resources/dataset/dataset.repository.js @@ -1,5 +1,5 @@ import Repository from '../base/repository'; -import { Dataset, type } from './dataset.model'; +import { Dataset } from './dataset.model'; export default class DatasetRepository extends Repository { constructor() { @@ -7,19 +7,14 @@ export default class DatasetRepository extends Repository { this.dataset = Dataset; } - async getDataset(id) { - return this.findOne({ type, datasetid: id }, { lean: true, populate: { path: 'submittedDataAccessRequests' } }); + async getDataset(id, query) { + query = { ...query, datasetid: id }; + const options = { lean: true, populate: { path: 'submittedDataAccessRequests' } }; + return this.findOne(query, options); } - async getDatasetExpanded(id) { - return this.findOne({ type, datasetid: id }, { lean: true, populate: { path: 'submittedDataAccessRequests' } }); - } - - async getDatasets() { - return this.find({ type }, { lean: true, populate: { path: 'submittedDataAccessRequests' } }); - } - - async getDatasetsExpanded() { - return this.find({ type }, { lean: true, populate: { path: 'submittedDataAccessRequests' } }); + async getDatasets(query) { + const options = { lean: true, populate: { path: 'submittedDataAccessRequests' } }; + return this.find(query, options); } } diff --git a/src/resources/dataset/dataset.service.js b/src/resources/dataset/dataset.service.js index 928f3d4e..427d350d 100644 --- a/src/resources/dataset/dataset.service.js +++ b/src/resources/dataset/dataset.service.js @@ -1,981 +1,19 @@ -import axios from 'axios'; -import * as Sentry from '@sentry/node'; -import { v4 as uuidv4 } from 'uuid'; - -import { Data } from '../tool/data.model'; -import { MetricsData } from '../stats/metrics.model'; - -export async function loadDataset(datasetID) { - var metadataCatalogueLink = process.env.metadataURL || 'https://metadata-catalogue.org/hdruk'; - const datasetCall = axios - .get(metadataCatalogueLink + '/api/facets/' + datasetID + '/profile/uk.ac.hdrukgateway/HdrUkProfilePluginService', { timeout: 5000 }) - .catch(err => { - console.log('Unable to get dataset details ' + err.message); - }); - const metadataQualityCall = axios - .get('https://raw.githubusercontent.com/HDRUK/datasets/master/reports/metadata_quality.json', { timeout: 5000 }) - .catch(err => { - console.log('Unable to get metadata quality value ' + err.message); - }); - const metadataSchemaCall = axios - .get(metadataCatalogueLink + '/api/profiles/uk.ac.hdrukgateway/HdrUkProfilePluginService/schema.org/' + datasetID, { timeout: 5000 }) - .catch(err => { - console.log('Unable to get metadata schema ' + err.message); - }); - const dataClassCall = axios.get(metadataCatalogueLink + '/api/dataModels/' + datasetID + '/dataClasses', { timeout: 5000 }).catch(err => { - console.log('Unable to get dataclass ' + err.message); - }); - const versionLinksCall = axios - .get(metadataCatalogueLink + '/api/catalogueItems/' + datasetID + '/semanticLinks', { timeout: 5000 }) - .catch(err => { - console.log('Unable to get version links ' + err.message); - }); - const phenotypesCall = await axios - .get('https://raw.githubusercontent.com/spiros/hdr-caliber-phenome-portal/master/_data/dataset2phenotypes.json', { timeout: 5000 }) - .catch(err => { - console.log('Unable to get phenotypes ' + err.message); - }); - const dataUtilityCall = await axios - .get('https://raw.githubusercontent.com/HDRUK/datasets/master/reports/data_utility.json', { timeout: 5000 }) - .catch(err => { - console.log('Unable to get data utility ' + err.message); - }); - const datasetV2Call = axios - .get(metadataCatalogueLink + '/api/facets/' + datasetID + '/metadata?all=true', { timeout: 5000 }) - .catch(err => { - console.log('Unable to get dataset version 2 ' + err.message); - }); - const [ - dataset, - metadataQualityList, - metadataSchema, - dataClass, - versionLinks, - phenotypesList, - dataUtilityList, - datasetV2, - ] = await axios.all([ - datasetCall, - metadataQualityCall, - metadataSchemaCall, - dataClassCall, - versionLinksCall, - phenotypesCall, - dataUtilityCall, - datasetV2Call, - ]); - - var technicaldetails = []; - - await dataClass.data.items.reduce( - (p, dataclassMDC) => - p.then( - () => - new Promise(resolve => { - setTimeout(async function () { - const dataClassElementCall = axios - .get(metadataCatalogueLink + '/api/dataModels/' + datasetID + '/dataClasses/' + dataclassMDC.id + '/dataElements', { - timeout: 5000, - }) - .catch(err => { - console.log('Unable to get dataclass element ' + err.message); - }); - const [dataClassElement] = await axios.all([dataClassElementCall]); - var dataClassElementArray = []; - - dataClassElement.data.items.forEach(element => { - dataClassElementArray.push({ - id: element.id, - domainType: element.domainType, - label: element.label, - description: element.description, - dataType: { - id: element.dataType.id, - domainType: element.dataType.domainType, - label: element.dataType.label, - }, - }); - }); - - technicaldetails.push({ - id: dataclassMDC.id, - domainType: dataclassMDC.domainType, - label: dataclassMDC.label, - description: dataclassMDC.description, - elements: dataClassElementArray, - }); - - resolve(null); - }, 500); - }) - ), - Promise.resolve(null) - ); - - let datasetv2Object = populateV2datasetObject(datasetV2.data.items); - - let uuid = uuidv4(); - let listOfVersions = []; - let pid = uuid; - let datasetVersion = '0.0.1'; - - if (versionLinks && versionLinks.data && versionLinks.data.items && versionLinks.data.items.length > 0) { - versionLinks.data.items.forEach(item => { - if (!listOfVersions.find(x => x.id === item.source.id)) { - listOfVersions.push({ id: item.source.id, version: item.source.documentationVersion }); - } - if (!listOfVersions.find(x => x.id === item.target.id)) { - listOfVersions.push({ id: item.target.id, version: item.target.documentationVersion }); - } - }); - - for (const item of listOfVersions) { - if (item.id !== dataset.data.id) { - let existingDataset = await Data.findOne({ datasetid: item.id }); - if (existingDataset && existingDataset.pid) pid = existingDataset.pid; - else { - await Data.findOneAndUpdate({ datasetid: item.id }, { pid: uuid, datasetVersion: item.version }); - } - } else { - datasetVersion = item.version; - } - } - } - - var uniqueID = ''; - while (uniqueID === '') { - uniqueID = parseInt(Math.random().toString().replace('0.', '')); - if ((await Data.find({ id: uniqueID }).length) === 0) { - uniqueID = ''; - } - } - - var keywordArray = splitString(dataset.data.keywords); - var physicalSampleAvailabilityArray = splitString(dataset.data.physicalSampleAvailability); - var geographicCoverageArray = splitString(dataset.data.geographicCoverage); - - const metadataQuality = metadataQualityList.data.find(x => x.id === datasetID); - const phenotypes = phenotypesList.data[datasetID] || []; - const dataUtility = dataUtilityList.data.find(x => x.id === datasetID); - - var data = new Data(); - data.pid = pid; - data.datasetVersion = datasetVersion; - data.id = uniqueID; - data.datasetid = dataset.data.id; - data.type = 'dataset'; - data.activeflag = 'archive'; - - data.name = dataset.data.title; - data.description = dataset.data.description; - data.license = dataset.data.license; - data.tags.features = keywordArray; - data.datasetfields.publisher = dataset.data.publisher; - data.datasetfields.geographicCoverage = geographicCoverageArray; - data.datasetfields.physicalSampleAvailability = physicalSampleAvailabilityArray; - data.datasetfields.abstract = dataset.data.abstract; - data.datasetfields.releaseDate = dataset.data.releaseDate; - data.datasetfields.accessRequestDuration = dataset.data.accessRequestDuration; - data.datasetfields.conformsTo = dataset.data.conformsTo; - data.datasetfields.accessRights = dataset.data.accessRights; - data.datasetfields.jurisdiction = dataset.data.jurisdiction; - data.datasetfields.datasetStartDate = dataset.data.datasetStartDate; - data.datasetfields.datasetEndDate = dataset.data.datasetEndDate; - data.datasetfields.statisticalPopulation = dataset.data.statisticalPopulation; - data.datasetfields.ageBand = dataset.data.ageBand; - data.datasetfields.contactPoint = dataset.data.contactPoint; - data.datasetfields.periodicity = dataset.data.periodicity; - - data.datasetfields.metadataquality = metadataQuality ? metadataQuality : {}; - data.datasetfields.metadataschema = metadataSchema && metadataSchema.data ? metadataSchema.data : {}; - data.datasetfields.technicaldetails = technicaldetails; - data.datasetfields.versionLinks = versionLinks && versionLinks.data && versionLinks.data.items ? versionLinks.data.items : []; - data.datasetfields.phenotypes = phenotypes; - data.datasetfields.datautility = dataUtility ? dataUtility : {}; - data.datasetv2 = datasetv2Object; - - return await data.save(); -} - -export async function loadDatasets(override) { - console.log('Starting run at ' + Date()); - var metadataCatalogueLink = process.env.metadataURL || 'https://metadata-catalogue.org/hdruk'; - - var datasetsMDCCount = await new Promise(function (resolve, reject) { - axios - .post( - metadataCatalogueLink + - '/api/profiles/uk.ac.hdrukgateway/HdrUkProfilePluginService/customSearch?searchTerm=&domainType=DataModel&limit=1' - ) - .then(function (response) { - resolve(response.data.count); - }) - .catch(err => { - Sentry.addBreadcrumb({ - category: 'Caching', - message: 'The caching run has failed because it was unable to get a count from the MDC', - level: Sentry.Severity.Fatal, - }); - Sentry.captureException(err); - reject(err); - }); - }).catch(() => { - return 'Update failed'; - }); - - if (datasetsMDCCount === 'Update failed') return; - - //Compare counts from HDR and MDC, if greater drop of 10%+ then stop process and email support queue - var datasetsHDRCount = await Data.countDocuments({ type: 'dataset', activeflag: 'active' }); - - if ((datasetsMDCCount / datasetsHDRCount) * 100 < 90 && !override) { - Sentry.addBreadcrumb({ - category: 'Caching', - message: `The caching run has failed because the counts from the MDC (${datasetsMDCCount}) where ${ - 100 - (datasetsMDCCount / datasetsHDRCount) * 100 - }% lower than the number stored in the DB (${datasetsHDRCount})`, - level: Sentry.Severity.Fatal, - }); - Sentry.captureException(); - return; - } - - //datasetsMDCCount = 10; //For testing to limit the number brought down - - var datasetsMDCList = await new Promise(function (resolve, reject) { - axios - .post( - metadataCatalogueLink + - '/api/profiles/uk.ac.hdrukgateway/HdrUkProfilePluginService/customSearch?searchTerm=&domainType=DataModel&limit=' + - datasetsMDCCount - ) - .then(function (response) { - resolve(response.data); - }) - .catch(err => { - Sentry.addBreadcrumb({ - category: 'Caching', - message: 'The caching run has failed because it was unable to pull the datasets from the MDC', - level: Sentry.Severity.Fatal, - }); - Sentry.captureException(err); - reject(err); - }); - }).catch(() => { - return 'Update failed'; - }); - - if (datasetsMDCList === 'Update failed') return; - - const metadataQualityList = await axios - .get('https://raw.githubusercontent.com/HDRUK/datasets/master/reports/metadata_quality.json', { timeout: 10000 }) - .catch(err => { - Sentry.addBreadcrumb({ - category: 'Caching', - message: 'Unable to get metadata quality value ' + err.message, - level: Sentry.Severity.Error, - }); - Sentry.captureException(err); - //console.log("Unable to get metadata quality value " + err.message); //Uncomment for local testing - }); - - const phenotypesList = await axios - .get('https://raw.githubusercontent.com/spiros/hdr-caliber-phenome-portal/master/_data/dataset2phenotypes.json', { timeout: 10000 }) - .catch(err => { - Sentry.addBreadcrumb({ - category: 'Caching', - message: 'Unable to get metadata quality value ' + err.message, - level: Sentry.Severity.Error, - }); - Sentry.captureException(err); - //console.log("Unable to get metadata quality value " + err.message); //Uncomment for local testing - }); - - const dataUtilityList = await axios - .get('https://raw.githubusercontent.com/HDRUK/datasets/master/reports/data_utility.json', { timeout: 10000 }) - .catch(err => { - Sentry.addBreadcrumb({ - category: 'Caching', - message: 'Unable to get data utility ' + err.message, - level: Sentry.Severity.Error, - }); - Sentry.captureException(err); - //console.log("Unable to get data utility " + err.message); //Uncomment for local testing - }); - - var datasetsMDCIDs = []; - var counter = 0; - - await datasetsMDCList.results.reduce( - (p, datasetMDC) => - p.then( - () => - new Promise(resolve => { - setTimeout(async function () { - try { - counter++; - var datasetHDR = await Data.findOne({ datasetid: datasetMDC.id }); - datasetsMDCIDs.push({ datasetid: datasetMDC.id }); - - const metadataQuality = metadataQualityList.data.find(x => x.id === datasetMDC.id); - const dataUtility = dataUtilityList.data.find(x => x.id === datasetMDC.id); - const phenotypes = phenotypesList.data[datasetMDC.id] || []; - - const metadataSchemaCall = axios - .get(metadataCatalogueLink + '/api/profiles/uk.ac.hdrukgateway/HdrUkProfilePluginService/schema.org/' + datasetMDC.id, { - timeout: 10000, - }) - .catch(err => { - Sentry.addBreadcrumb({ - category: 'Caching', - message: 'Unable to get metadata schema ' + err.message, - level: Sentry.Severity.Error, - }); - Sentry.captureException(err); - //console.log('Unable to get metadata schema ' + err.message); - }); - - const dataClassCall = axios - .get(metadataCatalogueLink + '/api/dataModels/' + datasetMDC.id + '/dataClasses?max=300', { timeout: 10000 }) - .catch(err => { - Sentry.addBreadcrumb({ - category: 'Caching', - message: 'Unable to get dataclass ' + err.message, - level: Sentry.Severity.Error, - }); - Sentry.captureException(err); - //console.log('Unable to get dataclass ' + err.message); - }); - - const versionLinksCall = axios - .get(metadataCatalogueLink + '/api/catalogueItems/' + datasetMDC.id + '/semanticLinks', { timeout: 10000 }) - .catch(err => { - Sentry.addBreadcrumb({ - category: 'Caching', - message: 'Unable to get version links ' + err.message, - level: Sentry.Severity.Error, - }); - Sentry.captureException(err); - //console.log('Unable to get version links ' + err.message); - }); - - const datasetV2Call = axios - .get(metadataCatalogueLink + '/api/facets/' + datasetMDC.id + '/metadata?all=true', { timeout: 5000 }) - .catch(err => { - Sentry.addBreadcrumb({ - category: 'Caching', - message: 'Unable to get dataset version 2 ' + err.message, - level: Sentry.Severity.Error, - }); - Sentry.captureException(err); - //console.log('Unable to get dataset version 2 ' + err.message); - }); - - const [metadataSchema, dataClass, versionLinks, datasetV2] = await axios.all([ - metadataSchemaCall, - dataClassCall, - versionLinksCall, - datasetV2Call, - ]); - - var technicaldetails = []; - - await dataClass.data.items.reduce( - (p, dataclassMDC) => - p.then( - () => - new Promise(resolve => { - setTimeout(async function () { - const dataClassElementCall = axios - .get( - metadataCatalogueLink + - '/api/dataModels/' + - datasetMDC.id + - '/dataClasses/' + - dataclassMDC.id + - '/dataElements?max=300', - { timeout: 5000 } - ) - .catch(err => { - console.log('Unable to get dataclass element ' + err.message); - }); - const [dataClassElement] = await axios.all([dataClassElementCall]); - var dataClassElementArray = []; - - dataClassElement.data.items.forEach(element => { - dataClassElementArray.push({ - id: element.id, - domainType: element.domainType, - label: element.label, - description: element.description, - dataType: { - id: element.dataType.id, - domainType: element.dataType.domainType, - label: element.dataType.label, - }, - }); - }); - - technicaldetails.push({ - id: dataclassMDC.id, - domainType: dataclassMDC.domainType, - label: dataclassMDC.label, - description: dataclassMDC.description, - elements: dataClassElementArray, - }); - - resolve(null); - }, 500); - }) - ), - Promise.resolve(null) - ); - - let datasetv2Object = populateV2datasetObject(datasetV2.data.items); - - if (datasetHDR) { - //Edit - if (!datasetHDR.pid) { - let uuid = uuidv4(); - let listOfVersions = []; - datasetHDR.pid = uuid; - datasetHDR.datasetVersion = '0.0.1'; - - if (versionLinks && versionLinks.data && versionLinks.data.items && versionLinks.data.items.length > 0) { - versionLinks.data.items.forEach(item => { - if (!listOfVersions.find(x => x.id === item.source.id)) { - listOfVersions.push({ id: item.source.id, version: item.source.documentationVersion }); - } - if (!listOfVersions.find(x => x.id === item.target.id)) { - listOfVersions.push({ id: item.target.id, version: item.target.documentationVersion }); - } - }); - - listOfVersions.forEach(async item => { - if (item.id !== datasetMDC.id) { - await Data.findOneAndUpdate({ datasetid: item.id }, { pid: uuid, datasetVersion: item.version }); - } else { - datasetHDR.pid = uuid; - datasetHDR.datasetVersion = item.version; - } - }); - } - } - - let keywordArray = splitString(datasetMDC.keywords); - let physicalSampleAvailabilityArray = splitString(datasetMDC.physicalSampleAvailability); - let geographicCoverageArray = splitString(datasetMDC.geographicCoverage); - - await Data.findOneAndUpdate( - { datasetid: datasetMDC.id }, - { - pid: datasetHDR.pid, - datasetVersion: datasetHDR.datasetVersion, - name: datasetMDC.title, - description: datasetMDC.description, - activeflag: 'active', - license: datasetMDC.license, - tags: { - features: keywordArray, - }, - datasetfields: { - publisher: datasetMDC.publisher, - geographicCoverage: geographicCoverageArray, - physicalSampleAvailability: physicalSampleAvailabilityArray, - abstract: datasetMDC.abstract, - releaseDate: datasetMDC.releaseDate, - accessRequestDuration: datasetMDC.accessRequestDuration, - conformsTo: datasetMDC.conformsTo, - accessRights: datasetMDC.accessRights, - jurisdiction: datasetMDC.jurisdiction, - datasetStartDate: datasetMDC.datasetStartDate, - datasetEndDate: datasetMDC.datasetEndDate, - statisticalPopulation: datasetMDC.statisticalPopulation, - ageBand: datasetMDC.ageBand, - contactPoint: datasetMDC.contactPoint, - periodicity: datasetMDC.periodicity, - - metadataquality: metadataQuality ? metadataQuality : {}, - datautility: dataUtility ? dataUtility : {}, - metadataschema: metadataSchema && metadataSchema.data ? metadataSchema.data : {}, - technicaldetails: technicaldetails, - versionLinks: versionLinks && versionLinks.data && versionLinks.data.items ? versionLinks.data.items : [], - phenotypes, - }, - datasetv2: datasetv2Object, - } - ); - } else { - //Add - let uuid = uuidv4(); - let listOfVersions = []; - let pid = uuid; - let datasetVersion = '0.0.1'; - - if (versionLinks && versionLinks.data && versionLinks.data.items && versionLinks.data.items.length > 0) { - versionLinks.data.items.forEach(item => { - if (!listOfVersions.find(x => x.id === item.source.id)) { - listOfVersions.push({ id: item.source.id, version: item.source.documentationVersion }); - } - if (!listOfVersions.find(x => x.id === item.target.id)) { - listOfVersions.push({ id: item.target.id, version: item.target.documentationVersion }); - } - }); - - for (const item of listOfVersions) { - if (item.id !== datasetMDC.id) { - var existingDataset = await Data.findOne({ datasetid: item.id }); - if (existingDataset && existingDataset.pid) pid = existingDataset.pid; - else { - await Data.findOneAndUpdate({ datasetid: item.id }, { pid: uuid, datasetVersion: item.version }); - } - } else { - datasetVersion = item.version; - } - } - } - - var uniqueID = ''; - while (uniqueID === '') { - uniqueID = parseInt(Math.random().toString().replace('0.', '')); - if ((await Data.find({ id: uniqueID }).length) === 0) { - uniqueID = ''; - } - } - - var keywordArray = splitString(datasetMDC.keywords); - var physicalSampleAvailabilityArray = splitString(datasetMDC.physicalSampleAvailability); - var geographicCoverageArray = splitString(datasetMDC.geographicCoverage); - - var data = new Data(); - data.pid = pid; - data.datasetVersion = datasetVersion; - data.id = uniqueID; - data.datasetid = datasetMDC.id; - data.type = 'dataset'; - data.activeflag = 'active'; - - data.name = datasetMDC.title; - data.description = datasetMDC.description; - data.license = datasetMDC.license; - data.tags.features = keywordArray; - data.datasetfields.publisher = datasetMDC.publisher; - data.datasetfields.geographicCoverage = geographicCoverageArray; - data.datasetfields.physicalSampleAvailability = physicalSampleAvailabilityArray; - data.datasetfields.abstract = datasetMDC.abstract; - data.datasetfields.releaseDate = datasetMDC.releaseDate; - data.datasetfields.accessRequestDuration = datasetMDC.accessRequestDuration; - data.datasetfields.conformsTo = datasetMDC.conformsTo; - data.datasetfields.accessRights = datasetMDC.accessRights; - data.datasetfields.jurisdiction = datasetMDC.jurisdiction; - data.datasetfields.datasetStartDate = datasetMDC.datasetStartDate; - data.datasetfields.datasetEndDate = datasetMDC.datasetEndDate; - data.datasetfields.statisticalPopulation = datasetMDC.statisticalPopulation; - data.datasetfields.ageBand = datasetMDC.ageBand; - data.datasetfields.contactPoint = datasetMDC.contactPoint; - data.datasetfields.periodicity = datasetMDC.periodicity; - - data.datasetfields.metadataquality = metadataQuality ? metadataQuality : {}; - data.datasetfields.datautility = dataUtility ? dataUtility : {}; - data.datasetfields.metadataschema = metadataSchema && metadataSchema.data ? metadataSchema.data : {}; - data.datasetfields.technicaldetails = technicaldetails; - data.datasetfields.versionLinks = - versionLinks && versionLinks.data && versionLinks.data.items ? versionLinks.data.items : []; - data.datasetfields.phenotypes = phenotypes; - data.datasetv2 = datasetv2Object; - await data.save(); - } - console.log(`Finished ${counter} of ${datasetsMDCCount} datasets (${datasetMDC.id})`); - resolve(null); - } catch (err) { - Sentry.addBreadcrumb({ - category: 'Caching', - message: `Failed to add ${datasetMDC.id} to the DB with the error of ${err.message}`, - level: Sentry.Severity.Fatal, - }); - Sentry.captureException(err); - //console.log(`Failed to add ${datasetMDC.id} to the DB with the error of ${err.message}`); //Uncomment for local testing - } - }, 500); - }) - ), - Promise.resolve(null) - ); - - var datasetsHDRIDs = await Data.aggregate([{ $match: { type: 'dataset' } }, { $project: { _id: 0, datasetid: 1 } }]); - - let datasetsNotFound = datasetsHDRIDs.filter(o1 => !datasetsMDCIDs.some(o2 => o1.datasetid === o2.datasetid)); - - await Promise.all( - datasetsNotFound.map(async dataset => { - //Archive - await Data.findOneAndUpdate( - { datasetid: dataset.datasetid }, - { - activeflag: 'archive', - } - ); - }) - ); - - saveUptime(); - - console.log('Update Completed at ' + Date()); - return; -} - -function populateV2datasetObject(v2Data) { - let datasetV2List = v2Data.filter(item => item.namespace === 'org.healthdatagateway'); - - let datasetv2Object = {}; - if (datasetV2List.length > 0) { - datasetv2Object = { - identifier: datasetV2List.find(x => x.key === 'properties/identifier') - ? datasetV2List.find(x => x.key === 'properties/identifier').value - : '', - version: datasetV2List.find(x => x.key === 'properties/version') ? datasetV2List.find(x => x.key === 'properties/version').value : '', - issued: datasetV2List.find(x => x.key === 'properties/issued') ? datasetV2List.find(x => x.key === 'properties/issued').value : '', - modified: datasetV2List.find(x => x.key === 'properties/modified') - ? datasetV2List.find(x => x.key === 'properties/modified').value - : '', - revisions: [], - summary: { - title: datasetV2List.find(x => x.key === 'properties/summary/title') - ? datasetV2List.find(x => x.key === 'properties/summary/title').value - : '', - abstract: datasetV2List.find(x => x.key === 'properties/summary/abstract') - ? datasetV2List.find(x => x.key === 'properties/summary/abstract').value - : '', - publisher: { - identifier: datasetV2List.find(x => x.key === 'properties/summary/publisher/identifier') - ? datasetV2List.find(x => x.key === 'properties/summary/publisher/identifier').value - : '', - name: datasetV2List.find(x => x.key === 'properties/summary/publisher/name') - ? datasetV2List.find(x => x.key === 'properties/summary/publisher/name').value - : '', - logo: datasetV2List.find(x => x.key === 'properties/summary/publisher/logo') - ? datasetV2List.find(x => x.key === 'properties/summary/publisher/logo').value - : '', - description: datasetV2List.find(x => x.key === 'properties/summary/publisher/description') - ? datasetV2List.find(x => x.key === 'properties/summary/publisher/description').value - : '', - contactPoint: checkForArray( - datasetV2List.find(x => x.key === 'properties/summary/publisher/contactPoint') - ? datasetV2List.find(x => x.key === 'properties/summary/publisher/contactPoint').value - : [] - ), - memberOf: datasetV2List.find(x => x.key === 'properties/summary/publisher/memberOf') - ? datasetV2List.find(x => x.key === 'properties/summary/publisher/memberOf').value - : '', - accessRights: checkForArray( - datasetV2List.find(x => x.key === 'properties/summary/publisher/accessRights') - ? datasetV2List.find(x => x.key === 'properties/summary/publisher/accessRights').value - : [] - ), - deliveryLeadTime: datasetV2List.find(x => x.key === 'properties/summary/publisher/deliveryLeadTime') - ? datasetV2List.find(x => x.key === 'properties/summary/publisher/deliveryLeadTime').value - : '', - accessService: datasetV2List.find(x => x.key === 'properties/summary/publisher/accessService') - ? datasetV2List.find(x => x.key === 'properties/summary/publisher/accessService').value - : '', - accessRequestCost: datasetV2List.find(x => x.key === 'properties/summary/publisher/accessRequestCost') - ? datasetV2List.find(x => x.key === 'properties/summary/publisher/accessRequestCost').value - : '', - dataUseLimitation: checkForArray( - datasetV2List.find(x => x.key === 'properties/summary/publisher/dataUseLimitation') - ? datasetV2List.find(x => x.key === 'properties/summary/publisher/dataUseLimitation').value - : [] - ), - dataUseRequirements: checkForArray( - datasetV2List.find(x => x.key === 'properties/summary/publisher/dataUseRequirements') - ? datasetV2List.find(x => x.key === 'properties/summary/publisher/dataUseRequirements').value - : [] - ), - }, - contactPoint: datasetV2List.find(x => x.key === 'properties/summary/contactPoint') - ? datasetV2List.find(x => x.key === 'properties/summary/contactPoint').value - : '', - keywords: checkForArray( - datasetV2List.find(x => x.key === 'properties/summary/keywords') - ? datasetV2List.find(x => x.key === 'properties/summary/keywords').value - : [] - ), - alternateIdentifiers: checkForArray( - datasetV2List.find(x => x.key === 'properties/summary/alternateIdentifiers') - ? datasetV2List.find(x => x.key === 'properties/summary/alternateIdentifiers').value - : [] - ), - doiName: datasetV2List.find(x => x.key === 'properties/summary/doiName') - ? datasetV2List.find(x => x.key === 'properties/summary/doiName').value - : '', - }, - documentation: { - description: datasetV2List.find(x => x.key === 'properties/documentation/description') - ? datasetV2List.find(x => x.key === 'properties/documentation/description').value - : '', - associatedMedia: checkForArray( - datasetV2List.find(x => x.key === 'properties/documentation/associatedMedia') - ? datasetV2List.find(x => x.key === 'properties/documentation/associatedMedia').value - : [] - ), - isPartOf: checkForArray( - datasetV2List.find(x => x.key === 'properties/documentation/isPartOf') - ? datasetV2List.find(x => x.key === 'properties/documentation/isPartOf').value - : [] - ), - }, - coverage: { - spatial: datasetV2List.find(x => x.key === 'properties/coverage/spatial') - ? datasetV2List.find(x => x.key === 'properties/coverage/spatial').value - : '', - typicalAgeRange: datasetV2List.find(x => x.key === 'properties/coverage/typicalAgeRange') - ? datasetV2List.find(x => x.key === 'properties/coverage/typicalAgeRange').value - : '', - physicalSampleAvailability: checkForArray( - datasetV2List.find(x => x.key === 'properties/coverage/physicalSampleAvailability') - ? datasetV2List.find(x => x.key === 'properties/coverage/physicalSampleAvailability').value - : [] - ), - followup: datasetV2List.find(x => x.key === 'properties/coverage/followup') - ? datasetV2List.find(x => x.key === 'properties/coverage/followup').value - : '', - pathway: datasetV2List.find(x => x.key === 'properties/coverage/pathway') - ? datasetV2List.find(x => x.key === 'properties/coverage/pathway').value - : '', - }, - provenance: { - origin: { - purpose: checkForArray( - datasetV2List.find(x => x.key === 'properties/provenance/origin/purpose') - ? datasetV2List.find(x => x.key === 'properties/provenance/origin/purpose').value - : [] - ), - source: checkForArray( - datasetV2List.find(x => x.key === 'properties/provenance/origin/source') - ? datasetV2List.find(x => x.key === 'properties/provenance/origin/source').value - : [] - ), - collectionSituation: checkForArray( - datasetV2List.find(x => x.key === 'properties/provenance/origin/collectionSituation') - ? datasetV2List.find(x => x.key === 'properties/provenance/origin/collectionSituation').value - : [] - ), - }, - temporal: { - accrualPeriodicity: datasetV2List.find(x => x.key === 'properties/provenance/temporal/accrualPeriodicity') - ? datasetV2List.find(x => x.key === 'properties/provenance/temporal/accrualPeriodicity').value - : '', - distributionReleaseDate: datasetV2List.find(x => x.key === 'properties/provenance/temporal/distributionReleaseDate') - ? datasetV2List.find(x => x.key === 'properties/provenance/temporal/distributionReleaseDate').value - : '', - startDate: datasetV2List.find(x => x.key === 'properties/provenance/temporal/startDate') - ? datasetV2List.find(x => x.key === 'properties/provenance/temporal/startDate').value - : '', - endDate: datasetV2List.find(x => x.key === 'properties/provenance/temporal/endDate') - ? datasetV2List.find(x => x.key === 'properties/provenance/temporal/endDate').value - : '', - timeLag: datasetV2List.find(x => x.key === 'properties/provenance/temporal/timeLag') - ? datasetV2List.find(x => x.key === 'properties/provenance/temporal/timeLag').value - : '', - }, - }, - accessibility: { - usage: { - dataUseLimitation: checkForArray( - datasetV2List.find(x => x.key === 'properties/accessibility/usage/dataUseLimitation') - ? datasetV2List.find(x => x.key === 'properties/accessibility/usage/dataUseLimitation').value - : [] - ), - dataUseRequirements: checkForArray( - datasetV2List.find(x => x.key === 'properties/accessibility/usage/dataUseRequirements') - ? datasetV2List.find(x => x.key === 'properties/accessibility/usage/dataUseRequirements').value - : [] - ), - resourceCreator: datasetV2List.find(x => x.key === 'properties/accessibility/usage/resourceCreator') - ? datasetV2List.find(x => x.key === 'properties/accessibility/usage/resourceCreator').value - : '', - investigations: checkForArray( - datasetV2List.find(x => x.key === 'properties/accessibility/usage/investigations') - ? datasetV2List.find(x => x.key === 'properties/accessibility/usage/investigations').value - : [] - ), - isReferencedBy: checkForArray( - datasetV2List.find(x => x.key === 'properties/accessibility/usage/isReferencedBy') - ? datasetV2List.find(x => x.key === 'properties/accessibility/usage/isReferencedBy').value - : [] - ), - }, - access: { - accessRights: checkForArray( - datasetV2List.find(x => x.key === 'properties/accessibility/access/accessRights') - ? datasetV2List.find(x => x.key === 'properties/accessibility/access/accessRights').value - : [] - ), - accessService: datasetV2List.find(x => x.key === 'properties/accessibility/access/accessService') - ? datasetV2List.find(x => x.key === 'properties/accessibility/access/accessService').value - : '', - accessRequestCost: datasetV2List.find(x => x.key === 'properties/accessibility/access/accessRequestCost') - ? datasetV2List.find(x => x.key === 'properties/accessibility/access/accessRequestCost').value - : '', - deliveryLeadTime: datasetV2List.find(x => x.key === 'properties/accessibility/access/deliveryLeadTime') - ? datasetV2List.find(x => x.key === 'properties/accessibility/access/deliveryLeadTime').value - : '', - jurisdiction: checkForArray( - datasetV2List.find(x => x.key === 'properties/accessibility/access/jurisdiction') - ? datasetV2List.find(x => x.key === 'properties/accessibility/access/jurisdiction').value - : [] - ), - dataProcessor: datasetV2List.find(x => x.key === 'properties/accessibility/access/dataProcessor') - ? datasetV2List.find(x => x.key === 'properties/accessibility/access/dataProcessor').value - : '', - dataController: datasetV2List.find(x => x.key === 'properties/accessibility/access/dataController') - ? datasetV2List.find(x => x.key === 'properties/accessibility/access/dataController').value - : '', - }, - formatAndStandards: { - vocabularyEncodingScheme: checkForArray( - datasetV2List.find(x => x.key === 'properties/accessibility/formatAndStandards/vocabularyEncodingScheme') - ? datasetV2List.find(x => x.key === 'properties/accessibility/formatAndStandards/vocabularyEncodingScheme').value - : [] - ), - conformsTo: checkForArray( - datasetV2List.find(x => x.key === 'properties/accessibility/formatAndStandards/conformsTo') - ? datasetV2List.find(x => x.key === 'properties/accessibility/formatAndStandards/conformsTo').value - : [] - ), - language: checkForArray( - datasetV2List.find(x => x.key === 'properties/accessibility/formatAndStandards/language') - ? datasetV2List.find(x => x.key === 'properties/accessibility/formatAndStandards/language').value - : [] - ), - format: checkForArray( - datasetV2List.find(x => x.key === 'properties/accessibility/formatAndStandards/format') - ? datasetV2List.find(x => x.key === 'properties/accessibility/formatAndStandards/format').value - : [] - ), - }, - }, - enrichmentAndLinkage: { - qualifiedRelation: checkForArray( - datasetV2List.find(x => x.key === 'properties/enrichmentAndLinkage/qualifiedRelation') - ? datasetV2List.find(x => x.key === 'properties/enrichmentAndLinkage/qualifiedRelation').value - : [] - ), - derivation: checkForArray( - datasetV2List.find(x => x.key === 'properties/enrichmentAndLinkage/derivation') - ? datasetV2List.find(x => x.key === 'properties/enrichmentAndLinkage/derivation').value - : [] - ), - tools: checkForArray( - datasetV2List.find(x => x.key === 'properties/enrichmentAndLinkage/tools') - ? datasetV2List.find(x => x.key === 'properties/enrichmentAndLinkage/tools').value - : [] - ), - }, - observations: [], - }; - } - - return datasetv2Object; -} - -function checkForArray(value) { - if (typeof value !== 'string') return value; - try { - const type = Object.prototype.toString.call(JSON.parse(value)); - if (type === '[object Object]' || type === '[object Array]') return JSON.parse(value); - } catch (err) { - return value; - } -} - -function splitString(array) { - var returnArray = []; - if (array !== null && array !== '' && array !== 'undefined' && array !== undefined) { - if (array.indexOf(',') === -1) { - returnArray.push(array.trim()); - } else { - array.split(',').forEach(term => { - returnArray.push(term.trim()); - }); - } - } - return returnArray; -} - -async function saveUptime() { - const monitoring = require('@google-cloud/monitoring'); - const projectId = 'hdruk-gateway'; - const client = new monitoring.MetricServiceClient(); - - var selectedMonthStart = new Date(); - selectedMonthStart.setMonth(selectedMonthStart.getMonth() - 1); - selectedMonthStart.setDate(1); - selectedMonthStart.setHours(0, 0, 0, 0); - - var selectedMonthEnd = new Date(); - selectedMonthEnd.setDate(0); - selectedMonthEnd.setHours(23, 59, 59, 999); - - const request = { - name: client.projectPath(projectId), - filter: - 'metric.type="monitoring.googleapis.com/uptime_check/check_passed" AND resource.type="uptime_url" AND metric.label."check_id"="check-production-web-app-qsxe8fXRrBo" AND metric.label."checker_location"="eur-belgium"', - - interval: { - startTime: { - seconds: selectedMonthStart.getTime() / 1000, - }, - endTime: { - seconds: selectedMonthEnd.getTime() / 1000, - }, - }, - aggregation: { - alignmentPeriod: { - seconds: '86400s', - }, - crossSeriesReducer: 'REDUCE_NONE', - groupByFields: ['metric.label."checker_location"', 'resource.label."instance_id"'], - perSeriesAligner: 'ALIGN_FRACTION_TRUE', - }, - }; - - // Writes time series data - const [timeSeries] = await client.listTimeSeries(request); - var dailyUptime = []; - var averageUptime; - - timeSeries.forEach(data => { - data.points.forEach(data => { - dailyUptime.push(data.value.doubleValue); - }); - - averageUptime = (dailyUptime.reduce((a, b) => a + b, 0) / dailyUptime.length) * 100; - }); - - var metricsData = new MetricsData(); - metricsData.uptime = averageUptime; - await metricsData.save(); -} - export default class DatasetService { constructor(datasetRepository) { this.datasetRepository = datasetRepository; } + + async getDataset(id, query = {}) { + let dataset = await this.datasetRepository.getDataset(id, query); - getDataset(id) { - return this.datasetRepository.getDataset(id); - } - - getDatasetExpanded(id) { - return this.datasetRepository.getDatasetExpanded(id); - } + if(dataset && query['expanded'] === true) { + dataset.isLatestVersion = dataset.checkLatestVersion(); + } - getDatasets() { - return this.datasetRepository.getDatasets(); + return dataset; } - getDatasetsExpanded() { - return this.datasetRepository.getDatasetsExpanded(); + getDatasets(query = {}) { + return this.datasetRepository.getDatasets(query); } } diff --git a/src/resources/dataset/v1/dataset.route.js b/src/resources/dataset/v1/dataset.route.js index 095e7f5f..cb9e7c78 100644 --- a/src/resources/dataset/v1/dataset.route.js +++ b/src/resources/dataset/v1/dataset.route.js @@ -1,6 +1,6 @@ import express from 'express'; import { Data } from '../../tool/data.model'; -import { loadDataset, loadDatasets } from '../dataset.service'; +import { loadDataset, loadDatasets } from './dataset.service'; import { getAllTools } from '../../tool/data.repository'; import _ from 'lodash'; import escape from 'escape-html'; diff --git a/src/resources/dataset/v1/dataset.service.js b/src/resources/dataset/v1/dataset.service.js new file mode 100644 index 00000000..77b65bb0 --- /dev/null +++ b/src/resources/dataset/v1/dataset.service.js @@ -0,0 +1,959 @@ +import axios from 'axios'; +import * as Sentry from '@sentry/node'; +import { v4 as uuidv4 } from 'uuid'; + +import { Data } from '../../tool/data.model'; +import { MetricsData } from '../../stats/metrics.model'; + +export async function loadDataset(datasetID) { + var metadataCatalogueLink = process.env.metadataURL || 'https://metadata-catalogue.org/hdruk'; + const datasetCall = axios + .get(metadataCatalogueLink + '/api/facets/' + datasetID + '/profile/uk.ac.hdrukgateway/HdrUkProfilePluginService', { timeout: 5000 }) + .catch(err => { + console.log('Unable to get dataset details ' + err.message); + }); + const metadataQualityCall = axios + .get('https://raw.githubusercontent.com/HDRUK/datasets/master/reports/metadata_quality.json', { timeout: 5000 }) + .catch(err => { + console.log('Unable to get metadata quality value ' + err.message); + }); + const metadataSchemaCall = axios + .get(metadataCatalogueLink + '/api/profiles/uk.ac.hdrukgateway/HdrUkProfilePluginService/schema.org/' + datasetID, { timeout: 5000 }) + .catch(err => { + console.log('Unable to get metadata schema ' + err.message); + }); + const dataClassCall = axios.get(metadataCatalogueLink + '/api/dataModels/' + datasetID + '/dataClasses', { timeout: 5000 }).catch(err => { + console.log('Unable to get dataclass ' + err.message); + }); + const versionLinksCall = axios + .get(metadataCatalogueLink + '/api/catalogueItems/' + datasetID + '/semanticLinks', { timeout: 5000 }) + .catch(err => { + console.log('Unable to get version links ' + err.message); + }); + const phenotypesCall = await axios + .get('https://raw.githubusercontent.com/spiros/hdr-caliber-phenome-portal/master/_data/dataset2phenotypes.json', { timeout: 5000 }) + .catch(err => { + console.log('Unable to get phenotypes ' + err.message); + }); + const dataUtilityCall = await axios + .get('https://raw.githubusercontent.com/HDRUK/datasets/master/reports/data_utility.json', { timeout: 5000 }) + .catch(err => { + console.log('Unable to get data utility ' + err.message); + }); + const datasetV2Call = axios + .get(metadataCatalogueLink + '/api/facets/' + datasetID + '/metadata?all=true', { timeout: 5000 }) + .catch(err => { + console.log('Unable to get dataset version 2 ' + err.message); + }); + const [ + dataset, + metadataQualityList, + metadataSchema, + dataClass, + versionLinks, + phenotypesList, + dataUtilityList, + datasetV2, + ] = await axios.all([ + datasetCall, + metadataQualityCall, + metadataSchemaCall, + dataClassCall, + versionLinksCall, + phenotypesCall, + dataUtilityCall, + datasetV2Call, + ]); + + var technicaldetails = []; + + await dataClass.data.items.reduce( + (p, dataclassMDC) => + p.then( + () => + new Promise(resolve => { + setTimeout(async function () { + const dataClassElementCall = axios + .get(metadataCatalogueLink + '/api/dataModels/' + datasetID + '/dataClasses/' + dataclassMDC.id + '/dataElements', { + timeout: 5000, + }) + .catch(err => { + console.log('Unable to get dataclass element ' + err.message); + }); + const [dataClassElement] = await axios.all([dataClassElementCall]); + var dataClassElementArray = []; + + dataClassElement.data.items.forEach(element => { + dataClassElementArray.push({ + id: element.id, + domainType: element.domainType, + label: element.label, + description: element.description, + dataType: { + id: element.dataType.id, + domainType: element.dataType.domainType, + label: element.dataType.label, + }, + }); + }); + + technicaldetails.push({ + id: dataclassMDC.id, + domainType: dataclassMDC.domainType, + label: dataclassMDC.label, + description: dataclassMDC.description, + elements: dataClassElementArray, + }); + + resolve(null); + }, 500); + }) + ), + Promise.resolve(null) + ); + + let datasetv2Object = populateV2datasetObject(datasetV2.data.items); + + let uuid = uuidv4(); + let listOfVersions = []; + let pid = uuid; + let datasetVersion = '0.0.1'; + + if (versionLinks && versionLinks.data && versionLinks.data.items && versionLinks.data.items.length > 0) { + versionLinks.data.items.forEach(item => { + if (!listOfVersions.find(x => x.id === item.source.id)) { + listOfVersions.push({ id: item.source.id, version: item.source.documentationVersion }); + } + if (!listOfVersions.find(x => x.id === item.target.id)) { + listOfVersions.push({ id: item.target.id, version: item.target.documentationVersion }); + } + }); + + for (const item of listOfVersions) { + if (item.id !== dataset.data.id) { + let existingDataset = await Data.findOne({ datasetid: item.id }); + if (existingDataset && existingDataset.pid) pid = existingDataset.pid; + else { + await Data.findOneAndUpdate({ datasetid: item.id }, { pid: uuid, datasetVersion: item.version }); + } + } else { + datasetVersion = item.version; + } + } + } + + var uniqueID = ''; + while (uniqueID === '') { + uniqueID = parseInt(Math.random().toString().replace('0.', '')); + if ((await Data.find({ id: uniqueID }).length) === 0) { + uniqueID = ''; + } + } + + var keywordArray = splitString(dataset.data.keywords); + var physicalSampleAvailabilityArray = splitString(dataset.data.physicalSampleAvailability); + var geographicCoverageArray = splitString(dataset.data.geographicCoverage); + + const metadataQuality = metadataQualityList.data.find(x => x.id === datasetID); + const phenotypes = phenotypesList.data[datasetID] || []; + const dataUtility = dataUtilityList.data.find(x => x.id === datasetID); + + var data = new Data(); + data.pid = pid; + data.datasetVersion = datasetVersion; + data.id = uniqueID; + data.datasetid = dataset.data.id; + data.type = 'dataset'; + data.activeflag = 'archive'; + + data.name = dataset.data.title; + data.description = dataset.data.description; + data.license = dataset.data.license; + data.tags.features = keywordArray; + data.datasetfields.publisher = dataset.data.publisher; + data.datasetfields.geographicCoverage = geographicCoverageArray; + data.datasetfields.physicalSampleAvailability = physicalSampleAvailabilityArray; + data.datasetfields.abstract = dataset.data.abstract; + data.datasetfields.releaseDate = dataset.data.releaseDate; + data.datasetfields.accessRequestDuration = dataset.data.accessRequestDuration; + data.datasetfields.conformsTo = dataset.data.conformsTo; + data.datasetfields.accessRights = dataset.data.accessRights; + data.datasetfields.jurisdiction = dataset.data.jurisdiction; + data.datasetfields.datasetStartDate = dataset.data.datasetStartDate; + data.datasetfields.datasetEndDate = dataset.data.datasetEndDate; + data.datasetfields.statisticalPopulation = dataset.data.statisticalPopulation; + data.datasetfields.ageBand = dataset.data.ageBand; + data.datasetfields.contactPoint = dataset.data.contactPoint; + data.datasetfields.periodicity = dataset.data.periodicity; + + data.datasetfields.metadataquality = metadataQuality ? metadataQuality : {}; + data.datasetfields.metadataschema = metadataSchema && metadataSchema.data ? metadataSchema.data : {}; + data.datasetfields.technicaldetails = technicaldetails; + data.datasetfields.versionLinks = versionLinks && versionLinks.data && versionLinks.data.items ? versionLinks.data.items : []; + data.datasetfields.phenotypes = phenotypes; + data.datasetfields.datautility = dataUtility ? dataUtility : {}; + data.datasetv2 = datasetv2Object; + + return await data.save(); +} + +export async function loadDatasets(override) { + console.log('Starting run at ' + Date()); + var metadataCatalogueLink = process.env.metadataURL || 'https://metadata-catalogue.org/hdruk'; + + var datasetsMDCCount = await new Promise(function (resolve, reject) { + axios + .post( + metadataCatalogueLink + + '/api/profiles/uk.ac.hdrukgateway/HdrUkProfilePluginService/customSearch?searchTerm=&domainType=DataModel&limit=1' + ) + .then(function (response) { + resolve(response.data.count); + }) + .catch(err => { + Sentry.addBreadcrumb({ + category: 'Caching', + message: 'The caching run has failed because it was unable to get a count from the MDC', + level: Sentry.Severity.Fatal, + }); + Sentry.captureException(err); + reject(err); + }); + }).catch(() => { + return 'Update failed'; + }); + + if (datasetsMDCCount === 'Update failed') return; + + //Compare counts from HDR and MDC, if greater drop of 10%+ then stop process and email support queue + var datasetsHDRCount = await Data.countDocuments({ type: 'dataset', activeflag: 'active' }); + + if ((datasetsMDCCount / datasetsHDRCount) * 100 < 90 && !override) { + Sentry.addBreadcrumb({ + category: 'Caching', + message: `The caching run has failed because the counts from the MDC (${datasetsMDCCount}) where ${ + 100 - (datasetsMDCCount / datasetsHDRCount) * 100 + }% lower than the number stored in the DB (${datasetsHDRCount})`, + level: Sentry.Severity.Fatal, + }); + Sentry.captureException(); + return; + } + + //datasetsMDCCount = 10; //For testing to limit the number brought down + + var datasetsMDCList = await new Promise(function (resolve, reject) { + axios + .post( + metadataCatalogueLink + + '/api/profiles/uk.ac.hdrukgateway/HdrUkProfilePluginService/customSearch?searchTerm=&domainType=DataModel&limit=' + + datasetsMDCCount + ) + .then(function (response) { + resolve(response.data); + }) + .catch(err => { + Sentry.addBreadcrumb({ + category: 'Caching', + message: 'The caching run has failed because it was unable to pull the datasets from the MDC', + level: Sentry.Severity.Fatal, + }); + Sentry.captureException(err); + reject(err); + }); + }).catch(() => { + return 'Update failed'; + }); + + if (datasetsMDCList === 'Update failed') return; + + const metadataQualityList = await axios + .get('https://raw.githubusercontent.com/HDRUK/datasets/master/reports/metadata_quality.json', { timeout: 10000 }) + .catch(err => { + Sentry.addBreadcrumb({ + category: 'Caching', + message: 'Unable to get metadata quality value ' + err.message, + level: Sentry.Severity.Error, + }); + Sentry.captureException(err); + //console.log("Unable to get metadata quality value " + err.message); //Uncomment for local testing + }); + + const phenotypesList = await axios + .get('https://raw.githubusercontent.com/spiros/hdr-caliber-phenome-portal/master/_data/dataset2phenotypes.json', { timeout: 10000 }) + .catch(err => { + Sentry.addBreadcrumb({ + category: 'Caching', + message: 'Unable to get metadata quality value ' + err.message, + level: Sentry.Severity.Error, + }); + Sentry.captureException(err); + //console.log("Unable to get metadata quality value " + err.message); //Uncomment for local testing + }); + + const dataUtilityList = await axios + .get('https://raw.githubusercontent.com/HDRUK/datasets/master/reports/data_utility.json', { timeout: 10000 }) + .catch(err => { + Sentry.addBreadcrumb({ + category: 'Caching', + message: 'Unable to get data utility ' + err.message, + level: Sentry.Severity.Error, + }); + Sentry.captureException(err); + //console.log("Unable to get data utility " + err.message); //Uncomment for local testing + }); + + var datasetsMDCIDs = []; + var counter = 0; + + await datasetsMDCList.results.reduce( + (p, datasetMDC) => + p.then( + () => + new Promise(resolve => { + setTimeout(async function () { + try { + counter++; + var datasetHDR = await Data.findOne({ datasetid: datasetMDC.id }); + datasetsMDCIDs.push({ datasetid: datasetMDC.id }); + + const metadataQuality = metadataQualityList.data.find(x => x.id === datasetMDC.id); + const dataUtility = dataUtilityList.data.find(x => x.id === datasetMDC.id); + const phenotypes = phenotypesList.data[datasetMDC.id] || []; + + const metadataSchemaCall = axios + .get(metadataCatalogueLink + '/api/profiles/uk.ac.hdrukgateway/HdrUkProfilePluginService/schema.org/' + datasetMDC.id, { + timeout: 10000, + }) + .catch(err => { + Sentry.addBreadcrumb({ + category: 'Caching', + message: 'Unable to get metadata schema ' + err.message, + level: Sentry.Severity.Error, + }); + Sentry.captureException(err); + //console.log('Unable to get metadata schema ' + err.message); + }); + + const dataClassCall = axios + .get(metadataCatalogueLink + '/api/dataModels/' + datasetMDC.id + '/dataClasses?max=300', { timeout: 10000 }) + .catch(err => { + Sentry.addBreadcrumb({ + category: 'Caching', + message: 'Unable to get dataclass ' + err.message, + level: Sentry.Severity.Error, + }); + Sentry.captureException(err); + //console.log('Unable to get dataclass ' + err.message); + }); + + const versionLinksCall = axios + .get(metadataCatalogueLink + '/api/catalogueItems/' + datasetMDC.id + '/semanticLinks', { timeout: 10000 }) + .catch(err => { + Sentry.addBreadcrumb({ + category: 'Caching', + message: 'Unable to get version links ' + err.message, + level: Sentry.Severity.Error, + }); + Sentry.captureException(err); + //console.log('Unable to get version links ' + err.message); + }); + + const datasetV2Call = axios + .get(metadataCatalogueLink + '/api/facets/' + datasetMDC.id + '/metadata?all=true', { timeout: 5000 }) + .catch(err => { + Sentry.addBreadcrumb({ + category: 'Caching', + message: 'Unable to get dataset version 2 ' + err.message, + level: Sentry.Severity.Error, + }); + Sentry.captureException(err); + //console.log('Unable to get dataset version 2 ' + err.message); + }); + + const [metadataSchema, dataClass, versionLinks, datasetV2] = await axios.all([ + metadataSchemaCall, + dataClassCall, + versionLinksCall, + datasetV2Call, + ]); + + var technicaldetails = []; + + await dataClass.data.items.reduce( + (p, dataclassMDC) => + p.then( + () => + new Promise(resolve => { + setTimeout(async function () { + const dataClassElementCall = axios + .get( + metadataCatalogueLink + + '/api/dataModels/' + + datasetMDC.id + + '/dataClasses/' + + dataclassMDC.id + + '/dataElements?max=300', + { timeout: 5000 } + ) + .catch(err => { + console.log('Unable to get dataclass element ' + err.message); + }); + const [dataClassElement] = await axios.all([dataClassElementCall]); + var dataClassElementArray = []; + + dataClassElement.data.items.forEach(element => { + dataClassElementArray.push({ + id: element.id, + domainType: element.domainType, + label: element.label, + description: element.description, + dataType: { + id: element.dataType.id, + domainType: element.dataType.domainType, + label: element.dataType.label, + }, + }); + }); + + technicaldetails.push({ + id: dataclassMDC.id, + domainType: dataclassMDC.domainType, + label: dataclassMDC.label, + description: dataclassMDC.description, + elements: dataClassElementArray, + }); + + resolve(null); + }, 500); + }) + ), + Promise.resolve(null) + ); + + let datasetv2Object = populateV2datasetObject(datasetV2.data.items); + + if (datasetHDR) { + //Edit + if (!datasetHDR.pid) { + let uuid = uuidv4(); + let listOfVersions = []; + datasetHDR.pid = uuid; + datasetHDR.datasetVersion = '0.0.1'; + + if (versionLinks && versionLinks.data && versionLinks.data.items && versionLinks.data.items.length > 0) { + versionLinks.data.items.forEach(item => { + if (!listOfVersions.find(x => x.id === item.source.id)) { + listOfVersions.push({ id: item.source.id, version: item.source.documentationVersion }); + } + if (!listOfVersions.find(x => x.id === item.target.id)) { + listOfVersions.push({ id: item.target.id, version: item.target.documentationVersion }); + } + }); + + listOfVersions.forEach(async item => { + if (item.id !== datasetMDC.id) { + await Data.findOneAndUpdate({ datasetid: item.id }, { pid: uuid, datasetVersion: item.version }); + } else { + datasetHDR.pid = uuid; + datasetHDR.datasetVersion = item.version; + } + }); + } + } + + let keywordArray = splitString(datasetMDC.keywords); + let physicalSampleAvailabilityArray = splitString(datasetMDC.physicalSampleAvailability); + let geographicCoverageArray = splitString(datasetMDC.geographicCoverage); + + await Data.findOneAndUpdate( + { datasetid: datasetMDC.id }, + { + pid: datasetHDR.pid, + datasetVersion: datasetHDR.datasetVersion, + name: datasetMDC.title, + description: datasetMDC.description, + activeflag: 'active', + license: datasetMDC.license, + tags: { + features: keywordArray, + }, + datasetfields: { + publisher: datasetMDC.publisher, + geographicCoverage: geographicCoverageArray, + physicalSampleAvailability: physicalSampleAvailabilityArray, + abstract: datasetMDC.abstract, + releaseDate: datasetMDC.releaseDate, + accessRequestDuration: datasetMDC.accessRequestDuration, + conformsTo: datasetMDC.conformsTo, + accessRights: datasetMDC.accessRights, + jurisdiction: datasetMDC.jurisdiction, + datasetStartDate: datasetMDC.datasetStartDate, + datasetEndDate: datasetMDC.datasetEndDate, + statisticalPopulation: datasetMDC.statisticalPopulation, + ageBand: datasetMDC.ageBand, + contactPoint: datasetMDC.contactPoint, + periodicity: datasetMDC.periodicity, + + metadataquality: metadataQuality ? metadataQuality : {}, + datautility: dataUtility ? dataUtility : {}, + metadataschema: metadataSchema && metadataSchema.data ? metadataSchema.data : {}, + technicaldetails: technicaldetails, + versionLinks: versionLinks && versionLinks.data && versionLinks.data.items ? versionLinks.data.items : [], + phenotypes, + }, + datasetv2: datasetv2Object, + } + ); + } else { + //Add + let uuid = uuidv4(); + let listOfVersions = []; + let pid = uuid; + let datasetVersion = '0.0.1'; + + if (versionLinks && versionLinks.data && versionLinks.data.items && versionLinks.data.items.length > 0) { + versionLinks.data.items.forEach(item => { + if (!listOfVersions.find(x => x.id === item.source.id)) { + listOfVersions.push({ id: item.source.id, version: item.source.documentationVersion }); + } + if (!listOfVersions.find(x => x.id === item.target.id)) { + listOfVersions.push({ id: item.target.id, version: item.target.documentationVersion }); + } + }); + + for (const item of listOfVersions) { + if (item.id !== datasetMDC.id) { + var existingDataset = await Data.findOne({ datasetid: item.id }); + if (existingDataset && existingDataset.pid) pid = existingDataset.pid; + else { + await Data.findOneAndUpdate({ datasetid: item.id }, { pid: uuid, datasetVersion: item.version }); + } + } else { + datasetVersion = item.version; + } + } + } + + var uniqueID = ''; + while (uniqueID === '') { + uniqueID = parseInt(Math.random().toString().replace('0.', '')); + if ((await Data.find({ id: uniqueID }).length) === 0) { + uniqueID = ''; + } + } + + var keywordArray = splitString(datasetMDC.keywords); + var physicalSampleAvailabilityArray = splitString(datasetMDC.physicalSampleAvailability); + var geographicCoverageArray = splitString(datasetMDC.geographicCoverage); + + var data = new Data(); + data.pid = pid; + data.datasetVersion = datasetVersion; + data.id = uniqueID; + data.datasetid = datasetMDC.id; + data.type = 'dataset'; + data.activeflag = 'active'; + + data.name = datasetMDC.title; + data.description = datasetMDC.description; + data.license = datasetMDC.license; + data.tags.features = keywordArray; + data.datasetfields.publisher = datasetMDC.publisher; + data.datasetfields.geographicCoverage = geographicCoverageArray; + data.datasetfields.physicalSampleAvailability = physicalSampleAvailabilityArray; + data.datasetfields.abstract = datasetMDC.abstract; + data.datasetfields.releaseDate = datasetMDC.releaseDate; + data.datasetfields.accessRequestDuration = datasetMDC.accessRequestDuration; + data.datasetfields.conformsTo = datasetMDC.conformsTo; + data.datasetfields.accessRights = datasetMDC.accessRights; + data.datasetfields.jurisdiction = datasetMDC.jurisdiction; + data.datasetfields.datasetStartDate = datasetMDC.datasetStartDate; + data.datasetfields.datasetEndDate = datasetMDC.datasetEndDate; + data.datasetfields.statisticalPopulation = datasetMDC.statisticalPopulation; + data.datasetfields.ageBand = datasetMDC.ageBand; + data.datasetfields.contactPoint = datasetMDC.contactPoint; + data.datasetfields.periodicity = datasetMDC.periodicity; + + data.datasetfields.metadataquality = metadataQuality ? metadataQuality : {}; + data.datasetfields.datautility = dataUtility ? dataUtility : {}; + data.datasetfields.metadataschema = metadataSchema && metadataSchema.data ? metadataSchema.data : {}; + data.datasetfields.technicaldetails = technicaldetails; + data.datasetfields.versionLinks = + versionLinks && versionLinks.data && versionLinks.data.items ? versionLinks.data.items : []; + data.datasetfields.phenotypes = phenotypes; + data.datasetv2 = datasetv2Object; + await data.save(); + } + console.log(`Finished ${counter} of ${datasetsMDCCount} datasets (${datasetMDC.id})`); + resolve(null); + } catch (err) { + Sentry.addBreadcrumb({ + category: 'Caching', + message: `Failed to add ${datasetMDC.id} to the DB with the error of ${err.message}`, + level: Sentry.Severity.Fatal, + }); + Sentry.captureException(err); + //console.log(`Failed to add ${datasetMDC.id} to the DB with the error of ${err.message}`); //Uncomment for local testing + } + }, 500); + }) + ), + Promise.resolve(null) + ); + + var datasetsHDRIDs = await Data.aggregate([{ $match: { type: 'dataset' } }, { $project: { _id: 0, datasetid: 1 } }]); + + let datasetsNotFound = datasetsHDRIDs.filter(o1 => !datasetsMDCIDs.some(o2 => o1.datasetid === o2.datasetid)); + + await Promise.all( + datasetsNotFound.map(async dataset => { + //Archive + await Data.findOneAndUpdate( + { datasetid: dataset.datasetid }, + { + activeflag: 'archive', + } + ); + }) + ); + + saveUptime(); + + console.log('Update Completed at ' + Date()); + return; +} + +function populateV2datasetObject(v2Data) { + let datasetV2List = v2Data.filter(item => item.namespace === 'org.healthdatagateway'); + + let datasetv2Object = {}; + if (datasetV2List.length > 0) { + datasetv2Object = { + identifier: datasetV2List.find(x => x.key === 'properties/identifier') + ? datasetV2List.find(x => x.key === 'properties/identifier').value + : '', + version: datasetV2List.find(x => x.key === 'properties/version') ? datasetV2List.find(x => x.key === 'properties/version').value : '', + issued: datasetV2List.find(x => x.key === 'properties/issued') ? datasetV2List.find(x => x.key === 'properties/issued').value : '', + modified: datasetV2List.find(x => x.key === 'properties/modified') + ? datasetV2List.find(x => x.key === 'properties/modified').value + : '', + revisions: [], + summary: { + title: datasetV2List.find(x => x.key === 'properties/summary/title') + ? datasetV2List.find(x => x.key === 'properties/summary/title').value + : '', + abstract: datasetV2List.find(x => x.key === 'properties/summary/abstract') + ? datasetV2List.find(x => x.key === 'properties/summary/abstract').value + : '', + publisher: { + identifier: datasetV2List.find(x => x.key === 'properties/summary/publisher/identifier') + ? datasetV2List.find(x => x.key === 'properties/summary/publisher/identifier').value + : '', + name: datasetV2List.find(x => x.key === 'properties/summary/publisher/name') + ? datasetV2List.find(x => x.key === 'properties/summary/publisher/name').value + : '', + logo: datasetV2List.find(x => x.key === 'properties/summary/publisher/logo') + ? datasetV2List.find(x => x.key === 'properties/summary/publisher/logo').value + : '', + description: datasetV2List.find(x => x.key === 'properties/summary/publisher/description') + ? datasetV2List.find(x => x.key === 'properties/summary/publisher/description').value + : '', + contactPoint: checkForArray( + datasetV2List.find(x => x.key === 'properties/summary/publisher/contactPoint') + ? datasetV2List.find(x => x.key === 'properties/summary/publisher/contactPoint').value + : [] + ), + memberOf: datasetV2List.find(x => x.key === 'properties/summary/publisher/memberOf') + ? datasetV2List.find(x => x.key === 'properties/summary/publisher/memberOf').value + : '', + accessRights: checkForArray( + datasetV2List.find(x => x.key === 'properties/summary/publisher/accessRights') + ? datasetV2List.find(x => x.key === 'properties/summary/publisher/accessRights').value + : [] + ), + deliveryLeadTime: datasetV2List.find(x => x.key === 'properties/summary/publisher/deliveryLeadTime') + ? datasetV2List.find(x => x.key === 'properties/summary/publisher/deliveryLeadTime').value + : '', + accessService: datasetV2List.find(x => x.key === 'properties/summary/publisher/accessService') + ? datasetV2List.find(x => x.key === 'properties/summary/publisher/accessService').value + : '', + accessRequestCost: datasetV2List.find(x => x.key === 'properties/summary/publisher/accessRequestCost') + ? datasetV2List.find(x => x.key === 'properties/summary/publisher/accessRequestCost').value + : '', + dataUseLimitation: checkForArray( + datasetV2List.find(x => x.key === 'properties/summary/publisher/dataUseLimitation') + ? datasetV2List.find(x => x.key === 'properties/summary/publisher/dataUseLimitation').value + : [] + ), + dataUseRequirements: checkForArray( + datasetV2List.find(x => x.key === 'properties/summary/publisher/dataUseRequirements') + ? datasetV2List.find(x => x.key === 'properties/summary/publisher/dataUseRequirements').value + : [] + ), + }, + contactPoint: datasetV2List.find(x => x.key === 'properties/summary/contactPoint') + ? datasetV2List.find(x => x.key === 'properties/summary/contactPoint').value + : '', + keywords: checkForArray( + datasetV2List.find(x => x.key === 'properties/summary/keywords') + ? datasetV2List.find(x => x.key === 'properties/summary/keywords').value + : [] + ), + alternateIdentifiers: checkForArray( + datasetV2List.find(x => x.key === 'properties/summary/alternateIdentifiers') + ? datasetV2List.find(x => x.key === 'properties/summary/alternateIdentifiers').value + : [] + ), + doiName: datasetV2List.find(x => x.key === 'properties/summary/doiName') + ? datasetV2List.find(x => x.key === 'properties/summary/doiName').value + : '', + }, + documentation: { + description: datasetV2List.find(x => x.key === 'properties/documentation/description') + ? datasetV2List.find(x => x.key === 'properties/documentation/description').value + : '', + associatedMedia: checkForArray( + datasetV2List.find(x => x.key === 'properties/documentation/associatedMedia') + ? datasetV2List.find(x => x.key === 'properties/documentation/associatedMedia').value + : [] + ), + isPartOf: checkForArray( + datasetV2List.find(x => x.key === 'properties/documentation/isPartOf') + ? datasetV2List.find(x => x.key === 'properties/documentation/isPartOf').value + : [] + ), + }, + coverage: { + spatial: datasetV2List.find(x => x.key === 'properties/coverage/spatial') + ? datasetV2List.find(x => x.key === 'properties/coverage/spatial').value + : '', + typicalAgeRange: datasetV2List.find(x => x.key === 'properties/coverage/typicalAgeRange') + ? datasetV2List.find(x => x.key === 'properties/coverage/typicalAgeRange').value + : '', + physicalSampleAvailability: checkForArray( + datasetV2List.find(x => x.key === 'properties/coverage/physicalSampleAvailability') + ? datasetV2List.find(x => x.key === 'properties/coverage/physicalSampleAvailability').value + : [] + ), + followup: datasetV2List.find(x => x.key === 'properties/coverage/followup') + ? datasetV2List.find(x => x.key === 'properties/coverage/followup').value + : '', + pathway: datasetV2List.find(x => x.key === 'properties/coverage/pathway') + ? datasetV2List.find(x => x.key === 'properties/coverage/pathway').value + : '', + }, + provenance: { + origin: { + purpose: checkForArray( + datasetV2List.find(x => x.key === 'properties/provenance/origin/purpose') + ? datasetV2List.find(x => x.key === 'properties/provenance/origin/purpose').value + : [] + ), + source: checkForArray( + datasetV2List.find(x => x.key === 'properties/provenance/origin/source') + ? datasetV2List.find(x => x.key === 'properties/provenance/origin/source').value + : [] + ), + collectionSituation: checkForArray( + datasetV2List.find(x => x.key === 'properties/provenance/origin/collectionSituation') + ? datasetV2List.find(x => x.key === 'properties/provenance/origin/collectionSituation').value + : [] + ), + }, + temporal: { + accrualPeriodicity: datasetV2List.find(x => x.key === 'properties/provenance/temporal/accrualPeriodicity') + ? datasetV2List.find(x => x.key === 'properties/provenance/temporal/accrualPeriodicity').value + : '', + distributionReleaseDate: datasetV2List.find(x => x.key === 'properties/provenance/temporal/distributionReleaseDate') + ? datasetV2List.find(x => x.key === 'properties/provenance/temporal/distributionReleaseDate').value + : '', + startDate: datasetV2List.find(x => x.key === 'properties/provenance/temporal/startDate') + ? datasetV2List.find(x => x.key === 'properties/provenance/temporal/startDate').value + : '', + endDate: datasetV2List.find(x => x.key === 'properties/provenance/temporal/endDate') + ? datasetV2List.find(x => x.key === 'properties/provenance/temporal/endDate').value + : '', + timeLag: datasetV2List.find(x => x.key === 'properties/provenance/temporal/timeLag') + ? datasetV2List.find(x => x.key === 'properties/provenance/temporal/timeLag').value + : '', + }, + }, + accessibility: { + usage: { + dataUseLimitation: checkForArray( + datasetV2List.find(x => x.key === 'properties/accessibility/usage/dataUseLimitation') + ? datasetV2List.find(x => x.key === 'properties/accessibility/usage/dataUseLimitation').value + : [] + ), + dataUseRequirements: checkForArray( + datasetV2List.find(x => x.key === 'properties/accessibility/usage/dataUseRequirements') + ? datasetV2List.find(x => x.key === 'properties/accessibility/usage/dataUseRequirements').value + : [] + ), + resourceCreator: datasetV2List.find(x => x.key === 'properties/accessibility/usage/resourceCreator') + ? datasetV2List.find(x => x.key === 'properties/accessibility/usage/resourceCreator').value + : '', + investigations: checkForArray( + datasetV2List.find(x => x.key === 'properties/accessibility/usage/investigations') + ? datasetV2List.find(x => x.key === 'properties/accessibility/usage/investigations').value + : [] + ), + isReferencedBy: checkForArray( + datasetV2List.find(x => x.key === 'properties/accessibility/usage/isReferencedBy') + ? datasetV2List.find(x => x.key === 'properties/accessibility/usage/isReferencedBy').value + : [] + ), + }, + access: { + accessRights: checkForArray( + datasetV2List.find(x => x.key === 'properties/accessibility/access/accessRights') + ? datasetV2List.find(x => x.key === 'properties/accessibility/access/accessRights').value + : [] + ), + accessService: datasetV2List.find(x => x.key === 'properties/accessibility/access/accessService') + ? datasetV2List.find(x => x.key === 'properties/accessibility/access/accessService').value + : '', + accessRequestCost: datasetV2List.find(x => x.key === 'properties/accessibility/access/accessRequestCost') + ? datasetV2List.find(x => x.key === 'properties/accessibility/access/accessRequestCost').value + : '', + deliveryLeadTime: datasetV2List.find(x => x.key === 'properties/accessibility/access/deliveryLeadTime') + ? datasetV2List.find(x => x.key === 'properties/accessibility/access/deliveryLeadTime').value + : '', + jurisdiction: checkForArray( + datasetV2List.find(x => x.key === 'properties/accessibility/access/jurisdiction') + ? datasetV2List.find(x => x.key === 'properties/accessibility/access/jurisdiction').value + : [] + ), + dataProcessor: datasetV2List.find(x => x.key === 'properties/accessibility/access/dataProcessor') + ? datasetV2List.find(x => x.key === 'properties/accessibility/access/dataProcessor').value + : '', + dataController: datasetV2List.find(x => x.key === 'properties/accessibility/access/dataController') + ? datasetV2List.find(x => x.key === 'properties/accessibility/access/dataController').value + : '', + }, + formatAndStandards: { + vocabularyEncodingScheme: checkForArray( + datasetV2List.find(x => x.key === 'properties/accessibility/formatAndStandards/vocabularyEncodingScheme') + ? datasetV2List.find(x => x.key === 'properties/accessibility/formatAndStandards/vocabularyEncodingScheme').value + : [] + ), + conformsTo: checkForArray( + datasetV2List.find(x => x.key === 'properties/accessibility/formatAndStandards/conformsTo') + ? datasetV2List.find(x => x.key === 'properties/accessibility/formatAndStandards/conformsTo').value + : [] + ), + language: checkForArray( + datasetV2List.find(x => x.key === 'properties/accessibility/formatAndStandards/language') + ? datasetV2List.find(x => x.key === 'properties/accessibility/formatAndStandards/language').value + : [] + ), + format: checkForArray( + datasetV2List.find(x => x.key === 'properties/accessibility/formatAndStandards/format') + ? datasetV2List.find(x => x.key === 'properties/accessibility/formatAndStandards/format').value + : [] + ), + }, + }, + enrichmentAndLinkage: { + qualifiedRelation: checkForArray( + datasetV2List.find(x => x.key === 'properties/enrichmentAndLinkage/qualifiedRelation') + ? datasetV2List.find(x => x.key === 'properties/enrichmentAndLinkage/qualifiedRelation').value + : [] + ), + derivation: checkForArray( + datasetV2List.find(x => x.key === 'properties/enrichmentAndLinkage/derivation') + ? datasetV2List.find(x => x.key === 'properties/enrichmentAndLinkage/derivation').value + : [] + ), + tools: checkForArray( + datasetV2List.find(x => x.key === 'properties/enrichmentAndLinkage/tools') + ? datasetV2List.find(x => x.key === 'properties/enrichmentAndLinkage/tools').value + : [] + ), + }, + observations: [], + }; + } + + return datasetv2Object; +} + +function checkForArray(value) { + if (typeof value !== 'string') return value; + try { + const type = Object.prototype.toString.call(JSON.parse(value)); + if (type === '[object Object]' || type === '[object Array]') return JSON.parse(value); + } catch (err) { + return value; + } +} + +function splitString(array) { + var returnArray = []; + if (array !== null && array !== '' && array !== 'undefined' && array !== undefined) { + if (array.indexOf(',') === -1) { + returnArray.push(array.trim()); + } else { + array.split(',').forEach(term => { + returnArray.push(term.trim()); + }); + } + } + return returnArray; +} + +async function saveUptime() { + const monitoring = require('@google-cloud/monitoring'); + const projectId = 'hdruk-gateway'; + const client = new monitoring.MetricServiceClient(); + + var selectedMonthStart = new Date(); + selectedMonthStart.setMonth(selectedMonthStart.getMonth() - 1); + selectedMonthStart.setDate(1); + selectedMonthStart.setHours(0, 0, 0, 0); + + var selectedMonthEnd = new Date(); + selectedMonthEnd.setDate(0); + selectedMonthEnd.setHours(23, 59, 59, 999); + + const request = { + name: client.projectPath(projectId), + filter: + 'metric.type="monitoring.googleapis.com/uptime_check/check_passed" AND resource.type="uptime_url" AND metric.label."check_id"="check-production-web-app-qsxe8fXRrBo" AND metric.label."checker_location"="eur-belgium"', + + interval: { + startTime: { + seconds: selectedMonthStart.getTime() / 1000, + }, + endTime: { + seconds: selectedMonthEnd.getTime() / 1000, + }, + }, + aggregation: { + alignmentPeriod: { + seconds: '86400s', + }, + crossSeriesReducer: 'REDUCE_NONE', + groupByFields: ['metric.label."checker_location"', 'resource.label."instance_id"'], + perSeriesAligner: 'ALIGN_FRACTION_TRUE', + }, + }; + + // Writes time series data + const [timeSeries] = await client.listTimeSeries(request); + var dailyUptime = []; + var averageUptime; + + timeSeries.forEach(data => { + data.points.forEach(data => { + dailyUptime.push(data.value.doubleValue); + }); + + averageUptime = (dailyUptime.reduce((a, b) => a + b, 0) / dailyUptime.length) * 100; + }); + + var metricsData = new MetricsData(); + metricsData.uptime = averageUptime; + await metricsData.save(); +} \ No newline at end of file diff --git a/src/resources/dataset/v2/dataset.route.js b/src/resources/dataset/v2/dataset.route.js index b8966616..b0df0f01 100644 --- a/src/resources/dataset/v2/dataset.route.js +++ b/src/resources/dataset/v2/dataset.route.js @@ -6,13 +6,13 @@ const router = express.Router(); const datasetController = new DatasetController(datasetService); // @route GET /api/v2/datasets/id -// @desc Returns a dataset based on either dataset ID or PID (persistent identifier) provided +// @desc Returns a dataset based on dataset ID provided // @access Public router.get('/:id', (req, res) => datasetController.getDataset(req, res)); // @route GET /api/v2/datasets // @desc Returns a collection of datasets based on supplied query parameters // @access Public -//router.get('/', datasetController.getDatasets); +router.get('/', (req, res) => datasetController.getDatasets(req, res)); module.exports = router; diff --git a/src/resources/paper/__mocks__/papers.js b/src/resources/paper/__mocks__/papers.js new file mode 100644 index 00000000..79b3554a --- /dev/null +++ b/src/resources/paper/__mocks__/papers.js @@ -0,0 +1,269 @@ +export const papersStub = [ + { + _id: '5f1ff663114dcd2cc53e5ece', + id: 32300742, + activeflag: 'active', + authors: [100000], + counter: 8, + createdAt: '2021-02-14T02:02:04.237Z', + datasetids: [], + description: + '\n\n**Lay Summary**\n\nBackground:The incidence of knife-related injuries is rising across the UK. This study aimed to determine the spectrum of knife-related injuries in a major UK city, with regards to patient and injury characteristics. A secondary aim was to quantify their impact on secondary care resources. Methods:Observational study of patients aged 16+ years admitted to a major trauma centre following knife-related injuries resulting from interpersonal violence (May 2015 to April 2018). Patients were identified using Emergency Department and discharge coding, blood bank and UK national Trauma Audit and Research prospective registries. Patient and injury characteristics, outcome and resource utilisation were collected from ambulance and hospital records. Findings:532 patients were identified; 93% male, median age 26 years (IQR 20-35). Median injury severity score was 9 (IQR 3-13). 346 (65%) underwent surgery; 133 (25%) required intensive care; 95 (17·9%) received blood transfusion. Median length of stay was 3·3 days (IQR 1·7-6·0). In-hospital mortality was 10/532 (1·9%). 98 patients (18·5%) had previous attendance with violence-related injuries. 24/37 females (64·9%) were injured in a domestic setting. Intoxication with alcohol (19·2%) and illicit drugs (17·6%) was common. Causative weapon was household knife in 9%, knife (other/unspecified) in 38·0%, machete in 13·9%, small folding blade (2·8%) and, unrecorded in 36·3%. Interpretation:Knife injuries constitute 12·9% of trauma team workload. Violence recidivism and intoxication are common, and females are predominantly injured in a domestic setting, presenting opportunities for targeted violence reduction interventions. 13·9% of injuries involved machetes, with implications for law enforcement strategies.\n\n\n\n**Authors:**\n\nMalik NS, Munoz B, de Courcey C, Imran R, Lee KC, Chernbumroong S, Bishop J, Lord JM, Gkoutos G, Bowley DM, Foster MA.\n\n', + journal: 'EClinicalMedicine', + journalYear: 2020, + link: 'https://doi.org/10.1016/j.eclinm.2020.100296', + name: 'Violence-related knife injuries in a UK city; epidemiology and impact on secondary care resources.', + relatedObjects: [], + source_url: 'https://raw.githubusercontent.com/HDRUK/papers/master/data/papers.csv', + tags: { + features: ['HDRUK Papers', 'Open Access'], + topics: [], + }, + toolids: [], + type: 'paper', + updatedAt: '2021-02-14T02:02:04.237Z', + updatedon: '2021-02-14T02:02:04.237Z', + document_links: [ + { + doi: 'https://doi.org/10.1016/j.eclinm.2020.100296', + }, + { + html: 'https://europepmc.org/articles/PMC7152819', + }, + { + pdf: 'https://europepmc.org/articles/PMC7152819?pdf=render', + }, + ], + persons: [ + { + _id: '5f1ff1e0114dcd2cc53e1e90', + id: 100000, + activeflag: 'active', + usedids: [], + projectids: [], + sociallinks: [''], + datasetids: [], + linkedids: [], + firstname: 'HDR UK', + type: 'person', + lastname: 'Papers Team', + tags: [], + updatedon: '2020-05-21T00:00:00.001Z', + bio: '', + counter: 38, + updatedAt: '2021-01-27T21:08:18.164Z', + organisation: '', + sector: '', + domain: '', + link: '', + orcid: '', + }, + ], + objects: [], + reviews: [], + }, + { + _id: '5f1ff69f114dcd2cc53e6288', + id: 32276644, + activeflag: 'active', + authors: [100000], + counter: 6, + createdAt: '2021-02-14T02:02:04.237Z', + datasetids: [], + description: + "\n\n**Lay Summary**\n\nBACKGROUND:Inaccurately modelled environmental exposures may have important implications for evidence-based policy targeting health promoting or hazardous facilities. Travel routes modelled using GIS generally use shortest network distances or Euclidean buffers to represent journeys with corresponding built-environment exposures calculated along these routes. These methods, however, are an unreliable proxy for calculating child built-environment exposures as child route choice is more complex than shortest network routes. METHODS:We hypothesised that a GIS model informed by characteristics of the built-environment known to influence child route choice could be developed to more accurately model exposures. Using GPS-derived walking commutes to and from school we used logistic regression models to highlight built-environment features important in child route choice (e.g. road type, traffic light count). We then recalculated walking commute routes using a weighted network to incorporate built-environment features. Multilevel regression analyses were used to validate exposure predictions to the retail food environment along the different routing methods. RESULTS:Children chose routes with more traffic lights and residential roads compared to the modelled shortest network routes. Compared to standard shortest network routes, the GPS-informed weighted network enabled GIS-based walking commutes to be derived with more than three times greater accuracy (38%) for the route to school and more than 12 times greater accuracy (92%) for the route home. CONCLUSIONS:This research advocates using weighted GIS networks to accurately reflect child walking journeys to school. The improved accuracy in route modelling has in turn improved estimates of children's exposures to potentially hazardous features in the environment. Further research is needed to explore if the built-environment features are important internationally. Route and corresponding exposure estimates can be scaled to the population level which will contribute to a better understanding of built-environment exposures on child health and contribute to mobility-based child health policy.\n\n\n\n**Authors:**\n\nMizen A, Fry R, Rodgers S.\n\n", + journal: 'International journal of health geographics', + journalYear: 2020, + link: 'https://doi.org/10.1186/s12942-020-00208-2', + name: 'GIS-modelled built-environment exposures reflecting daily mobility for applications in child health research.', + relatedObjects: [], + source_url: 'https://raw.githubusercontent.com/HDRUK/papers/master/data/papers.csv', + tags: { + features: [ + 'HDRUK Papers', + 'Open Access', + 'Environmental exposure', + 'Child Health', + 'Walking', + 'Weighted Network', + 'Daily Mobility', + 'School Commute', + ], + topics: [], + }, + toolids: [], + type: 'paper', + updatedAt: '2021-02-14T02:02:04.237Z', + updatedon: '2021-02-14T02:02:04.237Z', + document_links: [ + { + doi: 'https://doi.org/10.1186/s12942-020-00208-2', + }, + { + html: 'https://europepmc.org/articles/PMC7147039', + }, + { + pdf: 'https://europepmc.org/articles/PMC7147039?pdf=render', + }, + ], + persons: [ + { + _id: '5f1ff1e0114dcd2cc53e1e90', + id: 100000, + activeflag: 'active', + usedids: [], + projectids: [], + sociallinks: [''], + datasetids: [], + linkedids: [], + firstname: 'HDR UK', + type: 'person', + lastname: 'Papers Team', + tags: [], + updatedon: '2020-05-21T00:00:00.001Z', + bio: '', + counter: 38, + updatedAt: '2021-01-27T21:08:18.164Z', + organisation: '', + sector: '', + domain: '', + link: '', + orcid: '', + }, + ], + objects: [], + reviews: [], + }, + { + _id: '5f1ff69f114dcd2cc53e6293', + id: 32405103, + activeflag: 'active', + authors: [100000], + counter: 2, + createdAt: '2021-02-14T02:02:04.237Z', + datasetids: [], + description: + '\n\n**Lay Summary**\n\n

Background

The medical, societal, and economic impact of the coronavirus disease 2019 (COVID-19) pandemic has unknown effects on overall population mortality. Previous models of population mortality are based on death over days among infected people, nearly all of whom thus far have underlying conditions. Models have not incorporated information on high-risk conditions or their longer-term baseline (pre-COVID-19) mortality. We estimated the excess number of deaths over 1 year under different COVID-19 incidence scenarios based on varying levels of transmission suppression and differing mortality impacts based on different relative risks for the disease.

Methods

In this population-based cohort study, we used linked primary and secondary care electronic health records from England (Health Data Research UK-CALIBER). We report prevalence of underlying conditions defined by Public Health England guidelines (from March 16, 2020) in individuals aged 30 years or older registered with a practice between 1997 and 2017, using validated, openly available phenotypes for each condition. We estimated 1-year mortality in each condition, developing simple models (and a tool for calculation) of excess COVID-19-related deaths, assuming relative impact (as relative risks [RRs]) of the COVID-19 pandemic (compared with background mortality) of 1·5, 2·0, and 3·0 at differing infection rate scenarios, including full suppression (0·001%), partial suppression (1%), mitigation (10%), and do nothing (80%). We also developed an online, public, prototype risk calculator for excess death estimation.

Findings

We included 3 862 012 individuals (1 957 935 [50·7%] women and 1 904 077 [49·3%] men). We estimated that more than 20% of the study population are in the high-risk category, of whom 13·7% were older than 70 years and 6·3% were aged 70 years or younger with at least one underlying condition. 1-year mortality in the high-risk population was estimated to be 4·46% (95% CI 4·41-4·51). Age and underlying conditions combined to influence background risk, varying markedly across conditions. In a full suppression scenario in the UK population, we estimated that there would be two excess deaths (vs baseline deaths) with an RR of 1·5, four with an RR of 2·0, and seven with an RR of 3·0. In a mitigation scenario, we estimated 18 374 excess deaths with an RR of 1·5, 36 749 with an RR of 2·0, and 73 498 with an RR of 3·0. In a do nothing scenario, we estimated 146 996 excess deaths with an RR of 1·5, 293 991 with an RR of 2·0, and 587 982 with an RR of 3·0.

Interpretation

We provide policy makers, researchers, and the public a simple model and an online tool for understanding excess mortality over 1 year from the COVID-19 pandemic, based on age, sex, and underlying condition-specific estimates. These results signal the need for sustained stringent suppression measures as well as sustained efforts to target those at highest risk because of underlying conditions with a range of preventive interventions. Countries should assess the overall (direct and indirect) effects of the pandemic on excess mortality.

Funding

National Institute for Health Research University College London Hospitals Biomedical Research Centre, Health Data Research UK.\n\n\n\n**Authors:**\n\nBanerjee A, Pasea L, Harris S, Gonzalez-Izquierdo A, Torralbo A, Shallcross L, Noursadeghi M, Pillay D, Sebire N, Holmes C, Pagel C, Wong WK, Langenberg C, Williams B, Denaxas S, Hemingway H.\n\n', + journal: 'Lancet (London, England)', + journalYear: 2020, + link: 'https://doi.org/10.1016/s0140-6736(20)30854-0', + name: + 'Estimating excess 1-year mortality associated with the COVID-19 pandemic according to underlying conditions and age: a population-based cohort study.', + relatedObjects: [], + source_url: 'https://raw.githubusercontent.com/HDRUK/papers/master/data/papers.csv', + tags: { + features: ['HDRUK Papers', 'Open Access'], + topics: [], + }, + toolids: [], + type: 'paper', + updatedAt: '2021-02-14T02:02:04.237Z', + updatedon: '2021-02-14T02:02:04.237Z', + document_links: [ + { + doi: 'https://doi.org/10.1016/S0140-6736(20)30854-0', + }, + { + html: 'https://europepmc.org/articles/PMC7217641', + }, + ], + persons: [ + { + _id: '5f1ff1e0114dcd2cc53e1e90', + id: 100000, + activeflag: 'active', + usedids: [], + projectids: [], + sociallinks: [''], + datasetids: [], + linkedids: [], + firstname: 'HDR UK', + type: 'person', + lastname: 'Papers Team', + tags: [], + updatedon: '2020-05-21T00:00:00.001Z', + bio: '', + counter: 38, + updatedAt: '2021-01-27T21:08:18.164Z', + organisation: '', + sector: '', + domain: '', + link: '', + orcid: '', + }, + ], + objects: [], + reviews: [], + }, + { + _id: '5f1ff69f114dcd2cc53e629e', + id: 32499256, + activeflag: 'active', + authors: [100000], + counter: 3, + createdAt: '2021-02-14T02:02:04.237Z', + datasetids: [], + description: + '\n\n**Lay Summary**\n\nINTRODUCTION:Urinary tract infections (UTIs) are the second most common type of infection worldwide, accounting for a large number of primary care consultations and antibiotic prescribing. Current diagnosis is based on an empirical approach, relying on symptoms and occasional use of urine dipsticks. The diagnostic reference standard is still urine culture, although it is not routinely recommended for uncomplicated UTIs in the community, due to time to diagnosis (48 hours). Faster point-of-care tests have been developed, but their diagnostic accuracy has not been compared. Our objective is to systematically review and meta-analyse the diagnostic accuracy of currently available point-of-care tests for UTIs. METHODS AND ANALYSIS:Studies evaluating the diagnostic accuracy of point-of-care tests for UTIs will be included. PubMed, Web of Science, Embase and Cochrane Database of Systematic Reviews were searched from inception to 1 June 2019. Data extraction and risk-of-bias assessment will be assessed using the Quality Assessment of Diagnostic Accuracy Studies tool. Meta-analysis will be performed depending on data availability and heterogeneity. ETHICS AND DISSEMINATION:This is a systematic review protocol and therefore formal ethical approval is not required, as no primary, identifiable, personal data will be collected. Patients or the public were not involved in the design of our research. However, the findings from this review will be shared with key stakeholders, including patient groups, clinicians and guideline developers, and will also be presented and national and international conferences. PROSPERO REGISTRATION NUMBER:CRD42018112019.\n\n\n\n**Authors:**\n\nFraile Navarro D, Sullivan F, Azcoaga-Lorenzo A, Hernandez Santiago V.\n\n', + journal: 'BMJ open', + journalYear: 2020, + link: 'https://doi.org/10.1136/bmjopen-2019-033424', + name: + 'Point-of-care tests for urinary tract infections: protocol for a systematic review and meta-analysis of diagnostic test accuracy.', + relatedObjects: [], + source_url: 'https://raw.githubusercontent.com/HDRUK/papers/master/data/papers.csv', + tags: { + features: [ + 'HDRUK Papers', + 'Open Access', + 'Urinary tract infections', + 'epidemiology', + 'Molecular Diagnostics', + 'Diagnostic Microbiology', + ], + topics: [], + }, + toolids: [], + type: 'paper', + updatedAt: '2021-02-14T02:02:04.237Z', + updatedon: '2021-02-14T02:02:04.237Z', + document_links: [ + { + doi: 'https://doi.org/10.1136/bmjopen-2019-033424', + }, + { + html: 'https://europepmc.org/articles/PMC7282288', + }, + { + pdf: 'https://europepmc.org/articles/PMC7282288?pdf=render', + }, + ], + persons: [ + { + _id: '5f1ff1e0114dcd2cc53e1e90', + id: 100000, + activeflag: 'active', + usedids: [], + projectids: [], + sociallinks: [''], + datasetids: [], + linkedids: [], + firstname: 'HDR UK', + type: 'person', + lastname: 'Papers Team', + tags: [], + updatedon: '2020-05-21T00:00:00.001Z', + bio: '', + counter: 38, + updatedAt: '2021-01-27T21:08:18.164Z', + organisation: '', + sector: '', + domain: '', + link: '', + orcid: '', + }, + ], + objects: [], + reviews: [], + }, +]; diff --git a/src/resources/paper/__tests__/paper.controller.test.js b/src/resources/paper/__tests__/paper.controller.test.js new file mode 100644 index 00000000..b0041a67 --- /dev/null +++ b/src/resources/paper/__tests__/paper.controller.test.js @@ -0,0 +1,117 @@ +import sinon from 'sinon'; +import faker from 'faker'; + +import PaperController from '../paper.controller'; +import PaperService from '../paper.service'; + +describe('PaperController', function () { + beforeAll(() => { + console.log = sinon.stub(); + console.error = sinon.stub(); + }); + + describe('getPaper', function () { + let req, res, status, json, paperService, paperController; + + beforeEach(() => { + status = sinon.stub(); + json = sinon.spy(); + res = { json, status }; + status.returns(res); + paperService = new PaperService(); + }); + + it('should return a paper that matches the id param', async function () { + req = { params: { id: faker.random.number({ min: 1, max: 999999999 }) } }; + const stubValue = { + id: req.params.id, + }; + const serviceStub = sinon.stub(paperService, 'getPaper').returns(stubValue); + paperController = new PaperController(paperService); + await paperController.getPaper(req, res); + + expect(serviceStub.calledOnce).toBe(true); + expect(status.calledWith(200)).toBe(true); + expect(json.calledWith({ success: true, data: stubValue })).toBe(true); + }); + + it('should return a bad request response if no paper id is provided', async function () { + req = { params: {} }; + + const serviceStub = sinon.stub(paperService, 'getPaper').returns({}); + paperController = new PaperController(paperService); + await paperController.getPaper(req, res); + + expect(serviceStub.notCalled).toBe(true); + expect(status.calledWith(400)).toBe(true); + expect(json.calledWith({ success: false, message: 'You must provide a paper identifier' })).toBe(true); + }); + + it('should return a not found response if no paper could be found for the id provided', async function () { + req = { params: { id: faker.random.number({ min: 1, max: 999999999 }) } }; + + const serviceStub = sinon.stub(paperService, 'getPaper').returns(null); + paperController = new PaperController(paperService); + await paperController.getPaper(req, res); + + expect(serviceStub.calledOnce).toBe(true); + expect(status.calledWith(404)).toBe(true); + expect(json.calledWith({ success: false, message: 'A paper could not be found with the provided id' })).toBe(true); + }); + + it('should return a server error if an unexpected exception occurs', async function () { + req = { params: { id: faker.random.number({ min: 1, max: 999999999 }) } }; + + const error = new Error('A server error occurred'); + const serviceStub = sinon.stub(paperService, 'getPaper').throws(error); + paperController = new PaperController(paperService); + await paperController.getPaper(req, res); + + expect(serviceStub.calledOnce).toBe(true); + expect(status.calledWith(500)).toBe(true); + expect(json.calledWith({ success: false, message: 'A server error occurred, please try again' })).toBe(true); + }); + }); + + describe('getPapers', function () { + let req, res, status, json, paperService, paperController; + req = { params: {} }; + + beforeEach(() => { + status = sinon.stub(); + json = sinon.spy(); + res = { json, status }; + status.returns(res); + paperService = new PaperService(); + }); + + it('should return an array of papers', async function () { + const stubValue = [ + { + id: faker.random.number({ min: 1, max: 999999999 }), + }, + { + id: faker.random.number({ min: 1, max: 999999999 }), + }, + ]; + const serviceStub = sinon.stub(paperService, 'getPapers').returns(stubValue); + paperController = new PaperController(paperService); + await paperController.getPapers(req, res); + + expect(serviceStub.calledOnce).toBe(true); + expect(status.calledWith(200)).toBe(true); + expect(json.calledWith({ success: true, data: stubValue })).toBe(true); + }); + + it('should return a server error if an unexpected exception occurs', async function () { + const error = new Error('A server error occurred'); + const serviceStub = sinon.stub(paperService, 'getPapers').throws(error); + paperController = new PaperController(paperService); + await paperController.getPapers(req, res); + + expect(serviceStub.calledOnce).toBe(true); + expect(status.calledWith(500)).toBe(true); + expect(json.calledWith({ success: false, message: 'A server error occurred, please try again' })).toBe(true); + }); + }); +}); diff --git a/src/resources/paper/__tests__/paper.entity.test.js b/src/resources/paper/__tests__/paper.entity.test.js new file mode 100644 index 00000000..0c6e66a8 --- /dev/null +++ b/src/resources/paper/__tests__/paper.entity.test.js @@ -0,0 +1,54 @@ +import PaperClass from '../paper.entity'; + +describe('PaperEntity', function () { + describe('constructor', function () { + it('should create an instance of a paper entity with the expected properties', async function () { + const paper = new PaperClass( + 32300742, + 'Violence-related knife injuries in a UK city; epidemiology and impact on secondary care resources.', + '\n\n**Lay Summary**\n\nBackground:The incidence of knife-related injuries is rising across the UK. This study aimed to determine the spectrum of knife-related injuries in a major UK city, with regards to patient and injury characteristics. A secondary aim was to quantify their impact on secondary care resources. Methods:Observational study of patients aged 16+ years admitted to a major trauma centre following knife-related injuries resulting from interpersonal violence (May 2015 to April 2018). Patients were identified using Emergency Department and discharge coding, blood bank and UK national Trauma Audit and Research prospective registries. Patient and injury characteristics, outcome and resource utilisation were collected from ambulance and hospital records. Findings:532 patients were identified; 93% male, median age 26 years (IQR 20-35). Median injury severity score was 9 (IQR 3-13). 346 (65%) underwent surgery; 133 (25%) required intensive care; 95 (17·9%) received blood transfusion. Median length of stay was 3·3 days (IQR 1·7-6·0). In-hospital mortality was 10/532 (1·9%). 98 patients (18·5%) had previous attendance with violence-related injuries. 24/37 females (64·9%) were injured in a domestic setting. Intoxication with alcohol (19·2%) and illicit drugs (17·6%) was common. Causative weapon was household knife in 9%, knife (other/unspecified) in 38·0%, machete in 13·9%, small folding blade (2·8%) and, unrecorded in 36·3%. Interpretation:Knife injuries constitute 12·9% of trauma team workload. Violence recidivism and intoxication are common, and females are predominantly injured in a domestic setting, presenting opportunities for targeted violence reduction interventions. 13·9% of injuries involved machetes, with implications for law enforcement strategies.\n\n\n\n**Authors:**\n\nMalik NS, Munoz B, de Courcey C, Imran R, Lee KC, Chernbumroong S, Bishop J, Lord JM, Gkoutos G, Bowley DM, Foster MA.\n\n', + null, + 'https://doi.org/10.1016/j.eclinm.2020.100296', + 'paper', + {}, + null, + [100000], + { + features: ['HDRUK Papers', 'Open Access'], + topics: [], + }, + 'active', + 8, + null, + [], + null, + 'EClinicalMedicine', + 2020, + false + ); + + expect(paper.id).toEqual(32300742); + expect(paper.type).toEqual('paper'); + expect(paper.name).toEqual('Violence-related knife injuries in a UK city; epidemiology and impact on secondary care resources.'); + expect(paper.description).toEqual( + '\n\n**Lay Summary**\n\nBackground:The incidence of knife-related injuries is rising across the UK. This study aimed to determine the spectrum of knife-related injuries in a major UK city, with regards to patient and injury characteristics. A secondary aim was to quantify their impact on secondary care resources. Methods:Observational study of patients aged 16+ years admitted to a major trauma centre following knife-related injuries resulting from interpersonal violence (May 2015 to April 2018). Patients were identified using Emergency Department and discharge coding, blood bank and UK national Trauma Audit and Research prospective registries. Patient and injury characteristics, outcome and resource utilisation were collected from ambulance and hospital records. Findings:532 patients were identified; 93% male, median age 26 years (IQR 20-35). Median injury severity score was 9 (IQR 3-13). 346 (65%) underwent surgery; 133 (25%) required intensive care; 95 (17·9%) received blood transfusion. Median length of stay was 3·3 days (IQR 1·7-6·0). In-hospital mortality was 10/532 (1·9%). 98 patients (18·5%) had previous attendance with violence-related injuries. 24/37 females (64·9%) were injured in a domestic setting. Intoxication with alcohol (19·2%) and illicit drugs (17·6%) was common. Causative weapon was household knife in 9%, knife (other/unspecified) in 38·0%, machete in 13·9%, small folding blade (2·8%) and, unrecorded in 36·3%. Interpretation:Knife injuries constitute 12·9% of trauma team workload. Violence recidivism and intoxication are common, and females are predominantly injured in a domestic setting, presenting opportunities for targeted violence reduction interventions. 13·9% of injuries involved machetes, with implications for law enforcement strategies.\n\n\n\n**Authors:**\n\nMalik NS, Munoz B, de Courcey C, Imran R, Lee KC, Chernbumroong S, Bishop J, Lord JM, Gkoutos G, Bowley DM, Foster MA.\n\n' + ); + expect(paper.resultsInsights).toEqual(null); + expect(paper.categories).toEqual({}); + expect(paper.tags).toEqual({ + features: ['HDRUK Papers', 'Open Access'], + topics: [], + }); + expect(paper.license).toEqual(null); + expect(paper.authors).toEqual([100000]); + expect(paper.activeflag).toEqual('active'); + expect(paper.counter).toEqual(8); + expect(paper.discourseTopicId).toEqual(null); + expect(paper.relatedObjects).toEqual([]); + expect(paper.uploader).toEqual(null); + expect(paper.journal).toEqual('EClinicalMedicine'); + expect(paper.journalYear).toEqual(2020); + expect(paper.isPreprint).toEqual(false); + }); + }); +}); diff --git a/src/resources/paper/__tests__/paper.repository.it.test.js b/src/resources/paper/__tests__/paper.repository.it.test.js new file mode 100644 index 00000000..d8c634e6 --- /dev/null +++ b/src/resources/paper/__tests__/paper.repository.it.test.js @@ -0,0 +1,42 @@ +import dbHandler from '../../../config/in-memory-db'; +import PaperRepository from '../paper.repository'; +import { papersStub } from '../__mocks__/papers'; + +/** + * Connect to a new in-memory database before running any tests. + */ +beforeAll(async () => { + await dbHandler.connect(); + await dbHandler.loadData({ tools: papersStub }); +}); + +/** + * Revert to initial test data after every test. + */ +afterEach(async () => { + await dbHandler.clearDatabase(); + await dbHandler.loadData({ tools: papersStub }); +}); + +/** + * Remove and close the db and server. + */ +afterAll(async () => await dbHandler.closeDatabase()); + +describe('PaperRepository', function () { + describe('getPaper', () => { + it('should return a paper by a specified id', async function () { + const paperRepository = new PaperRepository(); + const paper = await paperRepository.getPaper(32300742); + expect(paper).toEqual(papersStub[0]); + }); + }); + + describe('getPapers', () => { + it('should return an array of papers', async function () { + const paperRepository = new PaperRepository(); + const papers = await paperRepository.getPapers(); + expect(papers.length).toBeGreaterThan(0); + }); + }); +}); diff --git a/src/resources/paper/__tests__/paper.repository.test.js b/src/resources/paper/__tests__/paper.repository.test.js new file mode 100644 index 00000000..62fa19d6 --- /dev/null +++ b/src/resources/paper/__tests__/paper.repository.test.js @@ -0,0 +1,59 @@ +import sinon from 'sinon'; + +import PaperRepository from '../paper.repository'; +import { papersStub } from '../__mocks__/papers'; + +describe('PaperRepository', function () { + describe('getPaper', function () { + it('should return a paper by a specified id', async function () { + const paperStub = papersStub[0]; + const paperRepository = new PaperRepository(); + const stub = sinon.stub(paperRepository, 'findOne').returns(paperStub); + const paper = await paperRepository.getPaper(paperStub.id); + + expect(stub.calledOnce).toBe(true); + + expect(paper.type).toEqual(paperStub.type); + expect(paper.id).toEqual(paperStub.id); + expect(paper.name).toEqual(paperStub.name); + expect(paper.description).toEqual(paperStub.description); + expect(paper.resultsInsights).toEqual(paperStub.resultsInsights); + expect(paper.paperid).toEqual(paperStub.paperid); + expect(paper.categories).toEqual(paperStub.categories); + expect(paper.license).toEqual(paperStub.license); + expect(paper.authors).toEqual(paperStub.authors); + expect(paper.activeflag).toEqual(paperStub.activeflag); + expect(paper.counter).toEqual(paperStub.counter); + expect(paper.discourseTopicId).toEqual(paperStub.discourseTopicId); + expect(paper.relatedObjects).toEqual(paperStub.relatedObjects); + expect(paper.uploader).toEqual(paperStub.uploader); + expect(paper.journal).toEqual(paperStub.journal); + expect(paper.journalYear).toEqual(paperStub.journalYear); + expect(paper.isPreprint).toEqual(paperStub.isPreprint); + }); + }); + + describe('getPapers', function () { + it('should return an array of papers', async function () { + const paperRepository = new PaperRepository(); + const stub = sinon.stub(paperRepository, 'find').returns(papersStub); + const papers = await paperRepository.getPapers(); + + expect(stub.calledOnce).toBe(true); + + expect(papers.length).toBeGreaterThan(0); + }); + }); + + describe('findCountOf', function () { + it('should return the number of documents found by a given query', async function () { + const paperRepository = new PaperRepository(); + const stub = sinon.stub(paperRepository, 'findCountOf').returns(1); + const paperCount = await paperRepository.findCountOf({ name: 'Admitted Patient Care Paper' }); + + expect(stub.calledOnce).toBe(true); + + expect(paperCount).toEqual(1); + }); + }); +}); \ No newline at end of file diff --git a/src/resources/paper/__tests__/paper.service.test.js b/src/resources/paper/__tests__/paper.service.test.js new file mode 100644 index 00000000..4936a7cd --- /dev/null +++ b/src/resources/paper/__tests__/paper.service.test.js @@ -0,0 +1,49 @@ +import sinon from 'sinon'; + +import PaperRepository from '../paper.repository'; +import PaperService from '../paper.service'; +import { papersStub } from '../__mocks__/papers'; + +describe('PaperService', function () { + describe('getPaper', function () { + it('should return a paper by a specified id', async function () { + const paperStub = papersStub[0]; + const paperRepository = new PaperRepository(); + const stub = sinon.stub(paperRepository, 'getPaper').returns(paperStub); + const paperService = new PaperService(paperRepository); + const paper = await paperService.getPaper(paperStub.id); + + expect(stub.calledOnce).toBe(true); + + expect(paper.type).toEqual(paperStub.type); + expect(paper.id).toEqual(paperStub.id); + expect(paper.name).toEqual(paperStub.name); + expect(paper.description).toEqual(paperStub.description); + expect(paper.resultsInsights).toEqual(paperStub.resultsInsights); + expect(paper.paperid).toEqual(paperStub.paperid); + expect(paper.categories).toEqual(paperStub.categories); + expect(paper.license).toEqual(paperStub.license); + expect(paper.authors).toEqual(paperStub.authors); + expect(paper.activeflag).toEqual(paperStub.activeflag); + expect(paper.counter).toEqual(paperStub.counter); + expect(paper.discourseTopicId).toEqual(paperStub.discourseTopicId); + expect(paper.relatedObjects).toEqual(paperStub.relatedObjects); + expect(paper.uploader).toEqual(paperStub.uploader); + expect(paper.journal).toEqual(paperStub.journal); + expect(paper.journalYear).toEqual(paperStub.journalYear); + expect(paper.isPreprint).toEqual(paperStub.isPreprint); + }); + }); + describe('getPapers', function () { + it('should return an array of papers', async function () { + const paperRepository = new PaperRepository(); + const stub = sinon.stub(paperRepository, 'getPapers').returns(papersStub); + const paperService = new PaperService(paperRepository); + const papers = await paperService.getPapers(); + + expect(stub.calledOnce).toBe(true); + + expect(papers.length).toBeGreaterThan(0); + }); + }); +}); diff --git a/src/resources/paper/dependency.js b/src/resources/paper/dependency.js new file mode 100644 index 00000000..14938adb --- /dev/null +++ b/src/resources/paper/dependency.js @@ -0,0 +1,5 @@ +import PaperRepository from './paper.repository'; +import PaperService from './paper.service'; + +export const paperRepository = new PaperRepository(); +export const paperService = new PaperService(paperRepository); diff --git a/src/resources/paper/paper.controller.js b/src/resources/paper/paper.controller.js new file mode 100644 index 00000000..f8db5a1f --- /dev/null +++ b/src/resources/paper/paper.controller.js @@ -0,0 +1,62 @@ +import Controller from '../base/controller'; + +export default class PaperController extends Controller { + constructor(paperService) { + super(paperService); + this.paperService = paperService; + } + + async getPaper(req, res) { + try { + // Extract id parameter from query string + const { id } = req.params; + // If no id provided, it is a bad request + if (!id) { + return res.status(400).json({ + success: false, + message: 'You must provide a paper identifier', + }); + } + // Find the paper + let paper = await this.paperService.getPaper(id, req.query); + // Return if no paper found + if (!paper) { + return res.status(404).json({ + success: false, + message: 'A paper could not be found with the provided id', + }); + } + // Return the paper + return res.status(200).json({ + success: true, + data: paper, + }); + } catch (err) { + // Return error response if something goes wrong + console.error(err); + return res.status(500).json({ + success: false, + message: 'A server error occurred, please try again', + }); + } + } + + async getPapers(req, res) { + try { + // Find the papers + let papers = await this.paperService.getPapers(req.query); + // Return the papers + return res.status(200).json({ + success: true, + data: papers + }); + } catch (err) { + // Return error response if something goes wrong + console.error(err); + return res.status(500).json({ + success: false, + message: 'A server error occurred, please try again', + }); + } + } +} diff --git a/src/resources/paper/paper.entity.js b/src/resources/paper/paper.entity.js new file mode 100644 index 00000000..4f62f7b2 --- /dev/null +++ b/src/resources/paper/paper.entity.js @@ -0,0 +1,44 @@ +import Entity from '../base/entity'; + +export default class PaperClass extends Entity { + constructor( + id, + name, + description, + resultsInsights, + link, + type, + categories, + license, + authors, + tags, + activeflag, + counter, + discourseTopicId, + relatedObjects, + uploader, + journal, + journalYear, + isPreprint + ) { + super(); + this.id = id; + this.name = name; + this.description = description; + this.resultsInsights = resultsInsights; + this.link = link; + this.type = type; + this.categories = categories; + this.license = license; + this.authors = authors; + this.tags = tags; + this.activeflag = activeflag; + this.counter = counter; + this.discourseTopicId = discourseTopicId; + this.relatedObjects = relatedObjects; + this.uploader = uploader; + this.journal = journal; + this.journalYear = journalYear; + this.isPreprint = isPreprint; + } +} diff --git a/src/resources/paper/paper.model.js b/src/resources/paper/paper.model.js new file mode 100644 index 00000000..204e6e4b --- /dev/null +++ b/src/resources/paper/paper.model.js @@ -0,0 +1,63 @@ +import { model, Schema } from 'mongoose'; + +import PaperClass from './paper.entity'; + +const paperSchema = new Schema( + { + id: Number, + type: String, + name: String, + description: String, + resultsInsights: String, + link: String, + categories: { + category: { type: String } + }, + license: String, + authors: [Number], + tags: { + features: [String], + topics: [String], + }, + activeflag: String, + updatedon: Date, + counter: Number, + discourseTopicId: Number, + relatedObjects: [ + { + objectId: String, + reason: String, + pid: String, + objectType: String, + user: String, + updated: String, + }, + ], + uploader: Number, + journal: String, + journalYear: Number, + isPreprint: Boolean, + }, + { + timestamps: true, + collection: 'tools', + toJSON: { virtuals: true }, + toObject: { virtuals: true }, + } +); + +// Virtuals + +// Pre hook query middleware +paperSchema.pre('find', function() { + this.where({type: 'paper'}); +}); + +paperSchema.pre('findOne', function() { + this.where({type: 'paper'}); +}); + +// Load entity class +paperSchema.loadClass(PaperClass); + +export const Paper = model('Paper', paperSchema, 'tools'); \ No newline at end of file diff --git a/src/resources/paper/paper.repository.js b/src/resources/paper/paper.repository.js new file mode 100644 index 00000000..346529b1 --- /dev/null +++ b/src/resources/paper/paper.repository.js @@ -0,0 +1,20 @@ +import Repository from '../base/repository'; +import { Paper } from './paper.model'; + +export default class PaperRepository extends Repository { + constructor() { + super(Paper); + this.paper = Paper; + } + + async getPaper(id, query) { + query = { ...query, id }; + const options = { lean: true }; + return this.findOne(query, options); + } + + async getPapers(query) { + const options = { lean: true }; + return this.find(query, options); + } +} diff --git a/src/resources/paper/paper.service.js b/src/resources/paper/paper.service.js new file mode 100644 index 00000000..33d2cce5 --- /dev/null +++ b/src/resources/paper/paper.service.js @@ -0,0 +1,13 @@ +export default class PaperService { + constructor(paperRepository) { + this.paperRepository = paperRepository; + } + + getPaper(id, query = {}) { + return this.paperRepository.getPaper(id, query); + } + + getPapers(query = {}) { + return this.paperRepository.getPapers(query); + } +} diff --git a/src/resources/paper/paper.route.js b/src/resources/paper/v1/paper.route.js similarity index 96% rename from src/resources/paper/paper.route.js rename to src/resources/paper/v1/paper.route.js index 13c0a9fe..afd662e2 100644 --- a/src/resources/paper/paper.route.js +++ b/src/resources/paper/v1/paper.route.js @@ -1,10 +1,10 @@ import express from 'express'; -import { Data } from '../tool/data.model'; -import { ROLES } from '../user/user.roles'; +import { Data } from '../../tool/data.model'; +import { ROLES } from '../../user/user.roles'; import passport from 'passport'; -import { utils } from '../auth'; -import { addTool, editTool, setStatus, getTools, getToolsAdmin, getAllTools } from '../tool/data.repository'; -import helper from '../utilities/helper.util'; +import { utils } from '../../auth'; +import { addTool, editTool, setStatus, getTools, getToolsAdmin, getAllTools } from '../../tool/data.repository'; +import helper from '../../utilities/helper.util'; import escape from 'escape-html'; const router = express.Router(); diff --git a/src/resources/paper/v2/paper.route.js b/src/resources/paper/v2/paper.route.js new file mode 100644 index 00000000..40c14ab0 --- /dev/null +++ b/src/resources/paper/v2/paper.route.js @@ -0,0 +1,18 @@ +import express from 'express'; +import PaperController from '../paper.controller'; +import { paperService } from '../dependency'; + +const router = express.Router(); +const paperController = new PaperController(paperService); + +// @route GET /api/v2/papers/id +// @desc Returns a paper based on identifier provided +// @access Public +router.get('/:id', (req, res) => paperController.getPaper(req, res)); + +// @route GET /api/v2/papers +// @desc Returns a collection of papers based on supplied query parameters +// @access Public +router.get('/', (req, res) => paperController.getPapers(req, res)); + +module.exports = router; diff --git a/src/resources/project/__mocks__/projects.js b/src/resources/project/__mocks__/projects.js new file mode 100644 index 00000000..91006330 --- /dev/null +++ b/src/resources/project/__mocks__/projects.js @@ -0,0 +1,207 @@ +export const projectsStub = [ + { + "_id": "5f441ce19e7d2fc067e6496f", + "categories": { + "category": "COVID-19 Research Question" + }, + "tags": { + "features": [ + "COVID-19", + "Population Health", + "BREATHE" + ], + "topics": [] + }, + "datasetfields": { + "physicalSampleAvailability": [], + "technicaldetails": [], + "versionLinks": [] + }, + "authors": [ + 1636225498764312 + ], + "toolids": [], + "datasetids": [], + "relatedObjects": [], + "id": 12234413426179104, + "type": "project", + "name": "BREATHE: COVID-19 Symptom Tracker – Calderdale", + "link": "https://www.ed.ac.uk/usher/breathe", + "description": "BREATHE: COVID-19 Symptom Tracker – Calderdale", + "activeflag": "active", + "updatedon": "2020-08-24T20:02:41.093Z", + "uploader": 22850500618628944, + "createdAt": "2020-08-24T20:02:41.099Z", + "updatedAt": "2021-02-14T12:19:02.892Z", + "counter": 11, + "persons": [], + "objects": [], + "reviews": [] + }, + { + "_id": "6026b04b81b24e35ad760091", + "categories": { + "category": "National Core Study", + "programmingLanguage": [], + "programmingLanguageVersion": null + }, + "tags": { + "features": [ + "Clinical Trials", + "Community-based", + "NCS" + ], + "topics": [ + "COVID-19" + ] + }, + "document_links": { + "doi": [], + "pdf": [], + "html": [] + }, + "datasetfields": { + "geographicCoverage": [], + "physicalSampleAvailability": [], + "technicaldetails": [], + "versionLinks": [], + "phenotypes": [] + }, + "authors": [ + 21846160652986812, + 21692037506716690, + 5785134142012602, + 146613086705361, + 190000 + ], + "showOrganisation": false, + "toolids": [], + "datasetids": [], + "relatedObjects": [], + "id": 3026710068758245, + "type": "project", + "name": "PRINCIPLE: Data Enablement", + "link": "https://www.principletrial.org/", + "description": "Development and evaluation of rapid data-enabled access to routine clinical information to enhance early recruitment to the national clinical platform trial of COVID-19 community treatments. \n\nThe Platform Randomised trial of INterventions against COVID-19 In older people (PRINCIPLE) was established in March 2020 as an Urgent Public Health, UK-wide National Priority Platform that aims to evaluate the effects of repurposed drugs for people in the community with SARS-CoV-2 infection who are at high risk of complications. The study currently aims to evaluate whether repurposed medicines speed recovery and reduce the need for hospitalisation and reduce deaths in people with suspected or proven COVID-19. The trial design allows trial arms to be stopped for proven effectiveness, futility or safety reasons, and for arms to be replaced or added. \n\nThe COVID-19 pandemic has presented unique challenges for rapidly designing, initiating, and delivering therapeutic clinical trials. PRINCIPLE is the UK national platform investigating repurposed therapies for COVID-19 treatment of older people in the community at high risk of complications. Standard methods of patient recruitment were failing to meet the required pace and scale of enrolment.\n\nThe project focused on the development and appraisal of a near real-time, data-driven, ethical approach for enhancing recruitment in community care by contacting people with a recent COVID-19 positive test result from the central NHS Test and Trace service within 24-48 hours of their test result.", + "resultsInsights": "Prior to establishing the data service, PRINCIPLE registered on average 87 participants per week. This increased by up to 87 additional people registered per week from the test data, contributing to an increase from 1,013 recruits to PRINCIPLE at the start of October 2020 to 2,802 recruits by 20th December 2020. By 12 February 2021, 4182 participants had been recruited.\n\nWhile procedural caveats were identified by the public consultation, out of 2,639 people contacted by PRINCIPLE following a positive test result, no one raised a concern about being approached.\n\nThis project developed a novel approach to using near-real time NHS operational data to recruit community-based patients within a few days of presentation with acute illness.\n\nThis approach increased recruitment, and reduced time between positive test and randomisation, allowing more rapid evaluation of treatments and increased safety for participants. End-to-end public and patient involvement in the design of the approach provided evidence to inform information governance decisions.\n\nLatest Preprint: [https://www.medrxiv.org/content/10.1101/2021.01.15.21249724v1.full](https://www.medrxiv.org/content/10.1101/2021.01.15.21249724v1.full)", + "activeflag": "active", + "updatedon": "2021-02-12T16:43:55.406Z", + "uploader": 21846160652986812, + "createdAt": "2021-02-12T16:43:55.410Z", + "updatedAt": "2021-02-12T18:28:48.872Z", + "counter": 11, + "isPreprint": null, + "journal": null, + "journalYear": null, + "license": null, + "programmingLanguage": null, + "persons": [], + "objects": [], + "reviews": [] + }, + { + "_id": "5f9936bc1e79d0f5729ca47d", + "categories": { + "category": "Digital Innovation Hub Project", + "programmingLanguage": [], + "programmingLanguageVersion": null + }, + "tags": { + "features": [ + "Inflammatory bowel disease", + "Gut Reaction" + ], + "topics": [] + }, + "datasetfields": { + "geographicCoverage": [], + "physicalSampleAvailability": [], + "technicaldetails": [], + "versionLinks": [], + "phenotypes": [] + }, + "authors": [ + 5083541653669739 + ], + "showOrganisation": false, + "toolids": [], + "datasetids": [], + "relatedObjects": [], + "id": 16090674102699244, + "type": "project", + "name": "A longitudinal study of childlessness in women with Inflammatory bowel disease", + "link": "https://bioresource.nihr.ac.uk/studies/nbr48/", + "description": "A longitudinal study of childlessness in women with Inflammatory bowel disease. Participant recall study.", + "resultsInsights": "", + "activeflag": "active", + "updatedon": "2020-10-28T09:15:40.975Z", + "uploader": 5083541653669739, + "createdAt": "2020-10-28T09:15:40.976Z", + "updatedAt": "2021-02-12T17:33:04.378Z", + "counter": 11, + "isPreprint": null, + "journal": null, + "journalYear": null, + "license": null, + "programmingLanguage": null, + "persons": [], + "objects": [], + "reviews": [] + }, + { + "_id": "5ffd6d0f056ce725fea791c0", + "categories": { + "category": "COVID-19 Research Question", + "programmingLanguage": [], + "programmingLanguageVersion": null + }, + "tags": { + "features": [ + "wastewater", + "surveillance", + "COVID-19", + "NCS" + ], + "topics": [ + "COVID-19", + "Epidemiology", + "geospatial analysis" + ] + }, + "datasetfields": { + "geographicCoverage": [], + "physicalSampleAvailability": [], + "technicaldetails": [], + "versionLinks": [], + "phenotypes": [] + }, + "authors": [ + 6744207091112187, + 9938600125923452 + ], + "showOrganisation": false, + "toolids": [], + "datasetids": [], + "relatedObjects": [], + "id": 7635028705590892, + "type": "project", + "name": "How can NCS healthcare data be connected with wastewater surveillance of COVID-19 in a privacy-preserving fashion to inform epidemiological models and democratise data access?", + "link": "https://web.www.healthdatagateway.org/project/7635028705590892", + "description": "Wastewater-based epidemiology (WBE), i.e. the monitoring of public health using samples collected from sewerage systems, offers the unique opportunity to make healthcare data widely available without compromising the privacy of individuals: by their very nature, sewerage systems aggregate the signal from thousands of people, diminishing the ability to identify any one individual. In addition to ongoing work modelling the relationship between case numbers and wastewater data, this project will deliver five additional benefits:\n* provide data on COVID-19 related hospital admissions and symptoms as a complementary signal to refine WBE modelling with higher spatiotemporal resolution than is currently available.\n* develop standards for exchanging WBE data that support automatic validation, increasing our confidence in the data.\nprovide consistent methods to aggregate healthcare data to catchment areas across England, Scotland, and Wales.\n* develop sound, privacy-preserving aggregation methods for sensitive data in the context of WBE that could help address future public health concerns, e.g. influenza outbreaks or illicit drug consumption.\n* subject to privacy due diligence, generate data products that can be shared publicly, aligning with the government’s open data strategy and allowing a broader audience to participate in WBE.\n", + "resultsInsights": "", + "activeflag": "active", + "updatedon": "2021-01-12T09:34:07.020Z", + "uploader": 6744207091112187, + "createdAt": "2021-01-12T09:34:07.026Z", + "updatedAt": "2021-02-12T17:07:57.473Z", + "counter": 75, + "isPreprint": null, + "journal": null, + "journalYear": null, + "license": null, + "programmingLanguage": null, + "persons": [], + "objects": [], + "reviews": [] + } +]; diff --git a/src/resources/project/__tests__/project.controller.test.js b/src/resources/project/__tests__/project.controller.test.js new file mode 100644 index 00000000..9abdd8e1 --- /dev/null +++ b/src/resources/project/__tests__/project.controller.test.js @@ -0,0 +1,117 @@ +import sinon from 'sinon'; +import faker from 'faker'; + +import ProjectController from '../project.controller'; +import ProjectService from '../project.service'; + +describe('ProjectController', function () { + beforeAll(() => { + console.log = sinon.stub(); + console.error = sinon.stub(); + }); + + describe('getProject', function () { + let req, res, status, json, projectService, projectController; + + beforeEach(() => { + status = sinon.stub(); + json = sinon.spy(); + res = { json, status }; + status.returns(res); + projectService = new ProjectService(); + }); + + it('should return a project that matches the id param', async function () { + req = { params: { id: faker.random.number({ min: 1, max: 999999999 }) } }; + const stubValue = { + id: req.params.id, + }; + const serviceStub = sinon.stub(projectService, 'getProject').returns(stubValue); + projectController = new ProjectController(projectService); + await projectController.getProject(req, res); + + expect(serviceStub.calledOnce).toBe(true); + expect(status.calledWith(200)).toBe(true); + expect(json.calledWith({ success: true, data: stubValue })).toBe(true); + }); + + it('should return a bad request response if no project id is provided', async function () { + req = { params: {} }; + + const serviceStub = sinon.stub(projectService, 'getProject').returns({}); + projectController = new ProjectController(projectService); + await projectController.getProject(req, res); + + expect(serviceStub.notCalled).toBe(true); + expect(status.calledWith(400)).toBe(true); + expect(json.calledWith({ success: false, message: 'You must provide a project identifier' })).toBe(true); + }); + + it('should return a not found response if no project could be found for the id provided', async function () { + req = { params: { id: faker.random.number({ min: 1, max: 999999999 }) } }; + + const serviceStub = sinon.stub(projectService, 'getProject').returns(null); + projectController = new ProjectController(projectService); + await projectController.getProject(req, res); + + expect(serviceStub.calledOnce).toBe(true); + expect(status.calledWith(404)).toBe(true); + expect(json.calledWith({ success: false, message: 'A project could not be found with the provided id' })).toBe(true); + }); + + it('should return a server error if an unexpected exception occurs', async function () { + req = { params: { id: faker.random.number({ min: 1, max: 999999999 }) } }; + + const error = new Error('A server error occurred'); + const serviceStub = sinon.stub(projectService, 'getProject').throws(error); + projectController = new ProjectController(projectService); + await projectController.getProject(req, res); + + expect(serviceStub.calledOnce).toBe(true); + expect(status.calledWith(500)).toBe(true); + expect(json.calledWith({ success: false, message: 'A server error occurred, please try again' })).toBe(true); + }); + }); + + describe('getProjects', function () { + let req, res, status, json, projectService, projectController; + req = { params: {} }; + + beforeEach(() => { + status = sinon.stub(); + json = sinon.spy(); + res = { json, status }; + status.returns(res); + projectService = new ProjectService(); + }); + + it('should return an array of projects', async function () { + const stubValue = [ + { + id: faker.random.number({ min: 1, max: 999999999 }), + }, + { + id: faker.random.number({ min: 1, max: 999999999 }), + }, + ]; + const serviceStub = sinon.stub(projectService, 'getProjects').returns(stubValue); + projectController = new ProjectController(projectService); + await projectController.getProjects(req, res); + + expect(serviceStub.calledOnce).toBe(true); + expect(status.calledWith(200)).toBe(true); + expect(json.calledWith({ success: true, data: stubValue })).toBe(true); + }); + + it('should return a server error if an unexpected exception occurs', async function () { + const error = new Error('A server error occurred'); + const serviceStub = sinon.stub(projectService, 'getProjects').throws(error); + projectController = new ProjectController(projectService); + await projectController.getProjects(req, res); + + expect(serviceStub.calledOnce).toBe(true); + expect(status.calledWith(500)).toBe(true); + expect(json.calledWith({ success: false, message: 'A server error occurred, please try again' })).toBe(true); + }); + }); +}); diff --git a/src/resources/project/__tests__/project.entity.test.js b/src/resources/project/__tests__/project.entity.test.js new file mode 100644 index 00000000..95d33cea --- /dev/null +++ b/src/resources/project/__tests__/project.entity.test.js @@ -0,0 +1,50 @@ +import ProjectClass from '../project.entity'; + +describe('ProjectEntity', function () { + describe('constructor', function () { + it('should create an instance of a project entity with the expected properties', async function () { + const project = new ProjectClass( + 12234413426179104, + 'BREATHE: COVID-19 Symptom Tracker – Calderdale', + 'BREATHE: COVID-19 Symptom Tracker – Calderdale', + null, + 'https://www.ed.ac.uk/usher/breathe', + 'project', + { + category: 'COVID-19 Research Question', + }, + null, + [1636225498764312], + { + features: ['COVID-19', 'Population Health', 'BREATHE'], + topics: [], + }, + 'active', + 11, + null, + [], + 22850500618628944 + ); + + expect(project.id).toEqual(12234413426179104); + expect(project.type).toEqual('project'); + expect(project.name).toEqual('BREATHE: COVID-19 Symptom Tracker – Calderdale'); + expect(project.description).toEqual('BREATHE: COVID-19 Symptom Tracker – Calderdale'); + expect(project.resultsInsights).toEqual(null); + expect(project.categories).toEqual({ + category: 'COVID-19 Research Question', + }); + expect(project.tags).toEqual({ + features: ['COVID-19', 'Population Health', 'BREATHE'], + topics: [], + }); + expect(project.license).toEqual(null); + expect(project.authors).toEqual([1636225498764312]); + expect(project.activeflag).toEqual('active'); + expect(project.counter).toEqual(11); + expect(project.discourseTopicId).toEqual(null); + expect(project.relatedObjects).toEqual([]); + expect(project.uploader).toEqual(22850500618628944); + }); + }); +}); diff --git a/src/resources/project/__tests__/project.repository.it.test.js b/src/resources/project/__tests__/project.repository.it.test.js new file mode 100644 index 00000000..30a70fb9 --- /dev/null +++ b/src/resources/project/__tests__/project.repository.it.test.js @@ -0,0 +1,42 @@ +import dbHandler from '../../../config/in-memory-db'; +import ProjectRepository from '../project.repository'; +import { projectsStub } from '../__mocks__/projects'; + +/** + * Connect to a new in-memory database before running any tests. + */ +beforeAll(async () => { + await dbHandler.connect(); + await dbHandler.loadData({ tools: projectsStub }); +}); + +/** + * Revert to initial test data after every test. + */ +afterEach(async () => { + await dbHandler.clearDatabase(); + await dbHandler.loadData({ tools: projectsStub }); +}); + +/** + * Remove and close the db and server. + */ +afterAll(async () => await dbHandler.closeDatabase()); + +describe('ProjectRepository', function () { + describe('getProject', () => { + it('should return a project by a specified id', async function () { + const projectRepository = new ProjectRepository(); + const project = await projectRepository.getProject(12234413426179104); + expect(project).toEqual(projectsStub[0]); + }); + }); + + describe('getProjects', () => { + it('should return an array of projects', async function () { + const projectRepository = new ProjectRepository(); + const projects = await projectRepository.getProjects(); + expect(projects.length).toBeGreaterThan(0); + }); + }); +}); diff --git a/src/resources/project/__tests__/project.repository.test.js b/src/resources/project/__tests__/project.repository.test.js new file mode 100644 index 00000000..7d8da570 --- /dev/null +++ b/src/resources/project/__tests__/project.repository.test.js @@ -0,0 +1,56 @@ +import sinon from 'sinon'; + +import ProjectRepository from '../project.repository'; +import { projectsStub } from '../__mocks__/projects'; + +describe('ProjectRepository', function () { + describe('getProject', function () { + it('should return a project by a specified id', async function () { + const projectStub = projectsStub[0]; + const projectRepository = new ProjectRepository(); + const stub = sinon.stub(projectRepository, 'findOne').returns(projectStub); + const project = await projectRepository.getProject(projectStub.id); + + expect(stub.calledOnce).toBe(true); + + expect(project.type).toEqual(projectStub.type); + expect(project.id).toEqual(projectStub.id); + expect(project.name).toEqual(projectStub.name); + expect(project.description).toEqual(projectStub.description); + expect(project.resultsInsights).toEqual(projectStub.resultsInsights); + expect(project.projectid).toEqual(projectStub.projectid); + expect(project.categories).toEqual(projectStub.categories); + expect(project.license).toEqual(projectStub.license); + expect(project.authors).toEqual(projectStub.authors); + expect(project.activeflag).toEqual(projectStub.activeflag); + expect(project.counter).toEqual(projectStub.counter); + expect(project.discourseTopicId).toEqual(projectStub.discourseTopicId); + expect(project.relatedObjects).toEqual(projectStub.relatedObjects); + expect(project.uploader).toEqual(projectStub.uploader); + }); + }); + + describe('getProjects', function () { + it('should return an array of projects', async function () { + const projectRepository = new ProjectRepository(); + const stub = sinon.stub(projectRepository, 'find').returns(projectsStub); + const projects = await projectRepository.getProjects(); + + expect(stub.calledOnce).toBe(true); + + expect(projects.length).toBeGreaterThan(0); + }); + }); + + describe('findCountOf', function () { + it('should return the number of documents found by a given query', async function () { + const projectRepository = new ProjectRepository(); + const stub = sinon.stub(projectRepository, 'findCountOf').returns(1); + const projectCount = await projectRepository.findCountOf({ name: 'Admitted Patient Care Project' }); + + expect(stub.calledOnce).toBe(true); + + expect(projectCount).toEqual(1); + }); + }); +}); \ No newline at end of file diff --git a/src/resources/project/__tests__/project.service.test.js b/src/resources/project/__tests__/project.service.test.js new file mode 100644 index 00000000..45112fe6 --- /dev/null +++ b/src/resources/project/__tests__/project.service.test.js @@ -0,0 +1,46 @@ +import sinon from 'sinon'; + +import ProjectRepository from '../project.repository'; +import ProjectService from '../project.service'; +import { projectsStub } from '../__mocks__/projects'; + +describe('ProjectService', function () { + describe('getProject', function () { + it('should return a project by a specified id', async function () { + const projectStub = projectsStub[0]; + const projectRepository = new ProjectRepository(); + const stub = sinon.stub(projectRepository, 'getProject').returns(projectStub); + const projectService = new ProjectService(projectRepository); + const project = await projectService.getProject(projectStub.id); + + expect(stub.calledOnce).toBe(true); + + expect(project.type).toEqual(projectStub.type); + expect(project.id).toEqual(projectStub.id); + expect(project.name).toEqual(projectStub.name); + expect(project.description).toEqual(projectStub.description); + expect(project.resultsInsights).toEqual(projectStub.resultsInsights); + expect(project.projectid).toEqual(projectStub.projectid); + expect(project.categories).toEqual(projectStub.categories); + expect(project.license).toEqual(projectStub.license); + expect(project.authors).toEqual(projectStub.authors); + expect(project.activeflag).toEqual(projectStub.activeflag); + expect(project.counter).toEqual(projectStub.counter); + expect(project.discourseTopicId).toEqual(projectStub.discourseTopicId); + expect(project.relatedObjects).toEqual(projectStub.relatedObjects); + expect(project.uploader).toEqual(projectStub.uploader); + }); + }); + describe('getProjects', function () { + it('should return an array of projects', async function () { + const projectRepository = new ProjectRepository(); + const stub = sinon.stub(projectRepository, 'getProjects').returns(projectsStub); + const projectService = new ProjectService(projectRepository); + const projects = await projectService.getProjects(); + + expect(stub.calledOnce).toBe(true); + + expect(projects.length).toBeGreaterThan(0); + }); + }); +}); diff --git a/src/resources/project/dependency.js b/src/resources/project/dependency.js new file mode 100644 index 00000000..32f6af73 --- /dev/null +++ b/src/resources/project/dependency.js @@ -0,0 +1,5 @@ +import ProjectRepository from './project.repository'; +import ProjectService from './project.service'; + +export const projectRepository = new ProjectRepository(); +export const projectService = new ProjectService(projectRepository); diff --git a/src/resources/project/project.controller.js b/src/resources/project/project.controller.js new file mode 100644 index 00000000..8c36018e --- /dev/null +++ b/src/resources/project/project.controller.js @@ -0,0 +1,62 @@ +import Controller from '../base/controller'; + +export default class ProjectController extends Controller { + constructor(projectService) { + super(projectService); + this.projectService = projectService; + } + + async getProject(req, res) { + try { + // Extract id parameter from query string + const { id } = req.params; + // If no id provided, it is a bad request + if (!id) { + return res.status(400).json({ + success: false, + message: 'You must provide a project identifier', + }); + } + // Find the project + let project = await this.projectService.getProject(id, req.query); + // Return if no project found + if (!project) { + return res.status(404).json({ + success: false, + message: 'A project could not be found with the provided id', + }); + } + // Return the project + return res.status(200).json({ + success: true, + data: project, + }); + } catch (err) { + // Return error response if something goes wrong + console.error(err); + return res.status(500).json({ + success: false, + message: 'A server error occurred, please try again', + }); + } + } + + async getProjects(req, res) { + try { + // Find the projects + let projects = await this.projectService.getProjects(req.query); + // Return the projects + return res.status(200).json({ + success: true, + data: projects + }); + } catch (err) { + // Return error response if something goes wrong + console.error(err); + return res.status(500).json({ + success: false, + message: 'A server error occurred, please try again', + }); + } + } +} diff --git a/src/resources/project/project.entity.js b/src/resources/project/project.entity.js new file mode 100644 index 00000000..0d2935b9 --- /dev/null +++ b/src/resources/project/project.entity.js @@ -0,0 +1,38 @@ +import Entity from '../base/entity'; + +export default class ProjectClass extends Entity { + constructor( + id, + name, + description, + resultsInsights, + link, + type, + categories, + license, + authors, + tags, + activeflag, + counter, + discourseTopicId, + relatedObjects, + uploader + ) { + super(); + this.id = id; + this.name = name; + this.description = description; + this.resultsInsights = resultsInsights; + this.link = link; + this.type = type; + this.categories = categories; + this.license = license; + this.authors = authors; + this.tags = tags; + this.activeflag = activeflag; + this.counter = counter; + this.discourseTopicId = discourseTopicId; + this.relatedObjects = relatedObjects; + this.uploader = uploader; + } +} diff --git a/src/resources/project/project.model.js b/src/resources/project/project.model.js new file mode 100644 index 00000000..a172196f --- /dev/null +++ b/src/resources/project/project.model.js @@ -0,0 +1,60 @@ +import { model, Schema } from 'mongoose'; + +import ProjectClass from './project.entity'; + +const projectSchema = new Schema( + { + id: Number, + type: String, + name: String, + description: String, + resultsInsights: String, + link: String, + categories: { + category: { type: String } + }, + license: String, + authors: [Number], + tags: { + features: [String], + topics: [String], + }, + activeflag: String, + updatedon: Date, + counter: Number, + discourseTopicId: Number, + relatedObjects: [ + { + objectId: String, + reason: String, + pid: String, + objectType: String, + user: String, + updated: String, + }, + ], + uploader: Number, + }, + { + timestamps: true, + collection: 'tools', + toJSON: { virtuals: true }, + toObject: { virtuals: true }, + } +); + +// Virtuals + +// Pre hook query middleware +projectSchema.pre('find', function() { + this.where({type: 'project'}); +}); + +projectSchema.pre('findOne', function() { + this.where({type: 'project'}); +}); + +// Load entity class +projectSchema.loadClass(ProjectClass); + +export const Project = model('Project', projectSchema, 'tools'); \ No newline at end of file diff --git a/src/resources/project/project.repository.js b/src/resources/project/project.repository.js new file mode 100644 index 00000000..26764dfe --- /dev/null +++ b/src/resources/project/project.repository.js @@ -0,0 +1,20 @@ +import Repository from '../base/repository'; +import { Project } from './project.model'; + +export default class ProjectRepository extends Repository { + constructor() { + super(Project); + this.project = Project; + } + + async getProject(id, query) { + query = { ...query, id }; + const options = { lean: true }; + return this.findOne(query, options); + } + + async getProjects(query) { + const options = { lean: true }; + return this.find(query, options); + } +} diff --git a/src/resources/project/project.service.js b/src/resources/project/project.service.js new file mode 100644 index 00000000..2b67325f --- /dev/null +++ b/src/resources/project/project.service.js @@ -0,0 +1,13 @@ +export default class ProjectService { + constructor(projectRepository) { + this.projectRepository = projectRepository; + } + + getProject(id, query = {}) { + return this.projectRepository.getProject(id, query); + } + + getProjects(query = {}) { + return this.projectRepository.getProjects(query); + } +} diff --git a/src/resources/project/project.route.js b/src/resources/project/v1/project.route.js similarity index 94% rename from src/resources/project/project.route.js rename to src/resources/project/v1/project.route.js index 14def5fe..4ade03a5 100644 --- a/src/resources/project/project.route.js +++ b/src/resources/project/v1/project.route.js @@ -1,11 +1,12 @@ import express from 'express'; -import { Data } from '../tool/data.model'; -import { ROLES } from '../user/user.roles'; -import passport from 'passport'; -import { utils } from '../auth'; -import { addTool, editTool, setStatus, getTools, getToolsAdmin, getAllTools } from '../tool/data.repository'; -import helper from '../utilities/helper.util'; +import helper from '../../utilities/helper.util'; import escape from 'escape-html'; +import passport from 'passport'; + +import { Data } from '../../tool/data.model'; +import { ROLES } from '../../user/user.roles'; +import { utils } from '../../auth'; +import { addTool, editTool, setStatus, getTools, getToolsAdmin, getAllTools } from '../../tool/data.repository'; const router = express.Router(); diff --git a/src/resources/project/v2/project.route.js b/src/resources/project/v2/project.route.js new file mode 100644 index 00000000..fed79dcb --- /dev/null +++ b/src/resources/project/v2/project.route.js @@ -0,0 +1,18 @@ +import express from 'express'; +import ProjectController from '../project.controller'; +import { projectService } from '../dependency'; + +const router = express.Router(); +const projectController = new ProjectController(projectService); + +// @route GET /api/v2/projects/id +// @desc Returns a project based on identifier provided +// @access Public +router.get('/:id', (req, res) => projectController.getProject(req, res)); + +// @route GET /api/v2/projects +// @desc Returns a collection of projects based on supplied query parameters +// @access Public +router.get('/', (req, res) => projectController.getProjects(req, res)); + +module.exports = router; diff --git a/src/resources/tool/__mocks__/tools.js b/src/resources/tool/__mocks__/tools.js new file mode 100644 index 00000000..346697b4 --- /dev/null +++ b/src/resources/tool/__mocks__/tools.js @@ -0,0 +1,147 @@ +export const toolsStub = [ + { + _id: '5ece82142abda8b3a06f1a22', + activeflag: 'active', + id: 19008, + type: 'tool', + license: '', + name: 'C19 Track', + description: 'Help paint a more accurate picture of covid-19 in your community', + tags: { + features: ['Community Tracking'], + topics: ['COVID-19'], + }, + link: 'https://c19track.com/', + updatedon: '2020-05-27T00:00:00.000Z', + datasetids: [], + categories: { + category: 'Questionnaire', + programmingLanguageVersion: '', + programmingLanguage: [], + }, + authors: [190000], + projectids: [], + authors_names: ['University of St. Andrews'], + counter: 65, + updatedAt: '2021-02-14T16:23:25.846Z', + persons: [], + objects: [], + reviews: [], + }, + { + _id: '5ece82142abda8b3a06f1a1b', + updatedon: '2020-05-27T00:00:00.000Z', + datasetids: [], + license: '', + description: '', + name: 'CMMID Interactive Applications', + tags: { + topics: ['COVID-19'], + features: ['Modelling', 'Visualisation'], + }, + link: 'https://cmmid.github.io/visualisations', + activeflag: 'active', + id: 19001, + type: 'tool', + authors_names: ['LSHTM'], + projectids: [], + authors: [190000], + categories: { + category: 'Data Modelling', + programmingLanguageVersion: '', + programmingLanguage: ['R'], + }, + counter: 99, + updatedAt: '2021-02-14T12:20:22.364Z', + discourseTopicId: 13, + programmingLanguage: [ + { + programmingLanguage: 'R', + version: '', + }, + ], + persons: [], + objects: [], + reviews: [], + }, + { + _id: '5f71c7537815d0195a1e9b74', + categories: { + programmingLanguage: ['Java'], + category: 'Software', + programmingLanguageVersion: '', + }, + tags: { + features: ['Data Cleaning', 'Data Import/Export', 'Data Integration', 'Data Reduction', 'Data Transformation', 'Data Visualization'], + topics: [], + }, + datasetfields: { + geographicCoverage: [], + physicalSampleAvailability: [], + technicaldetails: [], + versionLinks: [], + phenotypes: [], + }, + authors: [4442271343443529], + toolids: [], + datasetids: [], + relatedObjects: [], + id: 8656803550224388, + type: 'tool', + name: 'DataPreparator', + link: 'https://www.datapreparator.com/', + description: + 'DataPreparator is a free software tool designed to assist with common tasks of data preparation (or data preprocessing) in data analysis and data mining. \n\nDataPreparator provides:\n\n-A variety of techniques for data cleaning, transformation, and exploration\n-Chaining of preprocessing operators into a flow graph (operator tree)\n-Handling of large volumes of data (since data sets are not stored in the computer memory)\n-Stand alone tool independent of any other tools\n-User friendly graphical user interface\n\nDataPreparator can assist you with exploring and preparing data in various ways prior to data analysis or data mining. It includes operators for cleaning, discretization, numeration, scaling, attribute selection, missing values, outliers, statistics, visualization, balancing, sampling, row selection, and several other tasks. See Features for details.', + license: 'https://www.datapreparator.com/software-license-agreement-2.html', + activeflag: 'active', + updatedon: '2020-09-28T11:21:55.527Z', + uploader: 4442271343443529, + createdAt: '2020-09-28T11:21:55.530Z', + updatedAt: '2021-02-14T08:55:13.596Z', + counter: 12, + programmingLanguage: [ + { + programmingLanguage: 'Java', + version: '', + }, + ], + persons: [], + objects: [], + reviews: [], + }, + { + _id: '5eda27d4aac50ef405c861fd', + categories: { + programmingLanguage: ['R'], + category: 'Package', + programmingLanguageVersion: '', + }, + tags: { + features: ['Imputation', 'Phenomics', 'Data management', 'Natural language processing'], + topics: [], + }, + authors: [19451923771745828], + id: 22796788425888104, + type: 'tool', + name: 'CALIBER health records research toolkit', + link: 'http://caliberanalysis.r-forge.r-project.org/', + description: + 'This project comprises a set of R packages to assist in epidemiological studies using electronic health records databases.\n\nCALIBER (http://caliberresearch.org/) is led from the Farr Institute @ London. CALIBER investigators represent a collaboration between epidemiologists, clinicians, statisticians, health informaticians and computer scientists with initial funding from the Wellcome Trust and the National Institute for Health Research.\n\nThe goal of CALIBER is to provide evidence across different stages of translation, from discovery, through evaluation to implementation where electronic health records provide new scientific opportunities.\n', + license: 'GNU General Public License (GPL)', + activeflag: 'active', + updatedon: '2020-06-05T11:09:08.566Z', + createdAt: '2020-06-05T11:09:08.573Z', + updatedAt: '2021-02-13T22:55:00.027Z', + counter: 127, + discourseTopicId: 30, + programmingLanguage: [ + { + programmingLanguage: 'R', + version: '', + }, + ], + persons: [], + objects: [], + reviews: [], + }, +]; diff --git a/src/resources/tool/__tests__/tool.controller.test.js b/src/resources/tool/__tests__/tool.controller.test.js new file mode 100644 index 00000000..436621cb --- /dev/null +++ b/src/resources/tool/__tests__/tool.controller.test.js @@ -0,0 +1,117 @@ +import sinon from 'sinon'; +import faker from 'faker'; + +import ToolController from '../v2/tool.controller'; +import ToolService from '../v2/tool.service'; + +describe('ToolController', function () { + beforeAll(() => { + console.log = sinon.stub(); + console.error = sinon.stub(); + }); + + describe('getTool', function () { + let req, res, status, json, toolService, toolController; + + beforeEach(() => { + status = sinon.stub(); + json = sinon.spy(); + res = { json, status }; + status.returns(res); + toolService = new ToolService(); + }); + + it('should return a tool that matches the id param', async function () { + req = { params: { id: faker.random.number({ min: 1, max: 999999999 }) } }; + const stubValue = { + id: req.params.id, + }; + const serviceStub = sinon.stub(toolService, 'getTool').returns(stubValue); + toolController = new ToolController(toolService); + await toolController.getTool(req, res); + + expect(serviceStub.calledOnce).toBe(true); + expect(status.calledWith(200)).toBe(true); + expect(json.calledWith({ success: true, data: stubValue })).toBe(true); + }); + + it('should return a bad request response if no tool id is provided', async function () { + req = { params: {} }; + + const serviceStub = sinon.stub(toolService, 'getTool').returns({}); + toolController = new ToolController(toolService); + await toolController.getTool(req, res); + + expect(serviceStub.notCalled).toBe(true); + expect(status.calledWith(400)).toBe(true); + expect(json.calledWith({ success: false, message: 'You must provide a tool identifier' })).toBe(true); + }); + + it('should return a not found response if no tool could be found for the id provided', async function () { + req = { params: { id: faker.random.number({ min: 1, max: 999999999 }) } }; + + const serviceStub = sinon.stub(toolService, 'getTool').returns(null); + toolController = new ToolController(toolService); + await toolController.getTool(req, res); + + expect(serviceStub.calledOnce).toBe(true); + expect(status.calledWith(404)).toBe(true); + expect(json.calledWith({ success: false, message: 'A tool could not be found with the provided id' })).toBe(true); + }); + + it('should return a server error if an unexpected exception occurs', async function () { + req = { params: { id: faker.random.number({ min: 1, max: 999999999 }) } }; + + const error = new Error('A server error occurred'); + const serviceStub = sinon.stub(toolService, 'getTool').throws(error); + toolController = new ToolController(toolService); + await toolController.getTool(req, res); + + expect(serviceStub.calledOnce).toBe(true); + expect(status.calledWith(500)).toBe(true); + expect(json.calledWith({ success: false, message: 'A server error occurred, please try again' })).toBe(true); + }); + }); + + describe('getTools', function () { + let req, res, status, json, toolService, toolController; + req = { params: {} }; + + beforeEach(() => { + status = sinon.stub(); + json = sinon.spy(); + res = { json, status }; + status.returns(res); + toolService = new ToolService(); + }); + + it('should return an array of tools', async function () { + const stubValue = [ + { + id: faker.random.number({ min: 1, max: 999999999 }), + }, + { + id: faker.random.number({ min: 1, max: 999999999 }), + }, + ]; + const serviceStub = sinon.stub(toolService, 'getTools').returns(stubValue); + toolController = new ToolController(toolService); + await toolController.getTools(req, res); + + expect(serviceStub.calledOnce).toBe(true); + expect(status.calledWith(200)).toBe(true); + expect(json.calledWith({ success: true, data: stubValue })).toBe(true); + }); + + it('should return a server error if an unexpected exception occurs', async function () { + const error = new Error('A server error occurred'); + const serviceStub = sinon.stub(toolService, 'getTools').throws(error); + toolController = new ToolController(toolService); + await toolController.getTools(req, res); + + expect(serviceStub.calledOnce).toBe(true); + expect(status.calledWith(500)).toBe(true); + expect(json.calledWith({ success: false, message: 'A server error occurred, please try again' })).toBe(true); + }); + }); +}); diff --git a/src/resources/tool/__tests__/tool.entity.test.js b/src/resources/tool/__tests__/tool.entity.test.js new file mode 100644 index 00000000..396f9d72 --- /dev/null +++ b/src/resources/tool/__tests__/tool.entity.test.js @@ -0,0 +1,61 @@ +import ToolClass from '../v2/tool.entity'; + +describe('ToolEntity', function () { + describe('constructor', function () { + it('should create an instance of a tool entity with the expected properties', async function () { + const tool = new ToolClass( + 19008, + 'C19 Track', + 'Help paint a more accurate picture of covid-19 in your community', + null, + 'https://c19track.com/', + 'tool', + { + category: 'Questionnaire', + programmingLanguageVersion: '', + programmingLanguage: [], + }, + '', + [190000], + null, + 'active', + 65, + null, + [], + null, + [ + { + programmingLanguage: 'R', + version: '', + }, + ] + ); + + expect(tool.id).toEqual(19008); + expect(tool.type).toEqual('tool'); + expect(tool.name).toEqual('C19 Track'); + expect(tool.description).toEqual('Help paint a more accurate picture of covid-19 in your community'); + expect(tool.resultsInsights).toEqual(null); + expect(tool.link).toEqual('https://c19track.com/'); + expect(tool.categories).toEqual({ + category: 'Questionnaire', + programmingLanguageVersion: '', + programmingLanguage: [], + }); + expect(tool.tags).toEqual(null); + expect(tool.license).toEqual(''); + expect(tool.authors).toEqual([190000]); + expect(tool.activeflag).toEqual('active'); + expect(tool.counter).toEqual(65); + expect(tool.discourseTopicId).toEqual(null); + expect(tool.relatedObjects).toEqual([]); + expect(tool.uploader).toEqual(null); + expect(tool.programmingLanguage).toEqual([ + { + programmingLanguage: 'R', + version: '', + }, + ]); + }); + }); +}); diff --git a/src/resources/tool/__tests__/tool.repository.it.test.js b/src/resources/tool/__tests__/tool.repository.it.test.js new file mode 100644 index 00000000..e2cacda3 --- /dev/null +++ b/src/resources/tool/__tests__/tool.repository.it.test.js @@ -0,0 +1,42 @@ +import dbHandler from '../../../config/in-memory-db'; +import ToolRepository from '../v2/tool.repository'; +import { toolsStub } from '../__mocks__/tools'; + +/** + * Connect to a new in-memory database before running any tests. + */ +beforeAll(async () => { + await dbHandler.connect(); + await dbHandler.loadData({ tools: toolsStub }); +}); + +/** + * Revert to initial test data after every test. + */ +afterEach(async () => { + await dbHandler.clearDatabase(); + await dbHandler.loadData({ tools: toolsStub }); +}); + +/** + * Remove and close the db and server. + */ +afterAll(async () => await dbHandler.closeDatabase()); + +describe('ToolRepository', function () { + describe('getTool', () => { + it('should return a tool by a specified id', async function () { + const toolRepository = new ToolRepository(); + const tool = await toolRepository.getTool(19008); + expect(tool).toEqual(toolsStub[0]); + }); + }); + + describe('getTools', () => { + it('should return an array of tools', async function () { + const toolRepository = new ToolRepository(); + const tools = await toolRepository.getTools(); + expect(tools.length).toBeGreaterThan(0); + }); + }); +}); diff --git a/src/resources/tool/__tests__/tool.repository.test.js b/src/resources/tool/__tests__/tool.repository.test.js new file mode 100644 index 00000000..6b43fb24 --- /dev/null +++ b/src/resources/tool/__tests__/tool.repository.test.js @@ -0,0 +1,57 @@ +import sinon from 'sinon'; + +import ToolRepository from '../v2/tool.repository'; +import { toolsStub } from '../__mocks__/tools'; + +describe('ToolRepository', function () { + describe('getTool', function () { + it('should return a tool by a specified id', async function () { + const toolStub = toolsStub[0]; + const toolRepository = new ToolRepository(); + const stub = sinon.stub(toolRepository, 'findOne').returns(toolStub); + const tool = await toolRepository.getTool(toolStub.id); + + expect(stub.calledOnce).toBe(true); + + expect(tool.type).toEqual(toolStub.type); + expect(tool.id).toEqual(toolStub.id); + expect(tool.name).toEqual(toolStub.name); + expect(tool.description).toEqual(toolStub.description); + expect(tool.resultsInsights).toEqual(toolStub.resultsInsights); + expect(tool.toolid).toEqual(toolStub.toolid); + expect(tool.categories).toEqual(toolStub.categories); + expect(tool.license).toEqual(toolStub.license); + expect(tool.authors).toEqual(toolStub.authors); + expect(tool.activeflag).toEqual(toolStub.activeflag); + expect(tool.counter).toEqual(toolStub.counter); + expect(tool.discourseTopicId).toEqual(toolStub.discourseTopicId); + expect(tool.relatedObjects).toEqual(toolStub.relatedObjects); + expect(tool.uploader).toEqual(toolStub.uploader); + expect(tool.programmingLanguage).toEqual(toolStub.programmingLanguage); + }); + }); + + describe('getTools', function () { + it('should return an array of tools', async function () { + const toolRepository = new ToolRepository(); + const stub = sinon.stub(toolRepository, 'find').returns(toolsStub); + const tools = await toolRepository.getTools(); + + expect(stub.calledOnce).toBe(true); + + expect(tools.length).toBeGreaterThan(0); + }); + }); + + describe('findCountOf', function () { + it('should return the number of documents found by a given query', async function () { + const toolRepository = new ToolRepository(); + const stub = sinon.stub(toolRepository, 'findCountOf').returns(1); + const toolCount = await toolRepository.findCountOf({ name: 'Admitted Patient Care Tool' }); + + expect(stub.calledOnce).toBe(true); + + expect(toolCount).toEqual(1); + }); + }); +}); \ No newline at end of file diff --git a/src/resources/tool/__tests__/tool.service.test.js b/src/resources/tool/__tests__/tool.service.test.js new file mode 100644 index 00000000..e36b698d --- /dev/null +++ b/src/resources/tool/__tests__/tool.service.test.js @@ -0,0 +1,47 @@ +import sinon from 'sinon'; + +import ToolRepository from '../v2/tool.repository'; +import ToolService from '../v2/tool.service'; +import { toolsStub } from '../__mocks__/tools'; + +describe('ToolService', function () { + describe('getTool', function () { + it('should return a tool by a specified id', async function () { + const toolStub = toolsStub[0]; + const toolRepository = new ToolRepository(); + const stub = sinon.stub(toolRepository, 'getTool').returns(toolStub); + const toolService = new ToolService(toolRepository); + const tool = await toolService.getTool(toolStub.id); + + expect(stub.calledOnce).toBe(true); + + expect(tool.type).toEqual(toolStub.type); + expect(tool.id).toEqual(toolStub.id); + expect(tool.name).toEqual(toolStub.name); + expect(tool.description).toEqual(toolStub.description); + expect(tool.resultsInsights).toEqual(toolStub.resultsInsights); + expect(tool.toolid).toEqual(toolStub.toolid); + expect(tool.categories).toEqual(toolStub.categories); + expect(tool.license).toEqual(toolStub.license); + expect(tool.authors).toEqual(toolStub.authors); + expect(tool.activeflag).toEqual(toolStub.activeflag); + expect(tool.counter).toEqual(toolStub.counter); + expect(tool.discourseTopicId).toEqual(toolStub.discourseTopicId); + expect(tool.relatedObjects).toEqual(toolStub.relatedObjects); + expect(tool.uploader).toEqual(toolStub.uploader); + expect(tool.programmingLanguage).toEqual(toolStub.programmingLanguage); + }); + }); + describe('getTools', function () { + it('should return an array of tools', async function () { + const toolRepository = new ToolRepository(); + const stub = sinon.stub(toolRepository, 'getTools').returns(toolsStub); + const toolService = new ToolService(toolRepository); + const tools = await toolService.getTools(); + + expect(stub.calledOnce).toBe(true); + + expect(tools.length).toBeGreaterThan(0); + }); + }); +}); diff --git a/src/resources/tool/tool.route.js b/src/resources/tool/v1/tool.route.js similarity index 96% rename from src/resources/tool/tool.route.js rename to src/resources/tool/v1/tool.route.js index c03c9ee0..b5541317 100644 --- a/src/resources/tool/tool.route.js +++ b/src/resources/tool/v1/tool.route.js @@ -1,16 +1,16 @@ import express from 'express'; -import { ROLES } from '../user/user.roles'; -import { Reviews } from './review.model'; -import { Data } from '../tool/data.model'; +import { ROLES } from '../../user/user.roles'; +import { Reviews } from '../review.model'; +import { Data } from '../data.model'; import passport from 'passport'; -import { utils } from '../auth'; -import { UserModel } from '../user/user.model'; -import { MessagesModel } from '../message/message.model'; -import { addTool, editTool, setStatus, getTools, getToolsAdmin, getAllTools } from '../tool/data.repository'; -import emailGenerator from '../utilities/emailGenerator.util'; -import inputSanitizer from '../utilities/inputSanitizer'; +import { utils } from '../../auth'; +import { UserModel } from '../../user/user.model'; +import { MessagesModel } from '../../message/message.model'; +import { addTool, editTool, setStatus, getTools, getToolsAdmin, getAllTools } from '../data.repository'; +import emailGenerator from '../../utilities/emailGenerator.util'; +import inputSanitizer from '../../utilities/inputSanitizer'; import _ from 'lodash'; -import helper from '../utilities/helper.util'; +import helper from '../../utilities/helper.util'; import escape from 'escape-html'; const hdrukEmail = `enquiry@healthdatagateway.org`; const router = express.Router(); diff --git a/src/resources/tool/v2/dependency.js b/src/resources/tool/v2/dependency.js new file mode 100644 index 00000000..ff7badc3 --- /dev/null +++ b/src/resources/tool/v2/dependency.js @@ -0,0 +1,5 @@ +import ToolRepository from './tool.repository'; +import ToolService from './tool.service'; + +export const toolRepository = new ToolRepository(); +export const toolService = new ToolService(toolRepository); diff --git a/src/resources/tool/v2/tool.controller.js b/src/resources/tool/v2/tool.controller.js new file mode 100644 index 00000000..2f8e5c8f --- /dev/null +++ b/src/resources/tool/v2/tool.controller.js @@ -0,0 +1,62 @@ +import Controller from '../../base/controller'; + +export default class ToolController extends Controller { + constructor(toolService) { + super(toolService); + this.toolService = toolService; + } + + async getTool(req, res) { + try { + // Extract id parameter from query string + const { id } = req.params; + // If no id provided, it is a bad request + if (!id) { + return res.status(400).json({ + success: false, + message: 'You must provide a tool identifier', + }); + } + // Find the tool + let tool = await this.toolService.getTool(id, req.query); + // Return if no tool found + if (!tool) { + return res.status(404).json({ + success: false, + message: 'A tool could not be found with the provided id', + }); + } + // Return the tool + return res.status(200).json({ + success: true, + data: tool, + }); + } catch (err) { + // Return error response if something goes wrong + console.error(err); + return res.status(500).json({ + success: false, + message: 'A server error occurred, please try again', + }); + } + } + + async getTools(req, res) { + try { + // Find the tools + let tools = await this.toolService.getTools(req.query); + // Return the tools + return res.status(200).json({ + success: true, + data: tools + }); + } catch (err) { + // Return error response if something goes wrong + console.error(err); + return res.status(500).json({ + success: false, + message: 'A server error occurred, please try again', + }); + } + } +} diff --git a/src/resources/tool/v2/tool.entity.js b/src/resources/tool/v2/tool.entity.js new file mode 100644 index 00000000..c19609ae --- /dev/null +++ b/src/resources/tool/v2/tool.entity.js @@ -0,0 +1,40 @@ +import Entity from '../../base/entity'; + +export default class ToolClass extends Entity { + constructor( + id, + name, + description, + resultsInsights, + link, + type, + categories, + license, + authors, + tags, + activeflag, + counter, + discourseTopicId, + relatedObjects, + uploader, + programmingLanguage + ) { + super(); + this.id = id; + this.name = name; + this.description = description; + this.resultsInsights = resultsInsights; + this.link = link; + this.type = type; + this.categories = categories; + this.license = license; + this.authors = authors; + this.tags = tags; + this.activeflag = activeflag; + this.counter = counter; + this.discourseTopicId = discourseTopicId; + this.relatedObjects = relatedObjects; + this.uploader = uploader; + this.programmingLanguage = programmingLanguage; + } +} diff --git a/src/resources/tool/v2/tool.model.js b/src/resources/tool/v2/tool.model.js new file mode 100644 index 00000000..c858be99 --- /dev/null +++ b/src/resources/tool/v2/tool.model.js @@ -0,0 +1,68 @@ +import { model, Schema } from 'mongoose'; + +import ToolClass from './tool.entity'; + +const toolSchema = new Schema( + { + id: Number, + type: String, + name: String, + description: String, + resultsInsights: String, + link: String, + categories: { + category: { type: String }, + programmingLanguage: { type: [String] }, + programmingLanguageVersion: { type: String }, + }, + license: String, + authors: [Number], + tags: { + features: [String], + topics: [String], + }, + activeflag: String, + updatedon: Date, + counter: Number, + discourseTopicId: Number, + relatedObjects: [ + { + objectId: String, + reason: String, + pid: String, + objectType: String, + user: String, + updated: String, + }, + ], + uploader: Number, + programmingLanguage: [ + { + programmingLanguage: String, + version: String, + }, + ], + }, + { + timestamps: true, + collection: 'tools', + toJSON: { virtuals: true }, + toObject: { virtuals: true }, + } +); + +// Virtuals + +// Pre hook query middleware +toolSchema.pre('find', function() { + this.where({type: 'tool'}); +}); + +toolSchema.pre('findOne', function() { + this.where({type: 'tool'}); +}); + +// Load entity class +toolSchema.loadClass(ToolClass); + +export const Tool = model('Tool', toolSchema, 'tools'); \ No newline at end of file diff --git a/src/resources/tool/v2/tool.repository.js b/src/resources/tool/v2/tool.repository.js new file mode 100644 index 00000000..c224e376 --- /dev/null +++ b/src/resources/tool/v2/tool.repository.js @@ -0,0 +1,20 @@ +import Repository from '../../base/repository'; +import { Tool } from './tool.model'; + +export default class ToolRepository extends Repository { + constructor() { + super(Tool); + this.tool = Tool; + } + + async getTool(id, query) { + query = { ...query, id }; + const options = { lean: true }; + return this.findOne(query, options); + } + + async getTools(query) { + const options = { lean: true }; + return this.find(query, options); + } +} diff --git a/src/resources/tool/v2/tool.route.js b/src/resources/tool/v2/tool.route.js new file mode 100644 index 00000000..4eb0a7de --- /dev/null +++ b/src/resources/tool/v2/tool.route.js @@ -0,0 +1,18 @@ +import express from 'express'; +import ToolController from './tool.controller'; +import { toolService } from './dependency'; + +const router = express.Router(); +const toolController = new ToolController(toolService); + +// @route GET /api/v2/tools/id +// @desc Returns a tool based on identifier provided +// @access Public +router.get('/:id', (req, res) => toolController.getTool(req, res)); + +// @route GET /api/v2/tools +// @desc Returns a collection of tools based on supplied query parameters +// @access Public +router.get('/', (req, res) => toolController.getTools(req, res)); + +module.exports = router; diff --git a/src/resources/tool/v2/tool.service.js b/src/resources/tool/v2/tool.service.js new file mode 100644 index 00000000..86e1477c --- /dev/null +++ b/src/resources/tool/v2/tool.service.js @@ -0,0 +1,13 @@ +export default class ToolService { + constructor(toolRepository) { + this.toolRepository = toolRepository; + } + + getTool(id, query = {}) { + return this.toolRepository.getTool(id, query); + } + + getTools(query = {}) { + return this.toolRepository.getTools(query); + } +} diff --git a/src/resources/utilities/helper.util.js b/src/resources/utilities/helper.util.js index 7110d184..25312ab4 100644 --- a/src/resources/utilities/helper.util.js +++ b/src/resources/utilities/helper.util.js @@ -74,5 +74,5 @@ export default { generatedNumericId: _generatedNumericId, generateAlphaNumericString: _generateAlphaNumericString, hidePrivateProfileDetails: _hidePrivateProfileDetails, - getEnvironment: _getEnvironment, + getEnvironment: _getEnvironment }; From 8902f16c8b5462a7e54d88ca8a197f3238f86611 Mon Sep 17 00:00:00 2001 From: Richard Date: Mon, 15 Feb 2021 15:56:10 +0000 Subject: [PATCH 14/42] Handling retro structure of document_links for papers --- src/resources/paper/paper.route.js | 14 ++++++++++---- src/resources/tool/data.repository.js | 26 +++++++++++++++++++++++++- 2 files changed, 35 insertions(+), 5 deletions(-) diff --git a/src/resources/paper/paper.route.js b/src/resources/paper/paper.route.js index 13c0a9fe..d1d25f55 100644 --- a/src/resources/paper/paper.route.js +++ b/src/resources/paper/paper.route.js @@ -3,7 +3,7 @@ import { Data } from '../tool/data.model'; import { ROLES } from '../user/user.roles'; import passport from 'passport'; import { utils } from '../auth'; -import { addTool, editTool, setStatus, getTools, getToolsAdmin, getAllTools } from '../tool/data.repository'; +import { addTool, editTool, setStatus, getTools, getToolsAdmin, getAllTools, formatRetroDocumentLinks } from '../tool/data.repository'; import helper from '../utilities/helper.util'; import escape from 'escape-html'; const router = express.Router(); @@ -19,7 +19,7 @@ router.post('/', passport.authenticate('jwt'), utils.checkIsInRole(ROLES.Admin, .catch(err => { return res.json({ success: false, err }); }); -}); +}); // @router GET /api/v1/ // @desc Returns List of Paper Objects Authenticated @@ -27,7 +27,7 @@ router.post('/', passport.authenticate('jwt'), utils.checkIsInRole(ROLES.Admin, router.get('/getList', passport.authenticate('jwt'), utils.checkIsInRole(ROLES.Admin, ROLES.Creator), async (req, res) => { req.params.type = 'paper'; let role = req.user.role; - + if (role === ROLES.Admin) { await getToolsAdmin(req) .then(data => { @@ -118,7 +118,7 @@ router.put('/:id', passport.authenticate('jwt'), utils.checkIsInRole(ROLES.Admin * * Return the details on the paper based on the tool ID. */ -router.get('/:paperID', async (req, res) => { +router.get('/:paperID', async (req, res) => { var q = Data.aggregate([ { $match: { $and: [{ id: parseInt(req.params.paperID) }, { type: 'paper' }] } }, { $lookup: { from: 'tools', localField: 'authors', foreignField: 'id', as: 'persons' } }, @@ -146,6 +146,9 @@ router.get('/:paperID', async (req, res) => { if (err) return res.json({ success: false, error: err }); data[0].persons = helper.hidePrivateProfileDetails(data[0].persons); + if (Array.isArray(data[0].document_links)) { + data[0].document_links = formatRetroDocumentLinks(data[0].document_links); + } return res.json({ success: true, data: data }); }); } else { @@ -161,6 +164,9 @@ router.get('/edit/:paperID', async (req, res) => { ]); query.exec((err, data) => { if (data.length > 0) { + if (Array.isArray(data[0].document_links)) { + data[0].document_links = formatRetroDocumentLinks(data[0].document_links); + } return res.json({ success: true, data: data }); } else { return res.json({ success: false, error: `Paper not found for paper id ${req.params.id}` }); diff --git a/src/resources/tool/data.repository.js b/src/resources/tool/data.repository.js index cc3a205f..3144edcd 100644 --- a/src/resources/tool/data.repository.js +++ b/src/resources/tool/data.repository.js @@ -617,4 +617,28 @@ function validateDocumentLinks(document_links) { return documentLinksValidated; } -export { addTool, editTool, deleteTool, setStatus, getTools, getToolsAdmin, getAllTools }; +function formatRetroDocumentLinks(document_links) { + let documentLinksValidated = { doi: [], pdf: [], html: [] }; + + document_links.forEach(obj => { + for (const [key, value] of Object.entries(obj)) { + switch (key) { + case 'doi': + documentLinksValidated.doi.push(value); + break; + case 'pdf': + documentLinksValidated.pdf.push(value); + break; + case 'html': + documentLinksValidated.html.push(value); + break; + default: + break; + } + } + }); + + return documentLinksValidated; +} + +export { addTool, editTool, deleteTool, setStatus, getTools, getToolsAdmin, getAllTools, formatRetroDocumentLinks }; From 5ff58fd2250f445deef088523e84fcda871b122d Mon Sep 17 00:00:00 2001 From: Robin Kavanagh Date: Mon, 15 Feb 2021 16:08:23 +0000 Subject: [PATCH 15/42] Updated dataset service with latest dev changes --- src/resources/dataset/v1/dataset.service.js | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/resources/dataset/v1/dataset.service.js b/src/resources/dataset/v1/dataset.service.js index 77b65bb0..84251040 100644 --- a/src/resources/dataset/v1/dataset.service.js +++ b/src/resources/dataset/v1/dataset.service.js @@ -1,10 +1,9 @@ +import { Data } from '../../tool/data.model'; +import { MetricsData } from '../../stats/metrics.model'; import axios from 'axios'; import * as Sentry from '@sentry/node'; import { v4 as uuidv4 } from 'uuid'; -import { Data } from '../../tool/data.model'; -import { MetricsData } from '../../stats/metrics.model'; - export async function loadDataset(datasetID) { var metadataCatalogueLink = process.env.metadataURL || 'https://metadata-catalogue.org/hdruk'; const datasetCall = axios From e13d8e9079a8bd3e8128628d564a7b17d7ba1d70 Mon Sep 17 00:00:00 2001 From: Robin Kavanagh Date: Mon, 15 Feb 2021 16:21:44 +0000 Subject: [PATCH 16/42] Fixed references in dataset route --- src/resources/dataset/v1/dataset.route.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/resources/dataset/v1/dataset.route.js b/src/resources/dataset/v1/dataset.route.js index 4902e0d8..cc21b6a1 100644 --- a/src/resources/dataset/v1/dataset.route.js +++ b/src/resources/dataset/v1/dataset.route.js @@ -4,7 +4,7 @@ import { loadDataset, loadDatasets } from './dataset.service'; import { getAllTools } from '../../tool/data.repository'; import _ from 'lodash'; import escape from 'escape-html'; -import { Course } from '../course/course.model'; +import { Course } from '../../course/course.model'; const router = express.Router(); const rateLimit = require('express-rate-limit'); @@ -167,4 +167,4 @@ router.get('/', async (req, res) => { }); }); -module.exports = router; +module.exports = router; \ No newline at end of file From d4514b72b1bf106a6ad74b5fcd732970fc03aa47 Mon Sep 17 00:00:00 2001 From: Robin Kavanagh Date: Mon, 15 Feb 2021 17:01:32 +0000 Subject: [PATCH 17/42] Fixed merge issue --- src/resources/paper/v1/paper.route.js | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/resources/paper/v1/paper.route.js b/src/resources/paper/v1/paper.route.js index 84402757..1ae07567 100644 --- a/src/resources/paper/v1/paper.route.js +++ b/src/resources/paper/v1/paper.route.js @@ -2,15 +2,9 @@ import express from 'express'; import { Data } from '../../tool/data.model'; import { ROLES } from '../../user/user.roles'; import passport from 'passport'; -<<<<<<< HEAD:src/resources/paper/v1/paper.route.js import { utils } from '../../auth'; import { addTool, editTool, setStatus, getTools, getToolsAdmin, getAllTools, formatRetroDocumentLinks } from '../../tool/data.repository'; import helper from '../../utilities/helper.util'; -======= -import { utils } from '../auth'; -import { addTool, editTool, setStatus, getTools, getToolsAdmin, getAllTools, formatRetroDocumentLinks } from '../tool/data.repository'; -import helper from '../utilities/helper.util'; ->>>>>>> dev:src/resources/paper/paper.route.js import escape from 'escape-html'; const router = express.Router(); From e820c2862d5109da89dcc03b4cc8d910d928726b Mon Sep 17 00:00:00 2001 From: CiaraWardPA Date: Mon, 15 Feb 2021 17:09:53 +0000 Subject: [PATCH 18/42] IG-1243 updates to add collections to search and filters --- src/resources/search/filter.route.js | 22 ++- src/resources/search/search.repository.js | 199 ++++++++++++++-------- src/resources/search/search.router.js | 19 +-- 3 files changed, 152 insertions(+), 88 deletions(-) diff --git a/src/resources/search/filter.route.js b/src/resources/search/filter.route.js index 532c298d..a0d18626 100644 --- a/src/resources/search/filter.route.js +++ b/src/resources/search/filter.route.js @@ -171,7 +171,6 @@ router.get('/', async (req, res) => { let searchQuery = { $and: [{ activeflag: 'active' }] }; if (searchString.length > 0) searchQuery['$and'].push({ $text: { $search: searchString } }); var activeFiltersQuery = getObjectFilters(searchQuery, req, 'paper'); - await Promise.all([ getFilter(searchString, 'paper', 'tags.topics', true, activeFiltersQuery), getFilter(searchString, 'paper', 'tags.features', true, activeFiltersQuery), @@ -233,6 +232,27 @@ router.get('/', async (req, res) => { }, }); }); + } else if (tab === 'Collections') { + let searchQuery = { $and: [{ activeflag: 'active' }, { publicflag: true }] }; + if (searchString.length > 0) searchQuery['$and'].push({ $text: { $search: searchString } }); + var activeFiltersQuery = getObjectFilters(searchQuery, req, 'collection'); + + await Promise.all([ + getFilter(searchString, 'collection', 'keywords', true, activeFiltersQuery), + getFilter(searchString, 'collection', 'authors', true, activeFiltersQuery), + ]).then(values => { + return res.json({ + success: true, + allFilters: { + collectionKeywordFilter: values[0][0], + collectionPublisherFilter: values[1][0], + }, + filterOptions: { + collectionKeywordsFilterOptions: values[0][1], + collectionPublisherFilterOptions: values[1][1], + }, + }); + }); } }); diff --git a/src/resources/search/search.repository.js b/src/resources/search/search.repository.js index 0ba42b8b..01ba251a 100644 --- a/src/resources/search/search.repository.js +++ b/src/resources/search/search.repository.js @@ -11,10 +11,12 @@ export function getObjectResult(type, searchAll, searchQuery, startIndex, maxRes } else if (type === 'collection') { collection = Collections; } - console.log(`searchQuery: ${JSON.stringify(searchQuery, null, 2)}`); - var newSearchQuery = JSON.parse(JSON.stringify(searchQuery)); + + let newSearchQuery = JSON.parse(JSON.stringify(searchQuery)); if (type !== 'collection') { newSearchQuery['$and'].push({ type: type }); + } else { + newSearchQuery['$and'].push({ publicflag: true }); } if (type === 'course') { @@ -30,7 +32,7 @@ export function getObjectResult(type, searchAll, searchQuery, startIndex, maxRes }); } - var queryObject; + let queryObject; if (type === 'course') { queryObject = [ { $unwind: '$courseOptions' }, @@ -102,8 +104,6 @@ export function getObjectResult(type, searchAll, searchQuery, startIndex, maxRes ]; } - console.log(`queryObject: ${JSON.stringify(queryObject, null, 2)}`); - if (sort === '' || sort === 'relevance') { if (type === 'person') { if (searchAll) queryObject.push({ $sort: { lastname: 1 } }); @@ -144,9 +144,11 @@ export function getObjectCount(type, searchAll, searchQuery) { } else if (type === 'collection') { collection = Collections; } - var newSearchQuery = JSON.parse(JSON.stringify(searchQuery)); + let newSearchQuery = JSON.parse(JSON.stringify(searchQuery)); if (type !== 'collection') { newSearchQuery['$and'].push({ type: type }); + } else { + newSearchQuery['$and'].push({ publicflag: true }); } if (type === 'course') { newSearchQuery['$and'].forEach(x => { @@ -204,9 +206,7 @@ export function getObjectCount(type, searchAll, searchQuery) { ]) .sort({ score: { $meta: 'textScore' } }); } - } - // TODO - get count for collections - else if (type === 'collection') { + } else if (type === 'collection') { if (searchAll) { q = collection.aggregate([ { $match: newSearchQuery }, @@ -226,24 +226,25 @@ export function getObjectCount(type, searchAll, searchQuery) { }, ]); } else { - q = collection.aggregate([ - { $match: newSearchQuery }, - { - $group: { - _id: {}, - count: { - $sum: 1, + q = collection + .aggregate([ + { $match: newSearchQuery }, + { + $group: { + _id: {}, + count: { + $sum: 1, + }, }, }, - }, - { - $project: { - count: '$count', - _id: 0, + { + $project: { + count: '$count', + _id: 0, + }, }, - }, - ]); - // .sort({ score: { $meta: 'textScore' } }); + ]) + .sort({ score: { $meta: 'textScore' } }); } } else { if (searchAll) { @@ -325,12 +326,13 @@ export function getObjectFilters(searchQueryStart, req, type) { courseentrylevel = '', courseframework = '', coursepriority = '', - //TODO - add collection filter options in here + collectionpublisher = '', + collectionkeywords = '', } = req.query; if (type === 'dataset') { if (license.length > 0) { - var filterTermArray = []; + let filterTermArray = []; license.split('::').forEach(filterTerm => { filterTermArray.push({ license: filterTerm }); }); @@ -338,7 +340,7 @@ export function getObjectFilters(searchQueryStart, req, type) { } if (sampleavailability.length > 0) { - var filterTermArray = []; + let filterTermArray = []; sampleavailability.split('::').forEach(filterTerm => { filterTermArray.push({ 'datasetfields.physicalSampleAvailability': filterTerm }); }); @@ -346,7 +348,7 @@ export function getObjectFilters(searchQueryStart, req, type) { } if (keywords.length > 0) { - var filterTermArray = []; + let filterTermArray = []; keywords.split('::').forEach(filterTerm => { filterTermArray.push({ 'tags.features': filterTerm }); }); @@ -354,7 +356,7 @@ export function getObjectFilters(searchQueryStart, req, type) { } if (publisher.length > 0) { - var filterTermArray = []; + let filterTermArray = []; publisher.split('::').forEach(filterTerm => { filterTermArray.push({ 'datasetfields.publisher': filterTerm }); }); @@ -362,7 +364,7 @@ export function getObjectFilters(searchQueryStart, req, type) { } if (ageband.length > 0) { - var filterTermArray = []; + let filterTermArray = []; ageband.split('::').forEach(filterTerm => { filterTermArray.push({ 'datasetfields.ageBand': filterTerm }); }); @@ -370,7 +372,7 @@ export function getObjectFilters(searchQueryStart, req, type) { } if (geographiccover.length > 0) { - var filterTermArray = []; + let filterTermArray = []; geographiccover.split('::').forEach(filterTerm => { filterTermArray.push({ 'datasetfields.geographicCoverage': filterTerm }); }); @@ -378,7 +380,7 @@ export function getObjectFilters(searchQueryStart, req, type) { } if (phenotypes.length > 0) { - var filterTermArray = []; + let filterTermArray = []; phenotypes.split('::').forEach(filterTerm => { filterTermArray.push({ 'datasetfields.phenotypes.name': filterTerm }); }); @@ -388,7 +390,7 @@ export function getObjectFilters(searchQueryStart, req, type) { if (type === 'tool') { if (programmingLanguage.length > 0) { - var filterTermArray = []; + let filterTermArray = []; programmingLanguage.split('::').forEach(filterTerm => { filterTermArray.push({ 'programmingLanguage.programmingLanguage': filterTerm }); }); @@ -396,7 +398,7 @@ export function getObjectFilters(searchQueryStart, req, type) { } if (toolcategories.length > 0) { - var filterTermArray = []; + let filterTermArray = []; toolcategories.split('::').forEach(filterTerm => { filterTermArray.push({ 'categories.category': filterTerm }); }); @@ -404,7 +406,7 @@ export function getObjectFilters(searchQueryStart, req, type) { } if (features.length > 0) { - var filterTermArray = []; + let filterTermArray = []; features.split('::').forEach(filterTerm => { filterTermArray.push({ 'tags.features': filterTerm }); }); @@ -412,7 +414,7 @@ export function getObjectFilters(searchQueryStart, req, type) { } if (tooltopics.length > 0) { - var filterTermArray = []; + let filterTermArray = []; tooltopics.split('::').forEach(filterTerm => { filterTermArray.push({ 'tags.topics': filterTerm }); }); @@ -420,7 +422,7 @@ export function getObjectFilters(searchQueryStart, req, type) { } } else if (type === 'project') { if (projectcategories.length > 0) { - var filterTermArray = []; + let filterTermArray = []; projectcategories.split('::').forEach(filterTerm => { filterTermArray.push({ 'categories.category': filterTerm }); }); @@ -428,7 +430,7 @@ export function getObjectFilters(searchQueryStart, req, type) { } if (projectfeatures.length > 0) { - var filterTermArray = []; + let filterTermArray = []; projectfeatures.split('::').forEach(filterTerm => { filterTermArray.push({ 'tags.features': filterTerm }); }); @@ -436,7 +438,7 @@ export function getObjectFilters(searchQueryStart, req, type) { } if (projecttopics.length > 0) { - var filterTermArray = []; + let filterTermArray = []; projecttopics.split('::').forEach(filterTerm => { filterTermArray.push({ 'tags.topics': filterTerm }); }); @@ -444,7 +446,7 @@ export function getObjectFilters(searchQueryStart, req, type) { } } else if (type === 'paper') { if (paperfeatures.length > 0) { - var filterTermArray = []; + let filterTermArray = []; paperfeatures.split('::').forEach(filterTerm => { filterTermArray.push({ 'tags.features': filterTerm }); }); @@ -452,7 +454,7 @@ export function getObjectFilters(searchQueryStart, req, type) { } if (papertopics.length > 0) { - var filterTermArray = []; + let filterTermArray = []; papertopics.split('::').forEach(filterTerm => { filterTermArray.push({ 'tags.topics': filterTerm }); }); @@ -460,7 +462,7 @@ export function getObjectFilters(searchQueryStart, req, type) { } } else if (type === 'course') { if (coursestartdates.length > 0) { - var filterTermArray = []; + let filterTermArray = []; coursestartdates.split('::').forEach(filterTerm => { filterTermArray.push({ 'courseOptions.startDate': filterTerm }); }); @@ -468,7 +470,7 @@ export function getObjectFilters(searchQueryStart, req, type) { } if (courseprovider.length > 0) { - var filterTermArray = []; + let filterTermArray = []; courseprovider.split('::').forEach(filterTerm => { filterTermArray.push({ provider: filterTerm }); }); @@ -476,7 +478,7 @@ export function getObjectFilters(searchQueryStart, req, type) { } if (courselocation.length > 0) { - var filterTermArray = []; + let filterTermArray = []; courselocation.split('::').forEach(filterTerm => { filterTermArray.push({ location: filterTerm }); }); @@ -484,7 +486,7 @@ export function getObjectFilters(searchQueryStart, req, type) { } if (coursestudymode.length > 0) { - var filterTermArray = []; + let filterTermArray = []; coursestudymode.split('::').forEach(filterTerm => { filterTermArray.push({ 'courseOptions.studyMode': filterTerm }); }); @@ -492,7 +494,7 @@ export function getObjectFilters(searchQueryStart, req, type) { } if (courseaward.length > 0) { - var filterTermArray = []; + let filterTermArray = []; courseaward.split('::').forEach(filterTerm => { filterTermArray.push({ award: filterTerm }); }); @@ -500,7 +502,7 @@ export function getObjectFilters(searchQueryStart, req, type) { } if (courseentrylevel.length > 0) { - var filterTermArray = []; + let filterTermArray = []; courseentrylevel.split('::').forEach(filterTerm => { filterTermArray.push({ 'entries.level': filterTerm }); }); @@ -508,7 +510,7 @@ export function getObjectFilters(searchQueryStart, req, type) { } if (coursedomains.length > 0) { - var filterTermArray = []; + let filterTermArray = []; coursedomains.split('::').forEach(filterTerm => { filterTermArray.push({ domains: filterTerm }); }); @@ -516,7 +518,7 @@ export function getObjectFilters(searchQueryStart, req, type) { } if (coursekeywords.length > 0) { - var filterTermArray = []; + let filterTermArray = []; coursekeywords.split('::').forEach(filterTerm => { filterTermArray.push({ keywords: filterTerm }); }); @@ -524,7 +526,7 @@ export function getObjectFilters(searchQueryStart, req, type) { } if (courseframework.length > 0) { - var filterTermArray = []; + let filterTermArray = []; courseframework.split('::').forEach(filterTerm => { filterTermArray.push({ competencyFramework: filterTerm }); }); @@ -532,17 +534,28 @@ export function getObjectFilters(searchQueryStart, req, type) { } if (coursepriority.length > 0) { - var filterTermArray = []; + let filterTermArray = []; coursepriority.split('::').forEach(filterTerm => { filterTermArray.push({ nationalPriority: filterTerm }); }); searchQuery['$and'].push({ $or: filterTermArray }); } - } - // TODO - add in collection filters - else if (type === 'collection') { - searchQuery['$and'].push({ publicflag: true }); - console.log(`collection searchQuery: ${JSON.stringify(searchQuery, null, 2)}`); + } else if (type === 'collection') { + if (collectionkeywords.length > 0) { + let filterTermArray = []; + collectionkeywords.split('::').forEach(filterTerm => { + filterTermArray.push({ keywords: filterTerm }); + }); + searchQuery['$and'].push({ $or: filterTermArray }); + } + + if (collectionpublisher.length > 0) { + let filterTermArray = []; + collectionpublisher.split('::').forEach(filterTerm => { + filterTermArray.push({ authors: parseInt(filterTerm) }); + }); + searchQuery['$and'].push({ $or: filterTermArray }); + } } return searchQuery; } @@ -550,11 +563,16 @@ export function getObjectFilters(searchQueryStart, req, type) { export const getFilter = async (searchString, type, field, isArray, activeFiltersQuery) => { return new Promise(async (resolve, reject) => { let collection = Data; - if (type === 'course') collection = Course; - var q = '', + if (type === 'course') { + collection = Course; + } else if (type === 'collection') { + collection = Collections; + } + let q = '', p = ''; - var combinedResults = [], - activeCombinedResults = []; + let combinedResults = [], + activeCombinedResults = [], + publishers = []; if (searchString) q = collection.aggregate(filterQueryGenerator(field, searchString, type, isArray, {})); else q = collection.aggregate(filterQueryGenerator(field, '', type, isArray, {})); @@ -567,13 +585,21 @@ export const getFilter = async (searchString, type, field, isArray, activeFilter if (dat.result && dat.result !== '') { if (field === 'datasetfields.phenotypes') combinedResults.push(dat.result.name.trim()); else if (field === 'courseOptions.startDate') combinedResults.push(moment(dat.result).format('DD MMM YYYY')); - else combinedResults.push(dat.result.trim()); + else { + if (_.isString(dat.result)) { + combinedResults.push(dat.result.trim()); + } else if (field === 'authors' && dat.id === dat.result) { + combinedResults.push(dat); + } + } } }); } var newSearchQuery = JSON.parse(JSON.stringify(activeFiltersQuery)); - newSearchQuery['$and'].push({ type: type }); + if (type !== 'collection') { + newSearchQuery['$and'].push({ type: type }); + } if (searchString) p = collection.aggregate(filterQueryGenerator(field, searchString, type, isArray, newSearchQuery)); else p = collection.aggregate(filterQueryGenerator(field, '', type, isArray, newSearchQuery)); @@ -584,11 +610,17 @@ export const getFilter = async (searchString, type, field, isArray, activeFilter if (dat.result && dat.result !== '') { if (field === 'datasetfields.phenotypes') activeCombinedResults.push(dat.result.name.trim()); else if (field === 'courseOptions.startDate') activeCombinedResults.push(moment(dat.result).format('DD MMM YYYY')); - else activeCombinedResults.push(dat.result.trim()); + else { + if (_.isString(dat.result)) { + activeCombinedResults.push(dat.result.trim()); + } else if (field === 'authors' && dat.id === dat.result) { + activeCombinedResults.push(dat); + } + } } }); } - resolve([combinedResults, activeCombinedResults]); + resolve([combinedResults, activeCombinedResults, publishers]); }); }); }); @@ -607,17 +639,38 @@ export function filterQueryGenerator(filter, searchString, type, isArray, active if (!_.isEmpty(activeFiltersQuery)) { queryArray.push({ $match: activeFiltersQuery }); } else { - if (searchString !== '') - queryArray.push({ $match: { $and: [{ $text: { $search: searchString } }, { type: type }, { activeflag: 'active' }] } }); - else queryArray.push({ $match: { $and: [{ type: type }, { activeflag: 'active' }] } }); + if (searchString !== '') { + type !== 'collection' + ? queryArray.push({ $match: { $and: [{ $text: { $search: searchString } }, { type: type }, { activeflag: 'active' }] } }) + : queryArray.push({ $match: { $and: [{ $text: { $search: searchString } }, { activeflag: 'active' }, { publicflag: true }] } }); + } else { + type !== 'collection' + ? queryArray.push({ $match: { $and: [{ type: type }, { activeflag: 'active' }] } }) + : queryArray.push({ $match: { $and: [{ activeflag: 'active' }, { publicflag: true }] } }); + } } - queryArray.push({ - $project: { - result: '$' + filter, - _id: 0, - }, - }); + if (type === 'collection' && filter === 'authors') { + queryArray.push({ $lookup: { from: 'tools', localField: 'authors', foreignField: 'id', as: 'persons' } }); + queryArray.push( + { $unwind: '$persons' }, + { + $project: { + result: '$' + filter, + _id: 0, + name: { $concat: ['$persons.firstname', ' ', '$persons.lastname'] }, + id: '$persons.id', + }, + } + ); + } else { + queryArray.push({ + $project: { + result: '$' + filter, + _id: 0, + }, + }); + } if (isArray) { queryArray.push({ $unwind: '$result' }); diff --git a/src/resources/search/search.router.js b/src/resources/search/search.router.js index 96c32900..f32219e6 100644 --- a/src/resources/search/search.router.js +++ b/src/resources/search/search.router.js @@ -46,14 +46,13 @@ router.get('/', async (req, res) => { searchAll = true; } - var allResults = [], + let allResults = [], datasetResults = [], toolResults = [], projectResults = [], paperResults = [], personResults = [], courseResults = [], - // TODO collectionResults = []; if (tab === '') { @@ -99,14 +98,13 @@ router.get('/', async (req, res) => { req.query.maxResults || 40, 'startdate' ), - // TODO getObjectResult( 'collection', searchAll, getObjectFilters(searchQuery, req, 'collection'), req.query.collectionIndex || 0, req.query.maxResults || 40, - 'startdate' + req.query.collectionSort || '' ), ]); } else if (tab === 'Datasets') { @@ -168,9 +166,7 @@ router.get('/', async (req, res) => { 'startdate' ), ]); - } - // TODO - else if (tab === 'Collections') { + } else if (tab === 'Collections') { collectionResults = await Promise.all([ getObjectResult( 'collection', @@ -178,7 +174,7 @@ router.get('/', async (req, res) => { getObjectFilters(searchQuery, req, 'collection'), req.query.collectionIndex || 0, req.query.maxResults || 40, - 'startdate' + req.query.collectionSort || '' ), ]); } @@ -190,7 +186,6 @@ router.get('/', async (req, res) => { getObjectCount('paper', searchAll, getObjectFilters(searchQuery, req, 'paper')), getObjectCount('person', searchAll, searchQuery), getObjectCount('course', searchAll, getObjectFilters(searchQuery, req, 'course')), - //TODO getObjectCount('collection', searchAll, getObjectFilters(searchQuery, req, 'collection')), ]); @@ -201,8 +196,7 @@ router.get('/', async (req, res) => { papers: summaryCounts[3][0] !== undefined ? summaryCounts[3][0].count : 0, persons: summaryCounts[4][0] !== undefined ? summaryCounts[4][0].count : 0, courses: summaryCounts[5][0] !== undefined ? summaryCounts[5][0].count : 0, - //TODO - collections: summaryCounts[5][0] !== undefined ? summaryCounts[6][0].count : 0, + collections: summaryCounts[6][0] !== undefined ? summaryCounts[6][0].count : 0, }; let recordSearchData = new RecordSearchData(); @@ -213,7 +207,6 @@ router.get('/', async (req, res) => { recordSearchData.returned.paper = summaryCounts[3][0] !== undefined ? summaryCounts[3][0].count : 0; recordSearchData.returned.person = summaryCounts[4][0] !== undefined ? summaryCounts[4][0].count : 0; recordSearchData.returned.course = summaryCounts[5][0] !== undefined ? summaryCounts[5][0].count : 0; - //TODO recordSearchData.returned.collection = summaryCounts[6][0] !== undefined ? summaryCounts[6][0].count : 0; recordSearchData.datesearched = Date.now(); recordSearchData.save(err => {}); @@ -227,7 +220,6 @@ router.get('/', async (req, res) => { paperResults: allResults[3], personResults: allResults[4], courseResults: allResults[5], - // TODO collectionResults: allResults[6], summary: summary, }); @@ -240,7 +232,6 @@ router.get('/', async (req, res) => { paperResults: paperResults[0], personResults: personResults[0], courseResults: courseResults[0], - // TODO collectionResults: collectionResults[0], summary: summary, }); From e69127b6c4047067a4fac22679f0c1e42463c325 Mon Sep 17 00:00:00 2001 From: CiaraWardPA Date: Wed, 17 Feb 2021 11:49:49 +0000 Subject: [PATCH 19/42] IG-1327 Public flag added to collection model, add and edit --- .../collections/collections.model.js | 5 ++-- .../collections/collections.route.js | 24 +++++++++++-------- 2 files changed, 17 insertions(+), 12 deletions(-) diff --git a/src/resources/collections/collections.model.js b/src/resources/collections/collections.model.js index 59cc2b52..5c1bca0b 100644 --- a/src/resources/collections/collections.model.js +++ b/src/resources/collections/collections.model.js @@ -14,14 +14,15 @@ const CollectionSchema = new Schema( relatedObjects: [ { objectId: String, - reason: String, - pid: String, + reason: String, + pid: String, objectType: String, user: String, updated: String, }, ], activeflag: String, + publicflag: Boolean, }, { collection: 'collections', //will be created when first posting diff --git a/src/resources/collections/collections.route.js b/src/resources/collections/collections.route.js index 94ac7998..66fe1e46 100644 --- a/src/resources/collections/collections.route.js +++ b/src/resources/collections/collections.route.js @@ -48,12 +48,12 @@ router.get('/relatedobjects/:collectionID', async (req, res) => { }); router.get('/entityid/:entityID', async (req, res) => { - let entityID = req.params.entityID - let dataVersions = await Data.find({ pid: entityID }, {_id: 0, datasetid:1}); - let dataVersionsArray = dataVersions.map(a => a.datasetid); - dataVersionsArray.push(entityID); - - var q = Collections.aggregate([ + let entityID = req.params.entityID; + let dataVersions = await Data.find({ pid: entityID }, { _id: 0, datasetid: 1 }); + let dataVersionsArray = dataVersions.map(a => a.datasetid); + dataVersionsArray.push(entityID); + + var q = Collections.aggregate([ { $match: { $and: [ @@ -62,11 +62,11 @@ router.get('/entityid/:entityID', async (req, res) => { $elemMatch: { $or: [ { - objectId: { $in : dataVersionsArray }, + objectId: { $in: dataVersionsArray }, }, { pid: entityID, - } + }, ], }, }, @@ -88,9 +88,10 @@ router.get('/entityid/:entityID', async (req, res) => { }); }); +//Edit collection router.put('/edit', passport.authenticate('jwt'), utils.checkIsInRole(ROLES.Admin, ROLES.Creator), async (req, res) => { const collectionCreator = req.body.collectionCreator; - var { id, name, description, imageLink, authors, relatedObjects } = req.body; + var { id, name, description, imageLink, authors, relatedObjects, publicflag } = req.body; imageLink = urlValidator.validateURL(imageLink); Collections.findOneAndUpdate( @@ -101,6 +102,7 @@ router.put('/edit', passport.authenticate('jwt'), utils.checkIsInRole(ROLES.Admi imageLink: imageLink, authors: authors, relatedObjects: relatedObjects, + publicflag: publicflag, }, err => { if (err) { @@ -112,12 +114,13 @@ router.put('/edit', passport.authenticate('jwt'), utils.checkIsInRole(ROLES.Admi }); }); +//Add collection router.post('/add', passport.authenticate('jwt'), utils.checkIsInRole(ROLES.Admin, ROLES.Creator), async (req, res) => { let collections = new Collections(); const collectionCreator = req.body.collectionCreator; - const { name, description, imageLink, authors, relatedObjects } = req.body; + const { name, description, imageLink, authors, relatedObjects, publicflag } = req.body; collections.id = parseInt(Math.random().toString().replace('0.', '')); collections.name = inputSanitizer.removeNonBreakingSpaces(name); @@ -126,6 +129,7 @@ router.post('/add', passport.authenticate('jwt'), utils.checkIsInRole(ROLES.Admi collections.authors = authors; collections.relatedObjects = relatedObjects; collections.activeflag = 'active'; + collections.publicflag = publicflag; try { if (collections.authors) { From 9973fd2cf68ab64fcdb464fd39935e9ec12499fa Mon Sep 17 00:00:00 2001 From: CiaraWardPA Date: Wed, 17 Feb 2021 15:28:38 +0000 Subject: [PATCH 20/42] IG-1324 Start of collection keywords --- .../collections/collections.route.js | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/src/resources/collections/collections.route.js b/src/resources/collections/collections.route.js index 66fe1e46..1eb96582 100644 --- a/src/resources/collections/collections.route.js +++ b/src/resources/collections/collections.route.js @@ -21,6 +21,33 @@ const hdrukEmail = `enquiry@healthdatagateway.org`; const router = express.Router(); +//TODO +router.get('/keywords', async (req, res) => { + console.log(`KEYWORDS`); + Collections.find( + {}, + { + _id: 0, + keywords: 1, + }, + err => { + if (err) { + return res.json({ success: false, error: err }); + } + } + ).then(res => { + for (let item of res) { + console.log(`item: ${JSON.stringify(item, null, 2)} - ${typeof item} - ${item.length}`); + + if (_.isEmpty(item)) { + console.log(`YES`); + } else console.log(`NO`); + } + + return res; + }); +}); + router.get('/:collectionID', async (req, res) => { var q = Collections.aggregate([ { $match: { $and: [{ id: parseInt(req.params.collectionID) }] } }, From 5e110ac9422616483e5f82646b199f8517935ee4 Mon Sep 17 00:00:00 2001 From: CiaraWardPA Date: Wed, 17 Feb 2021 17:36:35 +0000 Subject: [PATCH 21/42] IG-1243 name variable renamed value --- src/resources/search/search.repository.js | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/resources/search/search.repository.js b/src/resources/search/search.repository.js index 7f9b5db7..9084b2c8 100644 --- a/src/resources/search/search.repository.js +++ b/src/resources/search/search.repository.js @@ -642,8 +642,8 @@ export function filterQueryGenerator(filter, searchString, type, isArray, active : queryArray.push({ $match: { $and: [{ activeflag: 'active' }, { publicflag: true }] } }); } } - - if (type === 'course') { + + if (type === 'course') { queryArray.push({ $match: { $or: [{ 'courseOptions.startDate': { $gte: new Date(Date.now()) } }, { 'courseOptions.flexibleDates': true }] }, }); @@ -658,7 +658,7 @@ export function filterQueryGenerator(filter, searchString, type, isArray, active $project: { result: '$' + filter, _id: 0, - name: { $concat: ['$persons.firstname', ' ', '$persons.lastname'] }, + value: { $concat: ['$persons.firstname', ' ', '$persons.lastname'] }, id: '$persons.id', }, } @@ -671,7 +671,6 @@ export function filterQueryGenerator(filter, searchString, type, isArray, active }, }); } - if (isArray) { queryArray.push({ $unwind: '$result' }); From 89a403652eb582e08adb550ee2b4bac52e9f6f82 Mon Sep 17 00:00:00 2001 From: Robin Kavanagh Date: Thu, 18 Feb 2021 10:37:59 +0000 Subject: [PATCH 22/42] Updating V2 Dataset endpoint --- .eslintrc.js | 306 ++++++++++++++++++++ package.json | 2 + src/resources/base/entity.js | 6 + src/resources/dataset/dataset.controller.js | 2 +- src/resources/dataset/dataset.entity.js | 34 ++- src/resources/dataset/dataset.repository.js | 5 +- src/resources/dataset/dataset.service.js | 31 +- 7 files changed, 377 insertions(+), 9 deletions(-) create mode 100644 .eslintrc.js diff --git a/.eslintrc.js b/.eslintrc.js new file mode 100644 index 00000000..4b0885e7 --- /dev/null +++ b/.eslintrc.js @@ -0,0 +1,306 @@ +module.exports = { + "env": { + "browser": true, + "es2021": true + }, + "extends": "eslint:recommended", + "parserOptions": { + "ecmaVersion": 12, + "sourceType": "module" + }, + "rules": { + "accessor-pairs": "error", + "array-bracket-newline": "off", + "array-bracket-spacing": "off", + "array-callback-return": "off", + "array-element-newline": "off", + "arrow-body-style": "off", + "arrow-parens": "off", + "arrow-spacing": [ + "error", + { + "after": true, + "before": true + } + ], + "block-scoped-var": "off", + "block-spacing": "off", + "brace-style": "off", + "camelcase": "off", + "capitalized-comments": "off", + "class-methods-use-this": "error", + "comma-dangle": "off", + "comma-spacing": "off", + "comma-style": [ + "error", + "last" + ], + "complexity": "off", + "computed-property-spacing": [ + "error", + "never" + ], + "consistent-return": "off", + "consistent-this": "error", + "curly": "off", + "default-case": "off", + "default-case-last": "error", + "default-param-last": "off", + "dot-location": [ + "error", + "property" + ], + "dot-notation": "off", + "eol-last": "off", + "eqeqeq": "off", + "func-call-spacing": "error", + "func-name-matching": "error", + "func-names": "off", + "func-style": [ + "error", + "declaration", + { + "allowArrowFunctions": true + } + ], + "function-call-argument-newline": [ + "error", + "consistent" + ], + "function-paren-newline": "off", + "generator-star-spacing": "error", + "grouped-accessor-pairs": "error", + "guard-for-in": "off", + "id-denylist": "error", + "id-length": "off", + "id-match": "error", + "implicit-arrow-linebreak": "off", + "indent": "off", + "init-declarations": "off", + "jsx-quotes": "error", + "key-spacing": "off", + "keyword-spacing": "off", + "line-comment-position": "off", + "linebreak-style": [ + "error", + "unix" + ], + "lines-around-comment": "off", + "lines-between-class-members": [ + "error", + "always" + ], + "max-classes-per-file": "error", + "max-depth": "off", + "max-len": "off", + "max-lines": "off", + "max-lines-per-function": "off", + "max-nested-callbacks": "error", + "max-params": "off", + "max-statements": "off", + "max-statements-per-line": "off", + "multiline-comment-style": "off", + "multiline-ternary": "off", + "new-parens": "error", + "newline-per-chained-call": "off", + "no-alert": "error", + "no-array-constructor": "error", + "no-await-in-loop": "off", + "no-bitwise": "error", + "no-caller": "error", + "no-cond-assign": [ + "error", + "except-parens" + ], + "no-confusing-arrow": [ + "error", + { + "allowParens": true + } + ], + "no-console": "off", + "no-constructor-return": "error", + "no-continue": "error", + "no-div-regex": "error", + "no-duplicate-imports": "off", + "no-else-return": "off", + "no-empty-function": "off", + "no-eq-null": "off", + "no-eval": "error", + "no-extend-native": "error", + "no-extra-bind": "error", + "no-extra-label": "error", + "no-extra-parens": "off", + "no-floating-decimal": "error", + "no-implicit-globals": "error", + "no-implied-eval": "error", + "no-inline-comments": "off", + "no-inner-declarations": [ + "error", + "functions" + ], + "no-invalid-this": "off", + "no-iterator": "error", + "no-label-var": "error", + "no-labels": "error", + "no-lone-blocks": "error", + "no-lonely-if": "off", + "no-loop-func": "off", + "no-loss-of-precision": "error", + "no-magic-numbers": "off", + "no-mixed-operators": "off", + "no-multi-assign": "error", + "no-multi-spaces": "off", + "no-multi-str": "error", + "no-multiple-empty-lines": "error", + "no-negated-condition": "off", + "no-nested-ternary": "off", + "no-new": "error", + "no-new-func": "error", + "no-new-object": "error", + "no-new-wrappers": "error", + "no-nonoctal-decimal-escape": "error", + "no-octal-escape": "error", + "no-param-reassign": "off", + "no-plusplus": "off", + "no-promise-executor-return": "off", + "no-proto": "error", + "no-restricted-exports": "error", + "no-restricted-globals": "error", + "no-restricted-imports": "error", + "no-restricted-properties": "error", + "no-restricted-syntax": "error", + "no-return-assign": "error", + "no-return-await": "off", + "no-script-url": "error", + "no-self-compare": "error", + "no-sequences": "off", + "no-shadow": "off", + "no-tabs": "off", + "no-template-curly-in-string": "error", + "no-ternary": "off", + "no-throw-literal": "error", + "no-trailing-spaces": "off", + "no-undef-init": "error", + "no-undefined": "off", + "no-underscore-dangle": "off", + "no-unmodified-loop-condition": "error", + "no-unneeded-ternary": "off", + "no-unreachable-loop": "error", + "no-unsafe-optional-chaining": "error", + "no-unused-expressions": "off", + "no-use-before-define": "off", + "no-useless-backreference": "error", + "no-useless-call": "error", + "no-useless-computed-key": "off", + "no-useless-concat": "error", + "no-useless-constructor": "error", + "no-useless-rename": "error", + "no-useless-return": "off", + "no-var": "off", + "no-void": "error", + "no-warning-comments": [ + "error", + { + "location": "start" + } + ], + "no-whitespace-before-property": "error", + "nonblock-statement-body-position": [ + "error", + "any" + ], + "object-curly-newline": "off", + "object-curly-spacing": "off", + "object-shorthand": "off", + "one-var": "off", + "one-var-declaration-per-line": "off", + "operator-assignment": [ + "error", + "always" + ], + "operator-linebreak": [ + "error", + null + ], + "padded-blocks": "off", + "padding-line-between-statements": "error", + "prefer-arrow-callback": "off", + "prefer-const": "off", + "prefer-destructuring": "off", + "prefer-exponentiation-operator": "error", + "prefer-named-capture-group": "off", + "prefer-numeric-literals": "error", + "prefer-object-spread": "error", + "prefer-promise-reject-errors": "off", + "prefer-regex-literals": "error", + "prefer-rest-params": "error", + "prefer-spread": "error", + "prefer-template": "off", + "quote-props": "off", + "quotes": "off", + "radix": [ + "error", + "as-needed" + ], + "require-atomic-updates": "off", + "require-await": "off", + "require-unicode-regexp": "off", + "rest-spread-spacing": [ + "error", + "never" + ], + "semi": "off", + "semi-spacing": [ + "error", + { + "after": true, + "before": false + } + ], + "semi-style": [ + "error", + "last" + ], + "sort-keys": "off", + "sort-vars": "off", + "space-before-blocks": "off", + "space-before-function-paren": "off", + "space-in-parens": "off", + "space-infix-ops": "error", + "space-unary-ops": "off", + "spaced-comment": "off", + "strict": "off", + "switch-colon-spacing": [ + "error", + { + "after": true, + "before": false + } + ], + "symbol-description": "error", + "template-curly-spacing": [ + "error", + "never" + ], + "template-tag-spacing": "error", + "unicode-bom": [ + "error", + "never" + ], + "valid-typeof": [ + "error", + { + "requireStringLiterals": false + } + ], + "vars-on-top": "off", + "wrap-iife": "error", + "wrap-regex": "off", + "yield-star-spacing": "error", + "yoda": [ + "error", + "never" + ] + } +}; diff --git a/package.json b/package.json index 898eae12..fdbe4857 100644 --- a/package.json +++ b/package.json @@ -50,6 +50,7 @@ "snyk": "^1.334.0", "swagger-ui-express": "^4.1.4", "test": "^0.6.0", + "transformobject": "^0.3.1", "uuid": "^8.3.1", "yamljs": "^0.3.0" }, @@ -57,6 +58,7 @@ "@babel/preset-env": "^7.12.1", "@shelf/jest-mongodb": "^1.2.3", "babel-jest": "^26.6.3", + "eslint": "^7.20.0", "jest": "^26.6.3", "mongodb-memory-server": "^6.9.2", "nodemon": "^2.0.3", diff --git a/src/resources/base/entity.js b/src/resources/base/entity.js index 8587ce68..2b21d3fe 100644 --- a/src/resources/base/entity.js +++ b/src/resources/base/entity.js @@ -1,3 +1,5 @@ +const transform = require('transformobject').transform; + class Entity { equals (other) { @@ -18,6 +20,10 @@ class Entity { toString () { return this.id; } + + transformTo(format, {strict} = {strict: false}) { + return transform(this, format, { strict }); + } } module.exports = Entity; \ No newline at end of file diff --git a/src/resources/dataset/dataset.controller.js b/src/resources/dataset/dataset.controller.js index 4bf0b79a..10315bd3 100644 --- a/src/resources/dataset/dataset.controller.js +++ b/src/resources/dataset/dataset.controller.js @@ -29,7 +29,7 @@ export default class DatasetController extends Controller { // Return the dataset return res.status(200).json({ success: true, - data: dataset, + ...dataset }); } catch (err) { // Return error response if something goes wrong diff --git a/src/resources/dataset/dataset.entity.js b/src/resources/dataset/dataset.entity.js index 30ca0efa..66f6c2df 100644 --- a/src/resources/dataset/dataset.entity.js +++ b/src/resources/dataset/dataset.entity.js @@ -47,8 +47,36 @@ export default class DatasetClass extends Entity { } checkLatestVersion() { - return this.activeflag === 'active'; + return this.activeflag === 'active'; } - - } + +export const v2Format = { + dataset: { + "@schema": { + "type": "Dataset", + "version": "2.0.0", + "url": "https://raw.githubusercontent.com/HDRUK/schemata/master/schema/dataset/latest/dataset.schema.json" + }, + pid: 'pid', + id: 'datasetid', + identifier: '', + version: '', + summary: '', + documentation: '', + revisions: '', + modified: '', + issued: '', + accessibility: '', + observations: '', + provenance: '', + coverage: '', + enrichmentAndLinkage: '', + sturcturalMetadata: '' + }, + relatedObjects: 'relatedObjects', + metadataQuality: 'datasetfields.metadataquality', + dataUtility: 'datasetfields.datautility', + viewCounter: 'counter', + submittedDataAccessRequests: 'submittedDataAccessRequests' +}; diff --git a/src/resources/dataset/dataset.repository.js b/src/resources/dataset/dataset.repository.js index 74007e22..413e99b2 100644 --- a/src/resources/dataset/dataset.repository.js +++ b/src/resources/dataset/dataset.repository.js @@ -7,9 +7,8 @@ export default class DatasetRepository extends Repository { this.dataset = Dataset; } - async getDataset(id, query) { - query = { ...query, datasetid: id }; - const options = { lean: true, populate: { path: 'submittedDataAccessRequests' } }; + async getDataset(query) { + const options = { lean: false, populate: { path: 'submittedDataAccessRequests' } }; return this.findOne(query, options); } diff --git a/src/resources/dataset/dataset.service.js b/src/resources/dataset/dataset.service.js index 427d350d..3d99202a 100644 --- a/src/resources/dataset/dataset.service.js +++ b/src/resources/dataset/dataset.service.js @@ -1,13 +1,40 @@ +import _ from 'lodash'; + +import { v2Format } from './dataset.entity'; + export default class DatasetService { constructor(datasetRepository) { this.datasetRepository = datasetRepository; } async getDataset(id, query = {}) { - let dataset = await this.datasetRepository.getDataset(id, query); + // Get dataset from Db by datasetid first + query = { ...query, datasetid: id}; + let dataset = await this.datasetRepository.getDataset(query); + + // Update query to find the latest dataset + if (!_.isNil(dataset)) { + id = dataset.pid; + } + + // Get latest data set + dataset = await this.datasetRepository.getDataset({ ...query, pid: id, activeflag: 'active' }); + + // If no dataset is found active, look for most recent archived dataset + if(!dataset) { + query.sort = '-createdAt'; + dataset = await this.datasetRepository.getDataset({ ...query, pid: id, activeflag: 'archive' }); + } + + // Return undefined if no dataset found + if(!dataset) return; - if(dataset && query['expanded'] === true) { + // Raw output responds with the data structure as defined in MongoDb + if(query['raw'] === true) { dataset.isLatestVersion = dataset.checkLatestVersion(); + } else { + // Transform to dataset v2 data structure + dataset = dataset.transformTo(v2Format, { strict: true }); } return dataset; From f5a24bcd55ef33e8ff7bb50bef2adb99f1809199 Mon Sep 17 00:00:00 2001 From: CiaraWardPA Date: Thu, 18 Feb 2021 12:09:37 +0000 Subject: [PATCH 23/42] IG-1324 keywords added to add/edit collection --- .../collections/collections.model.js | 1 + .../collections/collections.route.js | 35 +++---------------- 2 files changed, 5 insertions(+), 31 deletions(-) diff --git a/src/resources/collections/collections.model.js b/src/resources/collections/collections.model.js index 5c1bca0b..380204db 100644 --- a/src/resources/collections/collections.model.js +++ b/src/resources/collections/collections.model.js @@ -23,6 +23,7 @@ const CollectionSchema = new Schema( ], activeflag: String, publicflag: Boolean, + keywords: [String], }, { collection: 'collections', //will be created when first posting diff --git a/src/resources/collections/collections.route.js b/src/resources/collections/collections.route.js index 1eb96582..6d9e1928 100644 --- a/src/resources/collections/collections.route.js +++ b/src/resources/collections/collections.route.js @@ -21,33 +21,6 @@ const hdrukEmail = `enquiry@healthdatagateway.org`; const router = express.Router(); -//TODO -router.get('/keywords', async (req, res) => { - console.log(`KEYWORDS`); - Collections.find( - {}, - { - _id: 0, - keywords: 1, - }, - err => { - if (err) { - return res.json({ success: false, error: err }); - } - } - ).then(res => { - for (let item of res) { - console.log(`item: ${JSON.stringify(item, null, 2)} - ${typeof item} - ${item.length}`); - - if (_.isEmpty(item)) { - console.log(`YES`); - } else console.log(`NO`); - } - - return res; - }); -}); - router.get('/:collectionID', async (req, res) => { var q = Collections.aggregate([ { $match: { $and: [{ id: parseInt(req.params.collectionID) }] } }, @@ -115,10 +88,9 @@ router.get('/entityid/:entityID', async (req, res) => { }); }); -//Edit collection router.put('/edit', passport.authenticate('jwt'), utils.checkIsInRole(ROLES.Admin, ROLES.Creator), async (req, res) => { const collectionCreator = req.body.collectionCreator; - var { id, name, description, imageLink, authors, relatedObjects, publicflag } = req.body; + var { id, name, description, imageLink, authors, relatedObjects, publicflag, keywords } = req.body; imageLink = urlValidator.validateURL(imageLink); Collections.findOneAndUpdate( @@ -130,6 +102,7 @@ router.put('/edit', passport.authenticate('jwt'), utils.checkIsInRole(ROLES.Admi authors: authors, relatedObjects: relatedObjects, publicflag: publicflag, + keywords: keywords, }, err => { if (err) { @@ -141,13 +114,12 @@ router.put('/edit', passport.authenticate('jwt'), utils.checkIsInRole(ROLES.Admi }); }); -//Add collection router.post('/add', passport.authenticate('jwt'), utils.checkIsInRole(ROLES.Admin, ROLES.Creator), async (req, res) => { let collections = new Collections(); const collectionCreator = req.body.collectionCreator; - const { name, description, imageLink, authors, relatedObjects, publicflag } = req.body; + const { name, description, imageLink, authors, relatedObjects, publicflag, keywords } = req.body; collections.id = parseInt(Math.random().toString().replace('0.', '')); collections.name = inputSanitizer.removeNonBreakingSpaces(name); @@ -157,6 +129,7 @@ router.post('/add', passport.authenticate('jwt'), utils.checkIsInRole(ROLES.Admi collections.relatedObjects = relatedObjects; collections.activeflag = 'active'; collections.publicflag = publicflag; + collections.keywords = keywords; try { if (collections.authors) { From ffeba9769bdc2b05c505a4e822f33aaa32ffc714 Mon Sep 17 00:00:00 2001 From: Paul McCafferty <> Date: Thu, 18 Feb 2021 15:51:08 +0000 Subject: [PATCH 24/42] Fix for loading related resources for collections --- src/resources/collections/collections.repository.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/resources/collections/collections.repository.js b/src/resources/collections/collections.repository.js index dcf842c6..26d47c9e 100644 --- a/src/resources/collections/collections.repository.js +++ b/src/resources/collections/collections.repository.js @@ -36,7 +36,7 @@ function getCollectionObject(objectId, objectType, pid) { return new Promise(async (resolve, reject) => { let data; - if (!isNaN(id) && objectType !== 'course') { + if (objectType !== 'dataset' && objectType !== 'course') { data = await Data.find( { id: parseInt(id) }, { @@ -68,7 +68,7 @@ function getCollectionObject(objectId, objectType, pid) { ); // 2. If dataset not found search for a dataset based on datasetID if (!data || data.length <= 0) { - data = await Data.find({ datasetid: id }, { datasetid: 1, pid: 1 }); + data = await Data.find({ datasetid: objectId }, { datasetid: 1, pid: 1 }); // 3. Use retrieved dataset's pid to search by pid again data = await Data.find( { pid: data[0].pid, activeflag: 'active' }, From 1e7f4781eff8dc7e1fccc3469dce8e9b18cb59f9 Mon Sep 17 00:00:00 2001 From: Robin Kavanagh Date: Fri, 19 Feb 2021 10:34:45 +0000 Subject: [PATCH 25/42] Continued dev --- src/resources/dataset/dataset.entity.js | 80 +++++++++++++++--------- src/resources/dataset/dataset.service.js | 5 +- 2 files changed, 52 insertions(+), 33 deletions(-) diff --git a/src/resources/dataset/dataset.entity.js b/src/resources/dataset/dataset.entity.js index 66f6c2df..abb133cd 100644 --- a/src/resources/dataset/dataset.entity.js +++ b/src/resources/dataset/dataset.entity.js @@ -49,34 +49,54 @@ export default class DatasetClass extends Entity { checkLatestVersion() { return this.activeflag === 'active'; } -} -export const v2Format = { - dataset: { - "@schema": { - "type": "Dataset", - "version": "2.0.0", - "url": "https://raw.githubusercontent.com/HDRUK/schemata/master/schema/dataset/latest/dataset.schema.json" - }, - pid: 'pid', - id: 'datasetid', - identifier: '', - version: '', - summary: '', - documentation: '', - revisions: '', - modified: '', - issued: '', - accessibility: '', - observations: '', - provenance: '', - coverage: '', - enrichmentAndLinkage: '', - sturcturalMetadata: '' - }, - relatedObjects: 'relatedObjects', - metadataQuality: 'datasetfields.metadataquality', - dataUtility: 'datasetfields.datautility', - viewCounter: 'counter', - submittedDataAccessRequests: 'submittedDataAccessRequests' -}; + toV2Format() { + // Version 2 transformer map + const transformer = { + dataset: { + pid: 'pid', + id: 'datasetid', + version: 'datasetVersion', + identifier: 'datasetv2.identifier', + summary: 'datasetv2.summary', + documentation: 'datasetv2.documentation', + revisions: 'datasetv2.revisions', + modified: 'datasetv2.modified', + issued: 'datasetv2.issued', + accessibility: 'datasetv2.accessibility', + observations: 'datasetv2.observations', + provenance: 'datasetv2.provenance', + coverage: 'datasetv2.coverage', + enrichmentAndLinkage: 'datasetv2.enrichmentAndLinkage', + structuralMetadata: { + structuralMetadataCount: {}, + dataClasses: 'datasetfields.technicaldetails', + }, + }, + relatedObjects: 'relatedObjects', + metadataQuality: 'datasetfields.metadataquality', + dataUtility: 'datasetfields.datautility', + viewCounter: 'counter', + submittedDataAccessRequests: 'submittedDataAccessRequests', + }; + + // Transform entity into v2 using map, with stict applied to retain null values + const transformedObject = this.transformTo(transformer, { strict: true }); + + // Manually update identifier URL link + transformedObject.dataset.identifier = `https://web.www.healthdatagateway.org/dataset/${this.datasetid}`; + + // Append static schema details for v2 + const formattedObject = { + '@schema': { + type: `Dataset`, + version: `2.0.0`, + url: `https://raw.githubusercontent.com/HDRUK/schemata/master/schema/dataset/latest/dataset.schema.json`, + }, + ...transformedObject + }; + + // Return v2 object + return formattedObject; + } +} diff --git a/src/resources/dataset/dataset.service.js b/src/resources/dataset/dataset.service.js index 3d99202a..ef1640e7 100644 --- a/src/resources/dataset/dataset.service.js +++ b/src/resources/dataset/dataset.service.js @@ -1,7 +1,5 @@ import _ from 'lodash'; -import { v2Format } from './dataset.entity'; - export default class DatasetService { constructor(datasetRepository) { this.datasetRepository = datasetRepository; @@ -18,6 +16,7 @@ export default class DatasetService { } // Get latest data set + delete query.datasetid; dataset = await this.datasetRepository.getDataset({ ...query, pid: id, activeflag: 'active' }); // If no dataset is found active, look for most recent archived dataset @@ -34,7 +33,7 @@ export default class DatasetService { dataset.isLatestVersion = dataset.checkLatestVersion(); } else { // Transform to dataset v2 data structure - dataset = dataset.transformTo(v2Format, { strict: true }); + dataset = dataset.toV2Format(); } return dataset; From b9fbbbfff90e038ff9a8c429629ad4bdb7a30cfc Mon Sep 17 00:00:00 2001 From: Robin Kavanagh Date: Sun, 21 Feb 2021 11:18:41 +0000 Subject: [PATCH 26/42] Completed dataset output update --- src/resources/dataset/dataset.entity.js | 10 +- src/resources/dataset/dataset.repository.js | 20 +++- src/resources/dataset/dataset.service.js | 101 ++++++++++++++++++-- src/resources/dataset/dependency.js | 11 ++- 4 files changed, 125 insertions(+), 17 deletions(-) diff --git a/src/resources/dataset/dataset.entity.js b/src/resources/dataset/dataset.entity.js index abb133cd..9f757c55 100644 --- a/src/resources/dataset/dataset.entity.js +++ b/src/resources/dataset/dataset.entity.js @@ -60,9 +60,9 @@ export default class DatasetClass extends Entity { identifier: 'datasetv2.identifier', summary: 'datasetv2.summary', documentation: 'datasetv2.documentation', - revisions: 'datasetv2.revisions', - modified: 'datasetv2.modified', - issued: 'datasetv2.issued', + revisions: 'revisions', + modified: 'updatedAt', + issued: 'createdAt', accessibility: 'datasetv2.accessibility', observations: 'datasetv2.observations', provenance: 'datasetv2.provenance', @@ -81,7 +81,7 @@ export default class DatasetClass extends Entity { }; // Transform entity into v2 using map, with stict applied to retain null values - const transformedObject = this.transformTo(transformer, { strict: true }); + const transformedObject = this.transformTo(transformer, { strict: false }); // Manually update identifier URL link transformedObject.dataset.identifier = `https://web.www.healthdatagateway.org/dataset/${this.datasetid}`; @@ -95,7 +95,7 @@ export default class DatasetClass extends Entity { }, ...transformedObject }; - + // Return v2 object return formattedObject; } diff --git a/src/resources/dataset/dataset.repository.js b/src/resources/dataset/dataset.repository.js index 413e99b2..73c3ea96 100644 --- a/src/resources/dataset/dataset.repository.js +++ b/src/resources/dataset/dataset.repository.js @@ -13,7 +13,25 @@ export default class DatasetRepository extends Repository { } async getDatasets(query) { - const options = { lean: true, populate: { path: 'submittedDataAccessRequests' } }; + const options = { lean: false, populate: { path: 'submittedDataAccessRequests' } }; return this.find(query, options); } + + async getDatasetRevisions(pid) { + if (!pid) { + return {}; + } + // Get dataset versions using pid + const datasets = await Dataset.find({ pid }).select({ datasetid: 1, datasetVersion: 1, activeflag: 1 }).lean(); + // Create revision structure + return datasets.reduce((obj, dataset) => { + const { datasetVersion = 'default', datasetid = 'empty', activeflag = '' } = dataset; + obj[datasetVersion] = datasetid; + // Set the active dataset as the latest version + if (activeflag === 'active') { + obj['latest'] = datasetid; + } + return obj; + }, {}); + } } diff --git a/src/resources/dataset/dataset.service.js b/src/resources/dataset/dataset.service.js index ef1640e7..7a800820 100644 --- a/src/resources/dataset/dataset.service.js +++ b/src/resources/dataset/dataset.service.js @@ -1,45 +1,126 @@ import _ from 'lodash'; export default class DatasetService { - constructor(datasetRepository) { + constructor(datasetRepository, paperRepository, projectRepository, toolRepository, courseRepository) { this.datasetRepository = datasetRepository; + this.paperRepository = paperRepository; + this.projectRepository = projectRepository; + this.toolRepository = toolRepository; + this.courseRepository = courseRepository; } - + async getDataset(id, query = {}) { // Get dataset from Db by datasetid first - query = { ...query, datasetid: id}; + query = { ...query, datasetid: id }; let dataset = await this.datasetRepository.getDataset(query); // Update query to find the latest dataset if (!_.isNil(dataset)) { id = dataset.pid; } - + // Get latest data set delete query.datasetid; dataset = await this.datasetRepository.getDataset({ ...query, pid: id, activeflag: 'active' }); // If no dataset is found active, look for most recent archived dataset - if(!dataset) { + if (!dataset) { query.sort = '-createdAt'; dataset = await this.datasetRepository.getDataset({ ...query, pid: id, activeflag: 'archive' }); } // Return undefined if no dataset found - if(!dataset) return; + if (!dataset) return; + + // Populate derived fields + dataset.revisions = await this.datasetRepository.getDatasetRevisions(dataset.pid); + dataset.relatedObjects = await this.getRelatedObjects(dataset.pid); // Raw output responds with the data structure as defined in MongoDb - if(query['raw'] === true) { + if (query['raw'] === true) { dataset.isLatestVersion = dataset.checkLatestVersion(); } else { // Transform to dataset v2 data structure - dataset = dataset.toV2Format(); + let v2Response = dataset.toV2Format(); + // Temporary step of reformatting technical details until cache updated + v2Response.dataset = reformatTechnicalDetails(v2Response.dataset); + // Set full response + dataset = v2Response; } - return dataset; } - getDatasets(query = {}) { + async getDatasets(query = {}) { return this.datasetRepository.getDatasets(query); } + + async getRelatedObjects(pid) { + if (!pid) { + return {}; + } + + // Build query to find objects related to this pid + const query = { + relatedObjects: { + $elemMatch: { + pid, + }, + }, + activeflag: 'active', + fields: 'id, type, relatedObjects', + }; + + // Set query to be lean for performance optimisation + const lean = true; + + // Run query on each entity repository + const relatedEntities = await Promise.all([ + this.paperRepository.find(query, { lean }), + this.toolRepository.find(query, { lean }), + this.projectRepository.find(query, { lean }), + this.courseRepository.find(query, { lean }) + ]); + + // Flatten and reduce related entities into related objects + const relatedObjects = relatedEntities.flat().reduce((arr, entity) => { + let { relatedObjects: entityRelatedObjects = [] } = entity; + entityRelatedObjects = entityRelatedObjects.filter(obj => obj.pid === pid); + const formattedEntityRelatedObjects = entityRelatedObjects.map(obj => { + return { + objectId: entity.id, + reason: obj.reason, + objectType: entity.type, + user: obj.user, + updated: obj.updated, + } + }); + arr = [...arr, ...formattedEntityRelatedObjects]; + return arr; + }, []); + return relatedObjects; + } } + +const reformatTechnicalDetails = dataset => { + // Return if no technical details found + if (_.isNil(dataset.structuralMetadata) || _.isNil(dataset.structuralMetadata.dataClasses)) { + return dataset; + } + // Convert mongoose array to regular array + const dataClasses = Array.from([...dataset.structuralMetadata.dataClasses]) || []; + // Map data classes array into correct format + dataset.structuralMetadata.dataClasses = [...dataClasses].map(el => { + const { id = '', description = '', label: name = '', elements = [] } = el; + const dataElements = [...elements].map(dataEl => { + const { + id = '', + description = '', + label: name = '', + dataType: { domainType: type = '' }, + } = dataEl; + return { id, description, name, type }; + }); + return { id, description, name, dataElementsCount: dataElements.length || 0, dataElements }; + }); + return dataset; +}; diff --git a/src/resources/dataset/dependency.js b/src/resources/dataset/dependency.js index 06a5be78..1c7885c6 100644 --- a/src/resources/dataset/dependency.js +++ b/src/resources/dataset/dependency.js @@ -1,5 +1,14 @@ import DatasetRepository from './dataset.repository'; +import PaperRepository from '../paper/paper.repository'; +import ProjectRepository from '../project/project.repository'; +import ToolRepository from '../tool/v2/tool.repository'; +import CourseRepository from '../course/v2/course.repository'; import DatasetService from './dataset.service'; +const paperRepository = new PaperRepository(); +const projectRepository = new ProjectRepository(); +const toolRepository = new ToolRepository(); +const courseRepository = new CourseRepository(); + export const datasetRepository = new DatasetRepository(); -export const datasetService = new DatasetService(datasetRepository); +export const datasetService = new DatasetService(datasetRepository, paperRepository, projectRepository, toolRepository, courseRepository); From ef184394e70f82b620173758e14bc6599a05a776 Mon Sep 17 00:00:00 2001 From: Robin Kavanagh Date: Sun, 21 Feb 2021 18:52:07 +0000 Subject: [PATCH 27/42] Completed tests --- src/resources/base/__tests__/entity.test.js | 54 +-- src/resources/dataset/__mocks__/datasets.js | 365 +++++++++++---- .../__tests__/dataset.controller.test.js | 2 +- .../dataset/__tests__/dataset.entity.test.js | 126 +++-- .../__tests__/dataset.repository.it.test.js | 4 +- .../__tests__/dataset.repository.test.js | 49 +- .../dataset/__tests__/dataset.service.test.js | 441 +++++++++++++++--- src/resources/dataset/dataset.entity.js | 46 +- src/resources/dataset/dataset.repository.js | 4 +- src/resources/dataset/dataset.service.js | 75 ++- 10 files changed, 804 insertions(+), 362 deletions(-) diff --git a/src/resources/base/__tests__/entity.test.js b/src/resources/base/__tests__/entity.test.js index eb27a803..f474607c 100644 --- a/src/resources/base/__tests__/entity.test.js +++ b/src/resources/base/__tests__/entity.test.js @@ -3,33 +3,33 @@ import DatasetClass from '../../dataset/dataset.entity'; describe('Entity', function () { describe('equals', function () { it('should return true if equality exists where both objects are of the same instance', async function () { - const dataset = new DatasetClass(675584862177848, 'Admitted Patient Care Dataset'); - - const result = dataset.equals(dataset); + const dataset = new DatasetClass({ id: 675584862177848, name: 'Admitted Patient Care Dataset' }); + + const result = dataset.equals(dataset); expect(result).toBe(true); }); it('should return false if both objects are of the same type with differing properties', async function () { - const dataset = new DatasetClass(675584862177848, 'Admitted Patient Care Dataset'); - const referenceDataset = new DatasetClass(null, 'Admitted Patient Care Dataset'); - - const result = dataset.equals(referenceDataset); + const dataset = new DatasetClass({ id: 675584862177848, name: 'Admitted Patient Care Dataset' }); + const referenceDataset = new DatasetClass(null, 'Admitted Patient Care Dataset'); + + const result = dataset.equals(referenceDataset); expect(result).toBe(false); }); - it('should return false if equality does not exist', async function () { - const dataset = new DatasetClass(675584862177848, 'Admitted Patient Care Dataset'); + it('should return false if equality does not exist', async function () { + const dataset = new DatasetClass({ id: 675584862177848, name: 'Admitted Patient Care Dataset' }); const referenceDataset = new DatasetClass(246523922611217, 'Reference Dataset'); - const result = dataset.equals(referenceDataset); + const result = dataset.equals(referenceDataset); expect(result).toBe(false); }); it('should return immediate false result if reference object is not an entity', async function () { - const dataset = new DatasetClass(675584862177848, 'Admitted Patient Care Dataset'); + const dataset = new DatasetClass({ id: 675584862177848, name: 'Admitted Patient Care Dataset' }); const referenceObject = { id: 675584862177848 }; - const result = dataset.equals(referenceObject); + const result = dataset.equals(referenceObject); expect(result).toBe(false); }); @@ -37,26 +37,26 @@ describe('Entity', function () { describe('referenceEquals', function () { it('should return false if entity does not have an identifier assigned', async function () { - const dataset = new DatasetClass(null, 'Admitted Patient Care Dataset'); - const referenceId = 675584862177848; - - const result = dataset.referenceEquals(referenceId); + const dataset = new DatasetClass({ id: null, name: 'Admitted Patient Care Dataset' }); + const referenceId = 675584862177848; + + const result = dataset.referenceEquals(referenceId); expect(result).toBe(false); }); it('should return true if reference equality exists', async function () { - const dataset = new DatasetClass(675584862177848, 'Admitted Patient Care Dataset'); - const referenceId = 675584862177848; - - const result = dataset.referenceEquals(referenceId); + const dataset = new DatasetClass({ id: 675584862177848, name: 'Admitted Patient Care Dataset' }); + const referenceId = 675584862177848; + + const result = dataset.referenceEquals(referenceId); expect(result).toBe(true); }); - it('should return false if reference equality does not exist', async function () { - const dataset = new DatasetClass(675584862177848, 'Admitted Patient Care Dataset'); - const referenceId = 246523922611217; - - const result = dataset.referenceEquals(referenceId); + it('should return false if reference equality does not exist', async function () { + const dataset = new DatasetClass({ id: 675584862177848, name: 'Admitted Patient Care Dataset' }); + const referenceId = 246523922611217; + + const result = dataset.referenceEquals(referenceId); expect(result).toBe(false); }); @@ -64,9 +64,9 @@ describe('Entity', function () { describe('toString', function () { it('should return a string representation of an object identifier', async function () { - const dataset = new DatasetClass(675584862177848, 'Admitted Patient Care Dataset'); + const dataset = new DatasetClass({ id: 675584862177848, name: 'Admitted Patient Care Dataset' }); - const result = dataset.toString(); + const result = dataset.toString(); expect(result).toEqual(dataset.id); }); diff --git a/src/resources/dataset/__mocks__/datasets.js b/src/resources/dataset/__mocks__/datasets.js index cea577ec..f68c1101 100644 --- a/src/resources/dataset/__mocks__/datasets.js +++ b/src/resources/dataset/__mocks__/datasets.js @@ -1,100 +1,265 @@ -export const datasetsStub = [{ - id: '675584862177848', - submittedDataAccessRequests: 1, - name: 'Admitted Patient Care Dataset', - description: 'This is a dataset about admitted patient care', - resultsInsights: null, - link: null, - type: 'dataset', - datasetid: 'dfb21b3b-7fd9-40c4-892e-810edd6dfc25', - categories: {}, - license: null, - authors: [], - tags: {}, - activeflag: 'active', - counter: 15, - discourseTopicId: null, - relatedObjects: [], - uploader: null, - datasetid: 'dfb21b3b-7fd9-40c4-892e-810edd6dfc25', - pid: '4ef841d3-5e86-4f92-883f-1015ffd4b979', - datasetVersion: '0.0.1', - datasetfields: { - publisher: "Oxford University Hospitals NHS Foundation Trust", - geographicCoverage: [], - physicalSampleAvailability: [], - abstract: "Nationally defined dataset which ontaining administrative details for inpatient admissions (elective, emergency and maternity) and good coverage of clinical coding of diagnosis (ICD10) and procedures (OPCS4). Includes home birth and delivery spells.", - releaseDate: null, - accessRequestDuration: null, - conformsTo: null, - accessRights: "Available locally within Trust Clinical Data Warehouse System and authorised access by Trust staff only.\nDataset is also available via SUS/HES for government statistical purposes.", - jurisdiction: null, - datasetStartDate: null, - datasetEndDate: null, - statisticalPopulation: null, - ageBand: null, - contactPoint: "kinga.varnai@ouh.nhs.uk", - periodicity: null, - metadataquality: { - id: "dfb21b3b-7fd9-40c4-892e-810edd6dfc25", - publisher: "Oxford University Hospitals NHS Foundation Trust", - title: "Admitted Patient Care Dataset", - completeness_percent: 16.67, - weighted_completeness_percent: 14.29, - error_percent: 39.53, - weighted_error_percent: 39.68, - quality_score: 38.57, - quality_rating: "Not Rated", - weighted_quality_score: 37.3, - weighted_quality_rating: "Not Rated" - }, - datautility: { - id: "dfb21b3b-7fd9-40c4-892e-810edd6dfc25", - publisher: "Oxford University Hospitals NHS Foundation Trust", - title: "Admitted Patient Care Dataset", - metadata_richness: "Not Rated", - availability_of_additional_documentation_and_support: "", - data_model: "", - data_dictionary: "", - provenance: "", - data_quality_management_process: "", - dama_quality_dimensions: "", - pathway_coverage: "", - length_of_follow_up: "", - allowable_uses: "", - research_environment: "", - time_lag: "", - timeliness: "", - linkages: "", - data_enrichments: "" - }, - metadataschema: { - context: "http://schema.org/", - type: "Dataset", - identifier: "dfb21b3b-7fd9-40c4-892e-810edd6dfc25", - url: "https://healthdatagateway.org/detail/dfb21b3b-7fd9-40c4-892e-810edd6dfc25", - name: "Admitted Patient Care Dataset", - description: "", - keywords: [ - "Oxford University Hospitals NHS Foundation Trust", - "CDS" - ], - includedinDataCatalog: [ - { - type: "DataCatalog", - name: "Oxford University Hospitals NHS Foundation Trust", - url: "kinga.varnai@ouh.nhs.uk" - }, - { - type: "DataCatalog", - name: "HDR UK Health Data Gateway", - url: "http://healthdatagateway.org" - } - ] - }, - technicaldetails: [], - versionLinks: [], - phenotypes: [] - }, - datasetv2: {} - }]; \ No newline at end of file +export const datasetsStub = [ + { + id: 675584862177848, + submittedDataAccessRequests: 1, + name: 'Admitted Patient Care Dataset', + description: 'This is a dataset about admitted patient care', + resultsInsights: null, + link: null, + type: 'dataset', + license: null, + authors: [], + tags: { + features: [], + topics: [], + }, + activeflag: 'archive', + counter: 15, + discourseTopicId: null, + relatedObjects: [], + uploader: null, + datasetid: 'dfb21b3b-7fd9-40c4-892e-810edd6dfc25', + pid: '4ef841d3-5e86-4f92-883f-1015ffd4b979', + datasetVersion: '0.0.1', + datasetfields: { + publisher: 'Oxford University Hospitals NHS Foundation Trust', + geographicCoverage: [], + physicalSampleAvailability: [], + abstract: + 'Nationally defined dataset which ontaining administrative details for inpatient admissions (elective, emergency and maternity) and good coverage of clinical coding of diagnosis (ICD10) and procedures (OPCS4). Includes home birth and delivery spells.', + releaseDate: null, + accessRequestDuration: null, + conformsTo: null, + accessRights: + 'Available locally within Trust Clinical Data Warehouse System and authorised access by Trust staff only.\nDataset is also available via SUS/HES for government statistical purposes.', + jurisdiction: null, + datasetStartDate: null, + datasetEndDate: null, + statisticalPopulation: null, + ageBand: null, + contactPoint: 'kinga.varnai@ouh.nhs.uk', + periodicity: null, + metadataquality: { + id: 'dfb21b3b-7fd9-40c4-892e-810edd6dfc25', + publisher: 'Oxford University Hospitals NHS Foundation Trust', + title: 'Admitted Patient Care Dataset', + completeness_percent: 16.67, + weighted_completeness_percent: 14.29, + error_percent: 39.53, + weighted_error_percent: 39.68, + quality_score: 38.57, + quality_rating: 'Not Rated', + weighted_quality_score: 37.3, + weighted_quality_rating: 'Not Rated', + }, + datautility: { + id: 'dfb21b3b-7fd9-40c4-892e-810edd6dfc25', + publisher: 'Oxford University Hospitals NHS Foundation Trust', + title: 'Admitted Patient Care Dataset', + metadata_richness: 'Not Rated', + availability_of_additional_documentation_and_support: '', + data_model: '', + data_dictionary: '', + provenance: '', + data_quality_management_process: '', + dama_quality_dimensions: '', + pathway_coverage: '', + length_of_follow_up: '', + allowable_uses: '', + research_environment: '', + time_lag: '', + timeliness: '', + linkages: '', + data_enrichments: '', + }, + metadataschema: { + context: 'http://schema.org/', + type: 'Dataset', + identifier: 'dfb21b3b-7fd9-40c4-892e-810edd6dfc25', + url: 'https://healthdatagateway.org/detail/dfb21b3b-7fd9-40c4-892e-810edd6dfc25', + name: 'Admitted Patient Care Dataset', + description: '', + keywords: ['Oxford University Hospitals NHS Foundation Trust', 'CDS'], + includedinDataCatalog: [ + { + type: 'DataCatalog', + name: 'Oxford University Hospitals NHS Foundation Trust', + url: 'kinga.varnai@ouh.nhs.uk', + }, + { + type: 'DataCatalog', + name: 'HDR UK Health Data Gateway', + url: 'http://healthdatagateway.org', + }, + ], + }, + technicaldetails: [], + versionLinks: [], + phenotypes: [], + }, + }, + { + id: '675584862177848', + submittedDataAccessRequests: 1, + name: 'Admitted Patient Care Dataset', + description: 'This is a dataset about admitted patient care', + resultsInsights: null, + link: null, + type: 'dataset', + datasetid: '5b99795a-3db4-4020-a8f9-aa64bcc2c5f0', + categories: {}, + license: null, + authors: [], + tags: {}, + activeflag: 'active', + counter: 15, + discourseTopicId: null, + relatedObjects: [], + uploader: null, + pid: '4ef841d3-5e86-4f92-883f-1015ffd4b979', + datasetVersion: '1.0.0', + datasetfields: { + publisher: 'Oxford University Hospitals NHS Foundation Trust', + geographicCoverage: [], + physicalSampleAvailability: [], + abstract: + 'Nationally defined dataset which ontaining administrative details for inpatient admissions (elective, emergency and maternity) and good coverage of clinical coding of diagnosis (ICD10) and procedures (OPCS4). Includes home birth and delivery spells.', + releaseDate: null, + accessRequestDuration: null, + conformsTo: null, + accessRights: + 'Available locally within Trust Clinical Data Warehouse System and authorised access by Trust staff only.\nDataset is also available via SUS/HES for government statistical purposes.', + jurisdiction: null, + datasetStartDate: null, + datasetEndDate: null, + statisticalPopulation: null, + ageBand: null, + contactPoint: 'kinga.varnai@ouh.nhs.uk', + periodicity: null, + metadataquality: { + id: 'dfb21b3b-7fd9-40c4-892e-810edd6dfc25', + publisher: 'Oxford University Hospitals NHS Foundation Trust', + title: 'Admitted Patient Care Dataset', + completeness_percent: 16.67, + weighted_completeness_percent: 14.29, + error_percent: 39.53, + weighted_error_percent: 39.68, + quality_score: 38.57, + quality_rating: 'Not Rated', + weighted_quality_score: 37.3, + weighted_quality_rating: 'Not Rated', + }, + datautility: { + id: 'dfb21b3b-7fd9-40c4-892e-810edd6dfc25', + publisher: 'Oxford University Hospitals NHS Foundation Trust', + title: 'Admitted Patient Care Dataset', + metadata_richness: 'Not Rated', + availability_of_additional_documentation_and_support: '', + data_model: '', + data_dictionary: '', + provenance: '', + data_quality_management_process: '', + dama_quality_dimensions: '', + pathway_coverage: '', + length_of_follow_up: '', + allowable_uses: '', + research_environment: '', + time_lag: '', + timeliness: '', + linkages: '', + data_enrichments: '', + }, + metadataschema: { + context: 'http://schema.org/', + type: 'Dataset', + identifier: 'dfb21b3b-7fd9-40c4-892e-810edd6dfc25', + url: 'https://healthdatagateway.org/detail/dfb21b3b-7fd9-40c4-892e-810edd6dfc25', + name: 'Admitted Patient Care Dataset', + description: '', + keywords: ['Oxford University Hospitals NHS Foundation Trust', 'CDS'], + includedinDataCatalog: [ + { + type: 'DataCatalog', + name: 'Oxford University Hospitals NHS Foundation Trust', + url: 'kinga.varnai@ouh.nhs.uk', + }, + { + type: 'DataCatalog', + name: 'HDR UK Health Data Gateway', + url: 'http://healthdatagateway.org', + }, + ], + }, + technicaldetails: [], + versionLinks: [], + phenotypes: [], + }, + datasetv2: {}, + }, +]; + +export const v2DatasetsStub = [ + { + pid: 'b67f0edd-fed2-4d68-a25f-d225759aa3b0', + id: 'dfb21b3b-7fd9-40c4-892e-810edd6dfc25', + version: '0.0.1', + identifier: 'https://web.www.healthdatagateway.org/dataset/dfb21b3b-7fd9-40c4-892e-810edd6dfc25', + revisions: { + '0.0.1': 'dfb21b3b-7fd9-40c4-892e-810edd6dfc25', + latest: 'dfb21b3b-7fd9-40c4-892e-810edd6dfc25', + }, + modified: '2021-02-21T04:01:24.347Z', + issued: '2020-08-10T14:41:33.783Z', + structuralMetadata: { + structuralMetadataCount: {}, + dataClasses: [ + { + id: '857a5ee0-196c-4247-9b95-5b1aed70fea1', + domainType: 'DataClass', + label: 'Demography_Current.csv', + description: "This class was created from the White Rabbit profile data in 'Demography_Current.csv'", + elements: [ + { + id: 'd90bd4c2-df71-438c-879a-fe159e651517', + domainType: 'DataElement', + label: 'current_gp_accept_date', + description: null, + dataType: { + id: 'aff244ca-2f27-42b7-90a9-4de5b81a6063', + domainType: 'PrimitiveType', + label: 'VARCHAR', + }, + }, + { + id: '907a641c-86c4-4a95-adc4-3c2bbea35f59', + domainType: 'DataElement', + label: 'anon_date_of_birth', + description: null, + dataType: { + id: 'aff244ca-2f27-42b7-90a9-4de5b81a6063', + domainType: 'PrimitiveType', + label: 'VARCHAR', + }, + }, + ], + }, + ], + }, + }, + { + pid: 'b67f0edd-fed2-4d68-a25f-d225759aa3b0', + id: 'dfb21b3b-7fd9-40c4-892e-810edd6dfc25', + version: '0.0.1', + identifier: 'https://web.www.healthdatagateway.org/dataset/dfb21b3b-7fd9-40c4-892e-810edd6dfc25', + revisions: { + '0.0.1': 'dfb21b3b-7fd9-40c4-892e-810edd6dfc25', + latest: 'dfb21b3b-7fd9-40c4-892e-810edd6dfc25', + }, + modified: '2021-02-21T04:01:24.347Z', + issued: '2020-08-10T14:41:33.783Z', + structuralMetadata: { + structuralMetadataCount: {}, + dataClasses: [], + }, + }, +]; diff --git a/src/resources/dataset/__tests__/dataset.controller.test.js b/src/resources/dataset/__tests__/dataset.controller.test.js index 5d50f07c..ff39944b 100644 --- a/src/resources/dataset/__tests__/dataset.controller.test.js +++ b/src/resources/dataset/__tests__/dataset.controller.test.js @@ -32,7 +32,7 @@ describe('DatasetController', function () { expect(serviceStub.calledOnce).toBe(true); expect(status.calledWith(200)).toBe(true); - expect(json.calledWith({ success: true, data: stubValue })).toBe(true); + expect(json.calledWith({ success: true, ...stubValue })).toBe(true); }); it('should return a bad request response if no dataset id is provided', async function () { diff --git a/src/resources/dataset/__tests__/dataset.entity.test.js b/src/resources/dataset/__tests__/dataset.entity.test.js index 5cb85016..9cd9fea8 100644 --- a/src/resources/dataset/__tests__/dataset.entity.test.js +++ b/src/resources/dataset/__tests__/dataset.entity.test.js @@ -3,28 +3,26 @@ import DatasetClass from '../dataset.entity'; describe('DatasetEntity', function () { describe('constructor', function () { it('should create an instance of a dataset entity with the expected properties', async function () { - const dataset = new DatasetClass( - 675584862177848, - "Admitted Patient Care Dataset", - "This is a dataset about admitted patient care", - null, - null, - "dataset", - {}, - null, - [], - {}, - "active", - 15, - null, - [], - null, - "dfb21b3b-7fd9-40c4-892e-810edd6dfc25", - "4ef841d3-5e86-4f92-883f-1015ffd4b979", - "0.0.1", - { publisher: "Oxford University Hospitals NHS Foundation Trust" }, - {} - ); + const dataset = new DatasetClass({ + id: 675584862177848, + name: "Admitted Patient Care Dataset", + description: "This is a dataset about admitted patient care", + resultsInsights: null, + license: null, + type: "dataset", + categories: {}, + discourseTopicId: null, + relatedObjects: [], + datasetv2: {}, + activeflag: "active", + counter: 15, + uploader: null, + authors: [], + datasetid: "dfb21b3b-7fd9-40c4-892e-810edd6dfc25", + pid: "4ef841d3-5e86-4f92-883f-1015ffd4b979", + datasetVersion: "0.0.1", + datasetfields: { publisher: "Oxford University Hospitals NHS Foundation Trust" } + }); expect(dataset.datasetid).toEqual("dfb21b3b-7fd9-40c4-892e-810edd6dfc25"); expect(dataset.type).toEqual("dataset"); @@ -50,28 +48,26 @@ describe('DatasetEntity', function () { describe('checkLatestVersion', function () { it('should return a boolean indicating this is the latest version of the dataset', async function () { - const dataset = new DatasetClass( - 675584862177848, - "Admitted Patient Care Dataset", - "This is a dataset about admitted patient care", - null, - null, - "dataset", - {}, - null, - [], - {}, - "active", - 15, - null, - [], - null, - "dfb21b3b-7fd9-40c4-892e-810edd6dfc25", - "4ef841d3-5e86-4f92-883f-1015ffd4b979", - "0.0.1", - { publisher: "Oxford University Hospitals NHS Foundation Trust" }, - {} - ); + const dataset = new DatasetClass({ + id: 675584862177848, + name: "Admitted Patient Care Dataset", + description: "This is a dataset about admitted patient care", + resultsInsights: null, + license: null, + type: "dataset", + categories: {}, + discourseTopicId: null, + relatedObjects: [], + datasetv2: {}, + activeflag: "active", + counter: 15, + uploader: null, + authors: [], + datasetid: "dfb21b3b-7fd9-40c4-892e-810edd6dfc25", + pid: "4ef841d3-5e86-4f92-883f-1015ffd4b979", + datasetVersion: "0.0.1", + datasetfields: { publisher: "Oxford University Hospitals NHS Foundation Trust" } + }); const result = dataset.checkLatestVersion(); @@ -79,28 +75,26 @@ describe('DatasetEntity', function () { }); it('should return a boolean indicating this is not the latest version of the dataset', async function () { - const dataset = new DatasetClass( - 675584862177848, - "Admitted Patient Care Dataset", - "This is a dataset about admitted patient care", - null, - null, - "dataset", - {}, - null, - [], - {}, - "archive", - 15, - null, - [], - null, - "dfb21b3b-7fd9-40c4-892e-810edd6dfc25", - "4ef841d3-5e86-4f92-883f-1015ffd4b979", - "0.0.1", - { publisher: "Oxford University Hospitals NHS Foundation Trust" }, - {} - ); + const dataset = new DatasetClass({ + id: 675584862177848, + name: "Admitted Patient Care Dataset", + description: "This is a dataset about admitted patient care", + resultsInsights: null, + license: null, + type: "dataset", + categories: {}, + discourseTopicId: null, + relatedObjects: [], + datasetv2: {}, + activeflag: "archive", + counter: 15, + uploader: null, + authors: [], + datasetid: "dfb21b3b-7fd9-40c4-892e-810edd6dfc25", + pid: "4ef841d3-5e86-4f92-883f-1015ffd4b979", + datasetVersion: "0.0.1", + datasetfields: { publisher: "Oxford University Hospitals NHS Foundation Trust" } + }); const result = dataset.checkLatestVersion(); diff --git a/src/resources/dataset/__tests__/dataset.repository.it.test.js b/src/resources/dataset/__tests__/dataset.repository.it.test.js index bce45989..d20de253 100644 --- a/src/resources/dataset/__tests__/dataset.repository.it.test.js +++ b/src/resources/dataset/__tests__/dataset.repository.it.test.js @@ -28,8 +28,8 @@ describe('DatasetRepository', function () { describe('getDataset', () => { it('should return a dataset by a specified id', async function () { const datasetRepository = new DatasetRepository(); - const dataset = await datasetRepository.getDataset("dfb21b3b-7fd9-40c4-892e-810edd6dfc25"); - expect(dataset).toEqual(datasetsStub[0]); + const dataset = await datasetRepository.getDataset({datasetid: "dfb21b3b-7fd9-40c4-892e-810edd6dfc25"}); + expect(dataset.toObject()).toEqual(datasetsStub[0]); }); }); diff --git a/src/resources/dataset/__tests__/dataset.repository.test.js b/src/resources/dataset/__tests__/dataset.repository.test.js index 8cadab39..03e0981e 100644 --- a/src/resources/dataset/__tests__/dataset.repository.test.js +++ b/src/resources/dataset/__tests__/dataset.repository.test.js @@ -42,7 +42,6 @@ describe('DatasetRepository', function () { const datasets = await datasetRepository.getDatasets(); expect(stub.calledOnce).toBe(true); - expect(datasets.length).toBeGreaterThan(0); }); }); @@ -54,8 +53,52 @@ describe('DatasetRepository', function () { const datasetCount = await datasetRepository.findCountOf({ name: 'Admitted Patient Care Dataset' }); expect(stub.calledOnce).toBe(true); - expect(datasetCount).toEqual(1); }); }); -}); \ No newline at end of file + + describe('getDatasetRevisions', function () { + it('should return an empty object if an invalid persistent identifier is passed', async function () { + const datasetRepository = new DatasetRepository(); + const stub = sinon.stub(datasetRepository, 'find').returns({}); + + const datasetRevisions = await datasetRepository.getDatasetRevisions(null); + + expect(stub.notCalled).toBe(true); + expect(datasetRevisions).toEqual({}); + }); + }); + + describe('getDatasetRevisions', function () { + it('should return an object illustrating all versions of the dataset', async function () { + const datasetRepository = new DatasetRepository(); + const stub = sinon + .stub(datasetRepository, 'find') + .returns(datasetsStub.filter(obj => obj.pid === '4ef841d3-5e86-4f92-883f-1015ffd4b979')); + const datasetRevisions = await datasetRepository.getDatasetRevisions('4ef841d3-5e86-4f92-883f-1015ffd4b979'); + + expect(stub.calledOnce).toBe(true); + expect(datasetRevisions).toEqual({ + '0.0.1': 'dfb21b3b-7fd9-40c4-892e-810edd6dfc25', + '1.0.0': '5b99795a-3db4-4020-a8f9-aa64bcc2c5f0', + latest: '5b99795a-3db4-4020-a8f9-aa64bcc2c5f0', + }); + }); + it('should return a default object if version information is missing from datasets', async function () { + const filterDatasets = datasetsStub.filter(obj => obj.pid === '4ef841d3-5e86-4f92-883f-1015ffd4b979'); + filterDatasets.forEach(dataset => { + delete dataset.datasetVersion; + delete dataset.datasetid; + delete dataset.activeflag; + }); + const datasetRepository = new DatasetRepository(); + const stub = sinon.stub(datasetRepository, 'find').returns(filterDatasets); + const datasetRevisions = await datasetRepository.getDatasetRevisions('4ef841d3-5e86-4f92-883f-1015ffd4b979'); + + expect(stub.calledOnce).toBe(true); + expect(datasetRevisions).toEqual({ + default: 'empty', + }); + }); + }); +}); diff --git a/src/resources/dataset/__tests__/dataset.service.test.js b/src/resources/dataset/__tests__/dataset.service.test.js index 482a2fc0..b1cacd89 100644 --- a/src/resources/dataset/__tests__/dataset.service.test.js +++ b/src/resources/dataset/__tests__/dataset.service.test.js @@ -1,111 +1,402 @@ import sinon from 'sinon'; import DatasetRepository from '../dataset.repository'; +import PaperRepository from '../../paper/paper.repository'; +import ProjectRepository from '../../project/project.repository'; +import ToolRepository from '../../tool/v2/tool.repository'; +import CourseRepository from '../../course/v2/course.repository'; import DatasetService from '../dataset.service'; -import { datasetsStub } from '../__mocks__/datasets'; +import { datasetsStub, v2DatasetsStub } from '../__mocks__/datasets'; +import DatasetClass from '../dataset.entity'; describe('DatasetService', function () { describe('getDataset', function () { - it('should return a dataset by a specified id', async function () { - const datasetStub = datasetsStub[0]; + it('returns the matching dataset, when a valid dataset identifier for an existing dataset is provided', async function () { + // Fixtures + const datasetStub = new DatasetClass(datasetsStub[0]); + + // Dependencies const datasetRepository = new DatasetRepository(); - const stub = sinon.stub(datasetRepository, 'getDataset').returns(datasetStub); const datasetService = new DatasetService(datasetRepository); - const dataset = await datasetService.getDataset(datasetStub.id); - expect(stub.calledOnce).toBe(true); + // Stubs + const getDatasetStub = sinon.stub(datasetRepository, 'getDataset').returns(datasetStub); - expect(dataset.datasetid).toEqual(datasetStub.datasetid); - expect(dataset.type).toEqual(datasetStub.type); - expect(dataset.id).toEqual(datasetStub.id); - expect(dataset.name).toEqual(datasetStub.name); - expect(dataset.description).toEqual(datasetStub.description); - expect(dataset.resultsInsights).toEqual(datasetStub.resultsInsights); - expect(dataset.datasetid).toEqual(datasetStub.datasetid); - expect(dataset.categories).toEqual(datasetStub.categories); - expect(dataset.license).toEqual(datasetStub.license); - expect(dataset.authors).toEqual(datasetStub.authors); - expect(dataset.activeflag).toEqual(datasetStub.activeflag); - expect(dataset.counter).toEqual(datasetStub.counter); - expect(dataset.discourseTopicId).toEqual(datasetStub.discourseTopicId); - expect(dataset.relatedObjects).toEqual(datasetStub.relatedObjects); - expect(dataset.uploader).toEqual(datasetStub.uploader); - expect(dataset.pid).toEqual(datasetStub.pid); - expect(dataset.datasetVersion).toEqual(datasetStub.datasetVersion); - expect(dataset.datasetfields).toEqual(datasetStub.datasetfields); - expect(dataset.datasetv2).toEqual(datasetStub.datasetv2); + const getDatasetRevisionsStub = sinon.stub(datasetRepository, 'getDatasetRevisions').returns({ + '0.0.1': '92f668ee-5ae8-4fb4-8755-4639fe100fde', + '1.0.0': '5b99795a-3db4-4020-a8f9-aa64bcc2c5f0', + latest: '5b99795a-3db4-4020-a8f9-aa64bcc2c5f0', + }); + + const getRelatedObjectsStub = sinon.stub(datasetService, 'getRelatedObjects').returns([ + { + _id: '6032412570f7ff0299a5b7c6', + objectId: '7527250949367925', + objectType: 'project', + user: 'Example user', + updated: '17 Feb 2021', + }, + ]); + + // Act + const v2DatasetObj = await datasetService.getDataset(datasetStub.id); + + // Assert + expect(getDatasetStub.calledOnce).toBe(true); + expect(getDatasetRevisionsStub.calledOnce).toBe(true); + expect(getRelatedObjectsStub.calledOnce).toBe(true); + expect(v2DatasetObj.dataset.id).toEqual(datasetStub.datasetid); }); - it('should return a dataset by a specified id when a query parameter is passed as the second argument', async function () { - const datasetStub = datasetsStub[0]; + it('returns the matching dataset, when a valid dataset identifier and query parameter is passed for an existing dataset', async function () { + // Fixtures + const datasetStub = new DatasetClass(datasetsStub[0]); + + // Dependencies const datasetRepository = new DatasetRepository(); - const stub = sinon.stub(datasetRepository, 'getDataset').returns(datasetStub); const datasetService = new DatasetService(datasetRepository); - const dataset = await datasetService.getDataset(datasetStub.id, { expanded: false }); - expect(stub.calledOnce).toBe(true); + // Stubs + const getDatasetStub = sinon.stub(datasetRepository, 'getDataset').returns(datasetStub); - expect(dataset.datasetid).toEqual(datasetStub.datasetid); - expect(dataset.type).toEqual(datasetStub.type); - expect(dataset.id).toEqual(datasetStub.id); - expect(dataset.name).toEqual(datasetStub.name); - expect(dataset.description).toEqual(datasetStub.description); - expect(dataset.resultsInsights).toEqual(datasetStub.resultsInsights); - expect(dataset.datasetid).toEqual(datasetStub.datasetid); - expect(dataset.categories).toEqual(datasetStub.categories); - expect(dataset.license).toEqual(datasetStub.license); - expect(dataset.authors).toEqual(datasetStub.authors); - expect(dataset.activeflag).toEqual(datasetStub.activeflag); - expect(dataset.counter).toEqual(datasetStub.counter); - expect(dataset.discourseTopicId).toEqual(datasetStub.discourseTopicId); - expect(dataset.relatedObjects).toEqual(datasetStub.relatedObjects); - expect(dataset.uploader).toEqual(datasetStub.uploader); - expect(dataset.pid).toEqual(datasetStub.pid); - expect(dataset.datasetVersion).toEqual(datasetStub.datasetVersion); - expect(dataset.datasetfields).toEqual(datasetStub.datasetfields); - expect(dataset.datasetv2).toEqual(datasetStub.datasetv2); + const getDatasetRevisionsStub = sinon.stub(datasetRepository, 'getDatasetRevisions').returns({ + '0.0.1': '92f668ee-5ae8-4fb4-8755-4639fe100fde', + '1.0.0': '5b99795a-3db4-4020-a8f9-aa64bcc2c5f0', + latest: '5b99795a-3db4-4020-a8f9-aa64bcc2c5f0', + }); + + const getRelatedObjectsStub = sinon.stub(datasetService, 'getRelatedObjects').returns([ + { + _id: '6032412570f7ff0299a5b7c6', + objectId: '7527250949367925', + objectType: 'project', + user: 'Example user', + updated: '17 Feb 2021', + }, + ]); + + // Act + const v2DatasetObj = await datasetService.getDataset(datasetStub.id, { activeflag: 'active' }); + + // Assert + expect(getDatasetStub.calledOnce).toBe(true); + expect(getDatasetRevisionsStub.calledOnce).toBe(true); + expect(getRelatedObjectsStub.calledOnce).toBe(true); + expect(v2DatasetObj.dataset.id).toEqual(datasetStub.datasetid); }); - it('should return a dataset by a specified id when expanded', async function () { - const datasetStub = datasetsStub[0]; + it('returns the raw dataset in database format if the raw query parameter is passed', async function () { + // Fixtures + const datasetStub = new DatasetClass(datasetsStub[0]); + + // Dependencies const datasetRepository = new DatasetRepository(); - const stub = sinon.stub(datasetRepository, 'getDataset').returns({ ...datasetStub, checkLatestVersion: () => true}); const datasetService = new DatasetService(datasetRepository); - const dataset = await datasetService.getDataset(datasetStub.id, { expanded: true }); - expect(stub.calledOnce).toBe(true); + // Stubs + const getDatasetStub = sinon.stub(datasetRepository, 'getDataset').returns(datasetStub); + const getDatasetRevisionsStub = sinon.stub(datasetRepository, 'getDatasetRevisions').returns({ + '0.0.1': '92f668ee-5ae8-4fb4-8755-4639fe100fde', + '1.0.0': '5b99795a-3db4-4020-a8f9-aa64bcc2c5f0', + latest: '5b99795a-3db4-4020-a8f9-aa64bcc2c5f0', + }); + + const getRelatedObjectsStub = sinon.stub(datasetService, 'getRelatedObjects').returns([ + { + _id: '6032412570f7ff0299a5b7c6', + objectId: '7527250949367925', + objectType: 'project', + user: 'Example user', + updated: '17 Feb 2021', + }, + ]); + + // Spies + const toV2FormatSpy = sinon.spy(datasetStub.toV2Format); + const reformatTechnicalDetails = sinon.spy(datasetService.reformatTechnicalDetails); + + // Act + const dataset = await datasetService.getDataset(datasetStub.id, { activeflag: 'active', raw: true }); + + // Assert + expect(getDatasetStub.calledOnce).toBe(true); + expect(getDatasetRevisionsStub.calledOnce).toBe(true); + expect(getRelatedObjectsStub.calledOnce).toBe(true); + expect(toV2FormatSpy.notCalled).toBe(true); + expect(reformatTechnicalDetails.notCalled).toBe(true); expect(dataset.datasetid).toEqual(datasetStub.datasetid); - expect(dataset.type).toEqual(datasetStub.type); - expect(dataset.id).toEqual(datasetStub.id); - expect(dataset.name).toEqual(datasetStub.name); - expect(dataset.description).toEqual(datasetStub.description); - expect(dataset.resultsInsights).toEqual(datasetStub.resultsInsights); - expect(dataset.datasetid).toEqual(datasetStub.datasetid); - expect(dataset.categories).toEqual(datasetStub.categories); - expect(dataset.license).toEqual(datasetStub.license); - expect(dataset.authors).toEqual(datasetStub.authors); - expect(dataset.activeflag).toEqual(datasetStub.activeflag); - expect(dataset.counter).toEqual(datasetStub.counter); - expect(dataset.discourseTopicId).toEqual(datasetStub.discourseTopicId); - expect(dataset.relatedObjects).toEqual(datasetStub.relatedObjects); - expect(dataset.uploader).toEqual(datasetStub.uploader); - expect(dataset.pid).toEqual(datasetStub.pid); - expect(dataset.datasetVersion).toEqual(datasetStub.datasetVersion); - expect(dataset.datasetfields).toEqual(datasetStub.datasetfields); - expect(dataset.datasetv2).toEqual(datasetStub.datasetv2); + }); + + it('returns nothing when an invalid dataset identifier is passed', async function () { + // Dependencies + const datasetRepository = new DatasetRepository(); + const datasetService = new DatasetService(datasetRepository); + + // Stubs + const getDatasetStub = sinon.stub(datasetRepository, 'getDataset').returns(null); + + // Act + const dataset = await datasetService.getDataset(12345); + + // Assert + expect(getDatasetStub.calledOnce).toBe(true); + expect(dataset).toBeUndefined(); + }); + + it('returns nothing and does not attempt a database call when no dataset identifier is passed', async function () { + // Dependencies + const datasetRepository = new DatasetRepository(); + const datasetService = new DatasetService(datasetRepository); + + // Stubs + const getDatasetStub = sinon.stub(datasetRepository, 'getDataset').returns(null); + + // Act + const dataset = await datasetService.getDataset(null); + + // Assert + expect(getDatasetStub.notCalled).toBe(true); + expect(dataset).toBeUndefined(); }); }); describe('getDatasets', function () { - it('should return an array of datasets', async function () { + it('returns an array of datasets', async function () { + // Dependencies const datasetRepository = new DatasetRepository(); - const stub = sinon.stub(datasetRepository, 'getDatasets').returns(datasetsStub); const datasetService = new DatasetService(datasetRepository); + + // Stubs + const stub = sinon.stub(datasetRepository, 'getDatasets').returns(datasetsStub); + + // Act const datasets = await datasetService.getDatasets(); + // Assert expect(stub.calledOnce).toBe(true); - expect(datasets.length).toBeGreaterThan(0); }); }); + describe('reformatTechnicalDetails', function () { + it('returns a dataset object with structural metadata formatted for version two schema', async function () { + // Fixtures + const datasetStub = new DatasetClass(v2DatasetsStub[0]); + + // Dependencies + const datasetRepository = new DatasetRepository(); + const datasetService = new DatasetService(datasetRepository); + + // Act + const formattedDataset = datasetService.reformatTechnicalDetails(datasetStub); + + // Assert + expect(formattedDataset.structuralMetadata).toEqual({ + structuralMetadataCount: {}, + dataClasses: [ + { + id: '857a5ee0-196c-4247-9b95-5b1aed70fea1', + description: "This class was created from the White Rabbit profile data in 'Demography_Current.csv'", + name: 'Demography_Current.csv', + dataElementsCount: 2, + dataElements: [ + { + id: 'd90bd4c2-df71-438c-879a-fe159e651517', + description: null, + name: 'current_gp_accept_date', + type: 'PrimitiveType', + }, + { + id: '907a641c-86c4-4a95-adc4-3c2bbea35f59', + description: null, + name: 'anon_date_of_birth', + type: 'PrimitiveType', + }, + ], + }, + ], + }); + }); + it('returns a dataset object unmodified if no structural metadata data classes are present', async function () { + // Fixtures + const datasetStub = new DatasetClass(v2DatasetsStub[1]); + + // Dependencies + const datasetRepository = new DatasetRepository(); + const datasetService = new DatasetService(datasetRepository); + + // Act + const formattedDataset = datasetService.reformatTechnicalDetails(datasetStub); + + // Assert + expect(formattedDataset).toEqual(datasetStub); + }); + it('returns a dataset object unmodified if no technical details are present', async function () { + // Fixtures + const datasetStub = new DatasetClass(datasetsStub[0]); + + // Dependencies + const datasetRepository = new DatasetRepository(); + const datasetService = new DatasetService(datasetRepository); + + // Act + const formattedDataset = datasetService.reformatTechnicalDetails(datasetStub); + + // Assert + expect(formattedDataset).toEqual(datasetStub); + }); + }); + describe('getRelatedObjects', function () { + it('returns an empty array if no persistent identifier is provided', async function () { + // Fixtures + const datasetStub = new DatasetClass(datasetsStub[0]); + + // Dependencies + const datasetRepository = new DatasetRepository(); + const datasetService = new DatasetService(datasetRepository); + + // Act + const relatedObjects = await datasetService.getRelatedObjects(null); + + // Assert + expect(relatedObjects).toEqual({}); + }); + it('returns an empty array if no related objects are found', async function () { + // Fixtures + const datasetStub = new DatasetClass(datasetsStub[0]); + + // Dependencies + const datasetRepository = new DatasetRepository(); + const paperRepository = new PaperRepository(); + const projectRepository = new ProjectRepository(); + const toolRepository = new ToolRepository(); + const courseRepository = new CourseRepository(); + const datasetService = new DatasetService(datasetRepository, paperRepository, projectRepository, toolRepository, courseRepository); + + // Stubs + const getRelatedPapersStub = sinon.stub(paperRepository, 'find').returns([]); + const getRelatedProjectsStub = sinon.stub(projectRepository, 'find').returns([]); + const getRelatedToolsStub = sinon.stub(toolRepository, 'find').returns([]); + const getRelatedCoursesStub = sinon.stub(courseRepository, 'find').returns([]); + + // Act + const relatedObjects = await datasetService.getRelatedObjects(datasetStub.pid); + + // Assert + expect(getRelatedPapersStub.calledOnce).toBe(true); + expect(getRelatedProjectsStub.calledOnce).toBe(true); + expect(getRelatedToolsStub.calledOnce).toBe(true); + expect(getRelatedCoursesStub.calledOnce).toBe(true); + expect(relatedObjects).toEqual([]); + }); + it('returns an array of related objects by searching other entity repositories for relationships', async function () { + // Fixtures + const datasetStub = new DatasetClass(datasetsStub[0]); + + // Dependencies + const datasetRepository = new DatasetRepository(); + const paperRepository = new PaperRepository(); + const projectRepository = new ProjectRepository(); + const toolRepository = new ToolRepository(); + const courseRepository = new CourseRepository(); + const datasetService = new DatasetService(datasetRepository, paperRepository, projectRepository, toolRepository, courseRepository); + + // Stubs + const getRelatedPapersStub = sinon.stub(paperRepository, 'find').returns([ + { + id: 1, + type: 'paper', + relatedObjects: [ + { + pid: '4ef841d3-5e86-4f92-883f-1015ffd4b979', + reason: 'definition of paper relationship', + user: 'John Smith', + updated: '01/02/2020', + }, + ] + } + ]); + + const getRelatedProjectsStub = sinon.stub(projectRepository, 'find').returns([ + { + id: 2, + type: 'project', + relatedObjects: [ + { + pid: '4ef841d3-5e86-4f92-883f-1015ffd4b979', + reason: 'definition of project relationship', + user: 'Steve Smith', + updated: '02/03/2020', + }, + ], + }, + ]); + + const getRelatedToolsStub = sinon.stub(toolRepository, 'find').returns([ + { + id: 3, + type: 'tool', + relatedObjects: [ + { + pid: '4ef841d3-5e86-4f92-883f-1015ffd4b979', + reason: 'definition of tool relationship', + user: 'Linda Smith', + updated: '03/04/2020', + }, + ], + }, + ]); + + const getRelatedCoursesStub = sinon.stub(courseRepository, 'find').returns([ + { + id: 4, + type: 'course', + relatedObjects: [ + { + pid: '4ef841d3-5e86-4f92-883f-1015ffd4b979', + reason: 'definition of course relationship', + user: 'Caroline Smith', + updated: '04/05/2020', + }, + ], + }, + ]); + + // Act + const relatedObjects = await datasetService.getRelatedObjects(datasetStub.pid); + + // Assert + expect(getRelatedPapersStub.calledOnce).toBe(true); + expect(getRelatedProjectsStub.calledOnce).toBe(true); + expect(getRelatedToolsStub.calledOnce).toBe(true); + expect(getRelatedCoursesStub.calledOnce).toBe(true); + expect(relatedObjects).toContainEqual({ + objectId: 1, + reason: 'definition of paper relationship', + objectType: 'paper', + user: 'John Smith', + updated: '01/02/2020', + }); + expect(relatedObjects).toContainEqual({ + objectId: 2, + reason: 'definition of project relationship', + objectType: 'project', + user: 'Steve Smith', + updated: '02/03/2020', + }); + expect(relatedObjects).toContainEqual({ + objectId: 3, + reason: 'definition of tool relationship', + objectType: 'tool', + user: 'Linda Smith', + updated: '03/04/2020', + }); + expect(relatedObjects).toContainEqual({ + objectId: 4, + reason: 'definition of course relationship', + objectType: 'course', + user: 'Caroline Smith', + updated: '04/05/2020', + }); + }); + }); }); diff --git a/src/resources/dataset/dataset.entity.js b/src/resources/dataset/dataset.entity.js index 9f757c55..d6ac6ce0 100644 --- a/src/resources/dataset/dataset.entity.js +++ b/src/resources/dataset/dataset.entity.js @@ -1,49 +1,9 @@ import Entity from '../base/entity'; export default class DatasetClass extends Entity { - constructor( - id, - name, - description, - resultsInsights, - link, - type, - categories, - license, - authors, - tags, - activeflag, - counter, - discourseTopicId, - relatedObjects, - uploader, - datasetid, - pid, - datasetVersion, - datasetfields, - datasetv2 - ) { + constructor(obj) { super(); - this.id = id; - this.name = name; - this.description = description; - this.resultsInsights = resultsInsights; - this.link = link; - this.type = type; - this.categories = categories; - this.license = license; - this.authors = authors; - this.tags = tags; - this.activeflag = activeflag; - this.counter = counter; - this.discourseTopicId = discourseTopicId; - this.relatedObjects = relatedObjects; - this.uploader = uploader; - this.datasetid = datasetid; - this.pid = pid; - this.datasetVersion = datasetVersion; - this.datasetfields = datasetfields; - this.datasetv2 = datasetv2; + Object.assign(this, obj); } checkLatestVersion() { @@ -93,7 +53,7 @@ export default class DatasetClass extends Entity { version: `2.0.0`, url: `https://raw.githubusercontent.com/HDRUK/schemata/master/schema/dataset/latest/dataset.schema.json`, }, - ...transformedObject + ...transformedObject, }; // Return v2 object diff --git a/src/resources/dataset/dataset.repository.js b/src/resources/dataset/dataset.repository.js index 73c3ea96..0544292d 100644 --- a/src/resources/dataset/dataset.repository.js +++ b/src/resources/dataset/dataset.repository.js @@ -22,7 +22,9 @@ export default class DatasetRepository extends Repository { return {}; } // Get dataset versions using pid - const datasets = await Dataset.find({ pid }).select({ datasetid: 1, datasetVersion: 1, activeflag: 1 }).lean(); + const query = { pid, fields:'datasetid,datasetVersion,activeflag' }; + const options = { lean: true }; + const datasets = await this.find(query, options); // Create revision structure return datasets.reduce((obj, dataset) => { const { datasetVersion = 'default', datasetid = 'empty', activeflag = '' } = dataset; diff --git a/src/resources/dataset/dataset.service.js b/src/resources/dataset/dataset.service.js index 7a800820..1a79d7de 100644 --- a/src/resources/dataset/dataset.service.js +++ b/src/resources/dataset/dataset.service.js @@ -10,40 +10,27 @@ export default class DatasetService { } async getDataset(id, query = {}) { + // Protect for no id passed + if(!id) return; + // Get dataset from Db by datasetid first query = { ...query, datasetid: id }; let dataset = await this.datasetRepository.getDataset(query); - // Update query to find the latest dataset - if (!_.isNil(dataset)) { - id = dataset.pid; - } - - // Get latest data set - delete query.datasetid; - dataset = await this.datasetRepository.getDataset({ ...query, pid: id, activeflag: 'active' }); - - // If no dataset is found active, look for most recent archived dataset - if (!dataset) { - query.sort = '-createdAt'; - dataset = await this.datasetRepository.getDataset({ ...query, pid: id, activeflag: 'archive' }); - } - // Return undefined if no dataset found if (!dataset) return; // Populate derived fields dataset.revisions = await this.datasetRepository.getDatasetRevisions(dataset.pid); dataset.relatedObjects = await this.getRelatedObjects(dataset.pid); + dataset.isLatestVersion = dataset.checkLatestVersion(); - // Raw output responds with the data structure as defined in MongoDb - if (query['raw'] === true) { - dataset.isLatestVersion = dataset.checkLatestVersion(); - } else { + // Return v2 format for datasets if 'raw' isn't passed + if (!query['raw']) { // Transform to dataset v2 data structure let v2Response = dataset.toV2Format(); // Temporary step of reformatting technical details until cache updated - v2Response.dataset = reformatTechnicalDetails(v2Response.dataset); + v2Response.dataset = this.reformatTechnicalDetails(v2Response.dataset); // Set full response dataset = v2Response; } @@ -83,7 +70,7 @@ export default class DatasetService { // Flatten and reduce related entities into related objects const relatedObjects = relatedEntities.flat().reduce((arr, entity) => { - let { relatedObjects: entityRelatedObjects = [] } = entity; + let { relatedObjects: entityRelatedObjects } = entity; entityRelatedObjects = entityRelatedObjects.filter(obj => obj.pid === pid); const formattedEntityRelatedObjects = entityRelatedObjects.map(obj => { return { @@ -99,28 +86,28 @@ export default class DatasetService { }, []); return relatedObjects; } -} -const reformatTechnicalDetails = dataset => { - // Return if no technical details found - if (_.isNil(dataset.structuralMetadata) || _.isNil(dataset.structuralMetadata.dataClasses)) { - return dataset; - } - // Convert mongoose array to regular array - const dataClasses = Array.from([...dataset.structuralMetadata.dataClasses]) || []; - // Map data classes array into correct format - dataset.structuralMetadata.dataClasses = [...dataClasses].map(el => { - const { id = '', description = '', label: name = '', elements = [] } = el; - const dataElements = [...elements].map(dataEl => { - const { - id = '', - description = '', - label: name = '', - dataType: { domainType: type = '' }, - } = dataEl; - return { id, description, name, type }; + reformatTechnicalDetails (dataset) { + // Return if no technical details found + if (_.isNil(dataset.structuralMetadata) || _.isNil(dataset.structuralMetadata.dataClasses)) { + return dataset; + } + // Convert mongoose array to regular array + const dataClasses = Array.from([...dataset.structuralMetadata.dataClasses]) || []; + // Map data classes array into correct format + dataset.structuralMetadata.dataClasses = [...dataClasses].map(el => { + const { id = '', description = '', label: name = '', elements = [] } = el; + const dataElements = [...elements].map(dataEl => { + const { + id = '', + description = '', + label: name = '', + dataType: { domainType: type = '' }, + } = dataEl; + return { id, description, name, type }; + }); + return { id, description, name, dataElementsCount: dataElements.length || 0, dataElements }; }); - return { id, description, name, dataElementsCount: dataElements.length || 0, dataElements }; - }); - return dataset; -}; + return dataset; + }; +} From 6096f251ada31330ac9074e97108c535017a2e58 Mon Sep 17 00:00:00 2001 From: massimocianfroccaPA Date: Wed, 24 Feb 2021 15:37:44 +0100 Subject: [PATCH 28/42] Added API to update and read status of a file --- .../datarequest/datarequest.controller.js | 72 +++++++++++++++++++ .../datarequest/datarequest.route.js | 10 +++ src/resources/utilities/cloudStorage.util.js | 1 + 3 files changed, 83 insertions(+) diff --git a/src/resources/datarequest/datarequest.controller.js b/src/resources/datarequest/datarequest.controller.js index eb66c63f..2b0dcea1 100644 --- a/src/resources/datarequest/datarequest.controller.js +++ b/src/resources/datarequest/datarequest.controller.js @@ -955,6 +955,34 @@ module.exports = { } }, + //GET api/v1/data-access-request/:id/file/:fileId/status + getFileStatus: async (req, res) => { + try { + // 1. get params + const { + params: { id, fileId }, + } = req; + + // 2. get AccessRecord + let accessRecord = await DataRequestModel.findOne({ _id: id }); + if (!accessRecord) { + return res.status(404).json({ status: 'error', message: 'Application not found.' }); + } + + // 3. get file + const fileIndex = accessRecord.files.findIndex(file => file.fileId === fileId); + if(fileIndex === -1) + return res.status(404).json({ status: 'error', message: 'File not found.' }); + + // 4. Return successful response + return res.status(200).json({ status: accessRecord.files[fileIndex].status }); + + } catch (err) { + console.log(err.message); + res.status(500).json({ status: 'error', message: err }); + } + }, + //GET api/v1/data-access-request/:id/file/:fileId getFile: async (req, res) => { try { @@ -1620,6 +1648,50 @@ module.exports = { } }, + updateFileStatus: async (req, res) => { + try { + // 1. Get the required request params + const { + params: { id, fileId }, + } = req; + + let { status } = req.body; + + // 2. Find the relevant data request application + let accessRecord = await DataRequestModel.findOne({_id: id}); + + if (!accessRecord) { + return res.status(404).json({ status: 'error', message: 'Application not found.' }); + } + + //3. Check the status is valid + if(status!==fileStatus.UPLOADED && status!==fileStatus.SCANNED && status!==fileStatus.ERROR && status!==fileStatus.QUARANTINED ){ + return res.status(400).json({ status: 'error', message: 'File status not valid' }); + } + + //4. get the file + const fileIndex = accessRecord.files.findIndex(file => file.fileId === fileId); + if(fileIndex === -1) + return res.status(404).json({ status: 'error', message: 'File not found.' }); + + //5. update the status + accessRecord.files[fileIndex].status=status; + + //6. write back into mongo + await accessRecord.save(); + + return res.status(200).json({ + success: true, + }); + } catch (err) { + console.error(err.message); + return res.status(500).json({ + success: false, + message: err.message, + }); + } + }, + createNotifications: async (type, context, accessRecord, user) => { // Project details from about application if 5 Safes let { aboutApplication = {} } = accessRecord; diff --git a/src/resources/datarequest/datarequest.route.js b/src/resources/datarequest/datarequest.route.js index 8b9bd659..d6388859 100644 --- a/src/resources/datarequest/datarequest.route.js +++ b/src/resources/datarequest/datarequest.route.js @@ -44,6 +44,11 @@ router.get('/datasets/:datasetIds', passport.authenticate('jwt'), datarequestCon // @access Private router.get('/:id/file/:fileId', param('id').customSanitizer(value => {return value}), passport.authenticate('jwt'), datarequestController.getFile); +// @route GET api/v1/data-access-request/:id/file/:fileId/status +// @desc GET Status of a file +// @access Private +router.get('/:id/file/:fileId/status', passport.authenticate('jwt'), datarequestController.getFileStatus); + // @route PATCH api/v1/data-access-request/:id // @desc Update application passing single object to update database entry with specified key // @access Private - Applicant (Gateway User) @@ -109,4 +114,9 @@ router.post('/:id', passport.authenticate('jwt'), datarequestController.submitAc // @access Private router.post('/:id/notify', passport.authenticate('jwt'), datarequestController.notifyAccessRequestById); +// @route POST api/v1/data-access-request/:id/updatefilestatus +// @desc Update the status of a file. +// @access Private +router.post('/:id/file/:fileId/status', passport.authenticate('jwt'), datarequestController.updateFileStatus); + module.exports = router; \ No newline at end of file diff --git a/src/resources/utilities/cloudStorage.util.js b/src/resources/utilities/cloudStorage.util.js index fd5fe27d..84d89e9e 100644 --- a/src/resources/utilities/cloudStorage.util.js +++ b/src/resources/utilities/cloudStorage.util.js @@ -7,6 +7,7 @@ export const fileStatus = { UPLOADED: 'UPLOADED', ERROR: 'ERROR', SCANNED: 'SCANNED', + QUARANTINED: 'QUARANTINED' }; export const processFile = (file, id, uniqueId) => From 838255424b5ba7657f7a32b53ae81a7943989e8f Mon Sep 17 00:00:00 2001 From: massimocianfroccaPA Date: Thu, 25 Feb 2021 10:31:58 +0100 Subject: [PATCH 29/42] fix code indentation --- .../datarequest/datarequest.controller.js | 132 +++++++++--------- 1 file changed, 68 insertions(+), 64 deletions(-) diff --git a/src/resources/datarequest/datarequest.controller.js b/src/resources/datarequest/datarequest.controller.js index 2b0dcea1..8ecfb058 100644 --- a/src/resources/datarequest/datarequest.controller.js +++ b/src/resources/datarequest/datarequest.controller.js @@ -31,7 +31,9 @@ module.exports = { let { id: userId } = req.user; // 2. Find all data access request applications created with multi dataset version - let applications = await DataRequestModel.find({ $or: [{ userId: parseInt(userId) }, { authorIds: userId }] }).populate('datasets mainApplicant'); + let applications = await DataRequestModel.find({ $or: [{ userId: parseInt(userId) }, { authorIds: userId }] }).populate( + 'datasets mainApplicant' + ); // 3. Append project name and applicants let modifiedApplications = [...applications] @@ -413,11 +415,14 @@ module.exports = { if (typeof aboutApplication === 'string') { aboutApplication = JSON.parse(aboutApplication); } - const { datasetIds, datasetTitles } = aboutApplication.selectedDatasets.reduce((newObj, dataset) => { - newObj.datasetIds = [...newObj.datasetIds, dataset.datasetId]; - newObj.datasetTitles = [...newObj.datasetTitles, dataset.name]; - return newObj; - }, { datasetIds: [], datasetTitles: []}); + const { datasetIds, datasetTitles } = aboutApplication.selectedDatasets.reduce( + (newObj, dataset) => { + newObj.datasetIds = [...newObj.datasetIds, dataset.datasetId]; + newObj.datasetTitles = [...newObj.datasetTitles, dataset.name]; + return newObj; + }, + { datasetIds: [], datasetTitles: [] } + ); updateObj = { aboutApplication, datasetIds, datasetTitles }; } @@ -958,25 +963,23 @@ module.exports = { //GET api/v1/data-access-request/:id/file/:fileId/status getFileStatus: async (req, res) => { try { - // 1. get params - const { - params: { id, fileId }, - } = req; - - // 2. get AccessRecord - let accessRecord = await DataRequestModel.findOne({ _id: id }); - if (!accessRecord) { - return res.status(404).json({ status: 'error', message: 'Application not found.' }); - } + // 1. get params + const { + params: { id, fileId }, + } = req; - // 3. get file - const fileIndex = accessRecord.files.findIndex(file => file.fileId === fileId); - if(fileIndex === -1) - return res.status(404).json({ status: 'error', message: 'File not found.' }); - - // 4. Return successful response - return res.status(200).json({ status: accessRecord.files[fileIndex].status }); + // 2. get AccessRecord + let accessRecord = await DataRequestModel.findOne({ _id: id }); + if (!accessRecord) { + return res.status(404).json({ status: 'error', message: 'Application not found.' }); + } + // 3. get file + const fileIndex = accessRecord.files.findIndex(file => file.fileId === fileId); + if (fileIndex === -1) return res.status(404).json({ status: 'error', message: 'File not found.' }); + + // 4. Return successful response + return res.status(200).json({ status: accessRecord.files[fileIndex].status }); } catch (err) { console.log(err.message); res.status(500).json({ status: 'error', message: err }); @@ -1295,16 +1298,16 @@ module.exports = { //PUT api/v1/data-access-request/:id/deletefile updateAccessRequestDeleteFile: async (req, res) => { - try{ + try { const { params: { id }, } = req; // 1. Id of the file to delete let { fileId } = req.body; - + // 2. Find the relevant data request application - let accessRecord = await DataRequestModel.findOne({_id: id}); + let accessRecord = await DataRequestModel.findOne({ _id: id }); if (!accessRecord) { return res.status(404).json({ status: 'error', message: 'Application not found.' }); @@ -1312,7 +1315,7 @@ module.exports = { // 4. Ensure single datasets are mapped correctly into array if (_.isEmpty(accessRecord.datasets)) { - accessRecord.datasets = [accessRecord.dataset]; + accessRecord.datasets = [accessRecord.dataset]; } // 5. If application is not in progress, actions cannot be performed @@ -1337,18 +1340,15 @@ module.exports = { // 9. write back into mongo await accessRecord.save(); - + // 10. Return successful response return res.status(200).json({ status: 'success' }); - } catch (err) { console.log(err.message); res.status(500).json({ status: 'error', message: err }); } - }, - //POST api/v1/data-access-request/:id submitAccessRequestById: async (req, res) => { try { @@ -1649,48 +1649,52 @@ module.exports = { }, updateFileStatus: async (req, res) => { - try { - // 1. Get the required request params - const { - params: { id, fileId }, - } = req; + try { + // 1. Get the required request params + const { + params: { id, fileId }, + } = req; - let { status } = req.body; + let { status } = req.body; - // 2. Find the relevant data request application - let accessRecord = await DataRequestModel.findOne({_id: id}); + // 2. Find the relevant data request application + let accessRecord = await DataRequestModel.findOne({ _id: id }); - if (!accessRecord) { - return res.status(404).json({ status: 'error', message: 'Application not found.' }); - } + if (!accessRecord) { + return res.status(404).json({ status: 'error', message: 'Application not found.' }); + } //3. Check the status is valid - if(status!==fileStatus.UPLOADED && status!==fileStatus.SCANNED && status!==fileStatus.ERROR && status!==fileStatus.QUARANTINED ){ - return res.status(400).json({ status: 'error', message: 'File status not valid' }); + if ( + status !== fileStatus.UPLOADED && + status !== fileStatus.SCANNED && + status !== fileStatus.ERROR && + status !== fileStatus.QUARANTINED + ) { + return res.status(400).json({ status: 'error', message: 'File status not valid' }); } //4. get the file const fileIndex = accessRecord.files.findIndex(file => file.fileId === fileId); - if(fileIndex === -1) - return res.status(404).json({ status: 'error', message: 'File not found.' }); - - //5. update the status - accessRecord.files[fileIndex].status=status; - - //6. write back into mongo - await accessRecord.save(); - - return res.status(200).json({ - success: true, - }); - } catch (err) { - console.error(err.message); - return res.status(500).json({ - success: false, - message: err.message, - }); - } - }, + if (fileIndex === -1) return res.status(404).json({ status: 'error', message: 'File not found.' }); + + //5. update the status + accessRecord.files[fileIndex].status = status; + + //6. write back into mongo + await accessRecord.save(); + + return res.status(200).json({ + success: true, + }); + } catch (err) { + console.error(err.message); + return res.status(500).json({ + success: false, + message: err.message, + }); + } + }, createNotifications: async (type, context, accessRecord, user) => { // Project details from about application if 5 Safes From 9b575bf76042005d1ab1cf8f772d1488f7f682cb Mon Sep 17 00:00:00 2001 From: Richard Date: Thu, 25 Feb 2021 14:23:26 +0000 Subject: [PATCH 30/42] Added second sort column to sort to fix entities dashboard bug --- src/resources/course/course.repository.js | 92 ++++++++++------------- src/resources/tool/data.repository.js | 2 +- 2 files changed, 42 insertions(+), 52 deletions(-) diff --git a/src/resources/course/course.repository.js b/src/resources/course/course.repository.js index cb1e016f..583c6a11 100644 --- a/src/resources/course/course.repository.js +++ b/src/resources/course/course.repository.js @@ -164,11 +164,11 @@ const getAllCourses = async (req, res) => { let searchString = ''; if (req.query.offset) { - startIndex = req.query.offset; + startIndex = req.query.offset; } if (req.query.limit) { limit = req.query.limit; - } + } if (req.query.q) { searchString = req.query.q || ''; } @@ -205,11 +205,11 @@ const getCourseAdmin = async (req, res) => { searchString = req.query.q || ''; } if (req.query.status) { - status = req.query.status + status = req.query.status; } let searchQuery; - if(status === 'all'){ + if (status === 'all') { searchQuery = { $and: [{ type: 'course' }] }; } else { searchQuery = { $and: [{ type: 'course' }, { activeflag: status }] }; @@ -235,36 +235,36 @@ const getCourse = async (req, res) => { let idString = req.user.id; let status = 'all'; - if (req.query.offset) { - startIndex = req.query.offset; - } - if (req.query.limit) { - limit = req.query.limit; - } - if (req.query.id) { + if (req.query.offset) { + startIndex = req.query.offset; + } + if (req.query.limit) { + limit = req.query.limit; + } + if (req.query.id) { idString = req.query.id; - } - - let searchQuery; - if(status === 'all'){ - searchQuery = [{ type: 'course' }, { creator: parseInt(idString) }] - } else { - searchQuery = [{ type: 'course' }, { creator: parseInt(idString) }, { activeflag: status }] - } - + } + + let searchQuery; + if (status === 'all') { + searchQuery = [{ type: 'course' }, { creator: parseInt(idString) }]; + } else { + searchQuery = [{ type: 'course' }, { creator: parseInt(idString) }, { activeflag: status }]; + } + let query = Course.aggregate([ - { $match: { $and: searchQuery} }, + { $match: { $and: searchQuery } }, { $lookup: { from: 'tools', localField: 'creator', foreignField: 'id', as: 'persons' } }, { $sort: { updatedAt: -1 } }, ]) - .skip(parseInt(startIndex)) - .limit(parseInt(limit)); + .skip(parseInt(startIndex)) + .limit(parseInt(limit)); await Promise.all([getUserCourses(query), getCountsByStatusCreator(idString)]).then(values => { resolve(values); }); - function getUserCourses(query) { + function getUserCourses(query) { return new Promise((resolve, reject) => { query.exec((err, data) => { if (err) reject({ success: false, error: err }); @@ -505,7 +505,7 @@ function getObjectResult(type, searchAll, searchQuery, startIndex, limit) { { $lookup: { from: 'tools', localField: 'id', foreignField: 'authors', as: 'objects' } }, { $lookup: { from: 'reviews', localField: 'id', foreignField: 'toolID', as: 'reviews' } }, ]) - .sort({ updatedAt: -1 }) + .sort({ updatedAt: -1, _id: 1 }) .skip(parseInt(startIndex)) .limit(parseInt(limit)); } else { @@ -533,47 +533,37 @@ function getObjectResult(type, searchAll, searchQuery, startIndex, limit) { }); } -function getCountsByStatus() { +function getCountsByStatus() { + let q = Course.find({}, { id: 1, title: 1, activeflag: 1 }); - let q = Course.find({ }, { id: 1, title: 1, activeflag: 1 }); - return new Promise((resolve, reject) => { q.exec((err, data) => { - const activeCount = data.filter(dat => dat.activeflag === 'active').length - const reviewCount = data.filter(dat => dat.activeflag === 'review').length - const rejectedCount = data.filter(dat => dat.activeflag === 'rejected').length - const archiveCount = data.filter(dat => dat.activeflag === 'archive').length + const activeCount = data.filter(dat => dat.activeflag === 'active').length; + const reviewCount = data.filter(dat => dat.activeflag === 'review').length; + const rejectedCount = data.filter(dat => dat.activeflag === 'rejected').length; + const archiveCount = data.filter(dat => dat.activeflag === 'archive').length; - let countSummary = {'activeCount': activeCount, - 'reviewCount': reviewCount, - 'rejectedCount': rejectedCount, - 'archiveCount': archiveCount - } + let countSummary = { activeCount: activeCount, reviewCount: reviewCount, rejectedCount: rejectedCount, archiveCount: archiveCount }; resolve(countSummary); - }) + }); }); } -function getCountsByStatusCreator(idString) { - +function getCountsByStatusCreator(idString) { let q = Course.find({ $and: [{ type: 'course' }, { creator: parseInt(idString) }] }, { id: 1, title: 1, activeflag: 1 }); - + return new Promise((resolve, reject) => { q.exec((err, data) => { - const activeCount = data.filter(dat => dat.activeflag === 'active').length - const reviewCount = data.filter(dat => dat.activeflag === 'review').length - const rejectedCount = data.filter(dat => dat.activeflag === 'rejected').length - const archiveCount = data.filter(dat => dat.activeflag === 'archive').length + const activeCount = data.filter(dat => dat.activeflag === 'active').length; + const reviewCount = data.filter(dat => dat.activeflag === 'review').length; + const rejectedCount = data.filter(dat => dat.activeflag === 'rejected').length; + const archiveCount = data.filter(dat => dat.activeflag === 'archive').length; - let countSummary = {'activeCount': activeCount, - 'reviewCount': reviewCount, - 'rejectedCount': rejectedCount, - 'archiveCount': archiveCount - } + let countSummary = { activeCount: activeCount, reviewCount: reviewCount, rejectedCount: rejectedCount, archiveCount: archiveCount }; resolve(countSummary); - }) + }); }); } diff --git a/src/resources/tool/data.repository.js b/src/resources/tool/data.repository.js index 3144edcd..86ea1fef 100644 --- a/src/resources/tool/data.repository.js +++ b/src/resources/tool/data.repository.js @@ -530,7 +530,7 @@ function getObjectResult(type, searchAll, searchQuery, startIndex, limit) { { $lookup: { from: 'tools', localField: 'id', foreignField: 'authors', as: 'objects' } }, { $lookup: { from: 'reviews', localField: 'id', foreignField: 'toolID', as: 'reviews' } }, ]) - .sort({ updatedAt: -1 }) + .sort({ updatedAt: -1, _id: 1 }) .skip(parseInt(startIndex)) .limit(parseInt(limit)); } else { From 9da3978487f2a335be1548f92b15ab9e84309d02 Mon Sep 17 00:00:00 2001 From: Robin Kavanagh Date: Thu, 25 Feb 2021 15:18:57 +0000 Subject: [PATCH 31/42] Updated swagger docs for v2 entities --- swagger.yaml | 498 +++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 447 insertions(+), 51 deletions(-) diff --git a/swagger.yaml b/swagger.yaml index 7eda6efb..f572bea9 100644 --- a/swagger.yaml +++ b/swagger.yaml @@ -4,9 +4,9 @@ info: description: API for Tools and artefacts repository. version: 1.0.0 servers: - - url: https://api.www.healthdatagateway.org/api - - url: http://localhost:3001/api - - url: https://api.{environment}.healthdatagateway.org:{port}/api + - url: https://api.www.healthdatagateway.org/ + - url: http://localhost:3001/ + - url: https://api.{environment}.healthdatagateway.org:{port}/ variables: environment: default: latest @@ -710,6 +710,7 @@ paths: responses: '200': description: OK + /api/v1/publishers/{publisher}/dataaccessrequests: get: tags: @@ -1309,53 +1310,6 @@ paths: examples: 'Unauthorised': value: { 'status': 'failure', 'message': 'Unauthorised' } - /api/v1/datasets/{datasetID}: - get: - summary: Returns Dataset object. - tags: - - Datasets - parameters: - - in: path - name: datasetID - required: true - description: The ID of the datset - schema: - type: string - example: '756daeaa-6e47-4269-9df5-477c01cdd271' - responses: - '200': - description: OK - - /api/v1/datasets: - get: - summary: Returns List of Dataset objects. - tags: - - Datasets - parameters: - - in: query - name: limit - required: false - description: Limit the number of results - schema: - type: integer - example: 3 - - in: query - name: offset - required: false - description: Index to offset the search results - schema: - type: integer - example: 1 - - in: query - name: q - required: false - description: Filter using search query - schema: - type: string - example: epilepsy - responses: - '200': - description: OK /api/v1/data-access-request/{datasetID}: get: @@ -1816,6 +1770,179 @@ paths: '204': description: Ok + /api/v1/datasets/{datasetID}: + get: + summary: Returns Dataset object. + tags: + - Datasets + parameters: + - in: path + name: datasetID + required: true + description: The ID of the datset + schema: + type: string + example: '756daeaa-6e47-4269-9df5-477c01cdd271' + responses: + '200': + description: OK + + /api/v1/datasets: + get: + summary: Returns List of Dataset objects. + tags: + - Datasets + parameters: + - in: query + name: limit + required: false + description: Limit the number of results + schema: + type: integer + example: 3 + - in: query + name: offset + required: false + description: Index to offset the search results + schema: + type: integer + example: 1 + - in: query + name: q + required: false + description: Filter using search query + schema: + type: string + example: epilepsy + responses: + '200': + description: OK + + /api/v2/datasets: + get: + summary: Returns a list of dataset objects + tags: + - Datasets v2.0 + description: Version 2.0 of the datasets API introduces a large number of parameterised query string options to aid requests in collecting the data that is most relevant for a given use case. The query parameters defined below support a variety of comparison operators such as equals, contains, greater than, and less than. Using dot notation, any field can be queried, please see some examples below. + parameters: + - name: search + in: query + description: Full text index search function which searches for partial matches in various dataset fields including name, description and abstract. The response will contain a metascore indicating the relevancy of the match, by default results are sorted by the most relevant first unless a manual sort query parameter has been added. + schema: + type: string + example: COVID-19 + - name: page + in: query + description: A specific page of results to retrieve + schema: + type: number + example: 1 + - name: limit + in: query + description: Maximum number of results returned per page + schema: + type: number + example: 10 + - name: sort + in: query + description: Fields to apply sort operations to. Accepts multiple fields in ascending and descending. E.g. name for ascending or -name for descending. Multiple fields should be comma separated as shown in the example below. + schema: + type: string + example: datasetfields.publisher,name,-counter + - name: fields + in: query + description: Limit the size of the response by requesting only certain fields. Note that some additional derived fields are always returned. Multiple fields should be comma separate as shown in the example below. + schema: + type: string + example: name,counter,datasetid + - name: count + in: query + description: Returns the number of the number of entities matching the query parameters provided instead of the result payload + schema: + type: boolean + example: true + - name: datasetid + in: query + description: Filter by the unique identifier for a single version of a dataset + schema: + type: string + example: 0cfe60cd-038d-4c03-9a95-894c52135922 + - name: pid + in: query + description: Filter by the identifier for a dataset that persists across versions + schema: + type: string + example: 621dd611-adcf-4434-b538-eecdbe5f72cf + - name: name + in: query + description: Filter by dataset name + schema: + type: string + example: ARIA Dataset + - name: activeflag + in: query + description: Filter by the status of a single dataset version + schema: + type: string + enum: + - active + - archive + example: active + - name: datasetfields.publisher + in: query + description: Filter by the name of the Custodian holding the dataset + schema: + type: string + example: ALLIANCE > BARTS HEALTH NHS TRUST + - name: metadataquality.completeness_percent[gte] + in: query + description: Filter by the metadata quality completeness percentage using an operator [gte] for greater than or equal to, [gt] for greater than, [lte] for less than or equal to, [lt] for less than, and [eq] for equal to. + schema: + type: number + example: 90.5 + - name: metadataquality.weighted_completeness_percent[gte] + in: query + description: Filter by the metadata quality weighted completeness percentage using an operator [gte] for greater than or equal to, [gt] for greater than, [lte] for less than or equal to, [lt] for less than, and [eq] for equal to. + schema: + type: number + example: 71.2 + - name: metadataquality.weighted_quality_score[gte] + in: query + description: Filter by the metadata quality score using an operator [gte] for greater than or equal to, [gt] for greater than, [lte] for less than or equal to, [lt] for less than, and [eq] for equal to. + schema: + type: number + example: 35.3 + responses: + '200': + description: Successful response containing a list of datasets matching query parameters + + /api/v2/datasets/{datasetid}: + get: + summary: Returns a dataset object. + tags: + - Datasets v2.0 + parameters: + - in: path + name: datasetid + required: true + description: The unqiue identifier for a specific version of a dataset + schema: + type: string + example: af20ebb2-018a-4557-8ced-0bec75dba150 + - in: query + name: raw + required: false + description: A flag which determines if the response triggered is the raw structure in which the data is stored rather than the dataset v2.0 standard + schema: + type: boolean + example: false + description: Version 2.0 of the datasets API introduces the agreed dataset v2.0 schema as defined at the following link - https://github.com/HDRUK/schemata/edit/master/schema/dataset/2.0.0/dataset.schema.json + responses: + '200': + description: Successful response containing a single dataset object + '404': + description: A dataset could not be found by the provided dataset identifier + /api/v1/projects: post: summary: Returns a Project object with ID. @@ -2032,6 +2159,73 @@ paths: '200': description: OK + /api/v2/projects: + get: + summary: Returns a list of project objects + tags: + - Projects v2.0 + parameters: + - name: search + in: query + description: Full text index search function which searches for partial matches in various fields including name and description. The response will contain a metascore indicating the relevancy of the match, by default results are sorted by the most relevant first unless a manual sort query parameter has been added. + schema: + type: string + example: health service + - name: page + in: query + description: A specific page of results to retrieve + schema: + type: number + example: 1 + - name: limit + in: query + description: Maximum number of results returned per page + schema: + type: number + example: 10 + - name: sort + in: query + description: Fields to apply sort operations to. Accepts multiple fields in ascending and descending. E.g. name for ascending or -name for descending. Multiple fields should be comma separated as shown in the example below. + schema: + type: string + example: name,-counter + - name: fields + in: query + description: Limit the size of the response by requesting only certain fields. Note that some additional derived fields are always returned. Multiple fields should be comma separate as shown in the example below. + schema: + type: string + example: name,counter,description + - name: count + in: query + description: Returns the number of the number of entities matching the query parameters provided instead of the result payload + schema: + type: boolean + example: true + description: Version 2.0 of the courses API introduces a large number of parameterised query string options to aid requests in collecting the data that is most relevant for a given use case. The query parameters defined below support a variety of comparison operators such as equals, contains, greater than, and less than. Using dot notation, any field can be queried, please see some examples below. + responses: + '200': + description: Successful response containing a list of projects matching query parameters + + /api/v2/projects/{id}: + get: + summary: Returns a project object + tags: + - Projects v2.0 + parameters: + - in: path + name: id + required: true + description: The ID of the project + schema: + type: number + example: 100000001 + description: Returns a project object by matching unique identifier in the default format that is stored as within the Gateway + responses: + '200': + description: Successful response containing a single project object + '404': + description: A project could not be found by the provided project identifier + /api/v1/papers: post: summary: Returns a Paper object with ID. @@ -2253,6 +2447,73 @@ paths: '200': description: OK + /api/v2/papers: + get: + summary: Returns a list of paper objects + tags: + - Papers v2.0 + parameters: + - name: search + in: query + description: Full text index search function which searches for partial matches in various fields including name and description. The response will contain a metascore indicating the relevancy of the match, by default results are sorted by the most relevant first unless a manual sort query parameter has been added. + schema: + type: string + example: Exploration + - name: page + in: query + description: A specific page of results to retrieve + schema: + type: number + example: 1 + - name: limit + in: query + description: Maximum number of results returned per page + schema: + type: number + example: 10 + - name: sort + in: query + description: Fields to apply sort operations to. Accepts multiple fields in ascending and descending. E.g. name for ascending or -name for descending. Multiple fields should be comma separated as shown in the example below. + schema: + type: string + example: name,-counter + - name: fields + in: query + description: Limit the size of the response by requesting only certain fields. Note that some additional derived fields are always returned. Multiple fields should be comma separate as shown in the example below. + schema: + type: string + example: name,counter,description + - name: count + in: query + description: Returns the number of the number of entities matching the query parameters provided instead of the result payload + schema: + type: boolean + example: true + description: Version 2.0 of the courses API introduces a large number of parameterised query string options to aid requests in collecting the data that is most relevant for a given use case. The query parameters defined below support a variety of comparison operators such as equals, contains, greater than, and less than. Using dot notation, any field can be queried, please see some examples below. + responses: + '200': + description: Successful response containing a list of papers matching query parameters + + /api/v2/papers/{id}: + get: + summary: Returns paper object + tags: + - Papers v2.0 + parameters: + - in: path + name: id + required: true + description: The ID of the paper + schema: + type: number + example: 13296138992670704 + description: Returns a paper object by matching unique identifier in the default format that is stored as within the Gateway + responses: + '200': + description: Successful response containing a single paper object + '404': + description: A paper could not be found by the provided paper identifier + /api/v1/tools: get: summary: Return List of Tool objects. @@ -2341,7 +2602,7 @@ paths: /api/v1/tools/{id}: get: - summary: Returns Tool object. + summary: Returns Tool object tags: - Tools parameters: @@ -2468,6 +2729,141 @@ paths: '200': description: OK + /api/v2/tools: + get: + summary: Returns a list of tool objects + tags: + - Tools v2.0 + parameters: + - name: search + in: query + description: Full text index search function which searches for partial matches in various fields including name and description. The response will contain a metascore indicating the relevancy of the match, by default results are sorted by the most relevant first unless a manual sort query parameter has been added. + schema: + type: string + example: Regulation + - name: page + in: query + description: A specific page of results to retrieve + schema: + type: number + example: 1 + - name: limit + in: query + description: Maximum number of results returned per page + schema: + type: number + example: 10 + - name: sort + in: query + description: Fields to apply sort operations to. Accepts multiple fields in ascending and descending. E.g. name for ascending or -name for descending. Multiple fields should be comma separated as shown in the example below. + schema: + type: string + example: name,-counter + - name: fields + in: query + description: Limit the size of the response by requesting only certain fields. Note that some additional derived fields are always returned. Multiple fields should be comma separate as shown in the example below. + schema: + type: string + example: name,counter, description + - name: count + in: query + description: Returns the number of the number of entities matching the query parameters provided instead of the result payload + schema: + type: boolean + example: true + description: Version 2.0 of the courses API introduces a large number of parameterised query string options to aid requests in collecting the data that is most relevant for a given use case. The query parameters defined below support a variety of comparison operators such as equals, contains, greater than, and less than. Using dot notation, any field can be queried, please see some examples below. + responses: + '200': + description: Successful response containing a list of tools matching query parameters + + /api/v2/tools/{id}: + get: + summary: Returns a tool object + tags: + - Tools v2.0 + parameters: + - in: path + name: id + required: true + description: The ID of the tool + schema: + type: number + example: 100000006 + description: Returns a tool object by matching unique identifier in the default format that is stored as within the Gateway + responses: + '200': + description: Successful response containing a single tool object + '404': + description: A tool could not be found by the provided tool identifier + + /api/v2/courses: + get: + summary: Returns a list of courses + parameters: + - name: search + in: query + description: Full text index search function which searches for partial matches in various fields including name and description. The response will contain a metascore indicating the relevancy of the match, by default results are sorted by the most relevant first unless a manual sort query parameter has been added. + schema: + type: string + example: Research + - name: page + in: query + description: A specific page of results to retrieve + schema: + type: number + example: 1 + - name: limit + in: query + description: Maximum number of results returned per page + schema: + type: number + example: 10 + - name: sort + in: query + description: Fields to apply sort operations to. Accepts multiple fields in ascending and descending. E.g. provider for ascending or -provider for descending. Multiple fields should be comma separated as shown in the example below. + schema: + type: string + example: provider,-counter + - name: fields + in: query + description: Limit the size of the response by requesting only certain fields. Note that some additional derived fields are always returned. Multiple fields should be comma separate as shown in the example below. + schema: + type: string + example: provider,counter,description + - name: count + in: query + description: Returns the number of the number of entities matching the query parameters provided instead of the result payload + schema: + type: boolean + example: true + description: Version 2.0 of the courses API introduces a large number of parameterised query string options to aid requests in collecting the data that is most relevant for a given use case. The query parameters defined below support a variety of comparison operators such as equals, contains, greater than, and less than. Using dot notation, any field can be queried, please see some examples below. + tags: + - Courses v2.0 + responses: + '200': + description: Successful response containing a list of course objects matching query parameters + + /api/v2/courses/{id}: + summary: summary + get: + summary: Returns a course object + description: Returns a course object by matching unique identifier in the default format that is stored as within the Gateway + tags: + - Courses v2.0 + parameters: + - in: path + name: id + required: true + description: The ID of the course + schema: + type: number + example: 5540794872521069 + responses: + '200': + description: Successful response containing a single course object + '404': + description: A course could not be found by the provided course identifier + components: securitySchemes: oauth2: From 66538cae655548da2209f7352317a5cdcd7fa4d8 Mon Sep 17 00:00:00 2001 From: Robin Kavanagh Date: Fri, 26 Feb 2021 15:09:26 +0000 Subject: [PATCH 32/42] Updated Sentry and cache job entry point --- src/config/server.js | 6 ++-- src/resources/dataset/v1/dataset.route.js | 33 ++++++++++++--------- src/resources/dataset/v1/dataset.service.js | 4 +-- 3 files changed, 24 insertions(+), 19 deletions(-) diff --git a/src/config/server.js b/src/config/server.js index 1664736d..0ef7d5e5 100644 --- a/src/config/server.js +++ b/src/config/server.js @@ -19,9 +19,9 @@ require('dotenv').config(); if (helper.getEnvironment() !== 'local') { Sentry.init({ - dsn: 'https://c7c564a153884dc0a6b676943b172121@o444579.ingest.sentry.io/5419637', - environment: helper.getEnvironment(), - }); + dsn: "https://b6ea46f0fbe048c9974718d2c72e261b@o444579.ingest.sentry.io/5653683", + environment: helper.getEnvironment() + }); } const Account = require('./account'); diff --git a/src/resources/dataset/v1/dataset.route.js b/src/resources/dataset/v1/dataset.route.js index cc21b6a1..737e77da 100644 --- a/src/resources/dataset/v1/dataset.route.js +++ b/src/resources/dataset/v1/dataset.route.js @@ -15,20 +15,25 @@ const datasetLimiter = rateLimit({ }); router.post('/', async (req, res) => { - //Check to see if header is in json format - var parsedBody = {}; - if (req.header('content-type') === 'application/json') { - parsedBody = req.body; - } else { - parsedBody = JSON.parse(req.body); - } - //Check for key - if (parsedBody.key !== process.env.cachingkey) { - return res.json({ success: false, error: 'Caching failed' }); - } + try { + //Check to see if header is in json format + let parsedBody = {}; + if (req.header('content-type') === 'application/json') { + parsedBody = req.body; + } else { + parsedBody = JSON.parse(req.body); + } + //Check for key + if (parsedBody.key !== process.env.cachingkey) { + return res.status(400).json({ success: false, error: 'Caching could not be started' }); + } - loadDatasets(parsedBody.override || false); - return res.json({ success: true, message: 'Caching started' }); + loadDatasets(parsedBody.override || false); + return res.status(200).json({ success: true, message: 'Caching started' }); + } catch (err) { + console.error(err); + return res.status(500).json({ success: false, message: 'Caching failed' }); + } }); // @router GET /api/v1/datasets/pidList @@ -167,4 +172,4 @@ router.get('/', async (req, res) => { }); }); -module.exports = router; \ No newline at end of file +module.exports = router; diff --git a/src/resources/dataset/v1/dataset.service.js b/src/resources/dataset/v1/dataset.service.js index 84251040..0be798a1 100644 --- a/src/resources/dataset/v1/dataset.service.js +++ b/src/resources/dataset/v1/dataset.service.js @@ -198,9 +198,9 @@ export async function loadDataset(datasetID) { export async function loadDatasets(override) { console.log('Starting run at ' + Date()); - var metadataCatalogueLink = process.env.metadataURL || 'https://metadata-catalogue.org/hdruk'; + let metadataCatalogueLink = process.env.metadataURL || 'https://metadata-catalogue.org/hdruk'; - var datasetsMDCCount = await new Promise(function (resolve, reject) { + let datasetsMDCCount = await new Promise(function (resolve, reject) { axios .post( metadataCatalogueLink + From d84651260bbfbfe7cd76f8aa6e87c766b4098328 Mon Sep 17 00:00:00 2001 From: Robin Kavanagh Date: Fri, 26 Feb 2021 15:30:05 +0000 Subject: [PATCH 33/42] Added fail trigger --- src/resources/dataset/v1/dataset.route.js | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/resources/dataset/v1/dataset.route.js b/src/resources/dataset/v1/dataset.route.js index 737e77da..0f04cf63 100644 --- a/src/resources/dataset/v1/dataset.route.js +++ b/src/resources/dataset/v1/dataset.route.js @@ -5,6 +5,7 @@ import { getAllTools } from '../../tool/data.repository'; import _ from 'lodash'; import escape from 'escape-html'; import { Course } from '../../course/course.model'; +import * as Sentry from '@sentry/node'; const router = express.Router(); const rateLimit = require('express-rate-limit'); @@ -28,10 +29,15 @@ router.post('/', async (req, res) => { return res.status(400).json({ success: false, error: 'Caching could not be started' }); } + if (parsedBody.error === true) { + throw new Error('cache error test'); + } + loadDatasets(parsedBody.override || false); return res.status(200).json({ success: true, message: 'Caching started' }); } catch (err) { - console.error(err); + Sentry.captureException(err); + console.error(err.message); return res.status(500).json({ success: false, message: 'Caching failed' }); } }); From f4c0b71372a5a8a130ae30ecfebb2e6199c61e76 Mon Sep 17 00:00:00 2001 From: Robin Kavanagh Date: Fri, 26 Feb 2021 16:01:54 +0000 Subject: [PATCH 34/42] Updated Sentry init --- src/config/server.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/config/server.js b/src/config/server.js index 0ef7d5e5..338a6c85 100644 --- a/src/config/server.js +++ b/src/config/server.js @@ -17,10 +17,10 @@ import helper from '../resources/utilities/helper.util'; require('dotenv').config(); -if (helper.getEnvironment() !== 'local') { +if (process.env.api_url) { Sentry.init({ dsn: "https://b6ea46f0fbe048c9974718d2c72e261b@o444579.ingest.sentry.io/5653683", - environment: helper.getEnvironment() + environment: process.env.api_url }); } From 9a94362f5044665c0523e4ce65c24e1374c0cffc Mon Sep 17 00:00:00 2001 From: Robin Kavanagh Date: Fri, 26 Feb 2021 16:27:27 +0000 Subject: [PATCH 35/42] Extended Sentry config --- src/config/server.js | 4 +-- src/resources/utilities/helper.util.js | 41 ++++++++++++++++++-------- 2 files changed, 30 insertions(+), 15 deletions(-) diff --git a/src/config/server.js b/src/config/server.js index 338a6c85..0ef7d5e5 100644 --- a/src/config/server.js +++ b/src/config/server.js @@ -17,10 +17,10 @@ import helper from '../resources/utilities/helper.util'; require('dotenv').config(); -if (process.env.api_url) { +if (helper.getEnvironment() !== 'local') { Sentry.init({ dsn: "https://b6ea46f0fbe048c9974718d2c72e261b@o444579.ingest.sentry.io/5653683", - environment: process.env.api_url + environment: helper.getEnvironment() }); } diff --git a/src/resources/utilities/helper.util.js b/src/resources/utilities/helper.util.js index 25312ab4..c687b006 100644 --- a/src/resources/utilities/helper.util.js +++ b/src/resources/utilities/helper.util.js @@ -1,12 +1,9 @@ import crypto from 'crypto'; const _censorWord = str => { - if(str.length === 1) - return '*'; - else if(str.length === 2) - return `${str[0]}*`; - else - return str[0] + '*'.repeat(str.length - 2) + str.slice(-1); + if (str.length === 1) return '*'; + else if (str.length === 2) return `${str[0]}*`; + else return str[0] + '*'.repeat(str.length - 2) + str.slice(-1); }; const _censorEmail = email => { @@ -37,7 +34,7 @@ const _generatedNumericId = () => { return parseInt(Math.random().toString().replace('0.', '')); }; -const _generateAlphaNumericString = (length) => { +const _generateAlphaNumericString = length => { return crypto.randomBytes(length).toString('hex').substring(length); }; @@ -57,12 +54,30 @@ const _hidePrivateProfileDetails = persons => { }; const _getEnvironment = () => { - let environment = 'local'; + let environment = ''; - if (process.env.environment === 'www') environment = 'prod'; - else if (process.env.environment === 'uat') environment = 'uat'; - else if (process.env.environment === 'uatbeta') environment = 'uatbeta'; - else if (process.env.environment === 'latest') environment = 'latest'; + switch (process.env.api_url) { + case 'https://api.latest.healthdatagateway.org': + environment = 'latest'; + break; + case 'https://api.uatbeta.healthdatagateway.org': + environment = 'uatbeta'; + break; + case 'https://api.uat.healthdatagateway.org': + environment = 'uat'; + break; + case 'https://api.uat2.healthdatagateway.org': + environment = 'uat2'; + break; + case 'https://api.preprod.healthdatagateway.org': + environment = 'preprod'; + break; + case 'https://api.www.healthdatagateway.org': + environment = 'prod'; + break; + default: + environment = 'local'; + } return environment; }; @@ -74,5 +89,5 @@ export default { generatedNumericId: _generatedNumericId, generateAlphaNumericString: _generateAlphaNumericString, hidePrivateProfileDetails: _hidePrivateProfileDetails, - getEnvironment: _getEnvironment + getEnvironment: _getEnvironment, }; From 2edbc4b6e38ee264b2bb2f6d4e5460ec03f961a1 Mon Sep 17 00:00:00 2001 From: Robin Kavanagh Date: Mon, 1 Mar 2021 12:00:37 +0000 Subject: [PATCH 36/42] Updated job --- src/resources/dataset/v1/dataset.service.js | 98 ++++++++++----------- 1 file changed, 45 insertions(+), 53 deletions(-) diff --git a/src/resources/dataset/v1/dataset.service.js b/src/resources/dataset/v1/dataset.service.js index 0be798a1..d001b237 100644 --- a/src/resources/dataset/v1/dataset.service.js +++ b/src/resources/dataset/v1/dataset.service.js @@ -377,58 +377,50 @@ export async function loadDatasets(override) { datasetV2Call, ]); - var technicaldetails = []; - - await dataClass.data.items.reduce( - (p, dataclassMDC) => - p.then( - () => - new Promise(resolve => { - setTimeout(async function () { - const dataClassElementCall = axios - .get( - metadataCatalogueLink + - '/api/dataModels/' + - datasetMDC.id + - '/dataClasses/' + - dataclassMDC.id + - '/dataElements?max=300', - { timeout: 5000 } - ) - .catch(err => { - console.log('Unable to get dataclass element ' + err.message); - }); - const [dataClassElement] = await axios.all([dataClassElementCall]); - var dataClassElementArray = []; - - dataClassElement.data.items.forEach(element => { - dataClassElementArray.push({ - id: element.id, - domainType: element.domainType, - label: element.label, - description: element.description, - dataType: { - id: element.dataType.id, - domainType: element.dataType.domainType, - label: element.dataType.label, - }, - }); - }); - - technicaldetails.push({ - id: dataclassMDC.id, - domainType: dataclassMDC.domainType, - label: dataclassMDC.label, - description: dataclassMDC.description, - elements: dataClassElementArray, - }); - - resolve(null); - }, 500); - }) - ), - Promise.resolve(null) - ); + // Safely destructure data class items to protect against undefined and HTTP failures + const { data: { items: dataClassItems = [] } = {} } = dataClass || []; + + // Get technical details data classes + let technicaldetails = []; + + for (const dataClassMDC of dataClassItems) { + // Get data elements for each class + const { data: { items: dataClassElements = [] } = {} } = await axios + .get(`${metadataCatalogueLink}/api/dataModels/${datasetMDC.id}/dataClasses/${dataClassMDC.id}/dataElements?max=300`, { + timeout: 10000, + }) + .catch(err => { + console.log('Unable to get dataclass element ' + err.message); + }); + + // Map out data class elements to attach to class + const dataClassElementArray = dataClassElements.map(element => { + return { + id: element.id, + domainType: element.domainType, + label: element.label, + description: element.description, + dataType: { + id: element.dataType.id, + domainType: element.dataType.domainType, + label: element.dataType.label, + }, + }; + }); + + // Create class object + const technicalDetailClass = { + id: dataClassMDC.id, + domainType: dataClassMDC.domainType, + label: dataClassMDC.label, + description: dataClassMDC.description, + elements: dataClassElementArray, + }; + + technicaldetails = [...technicaldetails, technicalDetailClass]; + } + + console.log(JSON.stringify(technicaldetails)); let datasetv2Object = populateV2datasetObject(datasetV2.data.items); @@ -955,4 +947,4 @@ async function saveUptime() { var metricsData = new MetricsData(); metricsData.uptime = averageUptime; await metricsData.save(); -} \ No newline at end of file +} From 61960eaf024d5f37a47b2e71d5643e6d8eedbb6a Mon Sep 17 00:00:00 2001 From: Robin Kavanagh Date: Mon, 1 Mar 2021 12:01:58 +0000 Subject: [PATCH 37/42] Removed console log --- src/resources/dataset/v1/dataset.service.js | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/resources/dataset/v1/dataset.service.js b/src/resources/dataset/v1/dataset.service.js index d001b237..f403c45c 100644 --- a/src/resources/dataset/v1/dataset.service.js +++ b/src/resources/dataset/v1/dataset.service.js @@ -419,9 +419,7 @@ export async function loadDatasets(override) { technicaldetails = [...technicaldetails, technicalDetailClass]; } - - console.log(JSON.stringify(technicaldetails)); - + let datasetv2Object = populateV2datasetObject(datasetV2.data.items); if (datasetHDR) { From f87113d35086e38960f954a8329de9f2f1fb817b Mon Sep 17 00:00:00 2001 From: Robin Kavanagh Date: Mon, 1 Mar 2021 12:10:18 +0000 Subject: [PATCH 38/42] Completed adding of source --- src/resources/dataset/dataset.model.js | 1 + src/resources/dataset/v1/dataset.service.js | 44 +++++++++++---------- src/resources/tool/data.model.js | 1 + 3 files changed, 25 insertions(+), 21 deletions(-) diff --git a/src/resources/dataset/dataset.model.js b/src/resources/dataset/dataset.model.js index 0affd522..0f9db633 100644 --- a/src/resources/dataset/dataset.model.js +++ b/src/resources/dataset/dataset.model.js @@ -12,6 +12,7 @@ const datasetSchema = new Schema( id: Number, name: String, description: String, + source: String, resultsInsights: String, link: String, type: String, diff --git a/src/resources/dataset/v1/dataset.service.js b/src/resources/dataset/v1/dataset.service.js index f403c45c..ea2f969a 100644 --- a/src/resources/dataset/v1/dataset.service.js +++ b/src/resources/dataset/v1/dataset.service.js @@ -9,40 +9,40 @@ export async function loadDataset(datasetID) { const datasetCall = axios .get(metadataCatalogueLink + '/api/facets/' + datasetID + '/profile/uk.ac.hdrukgateway/HdrUkProfilePluginService', { timeout: 5000 }) .catch(err => { - console.log('Unable to get dataset details ' + err.message); + console.error('Unable to get dataset details ' + err.message); }); const metadataQualityCall = axios .get('https://raw.githubusercontent.com/HDRUK/datasets/master/reports/metadata_quality.json', { timeout: 5000 }) .catch(err => { - console.log('Unable to get metadata quality value ' + err.message); + console.error('Unable to get metadata quality value ' + err.message); }); const metadataSchemaCall = axios .get(metadataCatalogueLink + '/api/profiles/uk.ac.hdrukgateway/HdrUkProfilePluginService/schema.org/' + datasetID, { timeout: 5000 }) .catch(err => { - console.log('Unable to get metadata schema ' + err.message); + console.error('Unable to get metadata schema ' + err.message); }); const dataClassCall = axios.get(metadataCatalogueLink + '/api/dataModels/' + datasetID + '/dataClasses', { timeout: 5000 }).catch(err => { - console.log('Unable to get dataclass ' + err.message); + console.error('Unable to get dataclass ' + err.message); }); const versionLinksCall = axios .get(metadataCatalogueLink + '/api/catalogueItems/' + datasetID + '/semanticLinks', { timeout: 5000 }) .catch(err => { - console.log('Unable to get version links ' + err.message); + console.error('Unable to get version links ' + err.message); }); const phenotypesCall = await axios .get('https://raw.githubusercontent.com/spiros/hdr-caliber-phenome-portal/master/_data/dataset2phenotypes.json', { timeout: 5000 }) .catch(err => { - console.log('Unable to get phenotypes ' + err.message); + console.error('Unable to get phenotypes ' + err.message); }); const dataUtilityCall = await axios .get('https://raw.githubusercontent.com/HDRUK/datasets/master/reports/data_utility.json', { timeout: 5000 }) .catch(err => { - console.log('Unable to get data utility ' + err.message); + console.error('Unable to get data utility ' + err.message); }); const datasetV2Call = axios .get(metadataCatalogueLink + '/api/facets/' + datasetID + '/metadata?all=true', { timeout: 5000 }) .catch(err => { - console.log('Unable to get dataset version 2 ' + err.message); + console.error('Unable to get dataset version 2 ' + err.message); }); const [ dataset, @@ -77,7 +77,7 @@ export async function loadDataset(datasetID) { timeout: 5000, }) .catch(err => { - console.log('Unable to get dataclass element ' + err.message); + console.error('Unable to get dataclass element ' + err.message); }); const [dataClassElement] = await axios.all([dataClassElementCall]); var dataClassElementArray = []; @@ -197,7 +197,7 @@ export async function loadDataset(datasetID) { } export async function loadDatasets(override) { - console.log('Starting run at ' + Date()); + console.error('Starting run at ' + Date()); let metadataCatalogueLink = process.env.metadataURL || 'https://metadata-catalogue.org/hdruk'; let datasetsMDCCount = await new Promise(function (resolve, reject) { @@ -275,7 +275,7 @@ export async function loadDatasets(override) { level: Sentry.Severity.Error, }); Sentry.captureException(err); - //console.log("Unable to get metadata quality value " + err.message); //Uncomment for local testing + console.error("Unable to get metadata quality value " + err.message); }); const phenotypesList = await axios @@ -287,7 +287,7 @@ export async function loadDatasets(override) { level: Sentry.Severity.Error, }); Sentry.captureException(err); - //console.log("Unable to get metadata quality value " + err.message); //Uncomment for local testing + console.error("Unable to get metadata quality value " + err.message); }); const dataUtilityList = await axios @@ -299,7 +299,7 @@ export async function loadDatasets(override) { level: Sentry.Severity.Error, }); Sentry.captureException(err); - //console.log("Unable to get data utility " + err.message); //Uncomment for local testing + console.error("Unable to get data utility " + err.message); }); var datasetsMDCIDs = []; @@ -331,7 +331,7 @@ export async function loadDatasets(override) { level: Sentry.Severity.Error, }); Sentry.captureException(err); - //console.log('Unable to get metadata schema ' + err.message); + console.error('Unable to get metadata schema ' + err.message); }); const dataClassCall = axios @@ -343,7 +343,7 @@ export async function loadDatasets(override) { level: Sentry.Severity.Error, }); Sentry.captureException(err); - //console.log('Unable to get dataclass ' + err.message); + console.error('Unable to get dataclass ' + err.message); }); const versionLinksCall = axios @@ -355,7 +355,7 @@ export async function loadDatasets(override) { level: Sentry.Severity.Error, }); Sentry.captureException(err); - //console.log('Unable to get version links ' + err.message); + console.error('Unable to get version links ' + err.message); }); const datasetV2Call = axios @@ -367,7 +367,7 @@ export async function loadDatasets(override) { level: Sentry.Severity.Error, }); Sentry.captureException(err); - //console.log('Unable to get dataset version 2 ' + err.message); + console.error('Unable to get dataset version 2 ' + err.message); }); const [metadataSchema, dataClass, versionLinks, datasetV2] = await axios.all([ @@ -390,7 +390,7 @@ export async function loadDatasets(override) { timeout: 10000, }) .catch(err => { - console.log('Unable to get dataclass element ' + err.message); + console.error('Unable to get dataclass element ' + err.message); }); // Map out data class elements to attach to class @@ -419,7 +419,7 @@ export async function loadDatasets(override) { technicaldetails = [...technicaldetails, technicalDetailClass]; } - + let datasetv2Object = populateV2datasetObject(datasetV2.data.items); if (datasetHDR) { @@ -462,6 +462,7 @@ export async function loadDatasets(override) { datasetVersion: datasetHDR.datasetVersion, name: datasetMDC.title, description: datasetMDC.description, + source: 'HDRUK MDC', activeflag: 'active', license: datasetMDC.license, tags: { @@ -543,6 +544,7 @@ export async function loadDatasets(override) { data.datasetid = datasetMDC.id; data.type = 'dataset'; data.activeflag = 'active'; + data.source = 'HDRUK MDC'; data.name = datasetMDC.title; data.description = datasetMDC.description; @@ -574,7 +576,7 @@ export async function loadDatasets(override) { data.datasetv2 = datasetv2Object; await data.save(); } - console.log(`Finished ${counter} of ${datasetsMDCCount} datasets (${datasetMDC.id})`); + console.error(`Finished ${counter} of ${datasetsMDCCount} datasets (${datasetMDC.id})`); resolve(null); } catch (err) { Sentry.addBreadcrumb({ @@ -583,7 +585,7 @@ export async function loadDatasets(override) { level: Sentry.Severity.Fatal, }); Sentry.captureException(err); - //console.log(`Failed to add ${datasetMDC.id} to the DB with the error of ${err.message}`); //Uncomment for local testing + console.error(`Failed to add ${datasetMDC.id} to the DB with the error of ${err.message}`); } }, 500); }) diff --git a/src/resources/tool/data.model.js b/src/resources/tool/data.model.js index 8e37e21e..d199674b 100644 --- a/src/resources/tool/data.model.js +++ b/src/resources/tool/data.model.js @@ -74,6 +74,7 @@ const DataSchema = new Schema( profileComplete: Boolean, //dataset related fields + source: String, datasetid: String, pid: String, datasetVersion: String, From 3288557d64672279ba2e7c97cbcf9a2aa9c6c05c Mon Sep 17 00:00:00 2001 From: Robin Kavanagh Date: Tue, 2 Mar 2021 09:53:45 +0000 Subject: [PATCH 39/42] Standarised error logs --- src/config/db.js | 4 +- src/resources/account/account.route.js | 2 +- .../auth/sso/sso.discourse.router.js | 2 +- src/resources/auth/strategies/google.js | 2 +- src/resources/auth/strategies/linkedin.js | 2 +- src/resources/auth/strategies/oidc.js | 2 +- .../bpmnworkflow/bpmnworkflow.controller.js | 14 ++-- .../collections/collections.route.js | 2 +- src/resources/course/course.repository.js | 2 +- src/resources/course/v2/course.controller.js | 4 +- .../amendment/amendment.controller.js | 8 +-- .../datarequest/datarequest.controller.js | 66 +++++++++---------- .../datarequest/datarequest.schemas.route.js | 2 +- .../datarequest/utils/datarequest.util.js | 4 +- src/resources/dataset/dataset.controller.js | 4 +- src/resources/dataset/v1/dataset.service.js | 2 +- src/resources/discourse/discourse.route.js | 16 ++--- src/resources/discourse/discourse.service.js | 18 ++--- src/resources/message/message.controller.js | 8 +-- src/resources/message/message.route.js | 1 - src/resources/paper/paper.controller.js | 4 +- src/resources/paper/v1/paper.route.js | 4 +- src/resources/person/person.route.js | 2 - src/resources/project/project.controller.js | 4 +- .../publisher/publisher.controller.js | 4 +- .../relatedobjects/relatedobjects.route.js | 1 - src/resources/team/team.controller.js | 8 +-- src/resources/tool/v2/tool.controller.js | 4 +- src/resources/topic/topic.controller.js | 12 ++-- .../utilities/notificationBuilder.js | 2 +- src/resources/workflow/workflow.controller.js | 4 +- 31 files changed, 104 insertions(+), 110 deletions(-) diff --git a/src/config/db.js b/src/config/db.js index 4bb40f0f..13c94811 100644 --- a/src/config/db.js +++ b/src/config/db.js @@ -25,8 +25,8 @@ const connectToDatabase = async () => { }); console.log('MongoDB connected...'); - } catch (error) { - console.error(error.message); + } catch (err) { + console.error(err.message); process.exit(1); } diff --git a/src/resources/account/account.route.js b/src/resources/account/account.route.js index 66f303b4..e3b74634 100644 --- a/src/resources/account/account.route.js +++ b/src/resources/account/account.route.js @@ -202,7 +202,7 @@ router.put('/status', passport.authenticate('jwt'), utils.checkIsInRole(ROLES.Ad return res.json({ success: true }); } catch (err) { - console.log(err); + console.error(err.message); return res.status(500).json({ success: false, error: err }); } }); diff --git a/src/resources/auth/sso/sso.discourse.router.js b/src/resources/auth/sso/sso.discourse.router.js index a9555291..e7766983 100644 --- a/src/resources/auth/sso/sso.discourse.router.js +++ b/src/resources/auth/sso/sso.discourse.router.js @@ -19,7 +19,7 @@ router.get('/', function (req, res, next) { try { redirectUrl = discourseLogin(req.query.sso, req.query.sig, req.user); } catch (err) { - console.error(err); + console.error(err.message); return res.status(500).send('Error authenticating the user.'); } } diff --git a/src/resources/auth/strategies/google.js b/src/resources/auth/strategies/google.js index facd79cf..3587f180 100644 --- a/src/resources/auth/strategies/google.js +++ b/src/resources/auth/strategies/google.js @@ -117,7 +117,7 @@ const strategy = app => { try { redirectUrl = discourseLogin(queryStringParsed.sso, queryStringParsed.sig, req.user); } catch (err) { - console.error(err); + console.error(err.message); return res.status(500).send('Error authenticating the user.'); } } diff --git a/src/resources/auth/strategies/linkedin.js b/src/resources/auth/strategies/linkedin.js index 93d3cffb..09c2b488 100644 --- a/src/resources/auth/strategies/linkedin.js +++ b/src/resources/auth/strategies/linkedin.js @@ -115,7 +115,7 @@ const strategy = app => { try { redirectUrl = discourseLogin(queryStringParsed.sso, queryStringParsed.sig, req.user); } catch (err) { - console.error(err); + console.error(err.message); return res.status(500).send('Error authenticating the user.'); } } diff --git a/src/resources/auth/strategies/oidc.js b/src/resources/auth/strategies/oidc.js index 18842969..e8bd242f 100644 --- a/src/resources/auth/strategies/oidc.js +++ b/src/resources/auth/strategies/oidc.js @@ -119,7 +119,7 @@ const strategy = app => { try { redirectUrl = discourseLogin(queryStringParsed.sso, queryStringParsed.sig, req.user); } catch (err) { - console.error(err); + console.error(err.message); return res.status(500).send('Error authenticating the user.'); } } diff --git a/src/resources/bpmnworkflow/bpmnworkflow.controller.js b/src/resources/bpmnworkflow/bpmnworkflow.controller.js index 06452397..d568b4d8 100644 --- a/src/resources/bpmnworkflow/bpmnworkflow.controller.js +++ b/src/resources/bpmnworkflow/bpmnworkflow.controller.js @@ -48,7 +48,7 @@ module.exports = { businessKey: businessKey.toString(), }; await axios.post(`${bpmnBaseUrl}/engine-rest/process-definition/key/GatewayWorkflowSimple/start`, data, config).catch(err => { - console.error(err); + console.error(err.message); }); }, postUpdateProcess: async bpmContext => { @@ -79,7 +79,7 @@ module.exports = { }, }; await axios.post(`${bpmnBaseUrl}/engine-rest/task/${taskId}/complete`, data, config).catch(err => { - console.error(err); + console.error(err.message); }); }, @@ -105,7 +105,7 @@ module.exports = { businessKey: businessKey.toString(), }; await axios.post(`${bpmnBaseUrl}/engine-rest/process-definition/key/GatewayReviewWorkflowComplex/start`, data, config).catch(err => { - console.error(err); + console.error(err.message); }); }, postStartManagerReview: async bpmContext => { @@ -132,28 +132,28 @@ module.exports = { }, }; await axios.post(`${bpmnBaseUrl}/engine-rest/task/${taskId}/complete`, data, config).catch(err => { - console.error(err); + console.error(err.message); }); }, postManagerApproval: async bpmContext => { // Manager has approved sectoin let { businessKey } = bpmContext; await axios.post(`${bpmnBaseUrl}/api/gateway/workflow/v1/manager/completed/${businessKey}`, bpmContext.config).catch(err => { - console.error(err); + console.error(err.message); }); }, postStartStepReview: async bpmContext => { //Start Step-Review process let { businessKey } = bpmContext; await axios.post(`${bpmnBaseUrl}/api/gateway/workflow/v1/complete/review/${businessKey}`, bpmContext, config).catch(err => { - console.error(err); + console.error(err.message); }); }, postCompleteReview: async bpmContext => { //Start Next-Step process let { businessKey } = bpmContext; await axios.post(`${bpmnBaseUrl}/api/gateway/workflow/v1/reviewer/complete/${businessKey}`, bpmContext, config).catch(err => { - console.error(err); + console.error(err.message); }); }, }; diff --git a/src/resources/collections/collections.route.js b/src/resources/collections/collections.route.js index 6d9e1928..a8080f53 100644 --- a/src/resources/collections/collections.route.js +++ b/src/resources/collections/collections.route.js @@ -142,7 +142,7 @@ router.post('/add', passport.authenticate('jwt'), utils.checkIsInRole(ROLES.Admi // Send email notifications to all admins and authors who have opted in await sendEmailNotifications(collections, collections.activeflag, collectionCreator); } catch (err) { - console.log(err); + console.error(err.message); // return res.status(500).json({ success: false, error: err }); } diff --git a/src/resources/course/course.repository.js b/src/resources/course/course.repository.js index 583c6a11..84326c39 100644 --- a/src/resources/course/course.repository.js +++ b/src/resources/course/course.repository.js @@ -315,7 +315,7 @@ const setStatus = async (req, res) => { resolve(id); } catch (err) { - console.log(err); + console.error(err.message); reject(new Error(err)); } }); diff --git a/src/resources/course/v2/course.controller.js b/src/resources/course/v2/course.controller.js index 7b46ddb3..f337f0bd 100644 --- a/src/resources/course/v2/course.controller.js +++ b/src/resources/course/v2/course.controller.js @@ -33,7 +33,7 @@ export default class CourseController extends Controller { }); } catch (err) { // Return error response if something goes wrong - console.error(err); + console.error(err.message); return res.status(500).json({ success: false, message: 'A server error occurred, please try again', @@ -52,7 +52,7 @@ export default class CourseController extends Controller { }); } catch (err) { // Return error response if something goes wrong - console.error(err); + console.error(err.message); return res.status(500).json({ success: false, message: 'A server error occurred, please try again', diff --git a/src/resources/datarequest/amendment/amendment.controller.js b/src/resources/datarequest/amendment/amendment.controller.js index c9ba0391..608a502a 100644 --- a/src/resources/datarequest/amendment/amendment.controller.js +++ b/src/resources/datarequest/amendment/amendment.controller.js @@ -99,8 +99,8 @@ const setAmendment = async (req, res) => { // 9. Save changes to database await accessRecord.save(async err => { if (err) { - console.error(err); - return res.status(500).json({ status: 'error', message: err }); + console.error(err.message); + return res.status(500).json({ status: 'error', message: err.message }); } else { // 10. Update json schema and question answers with modifications since original submission let accessRecordObj = accessRecord.toObject(); @@ -214,8 +214,8 @@ const requestAmendments = async (req, res) => { // 9. Save changes to database await accessRecord.save(async err => { if (err) { - console.error(err); - return res.status(500).json({ status: 'error', message: err }); + console.error(err.message); + return res.status(500).json({ status: 'error', message: err.message }); } else { // 10. Send update request notifications createNotifications(constants.notificationTypes.RETURNED, accessRecord); diff --git a/src/resources/datarequest/datarequest.controller.js b/src/resources/datarequest/datarequest.controller.js index 8a03a2ce..236e2254 100644 --- a/src/resources/datarequest/datarequest.controller.js +++ b/src/resources/datarequest/datarequest.controller.js @@ -52,8 +52,8 @@ module.exports = { avgDecisionTime, canViewSubmitted: true, }); - } catch (error) { - console.error(error); + } catch (err) { + console.error(err.message); return res.status(500).json({ success: false, message: 'An error occurred searching for user applications', @@ -249,7 +249,7 @@ module.exports = { }, }); } catch (err) { - console.log(err.message); + console.error(err.message); res.status(500).json({ status: 'error', message: err.message }); } }, @@ -355,7 +355,7 @@ module.exports = { }, }); } catch (err) { - console.log(err.message); + console.error(err.message); res.status(500).json({ status: 'error', message: err.message }); } }, @@ -403,7 +403,7 @@ module.exports = { return res.status(200).json(data); }); } catch (err) { - console.log(err.message); + console.error(err.message); res.status(500).json({ status: 'error', message: err.message }); } }, @@ -445,7 +445,7 @@ module.exports = { if (applicationStatus === constants.applicationStatuses.INPROGRESS) { await DataRequestModel.findByIdAndUpdate(_id, updateObj, { new: true }, err => { if (err) { - console.error(err); + console.error(err.message); throw err; } }); @@ -462,7 +462,7 @@ module.exports = { accessRecord = amendmentController.handleApplicantAmendment(accessRecord.toObject(), updatedQuestionId, '', updatedAnswer, user); await DataRequestModel.replaceOne({ _id }, accessRecord, err => { if (err) { - console.error(err); + console.error(err.message); throw err; } }); @@ -612,8 +612,8 @@ module.exports = { if (isDirty) { await accessRecord.save(async err => { if (err) { - console.error(err); - return res.status(500).json({ status: 'error', message: err }); + console.error(err.message); + return res.status(500).json({ status: 'error', message: err.message }); } else { // If save has succeeded - send notifications // Send notifications to added/removed contributors @@ -763,7 +763,7 @@ module.exports = { // 12. Submit save accessRecord.save(function (err) { if (err) { - console.error(err); + console.error(err.message); return res.status(400).json({ success: false, message: err.message, @@ -845,8 +845,8 @@ module.exports = { // 7. Save update to access record await accessRecord.save(async err => { if (err) { - console.error(err); - res.status(500).json({ status: 'error', message: err }); + console.error(err.message); + res.status(500).json({ status: 'error', message: err.message }); } else { // 8. Call Camunda controller to get pre-review process let response = await bpmController.getProcess(id); @@ -872,8 +872,8 @@ module.exports = { // 14. Return aplication and successful response return res.status(200).json({ status: 'success' }); } catch (err) { - console.log(err.message); - res.status(500).json({ status: 'error', message: err }); + console.error(err.message); + res.status(500).json({ status: 'error', message: err.message }); } }, @@ -955,8 +955,8 @@ module.exports = { // 10. return response return res.status(200).json({ status: 'success', mediaFiles }); } catch (err) { - console.log(err.message); - res.status(500).json({ status: 'error', message: err }); + console.error(err.message); + res.status(500).json({ status: 'error', message: err.message }); } }, @@ -981,8 +981,8 @@ module.exports = { // 4. Return successful response return res.status(200).json({ status: accessRecord.files[fileIndex].status }); } catch (err) { - console.log(err.message); - res.status(500).json({ status: 'error', message: err }); + console.error(err.message); + res.status(500).json({ status: 'error', message: err.message }); } }, @@ -1021,8 +1021,8 @@ module.exports = { // 8. send file back to user return res.status(200).sendFile(`${process.env.TMPDIR}${id}/${dbFileId}_${name}`); } catch (err) { - console.log(err); - res.status(500).json({ status: 'error', message: err }); + console.error(err.message); + res.status(500).json({ status: 'error', message: err.message }); } }, @@ -1144,8 +1144,8 @@ module.exports = { // 14. Update MongoDb record for DAR await accessRecord.save(async err => { if (err) { - console.error(err); - res.status(500).json({ status: 'error', message: err }); + console.error(err.message); + res.status(500).json({ status: 'error', message: err.message }); } else { // 15. Create emails and notifications let relevantStepIndex = 0, @@ -1171,8 +1171,8 @@ module.exports = { // 17. Return aplication and successful response return res.status(200).json({ status: 'success', data: accessRecord._doc }); } catch (err) { - console.log(err.message); - res.status(500).json({ status: 'error', message: err }); + console.error(err.message); + res.status(500).json({ status: 'error', message: err.message }); } }, @@ -1260,8 +1260,8 @@ module.exports = { // 11. Save changes to the DAR await accessRecord.save(async err => { if (err) { - console.error(err); - res.status(500).json({ status: 'error', message: err }); + console.error(err.message); + res.status(500).json({ status: 'error', message: err.message }); } else { // 12. Gather context for notifications (active step) let emailContext = workflowController.getWorkflowEmailContext(accessRecord, workflow, activeStepIndex); @@ -1291,8 +1291,8 @@ module.exports = { // 16. Return aplication and successful response return res.status(200).json({ status: 'success' }); } catch (err) { - console.log(err.message); - res.status(500).json({ status: 'error', message: err }); + console.error(err.message); + res.status(500).json({ status: 'error', message: err.message }); } }, @@ -1344,8 +1344,8 @@ module.exports = { // 10. Return successful response return res.status(200).json({ status: 'success' }); } catch (err) { - console.log(err.message); - res.status(500).json({ status: 'error', message: err }); + console.error(err.message); + res.status(500).json({ status: 'error', message: err.message }); } }, @@ -1414,7 +1414,7 @@ module.exports = { // 7. Save changes to db await DataRequestModel.replaceOne({ _id: id }, accessRecord, async err => { if (err) { - console.error(err); + console.error(err.message); return res.status(500).json({ status: 'error', message: 'An error occurred saving the changes', @@ -1678,8 +1678,8 @@ module.exports = { await accessRecord.save(async err => { if (err) { - console.error(err); - return res.status(500).json({ status: 'error', message: err }); + console.error(err.message); + return res.status(500).json({ status: 'error', message: err.message }); } else { // 9. Append question actions for in progress applicant jsonSchema = datarequestUtil.injectQuestionActions( diff --git a/src/resources/datarequest/datarequest.schemas.route.js b/src/resources/datarequest/datarequest.schemas.route.js index d99450af..490598f3 100644 --- a/src/resources/datarequest/datarequest.schemas.route.js +++ b/src/resources/datarequest/datarequest.schemas.route.js @@ -52,6 +52,6 @@ async function archiveOtherVersions(id, dataSetId, status) { ); } } catch (err) { - console.log(err); + console.error(err.message); } } diff --git a/src/resources/datarequest/utils/datarequest.util.js b/src/resources/datarequest/utils/datarequest.util.js index 699df02b..7632bc8c 100644 --- a/src/resources/datarequest/utils/datarequest.util.js +++ b/src/resources/datarequest/utils/datarequest.util.js @@ -40,8 +40,8 @@ const getUserPermissionsForApplication = (application, userId, _id) => { } } return { authorised, userType }; - } catch (error) { - console.error(error); + } catch (err) { + console.error(err.message); return { authorised: false, userType: '' }; } }; diff --git a/src/resources/dataset/dataset.controller.js b/src/resources/dataset/dataset.controller.js index 10315bd3..df338a82 100644 --- a/src/resources/dataset/dataset.controller.js +++ b/src/resources/dataset/dataset.controller.js @@ -33,7 +33,7 @@ export default class DatasetController extends Controller { }); } catch (err) { // Return error response if something goes wrong - console.error(err); + console.error(err.message); return res.status(500).json({ success: false, message: 'A server error occurred, please try again', @@ -52,7 +52,7 @@ export default class DatasetController extends Controller { }); } catch (err) { // Return error response if something goes wrong - console.error(err); + console.error(err.message); return res.status(500).json({ success: false, message: 'A server error occurred, please try again', diff --git a/src/resources/dataset/v1/dataset.service.js b/src/resources/dataset/v1/dataset.service.js index ea2f969a..76358d45 100644 --- a/src/resources/dataset/v1/dataset.service.js +++ b/src/resources/dataset/v1/dataset.service.js @@ -576,7 +576,7 @@ export async function loadDatasets(override) { data.datasetv2 = datasetv2Object; await data.save(); } - console.error(`Finished ${counter} of ${datasetsMDCCount} datasets (${datasetMDC.id})`); + console.log(`Finished ${counter} of ${datasetsMDCCount} datasets (${datasetMDC.id})`); resolve(null); } catch (err) { Sentry.addBreadcrumb({ diff --git a/src/resources/discourse/discourse.route.js b/src/resources/discourse/discourse.route.js index 087cb278..37a77e77 100644 --- a/src/resources/discourse/discourse.route.js +++ b/src/resources/discourse/discourse.route.js @@ -39,7 +39,7 @@ router.get('/topic/:topicId', async (req, res) => { return res.status(500).json({ success: false, error: error.message }); }); } catch (err) { - console.error(err); + console.error(err.message); return res.status(500).json({ success: false, error: 'Error retrieving the topic, please try again later...' }); } }); @@ -67,7 +67,7 @@ router.get('/user/topic/:topicId', passport.authenticate('jwt'), utils.checkIsIn return res.status(500).json({ success: false, error: error.message }); }); } catch (err) { - console.error(err); + console.error(err.message); return res.status(500).json({ success: false, error: 'Error retrieving the topic, please try again later...' }); } }); @@ -99,7 +99,7 @@ router.put('/tool/:toolId', passport.authenticate('jwt'), utils.checkIsInRole(RO return res.status(500).json({ success: false, error: error.message }); }); } catch (err) { - console.error(err); + console.error(err.message); return res.status(500).json({ success: false, error: 'Error creating the topic, please try again later...' }); } }); @@ -167,7 +167,7 @@ router.post('/user/posts', passport.authenticate('jwt'), utils.checkIsInRole(ROL return res.json({ success: true, topic }); } } catch (err) { - console.error(err); + console.error(err.message); return res.status(500).json({ success: false, error: 'Error creating the topic, please try again later...' }); } }); @@ -194,7 +194,7 @@ router.put('/user/posts/:postId', passport.authenticate('jwt'), utils.checkIsInR // 5. Return the topic data return res.json({ success: true, topic }); } catch (err) { - console.error(err); + console.error(err.message); return res.status(500).json({ success: false, error: 'Error editing the post, please try again later...' }); } }); @@ -214,11 +214,11 @@ router.delete('/user/posts/:postId', passport.authenticate('jwt'), utils.checkIs // 3. Return success message return res.json({ success: true }); }) - .catch(error => { - return res.status(500).json({ success: false, error: error.message }); + .catch(err => { + return res.status(500).json({ success: false, error: err.message }); }); } catch (err) { - console.error(err); + console.error(err.message); return res.status(500).json({ success: false, error: 'Error deleting the topic, please try again later...' }); } }); diff --git a/src/resources/discourse/discourse.service.js b/src/resources/discourse/discourse.service.js index 39f1313f..ccfd163e 100644 --- a/src/resources/discourse/discourse.service.js +++ b/src/resources/discourse/discourse.service.js @@ -42,7 +42,7 @@ export async function getDiscourseTopic(topicId, user) { posts: posts, }; } catch (err) { - console.error(err); + console.error(err.message); } } @@ -110,7 +110,7 @@ export async function createDiscourseTopic(tool) { } } } catch (err) { - console.error(err); + console.error(err.message); } } @@ -141,7 +141,7 @@ export async function createDiscoursePost(topicId, comment, user) { try { const response = await axios.post(`${process.env.DISCOURSE_URL}/posts.json`, payload, config); } catch (err) { - console.error(err); + console.error(err.message); } } @@ -177,7 +177,7 @@ export async function updateDiscoursePost(postId, comment, user) { // 4. Return the post data return post; } catch (err) { - console.error(err); + console.error(err.message); } } @@ -209,7 +209,7 @@ export async function deleteDiscoursePost(postId, user) { try { const response = await axios.delete(`${process.env.DISCOURSE_URL}/posts/${postId}`, config); } catch (err) { - console.error(err); + console.error(err.message); } } @@ -249,7 +249,7 @@ async function createUser({ id, email, username }) { // 6. Return the new user object from Discourse return res.data; } catch (err) { - console.error(err); + console.error(err.message); } } @@ -287,7 +287,7 @@ async function generateAPIKey(discourseUsername) { // 3. Return key return key; } catch (err) { - console.error(err); + console.error(err.message); return ''; } } @@ -327,7 +327,7 @@ async function getCredentials(user, strict) { // 6. Update MongoDb to contain users Discourse credentials await UserModel.findOneAndUpdate({ id }, { $set: { discourseUsername, discourseKey } }); } catch (err) { - console.error(err); + console.error(err.message); } // 3. If user has username but no API key, generate new one } else if (_.isEmpty(discourseKey)) { @@ -337,7 +337,7 @@ async function getCredentials(user, strict) { // 5. Update MongoDb to contain users Discourse credentials await UserModel.findOneAndUpdate({ id }, { $set: { discourseUsername, discourseKey } }); } catch (err) { - console.error(err); + console.error(err.message); } } // Return identification payload of registered Discourse user diff --git a/src/resources/message/message.controller.js b/src/resources/message/message.controller.js index 2d895982..8a4d4c8f 100644 --- a/src/resources/message/message.controller.js +++ b/src/resources/message/message.controller.js @@ -88,7 +88,7 @@ module.exports = { return res.status(201).json({ success: true, message }); } catch (err) { console.error(err.message); - return res.status(500).json(err); + return res.status(500).json(err.message); } }, // DELETE /api/v1/messages/:id @@ -121,7 +121,7 @@ module.exports = { return res.status(204).json({ success: true }); } catch (err) { console.error(err.message); - return res.status(500).json(err); + return res.status(500).json(err.message); } }, // PUT /api/v1/messages @@ -153,7 +153,7 @@ module.exports = { return res.status(204).json({ success:true }); } catch(err) { console.error(err.message); - return res.status(500).json(err); + return res.status(500).json(err.message); } }, // GET api/v1/messages/unread/count @@ -180,7 +180,7 @@ module.exports = { } catch (err) { console.error(err.message); - return res.status(500).json(err); + return res.status(500).json(err.message); } } } diff --git a/src/resources/message/message.route.js b/src/resources/message/message.route.js index 7a3e58c5..54b2cea4 100644 --- a/src/resources/message/message.route.js +++ b/src/resources/message/message.route.js @@ -107,7 +107,6 @@ router.get('/admin/:personID', passport.authenticate('jwt'), utils.checkIsInRole }); router.post('/markasread', passport.authenticate('jwt'), utils.checkIsInRole(ROLES.Admin, ROLES.Creator), async (req, res) => { - console.log('in markAsRead'); const messageIds = req.body; MessagesModel.updateMany({ messageID: { $in: messageIds } }, { isRead: true }, err => { diff --git a/src/resources/paper/paper.controller.js b/src/resources/paper/paper.controller.js index f8db5a1f..9fd61b73 100644 --- a/src/resources/paper/paper.controller.js +++ b/src/resources/paper/paper.controller.js @@ -33,7 +33,7 @@ export default class PaperController extends Controller { }); } catch (err) { // Return error response if something goes wrong - console.error(err); + console.error(err.message); return res.status(500).json({ success: false, message: 'A server error occurred, please try again', @@ -52,7 +52,7 @@ export default class PaperController extends Controller { }); } catch (err) { // Return error response if something goes wrong - console.error(err); + console.error(err.message); return res.status(500).json({ success: false, message: 'A server error occurred, please try again', diff --git a/src/resources/paper/v1/paper.route.js b/src/resources/paper/v1/paper.route.js index 1ae07567..5ed07721 100644 --- a/src/resources/paper/v1/paper.route.js +++ b/src/resources/paper/v1/paper.route.js @@ -67,8 +67,8 @@ router.post('/validate', passport.authenticate('jwt'), utils.checkIsInRole(ROLES .json({ success: true, error: 'This link is already associated to another paper on the HDR-UK Innovation Gateway' }); // 5. Otherwise return valid return res.status(200).json({ success: true }); - } catch (error) { - console.error(error); + } catch (err) { + console.error(err.message); return res.status(500).json({ success: false, error: 'Paper link validation failed' }); } }); diff --git a/src/resources/person/person.route.js b/src/resources/person/person.route.js index fe0666be..7e0991e9 100644 --- a/src/resources/person/person.route.js +++ b/src/resources/person/person.route.js @@ -17,7 +17,6 @@ router.post('/', passport.authenticate('jwt'), utils.checkIsInRole(ROLES.Admin, let link = urlValidator.validateURL(inputSanitizer.removeNonBreakingSpaces(req.body.link)); let orcid = req.body.orcid !== '' ? urlValidator.validateOrcidURL(inputSanitizer.removeNonBreakingSpaces(req.body.orcid)) : ''; let data = Data(); - console.log(req.body); data.id = parseInt(Math.random().toString().replace('0.', '')); (data.firstname = inputSanitizer.removeNonBreakingSpaces(firstname)), (data.lastname = inputSanitizer.removeNonBreakingSpaces(lastname)), @@ -66,7 +65,6 @@ router.put('/', passport.authenticate('jwt'), utils.checkIsInRole(ROLES.Admin, R sector = inputSanitizer.removeNonBreakingSpaces(sector); organisation = inputSanitizer.removeNonBreakingSpaces(organisation); tags.topics = inputSanitizer.removeNonBreakingSpaces(tags.topics); - console.log(req.body); await Data.findOneAndUpdate( { id: id }, diff --git a/src/resources/project/project.controller.js b/src/resources/project/project.controller.js index 8c36018e..c96eb2ad 100644 --- a/src/resources/project/project.controller.js +++ b/src/resources/project/project.controller.js @@ -33,7 +33,7 @@ export default class ProjectController extends Controller { }); } catch (err) { // Return error response if something goes wrong - console.error(err); + console.error(err.message); return res.status(500).json({ success: false, message: 'A server error occurred, please try again', @@ -52,7 +52,7 @@ export default class ProjectController extends Controller { }); } catch (err) { // Return error response if something goes wrong - console.error(err); + console.error(err.message); return res.status(500).json({ success: false, message: 'A server error occurred, please try again', diff --git a/src/resources/publisher/publisher.controller.js b/src/resources/publisher/publisher.controller.js index d48a3bc2..1d858d21 100644 --- a/src/resources/publisher/publisher.controller.js +++ b/src/resources/publisher/publisher.controller.js @@ -24,7 +24,7 @@ module.exports = { return res.status(200).json({ success: true, publisher }); } catch (err) { console.error(err.message); - return res.status(500).json(err); + return res.status(500).json(err.message); } }, @@ -196,7 +196,7 @@ module.exports = { .status(200) .json({ success: true, data: modifiedApplications, avgDecisionTime, canViewSubmitted: isManager }); } catch (err) { - console.error(err); + console.error(err.message); return res.status(500).json({ success: false, message: 'An error occurred searching for custodian applications', diff --git a/src/resources/relatedobjects/relatedobjects.route.js b/src/resources/relatedobjects/relatedobjects.route.js index 0faef1a8..f9cde85a 100644 --- a/src/resources/relatedobjects/relatedobjects.route.js +++ b/src/resources/relatedobjects/relatedobjects.route.js @@ -11,7 +11,6 @@ const router = express.Router(); * Return the details on the relatedobject based on the ID. */ router.get('/:id', async (req, res) => { - console.log(`in relatedobjects.route`); let id = req.params.id; if (!isNaN(id)) { let q = Data.aggregate([ diff --git a/src/resources/team/team.controller.js b/src/resources/team/team.controller.js index fbdb70a8..4691e6a7 100644 --- a/src/resources/team/team.controller.js +++ b/src/resources/team/team.controller.js @@ -30,7 +30,7 @@ const getTeamById = async (req, res) => { return res.status(200).json({ success: true, team }); } catch (err) { console.error(err.message); - return res.status(500).json(err); + return res.status(500).json(err.message); } }; @@ -60,7 +60,7 @@ const getTeamMembers = async (req, res) => { return res.status(200).json({ success: true, members: users }); } catch (err) { console.error(err.message); - return res.status(500).json(err); + return res.status(500).json(err.message); } }; @@ -144,7 +144,7 @@ const addTeamMembers = async (req, res) => { // 9. Save members handling error callback if validation fails team.save(async (err) => { if (err) { - console.error(err); + console.error(err.message); return res.status(400).json({ success: false, message: err.message, @@ -254,7 +254,7 @@ const deleteTeamMember = async (req, res) => { team.members = updatedMembers; team.save(function (err) { if (err) { - console.error(err); + console.error(err.message); return res.status(400).json({ success: false, message: err.message, diff --git a/src/resources/tool/v2/tool.controller.js b/src/resources/tool/v2/tool.controller.js index 2f8e5c8f..77554cef 100644 --- a/src/resources/tool/v2/tool.controller.js +++ b/src/resources/tool/v2/tool.controller.js @@ -33,7 +33,7 @@ export default class ToolController extends Controller { }); } catch (err) { // Return error response if something goes wrong - console.error(err); + console.error(err.message); return res.status(500).json({ success: false, message: 'A server error occurred, please try again', @@ -52,7 +52,7 @@ export default class ToolController extends Controller { }); } catch (err) { // Return error response if something goes wrong - console.error(err); + console.error(err.message); return res.status(500).json({ success: false, message: 'A server error occurred, please try again', diff --git a/src/resources/topic/topic.controller.js b/src/resources/topic/topic.controller.js index 6cdf41a2..64081edc 100644 --- a/src/resources/topic/topic.controller.js +++ b/src/resources/topic/topic.controller.js @@ -59,7 +59,7 @@ module.exports = { tags.push(datasetTitle); break; default: - console.log('default'); + break; } }); // 7. Get recipients for topic/message using the first tool (same team exists as each publisher is the same) @@ -128,7 +128,7 @@ module.exports = { return res.status(201).json({ success: true, topic }); } catch (err) { console.error(err.message); - return res.status(500).json(err); + return res.status(500).json(err.message); } }, // DELETE api/v1/topics/:id @@ -137,11 +137,10 @@ module.exports = { const { id } = req.params; if (!id) return res.status(404).json({ success: false, message: 'Topic Id not found.' }); const topic = await TopicModel.findByIdAndUpdate(id, { isDeleted: true, status: 'closed', expiryDate: Date.now() }, { new: true }); - console.log(topic); return res.status(204).json({ success: true }); } catch (err) { console.error(err.message); - return res.status(500).json(err); + return res.status(500).json(err.message); } }, // GET api/v1/topics @@ -162,7 +161,6 @@ module.exports = { } // Calculate last unread message date at topic level topic.lastUnreadMessage = topic.topicMessages.reduce((a, b) => { - console.log(Date(a.createdDate) > new Date(b.createdDate) ? a : b); return (new Date(a.createdDate) > new Date(b.createdDate) ? a : b).createdDate; }); }); @@ -174,7 +172,7 @@ module.exports = { return res.status(200).json({ success: true, topics }); } catch (err) { console.error(err.message); - return res.status(500).json(err); + return res.status(500).json(err.message); } }, // GET api/v1/topics/:id @@ -199,7 +197,7 @@ module.exports = { return res.status(200).json({ success: true, topic: dispatchTopic }); } catch (err) { console.error(err.message); - return res.status(500).json(err); + return res.status(500).json(err.message); } }, }; diff --git a/src/resources/utilities/notificationBuilder.js b/src/resources/utilities/notificationBuilder.js index 5350a58f..6fda4be9 100644 --- a/src/resources/utilities/notificationBuilder.js +++ b/src/resources/utilities/notificationBuilder.js @@ -15,7 +15,7 @@ const triggerNotificationMessage = (messageRecipients, messageDescription, messa }); await message.save(async (err) => { if (err) { - console.error(`Failed to save ${messageType} message with error : ${err}`); + console.error(`Failed to save ${messageType} message with error : ${err.message}`); } }); }); diff --git a/src/resources/workflow/workflow.controller.js b/src/resources/workflow/workflow.controller.js index e3eddd52..2385650a 100644 --- a/src/resources/workflow/workflow.controller.js +++ b/src/resources/workflow/workflow.controller.js @@ -229,7 +229,7 @@ const updateWorkflow = async (req, res) => { if (isDirty) { workflow.save(async err => { if (err) { - console.error(err); + console.error(err.message); return res.status(400).json({ success: false, message: err.message, @@ -295,7 +295,7 @@ const deleteWorkflow = async (req, res) => { // 5. Delete workflow WorkflowModel.deleteOne({ _id: workflowId }, function (err) { if (err) { - console.error(err); + console.error(err.message); return res.status(400).json({ success: false, message: 'An error occurred deleting the workflow', From ea96109a1f106ae22a4494e812380bbfdb7c1919 Mon Sep 17 00:00:00 2001 From: Robin Kavanagh Date: Tue, 2 Mar 2021 10:26:49 +0000 Subject: [PATCH 40/42] Fixed LGTM issue --- src/resources/topic/topic.controller.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/resources/topic/topic.controller.js b/src/resources/topic/topic.controller.js index 64081edc..190eb0f6 100644 --- a/src/resources/topic/topic.controller.js +++ b/src/resources/topic/topic.controller.js @@ -136,7 +136,7 @@ module.exports = { try { const { id } = req.params; if (!id) return res.status(404).json({ success: false, message: 'Topic Id not found.' }); - const topic = await TopicModel.findByIdAndUpdate(id, { isDeleted: true, status: 'closed', expiryDate: Date.now() }, { new: true }); + TopicModel.findByIdAndUpdate(id, { isDeleted: true, status: 'closed', expiryDate: Date.now() }, { new: true }); return res.status(204).json({ success: true }); } catch (err) { console.error(err.message); From 7733ab283b6e537f64bce8121f628357f9a61b47 Mon Sep 17 00:00:00 2001 From: Paul McCafferty <> Date: Tue, 2 Mar 2021 16:55:31 +0000 Subject: [PATCH 41/42] First check in to UATBeta branch --- package.json | 158 +++++++++++++++++++++++++-------------------------- 1 file changed, 79 insertions(+), 79 deletions(-) diff --git a/package.json b/package.json index fdbe4857..c35e5de2 100644 --- a/package.json +++ b/package.json @@ -1,81 +1,81 @@ { - "name": "hdruk-rdt-api", - "version": "0.1.0", - "private": true, - "dependencies": { - "@google-cloud/monitoring": "^2.1.0", - "@google-cloud/storage": "^5.3.0", - "@sendgrid/mail": "^7.1.0", - "@sentry/node": "^5.29.0", - "async": "^3.2.0", - "await-to-js": "^2.1.1", - "axios": "0.19.2", - "axios-retry": "^3.1.9", - "base64url": "^3.0.1", - "bcrypt": "^5.0.0", - "body-parser": "^1.19.0", - "btoa": "^1.2.1", - "cookie-parser": "^1.4.5", - "cors": "^2.8.5", - "crypto": "^1.0.1", - "crypto-js": "^4.0.0", - "discourse-sso": "^1.0.3", - "dotenv": "^8.2.0", - "escape-html": "^1.0.3", - "esm": "^3.2.25", - "express": "^4.17.1", - "express-rate-limit": "^5.1.3", - "express-session": "^1.17.1", - "express-validator": "^6.6.1", - "faker": "^5.3.1", - "googleapis": "^55.0.0", - "jose": "^2.0.2", - "jsonwebtoken": "^8.5.1", - "keygrip": "^1.1.0", - "lodash": "^4.17.19", - "moment": "^2.27.0", - "mongoose": "^5.9.12", - "morgan": "^1.10.0", - "multer": "^1.4.2", - "oidc-provider": "^6.29.3", - "passport": "^0.4.1", - "passport-google-oauth": "^2.0.0", - "passport-jwt": "^4.0.0", - "passport-linkedin-oauth2": "^2.0.0", - "passport-openidconnect": "0.0.2", - "prettier": "^2.2.1", - "query-string": "^6.12.1", - "randomstring": "^1.1.5", - "sinon": "^9.2.4", - "snyk": "^1.334.0", - "swagger-ui-express": "^4.1.4", - "test": "^0.6.0", - "transformobject": "^0.3.1", - "uuid": "^8.3.1", - "yamljs": "^0.3.0" - }, - "devDependencies": { - "@babel/preset-env": "^7.12.1", - "@shelf/jest-mongodb": "^1.2.3", - "babel-jest": "^26.6.3", - "eslint": "^7.20.0", - "jest": "^26.6.3", - "mongodb-memory-server": "^6.9.2", - "nodemon": "^2.0.3", - "supertest": "^4.0.2" - }, - "scripts": { - "start": "node index.js", - "server": "nodemon index.js", - "debug": "nodemon --inspect=0.0.0.0:3001 index.js", - "build": "", - "test": "jest --runInBand", - "eject": "", - "snyk-protect": "snyk protect", - "prepublish": "npm run snyk-protect", - "prettify": "prettier --write \"src/**/*.{scss,js,jsx}\"", - "prettify-test": "prettier --write \"test/**/*.js\"" - }, - "proxy": "http://localhost:3001", - "snyk": true + "name": "hdruk-rdt-api", + "version": "0.1.0", + "private": true, + "dependencies": { + "@google-cloud/monitoring": "^2.1.0", + "@google-cloud/storage": "^5.3.0", + "@sendgrid/mail": "^7.1.0", + "@sentry/node": "^5.29.0", + "async": "^3.2.0", + "await-to-js": "^2.1.1", + "axios": "0.19.2", + "axios-retry": "^3.1.9", + "base64url": "^3.0.1", + "bcrypt": "^5.0.0", + "body-parser": "^1.19.0", + "btoa": "^1.2.1", + "cookie-parser": "^1.4.5", + "cors": "^2.8.5", + "crypto": "^1.0.1", + "crypto-js": "^4.0.0", + "discourse-sso": "^1.0.3", + "dotenv": "^8.2.0", + "escape-html": "^1.0.3", + "esm": "^3.2.25", + "express": "^4.17.1", + "express-rate-limit": "^5.1.3", + "express-session": "^1.17.1", + "express-validator": "^6.6.1", + "faker": "^5.3.1", + "googleapis": "^55.0.0", + "jose": "^2.0.2", + "jsonwebtoken": "^8.5.1", + "keygrip": "^1.1.0", + "lodash": "^4.17.19", + "moment": "^2.27.0", + "mongoose": "^5.9.12", + "morgan": "^1.10.0", + "multer": "^1.4.2", + "oidc-provider": "^6.29.3", + "passport": "^0.4.1", + "passport-google-oauth": "^2.0.0", + "passport-jwt": "^4.0.0", + "passport-linkedin-oauth2": "^2.0.0", + "passport-openidconnect": "0.0.2", + "prettier": "^2.2.1", + "query-string": "^6.12.1", + "randomstring": "^1.1.5", + "sinon": "^9.2.4", + "snyk": "^1.334.0", + "swagger-ui-express": "^4.1.4", + "test": "^0.6.0", + "transformobject": "^0.3.1", + "uuid": "^8.3.1", + "yamljs": "^0.3.0" + }, + "devDependencies": { + "@babel/preset-env": "^7.12.1", + "@shelf/jest-mongodb": "^1.2.3", + "babel-jest": "^26.6.3", + "eslint": "^7.20.0", + "jest": "^26.6.3", + "mongodb-memory-server": "^6.9.2", + "nodemon": "^2.0.3", + "supertest": "^4.0.2" + }, + "scripts": { + "start": "node index.js", + "server": "nodemon index.js", + "debug": "nodemon --inspect=0.0.0.0:3001 index.js", + "build": "", + "test": "jest --runInBand", + "eject": "", + "snyk-protect": "snyk protect", + "prepublish": "npm run snyk-protect", + "prettify": "prettier --write \"src/**/*.{scss,js,jsx}\"", + "prettify-test": "prettier --write \"test/**/*.js\"" + }, + "proxy": "http://localhost:3001", + "snyk": true } From cabe72c861a2e3dd0c60e69c36488863ea3da064 Mon Sep 17 00:00:00 2001 From: Paul McCafferty <> Date: Tue, 2 Mar 2021 17:04:33 +0000 Subject: [PATCH 42/42] Updating cloudbuild files --- cloudbuild.yaml | 47 +++++++++++++++++++++++----------------- cloudbuild_dynamic.yaml | 48 +++++++++++++++++++++++++++-------------- cloudbuild_uat.yaml | 26 ++++++++++++++++------ 3 files changed, 78 insertions(+), 43 deletions(-) diff --git a/cloudbuild.yaml b/cloudbuild.yaml index 92f08e45..ac7bc50e 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -1,22 +1,29 @@ steps: -- name: 'gcr.io/cloud-builders/docker' - entrypoint: 'bash' - args: ['-c', 'docker pull gcr.io/$PROJECT_ID/${_APP_NAME}:latest || exit 0'] -- name: 'gcr.io/cloud-builders/docker' - args: ['build', '-t', 'gcr.io/$PROJECT_ID/${_APP_NAME}:latest', '--cache-from', 'gcr.io/$PROJECT_ID/${_APP_NAME}:latest', '.'] -- name: 'gcr.io/cloud-builders/docker' - args: ['push', 'gcr.io/$PROJECT_ID/${_APP_NAME}:latest'] -- name: 'gcr.io/cloud-builders/gcloud' - args: ['run', 'deploy', 'latest-api', '--image', 'gcr.io/$PROJECT_ID/${_APP_NAME}:latest', '--platform', 'managed', '--region', '${_REGION}', '--allow-unauthenticated'] -- name: 'node' - args: ['npm','install'] -- name: 'node' - args: ['npm','test'] - env: - - 'URL=https://${_TEST_URL}' -- name: 'gcr.io/cloud-builders/gcloud' - args: ['run', 'deploy', 'uatbeta-api', '--image', 'gcr.io/$PROJECT_ID/${_APP_NAME}:latest', '--platform', 'managed', '--region', '${_REGION}', '--allow-unauthenticated'] -images: -- gcr.io/$PROJECT_ID/${_APP_NAME}:latest + - name: 'gcr.io/cloud-builders/docker' + entrypoint: 'bash' + args: ['-c', 'docker pull gcr.io/$PROJECT_ID/${_APP_NAME}:latest || exit 0'] + - name: 'gcr.io/cloud-builders/docker' + args: ['build', '-t', 'gcr.io/$PROJECT_ID/${_APP_NAME}:latest', '--cache-from', 'gcr.io/$PROJECT_ID/${_APP_NAME}:latest', '.'] + - name: 'gcr.io/cloud-builders/docker' + args: ['push', 'gcr.io/$PROJECT_ID/${_APP_NAME}:latest'] + - name: 'gcr.io/cloud-builders/gcloud' + args: + [ + 'run', + 'deploy', + 'latest-api', + '--image', + 'gcr.io/$PROJECT_ID/${_APP_NAME}:latest', + '--platform', + 'managed', + '--region', + '${_REGION}', + '--allow-unauthenticated', + ] + - name: 'node' + args: ['npm', 'test'] + env: + - 'URL=https://${_TEST_URL}' +timeout: 900s options: - machineType: 'E2_HIGHCPU_8' \ No newline at end of file + machineType: 'E2_HIGHCPU_8' diff --git a/cloudbuild_dynamic.yaml b/cloudbuild_dynamic.yaml index 3470fe8a..4b79be97 100644 --- a/cloudbuild_dynamic.yaml +++ b/cloudbuild_dynamic.yaml @@ -1,19 +1,35 @@ steps: -- name: 'gcr.io/cloud-builders/docker' - entrypoint: 'bash' - args: ['-c', 'docker pull gcr.io/$PROJECT_ID/${_APP_NAME}:${_ENVIRONMENT} || exit 0'] -- name: 'gcr.io/cloud-builders/docker' - args: [ - 'build', - '-t', 'gcr.io/$PROJECT_ID/${_APP_NAME}:${_ENVIRONMENT}', - '--cache-from', 'gcr.io/$PROJECT_ID/${_APP_NAME}:${_ENVIRONMENT}', - '.' - ] -- name: 'gcr.io/cloud-builders/docker' - args: ['push', 'gcr.io/$PROJECT_ID/${_APP_NAME}:${_ENVIRONMENT}'] -- name: 'gcr.io/cloud-builders/gcloud' - args: ['run', 'deploy', '${_ENVIRONMENT}-api', '--image', 'gcr.io/$PROJECT_ID/${_APP_NAME}:${_ENVIRONMENT}', '--platform', 'managed', '--region', '${_REGION}', '--allow-unauthenticated'] + - name: 'gcr.io/cloud-builders/docker' + entrypoint: 'bash' + args: ['-c', 'docker pull gcr.io/$PROJECT_ID/${_APP_NAME}:${_ENVIRONMENT} || exit 0'] + - name: 'gcr.io/cloud-builders/docker' + args: + [ + 'build', + '-t', + 'gcr.io/$PROJECT_ID/${_APP_NAME}:${_ENVIRONMENT}', + '--cache-from', + 'gcr.io/$PROJECT_ID/${_APP_NAME}:${_ENVIRONMENT}', + '.', + ] + - name: 'gcr.io/cloud-builders/docker' + args: ['push', 'gcr.io/$PROJECT_ID/${_APP_NAME}:${_ENVIRONMENT}'] + - name: 'gcr.io/cloud-builders/gcloud' + args: + [ + 'run', + 'deploy', + '${_ENVIRONMENT}-api', + '--image', + 'gcr.io/$PROJECT_ID/${_APP_NAME}:${_ENVIRONMENT}', + '--platform', + 'managed', + '--region', + '${_REGION}', + '--allow-unauthenticated', + ] images: -- gcr.io/$PROJECT_ID/${_APP_NAME}:${_ENVIRONMENT} + - gcr.io/$PROJECT_ID/${_APP_NAME}:${_ENVIRONMENT} +timeout: 900s options: - machineType: 'E2_HIGHCPU_8' \ No newline at end of file + machineType: 'E2_HIGHCPU_8' diff --git a/cloudbuild_uat.yaml b/cloudbuild_uat.yaml index 756cfa33..1bcb520a 100644 --- a/cloudbuild_uat.yaml +++ b/cloudbuild_uat.yaml @@ -1,9 +1,21 @@ steps: -- name: 'gcr.io/cloud-builders/docker' - args: ['build', '-t', 'gcr.io/$PROJECT_ID/${_APP_NAME}:uat', '.'] -- name: 'gcr.io/cloud-builders/docker' - args: ['push', 'gcr.io/$PROJECT_ID/${_APP_NAME}:uat'] -- name: 'gcr.io/cloud-builders/gcloud' - args: ['run', 'deploy', 'uat-api', '--image', 'gcr.io/$PROJECT_ID/${_APP_NAME}:uat', '--platform', 'managed', '--region', '${_REGION}', '--allow-unauthenticated'] + - name: 'gcr.io/cloud-builders/docker' + args: ['build', '-t', 'gcr.io/$PROJECT_ID/${_APP_NAME}:uat', '.'] + - name: 'gcr.io/cloud-builders/docker' + args: ['push', 'gcr.io/$PROJECT_ID/${_APP_NAME}:uat'] + - name: 'gcr.io/cloud-builders/gcloud' + args: + [ + 'run', + 'deploy', + 'uat-api', + '--image', + 'gcr.io/$PROJECT_ID/${_APP_NAME}:uat', + '--platform', + 'managed', + '--region', + '${_REGION}', + '--allow-unauthenticated', + ] images: -- gcr.io/$PROJECT_ID/${_APP_NAME}:uat + - gcr.io/$PROJECT_ID/${_APP_NAME}:uat