From 72bb0d5c300c391b4ce7ff2c14d3ebe03de69881 Mon Sep 17 00:00:00 2001 From: andrchoi Date: Sat, 9 May 2020 18:42:28 -0700 Subject: [PATCH 01/12] BS-34: create csv and send email to self --- .gitignore | 1 + package-lock.json | 139 ++++++++++++++++++++++++++---------- package.json | 2 + server.js | 2 + services/metricsDataDump.js | 134 ++++++++++++++++++++++++++++++++++ 5 files changed, 242 insertions(+), 36 deletions(-) create mode 100644 services/metricsDataDump.js diff --git a/.gitignore b/.gitignore index 99b800c..fbc6ad6 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ node_modules/**/* npm-debug.* .env +*.csv diff --git a/package-lock.json b/package-lock.json index 976da4c..57208c9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -987,16 +987,34 @@ } }, "@babel/register": { - "version": "7.6.2", - "resolved": "https://registry.npmjs.org/@babel/register/-/register-7.6.2.tgz", - "integrity": "sha512-xgZk2LRZvt6i2SAUWxc7ellk4+OYRgS3Zpsnr13nMS1Qo25w21Uu8o6vTOAqNaxiqrnv30KTYzh9YWY2k21CeQ==", + "version": "7.9.0", + "resolved": "https://registry.npmjs.org/@babel/register/-/register-7.9.0.tgz", + "integrity": "sha512-Tv8Zyi2J2VRR8g7pC5gTeIN8Ihultbmk0ocyNz8H2nEZbmhp1N6q0A1UGsQbDvGP/sNinQKUHf3SqXwqjtFv4Q==", "dev": true, "requires": { "find-cache-dir": "^2.0.0", "lodash": "^4.17.13", - "mkdirp": "^0.5.1", + "make-dir": "^2.1.0", "pirates": "^4.0.0", - "source-map-support": "^0.5.9" + "source-map-support": "^0.5.16" + }, + "dependencies": { + "make-dir": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-2.1.0.tgz", + "integrity": "sha512-LS9X+dc8KLxXCb8dni79fLIIUA5VyZoyjSMCwTluaXA0o27cCK0bhXkpgw+sTXVpPy/lSO57ilRixqk0vDmtRA==", + "dev": true, + "requires": { + "pify": "^4.0.1", + "semver": "^5.6.0" + } + }, + "pify": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/pify/-/pify-4.0.1.tgz", + "integrity": "sha512-uB80kBFb/tfd68bVleG9T5GGsGPjJrLAUpR5PZIrhBnIaRTQRjqdJSsIKkOP6OAIFbj7GOrcudc5pNjZ+geV2g==", + "dev": true + } } }, "@babel/template": { @@ -1949,6 +1967,11 @@ "resolved": "https://registry.npmjs.org/decode-uri-component/-/decode-uri-component-0.2.0.tgz", "integrity": "sha1-6zkTMzRYd1y4TNGh+uBiEGu4dUU=" }, + "deeks": { + "version": "2.2.5", + "resolved": "https://registry.npmjs.org/deeks/-/deeks-2.2.5.tgz", + "integrity": "sha512-a4N/8uC+OGAmr0pmqaF6ZikBIO3c4cLqLARoOkI0qFH/FXL2rcbvh/fUZmKhdEH0fDbwU5WblV+mn4o8sc9y/A==" + }, "deep-eql": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/deep-eql/-/deep-eql-3.0.1.tgz", @@ -2013,9 +2036,9 @@ "integrity": "sha1-TkMekrEalzFjaqH5yNHMvP2reN8=" }, "kind-of": { - "version": "6.0.2", - "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.2.tgz", - "integrity": "sha512-s5kLOcnH0XqDO+FvuaLX8DDjZ18CGFk7VygH40QoKPUQhW4e2rvM0rwUq0t8IQDOwYSeLK01U90OjzBTme2QqA==" + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==" } } }, @@ -2056,6 +2079,11 @@ "integrity": "sha512-A46qtFgd+g7pDZinpnwiRJtxbC1hpgf0uzP3iG89scHk0AUC7A1TGxf5OiiOUv/JMZR8GOt8hL900hV0bOy5xA==", "dev": true }, + "doc-path": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/doc-path/-/doc-path-2.0.3.tgz", + "integrity": "sha512-Wex5OlidOT7esfbuLxtUiDHnG2er5UaKzMbUW2Ns9nRrsx+DmEXdHwXb0+jp+FTKmjE9i6bgrwRakT+YcobjEg==" + }, "dot-prop": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/dot-prop/-/dot-prop-4.2.0.tgz", @@ -3707,6 +3735,15 @@ "esprima": "^4.0.0" } }, + "json-2-csv": { + "version": "3.7.0", + "resolved": "https://registry.npmjs.org/json-2-csv/-/json-2-csv-3.7.0.tgz", + "integrity": "sha512-3MZoyknul6W66uhJ0jZx3MQ9Q4ars7NNy8db/XhvcFNsqrRrKMoTeTYCnm7CH1YuZ9SmB8iNcU8s1KXACSgnxg==", + "requires": { + "deeks": "2.2.5", + "doc-path": "2.0.3" + } + }, "json5": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/json5/-/json5-2.1.1.tgz", @@ -3717,9 +3754,9 @@ }, "dependencies": { "minimist": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.0.tgz", - "integrity": "sha1-o1AIsg9BOD7sH7kU9M1d95omQoQ=", + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz", + "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==", "dev": true } } @@ -4035,9 +4072,9 @@ }, "dependencies": { "kind-of": { - "version": "6.0.2", - "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.2.tgz", - "integrity": "sha512-s5kLOcnH0XqDO+FvuaLX8DDjZ18CGFk7VygH40QoKPUQhW4e2rvM0rwUq0t8IQDOwYSeLK01U90OjzBTme2QqA==" + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==" } } }, @@ -4068,9 +4105,9 @@ } }, "minimist": { - "version": "0.0.8", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.0.8.tgz", - "integrity": "sha1-hX/Kv8M5fSYluCKCYuhqp6ARsF0=" + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz", + "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==" }, "minipass": { "version": "2.9.0", @@ -4109,11 +4146,11 @@ } }, "mkdirp": { - "version": "0.5.1", - "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz", - "integrity": "sha1-MAV0OOrGz3+MR2fzhkjWaX11yQM=", + "version": "0.5.5", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.5.tgz", + "integrity": "sha512-NKmAlESf6jMGym1++R0Ra7wvhV+wFW63FaSOFPwRahvea0gMUcGUhVeAg/0BC0wiv9ih5NYPB1Wn1UEI1/L+xQ==", "requires": { - "minimist": "0.0.8" + "minimist": "^1.2.5" } }, "mocha": { @@ -4170,11 +4207,36 @@ "path-is-absolute": "^1.0.0" } }, + "minimist": { + "version": "0.0.8", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.0.8.tgz", + "integrity": "sha1-hX/Kv8M5fSYluCKCYuhqp6ARsF0=", + "dev": true + }, + "mkdirp": { + "version": "0.5.1", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz", + "integrity": "sha1-MAV0OOrGz3+MR2fzhkjWaX11yQM=", + "dev": true, + "requires": { + "minimist": "0.0.8" + } + }, "ms": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.1.tgz", "integrity": "sha512-tgp+dl5cGk28utYktBsrFqA7HKgrhgPsg6Z/EfhWI4gl1Hwq8B/GmY/0oXZ6nF8hDVesS/FpnYaD/kOWhYQvyg==", "dev": true + }, + "yargs-parser": { + "version": "13.1.1", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-13.1.1.tgz", + "integrity": "sha512-oVAVsHz6uFrg3XQheFII8ESO2ssAf9luWuAd6Wexsu4F3OtIW0o8IribPXYrD4WC24LWtPrJlGy87y5udK+dxQ==", + "dev": true, + "requires": { + "camelcase": "^5.0.0", + "decamelize": "^1.2.0" + } } } }, @@ -4297,9 +4359,9 @@ "integrity": "sha1-qJS3XUvE9s1nnvMkSp/Y9Gri1Cg=" }, "kind-of": { - "version": "6.0.2", - "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.2.tgz", - "integrity": "sha512-s5kLOcnH0XqDO+FvuaLX8DDjZ18CGFk7VygH40QoKPUQhW4e2rvM0rwUq0t8IQDOwYSeLK01U90OjzBTme2QqA==" + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==" } } }, @@ -4388,6 +4450,11 @@ } } }, + "nodemailer": { + "version": "6.4.6", + "resolved": "https://registry.npmjs.org/nodemailer/-/nodemailer-6.4.6.tgz", + "integrity": "sha512-/kJ+FYVEm2HuUlw87hjSqTss+GU35D4giOpdSfGp7DO+5h6RlJj7R94YaYHOkoxu1CSaM0d3WRBtCzwXrY6MKA==" + }, "nodemon": { "version": "1.19.4", "resolved": "https://registry.npmjs.org/nodemon/-/nodemon-1.19.4.tgz", @@ -4976,9 +5043,9 @@ }, "dependencies": { "minimist": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.0.tgz", - "integrity": "sha1-o1AIsg9BOD7sH7kU9M1d95omQoQ=" + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz", + "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==" } } }, @@ -5261,9 +5328,9 @@ "dev": true }, "kind-of": { - "version": "6.0.2", - "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.2.tgz", - "integrity": "sha512-s5kLOcnH0XqDO+FvuaLX8DDjZ18CGFk7VygH40QoKPUQhW4e2rvM0rwUq0t8IQDOwYSeLK01U90OjzBTme2QqA==", + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", "dev": true }, "micromatch": { @@ -5714,9 +5781,9 @@ } }, "source-map-support": { - "version": "0.5.16", - "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.16.tgz", - "integrity": "sha512-efyLRJDr68D9hBBNIPWFjhpFzURh+KJykQwvMyW5UiZzYwoF6l4YMMDIJJEyFWxWCqfyxLzz6tSfUFR+kXXsVQ==", + "version": "0.5.19", + "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.19.tgz", + "integrity": "sha512-Wonm7zOCIJzBGQdB+thsPar0kYuCIzYvxZwlBa87yi/Mdjv7Tip2cyVbLj5o0cFPN4EVkuTwb3GDDyUx2DGnGw==", "dev": true, "requires": { "buffer-from": "^1.0.0", @@ -6404,9 +6471,9 @@ } }, "yargs-parser": { - "version": "13.1.1", - "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-13.1.1.tgz", - "integrity": "sha512-oVAVsHz6uFrg3XQheFII8ESO2ssAf9luWuAd6Wexsu4F3OtIW0o8IribPXYrD4WC24LWtPrJlGy87y5udK+dxQ==", + "version": "13.1.2", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-13.1.2.tgz", + "integrity": "sha512-3lbsNRf/j+A4QuSZfDRA7HRSfWrzO0YjqTJd5kjAq37Zep1CEgaYmrH9Q3GwPiB9cHyd1Y1UwggGhJGoxipbzg==", "dev": true, "requires": { "camelcase": "^5.0.0", diff --git a/package.json b/package.json index 91c1a11..6c26d2b 100644 --- a/package.json +++ b/package.json @@ -16,11 +16,13 @@ "expo-server-sdk": "^3.4.0", "express": "^4.17.1", "express-validator": "^6.3.0", + "json-2-csv": "^3.7.0", "jsonwebtoken": "^8.5.1", "knex": "^0.20.8", "mongodb": "^3.3.3", "mongoose": "^5.7.6", "morgan": "^1.9.1", + "nodemailer": "^6.4.6", "pg": "^7.18.1", "rand-token": "^0.4.0", "redis": "^2.8.0" diff --git a/server.js b/server.js index ecf6263..1c8a33b 100644 --- a/server.js +++ b/server.js @@ -1,6 +1,8 @@ require("dotenv").config({ path: __dirname + "/.env" }); const InitializationService = require("./services/initialization.service"); InitializationService.initialize(); +const metricDataDumpService = require("./Services/metricsDataDump"); +metricDataDumpService.sendDataDumpEmail(); const { validateSignup, validateLogin, validateUseRefreshToken, validateDeleteRefreshToken } = require("./utils/error_handling"); const user = require("./controllers/user"); const auth = require("./controllers/auth"); diff --git a/services/metricsDataDump.js b/services/metricsDataDump.js new file mode 100644 index 0000000..65aa447 --- /dev/null +++ b/services/metricsDataDump.js @@ -0,0 +1,134 @@ +import metrics from "../database/postgres"; +let fs = require("fs"); +let nodemailer = require("nodemailer"); +let converter = require("json-2-csv") + +let metricDB = metrics.getMetrics(); +let transporter = nodemailer.createTransport({ + service: process.env.METRIC_DATA_SERVICE, + auth: { + type: "OAuth2", + user: process.env.METRIC_DATA_USER, + clientId: process.env.METRIC_DATA_CLIENTID, + clientSecret: process.env.METRIC_DATA_CLIENT_SECRET, + refreshToken: process.env.METRIC_DATA_REFRESH_TOKEN, + accessToken: process.env.METRIC_DATA_ACCESS_TOKEN, + expires: 0 + } +}); + +async function sendDataDumpEmail() { + let data = await getAllMetricData(); + console.log(data.length) + + converter.json2csv(data, function (err, csv) { + if (err) { + sendEmailError(err) + } else { + makeAndEmailCSV(csv) + } + }) +} + +function makeAndEmailCSV(data) { + let date = new Date() + let year = String(date.getUTCFullYear()) + let month = String(1+date.getUTCMonth()).padStart(2,0) + let day = String(date.getUTCDate()).padStart(2,0) + let hour = String(date.getUTCHours()).padStart(2,0) + let min = String(date.getUTCMinutes()).padStart(2,0) + let sec = String(date.getUTCSeconds()).padStart(2,0) + let ms = String(date.getUTCMilliseconds()).padStart(3,0) + date = year + month + day + "-" + hour + min + sec + ms; + + let filename = "./"+"test"+".csv"; + fs.writeFile(filename, data, (err) => { + if (err) { + sendEmailError(err); + } else { + sendEmailWithCSV(filename) + } + }) +} + +function sendEmailWithCSV(file) { + let message = { + from: process.env.METRIC_DATA_USER, + to: process.env.METRIC_DATA_RECEIVER, + subject: "UHN App Backend Data Analytics for "+ file, + attachments: [ + { + path: "./test.csv" + } + ] + } + + transporter.sendMail(message, function (err, info) { + if (err) { + console.log(err) + } + else console.log(info) + }) + +} + + +async function getAllMetricData() { + let results = null; + try { + results = await metricDB.select( + "u.id as User_ID", + "mongoid as u_mongoid", + "username as u_username", + "lastlogin as u_lastlogin", + "al.id as AlarmLog_ID", + "userid as al_userid", + "alarmstart as al_alarmStart", + "alarmend as al_alarmEnd", + "alarmsent as al_alarmSent", + "rl.id as ResponseLog_ID", + "responderid as rl_responderid", + "alarmid as rl_alarmid", + "alertresponse as rl_alertResponse", + "responsetime as rl_responseTime", + "arl.id as ArrivalLog_ID", + "arl.responseid as arl_responseid", + "arl.arrivaltime as arl_arrivalTime", + "tl.id as TreatmentLog_ID", + "tl.responseid as tl_responseid", + "tl.alertsuccessful as tl_alertSuccessful", + "tl.treatmenttime as tl_treatmentTime" + ) + .from("users AS u") + .fullOuterJoin("alarmlog as al", "u.mongoid", "al.userid") + .fullOuterJoin("responselog as rl", "rl.alarmid", "al.id") + .fullOuterJoin("arrivallog as arl", "rl.id", "arl.responseid") + .fullOuterJoin("treatmentlog as tl", "rl.id", "tl.responseid"); + + return results; + } + catch (err) { + sendEmailError(err); + } +} + +function sendEmailError(err) { + console.log(err); + let message = { + from: process.env.METRIC_DATA_USER, + to: process.env.METRIC_DATA_RECEIVER, + subject: "Error: UHN App Backend Data Analytics", + text: "Unable to deliver data analytics due to error on backend server: \n" + err.message + } + + transporter.sendMail(message, function (err, info) { + if (err) { + console.log(err) + } + else console.log(info) + }) +} + +module.exports = { + sendDataDumpEmail +} \ No newline at end of file From 1d67e8a7e0e226bbb4a89ccd5aba61aa3194b569 Mon Sep 17 00:00:00 2001 From: andrchoi Date: Sat, 9 May 2020 19:58:09 -0700 Subject: [PATCH 02/12] BS-34: added instructions for gmail accounts to send emails --- README.md | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/README.md b/README.md index 24223d0..1999085 100644 --- a/README.md +++ b/README.md @@ -76,6 +76,42 @@ To fill database with example data, uncomment bottom statements in data.sql and For help with DATABASE_URL fields, go to root directory, connect to postgres with `psql --username=metric UHN-metrics` and enter `\conninfo` +### Set Up OAuth + Gmail for Data Analytics +## Getting credentials.json +1. Go to the [Google Developer Console](https://developers.google.com/gmail/api/quickstart/nodejs) and click "Enable the Gmail API". + +2. Select `Web Server` from the dropdown. + +3. Enter this link `https://developers.google.com/oauthplayground` in the "Authorized Redirect URIs" field. + +4. Download the client configuration to get `credentials.json` + +## Getting OAuth Tokens + +1. Go to `https://developers.google.com/oauthplayground` and open the settings menu on the right. + +2. Check the box labelled `Use your own OAuth credentials` and enter the OAuth Client ID and OAuth Client Secret provided in the saved `credentials.json` file. + +3. Go to the `Step 1: Select and authorize APIs` section on the left. Enter the URL `https://mail.google.com` in the field below and click `Authorize APIs`. + +4. Go to `Step 2: Exchange authorization code for tokens` and click the button labelled `Exchange authorization code for tokens`. + +5. Save the Refresh and Access tokens. + +## Setting Up The .env File + +Create the following variables: + + ``` + METRIC_DATA_SERVICE="gmail" + METRIC_DATA_USER="Gmail account used from the setup above" + METRIC_DATA_CLIENTID="Provided in credentials.json" + METRIC_DATA_CLIENT_SECRET="Provided in credentials.json" + METRIC_DATA_REFRESH_TOKEN="Saved from OAuth playground" + METRIC_DATA_ACCESS_TOKEN="Saved from OAuth playground" + METRIC_DATA_RECEIVER="Account to receive analytics" + ``` + ## Run tests Make sure you are in project directory and run From 09b8ef6930b4bb11fd82ea4ace143a2172a01239 Mon Sep 17 00:00:00 2001 From: andrchoi Date: Sat, 9 May 2020 20:00:17 -0700 Subject: [PATCH 03/12] BS-34: update readme --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 1999085..c8fb7bb 100644 --- a/README.md +++ b/README.md @@ -88,7 +88,7 @@ For help with DATABASE_URL fields, go to root directory, connect to postgres wit ## Getting OAuth Tokens -1. Go to `https://developers.google.com/oauthplayground` and open the settings menu on the right. +1. Go to https://developers.google.com/oauthplayground and open the settings menu on the right. 2. Check the box labelled `Use your own OAuth credentials` and enter the OAuth Client ID and OAuth Client Secret provided in the saved `credentials.json` file. @@ -102,7 +102,7 @@ For help with DATABASE_URL fields, go to root directory, connect to postgres wit Create the following variables: - ``` + ` METRIC_DATA_SERVICE="gmail" METRIC_DATA_USER="Gmail account used from the setup above" METRIC_DATA_CLIENTID="Provided in credentials.json" @@ -110,7 +110,7 @@ Create the following variables: METRIC_DATA_REFRESH_TOKEN="Saved from OAuth playground" METRIC_DATA_ACCESS_TOKEN="Saved from OAuth playground" METRIC_DATA_RECEIVER="Account to receive analytics" - ``` + ` ## Run tests From 1d493071b4dbd518518945d99c54c2882e8dc5df Mon Sep 17 00:00:00 2001 From: andrchoi Date: Sat, 9 May 2020 20:01:46 -0700 Subject: [PATCH 04/12] BS-34: fixed formatting --- README.md | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index c8fb7bb..5c22e27 100644 --- a/README.md +++ b/README.md @@ -77,7 +77,7 @@ For help with DATABASE_URL fields, go to root directory, connect to postgres wit ### Set Up OAuth + Gmail for Data Analytics -## Getting credentials.json +#### Getting credentials.json 1. Go to the [Google Developer Console](https://developers.google.com/gmail/api/quickstart/nodejs) and click "Enable the Gmail API". 2. Select `Web Server` from the dropdown. @@ -98,11 +98,10 @@ For help with DATABASE_URL fields, go to root directory, connect to postgres wit 5. Save the Refresh and Access tokens. -## Setting Up The .env File +#### Setting Up The .env File Create the following variables: - ` METRIC_DATA_SERVICE="gmail" METRIC_DATA_USER="Gmail account used from the setup above" METRIC_DATA_CLIENTID="Provided in credentials.json" @@ -110,9 +109,8 @@ Create the following variables: METRIC_DATA_REFRESH_TOKEN="Saved from OAuth playground" METRIC_DATA_ACCESS_TOKEN="Saved from OAuth playground" METRIC_DATA_RECEIVER="Account to receive analytics" - ` -## Run tests +#### Run tests Make sure you are in project directory and run From 3acffed0763e76a71259549b4eca238e8ee8fecc Mon Sep 17 00:00:00 2001 From: andrchoi Date: Sat, 9 May 2020 20:02:37 -0700 Subject: [PATCH 05/12] BS-34: formatting --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 5c22e27..741c9f3 100644 --- a/README.md +++ b/README.md @@ -86,7 +86,7 @@ For help with DATABASE_URL fields, go to root directory, connect to postgres wit 4. Download the client configuration to get `credentials.json` -## Getting OAuth Tokens +#### Getting OAuth Tokens 1. Go to https://developers.google.com/oauthplayground and open the settings menu on the right. @@ -110,7 +110,7 @@ Create the following variables: METRIC_DATA_ACCESS_TOKEN="Saved from OAuth playground" METRIC_DATA_RECEIVER="Account to receive analytics" -#### Run tests +## Run tests Make sure you are in project directory and run From ad742305623c70085db1c5b951628aa77579682c Mon Sep 17 00:00:00 2001 From: andrchoi Date: Sat, 9 May 2020 21:16:21 -0700 Subject: [PATCH 06/12] BS-34: added emailing on interval, formatting changes --- server.js | 2 +- services/metricsDataDump.js | 40 ++++++++++++++++++++----------------- 2 files changed, 23 insertions(+), 19 deletions(-) diff --git a/server.js b/server.js index 1c8a33b..b1ad889 100644 --- a/server.js +++ b/server.js @@ -2,7 +2,7 @@ require("dotenv").config({ path: __dirname + "/.env" }); const InitializationService = require("./services/initialization.service"); InitializationService.initialize(); const metricDataDumpService = require("./Services/metricsDataDump"); -metricDataDumpService.sendDataDumpEmail(); +metricDataDumpService.sendPeriodicEmail(); const { validateSignup, validateLogin, validateUseRefreshToken, validateDeleteRefreshToken } = require("./utils/error_handling"); const user = require("./controllers/user"); const auth = require("./controllers/auth"); diff --git a/services/metricsDataDump.js b/services/metricsDataDump.js index 65aa447..691ae3f 100644 --- a/services/metricsDataDump.js +++ b/services/metricsDataDump.js @@ -12,20 +12,18 @@ let transporter = nodemailer.createTransport({ clientId: process.env.METRIC_DATA_CLIENTID, clientSecret: process.env.METRIC_DATA_CLIENT_SECRET, refreshToken: process.env.METRIC_DATA_REFRESH_TOKEN, - accessToken: process.env.METRIC_DATA_ACCESS_TOKEN, - expires: 0 + accessToken: process.env.METRIC_DATA_ACCESS_TOKEN } }); async function sendDataDumpEmail() { let data = await getAllMetricData(); - console.log(data.length) converter.json2csv(data, function (err, csv) { if (err) { - sendEmailError(err) + sendEmailError(err); } else { - makeAndEmailCSV(csv) + makeAndEmailCSV(csv); } }) } @@ -35,13 +33,9 @@ function makeAndEmailCSV(data) { let year = String(date.getUTCFullYear()) let month = String(1+date.getUTCMonth()).padStart(2,0) let day = String(date.getUTCDate()).padStart(2,0) - let hour = String(date.getUTCHours()).padStart(2,0) - let min = String(date.getUTCMinutes()).padStart(2,0) - let sec = String(date.getUTCSeconds()).padStart(2,0) - let ms = String(date.getUTCMilliseconds()).padStart(3,0) - date = year + month + day + "-" + hour + min + sec + ms; + date = year + "-" + month + "-" + day; - let filename = "./"+"test"+".csv"; + let filename = "./"+date+".csv"; fs.writeFile(filename, data, (err) => { if (err) { sendEmailError(err); @@ -55,10 +49,10 @@ function sendEmailWithCSV(file) { let message = { from: process.env.METRIC_DATA_USER, to: process.env.METRIC_DATA_RECEIVER, - subject: "UHN App Backend Data Analytics for "+ file, + subject: "UHN App Backend Data Analytics for "+ file.substring(2, file.length-4), attachments: [ { - path: "./test.csv" + path: file } ] } @@ -112,23 +106,33 @@ async function getAllMetricData() { } } +function sendPeriodicEmail() { + let now = new Date(); + let emailTime = new Date(now.getFullYear(), now.getMonth(), now.getDate() + 1, 0, 0, 0, 0); + let countdown = emailTime-now; + + setTimeout(function() { + sendDataDumpEmail(); + sendPeriodicEmail(); + }, countdown); +} + function sendEmailError(err) { console.log(err); let message = { from: process.env.METRIC_DATA_USER, to: process.env.METRIC_DATA_RECEIVER, - subject: "Error: UHN App Backend Data Analytics", - text: "Unable to deliver data analytics due to error on backend server: \n" + err.message + subject: "ERROR: UHN App Backend Data Analytics", + text: "Unable to deliver data analytics due to the following error on backend server: \n\n" + err.message } - transporter.sendMail(message, function (err, info) { + transporter.sendMail(message, function (err) { if (err) { console.log(err) } - else console.log(info) }) } module.exports = { - sendDataDumpEmail + sendPeriodicEmail } \ No newline at end of file From ced3f65d66bc82e2b68c2fe8e53a935dd7d1b093 Mon Sep 17 00:00:00 2001 From: andrchoi Date: Mon, 11 May 2020 10:36:39 -0700 Subject: [PATCH 07/12] BS-34: removed console logs --- services/metricsDataDump.js | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/services/metricsDataDump.js b/services/metricsDataDump.js index 691ae3f..ae367be 100644 --- a/services/metricsDataDump.js +++ b/services/metricsDataDump.js @@ -57,11 +57,10 @@ function sendEmailWithCSV(file) { ] } - transporter.sendMail(message, function (err, info) { + transporter.sendMail(message, function (err) { if (err) { - console.log(err) + sendEmailError(err); } - else console.log(info) }) } From c5d934ed4da620c23a61d9f7e5de1b80e356ba31 Mon Sep 17 00:00:00 2001 From: andrchoi Date: Mon, 11 May 2020 10:45:03 -0700 Subject: [PATCH 08/12] BS-34: added log for next email --- services/metricsDataDump.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/services/metricsDataDump.js b/services/metricsDataDump.js index ae367be..c074dfc 100644 --- a/services/metricsDataDump.js +++ b/services/metricsDataDump.js @@ -110,6 +110,8 @@ function sendPeriodicEmail() { let emailTime = new Date(now.getFullYear(), now.getMonth(), now.getDate() + 1, 0, 0, 0, 0); let countdown = emailTime-now; + console.log("Next data analytics email scheduled for: "+ emailTime.toUTCString()); + setTimeout(function() { sendDataDumpEmail(); sendPeriodicEmail(); From 5c3fd4a8aca81cdcc6dec253c98c19d9c685dc33 Mon Sep 17 00:00:00 2001 From: andrchoi Date: Mon, 11 May 2020 19:00:34 -0700 Subject: [PATCH 09/12] BS-34: get data in last day, zip data --- .gitignore | 2 +- server.js | 1 + services/metricsDataDump.js | 36 +++++++++++++++++++++++++++++------- 3 files changed, 31 insertions(+), 8 deletions(-) diff --git a/.gitignore b/.gitignore index fbc6ad6..942b258 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,4 @@ node_modules/**/* npm-debug.* .env -*.csv +*.csv* diff --git a/server.js b/server.js index b1ad889..310f92e 100644 --- a/server.js +++ b/server.js @@ -3,6 +3,7 @@ const InitializationService = require("./services/initialization.service"); InitializationService.initialize(); const metricDataDumpService = require("./Services/metricsDataDump"); metricDataDumpService.sendPeriodicEmail(); +metricDataDumpService.sendDataDumpEmail(); const { validateSignup, validateLogin, validateUseRefreshToken, validateDeleteRefreshToken } = require("./utils/error_handling"); const user = require("./controllers/user"); const auth = require("./controllers/auth"); diff --git a/services/metricsDataDump.js b/services/metricsDataDump.js index c074dfc..b6db9b2 100644 --- a/services/metricsDataDump.js +++ b/services/metricsDataDump.js @@ -2,6 +2,7 @@ import metrics from "../database/postgres"; let fs = require("fs"); let nodemailer = require("nodemailer"); let converter = require("json-2-csv") +let zlib = require("zlib"); let metricDB = metrics.getMetrics(); let transporter = nodemailer.createTransport({ @@ -17,6 +18,7 @@ let transporter = nodemailer.createTransport({ }); async function sendDataDumpEmail() { + console.log('test') let data = await getAllMetricData(); converter.json2csv(data, function (err, csv) { @@ -40,16 +42,28 @@ function makeAndEmailCSV(data) { if (err) { sendEmailError(err); } else { - sendEmailWithCSV(filename) + zipAndSendFile(filename, date); } }) } -function sendEmailWithCSV(file) { +function zipAndSendFile(filename, date) { + let gzip = zlib.createGzip(); + let extension = ".gz"; + + let read = fs.createReadStream(filename); + let write = fs.createWriteStream(filename+extension); + + read.pipe(gzip).pipe(write); + + sendEmailWithZip(filename+extension, date); +} + +function sendEmailWithZip(file, date) { let message = { from: process.env.METRIC_DATA_USER, to: process.env.METRIC_DATA_RECEIVER, - subject: "UHN App Backend Data Analytics for "+ file.substring(2, file.length-4), + subject: "UHN App Backend Data Analytics for "+ date, attachments: [ { path: file @@ -65,7 +79,6 @@ function sendEmailWithCSV(file) { } - async function getAllMetricData() { let results = null; try { @@ -92,11 +105,19 @@ async function getAllMetricData() { "tl.alertsuccessful as tl_alertSuccessful", "tl.treatmenttime as tl_treatmentTime" ) - .from("users AS u") + .from("users AS u") .fullOuterJoin("alarmlog as al", "u.mongoid", "al.userid") .fullOuterJoin("responselog as rl", "rl.alarmid", "al.id") .fullOuterJoin("arrivallog as arl", "rl.id", "arl.responseid") - .fullOuterJoin("treatmentlog as tl", "rl.id", "tl.responseid"); + .fullOuterJoin("treatmentlog as tl", "rl.id", "tl.responseid") + .whereRaw( + "alarmstart >= current_date - interval '1' day \ + --or lastlogin >= current_date - interval '1' day \ + or alarmend >= current_date - interval '1' day \ + or responsetime >= current_date - interval '1' day \ + or arrivaltime >= current_date - interval '1' day \ + or treatmenttime >= current_date - interval '1' day" + ); return results; } @@ -135,5 +156,6 @@ function sendEmailError(err) { } module.exports = { - sendPeriodicEmail + sendPeriodicEmail, + sendDataDumpEmail } \ No newline at end of file From 9d19b14c364024192cd18cae205861477845f0c0 Mon Sep 17 00:00:00 2001 From: andrchoi Date: Mon, 11 May 2020 19:25:25 -0700 Subject: [PATCH 10/12] BS-34: added monthly login data do dump --- services/metricsDataDump.js | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/services/metricsDataDump.js b/services/metricsDataDump.js index b6db9b2..b9f5974 100644 --- a/services/metricsDataDump.js +++ b/services/metricsDataDump.js @@ -18,8 +18,10 @@ let transporter = nodemailer.createTransport({ }); async function sendDataDumpEmail() { - console.log('test') let data = await getAllMetricData(); + let monthlyLogins = await getMonthlyLogins(); + + data[0].monthly_unique_logins = monthlyLogins; converter.json2csv(data, function (err, csv) { if (err) { @@ -112,7 +114,6 @@ async function getAllMetricData() { .fullOuterJoin("treatmentlog as tl", "rl.id", "tl.responseid") .whereRaw( "alarmstart >= current_date - interval '1' day \ - --or lastlogin >= current_date - interval '1' day \ or alarmend >= current_date - interval '1' day \ or responsetime >= current_date - interval '1' day \ or arrivaltime >= current_date - interval '1' day \ @@ -126,6 +127,22 @@ async function getAllMetricData() { } } +async function getMonthlyLogins() { + let result = null; + try { + result = await metricDB("users").count("*").whereRaw( + "lastLogin >= current_date - interval '30' day" + ); + + result = result[0].count; + + return result; + } + catch (err) { + sendEmailError(err); + } +} + function sendPeriodicEmail() { let now = new Date(); let emailTime = new Date(now.getFullYear(), now.getMonth(), now.getDate() + 1, 0, 0, 0, 0); From 23523808e6cd5a17f32e249a319c40e7325a1876 Mon Sep 17 00:00:00 2001 From: andrchoi Date: Mon, 11 May 2020 21:55:46 -0700 Subject: [PATCH 11/12] BS-34: changed postgreSQL query to past day EST, save .csv to metricData folder --- metricData/.gitkeep | 0 services/metricsDataDump.js | 20 +++++++++++++------- 2 files changed, 13 insertions(+), 7 deletions(-) create mode 100644 metricData/.gitkeep diff --git a/metricData/.gitkeep b/metricData/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/services/metricsDataDump.js b/services/metricsDataDump.js index b9f5974..fb94dfd 100644 --- a/services/metricsDataDump.js +++ b/services/metricsDataDump.js @@ -39,7 +39,7 @@ function makeAndEmailCSV(data) { let day = String(date.getUTCDate()).padStart(2,0) date = year + "-" + month + "-" + day; - let filename = "./"+date+".csv"; + let filename = "./metricData/"+date+".csv"; fs.writeFile(filename, data, (err) => { if (err) { sendEmailError(err); @@ -113,11 +113,16 @@ async function getAllMetricData() { .fullOuterJoin("arrivallog as arl", "rl.id", "arl.responseid") .fullOuterJoin("treatmentlog as tl", "rl.id", "tl.responseid") .whereRaw( - "alarmstart >= current_date - interval '1' day \ - or alarmend >= current_date - interval '1' day \ - or responsetime >= current_date - interval '1' day \ - or arrivaltime >= current_date - interval '1' day \ - or treatmenttime >= current_date - interval '1' day" + "alarmstart >= current_timestamp at time zone 'utc' - interval '1' day - interval '4' hour \ + and alarmstart < current_timestamp at time zone 'utc' - interval '4' hour \ + or alarmend >= current_timestamp at time zone 'utc' - interval '1' day - interval '4' hour \ + and alarmend < current_timestamp at time zone 'utc' - interval '4' hour \ + or responsetime >= current_timestamp at time zone 'utc' - interval '1' day - interval '4' hour \ + and responsetime < current_timestamp at time zone 'utc' - interval '4' hour \ + or arrivaltime >= current_timestamp at time zone 'utc' - interval '1' day - interval '4' hour \ + and arrivaltime < current_timestamp at time zone 'utc' - interval '4' hour \ + or treatmenttime >= current_timestamp at time zone 'utc' - interval '1' day - interval '4' hour \ + and treatmenttime < current_timestamp at time zone 'utc' - interval '4' hour " ); return results; @@ -131,7 +136,8 @@ async function getMonthlyLogins() { let result = null; try { result = await metricDB("users").count("*").whereRaw( - "lastLogin >= current_date - interval '30' day" + "lastLogin >= current_timestamp at time zone 'utc' - interval '30' day - interval '4' hour \ + and lastlogin < current_timestamp at time zone 'utc' - interval '4' hour" ); result = result[0].count; From 487e4bfc9ceb31fbb9048fe9ed88c075a8571991 Mon Sep 17 00:00:00 2001 From: andrchoi Date: Wed, 20 May 2020 18:08:21 -0700 Subject: [PATCH 12/12] BS-34: changed to use node-cron --- package-lock.json | 19 ++++++++++++++++ package.json | 1 + services/metricsDataDump.js | 44 ++++++++++++++++++++++++++++--------- 3 files changed, 54 insertions(+), 10 deletions(-) diff --git a/package-lock.json b/package-lock.json index 57208c9..74262a9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -4395,6 +4395,15 @@ "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.2.tgz", "integrity": "sha512-hZXc7K2e+PgeI1eDBe/10Ard4ekbfrrqG8Ep+8Jmf4JID2bNg7NvCPOZN+kfF574pFQI7mum2AUqDidoKqcTOw==" }, + "node-cron": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/node-cron/-/node-cron-2.0.3.tgz", + "integrity": "sha512-eJI+QitXlwcgiZwNNSRbqsjeZMp5shyajMR81RZCqeW0ZDEj4zU9tpd4nTh/1JsBiKbF8d08FCewiipDmVIYjg==", + "requires": { + "opencollective-postinstall": "^2.0.0", + "tz-offset": "0.0.1" + } + }, "node-environment-flags": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/node-environment-flags/-/node-environment-flags-1.0.5.tgz", @@ -4683,6 +4692,11 @@ "wrappy": "1" } }, + "opencollective-postinstall": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/opencollective-postinstall/-/opencollective-postinstall-2.0.2.tgz", + "integrity": "sha512-pVOEP16TrAO2/fjej1IdOyupJY8KDUM1CvsaScRbw6oddvpQoOfGk4ywha0HKKVAD6RkW4x6Q+tNBwhf3Bgpuw==" + }, "os-homedir": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/os-homedir/-/os-homedir-1.0.2.tgz", @@ -6081,6 +6095,11 @@ "mime-types": "~2.1.24" } }, + "tz-offset": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/tz-offset/-/tz-offset-0.0.1.tgz", + "integrity": "sha512-kMBmblijHJXyOpKzgDhKx9INYU4u4E1RPMB0HqmKSgWG8vEcf3exEfLh4FFfzd3xdQOw9EuIy/cP0akY6rHopQ==" + }, "unc-path-regex": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/unc-path-regex/-/unc-path-regex-0.1.2.tgz", diff --git a/package.json b/package.json index 6c26d2b..53ec36d 100644 --- a/package.json +++ b/package.json @@ -22,6 +22,7 @@ "mongodb": "^3.3.3", "mongoose": "^5.7.6", "morgan": "^1.9.1", + "node-cron": "^2.0.3", "nodemailer": "^6.4.6", "pg": "^7.18.1", "rand-token": "^0.4.0", diff --git a/services/metricsDataDump.js b/services/metricsDataDump.js index fb94dfd..0a7390e 100644 --- a/services/metricsDataDump.js +++ b/services/metricsDataDump.js @@ -1,4 +1,5 @@ import metrics from "../database/postgres"; +let cron = require("node-cron"); let fs = require("fs"); let nodemailer = require("nodemailer"); let converter = require("json-2-csv") @@ -36,7 +37,7 @@ function makeAndEmailCSV(data) { let date = new Date() let year = String(date.getUTCFullYear()) let month = String(1+date.getUTCMonth()).padStart(2,0) - let day = String(date.getUTCDate()).padStart(2,0) + let day = String(date.getUTCDate()-1).padStart(2,0) date = year + "-" + month + "-" + day; let filename = "./metricData/"+date+".csv"; @@ -124,6 +125,32 @@ async function getAllMetricData() { or treatmenttime >= current_timestamp at time zone 'utc' - interval '1' day - interval '4' hour \ and treatmenttime < current_timestamp at time zone 'utc' - interval '4' hour " ); + + if (results.length === 0) { + results = [{ + User_ID: null, + u_mongoid: null, + u_username: null, + u_lastlogin: null, + AlarmLog_ID: null, + al_userid: null, + al_alarmStart: null, + al_alarmEnd: null, + al_alarmSent: null, + ResponseLog_ID: null, + rl_responderid: null, + rl_alarmid: null, + rl_alertResponse: null, + rl_responseTime: null, + ArrivalLog_ID: null, + arl_responseid: null, + arl_arrivalTime: null, + TreatmentLog_ID: null, + tl_responseid: null, + tl_alertSuccessful: null, + tl_treatmentTime: null + }] + } return results; } @@ -150,16 +177,13 @@ async function getMonthlyLogins() { } function sendPeriodicEmail() { - let now = new Date(); - let emailTime = new Date(now.getFullYear(), now.getMonth(), now.getDate() + 1, 0, 0, 0, 0); - let countdown = emailTime-now; - console.log("Next data analytics email scheduled for: "+ emailTime.toUTCString()); - - setTimeout(function() { - sendDataDumpEmail(); - sendPeriodicEmail(); - }, countdown); + cron.schedule('0 0 0 * * *', () => { + console.log("Sending scheduled data analytics email..."); + sendDataDumpEmail() + }, { + timezone: "Atlantic/South_Georgia" + }) } function sendEmailError(err) {