From cc23944af1c2b90a8f60593680d5b5fa13ebc7df Mon Sep 17 00:00:00 2001 From: Ryan Birmingham Date: Tue, 16 Aug 2022 14:11:56 -0400 Subject: [PATCH 1/9] file handler, add uuid pkg --- caracal.js | 2 ++ handlers/fileHandlers.js | 14 ++++++++++++++ package-lock.json | 5 +++++ package.json | 3 ++- 4 files changed, 23 insertions(+), 1 deletion(-) create mode 100644 handlers/fileHandlers.js diff --git a/caracal.js b/caracal.js index 6023c77..5fef2e0 100644 --- a/caracal.js +++ b/caracal.js @@ -18,6 +18,7 @@ const iipHandler = require('./handlers/iipHandler.js'); const proxyHandler = require('./handlers/proxyHandler.js'); const permissionHandler = require('./handlers/permssionHandler.js'); const dataHandlers = require('./handlers/dataHandlers.js'); +const fileHandlers = require('./handlers/fileHandlers.js'); const sanitizeBody = require('./handlers/sanitizeHandler.js'); const DataTransformationHandler = require('./handlers/dataTransformationHandler.js'); @@ -88,6 +89,7 @@ var HANDLERS = { "permissionHandler": permissionHandler, "editHandler": auth.editHandler, "proxyHandler": proxyHandler, + "writeFile": fileHandlers.writeFile, "iipHandler": function() { return iipHandler; }, diff --git a/handlers/fileHandlers.js b/handlers/fileHandlers.js new file mode 100644 index 0000000..f79e791 --- /dev/null +++ b/handlers/fileHandlers.js @@ -0,0 +1,14 @@ +const fs = require('fs').promises; +const uuid = require("uuid"); + +// write a file containing the request body +function writeFile(path, prefix) { + return function(req, res, next) { + let fn = prefix + "_" + uuid.v4(); + var data = JSON.parse(req.body); + fs.writeFile(path + fn, data).then(()=>next()).catch((e) => next(e)); + }; +}; + +fileHandlers = {}; +fileHandlers.writeFile = writeFile diff --git a/package-lock.json b/package-lock.json index 918501b..51cd2c0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2648,6 +2648,11 @@ "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", "integrity": "sha1-n5VxD1CiZ5R7LMwSR0HBAoQn5xM=" }, + "uuid": { + "version": "8.3.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", + "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==" + }, "v8-compile-cache": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/v8-compile-cache/-/v8-compile-cache-2.1.0.tgz", diff --git a/package.json b/package.json index 71c5649..ad5d43c 100644 --- a/package.json +++ b/package.json @@ -32,7 +32,8 @@ "jsonwebtoken": "^8.5.1", "jwks-rsa": "^1.12.3", "mongodb": "^3.6.6", - "throng": "^5.0.0" + "throng": "^5.0.0", + "uuid": "^8.3.2" }, "devDependencies": { "chai": "^4.3.4", From b9087290a97638d57c4dea493f5f4208025e977a Mon Sep 17 00:00:00 2001 From: Ryan Birmingham Date: Tue, 16 Aug 2022 14:22:59 -0400 Subject: [PATCH 2/9] export the file handler --- handlers/fileHandlers.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/handlers/fileHandlers.js b/handlers/fileHandlers.js index f79e791..db4980c 100644 --- a/handlers/fileHandlers.js +++ b/handlers/fileHandlers.js @@ -11,4 +11,5 @@ function writeFile(path, prefix) { }; fileHandlers = {}; -fileHandlers.writeFile = writeFile +fileHandlers.writeFile = writeFile; +module.exports = fileHandlers; From ec0dbd644f7226fdd042ea0885bbb2c87b9a8887 Mon Sep 17 00:00:00 2001 From: Ryan Birmingham Date: Tue, 16 Aug 2022 14:28:53 -0400 Subject: [PATCH 3/9] pass str to writefile --- handlers/fileHandlers.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/handlers/fileHandlers.js b/handlers/fileHandlers.js index db4980c..f4e4e3b 100644 --- a/handlers/fileHandlers.js +++ b/handlers/fileHandlers.js @@ -5,8 +5,7 @@ const uuid = require("uuid"); function writeFile(path, prefix) { return function(req, res, next) { let fn = prefix + "_" + uuid.v4(); - var data = JSON.parse(req.body); - fs.writeFile(path + fn, data).then(()=>next()).catch((e) => next(e)); + fs.writeFile(path + fn, req.body).then(()=>next()).catch((e) => next(e)); }; }; From d15bd8b6aeec15b0dc33c27139ec4e4773d81cc4 Mon Sep 17 00:00:00 2001 From: Ryan Birmingham Date: Tue, 16 Aug 2022 14:34:02 -0400 Subject: [PATCH 4/9] add writeFile handler --- handlers/fileHandlers.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/handlers/fileHandlers.js b/handlers/fileHandlers.js index f4e4e3b..c05bbb6 100644 --- a/handlers/fileHandlers.js +++ b/handlers/fileHandlers.js @@ -4,7 +4,7 @@ const uuid = require("uuid"); // write a file containing the request body function writeFile(path, prefix) { return function(req, res, next) { - let fn = prefix + "_" + uuid.v4(); + let fn = prefix + "_" + uuid.v4() + ".json"; fs.writeFile(path + fn, req.body).then(()=>next()).catch((e) => next(e)); }; }; From 54033921870631414cd10f774f97ee0090c2ef54 Mon Sep 17 00:00:00 2001 From: Ryan Birmingham Date: Tue, 30 Aug 2022 17:27:46 -0400 Subject: [PATCH 5/9] Curry another layer of auth header so usable --- caracal.js | 2 +- handlers/authHandlers.js | 56 +++++++++++++++++++++------------------- 2 files changed, 30 insertions(+), 28 deletions(-) diff --git a/caracal.js b/caracal.js index 5fef2e0..0291103 100644 --- a/caracal.js +++ b/caracal.js @@ -75,7 +75,7 @@ var HANDLERS = { "loginHandler": function() { return auth.loginHandler(auth.PUBKEY); }, - "loginWithHeader": auth.loginWithHeader, + "loginWithHeader": auth.loginWithHeader(auth.PRIKEY, userFunction), "sanitizeBody": function() { return sanitizeBody; }, diff --git a/handlers/authHandlers.js b/handlers/authHandlers.js index 430963f..1e55336 100644 --- a/handlers/authHandlers.js +++ b/handlers/authHandlers.js @@ -264,33 +264,35 @@ function firstSetupUserSignupExists() { } // Use a trusted header instead of a jwt for login. Use carefully if at all. -function loginWithHeader(header, signKey, userFunction) { - return function(req, res) { - // get the correct header, set it to use userFunction - let token = {"email": req.headers[header]}; - // login using that - userFunction(token).then((x) => { - if (x === false) { - res.status(401).send({ - 'err': 'User Unauthorized', - }); - } else { - data = x; - delete data['exp']; - // sign using the mounted key - var token = jwt.sign(data, signKey, { - algorithm: 'RS256', - expiresIn: EXPIRY, - }); - res.send({ - 'token': token, - }); - } - }).catch((e) => { - console.log(e); - res.status(401).send(e); - }); - }; +function loginWithHeader(signKey, userFunction) { + return function(header){ + return function(req, res) { + // get the correct header, set it to use userFunction + let token = {"email": req.headers[header]}; + // login using that + userFunction(token).then((x) => { + if (x === false) { + res.status(401).send({ + 'err': 'User Unauthorized', + }); + } else { + data = x; + delete data['exp']; + // sign using the mounted key + var token = jwt.sign(data, signKey, { + algorithm: 'RS256', + expiresIn: EXPIRY, + }); + res.send({ + 'token': token, + }); + } + }).catch((e) => { + console.log(e); + res.status(401).send(e); + }); + }; + } } auth = {}; From 0d74b04defc30fc5bccf8fd42f0631aa9648fa99 Mon Sep 17 00:00:00 2001 From: Ryan Birmingham Date: Wed, 5 Oct 2022 14:15:00 -0400 Subject: [PATCH 6/9] try silencing add errs --- idx_mongo.js | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/idx_mongo.js b/idx_mongo.js index 5ee21f4..f740f61 100644 --- a/idx_mongo.js +++ b/idx_mongo.js @@ -3,6 +3,15 @@ const mongodb = require("./service/database"); +function quiet_mongo_add(db, collection, data, x){ + try{ + mongodb.add(db, collection, data, x); + } + catch(err){ + console.error(err) + } +} + function indexes() { db = "camic"; mongodb.createIndex(db, "user", {"email": 1}, {unique: true}); @@ -176,7 +185,7 @@ function defaults() { }, }; - mongodb.add(db, 'template', defaultTemplate, true); + quiet_mongo_add(db, 'template', defaultTemplate, true); var defaultConfigs = [{ "configuration": [ @@ -376,7 +385,7 @@ function defaults() { }]; - mongodb.add(db, 'configuration', defaultConfigs, true); + quiet_mongo_add(db, 'configuration', defaultConfigs, true); var defaultLinks = { "configuration": [ @@ -390,7 +399,7 @@ function defaults() { "config_name": "additional_links", "version": "1.0.0", }; - mongodb.add(db, 'configuration', defaultLinks, true); + quiet_mongo_add(db, 'configuration', defaultLinks, true); var evaluationForm = { "config_name": "evaluation_form", @@ -470,7 +479,7 @@ function defaults() { }, "version": "1.0.0", }; - mongodb.add(db, 'configuration', evaluationForm, true); + quiet_mongo_add(db, 'configuration', evaluationForm, true); } module.exports = { From a2a6cd13165b49108d485e1a22b943c026997031 Mon Sep 17 00:00:00 2001 From: Ryan Birmingham Date: Wed, 5 Oct 2022 14:44:59 -0400 Subject: [PATCH 7/9] try another catch --- caracal.js | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/caracal.js b/caracal.js index 0291103..cbb9ed6 100644 --- a/caracal.js +++ b/caracal.js @@ -229,10 +229,15 @@ function masterHandler() { }).then(()=>{ if (RUN_INDEXER) { const indexer = require('./idx_mongo.js'); + try{ indexer.collections(); indexer.indexes(); indexer.defaults(); console.log("added indexes"); + } + catch(e){ + console.log("error in indexer, ", e); + } } }).catch((e) => { console.error(e); From fbb4bbdebbf5bc757195fc139e73cb41eeceaefb Mon Sep 17 00:00:00 2001 From: Ryan Birmingham Date: Thu, 13 Oct 2022 15:23:22 -0400 Subject: [PATCH 8/9] Add Pathdb Secure IIP Handler (#102) --- caracal.js | 11 ++++- handlers/authHandlers.js | 1 + handlers/iipHandler.js | 26 ++++++++++- handlers/pathdbIipHandler.js | 60 ++++++++++++++++++++++++ package-lock.json | 88 +++++++++++++++++++++++++++++++++--- package.json | 1 + routes.json.example | 4 ++ 7 files changed, 180 insertions(+), 11 deletions(-) create mode 100644 handlers/pathdbIipHandler.js diff --git a/caracal.js b/caracal.js index cbb9ed6..6eb81ff 100644 --- a/caracal.js +++ b/caracal.js @@ -14,7 +14,8 @@ const fs = require('fs'); const auth = require('./handlers/authHandlers.js'); const monitor = require('./handlers/monitorHandlers.js'); const userFunction = require('./handlers/userFunction.js'); -const iipHandler = require('./handlers/iipHandler.js'); +const iipHandlers = require('./handlers/iipHandler.js'); +const pdbIipHandlers = require('./handlers/pathdbIipHandler.js'); const proxyHandler = require('./handlers/proxyHandler.js'); const permissionHandler = require('./handlers/permssionHandler.js'); const dataHandlers = require('./handlers/dataHandlers.js'); @@ -91,7 +92,13 @@ var HANDLERS = { "proxyHandler": proxyHandler, "writeFile": fileHandlers.writeFile, "iipHandler": function() { - return iipHandler; + return iipHandlers.iipHandler; + }, + "preIip": function() { + return iipHandlers.preIip; + }, + "iipCheck": function() { + return pdbIipHandlers.iipCheck; }, "markMulti": function() { return dataHandlers.Mark.multi; diff --git a/handlers/authHandlers.js b/handlers/authHandlers.js index 1e55336..26a2df8 100644 --- a/handlers/authHandlers.js +++ b/handlers/authHandlers.js @@ -301,6 +301,7 @@ auth.tokenTrade = tokenTrade; auth.filterHandler = filterHandler; auth.loginHandler = loginHandler; auth.editHandler = editHandler; +auth.getToken = getToken; auth.firstSetupUserSignupExists = firstSetupUserSignupExists; auth.loginWithHeader = loginWithHeader; auth.CLIENT = CLIENT; diff --git a/handlers/iipHandler.js b/handlers/iipHandler.js index d7cacd5..379d91f 100644 --- a/handlers/iipHandler.js +++ b/handlers/iipHandler.js @@ -2,7 +2,7 @@ var proxy = require('http-proxy-middleware'); var IIP_PATH = process.env.IIP_PATH || 'http://ca-iip/'; -iipHandler = function(req, res, next) { +preIip = function(req, res, next) { if (req.query) { if (req.query.DeepZoom) { if (req.query.DeepZoom.endsWith('.dzi')) { @@ -12,13 +12,25 @@ iipHandler = function(req, res, next) { // just in case _files is in the filename for some reason req.iipFileRequested = req.query.DeepZoom.split('_files').slice(0, -1).join('/'); } + } else if (req.query.IIIF) { + req.iipFileRequested = req.query.IIIF.split("/")[0]; } else if (req.query.FIF) { req.iipFileRequested = req.query.FIF; } else { req.iipFileRequested = false; } } + console.log(req.iipFileRequested); + next(); +}; + +function RemoveParameterFromUrl(url, parameter) { + return url + .replace(new RegExp('[?&]' + parameter + '=[^&#]*(#.*)?$'), '$1') + .replace(new RegExp('([?&])' + parameter + '=[^&]*&'), '$1'); +} +iipHandler = function(req, res, next) { proxy({ secure: false, onError(err, req, res) { @@ -29,8 +41,14 @@ iipHandler = function(req, res, next) { changeOrigin: true, target: IIP_PATH, pathRewrite: function(path, req) { + if (req.newFilepath) { + path = path.replace(req.iipFileRequested, req.newFilepath); + } + // remove token if present + path = RemoveParameterFromUrl(path, "token"); // NOTE -- this may need to change if the original url has more subdirs or so added var splitPath = path.split('/'); + console.log(path); return '/' + splitPath.slice(2, splitPath.length).join('/'); }, onProxyReq: function(proxyReq, req, res) { @@ -42,5 +60,9 @@ iipHandler = function(req, res, next) { })(req, res, next); }; +iipHandlers = {}; +iipHandlers.preIip = preIip; +iipHandlers.iipHandler = iipHandler; + -module.exports = iipHandler; +module.exports = iipHandlers; diff --git a/handlers/pathdbIipHandler.js b/handlers/pathdbIipHandler.js new file mode 100644 index 0000000..78a7eec --- /dev/null +++ b/handlers/pathdbIipHandler.js @@ -0,0 +1,60 @@ +// EXTENDS authHandlers +const proxy = require('http-proxy-middleware'); +var jwt = require('jsonwebtoken'); +var EXPIRY = process.env.EXPIRY || '1d'; +var BYPASS_IIP_CHECK = process.env.BYPASS_IIP_CHECK == "Y"; +const auth = require('./authHandlers.js'); +const fetch = require('cross-fetch'); + +// internal function to issue a new jwt +function issueToken(data, signKey) { + return jwt.sign(data, signKey, { + algorithm: 'RS256', + expiresIn: EXPIRY, + }); +} + +iipCheck = function(req, res, next) { + if (!BYPASS_IIP_CHECK) { + if (req.iipFileRequested) { + // rewrite path first + const PDB_URL = process.env.PDB_URL || 'http://quip-pathdb'; + let requestedNode = req.iipFileRequested.replace("pathdb*", ""); + let lookupUrl = PDB_URL + "/node/" + requestedNode + "?_format=json"; + console.log(lookupUrl); + let pdbReqHeaders = {"Authorization": "Bearer " + auth.getToken(req)}; + console.log(pdbReqHeaders); + fetch(lookupUrl, {headers: pdbReqHeaders}).then((x)=>x.json()).then((x)=>{ + console.log(x); + // get path + if (x && x['field_iip_path'] && x['field_iip_path'].length && x['field_iip_path'][0]['value']) { + req.newFilepath = x['field_iip_path'][0]['value']; + console.log(req.newFilepath); + next(); + } else { + let err = {}; + err.message = "unauthorized slide request"; + err.statusCode = 401; + next(err); + } + }).catch((e)=>{ + console.error(e); + next(e); + }); + } else { + // do not return + let err = {}; + err.message = "unauthorized slide request"; + err.statusCode = 401; + next(err); + } + } else { + // NOTE -- this instead uses the actual value given instead + next(); + } +}; + +let pih = {}; +pih.iipCheck = iipCheck; + +module.exports = pih; diff --git a/package-lock.json b/package-lock.json index 51cd2c0..991e9d0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -284,17 +284,17 @@ "integrity": "sha512-Wm6ukoaOGJi/73p/cl2GvLjTI5JM1k/O14isD73YML8StrH/7/lRFgmg8nICZgD3bZZvjwCGxtMOD3wWNAu8cg==" }, "axios": { - "version": "0.21.4", - "resolved": "https://registry.npmjs.org/axios/-/axios-0.21.4.tgz", - "integrity": "sha512-ut5vewkiu8jjGBdqpM44XxjuCjq9LAKeHVmoVfHVzy8eHgxxq8SbAVQNovDA8mVi05kP0Ea/n/UzcSHcTJQfNg==", + "version": "0.26.0", + "resolved": "https://registry.npmjs.org/axios/-/axios-0.26.0.tgz", + "integrity": "sha512-lKoGLMYtHvFrPVt3r+RBMp9nh34N0M8zEfCWqdWZx6phynIEhQqAdydpyBAAG211zlhX9Rgu08cOamy6XjE5Og==", "requires": { - "follow-redirects": "^1.14.0" + "follow-redirects": "^1.14.8" }, "dependencies": { "follow-redirects": { - "version": "1.14.4", - "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.4.tgz", - "integrity": "sha512-zwGkiSXC1MUJG/qmeIFH2HBJx9u0V46QGUe3YR1fXG8bXQxq7fLj0RjLZQ5nubr9qNJUZrH+xUcwXEoXNpfS+g==" + "version": "1.14.9", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.9.tgz", + "integrity": "sha512-MQDfihBQYMcyy5dhRDJUHcw7lb2Pv/TuE6xP1vyraLukNDHKbDxDNaOE3NbCAdKQApno+GPRyo1YAp89yCjK4w==" } } }, @@ -572,6 +572,14 @@ "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=" }, + "cross-fetch": { + "version": "3.1.5", + "resolved": "https://registry.npmjs.org/cross-fetch/-/cross-fetch-3.1.5.tgz", + "integrity": "sha512-lvb1SBsI0Z7GDwmuid+mU3kWVBwTVUbe7S0H52yaaAdQOXq2YktTCZdlAcNKFzE6QtRz0snpw9bNiPeOIkkQvw==", + "requires": { + "node-fetch": "2.6.7" + } + }, "cross-spawn": { "version": "6.0.5", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-6.0.5.tgz", @@ -1526,6 +1534,14 @@ "debug": "4" } }, + "axios": { + "version": "0.21.4", + "resolved": "https://registry.npmjs.org/axios/-/axios-0.21.4.tgz", + "integrity": "sha512-ut5vewkiu8jjGBdqpM44XxjuCjq9LAKeHVmoVfHVzy8eHgxxq8SbAVQNovDA8mVi05kP0Ea/n/UzcSHcTJQfNg==", + "requires": { + "follow-redirects": "^1.14.0" + } + }, "debug": { "version": "4.3.1", "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.1.tgz", @@ -1541,6 +1557,11 @@ } } }, + "follow-redirects": { + "version": "1.14.9", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.9.tgz", + "integrity": "sha512-MQDfihBQYMcyy5dhRDJUHcw7lb2Pv/TuE6xP1vyraLukNDHKbDxDNaOE3NbCAdKQApno+GPRyo1YAp89yCjK4w==" + }, "https-proxy-agent": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.0.tgz", @@ -1960,6 +1981,40 @@ "integrity": "sha512-1nh45deeb5olNY7eX82BkPO7SSxR5SSYJiPTrTdFUVYwAl8CKMA5N9PjTYkHiRjisVcxcQ1HXdLhx2qxxJzLNQ==", "dev": true }, + "node-fetch": { + "version": "2.6.7", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.7.tgz", + "integrity": "sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ==", + "requires": { + "whatwg-url": "^5.0.0" + } + }, + "node-pre-gyp": { + "version": "0.14.0", + "resolved": "https://registry.npmjs.org/node-pre-gyp/-/node-pre-gyp-0.14.0.tgz", + "integrity": "sha512-+CvDC7ZttU/sSt9rFjix/P05iS43qHCOOGzcr3Ry99bXG7VX953+vFyEuph/tfqoYu8dttBkE86JSKBO2OzcxA==", + "requires": { + "detect-libc": "^1.0.2", + "mkdirp": "^0.5.1", + "needle": "^2.2.1", + "nopt": "^4.0.1", + "npm-packlist": "^1.1.6", + "npmlog": "^4.0.2", + "rc": "^1.2.7", + "rimraf": "^2.6.1", + "semver": "^5.3.0", + "tar": "^4.4.2" + } + }, + "nopt": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/nopt/-/nopt-4.0.3.tgz", + "integrity": "sha512-CvaGwVMztSMJLOeXPrez7fyfObdZqNUK1cPAEzLHrTybIua9pMdmmPR5YwtfNftIOMv3DPUhFaxsZMNTQO20Kg==", + "requires": { + "abbrev": "1", + "osenv": "^0.1.4" + } + }, "normalize-path": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", @@ -2589,6 +2644,11 @@ "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.0.tgz", "integrity": "sha512-yaOH/Pk/VEhBWWTlhI+qXxDFXlejDGcQipMlyxda9nthulaxLZUNcUqFxokp0vcYnvteJln5FNQDRrxj3YcbVw==" }, + "tr46": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", + "integrity": "sha1-gYT9NH2snNwYWZLzpmIuFLnZq2o=" + }, "tslib": { "version": "1.11.1", "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.11.1.tgz", @@ -2664,11 +2724,25 @@ "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", "integrity": "sha1-IpnwLG3tMNSllhsLn3RSShj2NPw=" }, + "webidl-conversions": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", + "integrity": "sha1-JFNCdeKnvGvnvIZhHMFq4KVlSHE=" + }, "webworkify": { "version": "1.5.0", "resolved": "https://registry.npmjs.org/webworkify/-/webworkify-1.5.0.tgz", "integrity": "sha512-AMcUeyXAhbACL8S2hqqdqOLqvJ8ylmIbNwUIqQujRSouf4+eUFaXbG6F1Rbu+srlJMmxQWsiU7mOJi0nMBfM1g==" }, + "whatwg-url": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", + "integrity": "sha1-lmRU6HZUYuN2RNNib2dCzotwll0=", + "requires": { + "tr46": "~0.0.3", + "webidl-conversions": "^3.0.0" + } + }, "which": { "version": "1.3.1", "resolved": "https://registry.npmjs.org/which/-/which-1.3.1.tgz", diff --git a/package.json b/package.json index ad5d43c..49bee32 100644 --- a/package.json +++ b/package.json @@ -24,6 +24,7 @@ "ajv": "^8.6.0", "ajv-keywords": "^5.0.0", "atob": "^2.1.2", + "cross-fetch": "^3.1.5", "dotenv": "^8.6.0", "express": "^4.17.1", "helmet": "^4.6.0", diff --git a/routes.json.example b/routes.json.example index 534364a..fd8fbf8 100644 --- a/routes.json.example +++ b/routes.json.example @@ -79,6 +79,10 @@ "function": "loginHandler", "args": [] }, + { + "function": "preIip", + "args": [] + }, { "function": "iipHandler", "args": [] From f6ce363e9bb4fbccf89085610ebd049643476cae Mon Sep 17 00:00:00 2001 From: Ryan Birmingham Date: Thu, 13 Oct 2022 15:36:30 -0400 Subject: [PATCH 9/9] no tfjs, back to alpine --- Dockerfile | 4 +- caracal.js | 45 +++------ handlers/authHandlers.js | 4 +- handlers/datasetHandler.js | 164 -------------------------------- handlers/iipHandler.js | 8 +- handlers/modelTrainer.js | 187 ------------------------------------- idx_mongo.js | 17 ++-- 7 files changed, 31 insertions(+), 398 deletions(-) delete mode 100644 handlers/datasetHandler.js delete mode 100644 handlers/modelTrainer.js diff --git a/Dockerfile b/Dockerfile index f5366fc..21441e8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,11 +1,11 @@ -FROM node:16-stretch-slim -RUN apt-get update && apt-get upgrade -y && apt-get install -y git build-essential python3 +FROM node:14-alpine RUN mkdir /src COPY . /src WORKDIR /src RUN npm install ARG viewer ARG fork +RUN apk add --no-cache git RUN git clone https://github.com/${fork:-camicroscope}/camicroscope.git --branch=${viewer:-master} EXPOSE 4010 diff --git a/caracal.js b/caracal.js index 6eb81ff..a06c1c7 100644 --- a/caracal.js +++ b/caracal.js @@ -23,14 +23,6 @@ const fileHandlers = require('./handlers/fileHandlers.js'); const sanitizeBody = require('./handlers/sanitizeHandler.js'); const DataTransformationHandler = require('./handlers/dataTransformationHandler.js'); -// TODO -- make optional -const DISABLE_TF = true; // DUE TO PRODUCTION STABILITY ISSUES WITH TFJS - -if (!DISABLE_TF) { - const DataSet = require('./handlers/datasetHandler.js'); - const Model = require('./handlers/modelTrainer.js'); -} - const {connector} = require("./service/database/connector"); @@ -129,22 +121,16 @@ var HANDLERS = { }, }; -if (!DISABLE_TF) { - HANDLERS["getDataset"] = DataSet.getDataset; - HANDLERS["trainModel"] = Model.trainModel; - HANDLERS["deleteDataset"] = DataSet.deleteData; - HANDLERS["sendTrainedModel"] = Model.sendTrainedModel; -} else { - function disabledRoute() { - return function(req, res) { - res.status(500).send('{"err":"This TF route is disabled"}'); - }; - } - HANDLERS["getDataset"] = disabledRoute; - HANDLERS["trainModel"] = disabledRoute; - HANDLERS["deleteDataset"] = disabledRoute; - HANDLERS["sendTrainedModel"] = disabledRoute; +// TODO! -- remove these by fully depreciating tfjs serverside +function disabledRoute() { + return function(req, res) { + res.status(500).send('{"err":"This TF route is disabled"}'); + }; } +HANDLERS["getDataset"] = disabledRoute; +HANDLERS["trainModel"] = disabledRoute; +HANDLERS["deleteDataset"] = disabledRoute; +HANDLERS["sendTrainedModel"] = disabledRoute; // register configurable services // TODO verify all @@ -236,13 +222,12 @@ function masterHandler() { }).then(()=>{ if (RUN_INDEXER) { const indexer = require('./idx_mongo.js'); - try{ - indexer.collections(); - indexer.indexes(); - indexer.defaults(); - console.log("added indexes"); - } - catch(e){ + try { + indexer.collections(); + indexer.indexes(); + indexer.defaults(); + console.log("added indexes"); + } catch (e) { console.log("error in indexer, ", e); } } diff --git a/handlers/authHandlers.js b/handlers/authHandlers.js index 26a2df8..fec64af 100644 --- a/handlers/authHandlers.js +++ b/handlers/authHandlers.js @@ -265,7 +265,7 @@ function firstSetupUserSignupExists() { // Use a trusted header instead of a jwt for login. Use carefully if at all. function loginWithHeader(signKey, userFunction) { - return function(header){ + return function(header) { return function(req, res) { // get the correct header, set it to use userFunction let token = {"email": req.headers[header]}; @@ -292,7 +292,7 @@ function loginWithHeader(signKey, userFunction) { res.status(401).send(e); }); }; - } + }; } auth = {}; diff --git a/handlers/datasetHandler.js b/handlers/datasetHandler.js deleted file mode 100644 index 726332e..0000000 --- a/handlers/datasetHandler.js +++ /dev/null @@ -1,164 +0,0 @@ -const tf = require('@tensorflow/tfjs-node'); - -// enable this and comment previous one for GPU (CUDA) training -- -// proper nvidia drivers needs to be installed on both base machine and container for gpu training -// const tf = require('@tensorflow/tfjs-node-gpu'); - -const fs = require('fs'); -const inkjet = require('inkjet'); -const crypto = require("crypto"); -const AdmZip = require('adm-zip'); -const path = require('path'); - -let LABELS_PATH = null; -let IMAGES_SPRITE_PATH = null; - - -class Data { - constructor() { - this.IMAGE_SIZE = null; - this.NUM_CLASSES = null; - this.NUM_TRAIN_ELEMENTS = null; - this.NUM_TEST_ELEMENTS = null; - this.IMAGES_SPRITE_PATH = IMAGES_SPRITE_PATH; - this.NUM_CHANNELS = null; // 1 for grayscale; 4 for rgba - this.LABELS_PATH = LABELS_PATH; - this.shuffledTrainIndex = null; - this.shuffledTestIndex = null; - this.datasetImages = null; - this.xs = null; - this.labels = null; - } - async load() { - let This = this; - const imgRequest = new Promise((resolve) => { - // console.log(this.IMAGES_SPRITE_PATH); - inkjet.decode(fs.readFileSync(this.IMAGES_SPRITE_PATH), function(err, decoded) { - const pixels = Float32Array.from(decoded.data).map((pixel) => { - return pixel / 255; - }); - This.datasetImages = pixels; - resolve(); - }); - }); - - let labelsRequest = fs.readFileSync(this.LABELS_PATH); - const [imgResponse, labelsResponse] = await Promise.all([ - imgRequest, - labelsRequest, - ]); - - this.datasetLabels = new Uint8Array(labelsResponse); - - // Create shuffled indices into the train/test set for when we select a - // random dataset element for training / validation. - this.trainIndices = tf.util.createShuffledIndices(this.NUM_TRAIN_ELEMENTS); - this.testIndices = tf.util.createShuffledIndices(this.NUM_TEST_ELEMENTS); - - // Slice the the images and labels into train and test sets. - this.trainImages = this.datasetImages.slice( - 0, - this.IMAGE_SIZE * this.NUM_TRAIN_ELEMENTS * this.NUM_CHANNELS, - ); - this.testImages = this.datasetImages.slice( - this.IMAGE_SIZE * this.NUM_TRAIN_ELEMENTS * this.NUM_CHANNELS, - ); - this.trainLabels = this.datasetLabels.slice( - 0, - this.NUM_CLASSES * this.NUM_TRAIN_ELEMENTS, - ); - this.testLabels = this.datasetLabels.slice( - this.NUM_CLASSES * this.NUM_TRAIN_ELEMENTS, - ); - } - - nextTrainBatch(batchSize) { - return this.nextBatch( - batchSize, - [this.trainImages, this.trainLabels], - () => { - this.shuffledTrainIndex = - (this.shuffledTrainIndex + 1) % this.trainIndices.length; - return this.trainIndices[this.shuffledTrainIndex]; - }, - ); - } - - nextTestBatch(batchSize) { - return this.nextBatch(batchSize, [this.testImages, this.testLabels], () => { - this.shuffledTestIndex = - (this.shuffledTestIndex + 1) % this.testIndices.length; - return this.testIndices[this.shuffledTestIndex]; - }); - } - - nextBatch(batchSize, data, index) { - const batchImagesArray = new Float32Array( - batchSize * this.IMAGE_SIZE * this.NUM_CHANNELS, - ); - const batchLabelsArray = new Uint8Array(batchSize * this.NUM_CLASSES); - - for (let i = 0; i < batchSize; i++) { - const idx = index(); - const image = data[0].slice( - idx * this.IMAGE_SIZE * this.NUM_CHANNELS, - idx * this.IMAGE_SIZE * this.NUM_CHANNELS + this.IMAGE_SIZE * this.NUM_CHANNELS, - ); - batchImagesArray.set(image, i * this.IMAGE_SIZE * this.NUM_CHANNELS); - - const label = data[1].slice( - idx * this.NUM_CLASSES, - idx * this.NUM_CLASSES + this.NUM_CLASSES, - ); - batchLabelsArray.set(label, i * this.NUM_CLASSES); - } - - this.xs = tf.tensor3d(batchImagesArray, [ - batchSize, - this.IMAGE_SIZE, - this.NUM_CHANNELS, - ]); - this.labels = tf.tensor2d(batchLabelsArray, [batchSize, this.NUM_CLASSES]).toFloat(); - return {xs: this.xs, labels: this.labels}; - } -} - -function getDataset() { - return function(req, res) { - let data = JSON.parse(req.body); - // console.log(req.body.ar); - let userFolder = crypto.randomBytes(20).toString('hex'); - if (!fs.existsSync('dataset')) { - fs.mkdirSync('dataset/'); - } - fs.mkdirSync('dataset/' + userFolder); - fs.writeFile('dataset/' + userFolder + '/dataset.zip', data.file, - {encoding: 'base64'}, - async function(err) { - let zip = new AdmZip('dataset/' + userFolder + '/dataset.zip'); - await zip.extractAllTo('dataset/' + userFolder, true); - LABELS_PATH = 'dataset/' + userFolder + '/labels.bin'; - IMAGES_SPRITE_PATH = 'dataset/' + userFolder + '/data.jpg'; - fs.unlink('dataset/' + userFolder + '/dataset.zip', () => {}); - res.json({status: 'DONE', userFolder: userFolder}); - }, - ); - }; -} - -function deleteData() { - return function(req, res) { - let data = JSON.parse(req.body); - let dir = path.normalize(data.userFolder).replace(/^(\.\.(\/|\\|$))+/, ''); - dir = path.join('./dataset/', dir); - fs.rmdir(dir, {recursive: true}, (err) => { - if (err) { - throw err; - } - console.log(`Temp folder deleted!`); - }); - res.json({status: 'Temp folder deleted!'}); - }; -} - -module.exports = {Data: Data, getDataset: getDataset, deleteData: deleteData}; diff --git a/handlers/iipHandler.js b/handlers/iipHandler.js index 379d91f..8049fb7 100644 --- a/handlers/iipHandler.js +++ b/handlers/iipHandler.js @@ -24,10 +24,10 @@ preIip = function(req, res, next) { next(); }; -function RemoveParameterFromUrl(url, parameter) { +function removeParameterFromUrl(url, parameter) { return url - .replace(new RegExp('[?&]' + parameter + '=[^&#]*(#.*)?$'), '$1') - .replace(new RegExp('([?&])' + parameter + '=[^&]*&'), '$1'); + .replace(new RegExp('[?&]' + parameter + '=[^&#]*(#.*)?$'), '$1') + .replace(new RegExp('([?&])' + parameter + '=[^&]*&'), '$1'); } iipHandler = function(req, res, next) { @@ -45,7 +45,7 @@ iipHandler = function(req, res, next) { path = path.replace(req.iipFileRequested, req.newFilepath); } // remove token if present - path = RemoveParameterFromUrl(path, "token"); + path = removeParameterFromUrl(path, "token"); // NOTE -- this may need to change if the original url has more subdirs or so added var splitPath = path.split('/'); console.log(path); diff --git a/handlers/modelTrainer.js b/handlers/modelTrainer.js deleted file mode 100644 index 123b8a2..0000000 --- a/handlers/modelTrainer.js +++ /dev/null @@ -1,187 +0,0 @@ -const tf = require('@tensorflow/tfjs-node'); - -// enable this and comment previous one for GPU (CUDA) training -- -// proper nvidia drivers needs to be installed on both base machine and container for gpu training -// const tf = require('@tensorflow/tfjs-node-gpu'); - -const Data = require('./datasetHandler.js'); -const AdmZip = require('adm-zip'); - -let Layers = []; -let Params = {}; - -function getModel(Layers, Params, res) { - // console.log(Params) - let model; - try { - model = tf.sequential({ - layers: Layers, - }); - - if (Params.advancedMode) { - model.compile({ - optimizer: Params.optimizer, - loss: Params.modelCompileLoss, - metrics: Params.modelCompileMetrics, - }); - } else { - model.compile({ - optimizer: Params.optimizer, - loss: 'categoricalCrossentropy', - metrics: ['accuracy'], - }); - } - } catch (error) { - res.status(400).json({message: error.message}); - // res.send(error); - } - - return model; -} - -async function train(model, data, Params) { - let TRAIN_DATA_SIZE = Params.trainDataSize; - let TEST_DATA_SIZE = Params.testDataSize; - let WIDTH = Params.width; - let HEIGHT = Params.height; - let d1; let d2; - const [trainXs, trainYs] = tf.tidy(() => { - d1 = data.nextTrainBatch(TRAIN_DATA_SIZE); - return [ - d1.xs.reshape([TRAIN_DATA_SIZE, HEIGHT, WIDTH, data.NUM_CHANNELS]), - d1.labels, - ]; - }); - - const [testXs, testYs] = tf.tidy(() => { - d2 = data.nextTestBatch(TEST_DATA_SIZE); - return [ - d2.xs.reshape([TEST_DATA_SIZE, HEIGHT, WIDTH, data.NUM_CHANNELS]), - d2.labels, - ]; - }); - - return model.fit(trainXs, trainYs, { - batchSize: Number(Params.batchSize), - validationData: [testXs, testYs], - epochs: Number(Params.epochs), - shuffle: Params.shuffle, - // callbacks: console.log(1), - }).then(() => { - tf.dispose([trainXs, trainYs, testXs, testYs, d2, d1]); - }); -} - -async function run(Layers, Params, res, userFolder) { - let model; let trained; let data; - try { - data = new Data.Data(); - data.IMAGE_SIZE = Params.height * Params.width; - data.NUM_CLASSES = Params.numClasses; - data.NUM_CHANNELS = Params.numChannels; - data.NUM_TEST_ELEMENTS = Params.testDataSize; - data.NUM_TRAIN_ELEMENTS = Params.trainDataSize; - - await data.load(); - - model = getModel(Layers, Params, res); - - model.summary(); - - trained = await train(model, data, Params); - console.log('TRAINING DONE'); - - await model.save('file://./dataset/' + userFolder + '/'); - - tf.dispose([model, trained, data.xs, data.labels]); - tf.disposeVariables(); - - let zip = new AdmZip(); - zip.addLocalFile('./dataset/' + userFolder + '/model.json'); - zip.addLocalFile('./dataset/' + userFolder + '/weights.bin'); - zip.writeZip('./dataset/' + userFolder + '/' + Params.modelName + '.zip'); - - res.json({status: 'done'}); - } catch (error) { - console.log(error); - tf.dispose([model, trained, data.xs, data.labels]); - tf.disposeVariables(); - res.status(400).json({message: error.message}); - } -} - -function makeLayers(layers, res, userFolder) { - delete layers[0].layer; - delete layers[layers.length - 1].layer; - Layers = [ - tf.layers.conv2d(layers[0]), - tf.layers.dense(layers[layers.length - 1]), - ]; - try { - for (let i = 1; i < layers.length - 1; i++) { - if (layers[i].layer == 'dense') { - delete layers[i].layer; - Layers.splice(Layers.length - 1, 0, tf.layers.dense(layers[i])); - } else if (layers[i].layer == 'conv2d') { - delete layers[i].layer; - Layers.splice(Layers.length - 1, 0, tf.layers.conv2d(layers[i])); - } else if (layers[i].layer == 'flatten') { - delete layers[i].layer; - Layers.splice(Layers.length - 1, 0, tf.layers.flatten(layers[i])); - } else if (layers[i].layer == 'batchNormalization') { - delete layers[i].layer; - Layers.splice(Layers.length - 1, 0, tf.layers.batchNormalization(layers[i])); - } else if (layers[i].layer == 'dropout') { - delete layers[i].layer; - Layers.splice(Layers.length - 1, 0, tf.layers.dropout(layers[i])); - } else if (layers[i].layer == 'maxpooling2d') { - delete layers[i].layer; - Layers.splice(Layers.length - 1, 0, tf.layers.maxPooling2d(layers[i])); - } else if (layers[i].layer == 'activation') { - delete layers[i].layer; - Layers.splice(Layers.length - 1, 0, tf.layers.activation(layers[i])); - } else if (layers[i].layer == 'conv2dTranspose') { - delete layers[i].layer; - Layers.splice(Layers.length - 1, 0, tf.layers.conv2dTranspose(layers[i])); - } else if (layers[i].layer == 'averagePooling2d') { - delete layers[i].layer; - Layers.splice(Layers.length - 1, 0, tf.layers.averagePooling2d(layers[i])); - } else if (layers[i].layer == 'globalAveragePooling2d') { - delete layers[i].layer; - Layers.splice(Layers.length - 1, 0, tf.layers.globalAveragePooling2d(layers[i])); - } else if (layers[i].layer == 'globalMaxPooling2d') { - delete layers[i].layer; - Layers.splice(Layers.length - 1, 0, tf.layers.globalMaxPooling2d(layers[i])); - } - // console.log(layers[i]); - } - } catch (error) { - console.log(error); - res.status(400).json({message: error.message}); - return; - } - - run(Layers, Params, res, userFolder); -} - -function trainModel() { - return function(req, res) { - let data = JSON.parse(req.body); - Params = data.Params; - makeLayers(data.Layers, res, data.userFolder); - }; -} - -function sendTrainedModel() { - return function(req, res) { - let data = JSON.parse(req.body); - let downloadURL = '/workbench/download/' + data.userFolder; - let app = require('../caracal.js'); - app.get(downloadURL, (req1, res1) => - res1.download('./dataset/' + data.userFolder + '/' + data.Params.modelName + '.zip'), - ); - res.json({url: downloadURL}); - }; -} - -module.exports = {trainModel: trainModel, sendTrainedModel: sendTrainedModel}; diff --git a/idx_mongo.js b/idx_mongo.js index f740f61..c2a2b0b 100644 --- a/idx_mongo.js +++ b/idx_mongo.js @@ -3,12 +3,11 @@ const mongodb = require("./service/database"); -function quiet_mongo_add(db, collection, data, x){ - try{ +function quietMongoAdd(db, collection, data, x) { + try { mongodb.add(db, collection, data, x); - } - catch(err){ - console.error(err) + } catch (err) { + console.error(err); } } @@ -185,7 +184,7 @@ function defaults() { }, }; - quiet_mongo_add(db, 'template', defaultTemplate, true); + quietMongoAdd(db, 'template', defaultTemplate, true); var defaultConfigs = [{ "configuration": [ @@ -385,7 +384,7 @@ function defaults() { }]; - quiet_mongo_add(db, 'configuration', defaultConfigs, true); + quietMongoAdd(db, 'configuration', defaultConfigs, true); var defaultLinks = { "configuration": [ @@ -399,7 +398,7 @@ function defaults() { "config_name": "additional_links", "version": "1.0.0", }; - quiet_mongo_add(db, 'configuration', defaultLinks, true); + quietMongoAdd(db, 'configuration', defaultLinks, true); var evaluationForm = { "config_name": "evaluation_form", @@ -479,7 +478,7 @@ function defaults() { }, "version": "1.0.0", }; - quiet_mongo_add(db, 'configuration', evaluationForm, true); + quietMongoAdd(db, 'configuration', evaluationForm, true); } module.exports = {