From b1d28a62314c343be9189c6910c8bb1e73c41c08 Mon Sep 17 00:00:00 2001 From: chughts Date: Wed, 31 May 2017 08:43:40 +0100 Subject: [PATCH 01/10] Bump dependency on watson-developer-cloud --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index aa654ebe..6f09a67d 100644 --- a/package.json +++ b/package.json @@ -11,7 +11,7 @@ "temp": "^0.8.3", "qs": "6.x", "image-type": "^2.0.2", - "watson-developer-cloud": "^2.31.2", + "watson-developer-cloud": "^2.32.1", "kuromoji": "^0.0.5", "is-docx": "^0.0.3" }, From d55c3694b391e5f43df83aeb0c15f82616af998c Mon Sep 17 00:00:00 2001 From: chughts Date: Sun, 4 Jun 2017 08:34:19 +0100 Subject: [PATCH 02/10] Error object messsage in err.message --- utilities/payload-utils.js | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/utilities/payload-utils.js b/utilities/payload-utils.js index 4c7ce480..779d71ca 100644 --- a/utilities/payload-utils.js +++ b/utilities/payload-utils.js @@ -90,8 +90,16 @@ PayloadUtils.prototype = { }, reportError: function (node, msg, message) { - var messageTxt = message.error ? message.error : message; - + var messageTxt = err; + + if (err.error) { + messageTxt = err.error; + } else if (err.description) { + messageTxt = err.description; + } else if (err.message) { + messageTxt = err.message; + } + msg.watsonerror = messageTxt; node.status({fill:'red', shape:'dot', text: messageTxt}); From f489ce6fc41e8362fc0da4a2c9d92e4c56891231 Mon Sep 17 00:00:00 2001 From: chughts Date: Sun, 4 Jun 2017 08:35:37 +0100 Subject: [PATCH 03/10] Bump version number --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 6f09a67d..fc06a671 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "node-red-node-watson", - "version": "0.5.7", + "version": "0.5.8", "description": "A collection of Node-RED nodes for IBM Watson services", "dependencies": { "alchemy-api": "^1.3.0", From a38dc6b1b17801ec6fc18a566c254e7363c7f6bc Mon Sep 17 00:00:00 2001 From: chughts Date: Sun, 4 Jun 2017 14:27:32 +0100 Subject: [PATCH 04/10] visual recognition to accept stream on msg.payload --- README.md | 3 + package.json | 3 +- services/visual_recognition/v3.js | 257 ++++++++++++++++-------------- utilities/payload-utils.js | 12 +- 4 files changed, 148 insertions(+), 127 deletions(-) diff --git a/README.md b/README.md index 261cfd33..f9e04207 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,9 @@ Node-RED Watson Nodes for IBM Bluemix CLA assistant +### New in version 0.5.8 +- Visual Reconition Node, now accepts readstream on msg.payload + ### New in version 0.5.7 - Fix to Tone Analyzer to allow JSON as input - Enabled Conversation Tone method to Tone Analyzer Node diff --git a/package.json b/package.json index fc06a671..82cdfef6 100644 --- a/package.json +++ b/package.json @@ -13,7 +13,8 @@ "image-type": "^2.0.2", "watson-developer-cloud": "^2.32.1", "kuromoji": "^0.0.5", - "is-docx": "^0.0.3" + "is-docx": "^0.0.3", + "stream-to-array" : "^2.3.0" }, "repository": { "type": "git", diff --git a/services/visual_recognition/v3.js b/services/visual_recognition/v3.js index a15b42cb..626d6c12 100644 --- a/services/visual_recognition/v3.js +++ b/services/visual_recognition/v3.js @@ -19,6 +19,7 @@ module.exports = function(RED) { const VisualRecognitionV3 = require('watson-developer-cloud/visual-recognition/v3'); var pkg = require('../../package.json'), serviceutils = require('../../utilities/service-utils'), + payloadutils = require('../../utilities/payload-utils'), //watson = require('watson-developer-cloud'), imageType = require('image-type'), url = require('url'), @@ -26,7 +27,8 @@ module.exports = function(RED) { fileType = require('file-type'), fs = require('fs'), async = require('async'), - sAPIKey = null, + toArray = require('stream-to-array') + sAPIKey = null, service = null; // temp is being used for file streaming to allow the file to arrive so it can be processed. @@ -72,17 +74,45 @@ module.exports = function(RED) { function verifyInputs(feature, msg) { switch (feature) { - case 'classifyImage': - case 'detectFaces': - case 'recognizeText': - if (typeof msg.payload === 'boolean' || typeof msg.payload === 'number') { - return Promise.reject('Bad format : msg.payload must be a URL string or a Node.js Buffer'); - } - break; + case 'classifyImage': + case 'detectFaces': + case 'recognizeText': + if (typeof msg.payload === 'boolean' || + typeof msg.payload === 'number') { + return Promise.reject('Bad format : msg.payload must be a URL string or a Node.js Buffer'); + } + break; } return Promise.resolve(); } + + // Even though Visual Reconition SDK can accept a filestream as input + // it can't handle the one on msg.payload, so read it into a buffer + function checkForStream(msg) { + var p = new Promise(function resolver(resolve, reject) { + if (payloadutils.isReadableStream(msg.payload)) { + msg.payload.resume(); + toArray(msg.payload) + .then(function(parts) { + var buffers = []; + + for (var i = 0; i < parts.length; ++i) { + var part = parts[i]; + buffers.push((part instanceof Buffer) ? part : new Buffer(part)); + } + msg.payload = Buffer.concat(buffers); + resolve(); + + }); + } else { + resolve(); + } + }); + return p; + + } + function verifyServiceCredentials(node, msg) { // If it is present the newly provided user entered key // takes precedence over the existing one. @@ -118,8 +148,26 @@ module.exports = function(RED) { } } + function setCommonParams(node, msg, params) { + if (msg.params != null && msg.params.classifier_ids != null) { + params['classifier_ids'] = msg.params['classifier_ids']; + } + if (msg.params != null && msg.params.owners != null) { + params['owners'] = msg.params['owners']; + } + if (msg.params != null && msg.params.threshold != null) { + params['threshold'] = msg.params['threshold']; + } + if (node.config != null && node.config.lang != null) { + params['Accept-Language'] = node.config.lang; + } + if (msg.params != null && msg.params.accept_language != null) { + params['Accept-Language'] = msg.params['accept_language']; + } + } + function prepareCommonParams(params, node, msg) { - var p = new Promise(function resolver(resolve, reject){ + var p = new Promise(function resolver(resolve, reject) { if (imageCheck(msg.payload)) { var ft = fileType(msg.payload); var ext = ft ? ft.ext : 'tmp'; @@ -131,41 +179,13 @@ module.exports = function(RED) { } stream_buffer(info.path, msg.payload, function() { params['images_file'] = fs.createReadStream(info.path); - if (msg.params != null && msg.params.classifier_ids != null) { - params['classifier_ids'] = msg.params['classifier_ids']; - } - if (msg.params != null && msg.params.owners != null) { - params['owners'] = msg.params['owners']; - } - if (msg.params != null && msg.params.threshold != null) { - params['threshold'] = msg.params['threshold']; - } - if (node.config != null && node.config.lang != null) { - params['Accept-Language'] = node.config.lang; - } - if (msg.params != null && msg.params.accept_language != null) { - params['Accept-Language'] = msg.params['accept_language']; - } + setCommonParams(node, msg, params); resolve(); }); }); } else if (urlCheck(msg.payload)) { params['url'] = msg.payload; - if (msg.params != null && msg.params.classifier_ids != null) { - params['classifier_ids'] = msg.params['classifier_ids']; - } - if (msg.params != null && msg.params.owners != null) { - params['owners'] = msg.params['owners']; - } - if (msg.params != null && msg.params.threshold != null) { - params['threshold'] = msg.params['threshold']; - } - if (node.config != null && node.config.lang != null) { - params['Accept-Language'] = node.config.lang; - } - if (msg.params != null && msg.params.accept_language != null) { - params['Accept-Language'] = msg.params['accept_language']; - } + setCommonParams(node, msg, params); resolve(); } else { reject('Payload must be either an image buffer or a string representing a url'); @@ -216,7 +236,7 @@ module.exports = function(RED) { // that contains a minimum of 10 images.(Optional) function prepareParamsCreateClassifier(params, node, msg) { - var p = new Promise(function resolver(resolve, reject){ + var p = new Promise(function resolver(resolve, reject) { var listParams = {}, asyncTasks = [], k = null; @@ -246,7 +266,7 @@ module.exports = function(RED) { } function performDeleteAllClassifiers(params, node, msg) { - var p = new Promise(function resolver(resolve, reject){ + var p = new Promise(function resolver(resolve, reject) { node.service.listClassifiers(params, function(err, body) { node.status({}); if (err) { @@ -288,35 +308,35 @@ module.exports = function(RED) { } function invokeService(feature, params, node, msg) { - var p = new Promise(function resolver(resolve, reject){ + var p = new Promise(function resolver(resolve, reject) { switch (feature) { - case 'classifyImage': - node.service.classify(params, function(err, body) { - if (err) { - reject(err); - } else { - resolve(body); - } - }); - break; - case 'detectFaces': - node.service.detectFaces(params, function(err, body) { - if (err) { - reject(err); - } else { - resolve(body); - } - }); - break; - case 'recognizeText': - node.service.recognizeText(params, function(err, body) { - if (err) { - reject(err); - } else { - resolve(body); - } - }); - break; + case 'classifyImage': + node.service.classify(params, function(err, body) { + if (err) { + reject(err); + } else { + resolve(body); + } + }); + break; + case 'detectFaces': + node.service.detectFaces(params, function(err, body) { + if (err) { + reject(err); + } else { + resolve(body); + } + }); + break; + case 'recognizeText': + node.service.recognizeText(params, function(err, body) { + if (err) { + reject(err); + } else { + resolve(body); + } + }); + break; } }); return p; @@ -324,17 +344,17 @@ module.exports = function(RED) { function executeService(feature, params, node, msg) { var p = prepareCommonParams(params, node, msg) - .then(function(){ + .then(function() { return invokeService(feature, params, node, msg); }) - .then(function(body){ + .then(function(body) { return processTheResponse(body, feature, node, msg); }); return p; } function invokeCreateClassifier(node, params) { - var p = new Promise(function resolver(resolve, reject){ + var p = new Promise(function resolver(resolve, reject) { node.service.createClassifier(params, function(err, body) { if (err) { reject(err); @@ -348,7 +368,7 @@ module.exports = function(RED) { } function invokeListClassifiers(node, params) { - var p = new Promise(function resolver(resolve, reject){ + var p = new Promise(function resolver(resolve, reject) { node.service.listClassifiers(params, function(err, body) { if (err) { reject(err); @@ -361,7 +381,7 @@ module.exports = function(RED) { } function invokeGetClassifier(node, params, msg) { - var p = new Promise(function resolver(resolve, reject){ + var p = new Promise(function resolver(resolve, reject) { params['classifier_id'] = msg.params['classifier_id']; node.service.getClassifier(params, function(err, body) { if (err) { @@ -375,7 +395,7 @@ module.exports = function(RED) { } function invokeDeleteClassifier(node, params, msg) { - var p = new Promise(function resolver(resolve, reject){ + var p = new Promise(function resolver(resolve, reject) { params['classifier_id'] = msg.params['classifier_id']; node.service.deleteClassifier(params, function(err, body) { if (err) { @@ -391,7 +411,7 @@ module.exports = function(RED) { function executeCreateClassifier(params, node, msg) { var p = prepareParamsCreateClassifier(params, node, msg) - .then(function(){ + .then(function() { return invokeCreateClassifier(node, params); }); @@ -401,40 +421,40 @@ module.exports = function(RED) { function executeUtilService(feature, params, node, msg) { var p = null; switch (feature) { - case 'createClassifier': - p = executeCreateClassifier(params, node, msg) - .then(function(body){ - return processTheResponse(body, feature, node, msg); - }); - break; + case 'createClassifier': + p = executeCreateClassifier(params, node, msg) + .then(function(body) { + return processTheResponse(body, feature, node, msg); + }); + break; - case 'retrieveClassifiersList': - p = invokeListClassifiers(node, params) - .then(function(body){ - return processTheResponse(body, feature, node, msg); - }); - break; + case 'retrieveClassifiersList': + p = invokeListClassifiers(node, params) + .then(function(body) { + return processTheResponse(body, feature, node, msg); + }); + break; - case 'retrieveClassifierDetails': - p = invokeGetClassifier(node, params, msg) - .then(function(body){ - return processTheResponse(body, feature, node, msg); - }); - break; + case 'retrieveClassifierDetails': + p = invokeGetClassifier(node, params, msg) + .then(function(body) { + return processTheResponse(body, feature, node, msg); + }); + break; - case 'deleteClassifier': - p = invokeDeleteClassifier(node, params, msg) - .then(function(body){ - return processTheResponse(body, feature, node, msg); - }); - break; + case 'deleteClassifier': + p = invokeDeleteClassifier(node, params, msg) + .then(function(body) { + return processTheResponse(body, feature, node, msg); + }); + break; - case 'deleteAllClassifiers': - p = performDeleteAllClassifiers(params, node, msg); - break; + case 'deleteAllClassifiers': + p = performDeleteAllClassifiers(params, node, msg); + break; - default: - p = Promise.reject('Mode ' + feature + ' not understood'); + default: + p = Promise.reject('Mode ' + feature + ' not understood'); } return p; } @@ -466,44 +486,35 @@ module.exports = function(RED) { var params = {}; node.status({}); - // so there is at most 1 temp file at a time (did not found a better solution...) - //temp.cleanup(); verifyPayload(msg) - .then(function(){ + .then(function() { + return checkForStream(msg); + }) + .then(function() { return verifyInputs(feature, msg); }) - .then(function(){ + .then(function() { return verifyServiceCredentials(node, msg); }) - .then(function(){ + .then(function() { return execute(feature, params, node, msg); }) - .then(function(){ + .then(function() { temp.cleanup(); node.status({}); node.send(msg); }) .catch(function(err) { - var messageTxt = err; - if (err.error) { - messageTxt = err.error; - } else if (err.description) { - messageTxt = err.description; - } - node.status({ - fill: 'red', - shape: 'dot', - text: messageTxt - }); + payloadutils.reportError(node, msg, err); msg.result = {}; msg.result['error'] = err; - node.error(messageTxt, msg); // Note: This node.send forwards the error to the next node, // if this isn't desired then this line needs to be removed. // Should be ok as the node.error would already have recorded // the error in the debug console. + temp.cleanup(); node.send(msg); }); }); diff --git a/utilities/payload-utils.js b/utilities/payload-utils.js index 779d71ca..64b6dc0a 100644 --- a/utilities/payload-utils.js +++ b/utilities/payload-utils.js @@ -18,6 +18,7 @@ var url = require('url'), fileType = require('file-type'), request = require('request'), path = require('path'); + stream = require('stream'); function PayloadUtils() {} @@ -89,7 +90,7 @@ PayloadUtils.prototype = { request(url).pipe(wstream); }, - reportError: function (node, msg, message) { + reportError: function (node, msg, err) { var messageTxt = err; if (err.error) { @@ -99,13 +100,18 @@ PayloadUtils.prototype = { } else if (err.message) { messageTxt = err.message; } - - msg.watsonerror = messageTxt; + msg.watsonerror = messageTxt; node.status({fill:'red', shape:'dot', text: messageTxt}); node.error(messageTxt, msg); }, + isReadableStream : function (obj) { + return obj instanceof stream.Stream && + typeof (obj._read === 'function') && + typeof (obj._readableState === 'object'); + }, + // Function that is returns a function to count // the characters in each language. word_count: function(ct) { From b61c3611acdc5cef9a2097f587d3554ddcbd4c63 Mon Sep 17 00:00:00 2001 From: chughts Date: Sun, 4 Jun 2017 14:28:41 +0100 Subject: [PATCH 05/10] visual recognition to accept stream on msg.payload --- services/visual_recognition/v3.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/services/visual_recognition/v3.js b/services/visual_recognition/v3.js index 626d6c12..3b31dffb 100644 --- a/services/visual_recognition/v3.js +++ b/services/visual_recognition/v3.js @@ -27,8 +27,8 @@ module.exports = function(RED) { fileType = require('file-type'), fs = require('fs'), async = require('async'), - toArray = require('stream-to-array') - sAPIKey = null, + toArray = require('stream-to-array'), + sAPIKey = null, service = null; // temp is being used for file streaming to allow the file to arrive so it can be processed. From a4ae56667f6bb758d6f98ef2c7aafdd1a9a158b6 Mon Sep 17 00:00:00 2001 From: chughts Date: Sun, 4 Jun 2017 14:38:49 +0100 Subject: [PATCH 06/10] visual recognition to accept stream on msg.payload --- services/visual_recognition/v3.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/visual_recognition/v3.js b/services/visual_recognition/v3.js index 3b31dffb..33b3685d 100644 --- a/services/visual_recognition/v3.js +++ b/services/visual_recognition/v3.js @@ -92,7 +92,7 @@ module.exports = function(RED) { function checkForStream(msg) { var p = new Promise(function resolver(resolve, reject) { if (payloadutils.isReadableStream(msg.payload)) { - msg.payload.resume(); + //msg.payload.resume(); toArray(msg.payload) .then(function(parts) { var buffers = []; From f0b9168fb57772960b38da3b548905607fcb62f5 Mon Sep 17 00:00:00 2001 From: chughts Date: Sun, 4 Jun 2017 16:18:38 +0100 Subject: [PATCH 07/10] Add passages parameter to Discovery Node --- README.md | 1 + services/discovery/discovery-utils.js | 5 ++++- services/discovery/v1-query-builder.html | 22 ++++++++++++++++++++-- services/discovery/v1.html | 18 ++++++++++++++++-- 4 files changed, 41 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index f9e04207..b963d62c 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,7 @@ Node-RED Watson Nodes for IBM Bluemix ### New in version 0.5.8 - Visual Reconition Node, now accepts readstream on msg.payload +- Add passages parameter to Discovery Node ### New in version 0.5.7 - Fix to Tone Analyzer to allow JSON as input diff --git a/services/discovery/discovery-utils.js b/services/discovery/discovery-utils.js index 8154396b..cac0015e 100644 --- a/services/discovery/discovery-utils.js +++ b/services/discovery/discovery-utils.js @@ -71,7 +71,7 @@ DiscoveryUtils.prototype = { ['environment_id','collection_id','configuration_id', 'collection_name', - 'query','description','size'].forEach(function(f) { + 'query','passages','description','size'].forEach(function(f) { params = me.buildParamsFor(msg, config, params, f); }); @@ -107,6 +107,9 @@ DiscoveryUtils.prototype = { } params.query += config.query3 + ':"' + config.queryvalue3 + '"'; } + if (config.passages) { + params.passages = config.passages; + } return params; }, diff --git a/services/discovery/v1-query-builder.html b/services/discovery/v1-query-builder.html index 089a8a75..0d3a5773 100644 --- a/services/discovery/v1-query-builder.html +++ b/services/discovery/v1-query-builder.html @@ -100,6 +100,15 @@ +
+ + + +
+
+ +
+