Merge pull request #9 from spidasoftware/retry-timedout-requests-1662…

…14942 [Delivers #166214942] Retry failed requests
spidasoftware · May 30, 2019 · 9dbefc3 · 9dbefc3
2 parents a4b6ed2 + 420c31a
commit 9dbefc3
Show file tree

Hide file tree

Showing 4 changed files with 84 additions and 32 deletions.
diff --git a/cmds/analyze.js b/cmds/analyze.js
@@ -3,18 +3,17 @@
 const BATCH_SIZE = 100;
 
 const Promise = require('bluebird');
-const request = Promise.promisify(require('request'));
 const fs = Promise.promisifyAll(require('fs'));
 const zlib = Promise.promisifyAll(require('zlib'));
 const mkdirp = Promise.promisify(require('mkdirp'));
-const querystring = require('querystring');
 const _ = require('lodash');
 const path = require('path');
 const co = require('co');
 const util = require('util');
 
 const api = require('../lib/api');
 const { loadClientData } = require('../lib/clientData');
+const { requestAndRetry } = require('../lib/retrier');
 
 const command = 'analyze [json..]';
 const desc =  'Send analysis or job in each file to CEE, then optionally poll CEE until analysis is complete  and retrieve results.  If polling is enabled analysis results will be written to a directory specified by the output parameter.';
@@ -59,7 +58,7 @@ module.exports = {
             batchJobs(argv).mapSeries(batch => 
                 zlib.gzipAsync(JSON.stringify(batch))
                     .then(jobData => 
-                        request({
+                        requestAndRetry({
                             url: config.server + '/job?apiToken=' + config.apiToken,
                             proxy: config.proxy,
                             gzip: true,
@@ -71,15 +70,15 @@ module.exports = {
                                 'Content-Encoding': 'gzip'
                             },
                             body: jobData
+                        }, (response) => {
+                            try {
+                                return JSON.parse(response.body);
+                            } catch (e) {
+                                console.log(`Unable to parse response JSON: ${response.body}`);
+                                throw e;
+                            }
                         })
-                    ).then(response => {
-                        try {
-                            return JSON.parse(response.body);
-                        } catch (e) {
-                            console.log(`Unable to parse response JSON: ${response.body}`);
-                            return [];
-                        }
-                    })
+                    )
             ).then(responses => {
                 responses = _.flatten(responses);
 
@@ -103,7 +102,6 @@ module.exports = {
                         console.log('The following errors occured:');
                         console.log(util.inspect(errors, { depth: null}));
                     }
-
                     return showProgress(jobIds,argv,config).then(() => console.log('Done.'));
                 }
             })
@@ -113,7 +111,7 @@ module.exports = {
 function pollFor(jobIds, status, config, onUpdate) {
     return co(function*() {
         while(jobIds.length > 0) {
-            const response = yield request({
+            const updatedJobIds = yield requestAndRetry({
                 url: config.server + '/job/poll',
                 proxy: config.proxy,
                 qs: { apiToken: config.apiToken, status, ids: JSON.stringify(jobIds.slice(0,BATCH_SIZE)) },
@@ -122,10 +120,15 @@ function pollFor(jobIds, status, config, onUpdate) {
                     'User-Agent': 'cee-cli',
                     'Accept': 'application/json',
                 }
+            }, (response) => {
+                try {
+                    return JSON.parse(response.body).map(j => j.id)
+                } catch(e) {
+                    console.log(`Failed to poll for jobs ${response.body}`)
+                    throw e
+                }
             });
 
-            const updatedJobIds = JSON.parse(response.body).map(j => j.id);
-
             _.pullAll(jobIds,updatedJobIds);
             const updateP = onUpdate(updatedJobIds);
             if (updateP) {
@@ -158,7 +161,7 @@ function showProgress(jobIds,argv,config) {
                 displayProgress(validCount, startedCount, finishedCount);
             }),
             pollFor(unfinishedJobs, 'FINISHED', config, jobIds =>
-                request({
+                requestAndRetry({
                     url: config.server + '/job',
                     qs: {
                         apiToken: config.apiToken,
@@ -170,7 +173,7 @@ function showProgress(jobIds,argv,config) {
                         'User-Agent': 'cee-cli',
                         'Accept': 'application/json',
                     }
-                }).then(response => 
+                }, (response) => 
                     Promise.all(
                         JSON.parse(response.body).map(result =>
                             fs.writeFileAsync(path.join(output,result.externalId),JSON.stringify(result))
@@ -358,3 +361,4 @@ function batchJobs(argv) {
 
 
 
+
diff --git a/cmds/get.js b/cmds/get.js
@@ -1,7 +1,7 @@
 'use strict';
 
 const Promise = require('bluebird');
-const request = Promise.promisify(require('request'));
+const { requestAndRetry } = require('../lib/retrier');
 const fs = Promise.promisifyAll(require('fs'));
 const mkdirp = Promise.promisify(require('mkdirp'));
 const path = require('path');
@@ -39,7 +39,7 @@ module.exports = {
             let jobIdsP;
 
             if (typeof argv.offset !== 'undefined') {
-                jobIdsP = request({
+                jobIdsP = requestAndRetry({
                     url: config.server + '/api/jobs',
                     proxy: config.proxy,
                     headers: {
@@ -52,14 +52,21 @@ module.exports = {
                         offset: argv.offset
                     },
                     gzip: true
-                }).then(resp => JSON.parse(resp.body).data.map(j => j.id));
+                }, (resp) => {
+                    try {
+                        return JSON.parse(resp.body).data.map(j => j.id);
+                    } catch(e) {
+                        console.log(`Failed to get jobs at offset: ${resp.body}`)
+                        throw e;
+                    }
+                });
             } else {
                 jobIdsP = Promise.resolve(argv.jobIds);
             }
 
             return jobIdsP.then(allJobIds =>
                 Promise.all(_.chunk(allJobIds, 100).map(jobIds =>
-                    request({
+                    requestAndRetry({
                         url: config.server + '/job',
                         proxy: config.proxy,
                         headers: {
@@ -71,10 +78,17 @@ module.exports = {
                             ids: JSON.stringify(jobIds)
                         },
                         gzip: true
+                    }, (response) => {
+                        try {
+                            return JSON.parse(response.body);
+                        } catch(e) {
+                            console.log(`Failed to get jobs: ${response.body}`)
+                            throw e;
+                        }
                     })
                 ))
-            ).then(responses => {
-                const jobs = _.flatten(responses.map(r => JSON.parse(r.body)));
+            ).then(bodies => {
+                const jobs = _.flatten(bodies);
                 if (argv.stdout) {
                     console.log(JSON.stringify(jobs));
                 } else {

diff --git a/cmds/status.js b/cmds/status.js
@@ -1,8 +1,6 @@
 'use strict';
 
-const Promise = require('bluebird');
-const request = Promise.promisify(require('request'));
-
+const { requestAndRetry } = require('../lib/retrier');
 const api = require('../lib/api');
 
 const command = 'status [jobId..]';
@@ -18,7 +16,7 @@ module.exports = {
         .default('config',api.defaultConfigPath),
     handler: argv =>
         api.loadConfig(argv.config).then(config =>
-            request({
+            requestAndRetry({
                 url: config.server + '/job/status',
                 proxy: config.proxy,
                 headers: {
@@ -30,10 +28,15 @@ module.exports = {
                     ids: JSON.stringify(argv.jobId)
                 },
                 gzip: true
-            }).then(response => 
-                JSON.parse(response.body).forEach(job => 
-                    console.log(`${job.id} -- ${job.status}`)
-                ) 
-            )
+            }, (response) => {
+                try {
+                    JSON.parse(response.body).forEach(job => 
+                        console.log(`${job.id} -- ${job.status}`)
+                    )
+                } catch(e) {
+                    console.log(`Failed to get job status: ${response.body}`)
+                    throw e;
+                }
+            })
         )
 };
diff --git a/lib/retrier.js b/lib/retrier.js
@@ -0,0 +1,31 @@
+'use strict';
+
+const Promise = require('bluebird');
+const request = Promise.promisify(require('request'));
+
+const REQUEST_RETRY_COUNT = 5;
+const RETRY_TIMEOUT_MS = 5000;
+const REQUEST_TIMEOUT_MS = 60000;
+
+function rejectDelay() {
+    return new Promise(function(resolve, reject) {
+        setTimeout(resolve, RETRY_TIMEOUT_MS);
+    })
+}
+
+function requestAndRetry(requestParams, responseHandler = (response) => response, retryCount = 0){
+	requestParams.timeout = REQUEST_TIMEOUT_MS;
+    return request(requestParams).then(responseHandler).catch((err) => {
+        if(retryCount < REQUEST_RETRY_COUNT){
+            console.log(`Could not complete last request. Retrying: ${retryCount+1}/${REQUEST_RETRY_COUNT}`)
+            return rejectDelay().then(() => requestAndRetry(requestParams, responseHandler, ++retryCount));
+        } else {
+            console.log("Could not complete action.");
+            throw err
+        }
+    });
+}
+
+module.exports = {
+    requestAndRetry
+};