Merge pull request #110 from LD4P/t102-throttle2

Throttles requests to Trellis.

config/default.js

@@ -17,5 +17,6 @@ module.exports = {
   indexUrl: process.env.INDEX_URL || 'http://localhost:9200',
   nonRdfTypeURI: process.env.NON_RDF_TYPE_URI || 'http://www.w3.org/ns/ldp#NonRDFSource',
   nonRdfMimeType: process.env.NON_RDF_MIME_TYPE || 'application/json',
-  debug:  process.env.DEBUG !== undefined ? process.env.DEBUG : true
+  debug:  process.env.DEBUG !== undefined ? process.env.DEBUG : true,
+  poolLimit: process.env.POOL_LIMIT || 2
 }

package.json

@@ -33,6 +33,7 @@
     "jsonpath-plus": "^1.1.0",
     "stomp-client": "^0.9.0",
     "superagent": "^5.1.0",
+    "tiny-async-pool": "^1.0.4",
     "url-parse": "^1.4.4",
     "wait-on": "^3.2.0"
   },

src/Crawler.js

@@ -1,6 +1,7 @@
 import config from 'config'
 import Logger from './Logger'
 import Request from './Request'
+import asyncPool from 'tiny-async-pool'
 
 const linkHeaderRegex = /<(?<link>.+)>; rel="type"/
 
@@ -41,17 +42,13 @@ export default class Crawler {
       // index.
       const response = await this.typeSpecificRequest(uri, types).response()
 
+      await onResource(response.body, uri, types)
+
       const containedResourcesArray = this.containedResourcesArray(response.body)
       this.logger.debug(`${uri} contains ${containedResourcesArray}`)
 
-      const containedResourcePromises = containedResourcesArray.map(async child => {
-        if (child)
-          // Recurse down into each child resource
-          await this.request(child, onResource)
-      })
-
-      // await promises for the callback on this resource as well as the requests for any child resources
-      await Promise.all([onResource(response.body, uri, types)].concat(containedResourcePromises))
+      await asyncPool(config.get('poolLimit'), containedResourcesArray, (child) => this.request(child, onResource))
+
     } catch(error) {
       this.logger.error(`during crawl, error making mime type-specific request to ${uri}: ${error}`)
     }