From 40f493b1cd4b0f3691dc5862952615c0fb85f400 Mon Sep 17 00:00:00 2001 From: Garrett Vargas Date: Fri, 23 Aug 2019 06:07:08 -0700 Subject: [PATCH 1/2] Add allowedExtensions option --- README.md | 10 ++++++++++ index.js | 10 +++++++++- test/index-spec.js | 36 ++++++++++++++++++++++++++++++++++++ 3 files changed, 55 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 09721f8..e619121 100644 --- a/README.md +++ b/README.md @@ -73,6 +73,16 @@ app.use(require('prerender-node').blacklisted('^/search')); app.use(require('prerender-node').blacklisted(['/search', '/users/.*/profile'])); ``` +### extensionsToAllow + +Allows a given file or files that would normally be ignored by prerender based on their extension to pass thru and be prerendered. Compares using regex, so be specific when possible. If extensionsToAllow is applied, these files will be prerendered. +```js +app.use(require('prerender-node').extensionsToAllow('sitemap.xml')); +``` +```js +app.use(require('prerender-node').extensionsToAllow(['favicon.ico', '/sitemap/*.xml'])); +``` + ### beforeRender This method is intended to be used for caching, but could be used to save analytics or anything else you need to do for each crawler request. If you return a string from beforeRender, the middleware will serve that to the crawler (with status `200`) instead of making a request to the prerender service. If you return an object the middleware will look for a `status` and `body` property (defaulting to `200` and `""` respectively) and serve those instead. diff --git a/index.js b/index.js index 4fc55f0..a1cb08a 100644 --- a/index.js +++ b/index.js @@ -129,6 +129,11 @@ prerender.blacklisted = function(blacklist) { }; +prerender.allowedExtensions = function(extensionsToAllow) { + prerender.extensionsToAllow = typeof extensionsToAllow === 'string' ? [extensionsToAllow] : extensionsToAllow; + return this; +} + prerender.shouldShowPrerenderedPage = function(req) { var userAgent = req.headers['user-agent'] , bufferAgent = req.headers['x-bufferbot'] @@ -149,7 +154,10 @@ prerender.shouldShowPrerenderedPage = function(req) { if(bufferAgent) isRequestingPrerenderedPage = true; //if it is a bot and is requesting a resource...dont prerender - if(prerender.extensionsToIgnore.some(function(extension){return req.url.toLowerCase().indexOf(extension) !== -1;})) return false; + if(prerender.extensionsToIgnore.some(function(extension){return req.url.toLowerCase().indexOf(extension) !== -1;})){ + //if there is a list of allowed files or extensions let them through + if(!Array.isArray(this.extensionsToAllow) || !this.extensionsToAllow.some(function(allowed){return (new RegExp(allowed)).test(req.url) === true;})) return false; + } //if it is a bot and not requesting a resource and is not whitelisted...dont prerender if(Array.isArray(this.whitelist) && this.whitelist.every(function(whitelisted){return (new RegExp(whitelisted)).test(req.url) === false;})) return false; diff --git a/test/index-spec.js b/test/index-spec.js index e85aae2..ca6dd8c 100644 --- a/test/index-spec.js +++ b/test/index-spec.js @@ -263,6 +263,42 @@ describe ('Prerender', function(){ }); + it('should return a prerendered response if the url is part of the regex specific allowedList', function(){ + var req = { method: 'GET', url: '/favicon.ico', headers: { 'user-agent': bot }, connection: { encrypted: false } }; + + nock('https://service.prerender.io', { + reqheaders: { + 'x-prerender-token': 'MY_TOKEN', + 'Accept-Encoding': 'gzip' + } + }) + .get('/http://google.com/favicon.ico') + .reply(200, ''); + + res.end = sandbox.spy(function(){ + assert.equal(next.callCount, 0); + assert.equal(res.writeHead.callCount, 1); + assert.equal(res.writeHead.getCall(0).args[0], 200); + assert.equal(res.end.callCount, 1); + assert.equal(res.end.getCall(0).args[0], ''); + done(); + }); + + prerender.allowedExtensions('favicon.ico')(req, res, next); + delete prerender.extensionsToAllow; + }); + + it('should call next() if the url is not part of the regex specific allowedList', function(){ + var req = { method: 'GET', url: '/favicon.ico', headers: { 'user-agent': bot }, connection: { encrypted: false } }; + + prerender.allowedExtensions(['sitemap/*.xml', '^/*.jpg'])(req, res, next); + + delete prerender.extensionsToAllow; + assert.equal(next.callCount, 1); + assert.equal(res.writeHead.callCount, 0); + assert.equal(res.end.callCount, 0); + }); + it('should return a prerendered response if a string is returned from beforeRender', function(){ var req = { method: 'GET', url: '/', headers: { 'user-agent': bot }, connection: { encrypted: false } }; From 33f69cc9ee60d1eb024ee5dc2eaacb1f240be39e Mon Sep 17 00:00:00 2001 From: Garrett Vargas Date: Fri, 23 Aug 2019 06:43:28 -0700 Subject: [PATCH 2/2] Update readme --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index e619121..f0a429e 100644 --- a/README.md +++ b/README.md @@ -73,14 +73,14 @@ app.use(require('prerender-node').blacklisted('^/search')); app.use(require('prerender-node').blacklisted(['/search', '/users/.*/profile'])); ``` -### extensionsToAllow +### allowedExtensions -Allows a given file or files that would normally be ignored by prerender based on their extension to pass thru and be prerendered. Compares using regex, so be specific when possible. If extensionsToAllow is applied, these files will be prerendered. +Allows a given file or files that would normally be ignored by prerender based on their extension to pass thru and be prerendered. Compares using regex, so be specific when possible. If allowedExtensions is applied, these files will be prerendered. ```js -app.use(require('prerender-node').extensionsToAllow('sitemap.xml')); +app.use(require('prerender-node').allowedExtensions('sitemap.xml')); ``` ```js -app.use(require('prerender-node').extensionsToAllow(['favicon.ico', '/sitemap/*.xml'])); +app.use(require('prerender-node').allowedExtensions(['favicon.ico', '/sitemap/*.xml'])); ``` ### beforeRender