From 1072f517e945b4c792d2d2eb27d0de036c6150f4 Mon Sep 17 00:00:00 2001 From: Mohamed Amin Boubaker Date: Sat, 28 Sep 2019 13:46:56 +0100 Subject: [PATCH] feat: added useZopfliForGzip option Closes #39 --- .travis.yml | 20 ----- README.md | 6 ++ brotli-compat.js | 18 ++--- index.js | 27 +++---- package.json | 5 +- test/compression.js | 177 +++++++++++++++++++++++++------------------- zopfli-compat.js | 9 ++- 7 files changed, 136 insertions(+), 126 deletions(-) diff --git a/.travis.yml b/.travis.yml index 2c5919b5..30f57916 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,23 +10,3 @@ cache: before_cache: rm -rf node_modules/.cache install: npm install --build-from-source after_success: cat ./coverage/lcov.info | coveralls - -#stages: -#- test -##- name: Deploy -## if: tag IS present - -#jobs: -# include: -# - stage: Deploy -# node_js: stable -# script: node ./prepublish.js -# after_success: false -## deploy: -## - provider: npm -## skip_cleanup: true -## on: -## tags: true -## email: a.molcanovas@gmail.com -## api_key: -## secure: iSr5sIvDsSP3c+55Wa91G3a3WXkb8Eb9iGc6wontJDZQx7PZ+xYHFFYEAQkK6LYw08i0eik/2xTdt76djb1jySOi/OlvAQIGsf/GPcTG0qRcRvW/rR6CM7yX36xQ9pWUn/PEuDHSmzBnKVHHrhbYR66olUrOBu7GcgTJoLDj0ppFSbtUMqqFCM8bNQs8d3gMgwF96+7jpzjHbFDVrB+jMN8qNZF29RO+Xqv9oaPSQyY2o7IJUwsbVGUs6g6EpnNlj4p7MfzBmhoNLikY73UtW2teLyobDTf8Qv0+PwuRq1WwB8UhlgSlqQ+TWaFcywqmhiZIm30fu69M1qfkuO6N51jIuvq19qCZekcl02vS5kUs0VnbOozp+5BuQNdEzkbmrQrtDkGXaI86ZVr7MUGfFZqPQlEcIoOQyRT6cA8BtZ4SrhCS73xLbFoHcLhbXR7i3gpNQK1Ap5YMnlwQS+z4e5NI6h/VQ+8GRXuVCTclAuJdpXkJWGuoOMfLK0keCkO1iqETW4Iv7t58tjJXiwfU8VBgxd/8bI5lyVoy9VpBoptaDjfWJpfaSskXjxTv+RVVmwhjr44wngvvNid93q0hB7+AFhr4kR52fQ8vXtZtfM3fdTUy036IUo/omQzXKV7EyL/QFkI62a4EDJAFWpuZOO+wtlCaijbcadTrtIV5wJk= diff --git a/README.md b/README.md index 049134dc..4ab0f7d4 100644 --- a/README.md +++ b/README.md @@ -46,6 +46,7 @@ project. - [API](#api) - [`shrinkRay([options])`](#shrinkrayoptions) - [Options](#options) + - [useZopfliForGzip](#usezopfliforgzip) - [filter](#filter) - [cache](#cache) - [cacheSize](#cachesize) @@ -129,6 +130,11 @@ we have also moved all of the gzip/deflate/zlib-specific parameters into a sub-object called `zlib`. If you use `zlib` parameters at the root level of options in `shrink-ray`, you will get a deprecation warning. +#### useZopfliForGzip + +Whether to use [node-zopfli-es](https://www.npmjs.com/package/node-zopfli-es) (`true`) or zlib (`false`) for gzip compression. +Defaults to `true`. + #### filter A function to decide if the response should be considered for compression. diff --git a/brotli-compat.js b/brotli-compat.js index 0f565141..c97049d1 100644 --- a/brotli-compat.js +++ b/brotli-compat.js @@ -1,6 +1,4 @@ -module.exports = getBrotliModule; - -function getBrotliModule() { +module.exports = function brotliCompat() { const zlib = require('zlib'); if (typeof zlib.createBrotliCompress === 'function') { @@ -12,8 +10,8 @@ function getBrotliModule() { quality: zlib.constants.BROTLI_PARAM_QUALITY, lgwin: zlib.constants.BROTLI_PARAM_LGWIN, lgblock: zlib.constants.BROTLI_PARAM_LGBLOCK, - disable_literal_context_modeling: - zlib.constants.BROTLI_PARAM_DISABLE_LITERAL_CONTEXT_MODELING, + disable_literal_context_modeling: + zlib.constants.BROTLI_PARAM_DISABLE_LITERAL_CONTEXT_MODELING, large_window: zlib.constants.BROTLI_PARAM_LARGE_WINDOW }; const iltorbOptionsToNodeZlibBrotliOpts = iltorbOpts => { @@ -23,11 +21,11 @@ function getBrotliModule() { if (ILTORB_OPTION_NAMES_TO_BROTLI_PARAM_NAMES.hasOwnProperty(key)) { params[ILTORB_OPTION_NAMES_TO_BROTLI_PARAM_NAMES[key]] = iltorbOpts[ key - ]; + ]; } }); return { params }; - } + }; /** * Replicate the 'iltorb' interface for backwards compatibility. @@ -42,7 +40,7 @@ function getBrotliModule() { }; } - + // If we get here, then our NodeJS does not support brotli natively. try { return require('iltorb'); @@ -55,8 +53,8 @@ function getBrotliModule() { } ); } - + // Return a signal value instead of throwing an exception, so the code in the // index file doesn't have to try/catch again. return false; -} \ No newline at end of file +}; diff --git a/index.js b/index.js index 7316c5ea..a298b3a9 100644 --- a/index.js +++ b/index.js @@ -2,6 +2,7 @@ /*! * compression + * Copyright(c) 2019 CodeIter (https://github.com/CodeIter) * Copyright(c) 2017 Arturas Molcanovas * Copyright(c) 2010 Sencha Inc. * Copyright(c) 2011 TJ Holowaychuk @@ -28,19 +29,18 @@ const zlib = require('zlib'); * Optional dependencies handling. If some binary dependencies cannot build in * this environment, or are incompatible with this version of Node, the rest of * the module should work! - * Known dependency issues: + * Known dependency issues: * - node-zopfli-es is not compatible with Node <8.11. * - iltorb is not required for Node >= 11.8, whose zlib has brotli built in. */ - const brotliCompat = require('./brotli-compat'); - const zopfliCompat = require('./zopfli-compat'); - - // These are factory functions because they dynamically require dependencies - // and may log errors. - // They need to be tested, so they shouldn't have side effects on load. - const brotli = brotliCompat(); - const zopfli = zopfliCompat(); +const brotliCompat = require('./brotli-compat'); +const zopfliCompat = require('./zopfli-compat'); + +// These are factory functions because they dynamically require dependencies +// and may log errors. +// They need to be tested, so they shouldn't have side effects on load. +const brotli = brotliCompat(); /** * Module exports. @@ -76,6 +76,7 @@ function compression(options) { const opts = options || {}; // options + const zopfli = zopfliCompat('useZopfliForGzip' in opts ? opts.useZopfliForGzip : true); const filter = opts.filter || shouldCompress; let threshold = bytes.parse(opts.threshold); @@ -93,7 +94,7 @@ function compression(options) { }); if (!opts.hasOwnProperty('cacheSize')) opts.cacheSize = '128mB'; - const cache = opts.cacheSize ? createCache(bytes(opts.cacheSize.toString())) : null; + const cache = opts.cacheSize ? createCache(bytes(opts.cacheSize.toString()), zopfli) : null; const shouldCache = opts.cache || stubTrue; @@ -359,7 +360,7 @@ function shouldTransform(req, res) { !cacheControlNoTransformRegExp.test(cacheControl); } -function createCache(size) { +function createCache(size, zopfli) { const index = {}; const lru = new lruCache({ max: size, @@ -411,7 +412,7 @@ function createCache(size) { const result = new BufferWritable(); new BufferReadable(buffer) - .pipe(getBestQualityReencoder(coding)) + .pipe(getBestQualityReencoder(coding, zopfli)) .pipe(result) .on('finish', function () { const itemInCache = lru.peek(key); @@ -489,7 +490,7 @@ BufferDuplex.prototype._write = function (chunk, encoding, callback) { // get a decode --> encode transform stream that will re-encode the content at // the best quality available for that coding method. -function getBestQualityReencoder(coding) { +function getBestQualityReencoder(coding, zopfli) { switch (coding) { case 'gzip': return multipipe(zlib.createGunzip(), zopfli.createGzip()); diff --git a/package.json b/package.json index 3f67ecc5..29a7daae 100644 --- a/package.json +++ b/package.json @@ -1,9 +1,10 @@ { "name": "shrink-ray-current", "description": "Node.js compression middleware with brotli and zopfli support", - "version": "4.0.0", + "version": "4.1.0", "contributors": [ - "Arturas Molcanovas (https://alorel.github.io)", + "Arturas Molcanovas (https://github.com/Alorel)", + "CodeIter (https://github.com/CodeIter)", "Douglas Christopher Wilson ", "Jonathan Ong (http://jongleberry.com)", "Scott Davis (https://github.com/scttdavs)", diff --git a/test/compression.js b/test/compression.js index f89c204d..fc4f032f 100644 --- a/test/compression.js +++ b/test/compression.js @@ -1,12 +1,12 @@ 'use strict'; -const assert = require('assert'); -const bytes = require('bytes'); -const crypto = require('crypto'); -const http = require('http'); -const iltorb = require('iltorb'); +const assert = require('assert'); +const bytes = require('bytes'); +const crypto = require('crypto'); +const http = require('http'); +const iltorb = require('iltorb'); const streamBuffers = require('stream-buffers'); -const request = require('supertest'); +const request = require('supertest'); const compression = require('..'); @@ -14,7 +14,8 @@ const createdServers = []; describe('compression()', function () { after(() => { - const cb = () => {}; + const cb = () => { + }; for (const server of createdServers) { server.close(cb); @@ -156,7 +157,8 @@ describe('compression()', function () { request(server) .get('/') - .end(function () {}); + .end(function () { + }); }); it('should back-pressure when compressed', function (done) { @@ -164,7 +166,7 @@ describe('compression()', function () { let client; let resp; let drained = false; - let wait = 2; + let wait = 2; const server = createServer({threshold: 0}, function (req, res) { resp = res; @@ -221,9 +223,13 @@ describe('compression()', function () { let client; let resp; let drained = false; - let wait = 2; + let wait = 2; - const server = createServer({filter: function () { return false; }}, function (req, res) { + const server = createServer({ + filter: function () { + return false; + } + }, function (req, res) { resp = res; res.on('drain', function () { drained = true; @@ -274,8 +280,8 @@ describe('compression()', function () { }); it('should transfer large bodies', function (done) { - const len = bytes('1mb'); - const buf = Buffer.alloc(len); + const len = bytes('1mb'); + const buf = Buffer.alloc(len); const server = createServer({threshold: 0}, function (req, res) { res.setHeader('Content-Type', 'text/plain'); res.end(buf); @@ -291,8 +297,8 @@ describe('compression()', function () { }); it('should transfer large bodies with multiple writes', function (done) { - const len = bytes('40kb'); - const buf = Buffer.alloc(len); + const len = bytes('40kb'); + const buf = Buffer.alloc(len); const server = createServer({threshold: 0}, function (req, res) { res.setHeader('Content-Type', 'text/plain'); res.write(buf); @@ -540,13 +546,13 @@ describe('compression()', function () { iltorb.compressSync(Buffer.from('hello, world', 'utf-8'), {quality: 8}) ); done(); - }); + }); }); }); describe('when caching is turned on', function () { it('should cache a gzipped response with the same ETag', function (done) { - let count = 0; + let count = 0; const server = createServer({threshold: 0}, function (req, res) { res.setHeader('Content-Type', 'text/plain'); res.setHeader('ETag', '12345'); @@ -560,7 +566,7 @@ describe('compression()', function () { }); it('should cache a deflate response with the same ETag', function (done) { - let count = 0; + let count = 0; const server = createServer({threshold: 0}, function (req, res) { res.setHeader('Content-Type', 'text/plain'); res.setHeader('ETag', '12345'); @@ -574,7 +580,7 @@ describe('compression()', function () { }); it('should cache a brotli response with the same ETag', function (done) { - let count = 0; + let count = 0; const server = createServer({threshold: 0, brotli: {quality: 1}}, function (req, res) { res.setHeader('Content-Type', 'text/plain'); res.setHeader('ETag', '12345'); @@ -598,8 +604,12 @@ describe('compression()', function () { }); it('should not cache when the cache function returns false', function (done) { - let count = 0; - const server = createServer({threshold: 0, cache: function (req, res) { return false; }}, function (req, res) { + let count = 0; + const server = createServer({ + threshold: 0, cache: function (req, res) { + return false; + } + }, function (req, res) { res.setHeader('Content-Type', 'text/plain'); res.setHeader('ETag', '12345'); res.end('hello, world #' + count); @@ -612,7 +622,7 @@ describe('compression()', function () { }); it('should not get a cached compressed response for a different ETag', function (done) { - let count = 0; + let count = 0; const server = createServer({threshold: 0}, function (req, res) { res.setHeader('Content-Type', 'text/plain'); res.setHeader('ETag', count.toString()); @@ -626,7 +636,7 @@ describe('compression()', function () { }); it('should not cache when there is no ETag', function (done) { - let count = 0; + let count = 0; const server = createServer({threshold: 0}, function (req, res) { res.setHeader('Content-Type', 'text/plain'); res.end('hello, world #' + count); @@ -639,7 +649,7 @@ describe('compression()', function () { }); it('should not cache when caching is disabled', function (done) { - let count = 0; + let count = 0; const server = createServer({threshold: 0, cacheSize: false}, function (req, res) { res.setHeader('Content-Type', 'text/plain'); res.setHeader('ETag', '12345'); @@ -653,8 +663,8 @@ describe('compression()', function () { }); it('should evict from the cache when over the limit', function (done) { - let etag = 'a'; - let count = 0; + let etag = 'a'; + let count = 0; const server = createServer({threshold: 0, cacheSize: 40}, function (req, res) { res.setHeader('Content-Type', 'text/plain'); res.setHeader('ETag', etag); @@ -662,13 +672,13 @@ describe('compression()', function () { }); gzipRequest(server).expect('hello, world #0', function () { - etag = 'b'; + etag = 'b'; count = 1; gzipRequest(server).expect('hello, world #1', function () { - etag = 'b'; + etag = 'b'; count = 2; gzipRequest(server).expect('hello, world #1', function () { - etag = 'a'; + etag = 'a'; count = 3; gzipRequest(server).expect('hello, world #3', done); }); @@ -677,8 +687,8 @@ describe('compression()', function () { }); it('should evict the oldest representation from the cache when over the limit', function (done) { - let etag = 'a'; - let count = 0; + let etag = 'a'; + let count = 0; const server = createServer({threshold: 0, cacheSize: 80}, function (req, res) { res.setHeader('Content-Type', 'text/plain'); res.setHeader('ETag', etag); @@ -686,16 +696,16 @@ describe('compression()', function () { }); gzipRequest(server).expect('hello, world #0', function () { - etag = 'b'; + etag = 'b'; count = 1; gzipRequest(server).expect('hello, world #1', function () { - etag = 'c'; + etag = 'c'; count = 2; gzipRequest(server).expect('hello, world #2', function () { - etag = 'b'; + etag = 'b'; count = 3; gzipRequest(server).expect('hello, world #1', function () { - etag = 'a'; + etag = 'a'; count = 4; gzipRequest(server).expect('hello, world #4', done); }); @@ -787,7 +797,7 @@ describe('compression()', function () { }); it('should flush the response', function (done) { - let chunks = 0; + let chunks = 0; let resp; const server = createServer({threshold: 0}, function (req, res) { resp = res; @@ -820,14 +830,14 @@ describe('compression()', function () { it('should flush the response for brotli', function (done) { var chunks = 0; var resp; - var server = createServer({ threshold: 0 }, function (req, res) { + var server = createServer({threshold: 0}, function (req, res) { resp = res; res.setHeader('Content-Type', 'text/plain'); res.setHeader('Content-Length', '2048'); write(); }); - function write () { + function write() { chunks++; if (chunks === 2) return resp.end(); if (chunks > 2) return chunks--; @@ -836,20 +846,20 @@ describe('compression()', function () { } brotliRequest(server) - .request() - .on('response', function (res) { - assert.equal(res.headers['content-encoding'], 'br'); - res.on('data', write); - res.on('end', function () { - assert.equal(chunks, 2); - done(); - }); - }) - .end(); + .request() + .on('response', function (res) { + assert.equal(res.headers['content-encoding'], 'br'); + res.on('data', write); + res.on('end', function () { + assert.equal(chunks, 2); + done(); + }); + }) + .end(); }); it('should flush small chunks for gzip', function (done) { - let chunks = 0; + let chunks = 0; let resp; const server = createServer({threshold: 0}, function (req, res) { resp = res; @@ -881,13 +891,13 @@ describe('compression()', function () { it('should flush small chunks for brotli', function (done) { var chunks = 0; var resp; - var server = createServer({ threshold: 0 }, function (req, res) { + var server = createServer({threshold: 0}, function (req, res) { resp = res; res.setHeader('Content-Type', 'text/plain'); write(); }); - function write () { + function write() { chunks++; if (chunks === 20) return resp.end(); if (chunks > 20) return chunks--; @@ -896,20 +906,20 @@ describe('compression()', function () { } brotliRequest(server) - .request() - .on('response', function (res) { - assert.equal(res.headers['content-encoding'], 'br'); - res.on('data', write); - res.on('end', function () { - assert.equal(chunks, 20); - done(); - }); - }) - .end(); + .request() + .on('response', function (res) { + assert.equal(res.headers['content-encoding'], 'br'); + res.on('data', write); + res.on('end', function () { + assert.equal(chunks, 20); + done(); + }); + }) + .end(); }); it('should flush small chunks for deflate', function (done) { - let chunks = 0; + let chunks = 0; let resp; const server = createServer({threshold: 0}, function (req, res) { resp = res; @@ -945,7 +955,7 @@ const proxyquire = require('proxyquire'); describe('compat factory for', function () { describe('brotli', function () { - it('returns iltorb facade on zlib when node supports brotli', function() { + it('returns iltorb facade on zlib when node supports brotli', function () { // simulate brotli compat no matter what node version for this test const zlibMock = { constants: { @@ -966,11 +976,11 @@ describe('compat factory for', function () { return zlibMock.stream; } }; - + const brotliCompat = proxyquire.noCallThru().load('../brotli-compat', { zlib: zlibMock }); - + const brotli = brotliCompat(); assert.equal(typeof brotli.compressStream, 'function'); assert.equal(typeof brotli.decompressStream, 'function'); @@ -993,7 +1003,7 @@ describe('compat factory for', function () { assert.equal(zlibMock.decompressOpts, undefined); }); - it('returns iltorb where native brotli is unavailable', function() { + it('returns iltorb where native brotli is unavailable', function () { const brotliCompat = proxyquire.noCallThru().load('../brotli-compat', { // simulate NO brotli compat no matter what node version for this test zlib: { @@ -1020,21 +1030,34 @@ describe('compat factory for', function () { }) }); describe('zopfli', function () { - const mockZop = {}; - it('returns zopfli if it exists', function () { - const zopfliCompat = proxyquire.noCallThru().load('../zopfli-compat', { - 'node-zopfli-es': mockZop + describe('Using zlib', () => { + it('returns zlib if it even if zopfli exists', function () { + const mockZlib = {}; + const zopfliCompat = proxyquire.noCallThru().load('../zopfli-compat', { + 'node-zopfli-es': {}, + zlib: mockZlib + }); + assert.equal(zopfliCompat(false), mockZlib); }); - assert.equal(zopfliCompat(), mockZop); }); - it('returns false if zopfli does not', function () { - const mockZlib = {}; - const zopfliCompat = proxyquire.noCallThru().load('../zopfli-compat', { - 'node-zopfli-es': null, - zlib: mockZlib + describe('Using zopfli', () => { + it('returns zopfli if it exists', function () { + const mockZop = {}; + const zopfliCompat = proxyquire.noCallThru().load('../zopfli-compat', { + 'node-zopfli-es': mockZop + }); + assert.equal(zopfliCompat(true), mockZop); + }); + + it('returns false if zopfli does not', function () { + const mockZlib = {}; + const zopfliCompat = proxyquire.noCallThru().load('../zopfli-compat', { + 'node-zopfli-es': null, + zlib: mockZlib + }); + assert.equal(zopfliCompat(true), mockZlib); }); - assert.equal(zopfliCompat(), mockZlib); }) }); }); diff --git a/zopfli-compat.js b/zopfli-compat.js index 742bd089..f84cc7ce 100644 --- a/zopfli-compat.js +++ b/zopfli-compat.js @@ -1,6 +1,5 @@ -module.exports = getZopfliModule; - -function getZopfliModule() { +module.exports = function zopfliCompat(useZopfliForGzip) { + if (useZopfliForGzip) { try { return require('node-zopfli-es'); } catch (e) { @@ -12,6 +11,8 @@ function getZopfliModule() { } ); } + } + // Fall back to plain zlib. return require('zlib'); -} \ No newline at end of file +};