From e043436bb148930ceaae95badac3445103af73d1 Mon Sep 17 00:00:00 2001 From: Tom Giratikanon Date: Thu, 5 Dec 2013 11:14:14 -0500 Subject: [PATCH 1/3] Fix bug so that sync verify works with gzip correctly Before, sync verify would compare md5 of gzipped remote file and non-gzipped local file, ensuring they would never match. --- tasks/lib/s3.js | 68 +++++++++++++++++++++++++++++++++---------------- 1 file changed, 46 insertions(+), 22 deletions(-) diff --git a/tasks/lib/s3.js b/tasks/lib/s3.js index 945978e..92b9813 100644 --- a/tasks/lib/s3.js +++ b/tasks/lib/s3.js @@ -431,35 +431,59 @@ exports.init = function (grunt) { // it. Strip them out. remoteHash = res.headers.etag.replace(/"/g, ''); - // Get an md5 of the local file so we can verify the upload. - localHash = crypto.createHash('md5').update(data).digest('hex'); - if (remoteHash === localHash) { - // the file exists and is the same so do nothing with that - return dfd.resolve(util.format(MSG_SKIP_MATCHES, prettySrc)); - } - fs.stat( src, function(err, stats) { - var remoteWhen, localWhen, upload; - if (err) { - return dfd.reject(makeError(MSG_ERR_UPLOAD, prettySrc, err)); - } + var onLocalHash = function() { - // which one is newer? if local is newer, we should upload it - remoteWhen = new Date(res.headers['last-modified'] || "0"); // earliest date possible if no header is returned - localWhen = new Date(stats.mtime || "1"); // make second earliest date possible if mtime isn't set + if (remoteHash === localHash) { + // the file exists and is the same so do nothing with that - if ( localWhen <= remoteWhen ) { - // Remote file was older - return dfd.resolve(util.format(MSG_SKIP_OLDER, prettySrc)); + return dfd.resolve(util.format(MSG_SKIP_MATCHES, prettySrc)); + } else { + // console.log("diff hash!", remoteHash, localHash) } - // default is that local is newer, only upload when it is - upload = exports.upload( src, dest, opts); - // pass through the dfd state - upload.then( dfd.resolve, dfd.reject ); - }); + fs.stat( src, function(err, stats) { + var remoteWhen, localWhen, upload; + + if (err) { + return dfd.reject(makeError(MSG_ERR_UPLOAD, prettySrc, err)); + } + + // which one is newer? if local is newer, we should upload it + remoteWhen = new Date(res.headers['last-modified'] || "0"); // earliest date possible if no header is returned + localWhen = new Date(stats.mtime || "1"); // make second earliest date possible if mtime isn't set + + if ( localWhen <= remoteWhen ) { + // console.log(localWhen, remoteWhen) + // Remote file was newer + return dfd.resolve(util.format(MSG_SKIP_OLDER, prettySrc)); + } + + // default is that local is newer, only upload when it is + upload = exports.upload( src, dest, opts); + // pass through the dfd state + upload.then( dfd.resolve, dfd.reject ); + }); + + } + + if (options.gzip) { + // If we're gzipping, assuming the remote file is also gzipped + zlib.gzip(data, function(err, buffer) { + if (!err) { + localHash = crypto.createHash('md5').update(buffer).digest('hex'); + } + onLocalHash(localHash); + }); + + } else { + // Get an md5 of the local file so we can verify the upload. + localHash = crypto.createHash('md5').update(data).digest('hex'); + onLocalHash(localHash); + } + }); }).end(); From fdf96a1da5bdb0313cf32717cdd3aa35d6ef096a Mon Sep 17 00:00:00 2001 From: Justin Ridgewell Date: Thu, 30 Jan 2014 13:02:19 -0500 Subject: [PATCH 2/3] Clean up s3.js --- tasks/lib/s3.js | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/tasks/lib/s3.js b/tasks/lib/s3.js index 92b9813..968b156 100644 --- a/tasks/lib/s3.js +++ b/tasks/lib/s3.js @@ -431,10 +431,7 @@ exports.init = function (grunt) { // it. Strip them out. remoteHash = res.headers.etag.replace(/"/g, ''); - - - - var onLocalHash = function() { + function onLocalHash() { if (remoteHash === localHash) { // the file exists and is the same so do nothing with that @@ -466,7 +463,6 @@ exports.init = function (grunt) { // pass through the dfd state upload.then( dfd.resolve, dfd.reject ); }); - } if (options.gzip) { From 2a7c40d90e6e0d6149031544f28123acbee1f465 Mon Sep 17 00:00:00 2001 From: Justin Ridgewell Date: Thu, 30 Jan 2014 13:02:49 -0500 Subject: [PATCH 3/3] Test Sync with gzipped uploads. --- test/sync.js | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/test/sync.js b/test/sync.js index 2a21fee..7fbd40a 100644 --- a/test/sync.js +++ b/test/sync.js @@ -1,4 +1,5 @@ +var fs = require('fs'); var grunt = require('grunt'); var yaml = require('libyaml'); var hashFile = require('../tasks/lib/common').hashFile; @@ -39,5 +40,41 @@ module.exports = { .always(function () { test.done(); }); + }, + + testSyncGZip: function (test) { + test.expect(1); + + var src = __dirname + '/files/a.txt'; + var destDir = __dirname + '/../s3/127/test/a.txt/.fakes3_metadataFFF/'; + var dest = destDir + 'content'; + var meta = destDir + 'metadata'; + var md5; + + var _config = _.clone(config, true); + _config.gzip = true; + _config.verify = true; + + s3.sync(src, '/a.txt', _config) + .then(function() { + // Change the file. We'll use this to detect if the file is changed again. + fs.writeFileSync(dest, 'b'); + + // Change the mtime, so the s3 file appears older. + // Need to do this so sync will upload the file if hashes don't match + // (uploaded file will have more recent mtime, preventing upload). + var doc = yaml.readFileSync(meta)[0]; + doc[':modified_date'] = 0; + md5 = doc[':md5']; + yaml.writeFileSync(meta, doc); + + // Re-upload. Files should have matching gzip md5s, so s3 file shouldn't change. + s3.sync(src, '/a.txt', _config).then(function() { + test.ok(md5 !== hashFile(dest), 'Does not reupload gzipped data if they match.'); + }) + .always(function() { + test.done(); + }); + }); } };