diff --git a/src/nodes/Merger.js b/src/nodes/Merger.js index 8343309..7ac2069 100644 --- a/src/nodes/Merger.js +++ b/src/nodes/Merger.js @@ -85,6 +85,17 @@ export default class Merger extends Node { }); } + getFileFromChecksum ( checksum ) { + let i = this.inputs.length; + let file; + + while ( i-- ) { + if ( file = this.inputs[i].getFileFromChecksum( checksum ) ) { + return file; + } + } + } + ready () { let aborted; let index; @@ -102,37 +113,35 @@ export default class Merger extends Node { this._ready = mkdir( outputdir ).then( () => { let start; - let inputdirs = []; - - return mapSeries( this.inputs, function ( input, i ) { - if ( aborted ) throw ABORTED; - return input.ready().then( inputdir => inputdirs[i] = inputdir ); - }).then( () => { - start = Date.now(); - - this.emit( 'info', { - code: 'MERGE_START', - id: this.id, - progressIndicator: true - }); - return mapSeries( inputdirs, inputdir => { + return mapSeries( this.inputs, x => x.ready() ) + .then( inputdirs => { + start = Date.now(); + + this.emit( 'info', { + code: 'MERGE_START', + id: this.id, + progressIndicator: true + }); + + return mapSeries( inputdirs, inputdir => { + if ( aborted ) throw ABORTED; + return mergeDirectories( inputdir, outputdir ); + }); + }) + .then( () => { if ( aborted ) throw ABORTED; - return mergeDirectories( inputdir, outputdir ); - }); - }).then( () => { - if ( aborted ) throw ABORTED; - this._cleanup( index ); + this._cleanup( index ); - this.emit( 'info', { - code: 'MERGE_COMPLETE', - id: this.id, - duration: Date.now() - start - }); + this.emit( 'info', { + code: 'MERGE_COMPLETE', + id: this.id, + duration: Date.now() - start + }); - return outputdir; - }); + return outputdir; + }); }); } diff --git a/src/nodes/Observer.js b/src/nodes/Observer.js index 7ba7ac6..6b64bf2 100644 --- a/src/nodes/Observer.js +++ b/src/nodes/Observer.js @@ -33,6 +33,10 @@ export default class Observer extends Node { this.input.on( 'info', this._oninfo ); } + getFileFromChecksum ( checksum ) { + return this.input.getFileFromChecksum( checksum ); + } + ready () { let observation; diff --git a/src/nodes/Source.js b/src/nodes/Source.js index c23f2f7..b0593b5 100644 --- a/src/nodes/Source.js +++ b/src/nodes/Source.js @@ -1,5 +1,7 @@ import { basename, relative, resolve } from 'path'; -import { link, linkSync, mkdirSync, statSync, Promise } from 'sander'; +import { lsr, link, linkSync, readFileSync, mkdirSync, statSync, unlinkSync, Promise } from 'sander'; +import queue from '../queue/index.js'; +import { crc32 } from 'crc'; import { watch } from 'chokidar'; import * as debounce from 'debounce'; import Node from './Node'; @@ -13,6 +15,8 @@ export default class Source extends Node { this.id = options.id || 'source'; this.dir = dir; + this.checksumByFile = {}; + this.fileByChecksum = {}; this.callbacks = []; this._entries = {}; @@ -21,16 +25,14 @@ export default class Source extends Node { const stats = statSync( this.dir ); if ( !stats.isDirectory() ) { + this.isFileSource = true; + this.file = dir; - this.dir = undefined; + this.dir = null; this.uid = uid( this.id ); - - this._ready = new Promise( ( ok, fail ) => { - this._deferred = { ok, fail }; - }); } else { - this._ready = Promise.resolve( this.dir ); + // this._ready = Promise.resolve( this.dir ); } } catch ( err ) { if ( err.code === 'ENOENT' ) { @@ -47,7 +49,41 @@ export default class Source extends Node { this.static = options && options.static; } + getFileFromChecksum ( checksum ) { + return this.fileByChecksum[ checksum ]; + } + ready () { + if ( !this._ready ) { + this._ready = queue.add( ( fulfil, reject ) => { + const start = Date.now(); + + this._makeReady(); + + lsr( this.dir ) + .then( files => { + files.forEach( file => { + const absolutePath = resolve( this.dir, file ); + const buffer = readFileSync( absolutePath ); + const checksum = crc32( buffer ); + + this.checksumByFile[ absolutePath ] = checksum; + this.fileByChecksum[ checksum ] = absolutePath; + }); + + // For most situations, generating checksums takes no time at all, + // but it's probably worth warning about this if it becomes a + // source of pain. TODO 'warn' event? + const duration = Date.now() - start; + if ( duration > 1000 ) { + this.emit( 'info', `the ${this.dir} directory took ${duration}ms to initialise - consider excluding unnecessary files from the build` ); + } + }) + .then( () => fulfil( this.dir ) ) + .catch( reject ); + }); + } + return this._ready; } @@ -56,17 +92,9 @@ export default class Source extends Node { return; } - this._active = true; - - // this is a file watch that isn't fully initialized - if ( this._deferred ) { - this._makeReady(); - } + this._makeReady(); - // make sure the file is in the appropriate target directory to start - if ( this.file ) { - linkSync( this.file ).to( this.targetFile ); - } + this._active = true; let changed = []; @@ -102,13 +130,14 @@ export default class Source extends Node { relay(); }); }); - } - - if ( this.file ) { - this._fileWatcher = watch( this.file, options ); + } else { + this._watcher = watch( this.dir, options ); - this._fileWatcher.on( 'change', () => { - link( this.file ).to( this.targetFile ); + [ 'add', 'change', 'unlink' ].forEach( type => { + this._watcher.on( type, path => { + changes.push({ type, path }); + relay(); + }); }); } } @@ -141,17 +170,12 @@ export default class Source extends Node { } _makeReady () { - this.dir = resolve( session.config.gobbledir, this.uid ); - this.targetFile = resolve( this.dir, basename( this.file ) ); + if ( this.isFileSource && !this._isReady ) { + this.dir = resolve( session.config.gobbledir, this.uid ); + this.targetFile = resolve( this.dir, basename( this.file ) ); - try { - mkdirSync( this.dir ); - this._deferred.ok( this.dir ); - } catch (e) { - this._deferred.fail( e ); - throw e; + linkSync( this.file ).to( this.targetFile ); + this._isReady = true; // TODO less conflicty flag name } - - delete this._deferred; } } diff --git a/src/nodes/Transformer.js b/src/nodes/Transformer.js index 4d61fba..d1e248e 100644 --- a/src/nodes/Transformer.js +++ b/src/nodes/Transformer.js @@ -46,6 +46,10 @@ export default class Transformer extends Node { this.input.on( 'info', this._oninfo ); } + getFileFromChecksum ( checksum ) { + return this.input.getFileFromChecksum( checksum ); + } + ready () { let outputdir; let transformation; diff --git a/src/nodes/build/index.js b/src/nodes/build/index.js index 45782ab..768c85d 100644 --- a/src/nodes/build/index.js +++ b/src/nodes/build/index.js @@ -35,7 +35,7 @@ export default function ( node, options ) { return node.ready() .then( inputdir => { return copydir( inputdir ).to( dest ) - .then( () => flattenSourcemaps( inputdir, dest, dest, task ) ); + .then( () => flattenSourcemaps( node, inputdir, dest, dest, task ) ); }) .then( () => { node.teardown(); diff --git a/src/nodes/serve/handleRequest.js b/src/nodes/serve/handleRequest.js index 5adecee..c08ab71 100644 --- a/src/nodes/serve/handleRequest.js +++ b/src/nodes/serve/handleRequest.js @@ -6,7 +6,7 @@ import serveDir from './serveDir'; import serveSourcemap from './serveSourcemap'; import serveError from './serveError'; -export default function handleRequest ( srcDir, error, sourcemapPromises, request, response ) { +export default function handleRequest ( node, srcDir, error, sourcemapPromises, request, response ) { const parsedUrl = parse( request.url ); const pathname = parsedUrl.pathname; @@ -31,7 +31,7 @@ export default function handleRequest ( srcDir, error, sourcemapPromises, reques filepath = join( srcDir, pathname ); if ( extname( filepath ) === '.map' ) { - return serveSourcemap( filepath, sourcemapPromises, request, response ) + return serveSourcemap( node, filepath, sourcemapPromises, request, response ) .catch( err => serveError( err, request, response ) ); } diff --git a/src/nodes/serve/index.js b/src/nodes/serve/index.js index dfb4eb4..b84fd2f 100644 --- a/src/nodes/serve/index.js +++ b/src/nodes/serve/index.js @@ -125,7 +125,7 @@ export default function serve ( node, options = {} ) { }); server.on( 'request', ( request, response ) => { - handleRequest( srcDir, error, sourcemapPromises, request, response ) + handleRequest( node, srcDir, error, sourcemapPromises, request, response ) .catch( err => task.emit( 'error', err ) ); }); diff --git a/src/nodes/serve/serveSourcemap.js b/src/nodes/serve/serveSourcemap.js index 01c8e0f..555c914 100644 --- a/src/nodes/serve/serveSourcemap.js +++ b/src/nodes/serve/serveSourcemap.js @@ -1,6 +1,9 @@ +import { crc32 } from 'crc'; +import { dirname, relative, resolve } from 'path'; +import { readFileSync } from 'sander'; import { load } from 'sorcery'; -export default function serveSourcemap ( filepath, sourcemapPromises, request, response ) { +export default function serveSourcemap ( node, filepath, sourcemapPromises, request, response ) { const owner = filepath.slice( 0, -4 ); if ( !sourcemapPromises[ filepath ] ) { @@ -10,7 +13,23 @@ export default function serveSourcemap ( filepath, sourcemapPromises, request, r throw new Error( 'Could not resolve sourcemap for ' + owner ); } - return chain.apply().toString(); + const map = chain.apply(); + const dir = dirname( owner ); + const cwd = process.cwd(); + + map.sources = map.sources.map( ( source, i ) => { + const content = map.sourcesContent[i]; + const checksum = crc32( content ); + const originalSource = node.getFileFromChecksum( checksum ); + + const absolutePath = resolve( dir, originalSource || source ); + + return relative( cwd, absolutePath ); + }); + + map.sourceRoot = 'file://' + process.cwd(); + + return map.toString(); }); } diff --git a/src/nodes/watch/index.js b/src/nodes/watch/index.js index cd17a51..24b5e37 100644 --- a/src/nodes/watch/index.js +++ b/src/nodes/watch/index.js @@ -40,7 +40,7 @@ export default function watch ( node, options ) { progressIndicator: true }); - return flattenSourcemaps( dir, dest, dest, task ).then( () => { + return flattenSourcemaps( node, dir, dest, dest, task ).then( () => { task.emit( 'info', { code: 'SOURCEMAP_PROCESS_COMPLETE', duration: Date.now() - sourcemapProcessStart diff --git a/src/utils/flattenSourcemaps.js b/src/utils/flattenSourcemaps.js index 314c070..88fbfd9 100644 --- a/src/utils/flattenSourcemaps.js +++ b/src/utils/flattenSourcemaps.js @@ -1,11 +1,13 @@ -import { extname, resolve } from 'path'; -import { lsr } from 'sander'; +import { basename, dirname, extname, relative, resolve } from 'path'; +import { lsr, readFileSync, writeFile } from 'sander'; import * as mapSeries from 'promise-map-series'; import { load } from 'sorcery'; +import { crc32 } from 'crc'; +import { SOURCEMAP_COMMENT, getSourcemapComment } from './sourcemap'; const whitelist = { '.js': true, '.css': true }; -export default function flattenSourcemaps ( inputdir, outputdir, base, task ) { +export default function flattenSourcemaps ( node, inputdir, outputdir, base, task ) { return lsr( inputdir ).then( files => { const jsAndCss = files.filter( file => whitelist[ extname( file ) ] ); @@ -13,7 +15,23 @@ export default function flattenSourcemaps ( inputdir, outputdir, base, task ) { return load( resolve( inputdir, file ) ) .then( chain => { if ( chain ) { - return chain.write( resolve( outputdir, file ), { base }); + const map = chain.apply({ base }); + + map.sources = map.sources.map( source => { + const checksum = crc32( readFileSync( base, source ) ); + const originalSource = node.getFileFromChecksum( checksum ); + + const dir = dirname( resolve( base, file ) ); + return originalSource ? relative( dir, originalSource ) : source; + }); + + const code = readFileSync( inputdir, file, { encoding: 'utf-8' }) + .replace( SOURCEMAP_COMMENT, getSourcemapComment( encodeURI( basename( file + '.map' ) ), extname( file ) ) ); + + return Promise.all([ + writeFile( outputdir, file, code ), + writeFile( outputdir, file + '.map', map.toString() ) + ]); } }) .catch( err => { diff --git a/test/sourcemaps.js b/test/sourcemaps.js index 178cdc5..ec2c01a 100644 --- a/test/sourcemaps.js +++ b/test/sourcemaps.js @@ -145,7 +145,7 @@ module.exports = function () { task.once( 'built', function () { task.once( 'built', function () { - var content = sander.readFileSync( 'tmp/output/baz' ); + var content = sander.readFileSync( 'tmp/output/baz' ).toString(); assert.equal( content, 'step2' ); done(); }); @@ -158,11 +158,7 @@ module.exports = function () { }); }); - task.on( 'error', function ( err ) { - setTimeout( function () { - throw err; - }); - }); + task.on( 'error', done ); }); it( 'does not use non-existent sourcemap files when reusing cached file transformer results', function ( done ) { @@ -392,6 +388,47 @@ module.exports = function () { }); }); + it( 'fixes `sources` with original files, where possible', function () { + var source = gobble( 'tmp/sourcemaps' ); + + return source + .include( 'app.coffee' ) + .moveTo( 'js' ) + .transform( 'coffee' ) + .build({ + dest: 'tmp/output' + }) + .then( function () { + return sander.readFile( 'tmp/output/js/app.js.map' ) + .then( String ) + .then( JSON.parse ) + .then( function ( map ) { + assert.deepEqual( map.sources, [ '../../sourcemaps/app.coffee' ]); + }); + }); + }); + + it( 'fixes `sources` with original files when serving', function ( done ) { + var source = gobble( 'tmp/sourcemaps' ); + + task = source + .include( 'app.coffee' ) + .moveTo( 'js' ) + .transform( 'coffee' ) + .serve(); + + task.on( 'error', done ); + + task.once( 'built', function () { + request( 'http://localhost:4567/js/app.js.map' ) + .then( JSON.parse ) + .then( function ( map ) { + assert.deepEqual( map.sources, [ 'tmp/sourcemaps/app.coffee' ]); + done(); + }); + }); + }); + it( 'should not get confused by filenames beginning with data', function () { var source = gobble( 'tmp/data' );