From e295106eb56e61797254b5e5b1870e3cc4112eac Mon Sep 17 00:00:00 2001 From: Timo Tijhof Date: Sat, 13 Mar 2021 20:42:32 +0000 Subject: [PATCH] Tests: Make CLI test cases easier to debug with normalised stack Remove use of regexes and instead strictly compare exact strings, but normalise two things: * collapse any Node internals into a single frame called "internal". * strip line/column numbers of our own build artefact. --- test/cli/fixtures/expected/tap-outputs.js | 58 +++-- test/cli/helpers/execute.js | 34 ++- test/cli/main.js | 39 +--- test/cli/watch.js | 245 +++++++++++----------- 4 files changed, 190 insertions(+), 186 deletions(-) diff --git a/test/cli/fixtures/expected/tap-outputs.js b/test/cli/fixtures/expected/tap-outputs.js index b4c7d3794..06081e155 100644 --- a/test/cli/fixtures/expected/tap-outputs.js +++ b/test/cli/fixtures/expected/tap-outputs.js @@ -60,14 +60,13 @@ not ok 1 Throws match > bad severity: failed actual : "Error: Match me with a pattern" expected: "/incorrect pattern/" - stack: .* + stack: at Object. (/qunit/test/cli/fixtures/fail/throws-match.js:3:10) ... 1..1 # pass 0 # skip 0 # todo 0 -# fail 1 -`, +# fail 1`, "qunit test single.js 'glob/**/*-test.js'": `TAP version 13 @@ -140,10 +139,9 @@ not ok 2 global failure # pass 0 # skip 0 # todo 0 -# fail 2 -`, +# fail 2`, - // Ignore the last frame about Node processing ticks (differs between Node 10 ad 12+) + // The last frame differs between Node 10 and 12+ (changes in processing of ticks) "qunit no-tests": `TAP version 13 not ok 1 global failure @@ -153,18 +151,17 @@ not ok 1 global failure actual : {} expected: undefined stack: Error: No tests were run. - at done (.*) - at advanceTestQueue (.*) - at Object.advance (.*) - at unblockAndAdvanceQueue (.*)(\n at )? - at .* + at done (/qunit/qunit/qunit.js) + at advanceTestQueue (/qunit/qunit/qunit.js) + at Object.advance (/qunit/qunit/qunit.js) + at unblockAndAdvanceQueue (/qunit/qunit/qunit.js) + at internal ... 1..1 # pass 0 # skip 0 # todo 0 -# fail 1 -`, +# fail 1`, "qunit sourcemap/source.js": `TAP version 13 @@ -181,8 +178,7 @@ not ok 2 Example > bad # pass 1 # skip 0 # todo 0 -# fail 1 -`, +# fail 1`, "NODE_OPTIONS='--enable-source-maps' qunit sourcemap/source.min.js": `TAP version 13 @@ -193,19 +189,18 @@ not ok 2 Example > bad severity: failed actual : false expected: true - stack: at .* \\(.*source.min.js:1:.*\\) - -> .*source.js:7:10 + stack: at Object. (/qunit/test/cli/fixtures/sourcemap/source.min.js:1:133) + -> /qunit/test/cli/fixtures/sourcemap/sourcemap/source.js:7:10 ... 1..2 # pass 1 # skip 0 # todo 0 -# fail 1 -`, +# fail 1`, "qunit ../../es2018/esm.mjs": `TAP version 13 -ok 1 ESM test suite > sum\\(\\) +ok 1 ESM test suite > sum() 1..1 # pass 1 # skip 0 @@ -220,16 +215,14 @@ not ok 1 timeout > first severity: failed actual : null expected: undefined - stack: at .* (.*timers.js.*) - at .*(\n at .*)?(\n at .*)? + stack: at internal ... ok 2 timeout > second 1..2 # pass 1 # skip 0 # todo 0 -# fail 1 -`, +# fail 1`, "qunit zero-assertions.js": `TAP version 13 @@ -250,18 +243,17 @@ not ok 1 global failure actual : {} expected: undefined stack: Error: No tests matched the filter "no matches". - at done (.*) - at advanceTestQueue (.*) - at Object.advance (.*) - at unblockAndAdvanceQueue (.*)(\n at )? - at .* + at done (/qunit/qunit/qunit.js) + at advanceTestQueue (/qunit/qunit/qunit.js) + at Object.advance (/qunit/qunit/qunit.js) + at unblockAndAdvanceQueue (/qunit/qunit/qunit.js) + at internal ... 1..1 # pass 0 # skip 0 # todo 0 -# fail 1 -`, +# fail 1`, "qunit single.js --require require-dep --require './node_modules/require-dep/module.js'": `required require-dep/index.js @@ -302,7 +294,7 @@ not ok 1 # TODO module B > Only this module should run > a todo test severity: todo actual : false expected: true - stack: .* + stack: at Object. (/qunit/test/cli/fixtures/only/module.js:17:15) ... ok 2 # SKIP module B > Only this module should run > implicitly skipped test ok 3 module B > Only this module should run > normal test @@ -323,7 +315,7 @@ not ok 1 # TODO module B > test B severity: todo actual : false expected: true - stack: .* + stack: at Object. (/qunit/test/cli/fixtures/only/module-flat.js:9:13) ... ok 2 # SKIP module B > test C ok 3 module B > test D diff --git a/test/cli/helpers/execute.js b/test/cli/helpers/execute.js index a82345ebf..baf7539fb 100644 --- a/test/cli/helpers/execute.js +++ b/test/cli/helpers/execute.js @@ -2,18 +2,48 @@ const path = require( "path" ); const exec = require( "execa" ).shell; +const reEscape = /([\\{}()|.?*+\-^$[\]])/g; + +// Apply light normalization to CLI output to allow strict string +// comparison across Node versions and OS platforms against the +// expected output in fixtures/. +function normalize( actual ) { + const dir = path.join( __dirname, "..", "..", ".." ); + const reDir = new RegExp( dir.replace( reEscape, "\\$1" ), "g" ); + + return actual + .replace( reDir, "/qunit" ) + .replace( /(\/qunit\/qunit\/qunit\.js):\d+:\d+\)/g, "$1)" ) + .replace( / at .+\([^/)][^)]*\)/g, " at internal" ) + + // merge successive lines after initial frame + .replace( /(\n\s+at internal)+/g, "$1" ) + + // merge successive line with initial frame + .replace( /(at internal)\n\s+at internal/g, "$1" ); +} // Executes the provided command from within the fixtures directory // The execaOptions parameter is used by test/cli/watch.js to // control the stdio stream. -module.exports = function execute( command, execaOptions ) { +module.exports = async function execute( command, execaOptions, hook ) { const cwd = process.cwd(); process.chdir( path.join( __dirname, "..", "fixtures" ) ); command = command.replace( /(^| )qunit\b/, "$1../../../bin/qunit.js" ); const execution = exec( command, execaOptions ); + if ( hook ) { + hook( execution ); + } process.chdir( cwd ); - return execution; + try { + const result = await execution; + result.stdout = normalize( String( result.stdout ).trimEnd() ); + return result; + } catch ( e ) { + e.stdout = normalize( String( e.stdout ).trimEnd() ); + throw e; + } }; diff --git a/test/cli/main.js b/test/cli/main.js index 4b7a078fd..e60578631 100644 --- a/test/cli/main.js +++ b/test/cli/main.js @@ -4,15 +4,6 @@ const expectedOutput = require( "./fixtures/expected/tap-outputs" ); const execute = require( "./helpers/execute" ); const semver = require( "semver" ); -QUnit.assert.matches = function( actual, expected, message ) { - this.pushResult( { - result: expected.test( actual ), - actual, - expected: expected.toString(), - message - } ); -}; - QUnit.module( "CLI Main", function() { QUnit.test( "defaults to running tests in 'test' directory", async function( assert ) { const command = "qunit"; @@ -93,8 +84,7 @@ QUnit.module( "CLI Main", function() { } catch ( e ) { assert.equal( e.code, 1 ); assert.equal( e.stderr, "" ); - const re = new RegExp( expectedOutput[ command ] ); - assert.equal( re.test( e.stdout ), true ); + assert.equal( e.stdout, expectedOutput[ command ] ); } } ); @@ -113,8 +103,7 @@ QUnit.module( "CLI Main", function() { } catch ( e ) { assert.equal( e.code, 1 ); assert.equal( e.stderr, "" ); - const re = new RegExp( expectedOutput[ command ] ); - assert.equal( re.test( e.stdout ), true ); + assert.equal( e.stdout, expectedOutput[ command ] ); } } ); @@ -158,11 +147,7 @@ QUnit.module( "CLI Main", function() { assert.equal( execution.stderr, "" ); } - const re = new RegExp( expectedOutput[ command ] ); - assert.equal( re.test( execution.stdout ), true ); - if ( !re.test( execution.stdout ) ) { - assert.equal( execution.stdout, expectedOutput[ command ] ); - } + assert.equal( execution.stdout, expectedOutput[ command ] ); } ); } @@ -188,11 +173,7 @@ QUnit.module( "CLI Main", function() { } catch ( e ) { assert.equal( e.code, 1 ); assert.equal( e.stderr, "" ); - const re = new RegExp( expectedOutput[ command ] ); - assert.equal( re.test( e.stdout ), true ); - if ( !re.test( e.stdout ) ) { - assert.equal( e.stdout, expectedOutput[ command ] ); - } + assert.equal( e.stdout, expectedOutput[ command ] ); } } ); } @@ -204,8 +185,7 @@ QUnit.module( "CLI Main", function() { } catch ( e ) { assert.equal( e.code, 1 ); assert.equal( e.stderr, "" ); - const re = new RegExp( expectedOutput[ command ] ); - assert.equal( re.test( e.stdout ), true ); + assert.equal( e.stdout, expectedOutput[ command ] ); } } ); @@ -249,8 +229,7 @@ QUnit.module( "CLI Main", function() { } catch ( e ) { assert.equal( e.code, 1 ); assert.equal( e.stderr, "" ); - const re = new RegExp( expectedOutput[ command ] ); - assert.equal( re.test( e.stdout ), true ); + assert.equal( e.stdout, expectedOutput[ command ] ); } } ); } ); @@ -323,8 +302,7 @@ QUnit.module( "CLI Main", function() { assert.equal( execution.code, 0 ); assert.equal( execution.stderr, "" ); - const re = new RegExp( expectedOutput[ command ] ); - assert.matches( execution.stdout, re ); + assert.equal( execution.stdout, expectedOutput[ command ] ); } ); QUnit.test( "flat modules", async function( assert ) { @@ -334,8 +312,7 @@ QUnit.module( "CLI Main", function() { assert.equal( execution.code, 0 ); assert.equal( execution.stderr, "" ); - const re = new RegExp( expectedOutput[ command ] ); - assert.matches( execution.stdout, re ); + assert.equal( execution.stdout, expectedOutput[ command ] ); } ); } ); } ); diff --git a/test/cli/watch.js b/test/cli/watch.js index 197ac6110..51d07e65a 100644 --- a/test/cli/watch.js +++ b/test/cli/watch.js @@ -9,8 +9,8 @@ const expectedWatchOutput = require( "./fixtures/expected/watch-tap-outputs" ); const executeHelper = require( "./helpers/execute" ); // Executes the provided command from within the fixtures directory -function execute( command ) { - return executeHelper( command, { stdio: [ null, null, null, "ipc" ] } ); +function execute( command, hook ) { + return executeHelper( command, { stdio: [ null, null, null, "ipc" ] }, hook ); } const fixturePath = path.join( __dirname, "fixtures", "watching" ); @@ -39,20 +39,21 @@ QUnit.module( "CLI Watch", function( hooks ) { rimraf.sync( fixturePath ); } ); - QUnit.test( "runs tests and waits until SIGTERM", async function( assert ) { + QUnit.test( "runs tests and waits until SIGTERM", async assert => { fixturify.writeSync( fixturePath, { "foo.js": "QUnit.test('foo', function(assert) { assert.true(true); });" } ); const command = "qunit watching"; - const execution = execute( `${command} --watch` ); - - execution.on( "message", function( data ) { - assert.step( data ); - kill( execution, "SIGTERM" ); - } ); - - const result = await execution; + const result = await execute( + `${command} --watch`, + execution => { + execution.on( "message", data => { + assert.step( data ); + kill( execution, "SIGTERM" ); + } ); + } + ); assert.verifySteps( [ "runEnd" ] ); assert.equal( result.code, 0 ); @@ -60,20 +61,21 @@ QUnit.module( "CLI Watch", function( hooks ) { assert.equal( result.stdout, expectedWatchOutput[ "no-change" ] ); } ); - QUnit.test( "runs tests and waits until SIGINT", async function( assert ) { + QUnit.test( "runs tests and waits until SIGINT", async assert => { fixturify.writeSync( fixturePath, { "foo.js": "QUnit.test('foo', function(assert) { assert.true(true); });" } ); const command = "qunit watching"; - const execution = execute( `${command} --watch` ); - - execution.on( "message", function( data ) { - assert.step( data ); - kill( execution ); - } ); - - const result = await execution; + const result = await execute( + `${command} --watch`, + execution => { + execution.on( "message", data => { + assert.step( data ); + kill( execution ); + } ); + } + ); assert.verifySteps( [ "runEnd" ] ); assert.equal( result.code, 0 ); @@ -81,27 +83,28 @@ QUnit.module( "CLI Watch", function( hooks ) { assert.equal( result.stdout, expectedWatchOutput[ "no-change" ] ); } ); - QUnit.test( "re-runs tests on file changed", async function( assert ) { + QUnit.test( "re-runs tests on file changed", async assert => { fixturify.writeSync( fixturePath, { "foo.js": "QUnit.test('foo', function(assert) { assert.true(true); });" } ); const command = "qunit watching"; - const execution = execute( `${command} --watch` ); - - execution.once( "message", function( data ) { - assert.step( data ); - fixturify.writeSync( fixturePath, { - "foo.js": "QUnit.test('bar', function(assert) { assert.true(true); });" - } ); - - execution.once( "message", function( data ) { - assert.step( data ); - kill( execution ); - } ); - } ); + const result = await execute( + `${command} --watch`, + execution => { + execution.once( "message", data => { + assert.step( data ); + fixturify.writeSync( fixturePath, { + "foo.js": "QUnit.test('bar', function(assert) { assert.true(true); });" + } ); - const result = await execution; + execution.once( "message", data => { + assert.step( data ); + kill( execution ); + } ); + } ); + } + ); assert.verifySteps( [ "runEnd", "runEnd" ] ); assert.equal( result.code, 0 ); @@ -109,27 +112,28 @@ QUnit.module( "CLI Watch", function( hooks ) { assert.equal( result.stdout, expectedWatchOutput[ "change-file" ] ); } ); - QUnit.test( "re-runs tests on file added", async function( assert ) { + QUnit.test( "re-runs tests on file added", async assert => { fixturify.writeSync( fixturePath, { "foo.js": "QUnit.test('foo', function(assert) { assert.true(true); });" } ); const command = "qunit watching"; - const execution = execute( `${command} --watch` ); - - execution.once( "message", function( data ) { - assert.step( data ); - fixturify.writeSync( fixturePath, { - "bar.js": "QUnit.test('bar', function(assert) { assert.true(true); });" - } ); - - execution.once( "message", function( data ) { - assert.step( data ); - kill( execution ); - } ); - } ); + const result = await execute( + `${command} --watch`, + execution => { + execution.once( "message", data => { + assert.step( data ); + fixturify.writeSync( fixturePath, { + "bar.js": "QUnit.test('bar', function(assert) { assert.true(true); });" + } ); - const result = await execution; + execution.once( "message", data => { + assert.step( data ); + kill( execution ); + } ); + } ); + } + ); assert.verifySteps( [ "runEnd", "runEnd" ] ); assert.equal( result.code, 0 ); @@ -137,28 +141,29 @@ QUnit.module( "CLI Watch", function( hooks ) { assert.equal( result.stdout, expectedWatchOutput[ "add-file" ] ); } ); - QUnit.test( "re-runs tests on file removed", async function( assert ) { + QUnit.test( "re-runs tests on file removed", async assert => { fixturify.writeSync( fixturePath, { "foo.js": "QUnit.test('foo', function(assert) { assert.true(true); });", "bar.js": "QUnit.test('bar', function(assert) { assert.true(true); });" } ); const command = "qunit watching"; - const execution = execute( `${command} --watch` ); - - execution.once( "message", function( data ) { - assert.step( data ); - fixturify.writeSync( fixturePath, { - "bar.js": null - } ); - - execution.once( "message", function( data ) { - assert.step( data ); - kill( execution ); - } ); - } ); + const result = await execute( + `${command} --watch`, + execution => { + execution.once( "message", data => { + assert.step( data ); + fixturify.writeSync( fixturePath, { + "bar.js": null + } ); - const result = await execution; + execution.once( "message", data => { + assert.step( data ); + kill( execution ); + } ); + } ); + } + ); assert.verifySteps( [ "runEnd", "runEnd" ] ); assert.equal( result.code, 0 ); @@ -166,7 +171,7 @@ QUnit.module( "CLI Watch", function( hooks ) { assert.equal( result.stdout, expectedWatchOutput[ "remove-file" ] ); } ); - QUnit.test( "aborts and restarts when in middle of run", async function( assert ) { + QUnit.test( "aborts and restarts when in middle of run", async assert => { // A proper abort finishes the currently running test and runs any remaining // afterEach/after hooks to ensure cleanup happens. @@ -196,32 +201,31 @@ QUnit.module( "CLI Watch", function( hooks ) { } ); const command = "qunit watching/tests"; - const execution = execute( `${command} --watch` ); - - function one( data ) { - if ( data === "testRunning" ) { - fixturify.writeSync( fixturePath, { - "bar.js": "module.exports = 'bar export second';" - } ); - } - - assert.step( data ); + const result = await execute( + `${command} --watch`, + execution => { + execution.on( "message", function handle( data ) { + if ( data === "testRunning" ) { + fixturify.writeSync( fixturePath, { + "bar.js": "module.exports = 'bar export second';" + } ); + } - if ( data === "runEnd" ) { - execution.removeListener( "message", one ); - execution.addListener( "message", function( data ) { assert.step( data ); if ( data === "runEnd" ) { - kill( execution ); + execution.off( "message", handle ); + execution.on( "message", data => { + assert.step( data ); + + if ( data === "runEnd" ) { + kill( execution ); + } + } ); } } ); } - } - - execution.addListener( "message", one ); - - const result = await execution; + ); assert.verifySteps( [ "bar export first", @@ -246,7 +250,7 @@ QUnit.module( "CLI Watch", function( hooks ) { assert.equal( result.stdout, expectedWatchOutput[ "change-file-mid-run" ] ); } ); - QUnit.test( "properly watches files after initial run", async function( assert ) { + QUnit.test( "properly watches files after initial run", async assert => { fixturify.writeSync( fixturePath, { "tests": { @@ -259,43 +263,44 @@ QUnit.module( "CLI Watch", function( hooks ) { } } ); - const command = "qunit watching/tests"; - const execution = execute( `${command} --watch` ); let count = 0; + const command = "qunit watching/tests"; + const result = await execute( + `${command} --watch`, + execution => { + execution.on( "message", data => { + assert.step( data ); - execution.addListener( "message", function( data ) { - assert.step( data ); - - if ( data === "runEnd" ) { - count++; - - if ( count === 1 ) { - fixturify.writeSync( fixturePath, { - "tests": { - "foo.js": ` - process.send(require('../bar.js')); - QUnit.module('Module'); - QUnit.test('Test', function(assert) { - assert.true(true); - });` - }, - "bar.js": "module.exports = 'bar export first';" - } ); - } - - if ( count === 2 ) { - fixturify.writeSync( fixturePath, { - "bar.js": "module.exports = 'bar export second';" - } ); - } - - if ( count === 3 ) { - kill( execution ); - } + if ( data === "runEnd" ) { + count++; + + if ( count === 1 ) { + fixturify.writeSync( fixturePath, { + "tests": { + "foo.js": ` + process.send(require('../bar.js')); + QUnit.module('Module'); + QUnit.test('Test', function(assert) { + assert.true(true); + });` + }, + "bar.js": "module.exports = 'bar export first';" + } ); + } + + if ( count === 2 ) { + fixturify.writeSync( fixturePath, { + "bar.js": "module.exports = 'bar export second';" + } ); + } + + if ( count === 3 ) { + kill( execution ); + } + } + } ); } - } ); - - const result = await execution; + ); assert.verifySteps( [ "runEnd",