From 7f1e900627c41faf833c02a298b3f158d592e9e5 Mon Sep 17 00:00:00 2001 From: mistakia <1823355+mistakia@users.noreply.github.com> Date: Tue, 2 Feb 2021 17:22:17 -0500 Subject: [PATCH] feat: save/compare results to/from file --- .gitignore | 1 + package-lock.json | 33 +++++++++++++++++++++++++++++++-- package.json | 1 + src/cli.js | 12 ++++++++++++ src/index.js | 9 ++++++++- src/report.js | 37 +++++++++++++++++++++++++++---------- test/cli.test.js | 3 +++ test/compare.test.js | 31 +++++++++++++++++++++++++++++++ test/save.test.js | 41 +++++++++++++++++++++++++++++++++++++++++ 9 files changed, 155 insertions(+), 13 deletions(-) create mode 100644 test/compare.test.js create mode 100644 test/save.test.js diff --git a/.gitignore b/.gitignore index c9106a7..a30cecf 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ node_modules .nyc_output +test/tmp diff --git a/package-lock.json b/package-lock.json index 256f63f..0b743ca 100644 --- a/package-lock.json +++ b/package-lock.json @@ -489,6 +489,11 @@ "integrity": "sha512-+Ryf6g3BKoRc7jfp7ad8tM4TtMiaWvbF/1/sQcZPkkS7ag3D5nMBCe2UfOTONtAkaG0tO0ij3C5Lwmf1EiyjHg==", "dev": true }, + "at-least-node": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/at-least-node/-/at-least-node-1.0.0.tgz", + "integrity": "sha512-+q/t7Ekv1EDY2l6Gda6LLiX14rU9TV20Wa3ofeQmwPFZbOMo9DXrLbOjFaaclkXKWidIaopwAObQDqwWtGUjqg==" + }, "balanced-match": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz", @@ -1520,6 +1525,17 @@ "integrity": "sha512-Xu2Qh8yqYuDhQGOhD5iJGninErSfI9A3FrriD3tjUgV5VbJFeH8vfgZ9HnC6jWN80QDVNQK5vmxRAmEAp7Mevw==", "dev": true }, + "fs-extra": { + "version": "9.1.0", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-9.1.0.tgz", + "integrity": "sha512-hcg3ZmepS30/7BSFqRvoo3DOMQu7IjqxO5nCDt+zM9XWjb33Wg7ziNT+Qvqbuc3+gWpzO02JubVyk2G4Zvo1OQ==", + "requires": { + "at-least-node": "^1.0.0", + "graceful-fs": "^4.2.0", + "jsonfile": "^6.0.1", + "universalify": "^2.0.0" + } + }, "fs.realpath": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", @@ -1606,8 +1622,7 @@ "graceful-fs": { "version": "4.2.4", "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.4.tgz", - "integrity": "sha512-WjKPNJF79dtJAVniUlGGWHYGz2jWxT6VhN/4m1NdkbZ2nOsEF+cI1Edgql5zCRhs/VsQYRvrXctxktVXZUkixw==", - "dev": true + "integrity": "sha512-WjKPNJF79dtJAVniUlGGWHYGz2jWxT6VhN/4m1NdkbZ2nOsEF+cI1Edgql5zCRhs/VsQYRvrXctxktVXZUkixw==" }, "growl": { "version": "1.10.5", @@ -2030,6 +2045,15 @@ "minimist": "^1.2.5" } }, + "jsonfile": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-6.1.0.tgz", + "integrity": "sha512-5dgndWOriYSm5cnYaJNhalLNDKOqFwyDB/rr1E9ZsGciGvKPs8R2xYGCacuf3z6K1YKDz182fd+fY3cn3pMqXQ==", + "requires": { + "graceful-fs": "^4.1.6", + "universalify": "^2.0.0" + } + }, "jsx-ast-utils": { "version": "2.4.1", "resolved": "https://registry.npmjs.org/jsx-ast-utils/-/jsx-ast-utils-2.4.1.tgz", @@ -3457,6 +3481,11 @@ "integrity": "sha1-sxxa6CVIRKOoKBVBzisEuGWnNP8=", "dev": true }, + "universalify": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/universalify/-/universalify-2.0.0.tgz", + "integrity": "sha512-hAZsKq7Yy11Zu1DE0OzWjw7nnLZmJZYTDZZyEFHZdUhV8FkH5MCfoU1XMaxXovpyW5nq5scPqq0ZDP9Zyl04oQ==" + }, "uri-js": { "version": "4.3.0", "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.3.0.tgz", diff --git a/package.json b/package.json index c8f7b34..77aff1e 100644 --- a/package.json +++ b/package.json @@ -13,6 +13,7 @@ "license": "MIT", "dependencies": { "expose-gc": "^1.0.0", + "fs-extra": "^9.1.0", "yargs": "^15.4.1" }, "localMaintainers": [ diff --git a/src/cli.js b/src/cli.js index b0d21a6..0109eef 100755 --- a/src/cli.js +++ b/src/cli.js @@ -9,6 +9,18 @@ const argv = yargs .version(false) .help('help').alias('help', 'h') .options({ + save: { + alias: 's', + description: 'Save results to file', + requiresArg: true, + required: false, + }, + compare: { + alias: 'c', + description: 'Compare results to file', + requiresArg: true, + required: false, + }, baseline: { alias: 'b', description: 'Run baseline benchmarks only', diff --git a/src/index.js b/src/index.js index b0b41b7..1562906 100644 --- a/src/index.js +++ b/src/index.js @@ -2,6 +2,7 @@ require('expose-gc') global.gc() const os = require('os') +const fs = require('fs-extra') const report = require('./report') @@ -52,8 +53,14 @@ const start = async (benchmarks, argv) => { output += ` in ${(runnerElapsed / 1000000000).toFixed(2)} seconds` process.stdout.write(output) + if (baselineOnly && argv.save) { + fs.ensureFileSync(argv.save) + fs.writeJsonSync(argv.save, results, { spaces: 2 }) + } + if (argv.report) { - report(results) + const compare = argv.compare ? fs.readJsonSync(argv.compare) : undefined + report(results, compare) } } catch (e) { console.log(e) diff --git a/src/report.js b/src/report.js index 55bfe62..02d844d 100644 --- a/src/report.js +++ b/src/report.js @@ -9,7 +9,7 @@ const padStr = (str, max) => { return str + repeatStr(' ', length) } -const reporter = (results) => { +const reporter = (results, compare = []) => { const reports = [{ name: 'Benchmark Name', ops: 'Ops / ms', @@ -22,34 +22,51 @@ const reporter = (results) => { let maxMemWidth = reports[0].mem.length for (const benchmark of results) { + const savedBenchmark = compare.find(c => c.name === benchmark.name) + const nameWidth = benchmark.name.length if (maxNameWidth < nameWidth) { maxNameWidth = nameWidth } - const totalMs = (benchmark.elapsed / 1000000).toFixed(4) - const opsPerMs = (benchmark.stats.count / totalMs).toFixed(4) - const memUsed = benchmark.memory.after.heapUsed - benchmark.memory.before.heapUsed - const memUsedMb = (memUsed / 1024 / 1024).toFixed(2) + const getCalculations = (b) => { + const totalMs = (b.elapsed / 1000000).toFixed(4) + const opsPerMs = (b.stats.count / totalMs).toFixed(4) + const memUsed = b.memory.after.heapUsed - b.memory.before.heapUsed + const memUsedMb = (memUsed / 1024 / 1024).toFixed(2) + return { totalMs, opsPerMs, memUsed, memUsedMb } + } + + const calculations = getCalculations(benchmark) + const savedCalculations = savedBenchmark ? getCalculations(savedBenchmark) : undefined + const getOutput = (type) => { + const delta = savedCalculations ? (calculations[type] - savedCalculations[type]).toFixed(2) : 0 + return delta ? `${calculations[type]} (${delta})` : calculations[type] + } - const opsWidth = opsPerMs.toString().length + const ops = getOutput('opsPerMs') + const opsWidth = ops.toString().length if (maxOpsWidth < opsWidth) { maxOpsWidth = opsWidth } + + const totalMs = getOutput('totalMs') const totalWidth = totalMs.toString().length if (maxTotalWidth < totalWidth) { maxTotalWidth = totalWidth } - const memWidth = memUsedMb.toString().length + + const mem = getOutput('memUsedMb') + const memWidth = mem.toString().length if (maxMemWidth < memWidth) { maxMemWidth = memWidth } reports.push({ name: benchmark.name, - ops: opsPerMs, - totalMs: totalMs, - mem: memUsedMb + ops, + totalMs, + mem }) } diff --git a/test/cli.test.js b/test/cli.test.js index 191b155..a8795be 100644 --- a/test/cli.test.js +++ b/test/cli.test.js @@ -24,6 +24,8 @@ describe('CLI Test', () => { expect(output).to.contain('stressLimit') expect(output).to.contain('baselineLimit') expect(output).to.contain('logLimit') + expect(output).to.contain('save') + expect(output).to.contain('compare') }) describe('baseline', () => { @@ -270,4 +272,5 @@ describe('CLI Test', () => { await start(benchmarks, yargsResult) }) }) + }) diff --git a/test/compare.test.js b/test/compare.test.js new file mode 100644 index 0000000..afc20ef --- /dev/null +++ b/test/compare.test.js @@ -0,0 +1,31 @@ +/* eslint-disable no-unused-expressions */ + +const yargs = require('yargs') +const path = require('path') +const fs = require('fs-extra') +const cli = require('../src/cli') +const expect = require('chai').expect +const { start } = require('../src/index') + +const benchmarks = require(process.cwd() + '/test/benchmarks') + +describe('CLI Compare', function () { + this.timeout(6000) + + it('should compare results to file', async () => { + const yargsCmd = yargs.command(cli) + const tmpFile = path.join(__dirname, 'tmp/benchmark.json') + const yargsResult1 = yargsCmd.parse( + `cli --baseline -s ${tmpFile}`, {} + ) + + await start(benchmarks, yargsResult1) + + const yargsResult2 = yargsCmd.parse( + `cli --baseline -r -c ${tmpFile}`, {} + ) + await start(benchmarks, yargsResult2) + + // TODO - test reporter output + }) +}) diff --git a/test/save.test.js b/test/save.test.js new file mode 100644 index 0000000..ee96923 --- /dev/null +++ b/test/save.test.js @@ -0,0 +1,41 @@ +/* eslint-disable no-unused-expressions */ + +const yargs = require('yargs') +const path = require('path') +const fs = require('fs-extra') +const cli = require('../src/cli') +const expect = require('chai').expect +const { start } = require('../src/index') + +const benchmarks = require(process.cwd() + '/test/benchmarks') + +describe('CLI Save', function () { + this.timeout(6000) + + it('should save results to file', async () => { + const yargsCmd = yargs.command(cli) + const tmpFile = path.join(__dirname, 'tmp/benchmark.json') + const yargsResult = yargsCmd.parse( + `cli --baseline -s ${tmpFile}`, {} + ) + + await start(benchmarks, yargsResult) + + const results = fs.readJsonSync(tmpFile) + expect(results).to.be.an('array') + expect(results.length).to.be.equal(1) + + const benchmark = results[0] + expect(benchmark).to.have.property('name') + expect(benchmark).to.have.property('cpus') + expect(benchmark).to.have.property('loadavg') + expect(benchmark).to.have.property('elapsed') + expect(benchmark).to.have.property('stats') + expect(benchmark).to.have.property('memory') + + expect(benchmark.name).to.be.equal('console-baseline') + expect(benchmark.memory).to.have.property('before') + expect(benchmark.memory).to.have.property('after') + expect(benchmark.stats).to.have.property('count') + }) +})