Move core functionality to index.js

WIP

refactor: move core functionality to index.js

suppressing output for CI

adding back in missing return statement

removing stray console.log

putting output back, sorry CI

rename IPFS log to OrbitDB

Removing rimraf

moving files to src

moving report and benchmarks folder

Adding global gc to index.js

linting

package.json

@@ -2,12 +2,12 @@
   "name": "orbit-db-benchmark-runner",
   "version": "1.0.4",
   "description": "OrbitDB Benchmark Runner",
-  "main": "index.js",
+  "main": "./src/index.js",
   "bin": {
-    "benchmark-runner": "./cli.js"
+    "benchmark-runner": "./src/cli.js"
   },
   "scripts": {
-    "test": "nyc mocha --expose-gc"
+    "test": "nyc mocha"
   },
   "author": "mistakia",
   "license": "MIT",

src/cli.js

@@ -0,0 +1,68 @@
+#!/usr/bin/env node
+
+/* global process */
+const { start } = require('./index')
+
+const yargs = require('yargs')
+const argv = yargs
+  .usage('OrbitDB benchmark runner\n\nUsage: node --expose-gc $0 [options]')
+  .version(false)
+  .help('help').alias('help', 'h')
+  .options({
+    baseline: {
+      alias: 'b',
+      description: 'Run baseline benchmarks only',
+      boolean: true,
+      requiresArg: false,
+      required: false
+    },
+    report: {
+      alias: 'r',
+      description: 'Output report (Default: false)',
+      requiresArg: false,
+      boolean: true,
+      required: false
+    },
+    list: {
+      alias: 'l',
+      description: 'List all benchmarks',
+      requiresArg: false,
+      required: false,
+      boolean: true
+    },
+    grep: {
+      alias: 'g',
+      description: '<regexp> Regular expression used to match benchmarks (Default: /.*/)',
+      requiresArg: true,
+      required: false
+    },
+    stressLimit: {
+      description: '<Int or Infinity> seconds to run a stress benchmark (Default: 300)',
+      requiresArg: true,
+      required: false
+    },
+    baselineLimit: {
+      description: '<Int> benchmark cycle limit for baseline benchmarks (Default: 1000)',
+      requiresArg: true,
+      required: false
+    },
+    logLimit: {
+      description: '<Int> max log size used for baseline benchmarks (inclusive) (Default: 10000)',
+      requiresArg: true,
+      required: false
+    }
+  })
+  .example('$0 -r -g append-baseline', 'Run a single benchmark (append-baseline)')
+  .example('$0 -r -g values-.*-baseline', 'Run all of the values baseline benchmarks')
+  .argv
+
+// Was this called directly from the CLI?
+// For mocha tests - maybe we can mitigate this but for now it works
+if (require.main === module) {
+  try {
+    const benchmarks = require(process.cwd() + '/benchmarks')
+    start(benchmarks, argv)
+  } catch (e) {
+    throw new Error(e.message)
+  }
+}

src/index.js

@@ -0,0 +1,115 @@
+require('expose-gc')
+global.gc()
+
+const os = require('os')
+
+const report = require('./report')
+
+const BASELINE_GREP = /[\d\w-]*-baseline/
+const DEFAULT_GREP = /.*/
+
+// TODO: Remove all OS-specific bindings here
+const start = async (benchmarks, argv) => {
+  if (argv.list) {
+    benchmarks.forEach(b => console.log(b.name))
+    return
+  }
+
+  const grep = argv.grep ? new RegExp(argv.grep) : DEFAULT_GREP
+  const stressLimit = argv.stressLimit || 5
+  const logLimit = argv.logLimit || 10000
+  const baselineLimit = argv.baselineLimit || 1000
+  const results = []
+  const baselineOnly = argv.baseline
+  const runnerStartTime = process.hrtime()
+
+  process.stdout.write(`Running ${baselineOnly ? 'baseline ' : ''}benchmarks matching: ${grep}`)
+  process.stdout.write('\n')
+
+  try {
+    for (const benchmark of benchmarks) {
+      benchmark.stressLimit = stressLimit
+      benchmark.baselineLimit = baselineLimit
+
+      if (baselineOnly && !BASELINE_GREP.test(benchmark.name)) {
+        continue
+      }
+
+      if (!grep.test(benchmark.name)) {
+        continue
+      }
+
+      if (benchmark.count && benchmark.count > logLimit) {
+        continue
+      }
+
+      const result = await runOne(benchmark)
+      results.push(result)
+    }
+
+    const runnerElapsed = getElapsed(process.hrtime(runnerStartTime))
+    let output = `\rCompleted ${results.length} benchmark${results.length > 1 ? 's' : ''}`
+    output += ` in ${(runnerElapsed / 1000000000).toFixed(2)} seconds`
+    process.stdout.write(output)
+
+    if (argv.report) {
+      report(results)
+    }
+  } catch (e) {
+    console.log(e)
+  }
+}
+
+const getElapsed = (time) => {
+  return +time[0] * 1e9 + +time[1]
+}
+
+const runOne = async (benchmark) => {
+  const { stressLimit, baselineLimit } = benchmark
+
+  const stats = {
+    count: 0
+  }
+
+  if (global.gc) {
+    global.gc()
+  }
+
+  const memory = {
+    before: process.memoryUsage()
+  }
+
+  process.stdout.write(`\r${benchmark.name} / Preparing`)
+  const params = await benchmark.prepare()
+
+  process.stdout.clearLine()
+  const startTime = process.hrtime() // eventually convert to hrtime.bigint
+  while (benchmark.while({ stats, startTime, stressLimit, baselineLimit })) {
+    const elapsed = getElapsed(process.hrtime(startTime))
+    const totalSeconds = (elapsed / 1000000000).toFixed(4)
+    const opsPerSec = (stats.count / totalSeconds).toFixed(4)
+    process.stdout.write(`\r${benchmark.name} / Cycles: ${stats.count} (${opsPerSec.toString()} ops/sec)`)
+    await benchmark.cycle(params)
+    stats.count++
+  }
+
+  const elapsed = getElapsed(process.hrtime(startTime))
+  memory.after = process.memoryUsage()
+
+  process.stdout.write(`\r${benchmark.name} / Finishing`)
+  await benchmark.teardown(params)
+  process.stdout.clearLine()
+
+  return {
+    name: benchmark.name,
+    cpus: os.cpus(),
+    loadavg: os.loadavg(),
+    elapsed,
+    stats,
+    memory
+  }
+}
+
+module.exports = {
+  start
+}