diff --git a/.gitignore b/.gitignore index 45a43de..ff4c02c 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,6 @@ Gemfile.lock .tebako *-test-package +site-* +relaton/ +iev/ diff --git a/lib/tebako/benchmarking/cli.rb b/lib/tebako/benchmarking/cli.rb index 92d408b..c8f635f 100755 --- a/lib/tebako/benchmarking/cli.rb +++ b/lib/tebako/benchmarking/cli.rb @@ -40,8 +40,12 @@ module Benchmarking class Cli < Thor package_name "Tebako benchmarking" - class_option :repetitions, type: :array, aliases: "-r", required: false, - desc: "Repetitions to run (array of positive integers)", default: ["1"] + class_option :repetitions, + type: :array, aliases: "-r", + required: false, + desc: "Repetitions to run (array of numbers or 'I' for a single run of original application)", + default: ["1"] + class_option :verbose, type: :boolean, aliases: "-v", default: false, desc: "Print benchmarking data for each repetition value" @@ -55,7 +59,7 @@ def compare exit 1 if (repetitions = preprocess).nil? cmd1 = options["first"] cmd2 = options["second"] - exit 1 if repetitions[0] != 1 && !(Tebako::Benchmarking.test_cmd(cmd1) && Tebako::Benchmarking.test_cmd(cmd2)) + exit 1 if repetitions[0] > 1 && !(Tebako::Benchmarking.test_cmd(cmd1) && Tebako::Benchmarking.test_cmd(cmd2)) do_compare(cmd1, cmd2, repetitions, options["verbose"]) end @@ -66,7 +70,7 @@ def compare def measure exit 1 if (repetitions = preprocess).nil? cmd = options["cmd"] - exit 1 unless repetitions[0] == 1 || Tebako::Benchmarking.test_cmd(cmd) + exit 1 if repetitions[0] > 1 && Tebako::Benchmarking.test_cmd(cmd) mea = iterate(cmd, repetitions, options["verbose"]) print_results(mea) @@ -106,12 +110,14 @@ def options end def preprocess + return [0] if options["repetitions"] == ["I"] + repetitions = options["repetitions"].map(&:to_i) repetitions.sort! return repetitions unless repetitions[0] < 1 - puts "Repetitions must be positive integers" + puts "Repetitions must be positive integers or I (for a single run of original application)" nil end @@ -136,8 +142,10 @@ def print_comparison_headers(cmd1, cmd2) def print_comparison(cmd1, cmd2, mea1, mea2) rows = mea1.keys.zip(mea1.values, mea2.values).map do |r, m1, m2| - format("%-15s| %-#{cmd1.length}s| %-#{cmd2.length}s", key: r, value1: m1["total"], - value2: m2["total"]) + format("%-15s| %-#{cmd1.length}s| %-#{cmd2.length}s", + key: (r.zero? ? 1 : r), + value1: m1["total"], + value2: m2["total"]) end puts print_comparison_headers(cmd1, cmd2) @@ -147,7 +155,9 @@ def print_comparison(cmd1, cmd2, mea1, mea2) def print_results(mea) header = format("%-15s %-15s", key: "Repetitions", value: "Total time") separator = "-" * header.length - rows = mea.map { |r, m| format("%-15s %-20s", key: r, value: m["total"]) } + rows = mea.map do |r, m| + format("%-15s %-20s", key: (r.zero? ? 1 : r), value: m["total"]) + end puts puts header @@ -177,23 +187,41 @@ def err_parse(msg, output) ERROR_MESSAGE end - def measure(package, repetitions, verbose) - print "Collecting data for '#{package}' with #{repetitions} repetitions ... " - stdout_str, stderr_str, status = do_measure(package, repetitions, verbose) + def measure(package, rpt, verbose) + print "Collecting data for '#{package}' with #{rp_print_v(rpt)} repetition#{rp_print_e(rpt)} ... " + stdout_str, stderr_str, status = do_measure(package, rpt, verbose) + mtr = nil if status.success? puts "OK" - metrics = parse_time_output(stderr_str) - metrics["total"] = metrics["user"].to_f + metrics["sys"].to_f - print_map_as_table(metrics) if verbose + mtr = metrics(stderr_str, verbose) else puts "Failed" err_bench(stdout_str, stderr_str) end - status.success? ? metrics : nil + mtr + end + + def rp_print_v(repetitions) + repetitions.zero? ? 1 : repetitions + end + + def rp_print_e(repetitions) + repetitions < 2 ? "" : "s" + end + + def metrics(stderr_str, verbose) + metrics = parse_time_output(stderr_str) + metrics["total"] = metrics["user"].to_f + metrics["sys"].to_f + print_map_as_table(metrics) if verbose + metrics end def do_measure(package, repetitions, verbose) - cmd = "#{package} #{repetitions} > /dev/null" + cmd = if repetitions.zero? + "#{package} > /dev/null" + else + "#{package} #{repetitions} > /dev/null" + end Open3.capture3("/usr/bin/time", verbose ? "-lp" : "-p", "sh", "-c", cmd) end diff --git a/results/RESULTS.adoc b/results/RESULTS.adoc index 1e7f0cf..ad3e81a 100644 --- a/results/RESULTS.adoc +++ b/results/RESULTS.adoc @@ -2,6 +2,12 @@ = Benchmarking results +== Executive summary +* Peformance of tebako package does not change comparing to original application +* Tebako package introduces additional time that is consumed during initialization. This time depends on the application size and the size of its native extensions, +it dows not depend on complexity of the data processed by the application or any other volatile factors +* In our tests additional time mentioned above varies from 0.03 seconds for 'Hello, world" script to 3 seconds for metanorma application + == "Hello, world!" script ``` puts "Hello! This is simple benchmarking test." @@ -131,6 +137,17 @@ The test shows that Tebako package takes extra time to setup native extension be image::vectory-benchmarking.jpg[vectory benchmarking results] + +== https://metanorma.org[metanorma application] + +Metanorma application benchmarking included execution of utility commands: ```metanorma help```, ```metanorma version``` and generation of sample sites (ietf, ieee, iec, iso, iho) using +``` +metanorma site generate samples -c samples/metanorma.yml -o site- --agree-to-terms +``` + + +image::metanorma-benchmarking.jpg[metanorma benchmarking results] + == Execution environment ``` diff --git a/results/img/metanorma-benchmarking.jpg b/results/img/metanorma-benchmarking.jpg new file mode 100644 index 0000000..25c56ee Binary files /dev/null and b/results/img/metanorma-benchmarking.jpg differ diff --git a/results/raw/results.xlsx b/results/raw/results.xlsx index 1a58d3a..5dd848d 100644 Binary files a/results/raw/results.xlsx and b/results/raw/results.xlsx differ diff --git a/tebako-benchmarking.gemspec b/tebako-benchmarking.gemspec index 2bf6657..b1958cb 100644 --- a/tebako-benchmarking.gemspec +++ b/tebako-benchmarking.gemspec @@ -59,6 +59,7 @@ Gem::Specification.new do |spec| spec.add_dependency "coradoc", "~> 0.1.0" spec.add_dependency "emf2svg", "~> 1.4.3" + spec.add_dependency "metanorma-cli", "~> 1.8.8" spec.add_dependency "tebako", "~> 0.5.5" spec.add_dependency "tebako-runtime", "~> 0.2.1" spec.add_dependency "thor", "~> 1.2" diff --git a/tests/metanorma-test/Gemfile b/tests/metanorma-test/Gemfile new file mode 100644 index 0000000..dc9c695 --- /dev/null +++ b/tests/metanorma-test/Gemfile @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +source "https://rubygems.org" + +# rubocop:disable Gemspec/DevelopmentDependencies +gem "iso-639", "<= 0.2.10" # https://github.com/metanorma/packed-mn/issues/26 + +gem "ffi" +# nokogiry asks for psych >= 4 +# psych 5 does not support --enable-bundled-libyaml configuration option +# that is required by tebako +# we set "psych ~> 4" temporarily until +# https://github.com/tamatebako/tebako/issues/93 is fixed +gem "psych", "~> 4" +gem "rake" +gem "sassc" +gem "seven-zip" + +group :development do + gem "byebug" +end + +gem "metanorma-cli", "= 1.8.8" + +# rubocop:enable Gemspec/DevelopmentDependencies diff --git a/tests/metanorma-test/metanorma-test-run.rb b/tests/metanorma-test/metanorma-test-run.rb new file mode 100755 index 0000000..02725e2 --- /dev/null +++ b/tests/metanorma-test/metanorma-test-run.rb @@ -0,0 +1,40 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +require "rubygems" +require "openssl" +require "open-uri" +require "net/https" + +require "jing" +require "optout" + +# Limit Relaton to concurrent fetches of 1 +ENV["RELATON_FETCH_PARALLEL"] = "1" + +unless Gem.win_platform? # because on windows we use aibika + # This code was bundled with ruby-packer/tebako hack but is not related + class Jing + def self.option_builder + @@option_builder + end + end + + class Optout + def []=(name, value) + @options[name] = value + end + end + + Jing.option_builder[:jar] = Optout::Option.create(:jar, "-jar", + index: 1, + validator: Optout::File.exists, + default: Jing::DEFAULT_JAR) +end + +# explicitly load all dependent gems +# ruby packer cannot use gem load path correctly. +require "metanorma/cli" + +# start up the CLI +Metanorma::Cli.start(ARGV)