Finalized benchmarking results (#6)

.gitignore

@@ -10,3 +10,6 @@
 Gemfile.lock
 .tebako
 *-test-package
+site-*
+relaton/
+iev/

lib/tebako/benchmarking/cli.rb

@@ -40,8 +40,12 @@ module Benchmarking
     class Cli < Thor
       package_name "Tebako benchmarking"
 
-      class_option :repetitions, type: :array, aliases: "-r", required: false,
-                                 desc: "Repetitions to run (array of positive integers)", default: ["1"]
+      class_option :repetitions,
+                   type: :array, aliases: "-r",
+                   required: false,
+                   desc: "Repetitions to run (array of numbers or 'I' for a single run of original application)",
+                   default: ["1"]
+
       class_option :verbose, type: :boolean, aliases: "-v", default: false,
                              desc: "Print benchmarking data for each repetition value"
 
@@ -55,7 +59,7 @@ def compare
         exit 1 if (repetitions = preprocess).nil?
         cmd1 = options["first"]
         cmd2 = options["second"]
-        exit 1 if repetitions[0] != 1 && !(Tebako::Benchmarking.test_cmd(cmd1) && Tebako::Benchmarking.test_cmd(cmd2))
+        exit 1 if repetitions[0] > 1 && !(Tebako::Benchmarking.test_cmd(cmd1) && Tebako::Benchmarking.test_cmd(cmd2))
 
         do_compare(cmd1, cmd2, repetitions, options["verbose"])
       end
@@ -66,7 +70,7 @@ def compare
       def measure
         exit 1 if (repetitions = preprocess).nil?
         cmd = options["cmd"]
-        exit 1 unless repetitions[0] == 1 || Tebako::Benchmarking.test_cmd(cmd)
+        exit 1 if repetitions[0] > 1 && Tebako::Benchmarking.test_cmd(cmd)
 
         mea = iterate(cmd, repetitions, options["verbose"])
         print_results(mea)
@@ -106,12 +110,14 @@ def options
         end
 
         def preprocess
+          return [0] if options["repetitions"] == ["I"]
+
           repetitions = options["repetitions"].map(&:to_i)
           repetitions.sort!
 
           return repetitions unless repetitions[0] < 1
 
-          puts "Repetitions must be positive integers"
+          puts "Repetitions must be positive integers or I (for a single run of original application)"
           nil
         end
 
@@ -136,8 +142,10 @@ def print_comparison_headers(cmd1, cmd2)
 
         def print_comparison(cmd1, cmd2, mea1, mea2)
           rows = mea1.keys.zip(mea1.values, mea2.values).map do |r, m1, m2|
-            format("%<key>-15s| %<value1>-#{cmd1.length}s| %<value2>-#{cmd2.length}s", key: r, value1: m1["total"],
-                                                                                       value2: m2["total"])
+            format("%<key>-15s| %<value1>-#{cmd1.length}s| %<value2>-#{cmd2.length}s",
+                   key: (r.zero? ? 1 : r),
+                   value1: m1["total"],
+                   value2: m2["total"])
           end
 
           puts print_comparison_headers(cmd1, cmd2)
@@ -147,7 +155,9 @@ def print_comparison(cmd1, cmd2, mea1, mea2)
         def print_results(mea)
           header = format("%<key>-15s %<value>-15s", key: "Repetitions", value: "Total time")
           separator = "-" * header.length
-          rows = mea.map { |r, m| format("%<key>-15s %<value>-20s", key: r, value: m["total"]) }
+          rows = mea.map do |r, m|
+            format("%<key>-15s %<value>-20s", key: (r.zero? ? 1 : r), value: m["total"])
+          end
 
           puts
           puts header
@@ -177,23 +187,41 @@ def err_parse(msg, output)
         ERROR_MESSAGE
       end
 
-      def measure(package, repetitions, verbose)
-        print "Collecting data for '#{package}' with #{repetitions} repetitions ... "
-        stdout_str, stderr_str, status = do_measure(package, repetitions, verbose)
+      def measure(package, rpt, verbose)
+        print "Collecting data for '#{package}' with #{rp_print_v(rpt)} repetition#{rp_print_e(rpt)} ... "
+        stdout_str, stderr_str, status = do_measure(package, rpt, verbose)
+        mtr = nil
         if status.success?
           puts "OK"
-          metrics = parse_time_output(stderr_str)
-          metrics["total"] = metrics["user"].to_f + metrics["sys"].to_f
-          print_map_as_table(metrics) if verbose
+          mtr = metrics(stderr_str, verbose)
         else
           puts "Failed"
           err_bench(stdout_str, stderr_str)
         end
-        status.success? ? metrics : nil
+        mtr
+      end
+
+      def rp_print_v(repetitions)
+        repetitions.zero? ? 1 : repetitions
+      end
+
+      def rp_print_e(repetitions)
+        repetitions < 2 ? "" : "s"
+      end
+
+      def metrics(stderr_str, verbose)
+        metrics = parse_time_output(stderr_str)
+        metrics["total"] = metrics["user"].to_f + metrics["sys"].to_f
+        print_map_as_table(metrics) if verbose
+        metrics
       end
 
       def do_measure(package, repetitions, verbose)
-        cmd = "#{package} #{repetitions} > /dev/null"
+        cmd = if repetitions.zero?
+                "#{package} > /dev/null"
+              else
+                "#{package} #{repetitions} > /dev/null"
+              end
         Open3.capture3("/usr/bin/time", verbose ? "-lp" : "-p", "sh", "-c", cmd)
       end
 

results/RESULTS.adoc

@@ -2,6 +2,12 @@
 
 = Benchmarking results
 
+== Executive summary
+* Peformance of tebako package does not change comparing to original application
+* Tebako package introduces additional time that is consumed during initialization. This time depends on the application size and the size of its native extensions, 
+it dows not depend on complexity of the data processed by the application or any other volatile factors
+* In our tests additional time mentioned above varies from 0.03 seconds for 'Hello, world" script to 3 seconds for metanorma application
+
 == "Hello, world!" script
 ```
 puts "Hello! This is simple benchmarking test."
@@ -131,6 +137,17 @@ The test shows that Tebako package takes extra time to setup native extension be
 
 image::vectory-benchmarking.jpg[vectory benchmarking results]
 
+
+== https://metanorma.org[metanorma application]
+
+Metanorma application benchmarking included execution of utility commands: ```metanorma help```, ```metanorma version``` and generation of sample sites (ietf, ieee, iec, iso, iho) using 
+```
+metanorma site generate samples -c samples/metanorma.yml  -o site-<site name> --agree-to-terms
+```
+
+
+image::metanorma-benchmarking.jpg[metanorma benchmarking results]
+
 == Execution environment
 
 ```

tebako-benchmarking.gemspec

@@ -59,6 +59,7 @@ Gem::Specification.new do |spec|
 
   spec.add_dependency "coradoc", "~> 0.1.0"
   spec.add_dependency "emf2svg", "~> 1.4.3"
+  spec.add_dependency "metanorma-cli", "~> 1.8.8"
   spec.add_dependency "tebako", "~> 0.5.5"
   spec.add_dependency "tebako-runtime", "~> 0.2.1"
   spec.add_dependency "thor", "~> 1.2"

tests/metanorma-test/Gemfile

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+source "https://rubygems.org"
+
+# rubocop:disable Gemspec/DevelopmentDependencies
+gem "iso-639", "<= 0.2.10" # https://github.com/metanorma/packed-mn/issues/26
+
+gem "ffi"
+# nokogiry asks for psych >= 4
+# psych 5 does not support --enable-bundled-libyaml configuration option
+# that is required by tebako
+# we set "psych ~> 4" temporarily until
+# https://github.com/tamatebako/tebako/issues/93  is fixed
+gem "psych", "~> 4"
+gem "rake"
+gem "sassc"
+gem "seven-zip"
+
+group :development do
+  gem "byebug"
+end
+
+gem "metanorma-cli", "= 1.8.8"
+
+# rubocop:enable Gemspec/DevelopmentDependencies

tests/metanorma-test/metanorma-test-run.rb

@@ -0,0 +1,40 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require "rubygems"
+require "openssl"
+require "open-uri"
+require "net/https"
+
+require "jing"
+require "optout"
+
+# Limit Relaton to concurrent fetches of 1
+ENV["RELATON_FETCH_PARALLEL"] = "1"
+
+unless Gem.win_platform? # because on windows we use aibika
+  # This code was bundled with ruby-packer/tebako hack but is not related
+  class Jing
+    def self.option_builder
+      @@option_builder
+    end
+  end
+
+  class Optout
+    def []=(name, value)
+      @options[name] = value
+    end
+  end
+
+  Jing.option_builder[:jar] = Optout::Option.create(:jar, "-jar",
+                                                    index: 1,
+                                                    validator: Optout::File.exists,
+                                                    default: Jing::DEFAULT_JAR)
+end
+
+# explicitly load all dependent gems
+# ruby packer cannot use gem load path correctly.
+require "metanorma/cli"
+
+# start up the CLI
+Metanorma::Cli.start(ARGV)