Added coradoc Gem benchmarking (#3)

.github/workflows/main.yml

@@ -26,3 +26,6 @@ jobs:
 
     - name: Run the default task
       run: bundle exec rake
+
+    - name: Try "Hello, world" benchmarking
+      run:  bundle exec exe/tebako-benchmarking measure -p 'ruby tests/simple-test/simple-test-run.rb'  -r 1 10 100

.gitignore

@@ -6,6 +6,7 @@
 /pkg/
 /spec/reports/
 /tmp/
+.vscode
 Gemfile.lock
 .tebako
 *-test-package

Gemfile

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-# Copyright (c) 2021-2023 [Ribose Inc](https://www.ribose.com).
+# Copyright (c) 2023 [Ribose Inc](https://www.ribose.com).
 # All rights reserved.
 # This file is a part of tebako
 #

Rakefile

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-# Copyright (c) 2021-2023 [Ribose Inc](https://www.ribose.com).
+# Copyright (c) 2023 [Ribose Inc](https://www.ribose.com).
 # All rights reserved.
 # This file is a part of tebako
 #

lib/tebako/benchmarking.rb

@@ -30,6 +30,5 @@
 module Tebako
   module Benchmarking
     class Error < StandardError; end
-    # Your code goes here...
   end
 end

lib/tebako/benchmarking/cli.rb

@@ -1,7 +1,7 @@
 #!/usr/bin/env ruby
 # frozen_string_literal: true
 
-# Copyright (c) 2021-2023 [Ribose Inc](https://www.ribose.com).
+# Copyright (c) 2023 [Ribose Inc](https://www.ribose.com).
 # All rights reserved.
 # This file is a part of tebako
 #
@@ -28,7 +28,6 @@
 
 require "fileutils"
 require "open3"
-require "tebako"
 require "thor"
 require "yaml"
 
@@ -45,19 +44,37 @@ class Cli < Thor
       method_option :package, type: :string, aliases: "-p", required: true,
                               desc: "Tebako package to benchmark"
 
-      method_option :repetitions, type: :numeric, aliases: "-r", required: true,
-                                  desc: "The number of repetitions", default: 10
+      method_option :repetitions, type: :array, aliases: "-r", required: true,
+                                  desc: "Repetitions to run (array of positive integers)", default: ["1"]
+      method_option :verbose, type: :boolean, aliases: "-v", default: false,
+                              desc: "Print benchmarking data for each repetition value"
       def measure
-        Tebako::Benchmarking.measure(options["package"], options["repetitions"])
+        exit 1 if (repetitions = preprocess).nil?
+        package = options["package"]
+        exit 1 unless repetitions[0] == 1 || Tebako::Benchmarking.test_cmd(package)
+
+        mea = iterate(package, repetitions, options["verbose"])
+        print_results(mea)
       end
 
       default_task :help
 
       def self.exit_on_failure?
         true
       end
-
+      # rubocop:disable Metrics/BlockLength
       no_commands do
+        def iterate(package, repetitions, verbose)
+          mea = {}
+
+          repetitions.each do |r|
+            mea[r] = Tebako::Benchmarking.measure(package, r, verbose)
+            exit 1 if mea[r].nil?
+          end
+
+          mea
+        end
+
         def options
           original_options = super
 
@@ -66,17 +83,39 @@ def options
           defaults = ::YAML.load_file(OPTIONS_FILE) || {}
           Thor::CoreExt::HashWithIndifferentAccess.new(defaults.merge(original_options))
         end
+
+        def preprocess
+          repetitions = options["repetitions"].map(&:to_i)
+          repetitions.sort!
+
+          return repetitions unless repetitions[0] < 1
+
+          puts "Repetitions must be positive integers"
+          nil
+        end
+
+        def print_results(mea)
+          header = format("%<key>-15s %<value>-15s", key: "Repetitions", value: "Total time")
+          separator = "-" * header.length
+          rows = mea.map { |r, m| format("%<key>-15s %<value>-20s", key: r, value: m["total"]) }
+
+          puts
+          puts header
+          puts separator
+          puts rows
+        end
       end
+      # rubocop:enable Metrics/BlockLength
     end
 
     class << self
       def err_bench(stdout_str, stderr_str)
         puts <<~ERROR_MESSAGE
-          Benchmarking failed
-          Ran '/usr/bin/time -l -p sh -c #{cmd}'
-          Output:
+          ----- Stdout -----
           #{stdout_str}
+          ----- Stderr -----
           #{stderr_str}
+          ------------------
         ERROR_MESSAGE
       end
 
@@ -88,24 +127,24 @@ def err_parse(msg, output)
         ERROR_MESSAGE
       end
 
-      def measure(package, repetitions)
-        return unless repetitions == 1 || test_cmd(package)
-
-        stdout_str, stderr_str, status = do_measure(package, repetitions)
+      def measure(package, repetitions, verbose)
+        print "Collecting data for '#{package}' with #{repetitions} repetitions ... "
+        stdout_str, stderr_str, status = do_measure(package, repetitions, verbose)
         if status.success?
-          puts "Benchmarking succeeded"
+          puts "OK"
           metrics = parse_time_output(stderr_str)
-          print_map_as_table(metrics)
+          metrics["total"] = metrics["user"].to_f + metrics["sys"].to_f
+          print_map_as_table(metrics) if verbose
         else
+          puts "Failed"
           err_bench(stdout_str, stderr_str)
         end
+        status.success? ? metrics : nil
       end
 
-      def do_measure(package, repetitions)
-        puts "Collecting data for '#{package}' with #{repetitions} repetitions."
-
+      def do_measure(package, repetitions, verbose)
         cmd = "#{package} #{repetitions} > /dev/null"
-        Open3.capture3("/usr/bin/time", "-l", "-p", "sh", "-c", cmd)
+        Open3.capture3("/usr/bin/time", verbose ? "-lp" : "-p", "sh", "-c", cmd)
       end
 
       def print_map_as_table(map)
@@ -145,7 +184,6 @@ def test_cmd(cmd)
           puts "Output:"
           puts stdout2e
         end
-
         status.success?
       end
     end

results/RESULTS.adoc

@@ -4,7 +4,7 @@
 
 == "Hello, world!" script
 ```
-puts "Hello!  This is test-01 talking from inside DwarFS"
+puts "Hello! This is simple benchmarking test."
 
 if (argv = ARGV).empty?
   puts "No arguments given"
@@ -25,14 +25,58 @@ With this script we compare the time that is require to load the most used part
 
 image::hello-world-benchmarking.jpg["Hello, world!" benchmarking results]
 
-A comparative analysis of the simpliest application shows that the Tebako package has a performance advantage over long runs. 
+A comparative analysis of the simpliest application shows that the Tebako package has a performance advantage over long runs.
 This is because the runtime library is served from an in-memory file system with significantly better access times.
 
 In short runs, Tebako loses because the package includes many files and components that are not used by the application, but are still loaded into memory.
 This creates a penalty of 0.3 seconds, which is however negligible in runs experiments.
 
-== Execution environment
+== coradoc Gem
+
+https://rubygems.org/gems/coradoc
+
+```
+puts "Hello! This is coradoc benchmarking test."
+
+if (argv = ARGV).empty?
+  puts "No arguments given"
+  exit(1)
+end
+
+if argv[0].to_i < 1
+  puts "Argument must be a positive integer"
+  exit(1)
+end
 
-... To be documented ...
+argv[0].to_i.times do
+  require "coradoc"
+  sample_file = File.join(__dir__, "fixtures", "sample.adoc")
+  require "coradoc/legacy_parser"
+  Coradoc::LegacyParser.parse(sample_file)[:document]
 
+  require "coradoc/oscal"
+  sample_file = File.join(__dir__, "fixtures", "sample-oscal.adoc")
+  document = Coradoc::Document.from_adoc(sample_file)
+  Coradoc::Oscal.to_oscal(document)
+
+  syntax_tree = Coradoc::Parser.parse(sample_file)
+  Coradoc::Transformer.transform(syntax_tree)
+end
+```
+With this test we compare intensive native Ruby code processing. Tebako package shows stable 20% performance gain.
+This gain is a result of faster access to Ruby standard library as explained above.
+Also the test shows that there is no difference in code execution between native Ruby and tebako package.
 
+image::coradoc-benchmarking.jpg[coradoc benchmarking results]
+
+== Execution environment
+
+```
+Model Name:	Mac mini
+Model Identifier:	Macmini9,1
+Chip:	Apple M1
+Total Number of Cores:	8 (4 performance and 4 efficiency)
+Memory:	16 GB
+Ruby 3.1.4p223 (2023-03-30 revision 957bb7cb81) [arm64-darwin21]
+tebako executable packager 0.5.5
+```

tebako-benchmarking.gemspec

@@ -57,6 +57,7 @@ Gem::Specification.new do |spec|
   spec.executables = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
+  spec.add_dependency "coradoc", "~> 0.1.0"
   spec.add_dependency "tebako", "~> 0.5.5"
   spec.add_dependency "tebako-runtime", "~> 0.2.1"
   spec.add_dependency "thor", "~> 1.2"

tests/coradoc-test/Gemfile

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+source "https://rubygems.org"
+
+gem "coradoc", "~> 0.1.0"

tests/coradoc-test/coradoc-test-run.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+puts "Hello! This is coradoc benchmarking test."
+
+if (argv = ARGV).empty?
+  puts "No arguments given"
+  exit(1)
+end
+
+if argv[0].to_i < 1
+  puts "Argument must be a positive integer"
+  exit(1)
+end
+
+argv[0].to_i.times do
+  require "coradoc"
+  sample_file = File.join(__dir__, "fixtures", "sample.adoc")
+  require "coradoc/legacy_parser"
+  Coradoc::LegacyParser.parse(sample_file)[:document]
+
+  require "coradoc/oscal"
+  sample_file = File.join(__dir__, "fixtures", "sample-oscal.adoc")
+  document = Coradoc::Document.from_adoc(sample_file)
+  Coradoc::Oscal.to_oscal(document)
+
+  syntax_tree = Coradoc::Parser.parse(sample_file)
+  Coradoc::Transformer.transform(syntax_tree)
+end