Version command

rebench/configurator.py

@@ -225,6 +225,22 @@ def __init__(self, raw_config, data_store, ui, cli_options=None, cli_reporter=No
         experiments = raw_config.get('experiments', {})
         self._experiments = self._compile_experiments(experiments)
 
+    def _parse_executor(self, executor_name, executor_config):
+        path = executor_config.get('path')
+        executable = executor_config.get('executable')
+        cores = executor_config.get('cores')
+        version_command = executor_config.get('version_command')
+
+        executor = Executor([], False, self.ui, config_dir=self.config_dir)
+        if version_command:
+            executor.set_version_command(version_command)
+        return executor
+
+    def _compile_experiment(self, exp_name, experiment):
+        experiment['executors'] = {name: self._parse_executor(name, config) for name, config in
+                                   experiment.get('executors', {}).items()}
+        return Experiment.compile(exp_name, experiment, self)
+
     @property
     def use_rebench_db(self):
         report_results = self.options is None or self.options.use_data_reporting
@@ -291,9 +307,15 @@ def get_executor(self, executor_name, run_details, variables, action):
             raise ConfigurationError(
                 "An experiment tries to use an undefined executor: %s" % executor_name)
 
+        executor_config = self._executors[executor_name]
         executor = Executor.compile(
-            executor_name, self._executors[executor_name],
+            executor_name, executor_config,
             run_details, variables, self.build_commands, action)
+
+        version_command = executor_config.get('version_command')
+        if version_command:
+            executor.version_command = version_command
+
         return executor
 
     def get_suite(self, suite_name):
@@ -342,6 +364,3 @@ def _compile_experiments(self, experiments):
                 self._exp_name, experiments[self._exp_name])
 
         return results
-
-    def _compile_experiment(self, exp_name, experiment):
-        return Experiment.compile(exp_name, experiment, self)

rebench/model/executor.py

@@ -18,6 +18,7 @@
 # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 # IN THE SOFTWARE.
 import os
+import subprocess
 
 from .build_cmd import BuildCommand
 from .exp_run_details import ExpRunDetails
@@ -26,59 +27,70 @@
 from ..configuration_error import ConfigurationError
 
 
-class Executor(object):
-
+class Executor:
     @classmethod
     def compile(cls, executor_name, executor, run_details, variables, build_commands, action):
         path = executor.get('path')
         if path and not path.startswith('~'):
             path = os.path.abspath(path)
         executable = executor.get('executable')
         args = executor.get('args')
+        version_command = executor.get('version_command')
+        version_string = executor.get('version_string')
+        version_git = executor.get('version_git')
 
         build = BuildCommand.create_commands(executor.get('build'), build_commands, path)
-
         description = executor.get('description')
         desc = executor.get('desc')
         env = executor.get('env')
-
         profiler = Profiler.compile(executor.get('profiler'))
-
         run_details = ExpRunDetails.compile(executor, run_details)
         variables = ExpVariables.compile(executor, variables)
 
         if action == "profile" and len(profiler) == 0:
-            raise ConfigurationError("Executor " + executor_name + " is configured for profiling, "
-                                     + "but no profiler details are given.")
+            raise ConfigurationError(f"Executor {executor_name} is configured for profiling, "
+                                     "but no profiler details are given.")
 
-        return Executor(executor_name, path, executable, args, build, description or desc,
-                        profiler, run_details, variables, action, env)
+        return Executor(executor_name, path, executable, args, version_command, version_string, version_git, build,
+                        description or desc, profiler, run_details, variables, action, env)
 
-    def __init__(self, name, path, executable, args, build, description,
+    def __init__(self, name, path, executable, args, version_command, version_string, version_git, build, description,
                  profiler, run_details, variables, action, env):
-        """Specializing the executor details in the run definitions with the settings from
-           the executor definitions
-        """
         self.name = name
         self.path = path
         self.executable = executable
         self.args = args
-
+        self.version_command = version_command
+        self.version_string = version_string
+        self.version_git = version_git
         self.build = build
         self.description = description
         self.profiler = profiler
-
         self.run_details = run_details
         self.variables = variables
         self.env = env
-
         self.action = action
 
+    def get_version(self):
+        if self.version_command:
+            try:
+                result = subprocess.run(self.version_command, shell=True, check=True, capture_output=True, text=True)
+                return result.stdout.strip()
+            except subprocess.CalledProcessError as e:
+                return e.stderr.strip()
+        elif self.version_string:
+            return self.version_string
+        elif self.version_git:
+            try:
+                result = subprocess.run(self.version_git, shell=True, check=True, capture_output=True, text=True)
+                return result.stdout.strip()
+            except subprocess.CalledProcessError as e:
+                return e.stderr.strip()
+        else:
+            return "Unknown version"
+
     def as_dict(self):
-        result = {
-            'name': self.name,
-            'desc': self.description
-        }
+        result = {'name': self.name, 'desc': self.description}
         if self.build:
             result['build'] = [b.as_dict() for b in self.build]
-        return result
+        return result

rebench/rebench-schema.yml

@@ -277,6 +277,18 @@ schema;executor_type:
               type: str
               desc: Argument given to `perf` when processing the recording
               default: report -g graph --no-children --stdio
+    version_command:
+      type: str
+      required: false
+      desc: Command to retrieve the version of the executable.
+    version_string:
+      type: str
+      required: false
+      desc: Explicit version string provided by the user.
+    version_git:
+      type: str
+      required: false
+      desc: Command to retrieve the Git version of the executable.
 
 schema;exp_suite_type:
   desc: A list of suites

rebench/tests/bugs/issue_4_run_equality_and_params_test.py

@@ -36,7 +36,7 @@ def setUp(self):
     @staticmethod
     def _create_template_run_id():
         executor = Executor('MyVM', 'foo_bar_path', 'foo_bar_bin',
-                            None, None, None, None, None, None, "benchmark", {})
+                            None, None, None, None, None, None, None, None, None, "benchmark", {})
         suite = BenchmarkSuite("MySuite", executor, '', '%(benchmark)s %(cores)s %(input)s',
                                None, None, [], None, None, None)
         benchmark = Benchmark("TestBench", "TestBench", None, suite, None,
@@ -46,7 +46,7 @@ def _create_template_run_id():
     @staticmethod
     def _create_hardcoded_run_id():
         executor = Executor('MyVM', 'foo_bar_path', 'foo_bar_bin',
-                            None, None, None, None, None, None, "benchmark", {})
+                            None, None, None, None, None, None, None, None, None, "benchmark", {})
         suite = BenchmarkSuite('MySuite', executor, '', '%(benchmark)s %(cores)s 2 3',
                                None, None, [], None, None, None)
         benchmark = Benchmark("TestBench", "TestBench", None, suite,

rebench/tests/executor_test.py

@@ -19,17 +19,18 @@
 # IN THE SOFTWARE.
 import unittest
 import os
+import subprocess
 
+from ..model.executor import Executor as RebenchExecutor
 from .persistence import TestPersistence
 from .rebench_test_case import ReBenchTestCase
-from ..rebench           import ReBench
-from ..executor          import Executor, BatchScheduler, RandomScheduler, RoundRobinScheduler
-from ..configurator      import Configurator, load_config
+from ..rebench import ReBench
+from ..executor import Executor, BatchScheduler, RandomScheduler, RoundRobinScheduler
+from ..configurator import Configurator, load_config
 from ..model.measurement import Measurement
-from ..persistence       import DataStore
+from ..persistence import DataStore
 from ..ui import UIError
-from ..reporter          import Reporter
-
+from ..reporter import Reporter
 
 
 class ExecutorTest(ReBenchTestCase):
@@ -219,6 +220,96 @@ def test_determine_exp_name_and_filters_only_others(self):
         self.assertEqual(exp_name, None)
         self.assertEqual(exp_filter, ['e:bar', 's:b'])
 
+    def test_version_command(self):
+        executor = RebenchExecutor(
+            "TestExecutor", None, None, None, "python --version",
+            None, None, None, None, None, None, None, None, None
+        )
+
+        try:
+            result = subprocess.run(
+                executor.version_command, shell=True, check=True,
+                stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True
+            )
+            version_output = result.stdout.strip()
+        except subprocess.CalledProcessError as e:
+            version_output = e.stderr.strip()
+        self.assertTrue("Python" in version_output)
+
+    def test_version_command_in_config(self):
+        self._cnf = Configurator(load_config(self._path + '/small_with_version_command.conf'), DataStore(self.ui),
+                                 self.ui, None, data_file=self._tmp_file)
+        runs = self._cnf.get_runs()
+        executor = list(runs)[0].benchmark.suite.executor
+
+        self.assertEqual(executor.version_command, "python --version")
+
+        try:
+            result = subprocess.run(
+                executor.version_command, shell=True, check=True,
+                stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True
+            )
+            version_output = result.stdout.strip()
+        except subprocess.CalledProcessError as e:
+            version_output = e.stderr.strip()
+
+        self.assertTrue("Python" in version_output)
+
+    def test_version_string(self):
+        executor = RebenchExecutor(
+            "TestExecutor", None, None, None, None, "7.42",
+            None, None, None, None, None, None, None, None
+        )
+
+        version_output = executor.version_string
+        self.assertTrue("7.42" in version_output)
+
+    def test_version_string_in_config(self):
+        self._cnf = Configurator(load_config(self._path + '/small_with_version_string.conf'), DataStore(self.ui),
+                                 self.ui, None, data_file=self._tmp_file)
+        runs = self._cnf.get_runs()
+        executor = list(runs)[0].benchmark.suite.executor
+
+        self.assertEqual(executor.version_string, "7.42")
+
+        version_output = executor.version_string
+        self.assertTrue("7.42" in version_output)
+
+    def test_version_git(self):
+        executor = RebenchExecutor(
+            "TestExecutor", None, None, None, None, None,
+            "git rev-parse HEAD", None, None, None, None, None, None, None
+        )
+
+        try:
+            result = subprocess.run(
+                executor.version_git, shell=True, check=True,
+                stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True
+            )
+            version_output = result.stdout.strip()
+        except subprocess.CalledProcessError as e:
+            version_output = e.stderr.strip()
+        self.assertTrue(len(version_output) > 0)
+
+    def test_version_git_in_config(self):
+        self._cnf = Configurator(load_config(self._path + '/small_with_version_git.conf'), DataStore(self.ui),
+                                 self.ui, None, data_file=self._tmp_file)
+        runs = self._cnf.get_runs()
+        executor = list(runs)[0].benchmark.suite.executor
+
+        self.assertEqual(executor.version_git, "git rev-parse HEAD")
+
+        try:
+            result = subprocess.run(
+                executor.version_git, shell=True, check=True,
+                stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True
+            )
+            version_output = result.stdout.strip()
+        except subprocess.CalledProcessError as e:
+            version_output = e.stderr.strip()
+
+        self.assertTrue(len(version_output) > 0)
+
 
 class _TestReporter(Reporter):
     __test__ = False  # This is not a test class

rebench/tests/persistency.conf

@@ -3,7 +3,7 @@
 
 # this run definition will be chosen if no parameters are given to rebench.py
 default_experiment: Test
-default_data_file:  'persistency.data'
+default_data_file: 'persistency.data'
 
 reporting:
   codespeed:

rebench/tests/persistency_test.py

@@ -45,7 +45,7 @@ class PersistencyTest(ReBenchTestCase):
     def test_de_serialization(self):
         data_store = DataStore(self.ui)
         executor = ExecutorConf("MyVM", '', '',
-                                None, None, None, None, None, None, "benchmark", {})
+                                None, None, None, None,None, None, None, None, None, "benchmark", {})
         suite = BenchmarkSuite("MySuite", executor, '', '', None, None,
                                None, None, None, None)
         benchmark = Benchmark("Test Bench [>", "Test Bench [>", None,

rebench/tests/small_with_version_command.conf

@@ -0,0 +1,33 @@
+# Config file for ReBench
+# Config format is YAML (see http://yaml.org/ for detailed spec)
+
+# this run definition will be chosen if no parameters are given to rebench.py
+default_experiment: Test
+default_data_file:  'small.data'
+
+# general configuration for runs
+runs:
+    invocations:  10
+    retries_after_failure: 3
+
+benchmark_suites:
+    Suite:
+        gauge_adapter: TestExecutor
+        command: TestBenchMarks ~/suiteFolder/%(benchmark)s
+        benchmarks:
+            - Bench1
+            - Bench2
+
+executors:
+    TestRunner1:
+        path: ~/PycharmProjects/ReBench/rebench/tests
+        executable: test-vm1.py %(cores)s
+        cores: [1]
+        version_command: "python --version"
+
+experiments:
+    Test:
+        suites:
+            - Suite
+        executions:
+            - TestRunner1

rebench/tests/small_with_version_git.conf

@@ -0,0 +1,27 @@
+# Config file for ReBench
+# Config format is YAML (see http://yaml.org/ for detailed spec)
+
+# this run definition will be chosen if no parameters are given to rebench.py
+default_experiment: Test
+default_data_file:  'small.data'
+
+# general configuration for runs
+benchmark_suites:
+    Suite:
+        gauge_adapter: TestExecutor
+        command: TestBenchMarks ~/suiteFolder/%(benchmark)s
+        benchmarks:
+            - Bench1
+            - Bench2
+executors:
+    TestRunner1:
+        path: ~/PycharmProjects/ReBench/rebench/tests
+        executable: test-vm1.py %(cores)s
+        cores: [1]
+        version_git: "git rev-parse HEAD"
+experiments:
+    Test:
+        suites:
+            - Suite
+        executions:
+            - TestRunner1