Ignore tests when the app version is too old (davidchall#25)

docs/config_test.rst

@@ -26,6 +26,8 @@ Optional fields
 
 **description** *[string]*
     A short description to help identification of this version.
+**minimum_app_version** *[string]*
+    The minimum software version required for the test to be executed (see :ref:`config_sw`). If the software under test does not satisfy this requirement, then the test is removed from the test suite before execution. This allows you to run the latest test suite on old software without test failures.
 **input_files** *[list of strings]*
     A list of required input files. Each path is specified relative to the location of the configuration file itself.
 **output_files** *[dict of string-string pairs]*

nrtest/__init__.py

@@ -119,6 +119,7 @@ class Test(Metadata):
     ]
     execute_optional_fields = {
         'description': None,
+        'minimum_app_version': None,
         'input_files': [],
         'output_files': {},
         'fail_strings': [],

nrtest/execute.py

@@ -7,6 +7,9 @@
 import json
 import datetime
 
+# third-party imports
+from packaging import version
+
 # project imports
 from .process import source, execute, monitor
 from .utility import color, copy_file_and_path, rmtree, which
@@ -119,6 +122,16 @@ def _postcheck(test):
             raise TestFailure('Output file not generated: "%s"' % fname)
 
 
+def _skip_test(test, app):
+    skip = False
+
+    if test.minimum_app_version and version.parse(app.version) < version.parse(test.minimum_app_version):
+        skip = True
+        logging.info('Skipping test (app version too old): "%s"' % test.name)
+
+    return skip
+
+
 def validate_testsuite(ts):
     p = ts.app.setup_script
     if p and not os.path.exists(p):
@@ -130,6 +143,8 @@ def validate_testsuite(ts):
         logging.error('Unable to find executable: "%s"' % ts.app.exe)
         return False
 
+    ts.tests[:] = [t for t in ts.tests if not _skip_test(t, ts.app)]
+
     for t in ts.tests:
         if not validate_test(t):
             return False
@@ -149,7 +164,6 @@ def validate_test(test):
 
     # not specified by user, but should be set by now
     additional_required_fields = [
-        'input_dir',
         'output_dir',
     ]
 
@@ -158,15 +172,16 @@ def validate_test(test):
             logger.error('Unable to find "%s" property' % field)
             return False
 
-    p = test.input_dir
-    if not os.path.isdir(p):
-        logger.error('Input directory not found: "%s"' % p)
-        return False
-
-    for fname in test.input_files:
-        p = os.path.join(test.input_dir, fname)
-        if not os.path.isfile(p):
-            logger.error('Input file not found: "%s"' % p)
+    if len(test.input_files) > 0:
+        p = test.input_dir
+        if not os.path.isdir(p):
+            logger.error('Input directory not found: "%s"' % p)
             return False
 
+        for fname in test.input_files:
+            p = os.path.join(test.input_dir, fname)
+            if not os.path.isfile(p):
+                logger.error('Input file not found: "%s"' % p)
+                return False
+
     return True

nrtest/process.py

@@ -143,9 +143,9 @@ def _measure_performance(proc, time):
     if not sys.platform.startswith('darwin') and \
        not sys.platform.lower().startswith('sunos'):
 
-            read_MB = float(proc.io_counters().read_bytes) / 1024 / 1024
-            write_MB = float(proc.io_counters().write_bytes) / 1024 / 1024
-            datum['read_MB'] = read_MB
-            datum['write_MB'] = write_MB
+        read_MB = float(proc.io_counters().read_bytes) / 1024 / 1024
+        write_MB = float(proc.io_counters().write_bytes) / 1024 / 1024
+        datum['read_MB'] = read_MB
+        datum['write_MB'] = write_MB
 
     return datum

nrtest/testsuite.py

@@ -27,6 +27,9 @@ def __init__(self, app, tests, benchmark_path):
         self.tests = sorted(tests)
         self.benchmark_path = benchmark_path
 
+        for t in self.tests:
+            t.output_dir = os.path.join(benchmark_path, slugify(t.name))
+
     @classmethod
     def read_config(cls, app_config_path, test_config_paths, benchmark_path):
         """Constructs a TestSuite instance from a set of JSON config files,
@@ -45,7 +48,6 @@ def read_config(cls, app_config_path, test_config_paths, benchmark_path):
             with open(p) as f:
                 t = Test.for_execution(json.load(f))
                 t.input_dir = os.path.dirname(p)
-                t.output_dir = os.path.join(benchmark_path, slugify(t.name))
                 tests.append(t)
 
         return cls(app, tests, benchmark_path)
@@ -64,8 +66,6 @@ def read_benchmark(cls, benchmark_path):
 
         app = Application.for_comparison(manifest['Application'])
         tests = [Test.for_comparison(test) for test in manifest['Tests']]
-        for t in tests:
-            t.output_dir = os.path.join(benchmark_path, slugify(t.name))
 
         return cls(app, tests, benchmark_path)
 

setup.py

@@ -17,6 +17,7 @@
 requirements = [
     'six',
     'psutil>=2.0',
+    'packaging',
 ]
 
 # bundled actions
@@ -55,9 +56,11 @@
         "Programming Language :: Python :: 2",
         'Programming Language :: Python :: 2.7',
         'Programming Language :: Python :: 3',
-        'Programming Language :: Python :: 3.3',
         'Programming Language :: Python :: 3.4',
         'Programming Language :: Python :: 3.5',
+        'Programming Language :: Python :: 3.6',
+        'Programming Language :: Python :: 3.7',
+        'Programming Language :: Python :: 3.8',
     ],
     test_suite='tests'
 )

tests/test_testsuite.py

@@ -0,0 +1,66 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""
+test_testsuite
+----------------------------------
+
+Tests for validation of TestSuite.
+"""
+
+import os.path
+import tempfile
+import unittest
+
+from nrtest import Application, Test
+from nrtest.execute import validate_testsuite
+from nrtest.testsuite import TestSuite
+from nrtest.utility import rmtree
+
+app1 = Application.for_execution({
+    'name': 'app',
+    'version': '1.3.0',
+    'exe': 'echo',
+})
+
+app2 = Application.for_execution({
+    'name': 'app',
+    'version': '1.2.0',
+    'exe': 'echo',
+})
+
+test1 = Test.for_execution({
+    'name': 'cat',
+    'version': '1.0',
+    'args': ['cat'],
+})
+
+test2 = Test.for_execution({
+    'name': 'cat',
+    'version': '1.0',
+    'args': ['dog'],
+    'minimum_app_version': '1.3.0',
+})
+
+
+class TestMinimumAppVersion(unittest.TestCase):
+    def setUp(self):
+        self.benchmark_path = os.path.join(tempfile.gettempdir(), "benchmark")
+
+    def tearDown(self):
+        rmtree(self.benchmark_path)
+
+    def test_sufficient_app_version(self):
+        ts = TestSuite(app1, [test1, test2], self.benchmark_path)
+        self.assertTrue(validate_testsuite(ts))
+        self.assertEqual(len(ts.tests), 2)
+
+    def test_insufficient_app_version(self):
+        ts = TestSuite(app2, [test1, test2], self.benchmark_path)
+        self.assertTrue(validate_testsuite(ts))
+        self.assertEqual(len(ts.tests), 1)
+
+
+if __name__ == '__main__':
+    import sys
+    sys.exit(unittest.main())