Merge pull request #51 from common-workflow-language/short-names

Add short names to cwltest and junit-xml

cwltest/__init__.py

@@ -14,7 +14,6 @@
 import threading
 import time
 
-import junit_xml
 import ruamel.yaml as yaml
 import ruamel.yaml.scanner as yamlscanner
 import schema_salad.ref_resolver
@@ -23,7 +22,8 @@
 from six.moves import zip
 from typing import Any, Dict, List
 
-from cwltest.utils import compare, CompareFail, TestResult, REQUIRED
+import junit_xml
+from cwltest.utils import compare, CompareFail, TestResult, REQUIRED, get_test_number_by_key
 
 _logger = logging.getLogger("cwltest")
 _logger.addHandler(logging.StreamHandler())
@@ -86,7 +86,10 @@ def run_test(args, i, tests, timeout):
     try:
         test_command = prepare_test_command(args, i, tests)
 
-        sys.stderr.write("%sTest [%i/%i] %s\n" % (prefix, i + 1, len(tests), suffix))
+        if t.get("short_name"):
+            sys.stderr.write("%sTest [%i/%i] %s: %s%s\n" % (prefix, i + 1, len(tests), t.get("short_name"), t.get("doc"), suffix))
+        else:
+            sys.stderr.write("%sTest [%i/%i] %s%s\n" % (prefix, i + 1, len(tests), t.get("doc"), suffix))
         sys.stderr.flush()
 
         start_time = time.time()
@@ -154,7 +157,8 @@ def arg_parser():  # type: () -> argparse.ArgumentParser
     parser.add_argument("--test", type=str, help="YAML file describing test cases", required=True)
     parser.add_argument("--basedir", type=str, help="Basedir to use for tests", default=".")
     parser.add_argument("-l", action="store_true", help="List tests then exit")
-    parser.add_argument("-n", type=str, default=None, help="Run a specific tests, format is 1,3-6,9")
+    parser.add_argument("-n", type=str, default=None, help="Run specific tests, format is 1,3-6,9")
+    parser.add_argument("-s", type=str, default=None, help="Run specific tests using their short names separated by comma")
     parser.add_argument("--tool", type=str, default="cwl-runner",
                         help="CWL runner executable to use (default 'cwl-runner'")
     parser.add_argument("--only-tools", action="store_true", help="Only test CommandLineTools")
@@ -210,17 +214,30 @@ def main():  # type: () -> int
 
     if args.l:
         for i, t in enumerate(tests):
-            print(u"[%i] %s" % (i + 1, t["doc"].strip()))
+            if t.get("short_name"):
+                print(u"[%i] %s: %s" % (i + 1, t["short_name"], t["doc"].strip()))
+            else:
+                print(u"[%i] %s" % (i + 1, t["doc"].strip()))
+
         return 0
 
-    if args.n is not None:
+    if args.n is not None or args.s is not None:
         ntest = []
-        for s in args.n.split(","):
-            sp = s.split("-")
-            if len(sp) == 2:
-                ntest.extend(list(range(int(sp[0]) - 1, int(sp[1]))))
-            else:
-                ntest.append(int(s) - 1)
+        if args.n is not None:
+            for s in args.n.split(","):
+                sp = s.split("-")
+                if len(sp) == 2:
+                    ntest.extend(list(range(int(sp[0]) - 1, int(sp[1]))))
+                else:
+                    ntest.append(int(s) - 1)
+        if args.s is not None:
+            for s in args.s.split(","):
+                test_number = get_test_number_by_key(tests, "short_name", s)
+                if test_number:
+                    ntest.append(test_number)
+                else:
+                    _logger.error('Test with short name "%s" not found ' % s)
+                    return 1
     else:
         ntest = list(range(0, len(tests)))
 

cwltest/utils.py

@@ -1,10 +1,9 @@
 import json
 
-import junit_xml
-from typing import Any, Dict, Set, Text
-
 from six.moves import range
+from typing import Any, Dict, Set, Text, List, Optional
 
+import junit_xml
 REQUIRED = "required"
 
 
@@ -28,8 +27,9 @@ def create_test_case(self, test):
             category = ", ".join(test['tags'])
         else:
             category = REQUIRED
+        short_name = test.get(u'short_name')
         case = junit_xml.TestCase(
-            doc, elapsed_sec=self.duration, classname=self.classname,
+            doc, elapsed_sec=self.duration, file=short_name,
             category=category, stdout=self.standard_output, stderr=self.error_output,
         )
         if self.return_code > 0:
@@ -167,3 +167,11 @@ def compare(expected, actual):  # type: (Any, Any) -> None
 
     except Exception as e:
         raise CompareFail(str(e))
+
+
+def get_test_number_by_key(tests, key, value):
+    # type: (List[Dict[str, str]], str, str) -> Optional[int]
+    for i, test in enumerate(tests):
+        if key in test and test[key] == value:
+            return i
+    return None

tests/test-data/short-names.yml

@@ -0,0 +1,7 @@
+- job: v1.0/cat-job.json
+  output: {}
+  tool: return-0.cwl
+  doc: Test with a short name
+  short_name: opt-error
+  tags: [ js, init_work_dir ]
+

tests/test-data/with-and-without-short-names.yml

@@ -0,0 +1,9 @@
+- job: v1.0/cat1-job.json
+  output: {}
+  tool: return-0.cwl
+  doc: Test without a short name
+- job: v1.0/cat2-job.json
+  output: {}
+  tool: return-0.cwl
+  doc: Test with a short name
+  short_name: opt-error

tests/test_categories.py

@@ -12,14 +12,15 @@ def test_unsupported_with_required_tests(self):
         args = ["--test", get_data("tests/test-data/required-unsupported.yml")]
         error_code, stdout, stderr = run_with_mock_cwl_runner(args)
         self.assertEquals(error_code, 1)
-        self.assertEquals("Test [1/2] \n\nTest [2/2] \n\n"
+        self.assertEquals("Test [1/2] Required test that is unsupported (without tags)\n\n"
+                          "Test [2/2] Required test that is unsupported (with tags)\n\n"
                           "0 tests passed, 2 failures, 0 unsupported features\n", stderr)
 
     def test_unsupported_with_optional_tests(self):
         args = ["--test", get_data("tests/test-data/optional-unsupported.yml")]
         error_code, stdout, stderr = run_with_mock_cwl_runner(args)
         self.assertEquals(error_code, 0)
-        self.assertEquals("Test [1/1] \n\n"
+        self.assertEquals("Test [1/1] Optional test that is unsupported\n\n"
                           "0 tests passed, 1 unsupported features\n", stderr)
 
     def test_error_with_optional_tests(self):

tests/test_short_names.py

@@ -0,0 +1,37 @@
+import unittest
+
+import os
+
+from .util import run_with_mock_cwl_runner, get_data
+import xml.etree.ElementTree as ET
+
+
+class TestShortNames(unittest.TestCase):
+
+    def test_stderr_output(self):
+        args = ["--test", get_data("tests/test-data/short-names.yml")]
+        error_code, stdout, stderr = run_with_mock_cwl_runner(args)
+        self.assertIn("Test [1/1] opt-error: Test with a short name\n", stderr)
+
+    def test_run_by_short_name(self):
+        short_name = "opt-error"
+        args = ["--test", get_data("tests/test-data/with-and-without-short-names.yml"), "-s", short_name]
+        error_code, stdout, stderr = run_with_mock_cwl_runner(args)
+        self.assertIn("Test [2/2] opt-error: Test with a short name", stderr)
+        self.assertNotIn("Test [1/2]", stderr)
+
+    def test_list_tests(self):
+        args = ["--test", get_data("tests/test-data/with-and-without-short-names.yml"), "-l"]
+        error_code, stdout, stderr = run_with_mock_cwl_runner(args)
+        self.assertEquals("[1] Test without a short name\n"
+                          "[2] opt-error: Test with a short name\n", stdout)
+
+    def test_short_name_in_junit_xml(self):
+        junit_xml_report = get_data("tests/test-data/junit-report.xml")
+        args = ["--test", get_data("tests/test-data/short-names.yml"), "--junit-xml", junit_xml_report]
+        run_with_mock_cwl_runner(args)
+        tree = ET.parse(junit_xml_report)
+        root = tree.getroot()
+        category = root.find("testsuite").find("testcase").attrib['file']
+        self.assertEquals(category, "opt-error")
+        os.remove(junit_xml_report)