diff --git a/cwltest/__init__.py b/cwltest/__init__.py index 9084d13..fc5e2c9 100755 --- a/cwltest/__init__.py +++ b/cwltest/__init__.py @@ -14,7 +14,6 @@ import threading import time -import junit_xml import ruamel.yaml as yaml import ruamel.yaml.scanner as yamlscanner import schema_salad.ref_resolver @@ -23,7 +22,8 @@ from six.moves import zip from typing import Any, Dict, List -from cwltest.utils import compare, CompareFail, TestResult, REQUIRED +import junit_xml +from cwltest.utils import compare, CompareFail, TestResult, REQUIRED, get_test_number_by_key _logger = logging.getLogger("cwltest") _logger.addHandler(logging.StreamHandler()) @@ -86,7 +86,10 @@ def run_test(args, i, tests, timeout): try: test_command = prepare_test_command(args, i, tests) - sys.stderr.write("%sTest [%i/%i] %s\n" % (prefix, i + 1, len(tests), suffix)) + if t.get("short_name"): + sys.stderr.write("%sTest [%i/%i] %s: %s%s\n" % (prefix, i + 1, len(tests), t.get("short_name"), t.get("doc"), suffix)) + else: + sys.stderr.write("%sTest [%i/%i] %s%s\n" % (prefix, i + 1, len(tests), t.get("doc"), suffix)) sys.stderr.flush() start_time = time.time() @@ -154,7 +157,8 @@ def arg_parser(): # type: () -> argparse.ArgumentParser parser.add_argument("--test", type=str, help="YAML file describing test cases", required=True) parser.add_argument("--basedir", type=str, help="Basedir to use for tests", default=".") parser.add_argument("-l", action="store_true", help="List tests then exit") - parser.add_argument("-n", type=str, default=None, help="Run a specific tests, format is 1,3-6,9") + parser.add_argument("-n", type=str, default=None, help="Run specific tests, format is 1,3-6,9") + parser.add_argument("-s", type=str, default=None, help="Run specific tests using their short names separated by comma") parser.add_argument("--tool", type=str, default="cwl-runner", help="CWL runner executable to use (default 'cwl-runner'") parser.add_argument("--only-tools", action="store_true", help="Only test CommandLineTools") @@ -210,17 +214,30 @@ def main(): # type: () -> int if args.l: for i, t in enumerate(tests): - print(u"[%i] %s" % (i + 1, t["doc"].strip())) + if t.get("short_name"): + print(u"[%i] %s: %s" % (i + 1, t["short_name"], t["doc"].strip())) + else: + print(u"[%i] %s" % (i + 1, t["doc"].strip())) + return 0 - if args.n is not None: + if args.n is not None or args.s is not None: ntest = [] - for s in args.n.split(","): - sp = s.split("-") - if len(sp) == 2: - ntest.extend(list(range(int(sp[0]) - 1, int(sp[1])))) - else: - ntest.append(int(s) - 1) + if args.n is not None: + for s in args.n.split(","): + sp = s.split("-") + if len(sp) == 2: + ntest.extend(list(range(int(sp[0]) - 1, int(sp[1])))) + else: + ntest.append(int(s) - 1) + if args.s is not None: + for s in args.s.split(","): + test_number = get_test_number_by_key(tests, "short_name", s) + if test_number: + ntest.append(test_number) + else: + _logger.error('Test with short name "%s" not found ' % s) + return 1 else: ntest = list(range(0, len(tests))) diff --git a/cwltest/utils.py b/cwltest/utils.py index 9f9c3a9..f831151 100644 --- a/cwltest/utils.py +++ b/cwltest/utils.py @@ -1,10 +1,9 @@ import json -import junit_xml -from typing import Any, Dict, Set, Text - from six.moves import range +from typing import Any, Dict, Set, Text, List, Optional +import junit_xml REQUIRED = "required" @@ -28,8 +27,9 @@ def create_test_case(self, test): category = ", ".join(test['tags']) else: category = REQUIRED + short_name = test.get(u'short_name') case = junit_xml.TestCase( - doc, elapsed_sec=self.duration, classname=self.classname, + doc, elapsed_sec=self.duration, file=short_name, category=category, stdout=self.standard_output, stderr=self.error_output, ) if self.return_code > 0: @@ -167,3 +167,11 @@ def compare(expected, actual): # type: (Any, Any) -> None except Exception as e: raise CompareFail(str(e)) + + +def get_test_number_by_key(tests, key, value): + # type: (List[Dict[str, str]], str, str) -> Optional[int] + for i, test in enumerate(tests): + if key in test and test[key] == value: + return i + return None diff --git a/tests/test-data/short-names.yml b/tests/test-data/short-names.yml new file mode 100644 index 0000000..4cf6a8e --- /dev/null +++ b/tests/test-data/short-names.yml @@ -0,0 +1,7 @@ +- job: v1.0/cat-job.json + output: {} + tool: return-0.cwl + doc: Test with a short name + short_name: opt-error + tags: [ js, init_work_dir ] + diff --git a/tests/test-data/with-and-without-short-names.yml b/tests/test-data/with-and-without-short-names.yml new file mode 100644 index 0000000..9194548 --- /dev/null +++ b/tests/test-data/with-and-without-short-names.yml @@ -0,0 +1,9 @@ +- job: v1.0/cat1-job.json + output: {} + tool: return-0.cwl + doc: Test without a short name +- job: v1.0/cat2-job.json + output: {} + tool: return-0.cwl + doc: Test with a short name + short_name: opt-error diff --git a/tests/test_categories.py b/tests/test_categories.py index d7269af..297a8a8 100644 --- a/tests/test_categories.py +++ b/tests/test_categories.py @@ -12,14 +12,15 @@ def test_unsupported_with_required_tests(self): args = ["--test", get_data("tests/test-data/required-unsupported.yml")] error_code, stdout, stderr = run_with_mock_cwl_runner(args) self.assertEquals(error_code, 1) - self.assertEquals("Test [1/2] \n\nTest [2/2] \n\n" + self.assertEquals("Test [1/2] Required test that is unsupported (without tags)\n\n" + "Test [2/2] Required test that is unsupported (with tags)\n\n" "0 tests passed, 2 failures, 0 unsupported features\n", stderr) def test_unsupported_with_optional_tests(self): args = ["--test", get_data("tests/test-data/optional-unsupported.yml")] error_code, stdout, stderr = run_with_mock_cwl_runner(args) self.assertEquals(error_code, 0) - self.assertEquals("Test [1/1] \n\n" + self.assertEquals("Test [1/1] Optional test that is unsupported\n\n" "0 tests passed, 1 unsupported features\n", stderr) def test_error_with_optional_tests(self): diff --git a/tests/test_short_names.py b/tests/test_short_names.py new file mode 100644 index 0000000..23e9cc8 --- /dev/null +++ b/tests/test_short_names.py @@ -0,0 +1,37 @@ +import unittest + +import os + +from .util import run_with_mock_cwl_runner, get_data +import xml.etree.ElementTree as ET + + +class TestShortNames(unittest.TestCase): + + def test_stderr_output(self): + args = ["--test", get_data("tests/test-data/short-names.yml")] + error_code, stdout, stderr = run_with_mock_cwl_runner(args) + self.assertIn("Test [1/1] opt-error: Test with a short name\n", stderr) + + def test_run_by_short_name(self): + short_name = "opt-error" + args = ["--test", get_data("tests/test-data/with-and-without-short-names.yml"), "-s", short_name] + error_code, stdout, stderr = run_with_mock_cwl_runner(args) + self.assertIn("Test [2/2] opt-error: Test with a short name", stderr) + self.assertNotIn("Test [1/2]", stderr) + + def test_list_tests(self): + args = ["--test", get_data("tests/test-data/with-and-without-short-names.yml"), "-l"] + error_code, stdout, stderr = run_with_mock_cwl_runner(args) + self.assertEquals("[1] Test without a short name\n" + "[2] opt-error: Test with a short name\n", stdout) + + def test_short_name_in_junit_xml(self): + junit_xml_report = get_data("tests/test-data/junit-report.xml") + args = ["--test", get_data("tests/test-data/short-names.yml"), "--junit-xml", junit_xml_report] + run_with_mock_cwl_runner(args) + tree = ET.parse(junit_xml_report) + root = tree.getroot() + category = root.find("testsuite").find("testcase").attrib['file'] + self.assertEquals(category, "opt-error") + os.remove(junit_xml_report)