From 19319561730aa2156783ca2b0702b6a0f7a95e01 Mon Sep 17 00:00:00 2001 From: Anton Khodak Date: Wed, 7 Feb 2018 12:45:17 +0200 Subject: [PATCH 1/2] Add short names to cwltest and junit-xml --- cwltest/__init__.py | 45 +++++++---- cwltest/cwl_junit_xml.py | 157 +++++++++++++++++++++++++++++++++++++++ cwltest/utils.py | 21 ++++-- 3 files changed, 203 insertions(+), 20 deletions(-) create mode 100644 cwltest/cwl_junit_xml.py diff --git a/cwltest/__init__.py b/cwltest/__init__.py index 9084d13..bdf2a2e 100755 --- a/cwltest/__init__.py +++ b/cwltest/__init__.py @@ -14,7 +14,6 @@ import threading import time -import junit_xml import ruamel.yaml as yaml import ruamel.yaml.scanner as yamlscanner import schema_salad.ref_resolver @@ -23,7 +22,8 @@ from six.moves import zip from typing import Any, Dict, List -from cwltest.utils import compare, CompareFail, TestResult, REQUIRED +import cwltest.cwl_junit_xml as cwl_junit_xml +from cwltest.utils import compare, CompareFail, TestResult, REQUIRED, get_test_number_by_key _logger = logging.getLogger("cwltest") _logger.addHandler(logging.StreamHandler()) @@ -86,7 +86,10 @@ def run_test(args, i, tests, timeout): try: test_command = prepare_test_command(args, i, tests) - sys.stderr.write("%sTest [%i/%i] %s\n" % (prefix, i + 1, len(tests), suffix)) + if t.get("short_name"): + sys.stderr.write("%sTest [%i/%i] %s: %s%s\n" % (prefix, i + 1, len(tests), t.get("short_name"), t.get("doc"), suffix)) + else: + sys.stderr.write("%sTest [%i/%i] %s%s\n" % (prefix, i + 1, len(tests), t.get("doc"), suffix)) sys.stderr.flush() start_time = time.time() @@ -154,7 +157,8 @@ def arg_parser(): # type: () -> argparse.ArgumentParser parser.add_argument("--test", type=str, help="YAML file describing test cases", required=True) parser.add_argument("--basedir", type=str, help="Basedir to use for tests", default=".") parser.add_argument("-l", action="store_true", help="List tests then exit") - parser.add_argument("-n", type=str, default=None, help="Run a specific tests, format is 1,3-6,9") + parser.add_argument("-n", type=str, default=None, help="Run specific tests, format is 1,3-6,9") + parser.add_argument("-s", type=str, default=None, help="Run specific tests using their short names separated by comma") parser.add_argument("--tool", type=str, default="cwl-runner", help="CWL runner executable to use (default 'cwl-runner'") parser.add_argument("--only-tools", action="store_true", help="Only test CommandLineTools") @@ -194,7 +198,7 @@ def main(): # type: () -> int unsupported = 0 passed = 0 suite_name, _ = os.path.splitext(os.path.basename(args.test)) - report = junit_xml.TestSuite(suite_name, []) + report = cwl_junit_xml.CWLTestSuite(suite_name, []) if args.only_tools: alltests = tests @@ -210,17 +214,30 @@ def main(): # type: () -> int if args.l: for i, t in enumerate(tests): - print(u"[%i] %s" % (i + 1, t["doc"].strip())) + if t.get("short_name"): + print(u"[%i] %s: %s" % (i + 1, t["short_name"], t["doc"].strip())) + else: + print(u"[%i] %s" % (i + 1, t["doc"].strip())) + return 0 - if args.n is not None: + if args.n is not None or args.s is not None: ntest = [] - for s in args.n.split(","): - sp = s.split("-") - if len(sp) == 2: - ntest.extend(list(range(int(sp[0]) - 1, int(sp[1])))) - else: - ntest.append(int(s) - 1) + if args.n is not None: + for s in args.n.split(","): + sp = s.split("-") + if len(sp) == 2: + ntest.extend(list(range(int(sp[0]) - 1, int(sp[1])))) + else: + ntest.append(int(s) - 1) + if args.s is not None: + for s in args.s.split(","): + test_number = get_test_number_by_key(tests, "short_name", s) + if test_number: + ntest.append(test_number) + else: + _logger.error('Test with short name "%s" not found ' % s) + return 1 else: ntest = list(range(0, len(tests))) @@ -250,7 +267,7 @@ def main(): # type: () -> int if args.junit_xml: with open(args.junit_xml, 'w') as fp: - junit_xml.TestSuite.to_file(fp, [report]) + cwl_junit_xml.CWLTestSuite.to_file(fp, [report]) if failures == 0 and unsupported == 0: _logger.info("All tests passed") diff --git a/cwltest/cwl_junit_xml.py b/cwltest/cwl_junit_xml.py new file mode 100644 index 0000000..00c9aef --- /dev/null +++ b/cwltest/cwl_junit_xml.py @@ -0,0 +1,157 @@ +import xml.etree.ElementTree as ET + +from junit_xml import TestCase, TestSuite, decode +from typing import Any + + +class CWLTestCase(TestCase): + + def __init__(self, name, classname=None, elapsed_sec=None, stdout=None, + stderr=None, assertions=None, timestamp=None, status=None, + category=None, file=None, line=None, log=None, group=None, + url=None, short_name=None): + # type: (Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any) -> None + super(CWLTestCase, self).__init__(name, classname=classname, elapsed_sec=elapsed_sec, stdout=stdout, + stderr=stderr, assertions=assertions, timestamp=timestamp, status=status, + category=category, file=file, line=line, log=log, group=group, + url=url) + self.short_name = short_name + + +class CWLTestSuite(TestSuite): + def build_xml_doc(self, encoding=None): + """ + This code duplicates the code in junit_xml.TestSuite.build_xml_doc + but allows to add `short_name` attribute from TestCase + """ + + # build the test suite element + test_suite_attributes = dict() + test_suite_attributes['name'] = decode(self.name, encoding) + if any(c.assertions for c in self.test_cases): + test_suite_attributes['assertions'] = \ + str(sum([int(c.assertions) for c in self.test_cases if c.assertions])) + test_suite_attributes['disabled'] = \ + str(len([c for c in self.test_cases if not c.is_enabled])) + test_suite_attributes['failures'] = \ + str(len([c for c in self.test_cases if c.is_failure()])) + test_suite_attributes['errors'] = \ + str(len([c for c in self.test_cases if c.is_error()])) + test_suite_attributes['skipped'] = \ + str(len([c for c in self.test_cases if c.is_skipped()])) + test_suite_attributes['time'] = \ + str(sum(c.elapsed_sec for c in self.test_cases if c.elapsed_sec)) + test_suite_attributes['tests'] = str(len(self.test_cases)) + + if self.hostname: + test_suite_attributes['hostname'] = decode(self.hostname, encoding) + if self.id: + test_suite_attributes['id'] = decode(self.id, encoding) + if self.package: + test_suite_attributes['package'] = decode(self.package, encoding) + if self.timestamp: + test_suite_attributes['timestamp'] = decode(self.timestamp, encoding) + if self.timestamp: + test_suite_attributes['file'] = decode(self.file, encoding) + if self.timestamp: + test_suite_attributes['log'] = decode(self.log, encoding) + if self.timestamp: + test_suite_attributes['url'] = decode(self.url, encoding) + + xml_element = ET.Element("testsuite", test_suite_attributes) + + # add any properties + if self.properties: + props_element = ET.SubElement(xml_element, "properties") + for k, v in self.properties.items(): + attrs = {'name': decode(k, encoding), 'value': decode(v, encoding)} + ET.SubElement(props_element, "property", attrs) + + # add test suite stdout + if self.stdout: + stdout_element = ET.SubElement(xml_element, "system-out") + stdout_element.text = decode(self.stdout, encoding) + + # add test suite stderr + if self.stderr: + stderr_element = ET.SubElement(xml_element, "system-err") + stderr_element.text = decode(self.stderr, encoding) + + # test cases + for case in self.test_cases: + test_case_attributes = dict() + test_case_attributes['name'] = decode(case.name, encoding) + if case.assertions: + # Number of assertions in the test case + test_case_attributes['assertions'] = "%d" % case.assertions + if case.elapsed_sec: + test_case_attributes['time'] = "%f" % case.elapsed_sec + if case.timestamp: + test_case_attributes['timestamp'] = decode(case.timestamp, encoding) + if case.classname: + test_case_attributes['classname'] = decode(case.classname, encoding) + if case.status: + test_case_attributes['status'] = decode(case.status, encoding) + if case.category: + test_case_attributes['class'] = decode(case.category, encoding) + if case.file: + test_case_attributes['file'] = decode(case.file, encoding) + if case.line: + test_case_attributes['line'] = decode(case.line, encoding) + if case.log: + test_case_attributes['log'] = decode(case.log, encoding) + if case.url: + test_case_attributes['url'] = decode(case.url, encoding) + if case.short_name: + test_case_attributes['short_name'] = decode(case.short_name, encoding) + + test_case_element = ET.SubElement( + xml_element, "testcase", test_case_attributes) + + # failures + if case.is_failure(): + attrs = {'type': 'failure'} + if case.failure_message: + attrs['message'] = decode(case.failure_message, encoding) + if case.failure_type: + attrs['type'] = decode(case.failure_type, encoding) + failure_element = ET.Element("failure", attrs) + if case.failure_output: + failure_element.text = decode(case.failure_output, encoding) + test_case_element.append(failure_element) + + # errors + if case.is_error(): + attrs = {'type': 'error'} + if case.error_message: + attrs['message'] = decode(case.error_message, encoding) + if case.error_type: + attrs['type'] = decode(case.error_type, encoding) + error_element = ET.Element("error", attrs) + if case.error_output: + error_element.text = decode(case.error_output, encoding) + test_case_element.append(error_element) + + # skippeds + if case.is_skipped(): + attrs = {'type': 'skipped'} + if case.skipped_message: + attrs['message'] = decode(case.skipped_message, encoding) + skipped_element = ET.Element("skipped", attrs) + if case.skipped_output: + skipped_element.text = decode(case.skipped_output, encoding) + test_case_element.append(skipped_element) + + # test stdout + if case.stdout: + stdout_element = ET.Element("system-out") + stdout_element.text = decode(case.stdout, encoding) + test_case_element.append(stdout_element) + + # test stderr + if case.stderr: + stderr_element = ET.Element("system-err") + stderr_element.text = decode(case.stderr, encoding) + test_case_element.append(stderr_element) + + return xml_element diff --git a/cwltest/utils.py b/cwltest/utils.py index 9f9c3a9..d1feb27 100644 --- a/cwltest/utils.py +++ b/cwltest/utils.py @@ -1,9 +1,9 @@ import json -import junit_xml -from typing import Any, Dict, Set, Text - from six.moves import range +from typing import Any, Dict, Set, Text, List, Optional + +from cwltest import cwl_junit_xml REQUIRED = "required" @@ -22,14 +22,15 @@ def __init__(self, return_code, standard_output, error_output, duration, classna self.classname = classname def create_test_case(self, test): - # type: (Dict[Text, Any]) -> junit_xml.TestCase + # type: (Dict[Text, Any]) -> cwl_junit_xml.TestCase doc = test.get(u'doc', 'N/A').strip() if test.get("tags"): category = ", ".join(test['tags']) else: category = REQUIRED - case = junit_xml.TestCase( - doc, elapsed_sec=self.duration, classname=self.classname, + short_name = test.get(u'short_name') + case = cwl_junit_xml.TestCase( + doc, elapsed_sec=self.duration, short_name=short_name, category=category, stdout=self.standard_output, stderr=self.error_output, ) if self.return_code > 0: @@ -167,3 +168,11 @@ def compare(expected, actual): # type: (Any, Any) -> None except Exception as e: raise CompareFail(str(e)) + + +def get_test_number_by_key(tests, key, value): + # type: (List[Dict[str, str]], str, str) -> Optional[int] + for i, test in enumerate(tests): + if key in test and test[key] == value: + return i + return None From 05dcdbd5524d3b9fc38087724e3674b04f95500c Mon Sep 17 00:00:00 2001 From: Anton Khodak Date: Mon, 5 Mar 2018 09:20:49 +0200 Subject: [PATCH 2/2] Add tests for short names & use junit_xml `file` attribute * instead of non-existent `short_name` attribute --- cwltest/__init__.py | 6 +- cwltest/cwl_junit_xml.py | 157 ------------------ cwltest/utils.py | 9 +- tests/test-data/short-names.yml | 7 + .../with-and-without-short-names.yml | 9 + tests/test_categories.py | 5 +- tests/test_short_names.py | 37 +++++ 7 files changed, 63 insertions(+), 167 deletions(-) delete mode 100644 cwltest/cwl_junit_xml.py create mode 100644 tests/test-data/short-names.yml create mode 100644 tests/test-data/with-and-without-short-names.yml create mode 100644 tests/test_short_names.py diff --git a/cwltest/__init__.py b/cwltest/__init__.py index bdf2a2e..fc5e2c9 100755 --- a/cwltest/__init__.py +++ b/cwltest/__init__.py @@ -22,7 +22,7 @@ from six.moves import zip from typing import Any, Dict, List -import cwltest.cwl_junit_xml as cwl_junit_xml +import junit_xml from cwltest.utils import compare, CompareFail, TestResult, REQUIRED, get_test_number_by_key _logger = logging.getLogger("cwltest") @@ -198,7 +198,7 @@ def main(): # type: () -> int unsupported = 0 passed = 0 suite_name, _ = os.path.splitext(os.path.basename(args.test)) - report = cwl_junit_xml.CWLTestSuite(suite_name, []) + report = junit_xml.TestSuite(suite_name, []) if args.only_tools: alltests = tests @@ -267,7 +267,7 @@ def main(): # type: () -> int if args.junit_xml: with open(args.junit_xml, 'w') as fp: - cwl_junit_xml.CWLTestSuite.to_file(fp, [report]) + junit_xml.TestSuite.to_file(fp, [report]) if failures == 0 and unsupported == 0: _logger.info("All tests passed") diff --git a/cwltest/cwl_junit_xml.py b/cwltest/cwl_junit_xml.py deleted file mode 100644 index 00c9aef..0000000 --- a/cwltest/cwl_junit_xml.py +++ /dev/null @@ -1,157 +0,0 @@ -import xml.etree.ElementTree as ET - -from junit_xml import TestCase, TestSuite, decode -from typing import Any - - -class CWLTestCase(TestCase): - - def __init__(self, name, classname=None, elapsed_sec=None, stdout=None, - stderr=None, assertions=None, timestamp=None, status=None, - category=None, file=None, line=None, log=None, group=None, - url=None, short_name=None): - # type: (Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any) -> None - super(CWLTestCase, self).__init__(name, classname=classname, elapsed_sec=elapsed_sec, stdout=stdout, - stderr=stderr, assertions=assertions, timestamp=timestamp, status=status, - category=category, file=file, line=line, log=log, group=group, - url=url) - self.short_name = short_name - - -class CWLTestSuite(TestSuite): - def build_xml_doc(self, encoding=None): - """ - This code duplicates the code in junit_xml.TestSuite.build_xml_doc - but allows to add `short_name` attribute from TestCase - """ - - # build the test suite element - test_suite_attributes = dict() - test_suite_attributes['name'] = decode(self.name, encoding) - if any(c.assertions for c in self.test_cases): - test_suite_attributes['assertions'] = \ - str(sum([int(c.assertions) for c in self.test_cases if c.assertions])) - test_suite_attributes['disabled'] = \ - str(len([c for c in self.test_cases if not c.is_enabled])) - test_suite_attributes['failures'] = \ - str(len([c for c in self.test_cases if c.is_failure()])) - test_suite_attributes['errors'] = \ - str(len([c for c in self.test_cases if c.is_error()])) - test_suite_attributes['skipped'] = \ - str(len([c for c in self.test_cases if c.is_skipped()])) - test_suite_attributes['time'] = \ - str(sum(c.elapsed_sec for c in self.test_cases if c.elapsed_sec)) - test_suite_attributes['tests'] = str(len(self.test_cases)) - - if self.hostname: - test_suite_attributes['hostname'] = decode(self.hostname, encoding) - if self.id: - test_suite_attributes['id'] = decode(self.id, encoding) - if self.package: - test_suite_attributes['package'] = decode(self.package, encoding) - if self.timestamp: - test_suite_attributes['timestamp'] = decode(self.timestamp, encoding) - if self.timestamp: - test_suite_attributes['file'] = decode(self.file, encoding) - if self.timestamp: - test_suite_attributes['log'] = decode(self.log, encoding) - if self.timestamp: - test_suite_attributes['url'] = decode(self.url, encoding) - - xml_element = ET.Element("testsuite", test_suite_attributes) - - # add any properties - if self.properties: - props_element = ET.SubElement(xml_element, "properties") - for k, v in self.properties.items(): - attrs = {'name': decode(k, encoding), 'value': decode(v, encoding)} - ET.SubElement(props_element, "property", attrs) - - # add test suite stdout - if self.stdout: - stdout_element = ET.SubElement(xml_element, "system-out") - stdout_element.text = decode(self.stdout, encoding) - - # add test suite stderr - if self.stderr: - stderr_element = ET.SubElement(xml_element, "system-err") - stderr_element.text = decode(self.stderr, encoding) - - # test cases - for case in self.test_cases: - test_case_attributes = dict() - test_case_attributes['name'] = decode(case.name, encoding) - if case.assertions: - # Number of assertions in the test case - test_case_attributes['assertions'] = "%d" % case.assertions - if case.elapsed_sec: - test_case_attributes['time'] = "%f" % case.elapsed_sec - if case.timestamp: - test_case_attributes['timestamp'] = decode(case.timestamp, encoding) - if case.classname: - test_case_attributes['classname'] = decode(case.classname, encoding) - if case.status: - test_case_attributes['status'] = decode(case.status, encoding) - if case.category: - test_case_attributes['class'] = decode(case.category, encoding) - if case.file: - test_case_attributes['file'] = decode(case.file, encoding) - if case.line: - test_case_attributes['line'] = decode(case.line, encoding) - if case.log: - test_case_attributes['log'] = decode(case.log, encoding) - if case.url: - test_case_attributes['url'] = decode(case.url, encoding) - if case.short_name: - test_case_attributes['short_name'] = decode(case.short_name, encoding) - - test_case_element = ET.SubElement( - xml_element, "testcase", test_case_attributes) - - # failures - if case.is_failure(): - attrs = {'type': 'failure'} - if case.failure_message: - attrs['message'] = decode(case.failure_message, encoding) - if case.failure_type: - attrs['type'] = decode(case.failure_type, encoding) - failure_element = ET.Element("failure", attrs) - if case.failure_output: - failure_element.text = decode(case.failure_output, encoding) - test_case_element.append(failure_element) - - # errors - if case.is_error(): - attrs = {'type': 'error'} - if case.error_message: - attrs['message'] = decode(case.error_message, encoding) - if case.error_type: - attrs['type'] = decode(case.error_type, encoding) - error_element = ET.Element("error", attrs) - if case.error_output: - error_element.text = decode(case.error_output, encoding) - test_case_element.append(error_element) - - # skippeds - if case.is_skipped(): - attrs = {'type': 'skipped'} - if case.skipped_message: - attrs['message'] = decode(case.skipped_message, encoding) - skipped_element = ET.Element("skipped", attrs) - if case.skipped_output: - skipped_element.text = decode(case.skipped_output, encoding) - test_case_element.append(skipped_element) - - # test stdout - if case.stdout: - stdout_element = ET.Element("system-out") - stdout_element.text = decode(case.stdout, encoding) - test_case_element.append(stdout_element) - - # test stderr - if case.stderr: - stderr_element = ET.Element("system-err") - stderr_element.text = decode(case.stderr, encoding) - test_case_element.append(stderr_element) - - return xml_element diff --git a/cwltest/utils.py b/cwltest/utils.py index d1feb27..f831151 100644 --- a/cwltest/utils.py +++ b/cwltest/utils.py @@ -3,8 +3,7 @@ from six.moves import range from typing import Any, Dict, Set, Text, List, Optional -from cwltest import cwl_junit_xml - +import junit_xml REQUIRED = "required" @@ -22,15 +21,15 @@ def __init__(self, return_code, standard_output, error_output, duration, classna self.classname = classname def create_test_case(self, test): - # type: (Dict[Text, Any]) -> cwl_junit_xml.TestCase + # type: (Dict[Text, Any]) -> junit_xml.TestCase doc = test.get(u'doc', 'N/A').strip() if test.get("tags"): category = ", ".join(test['tags']) else: category = REQUIRED short_name = test.get(u'short_name') - case = cwl_junit_xml.TestCase( - doc, elapsed_sec=self.duration, short_name=short_name, + case = junit_xml.TestCase( + doc, elapsed_sec=self.duration, file=short_name, category=category, stdout=self.standard_output, stderr=self.error_output, ) if self.return_code > 0: diff --git a/tests/test-data/short-names.yml b/tests/test-data/short-names.yml new file mode 100644 index 0000000..4cf6a8e --- /dev/null +++ b/tests/test-data/short-names.yml @@ -0,0 +1,7 @@ +- job: v1.0/cat-job.json + output: {} + tool: return-0.cwl + doc: Test with a short name + short_name: opt-error + tags: [ js, init_work_dir ] + diff --git a/tests/test-data/with-and-without-short-names.yml b/tests/test-data/with-and-without-short-names.yml new file mode 100644 index 0000000..9194548 --- /dev/null +++ b/tests/test-data/with-and-without-short-names.yml @@ -0,0 +1,9 @@ +- job: v1.0/cat1-job.json + output: {} + tool: return-0.cwl + doc: Test without a short name +- job: v1.0/cat2-job.json + output: {} + tool: return-0.cwl + doc: Test with a short name + short_name: opt-error diff --git a/tests/test_categories.py b/tests/test_categories.py index d7269af..297a8a8 100644 --- a/tests/test_categories.py +++ b/tests/test_categories.py @@ -12,14 +12,15 @@ def test_unsupported_with_required_tests(self): args = ["--test", get_data("tests/test-data/required-unsupported.yml")] error_code, stdout, stderr = run_with_mock_cwl_runner(args) self.assertEquals(error_code, 1) - self.assertEquals("Test [1/2] \n\nTest [2/2] \n\n" + self.assertEquals("Test [1/2] Required test that is unsupported (without tags)\n\n" + "Test [2/2] Required test that is unsupported (with tags)\n\n" "0 tests passed, 2 failures, 0 unsupported features\n", stderr) def test_unsupported_with_optional_tests(self): args = ["--test", get_data("tests/test-data/optional-unsupported.yml")] error_code, stdout, stderr = run_with_mock_cwl_runner(args) self.assertEquals(error_code, 0) - self.assertEquals("Test [1/1] \n\n" + self.assertEquals("Test [1/1] Optional test that is unsupported\n\n" "0 tests passed, 1 unsupported features\n", stderr) def test_error_with_optional_tests(self): diff --git a/tests/test_short_names.py b/tests/test_short_names.py new file mode 100644 index 0000000..23e9cc8 --- /dev/null +++ b/tests/test_short_names.py @@ -0,0 +1,37 @@ +import unittest + +import os + +from .util import run_with_mock_cwl_runner, get_data +import xml.etree.ElementTree as ET + + +class TestShortNames(unittest.TestCase): + + def test_stderr_output(self): + args = ["--test", get_data("tests/test-data/short-names.yml")] + error_code, stdout, stderr = run_with_mock_cwl_runner(args) + self.assertIn("Test [1/1] opt-error: Test with a short name\n", stderr) + + def test_run_by_short_name(self): + short_name = "opt-error" + args = ["--test", get_data("tests/test-data/with-and-without-short-names.yml"), "-s", short_name] + error_code, stdout, stderr = run_with_mock_cwl_runner(args) + self.assertIn("Test [2/2] opt-error: Test with a short name", stderr) + self.assertNotIn("Test [1/2]", stderr) + + def test_list_tests(self): + args = ["--test", get_data("tests/test-data/with-and-without-short-names.yml"), "-l"] + error_code, stdout, stderr = run_with_mock_cwl_runner(args) + self.assertEquals("[1] Test without a short name\n" + "[2] opt-error: Test with a short name\n", stdout) + + def test_short_name_in_junit_xml(self): + junit_xml_report = get_data("tests/test-data/junit-report.xml") + args = ["--test", get_data("tests/test-data/short-names.yml"), "--junit-xml", junit_xml_report] + run_with_mock_cwl_runner(args) + tree = ET.parse(junit_xml_report) + root = tree.getroot() + category = root.find("testsuite").find("testcase").attrib['file'] + self.assertEquals(category, "opt-error") + os.remove(junit_xml_report)