From 2843e2be888656045560e1702e8ff019f4ba3b69 Mon Sep 17 00:00:00 2001 From: Daniel J Magee Date: Mon, 14 Feb 2022 16:56:32 -0700 Subject: [PATCH 01/10] Remove json flag for query commands (result/status). Replace it with outfile flag, default is self.outfile, which, if set, will redirect output to the specified file. If that file ends in '.json', then json output of the specified type will be dumped to that file. --- lib/pavilion/commands/result.py | 32 +++++++++++++++----------------- lib/pavilion/commands/status.py | 12 ++++++------ lib/pavilion/status_utils.py | 8 ++++---- test/tests/status_cmd_tests.py | 4 +--- 4 files changed, 26 insertions(+), 30 deletions(-) diff --git a/lib/pavilion/commands/result.py b/lib/pavilion/commands/result.py index 6b3a5ab87..56563e47b 100644 --- a/lib/pavilion/commands/result.py +++ b/lib/pavilion/commands/result.py @@ -35,10 +35,8 @@ def __init__(self): def _setup_arguments(self, parser): parser.add_argument( - "-j", "--json", - action="store_true", default=False, - help="Give the results in json." - ) + '--outfile', '-o', action='store', default=self.outfile, + help='Send output to file, type dependent on extension (json).') group = parser.add_mutually_exclusive_group() group.add_argument( "-k", "--key", type=str, default='', @@ -93,21 +91,21 @@ def run(self, pav_cfg, args): results = result_utils.get_results(pav_cfg, tests) - if args.json or args.full: + if args.outfile.endswith('json'): + output.json_dump(results, args.outfile) + + elif args.full: if not results: output.fprint("Could not find any matching tests.", - color=output.RED, file=self.outfile) + color=output.RED, file=args.outfile) return errno.EINVAL width = shutil.get_terminal_size().columns or 80 try: - if args.json: - output.json_dump(results, self.outfile) - else: - pprint.pprint(results, # ext-print: ignore - stream=self.outfile, width=width, - compact=True) + pprint.pprint(results, # ext-print: ignore + stream=args.outfile, width=width, + compact=True) except OSError: # It's ok if this fails. Generally means we're piping to # another command. @@ -117,7 +115,7 @@ def run(self, pav_cfg, args): fields = result_utils.BASE_FIELDS + args.key.replace(',', ' ').split() output.draw_table( - outfile=self.outfile, + outfile=args.outfile, field_info={ 'started': {'transform': output.get_relative_timestamp}, 'finished': {'transform': output.get_relative_timestamp}, @@ -129,7 +127,7 @@ def run(self, pav_cfg, args): if args.show_log: if log_file is not None: - output.fprint(log_file.getvalue(), file=self.outfile, + output.fprint(log_file.getvalue(), file=args.outfile, color=output.GREY) else: if len(results) > 1: @@ -141,15 +139,15 @@ def run(self, pav_cfg, args): result_set = results[0] log_path = pathlib.Path(result_set['results_log']) output.fprint("\nResult logs for test {}\n" - .format(result_set['name']), file=self.outfile) + .format(result_set['name']), file=args.outfile) if log_path.exists(): with log_path.open() as log_file: output.fprint( log_file.read(), color=output.GREY, - file=self.outfile) + file=args.outfile) else: output.fprint("Log file '{}' missing>".format(log_path), - file=self.outfile, color=output.YELLOW) + file=args.outfile, color=output.YELLOW) return 0 diff --git a/lib/pavilion/commands/status.py b/lib/pavilion/commands/status.py index 273b94625..d21ca4d02 100644 --- a/lib/pavilion/commands/status.py +++ b/lib/pavilion/commands/status.py @@ -2,6 +2,7 @@ other commands to print statuses.""" import errno +import sys from pavilion import cmd_utils from pavilion import filters @@ -21,9 +22,8 @@ def __init__(self): def _setup_arguments(self, parser): parser.add_argument( - '-j', '--json', action='store_true', default=False, - help='Give output as json, rather than as standard human readable.' - ) + '--outfile', '-o', action='store', default=self.outfile, + help='Send output to file, type dependent on extension (json).') parser.add_argument( '--series', action='store_true', default=False, help='Show the series the test belongs to.') @@ -68,7 +68,7 @@ def run(self, pav_cfg, args): file=self.errfile, color=output.RED) return 1 - return status_utils.print_status_history(tests[-1], self.outfile, args.json) + return status_utils.print_status_history(tests[-1], args.outfile) tests = cmd_utils.get_tests_by_paths(pav_cfg, test_paths, self.errfile) @@ -76,8 +76,8 @@ def run(self, pav_cfg, args): if args.summary: return self.print_summary(statuses) else: - return status_utils.print_status(statuses, self.outfile, json=args.json, - series=args.series, note=args.note) + return status_utils.print_status(statuses, args.outfile, series=args.series, + note=args.note) def print_summary(self, statuses): """Print_summary takes in a list of test statuses. diff --git a/lib/pavilion/status_utils.py b/lib/pavilion/status_utils.py index b170c929c..e32b48270 100644 --- a/lib/pavilion/status_utils.py +++ b/lib/pavilion/status_utils.py @@ -118,7 +118,7 @@ def get_statuses(pav_cfg, tests: List[TestRun]): return list(pool.map(get_this_status, tests)) -def print_status(statuses: List[dict], outfile, note=False, series=False, json=False): +def print_status(statuses: List[dict], outfile, note=False, series=False): """Prints the statuses provided in the statuses parameter. :param statuses: list of dictionary objects containing the test_id, @@ -133,7 +133,7 @@ def print_status(statuses: List[dict], outfile, note=False, series=False, json=F :rtype: int """ - if json: + if outfile.endswith('json'): json_data = {'statuses': statuses} output.json_dump(json_data, outfile) else: @@ -195,7 +195,7 @@ def status_history_from_test_obj(test: TestRun) -> List[dict]: return status_history -def print_status_history(test: TestRun, outfile: TextIO, json: bool = False): +def print_status_history(test: TestRun, outfile: TextIO): """Print the status history for a given test object. :param test: Single test object. @@ -210,7 +210,7 @@ def print_status_history(test: TestRun, outfile: TextIO, json: bool = False): for status in status_history: if status['note'] != "Test not found.": ret_val = 0 - if json: + if outfile.endswith('json'): json_data = {'status_history': status_history} output.json_dump(json_data, outfile) else: diff --git a/test/tests/status_cmd_tests.py b/test/tests/status_cmd_tests.py index 924582182..3519c30de 100644 --- a/test/tests/status_cmd_tests.py +++ b/test/tests/status_cmd_tests.py @@ -28,15 +28,13 @@ def test_status_arguments(self): self.assertEqual(args.tests[0], 'test1') self.assertEqual(args.tests[1], 'test2') - self.assertEqual(args.json, False) parser = argparse.ArgumentParser() status_cmd._setup_arguments(parser) - args = parser.parse_args(['-j', 'test0', 'test9']) + args = parser.parse_args(['test0', 'test9']) self.assertEqual(args.tests[0], 'test0') self.assertEqual(args.tests[1], 'test9') - self.assertEqual(args.json, True) def test_status_command(self): """Test status command by generating a suite of tests.""" From 59157edbebd2fb672edfd4fb04a0200de859c14a Mon Sep 17 00:00:00 2001 From: Daniel J Magee Date: Tue, 22 Feb 2022 10:17:18 -0700 Subject: [PATCH 02/10] Check if the outfile argument is a string before checking if it ends in json. --- lib/pavilion/commands/result.py | 5 +++-- lib/pavilion/status_utils.py | 40 +++++++++++++++++---------------- 2 files changed, 24 insertions(+), 21 deletions(-) diff --git a/lib/pavilion/commands/result.py b/lib/pavilion/commands/result.py index 56563e47b..65bab7179 100644 --- a/lib/pavilion/commands/result.py +++ b/lib/pavilion/commands/result.py @@ -91,8 +91,9 @@ def run(self, pav_cfg, args): results = result_utils.get_results(pav_cfg, tests) - if args.outfile.endswith('json'): - output.json_dump(results, args.outfile) + if isinstance(args.outfile, str): + if args.outfile.endswith('json'): + output.json_dump(results, args.outfile) elif args.full: if not results: diff --git a/lib/pavilion/status_utils.py b/lib/pavilion/status_utils.py index e32b48270..46bb7daa0 100644 --- a/lib/pavilion/status_utils.py +++ b/lib/pavilion/status_utils.py @@ -133,25 +133,27 @@ def print_status(statuses: List[dict], outfile, note=False, series=False): :rtype: int """ - if outfile.endswith('json'): - json_data = {'statuses': statuses} - output.json_dump(json_data, outfile) - else: - fields = ['test_id', 'job_id', 'name', 'nodes', 'state', 'result', 'time'] - if series: - fields.insert(0, 'series') - if note: - fields.append('note') - - output.draw_table( - outfile=outfile, - field_info={ - 'time': {'transform': output.get_relative_timestamp}, - 'test_id': {'title': 'Test'}, - }, - fields=fields, - rows=statuses, - title='Test statuses') + if isinstance(outfile, str): + if outfile.endswith('json'): + json_data = {'statuses': statuses} + output.json_dump(json_data, outfile) + return 0 + + fields = ['test_id', 'job_id', 'name', 'nodes', 'state', 'result', 'time'] + if series: + fields.insert(0, 'series') + if note: + fields.append('note') + + output.draw_table( + outfile=outfile, + field_info={ + 'time': {'transform': output.get_relative_timestamp}, + 'test_id': {'title': 'Test'}, + }, + fields=fields, + rows=statuses, + title='Test statuses') return 0 From 125c292afa186ec3fac4d0c9f0894817ab46f4a4 Mon Sep 17 00:00:00 2001 From: Daniel J Magee Date: Tue, 22 Feb 2022 10:30:52 -0700 Subject: [PATCH 03/10] Whitespace --- lib/pavilion/commands/result.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pavilion/commands/result.py b/lib/pavilion/commands/result.py index ef64353ac..83b166255 100644 --- a/lib/pavilion/commands/result.py +++ b/lib/pavilion/commands/result.py @@ -100,7 +100,7 @@ def run(self, pav_cfg, args): flat_results = [] for result in results: flat_results.append(utils.flatten_dictionary(result)) - + field_info = {} if isinstance(args.outfile, str): From 63b9d3112ef68281bf5d9a1d90da46e6f69d4ef3 Mon Sep 17 00:00:00 2001 From: Daniel J Magee Date: Tue, 22 Feb 2022 10:58:09 -0700 Subject: [PATCH 04/10] Removed json cmd from pav tests. Fixed status history print. --- lib/pavilion/status_utils.py | 28 +++++++++++++++------------- test/tests/result_tests.py | 2 +- test/tests/status_cmd_tests.py | 6 +++--- 3 files changed, 19 insertions(+), 17 deletions(-) diff --git a/lib/pavilion/status_utils.py b/lib/pavilion/status_utils.py index 46bb7daa0..a05c737b0 100644 --- a/lib/pavilion/status_utils.py +++ b/lib/pavilion/status_utils.py @@ -212,18 +212,20 @@ def print_status_history(test: TestRun, outfile: TextIO): for status in status_history: if status['note'] != "Test not found.": ret_val = 0 - if outfile.endswith('json'): - json_data = {'status_history': status_history} - output.json_dump(json_data, outfile) - else: - fields = ['state', 'time', 'note'] - output.draw_table( - outfile=outfile, - field_info={ - 'time': {'transform': output.get_relative_timestamp} - }, - fields=fields, - rows=status_history, - title='Test {} Status History ({})'.format(test.id, test.name)) + + if isinstance(outfile, str): + if outfile.endswith('json'): + json_data = {'status_history': status_history} + output.json_dump(json_data, outfile) + + fields = ['state', 'time', 'note'] + output.draw_table( + outfile=outfile, + field_info={ + 'time': {'transform': output.get_relative_timestamp} + }, + fields=fields, + rows=status_history, + title='Test {} Status History ({})'.format(test.id, test.name)) return ret_val diff --git a/test/tests/result_tests.py b/test/tests/result_tests.py index e8bb8d0d4..f7f6cde91 100644 --- a/test/tests/result_tests.py +++ b/test/tests/result_tests.py @@ -597,7 +597,7 @@ def test_result_command(self): # Check that the changed results are what we expected. result_cmd.clear_output() res_args = arg_parser.parse_args( - ('result', '--re-run', '--json') + + ('result', '--re-run', '--full') + tuple(t.full_id for t in run_cmd.last_tests)) result_cmd.run(rerun_cfg, res_args) diff --git a/test/tests/status_cmd_tests.py b/test/tests/status_cmd_tests.py index 3519c30de..68d16c100 100644 --- a/test/tests/status_cmd_tests.py +++ b/test/tests/status_cmd_tests.py @@ -98,14 +98,14 @@ def test_status_command(self): for test in series.tests.values(): parser = argparse.ArgumentParser() status_cmd._setup_arguments(parser) - arg_list = ['-j', test.full_id] + arg_list = ['--full', test.full_id] args = parser.parse_args(arg_list) self.assertEqual(status_cmd.run(self.pav_cfg, args), 0) # Testing for multiple tests with json output parser = argparse.ArgumentParser() status_cmd._setup_arguments(parser) - arg_list = ['-j'] + test_str.split() + arg_list = ['--full'] + test_str.split() args = parser.parse_args(arg_list) self.assertEqual(status_cmd.run(self.pav_cfg, args), 0) @@ -224,7 +224,7 @@ def test_status_command_with_sched(self): parser = argparse.ArgumentParser() status_cmd._setup_arguments(parser) - args = parser.parse_args(['-j', 'test.{}'.format(test.id)]) + args = parser.parse_args(['--full', 'test.{}'.format(test.id)]) test.status.set(status_file.STATES.SCHEDULED, "faker") self.assertEqual(status_cmd.run(self.pav_cfg, args), 0) From 84d7855002cbc01f97a32ceff9b5cdce76a081a2 Mon Sep 17 00:00:00 2001 From: Daniel J Magee Date: Tue, 22 Feb 2022 11:24:11 -0700 Subject: [PATCH 05/10] Removed full flag from status tests. --- lib/pavilion/commands/result.py | 2 +- test/tests/status_cmd_tests.py | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/lib/pavilion/commands/result.py b/lib/pavilion/commands/result.py index 83b166255..4dd73f13d 100644 --- a/lib/pavilion/commands/result.py +++ b/lib/pavilion/commands/result.py @@ -136,7 +136,7 @@ def run(self, pav_cfg, args): try: pprint.pprint(results, # ext-print: ignore - stream=args.outfile, width=width, + stream=self.outfile, width=width, compact=True) except OSError: # It's ok if this fails. Generally means we're piping to diff --git a/test/tests/status_cmd_tests.py b/test/tests/status_cmd_tests.py index 68d16c100..13e135115 100644 --- a/test/tests/status_cmd_tests.py +++ b/test/tests/status_cmd_tests.py @@ -93,19 +93,20 @@ def test_status_command(self): status_cmd = commands.get_command('status') status_cmd.outfile = io.StringIO() + jsontest_file = ['-o', 'test.json'] # Testing for individual tests with json output for test in series.tests.values(): parser = argparse.ArgumentParser() status_cmd._setup_arguments(parser) - arg_list = ['--full', test.full_id] + arg_list = jsontest_file + [test.full_id] args = parser.parse_args(arg_list) self.assertEqual(status_cmd.run(self.pav_cfg, args), 0) # Testing for multiple tests with json output parser = argparse.ArgumentParser() status_cmd._setup_arguments(parser) - arg_list = ['--full'] + test_str.split() + arg_list = jsontest_file + test_str.split() args = parser.parse_args(arg_list) self.assertEqual(status_cmd.run(self.pav_cfg, args), 0) @@ -214,6 +215,7 @@ def test_status_command_with_sched(self): status_cmd = commands.get_command('status') status_cmd.silence() + jsontest_file = ['-o', 'test.json'] parser = argparse.ArgumentParser() status_cmd._setup_arguments(parser) @@ -224,7 +226,7 @@ def test_status_command_with_sched(self): parser = argparse.ArgumentParser() status_cmd._setup_arguments(parser) - args = parser.parse_args(['--full', 'test.{}'.format(test.id)]) + args = parser.parse_args(jsontest_file + ['test.{}'.format(test.id)]) test.status.set(status_file.STATES.SCHEDULED, "faker") self.assertEqual(status_cmd.run(self.pav_cfg, args), 0) From da5428770c4bf777a91959b0ff56b963e151eb8e Mon Sep 17 00:00:00 2001 From: Daniel J Magee Date: Tue, 22 Feb 2022 11:58:33 -0700 Subject: [PATCH 06/10] Open the json file before passing its handle to output.json_dump. --- lib/pavilion/commands/result.py | 3 ++- lib/pavilion/output.py | 4 +++- lib/pavilion/status_utils.py | 9 ++++++--- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/lib/pavilion/commands/result.py b/lib/pavilion/commands/result.py index 4dd73f13d..7b598e9a7 100644 --- a/lib/pavilion/commands/result.py +++ b/lib/pavilion/commands/result.py @@ -105,7 +105,8 @@ def run(self, pav_cfg, args): if isinstance(args.outfile, str): if args.outfile.endswith('json'): - output.json_dump(results, args.outfile) + with args.outfile.open('w') as json_file: + output.json_dump(results, json_file) elif args.list_keys: flat_keys = result_utils.keylist(flat_results) diff --git a/lib/pavilion/output.py b/lib/pavilion/output.py index 5be32cd0c..a9ee95934 100644 --- a/lib/pavilion/output.py +++ b/lib/pavilion/output.py @@ -227,7 +227,7 @@ def json_dump(obj, file, skipkeys=False, ensure_ascii=True, """Dump data to string as per the json dumps function, but using our custom encoder.""" - return json.dump(obj, file, cls=PavEncoder, + json_rtn = json.dump(obj, file, cls=PavEncoder, skipkeys=skipkeys, ensure_ascii=ensure_ascii, check_circular=check_circular, @@ -238,6 +238,8 @@ def json_dump(obj, file, skipkeys=False, ensure_ascii=True, sort_keys=sort_keys, **kw) + return json_rtn + def output_csv(outfile, fields, rows, field_info=None, header=False): """Write the given rows out as a CSV. diff --git a/lib/pavilion/status_utils.py b/lib/pavilion/status_utils.py index a05c737b0..e5645d8d4 100644 --- a/lib/pavilion/status_utils.py +++ b/lib/pavilion/status_utils.py @@ -136,8 +136,9 @@ def print_status(statuses: List[dict], outfile, note=False, series=False): if isinstance(outfile, str): if outfile.endswith('json'): json_data = {'statuses': statuses} - output.json_dump(json_data, outfile) - return 0 + with outfile.open('w') as json_file: + output.json_dump(json_data, json_file) + return 0 fields = ['test_id', 'job_id', 'name', 'nodes', 'state', 'result', 'time'] if series: @@ -216,7 +217,9 @@ def print_status_history(test: TestRun, outfile: TextIO): if isinstance(outfile, str): if outfile.endswith('json'): json_data = {'status_history': status_history} - output.json_dump(json_data, outfile) + with outfile.open('w') as json_file: + output.json_dump(json_data, json_file) + return 0 fields = ['state', 'time', 'note'] output.draw_table( From 566d16beba18c26d5fe0954eef97ee21378cc8bd Mon Sep 17 00:00:00 2001 From: Daniel J Magee Date: Tue, 22 Feb 2022 12:13:31 -0700 Subject: [PATCH 07/10] Open the outfile properly. --- lib/pavilion/commands/result.py | 2 +- lib/pavilion/status_utils.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/pavilion/commands/result.py b/lib/pavilion/commands/result.py index 7b598e9a7..0b89ddf93 100644 --- a/lib/pavilion/commands/result.py +++ b/lib/pavilion/commands/result.py @@ -105,7 +105,7 @@ def run(self, pav_cfg, args): if isinstance(args.outfile, str): if args.outfile.endswith('json'): - with args.outfile.open('w') as json_file: + with open(args.outfile, 'w') as json_file: output.json_dump(results, json_file) elif args.list_keys: diff --git a/lib/pavilion/status_utils.py b/lib/pavilion/status_utils.py index e5645d8d4..5406bf2e8 100644 --- a/lib/pavilion/status_utils.py +++ b/lib/pavilion/status_utils.py @@ -136,7 +136,7 @@ def print_status(statuses: List[dict], outfile, note=False, series=False): if isinstance(outfile, str): if outfile.endswith('json'): json_data = {'statuses': statuses} - with outfile.open('w') as json_file: + with open(outfile, 'w') as json_file: output.json_dump(json_data, json_file) return 0 @@ -217,7 +217,7 @@ def print_status_history(test: TestRun, outfile: TextIO): if isinstance(outfile, str): if outfile.endswith('json'): json_data = {'status_history': status_history} - with outfile.open('w') as json_file: + with open(outfile, 'w') as json_file: output.json_dump(json_data, json_file) return 0 From 1f115cfbe3dcbb395329823dd733057a96322a15 Mon Sep 17 00:00:00 2001 From: Daniel J Magee Date: Tue, 22 Feb 2022 15:14:53 -0700 Subject: [PATCH 08/10] The full flag didn't produce real json. But this method does and looks nicer. --- lib/pavilion/commands/result.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/lib/pavilion/commands/result.py b/lib/pavilion/commands/result.py index 0b89ddf93..904eba24d 100644 --- a/lib/pavilion/commands/result.py +++ b/lib/pavilion/commands/result.py @@ -2,9 +2,9 @@ import datetime import errno +import json import io import pathlib -import pprint import shutil from typing import List, IO @@ -136,9 +136,8 @@ def run(self, pav_cfg, args): result['finished'], fullstamp=True) try: - pprint.pprint(results, # ext-print: ignore - stream=self.outfile, width=width, - compact=True) + json_string = json.dumps(results, indent=2) + print(json_string, file=self.outfile) except OSError: # It's ok if this fails. Generally means we're piping to # another command. From 73e51bed7f4a80d002e2816e335862dfc47aa930 Mon Sep 17 00:00:00 2001 From: Daniel J Magee Date: Tue, 22 Feb 2022 15:40:56 -0700 Subject: [PATCH 09/10] Now produces the correct json (full flag) output with output.fprint. --- lib/pavilion/commands/result.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/lib/pavilion/commands/result.py b/lib/pavilion/commands/result.py index 904eba24d..4e646ac56 100644 --- a/lib/pavilion/commands/result.py +++ b/lib/pavilion/commands/result.py @@ -5,7 +5,6 @@ import json import io import pathlib -import shutil from typing import List, IO import pavilion.exceptions @@ -129,15 +128,13 @@ def run(self, pav_cfg, args): color=output.RED, file=args.outfile) return errno.EINVAL - width = shutil.get_terminal_size().columns or 80 - for result in results: result['finish_date'] = output.get_relative_timestamp( result['finished'], fullstamp=True) try: json_string = json.dumps(results, indent=2) - print(json_string, file=self.outfile) + output.fprint(json_string, width=None, file=self.outfile) except OSError: # It's ok if this fails. Generally means we're piping to # another command. From 5b8d1259e0e16bf985dd8e1bba780a30d5a462cb Mon Sep 17 00:00:00 2001 From: Daniel J Magee Date: Wed, 23 Feb 2022 09:31:24 -0700 Subject: [PATCH 10/10] Revert unnecessary changes to output.py. --- lib/pavilion/output.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lib/pavilion/output.py b/lib/pavilion/output.py index a9ee95934..5be32cd0c 100644 --- a/lib/pavilion/output.py +++ b/lib/pavilion/output.py @@ -227,7 +227,7 @@ def json_dump(obj, file, skipkeys=False, ensure_ascii=True, """Dump data to string as per the json dumps function, but using our custom encoder.""" - json_rtn = json.dump(obj, file, cls=PavEncoder, + return json.dump(obj, file, cls=PavEncoder, skipkeys=skipkeys, ensure_ascii=ensure_ascii, check_circular=check_circular, @@ -238,8 +238,6 @@ def json_dump(obj, file, skipkeys=False, ensure_ascii=True, sort_keys=sort_keys, **kw) - return json_rtn - def output_csv(outfile, fields, rows, field_info=None, header=False): """Write the given rows out as a CSV.