From a4f74686596059d95fd2471be6f656cdeec60c33 Mon Sep 17 00:00:00 2001 From: Joel Tony Date: Fri, 7 Jun 2024 19:32:49 +0530 Subject: [PATCH] feat: Export directory prefix (Darshan) --- drishti/handlers/handle_darshan.py | 15 ++- drishti/includes/module.py | 147 +++++++++++++++++------------ drishti/includes/parser.py | 11 +++ 3 files changed, 102 insertions(+), 71 deletions(-) diff --git a/drishti/handlers/handle_darshan.py b/drishti/handlers/handle_darshan.py index d47fbea..1270d7c 100644 --- a/drishti/handlers/handle_darshan.py +++ b/drishti/handlers/handle_darshan.py @@ -752,15 +752,12 @@ def handler(): display_thresholds(console) display_footer(console, insights_start_time, insights_end_time) - filename = '{}.html'.format(args.log_path) - export_html(console, filename) + input_filename = os.path.basename(args.log_path).replace('.darshan', '') + out_dir = args.export_dir if args.export_dir != "" else os.getcwd() - filename = '{}.svg'.format(args.log_path) - export_svg(console, filename) + print(f"DEBUG: outfile_name: {input_filename}") - filename = '{}-summary.csv'.format( - args.log_path.replace('.darshan', '') - ) - - export_csv(filename, job['job']['jobid']) + export_html(console, out_dir, input_filename) + export_svg(console, out_dir, input_filename) + export_csv(out_dir, input_filename, job['job']['jobid']) diff --git a/drishti/includes/module.py b/drishti/includes/module.py index dedaa09..0e69430 100644 --- a/drishti/includes/module.py +++ b/drishti/includes/module.py @@ -1823,76 +1823,99 @@ def display_footer(console, insights_start_time, insights_end_time): ) ) -def export_html(console, filename): +def export_html(console, export_dir, filename): ''' ''' - if args.export_html: - console.save_html( - filename, - theme=set_export_theme(), - clear=False - ) + if not args.export_html: + print("DEBUG: export_html() - return") + return + os.makedirs(export_dir, exist_ok=True) + filepath = os.path.join(export_dir, f"{filename}.html") -def export_svg(console, filename): - if args.export_svg: - console.save_svg( - filename, - title='Drishti', - theme=set_export_theme(), - clear=False - ) + console.save_html( + filepath, + theme=set_export_theme(), + clear=False + ) + print("DEBUG: END export_html()") -def export_csv(filename, jobid=None): - if args.export_csv: - issues = [ - 'JOB', - INSIGHTS_STDIO_HIGH_USAGE, - INSIGHTS_POSIX_WRITE_COUNT_INTENSIVE, - INSIGHTS_POSIX_READ_COUNT_INTENSIVE, - INSIGHTS_POSIX_WRITE_SIZE_INTENSIVE, - INSIGHTS_POSIX_READ_SIZE_INTENSIVE, - INSIGHTS_POSIX_HIGH_SMALL_READ_REQUESTS_USAGE, - INSIGHTS_POSIX_HIGH_SMALL_WRITE_REQUESTS_USAGE, - INSIGHTS_POSIX_HIGH_MISALIGNED_MEMORY_USAGE, - INSIGHTS_POSIX_HIGH_MISALIGNED_FILE_USAGE, - INSIGHTS_POSIX_REDUNDANT_READ_USAGE, - INSIGHTS_POSIX_REDUNDANT_WRITE_USAGE, - INSIGHTS_POSIX_HIGH_RANDOM_READ_USAGE, - INSIGHTS_POSIX_HIGH_SEQUENTIAL_READ_USAGE, - INSIGHTS_POSIX_HIGH_RANDOM_WRITE_USAGE, - INSIGHTS_POSIX_HIGH_SEQUENTIAL_WRITE_USAGE, - INSIGHTS_POSIX_HIGH_SMALL_READ_REQUESTS_SHARED_FILE_USAGE, - INSIGHTS_POSIX_HIGH_SMALL_WRITE_REQUESTS_SHARED_FILE_USAGE, - INSIGHTS_POSIX_HIGH_METADATA_TIME, - INSIGHTS_POSIX_SIZE_IMBALANCE, - INSIGHTS_POSIX_TIME_IMBALANCE, - INSIGHTS_POSIX_INDIVIDUAL_WRITE_SIZE_IMBALANCE, - INSIGHTS_POSIX_INDIVIDUAL_READ_SIZE_IMBALANCE, - INSIGHTS_MPI_IO_NO_USAGE, - INSIGHTS_MPI_IO_NO_COLLECTIVE_READ_USAGE, - INSIGHTS_MPI_IO_NO_COLLECTIVE_WRITE_USAGE, - INSIGHTS_MPI_IO_COLLECTIVE_READ_USAGE, - INSIGHTS_MPI_IO_COLLECTIVE_WRITE_USAGE, - INSIGHTS_MPI_IO_BLOCKING_READ_USAGE, - INSIGHTS_MPI_IO_BLOCKING_WRITE_USAGE, - INSIGHTS_MPI_IO_AGGREGATORS_INTRA, - INSIGHTS_MPI_IO_AGGREGATORS_INTER, - INSIGHTS_MPI_IO_AGGREGATORS_OK - ] - if codes: - issues.extend(codes) - detected_issues = dict.fromkeys(issues, False) - detected_issues['JOB'] = jobid +def export_svg(console, export_dir, filename): + if not args.export_svg: + return + + os.makedirs(export_dir, exist_ok=True) + filepath = os.path.join(export_dir, f"{filename}.svg") + + console.save_svg( + filepath, + title='Drishti', + theme=set_export_theme(), + clear=False + ) - for report in csv_report: - detected_issues[report] = True - with open(filename, 'w') as f: - w = csv.writer(f) - w.writerow(detected_issues.keys()) - w.writerow(detected_issues.values()) +def export_csv(export_dir, filename, jobid=None): + if not args.export_csv: + return + + issues = [ + 'JOB', + INSIGHTS_STDIO_HIGH_USAGE, + INSIGHTS_POSIX_WRITE_COUNT_INTENSIVE, + INSIGHTS_POSIX_READ_COUNT_INTENSIVE, + INSIGHTS_POSIX_WRITE_SIZE_INTENSIVE, + INSIGHTS_POSIX_READ_SIZE_INTENSIVE, + INSIGHTS_POSIX_HIGH_SMALL_READ_REQUESTS_USAGE, + INSIGHTS_POSIX_HIGH_SMALL_WRITE_REQUESTS_USAGE, + INSIGHTS_POSIX_HIGH_MISALIGNED_MEMORY_USAGE, + INSIGHTS_POSIX_HIGH_MISALIGNED_FILE_USAGE, + INSIGHTS_POSIX_REDUNDANT_READ_USAGE, + INSIGHTS_POSIX_REDUNDANT_WRITE_USAGE, + INSIGHTS_POSIX_HIGH_RANDOM_READ_USAGE, + INSIGHTS_POSIX_HIGH_SEQUENTIAL_READ_USAGE, + INSIGHTS_POSIX_HIGH_RANDOM_WRITE_USAGE, + INSIGHTS_POSIX_HIGH_SEQUENTIAL_WRITE_USAGE, + INSIGHTS_POSIX_HIGH_SMALL_READ_REQUESTS_SHARED_FILE_USAGE, + INSIGHTS_POSIX_HIGH_SMALL_WRITE_REQUESTS_SHARED_FILE_USAGE, + INSIGHTS_POSIX_HIGH_METADATA_TIME, + INSIGHTS_POSIX_SIZE_IMBALANCE, + INSIGHTS_POSIX_TIME_IMBALANCE, + INSIGHTS_POSIX_INDIVIDUAL_WRITE_SIZE_IMBALANCE, + INSIGHTS_POSIX_INDIVIDUAL_READ_SIZE_IMBALANCE, + INSIGHTS_MPI_IO_NO_USAGE, + INSIGHTS_MPI_IO_NO_COLLECTIVE_READ_USAGE, + INSIGHTS_MPI_IO_NO_COLLECTIVE_WRITE_USAGE, + INSIGHTS_MPI_IO_COLLECTIVE_READ_USAGE, + INSIGHTS_MPI_IO_COLLECTIVE_WRITE_USAGE, + INSIGHTS_MPI_IO_BLOCKING_READ_USAGE, + INSIGHTS_MPI_IO_BLOCKING_WRITE_USAGE, + INSIGHTS_MPI_IO_AGGREGATORS_INTRA, + INSIGHTS_MPI_IO_AGGREGATORS_INTER, + INSIGHTS_MPI_IO_AGGREGATORS_OK + ] + if codes: + issues.extend(codes) + + detected_issues = dict.fromkeys(issues, False) + detected_issues['JOB'] = jobid + + for report in csv_report: + detected_issues[report] = True + + # ensure dir exists + os.makedirs(export_dir, exist_ok=True) + filepath = os.path.join(export_dir, f"{filename}.csv") + + print(f"DEBUG: export_dir: {export_dir}") + print(f"DEBUG: filename: {filename}") + print(f"DEBUG: filepath: {filepath}") + + with open(filepath, 'w') as f: + w = csv.writer(f) + w.writerow(detected_issues.keys()) + w.writerow(detected_issues.values()) diff --git a/drishti/includes/parser.py b/drishti/includes/parser.py index 8659520..afa4247 100644 --- a/drishti/includes/parser.py +++ b/drishti/includes/parser.py @@ -96,6 +96,13 @@ help='Export a CSV with the code of all issues that were triggered' ) +parser.add_argument( + '--export_dir', + default="", + dest='export_dir', + help='Specify the directory prefix for the output files (if any)' +) + parser.add_argument( '--json', default=False, @@ -119,3 +126,7 @@ ) args = parser.parse_args() + +print(f"DEBUG: log_path: {args.log_path}") +print(f"DEBUG: export_path: {args.export_dir}") +print(f"DEBUG: export_csv: {args.export_csv}") \ No newline at end of file