diff --git a/drishti/handlers/handle_darshan.py b/drishti/handlers/handle_darshan.py index faebccb..c1eb2b2 100644 --- a/drishti/handlers/handle_darshan.py +++ b/drishti/handlers/handle_darshan.py @@ -11,22 +11,24 @@ import sys import time import typing +from typing import Dict from dataclasses import dataclass, field from typing import List, Optional import darshan import darshan.backend.cffi_backend as darshanll -import includes.config as config -import includes.module as module import pandas as pd -from includes.module import HIGH, RECOMMENDATIONS, WARN, Panel, insights_total - -# from includes.module import * -from includes.parser import args from packaging import version from rich import print from rich.padding import Padding +import drishti.includes.config as config +import drishti.includes.module as module +from drishti.includes.module import HIGH, RECOMMENDATIONS, WARN, Panel, insights_total + +# from includes.module import * +from drishti.includes.parser import args + def is_available(name): """Check whether `name` is on PATH and marked as executable.""" @@ -98,7 +100,7 @@ class AbstractDarshanTrace(abc.ABC): # Report modules: typing.Iterable[str] - name_records: dict[str, str] = field(default_factory=dict) + name_records: Dict[str, str] = field(default_factory=dict) max_read_offset: int = float('-inf') max_write_offset: int = float('-inf') @@ -108,7 +110,7 @@ class AbstractDarshanTrace(abc.ABC): total_files_posix: int = 0 total_files_mpiio: int = 0 - files: dict[str, dict[str, int]] = None + files: Dict[str, Dict[str, int]] = None total_reads: int = 0 total_writes: int = 0 @@ -165,7 +167,7 @@ class AbstractDarshanTrace(abc.ABC): # TODO: Should be a list of CB nodes for agg cb_nodes: Optional[int] = None number_of_compute_nodes: int = 0 - hints: list[str] = dataclasses.field(default_factory=list) + hints: List[str] = dataclasses.field(default_factory=list) job_start: Optional[datetime.datetime] = None job_end: Optional[datetime.datetime] = None @@ -346,49 +348,49 @@ def _do_something(self): # TODO: for trace in traces for trace in self.traces: pass - module.check_misaligned(self.total_operations, self.total_mem_not_aligned, self.total_file_not_aligned, - self.modules, self.name_records, self.lustre_df, self.dxt_posix, - self.dxt_posix_read_data) # posix alignment + # module.check_misaligned(self.total_operations, self.total_mem_not_aligned, self.total_file_not_aligned, + # self.modules, self.name_records, self.lustre_df, self.dxt_posix, + # self.dxt_posix_read_data) # posix alignment - module.check_traffic(self.max_read_offset, self.total_read_size, self.max_write_offset, self.total_written_size, - self.dxt_posix, self.dxt_posix_read_data, self.dxt_posix_write_data) # redundant reads + # module.check_traffic(self.max_read_offset, self.total_read_size, self.max_write_offset, self.total_written_size, + # self.dxt_posix, self.dxt_posix_read_data, self.dxt_posix_write_data) # redundant reads - module.check_random_operation(self.read_consecutive, self.read_sequential, self.read_random, self.total_reads, - self.write_consecutive, self.write_sequential, self.write_random, - self.total_writes, self.dxt_posix, - self.dxt_posix_read_data, self.dxt_posix_write_data) # random check + # module.check_random_operation(self.read_consecutive, self.read_sequential, self.read_random, self.total_reads, + # self.write_consecutive, self.write_sequential, self.write_random, + # self.total_writes, self.dxt_posix, + # self.dxt_posix_read_data, self.dxt_posix_write_data) # random check - module.check_shared_small_operation(self.total_shared_reads, self.total_shared_reads_small, - self.total_shared_writes, - self.total_shared_writes_small, self.shared_files, self.report.name_records) + # module.check_shared_small_operation(self.total_shared_reads, self.total_shared_reads_small, + # self.total_shared_writes, + # self.total_shared_writes_small, self.shared_files, self.report.name_records) module.check_long_metadata(self.count_long_metadata, self.modules) - module.check_shared_data_imblance(self.posix_shared_data_imbalance_stragglers_count, - self.posix_data_straggler_files, - self.report.name_records, self.dxt_posix, - self.dxt_posix_read_data, - self.dxt_posix_write_data) - - module.check_shared_time_imbalance(self.posix_stragglers_shared_file_time_imbalance_count, - self.posix_shared_time_imbalance_detected_files1, self.report.name_records) - - module.check_individual_write_imbalance(self.posix_data_imbalance_count, - self.posix_shared_time_imbalance_detected_files2, - self.report.name_records, self.dxt_posix, self.dxt_posix_write_data) + # module.check_shared_data_imblance(self.posix_shared_data_imbalance_stragglers_count, + # self.posix_data_straggler_files, + # self.report.name_records, self.dxt_posix, + # self.dxt_posix_read_data, + # self.dxt_posix_write_data) - module.check_mpi_collective_read_operation(self.mpiio_coll_reads, self.mpiio_indep_reads, - self.total_mpiio_read_operations, - self.detected_files_mpi_coll_reads, self.report.name_records, - self.dxt_mpiio) + # module.check_shared_time_imbalance(self.posix_stragglers_shared_file_time_imbalance_count, + # self.posix_shared_time_imbalance_detected_files1, self.report.name_records) - module.check_mpi_collective_write_operation(self.mpiio_coll_writes, self.mpiio_indep_writes, - self.total_mpiio_write_operations, - self.detected_files_mpiio_coll_writes, self.report.name_records, self.dxt_mpiio) + # module.check_individual_write_imbalance(self.posix_data_imbalance_count, + # self.posix_shared_time_imbalance_detected_files2, + # self.report.name_records, self.dxt_posix, self.dxt_posix_write_data) - module.check_individual_read_imbalance(self.imbalance_count_posix_shared_time, - self.posix_shared_time_imbalance_detected_files3, - self.report.name_records, self.dxt_posix, self.dxt_posix_read_data) + # module.check_mpi_collective_read_operation(self.mpiio_coll_reads, self.mpiio_indep_reads, + # self.total_mpiio_read_operations, + # self.detected_files_mpi_coll_reads, self.report.name_records, + # self.dxt_mpiio) + + # module.check_mpi_collective_write_operation(self.mpiio_coll_writes, self.mpiio_indep_writes, + # self.total_mpiio_write_operations, + # self.detected_files_mpiio_coll_writes, self.report.name_records, self.dxt_mpiio) + # + # module.check_individual_read_imbalance(self.imbalance_count_posix_shared_time, + # self.posix_shared_time_imbalance_detected_files3, + # self.report.name_records, self.dxt_posix, self.dxt_posix_read_data) module.check_mpi_none_block_operation(self.mpiio_nb_reads, self.mpiio_nb_writes, self.has_hdf5_extension, self.modules) @@ -396,11 +398,12 @@ def _do_something(self): def _small_operation_insight(self): - module.check_small_operation(self.total_reads, self.total_reads_small, self.total_writes, - self.total_writes_small, - self.small_operation_detected_files, - self.modules, self.report.name_records, self.dxt_posix, self.dxt_posix_read_data, - self.dxt_posix_write_data) + pass + # module.check_small_operation(self.total_reads, self.total_reads_small, self.total_writes, + # self.total_writes_small, + # self.small_operation_detected_files, + # self.modules, self.report.name_records, self.dxt_posix, self.dxt_posix_read_data, + # self.dxt_posix_write_data) @@ -921,7 +924,7 @@ def __init__(self, traces: List[DarshanTrace]): reports = [current_trace.report for current_trace in traces] self.name_records = dict() for report in reports: - self.name_records |= report.name_records + self.name_records.update(report.name_records) # self.name_records |= report.name_records def aggregate_traces(self): self.modules = set() @@ -953,7 +956,7 @@ def aggregate_traces(self): self.total_files_posix += current_trace.total_size_posix self.total_files_mpiio += current_trace.total_files_mpiio - self.files |= current_trace.files + self.files.update(current_trace.files) # self.files |= current_trace.files self.total_reads += current_trace.total_reads self.total_writes += current_trace.total_writes diff --git a/drishti/includes/config.py b/drishti/includes/config.py index 4338467..edad899 100644 --- a/drishti/includes/config.py +++ b/drishti/includes/config.py @@ -3,7 +3,7 @@ import json import os -from includes.parser import * +from drishti.includes.parser import * from rich.console import Console, Group from rich.padding import Padding from rich.panel import Panel diff --git a/drishti/includes/module.py b/drishti/includes/module.py index 15e1ae3..3e7bd94 100644 --- a/drishti/includes/module.py +++ b/drishti/includes/module.py @@ -5,7 +5,7 @@ import time import pandas as pd -from includes.config import * +from drishti.includes.config import * from rich import box from rich.syntax import Syntax diff --git a/drishti/reporter.py b/drishti/reporter.py index 9aa12b5..c07be9a 100644 --- a/drishti/reporter.py +++ b/drishti/reporter.py @@ -3,9 +3,10 @@ import os import sys from subprocess import call +from typing import List, Optional # from includes.parser import * # imports {'parser', 'args', 'argparse'} # TODO: Is next line enuf -from includes.parser import args +from drishti.includes.parser import args ''' |- handler_darshan -| @@ -29,7 +30,7 @@ def clear(): _ = call('clear' if os.name == 'posix' else 'cls') -def check_log_type(paths: list[str]) -> int | None: +def check_log_type(paths: List[str]) -> Optional[int]: is_darshan = True is_recorder = True multiple_logs = len(paths) > 1 @@ -70,10 +71,10 @@ def main(): log_type = check_log_type(args.log_paths) if log_type == LOG_TYPE_DARSHAN: - from handlers.handle_darshan import handler + from drishti.handlers.handle_darshan import handler elif log_type == LOG_TYPE_RECORDER: - from handlers.handle_recorder import handler + from drishti.handlers.handle_recorder import handler handler()