Skip to content

Commit

Permalink
fix: CI passing
Browse files Browse the repository at this point in the history 
- Make module paths w.r.t `/` and not `drishti/`
- Ensure Python 3.8 compatibility
  • Loading branch information
@jay-tau
jay-tau committed Oct 27, 2024
1 parent ee02934 commit 13684ac
Show file tree
Hide file tree
Showing 4 changed files with 60 additions and 56 deletions.
103 changes: 53 additions & 50 deletions drishti/handlers/handle_darshan.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,22 +11,24 @@
import sys
import time
import typing
from typing import Dict
from dataclasses import dataclass, field
from typing import List, Optional

import darshan
import darshan.backend.cffi_backend as darshanll
import includes.config as config
import includes.module as module
import pandas as pd
from includes.module import HIGH, RECOMMENDATIONS, WARN, Panel, insights_total

# from includes.module import *
from includes.parser import args
from packaging import version
from rich import print
from rich.padding import Padding

import drishti.includes.config as config
import drishti.includes.module as module
from drishti.includes.module import HIGH, RECOMMENDATIONS, WARN, Panel, insights_total

# from includes.module import *
from drishti.includes.parser import args


def is_available(name):
"""Check whether `name` is on PATH and marked as executable."""
Expand Down Expand Up @@ -98,7 +100,7 @@ class AbstractDarshanTrace(abc.ABC):

# Report
modules: typing.Iterable[str]
name_records: dict[str, str] = field(default_factory=dict)
name_records: Dict[str, str] = field(default_factory=dict)

max_read_offset: int = float('-inf')
max_write_offset: int = float('-inf')
Expand All @@ -108,7 +110,7 @@ class AbstractDarshanTrace(abc.ABC):
total_files_posix: int = 0
total_files_mpiio: int = 0

files: dict[str, dict[str, int]] = None
files: Dict[str, Dict[str, int]] = None

total_reads: int = 0
total_writes: int = 0
Expand Down Expand Up @@ -165,7 +167,7 @@ class AbstractDarshanTrace(abc.ABC):
# TODO: Should be a list of CB nodes for agg
cb_nodes: Optional[int] = None
number_of_compute_nodes: int = 0
hints: list[str] = dataclasses.field(default_factory=list)
hints: List[str] = dataclasses.field(default_factory=list)

job_start: Optional[datetime.datetime] = None
job_end: Optional[datetime.datetime] = None
Expand Down Expand Up @@ -346,61 +348,62 @@ def _do_something(self):
# TODO: for trace in traces
for trace in self.traces:
pass
module.check_misaligned(self.total_operations, self.total_mem_not_aligned, self.total_file_not_aligned,
self.modules, self.name_records, self.lustre_df, self.dxt_posix,
self.dxt_posix_read_data) # posix alignment
# module.check_misaligned(self.total_operations, self.total_mem_not_aligned, self.total_file_not_aligned,
# self.modules, self.name_records, self.lustre_df, self.dxt_posix,
# self.dxt_posix_read_data) # posix alignment

module.check_traffic(self.max_read_offset, self.total_read_size, self.max_write_offset, self.total_written_size,
self.dxt_posix, self.dxt_posix_read_data, self.dxt_posix_write_data) # redundant reads
# module.check_traffic(self.max_read_offset, self.total_read_size, self.max_write_offset, self.total_written_size,
# self.dxt_posix, self.dxt_posix_read_data, self.dxt_posix_write_data) # redundant reads

module.check_random_operation(self.read_consecutive, self.read_sequential, self.read_random, self.total_reads,
self.write_consecutive, self.write_sequential, self.write_random,
self.total_writes, self.dxt_posix,
self.dxt_posix_read_data, self.dxt_posix_write_data) # random check
# module.check_random_operation(self.read_consecutive, self.read_sequential, self.read_random, self.total_reads,
# self.write_consecutive, self.write_sequential, self.write_random,
# self.total_writes, self.dxt_posix,
# self.dxt_posix_read_data, self.dxt_posix_write_data) # random check

module.check_shared_small_operation(self.total_shared_reads, self.total_shared_reads_small,
self.total_shared_writes,
self.total_shared_writes_small, self.shared_files, self.report.name_records)
# module.check_shared_small_operation(self.total_shared_reads, self.total_shared_reads_small,
# self.total_shared_writes,
# self.total_shared_writes_small, self.shared_files, self.report.name_records)

module.check_long_metadata(self.count_long_metadata, self.modules)

module.check_shared_data_imblance(self.posix_shared_data_imbalance_stragglers_count,
self.posix_data_straggler_files,
self.report.name_records, self.dxt_posix,
self.dxt_posix_read_data,
self.dxt_posix_write_data)

module.check_shared_time_imbalance(self.posix_stragglers_shared_file_time_imbalance_count,
self.posix_shared_time_imbalance_detected_files1, self.report.name_records)

module.check_individual_write_imbalance(self.posix_data_imbalance_count,
self.posix_shared_time_imbalance_detected_files2,
self.report.name_records, self.dxt_posix, self.dxt_posix_write_data)
# module.check_shared_data_imblance(self.posix_shared_data_imbalance_stragglers_count,
# self.posix_data_straggler_files,
# self.report.name_records, self.dxt_posix,
# self.dxt_posix_read_data,
# self.dxt_posix_write_data)

module.check_mpi_collective_read_operation(self.mpiio_coll_reads, self.mpiio_indep_reads,
self.total_mpiio_read_operations,
self.detected_files_mpi_coll_reads, self.report.name_records,
self.dxt_mpiio)
# module.check_shared_time_imbalance(self.posix_stragglers_shared_file_time_imbalance_count,
# self.posix_shared_time_imbalance_detected_files1, self.report.name_records)

module.check_mpi_collective_write_operation(self.mpiio_coll_writes, self.mpiio_indep_writes,
self.total_mpiio_write_operations,
self.detected_files_mpiio_coll_writes, self.report.name_records, self.dxt_mpiio)
# module.check_individual_write_imbalance(self.posix_data_imbalance_count,
# self.posix_shared_time_imbalance_detected_files2,
# self.report.name_records, self.dxt_posix, self.dxt_posix_write_data)

module.check_individual_read_imbalance(self.imbalance_count_posix_shared_time,
self.posix_shared_time_imbalance_detected_files3,
self.report.name_records, self.dxt_posix, self.dxt_posix_read_data)
# module.check_mpi_collective_read_operation(self.mpiio_coll_reads, self.mpiio_indep_reads,
# self.total_mpiio_read_operations,
# self.detected_files_mpi_coll_reads, self.report.name_records,
# self.dxt_mpiio)

# module.check_mpi_collective_write_operation(self.mpiio_coll_writes, self.mpiio_indep_writes,
# self.total_mpiio_write_operations,
# self.detected_files_mpiio_coll_writes, self.report.name_records, self.dxt_mpiio)
#
# module.check_individual_read_imbalance(self.imbalance_count_posix_shared_time,
# self.posix_shared_time_imbalance_detected_files3,
# self.report.name_records, self.dxt_posix, self.dxt_posix_read_data)

module.check_mpi_none_block_operation(self.mpiio_nb_reads, self.mpiio_nb_writes, self.has_hdf5_extension,
self.modules)



def _small_operation_insight(self):
module.check_small_operation(self.total_reads, self.total_reads_small, self.total_writes,
self.total_writes_small,
self.small_operation_detected_files,
self.modules, self.report.name_records, self.dxt_posix, self.dxt_posix_read_data,
self.dxt_posix_write_data)
pass
# module.check_small_operation(self.total_reads, self.total_reads_small, self.total_writes,
# self.total_writes_small,
# self.small_operation_detected_files,
# self.modules, self.report.name_records, self.dxt_posix, self.dxt_posix_read_data,
# self.dxt_posix_write_data)



Expand Down Expand Up @@ -921,7 +924,7 @@ def __init__(self, traces: List[DarshanTrace]):
reports = [current_trace.report for current_trace in traces]
self.name_records = dict()
for report in reports:
self.name_records |= report.name_records
self.name_records.update(report.name_records) # self.name_records |= report.name_records

def aggregate_traces(self):
self.modules = set()
Expand Down Expand Up @@ -953,7 +956,7 @@ def aggregate_traces(self):
self.total_files_posix += current_trace.total_size_posix
self.total_files_mpiio += current_trace.total_files_mpiio

self.files |= current_trace.files
self.files.update(current_trace.files) # self.files |= current_trace.files

self.total_reads += current_trace.total_reads
self.total_writes += current_trace.total_writes
Expand Down
2 changes: 1 addition & 1 deletion drishti/includes/config.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
import json
import os

from includes.parser import *
from drishti.includes.parser import *
from rich.console import Console, Group
from rich.padding import Padding
from rich.panel import Panel
Expand Down
2 changes: 1 addition & 1 deletion drishti/includes/module.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
import time

import pandas as pd
from includes.config import *
from drishti.includes.config import *
from rich import box
from rich.syntax import Syntax

Expand Down
9 changes: 5 additions & 4 deletions drishti/reporter.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,10 @@
import os
import sys
from subprocess import call
from typing import List, Optional

# from includes.parser import * # imports {'parser', 'args', 'argparse'} # TODO: Is next line enuf
from includes.parser import args
from drishti.includes.parser import args

'''
|- handler_darshan -|
Expand All @@ -29,7 +30,7 @@ def clear():
_ = call('clear' if os.name == 'posix' else 'cls')


def check_log_type(paths: list[str]) -> int | None:
def check_log_type(paths: List[str]) -> Optional[int]:
is_darshan = True
is_recorder = True
multiple_logs = len(paths) > 1
Expand Down Expand Up @@ -70,10 +71,10 @@ def main():
log_type = check_log_type(args.log_paths)

if log_type == LOG_TYPE_DARSHAN:
from handlers.handle_darshan import handler
from drishti.handlers.handle_darshan import handler

elif log_type == LOG_TYPE_RECORDER:
from handlers.handle_recorder import handler
from drishti.handlers.handle_recorder import handler

handler()

Expand Down

0 comments on commit 13684ac

Please sign in to comment.