diff --git a/guider/guider.py b/guider/guider.py index 19e0cdf4..489d6652 100755 --- a/guider/guider.py +++ b/guider/guider.py @@ -7,7 +7,7 @@ __credits__ = "Peace Lee" __license__ = "GPLv2" __version__ = "3.9.8" -__revision__ = "240905" +__revision__ = "240907" __maintainer__ = "Peace Lee" __email__ = "iipeace5@gmail.com" __repository__ = "https://github.com/iipeace/guider" @@ -6256,6 +6256,35 @@ def removeColor(string): SysMgr.ansiObj = re.compile(ansi) return SysMgr.ansiObj.sub("", string) + @staticmethod + def calcLists(calc, *lists): + # check length # + lenList = [len(l) for l in lists] + if max(lenList) != min(lenList): + SysMgr.printErr( + "failed to calculate lists because of length difference %s" + % lenList + ) + return [] + + resList = [] + for l in lists: + if not resList: + resList = l + continue + + for i, k in enumerate(l): + if calc == "add": + resList[i] += k + elif calc == "sub": + resList[i] -= k + elif calc == "mul": + resList[i] *= k + elif calc == "div": + resList[i] *= k + + return resList + @staticmethod def convFloat2Str(val): val = ("%f" % val).rstrip("0") @@ -27423,7 +27452,7 @@ class SysMgr(object): # list # signalCmd = "trap 'kill $$' INT\n" saveCmd = None - boundaryLine = None + boundaryLine = [] nrTop = 0 pipeForPager = None printFd = None @@ -37328,6 +37357,7 @@ def getCmdList(): "drawcpu": ("CPU", "Linux/MacOS/Windows"), "drawcpuavg": ("CPU", "Linux/MacOS/Windows"), "drawdelay": ("Delay", "Linux/MacOS/Windows"), + "drawdiff": ("Diff", "Linux/MacOS/Windows"), "drawflame": ("Function", "Linux/MacOS/Windows"), "drawhist": ("Histogram", "Linux/MacOS/Windows"), "drawio": ("I/O", "Linux/MacOS/Windows"), @@ -38357,6 +38387,9 @@ def printHelp(force=False, isExit=True): - {2:1} of total resource usage with multiple files for comparison # {0:1} {1:1} "guider*.out" worstcase.out -a -g TOTAL + - {2:1} of total resource usage with multiple files for comparison with thin plots + # {0:1} {1:1} "guider*.out" worstcase.out -a -g TOTAL, NOBOLD + - Draw fixed-size items for timeline segments # {0:1} {1:1} timeline.json -q DURATION:500 @@ -42201,6 +42234,12 @@ def _getDesc(s, t=0): helpStr += drawSubStr + drawExamStr + # Diff draw # + elif SysMgr.checkMode("drawcpu"): + helpStr = _getDesc("Draw diff graphs", t=5) + + helpStr += drawSubStr + drawExamStr + # CPU Delay draw # elif SysMgr.checkMode("drawdelay"): helpStr = _getDesc("Draw CPU delay graphs", t=5) @@ -42239,6 +42278,23 @@ def _getDesc(s, t=0): helpStr += drawSubStr + drawExamStr + myStr + # draw diff # + elif SysMgr.checkMode("drawdiff"): + helpStr = _getDesc( + "Draw diff graphs between report files", t=5 + ) + + myStr = """ + - {2:1} + # {0:1} {1:1} "guider*.out" + """.format( + cmd, + mode, + "Draw diff graphs between report files", + ) + + helpStr += drawSubStr + drawExamStr + myStr + # flamegraph draw # elif SysMgr.checkMode("drawflame"): helpStr = _getDesc("Draw flame graph", t=5) @@ -117587,16 +117643,27 @@ def __init__(self, fpath=None, onlyInstance=False): else: drawFunc = self.drawStats + # check diff option # + isDrawDiff = SysMgr.checkMode("drawdiff", True) + if isDrawDiff: + SysMgr.boundaryLine.append(0) + # define resDict # resDict = {} # draw # if "NOMERGE" in SysMgr.environList: for fpath in list(SysMgr.inputParam): - ret = drawFunc(fpath) + if isDrawDiff: + ret = drawFunc(fpath, diff=True) + else: + ret = drawFunc(fpath) resDict[fpath] = ret else: - ret = drawFunc(SysMgr.inputParam) + if isDrawDiff: + ret = drawFunc(SysMgr.inputParam, diff=True) + else: + ret = drawFunc(SysMgr.inputParam) resDict[",".join(SysMgr.inputParam)] = ret # print output results # @@ -120537,6 +120604,7 @@ def drawYticks( ymin=-1, margin=True, yunit="", + negYmin=True, ): # pylint: disable=undefined-variable @@ -120605,17 +120673,18 @@ def drawYticks( ylim([0, ymaxval]) # check ymin # - if ymin < 0: - ylist = ax.get_yticks().tolist() - ymin = long(min(ylist)) - else: - yminval = ymin - int(ymin / unit) - ax.set_ylim(bottom=yminval) + if not negYmin: + if ymin < 0: + ylist = ax.get_yticks().tolist() + ymin = long(min(ylist)) + else: + yminval = ymin - int(ymin / unit) + ax.set_ylim(bottom=yminval) - # adjust ymin # - if ymin < 0: - ymin = 0 - ax.set_ylim(bottom=0) + # adjust ymin # + if ymin < 0: + ymin = 0 + ax.set_ylim(bottom=0) # adjust yticks # if adjust: @@ -120887,6 +120956,7 @@ def drawStats( onlyGraph=False, onlyChart=False, applyOpt=True, + diff=False, ): def _printMemUsage(signum=None, frame=None): pid = SysMgr.pid @@ -120926,6 +120996,10 @@ def _printMemUsage(signum=None, frame=None): # get stats from a single file # if len(targets) == 1: + if diff: + SysMgr.printErr("failed to diff because only one file checked") + return + logFile = targets[0] # parse stats # @@ -120942,12 +121016,14 @@ def _printMemUsage(signum=None, frame=None): graphStats = {} chartStats = {} timeList = {} + prevStats = {} + prevFname = "" # get only start flag # concatenated = "CONCATENATE" in SysMgr.environList # parse stats from multiple files # - for lfile in targets: + for fidx, lfile in enumerate(targets): try: gstats, _ = TaskAnalyzer.getStatsFile( lfile, onlyStart=concatenated, applyOpt=applyOpt @@ -120959,6 +121035,189 @@ def _printMemUsage(signum=None, frame=None): except: continue + # calculate diff # + if diff: + # skip first file # + if not prevStats: + prevStats = gstats + prevFname = lfile + continue + + # set stat list # + statLen = min( + len(prevStats.get("timeline", [])), + len(gstats.get("timeline", [])), + ) + + # set name # + groupName = "DiffGroup%s" % fidx + diffName = "%s-%s" % (lfile, prevFname) + groupInfo = "{0:20} {1:<100}\n\n".format( + groupName, diffName + ) + SysMgr.sysinfoBuffer += groupInfo + + # define new stat dict # + nstats = {"graphTitle": "Guider Diff Graph"} + + # define merge function # + def _mergeTasks(atype): + # register new stat # + nstats.setdefault(n, {}) + + # define file merge function # + def _getMergedStats(stats): + merged = {} + + for comm, attr in stats[n].items(): + # get target values # + if atype: + vals = attr.get(atype) + if not vals: + continue + else: + vals = attr + + # merge values # + comm = comm.lstrip("*").rsplit("(", 1)[0] + usage = list(map(long, vals.split())) + if comm in merged: + merged[comm] = UtilMgr.calcLists( + "add", merged[comm], usage[:statLen] + ) + else: + merged[comm] = usage[:statLen] + + return merged + + # merge stats for common tasks # + prevStatsMerged = _getMergedStats(prevStats) + statsMerged = _getMergedStats(gstats) + + # set max name # + if atype == "rssUsage": + maxName = "maxRss" + defMax = -(SysMgr.maxSize) + elif atype == "vssUsage": + maxName = "maxVss" + defMax = -(SysMgr.maxSize) + else: + maxName = "maximum" + defMax = 1 + + # calculate diffs between stats # + for comm, vals in statsMerged.items(): + if comm in prevStatsMerged: + diffList = UtilMgr.calcLists( + "sub", vals, prevStatsMerged[comm] + ) + statStr = " ".join(list(map(str, diffList))) + sums = sum(diffList) + maxval = max(defMax, max(diffList)) + else: + statStr = " ".join(list(map(str, vals))) + sums = sum(vals) + maxval = max(defMax, max(vals)) + + # save max # + nstats[n].setdefault( + comm, {"average": sums, "minimum": 1} + ) + nstats[n][comm][maxName] = maxval + + # save diff # + if atype: + nstats[n][comm][atype] = statStr + else: + nstats[n][comm] = statStr + + # calculate diffs from previous stats # + for comm in set(prevStatsMerged.keys()) - set( + statsMerged.keys() + ): + # save diff # + diffList = UtilMgr.calcLists( + "sub", + [0] * len(prevStatsMerged[comm]), + prevStatsMerged[comm], + ) + + # save max # + statStr = " ".join(list(map(str, diffList))) + nstats[n].setdefault( + comm, {"average": sum(diffList), "minimum": 1} + ) + nstats[n][comm][maxName] = max( + defMax, max(diffList) + ) + + # save diff # + if atype: + nstats[n][comm][atype] = statStr + else: + nstats[n][comm] = statStr + + # traverse stats # + for n in gstats.keys(): + # TODO: dict format stats # + + if n in ( + "cpuUsage", + "memFree", + "memAnon", + "memCache", + "memKernel", + "blkWait", + "blkRead", + "blkWrite", + "netRead", + "netWrite", + "swapUsage", + "reclaimBg", + "reclaimDr", + "nrCore", + "eventList", + ): + diffList = UtilMgr.calcLists( + "add" if n == "eventList" else "sub", + gstats[n][:statLen], + prevStats[n][:statLen], + ) + nstats[n] = diffList + elif n in ("timeline"): + nstats[n] = gstats[n][:statLen] + elif n in ("totalRam", "totalSwap"): + nstats[n] = gstats[n] + elif n in ("cpuProcUsage", "cpuProcDelay"): + _mergeTasks("usage") + elif n in ("cpuProcPrio"): + pass + elif n in ("gpuUsage"): + _mergeTasks(None) + elif n in ("memProcUsage"): + _mergeTasks("vssUsage") + _mergeTasks("rssUsage") + elif n in ("blkProcUsage"): + pass + elif n in ("networkUsage"): + pass + elif n in ("storageUsage"): + pass + else: + pass + + # merge stats # + for key, val in nstats.items(): + if key == "graphTitle": + graphStats[key] = val + else: + graphStats["%s:%s" % (groupName, key)] = val + + # update prev context # + prevStats = gstats + prevFname = lfile + continue + # merge stats for concatenation temporally # if concatenated: try: @@ -121170,6 +121429,7 @@ def _printMemUsage(signum=None, frame=None): outFile=outFile, outFd=outFd, logEvents=logEvents, + diffOpt=diff, ) resDict["graph"] = ret except SystemExit: @@ -121864,7 +122124,13 @@ def drawBoundary(self, gtype, labelList): continue def drawGraph( - self, graphStats, logFile, outFile=None, outFd=None, logEvents=[] + self, + graphStats, + logFile, + outFile=None, + outFd=None, + logEvents=[], + diffOpt=False, ): # pylint: disable=undefined-variable @@ -122546,10 +122812,13 @@ def _drawCpu(graphStats, xtype, pos, size, delay=False): lent = len(timeline) # get stacked plot option # - stacked = "STACKEDPLOT" in SysMgr.environList + stacked = "STACKEDPLOT" in SysMgr.environList and not diffOpt stackedStats = [] stackedTexts = [] + # get bold effect flag # + bold = not "NOBOLD" in SysMgr.environList + conv = UtilMgr.convNum # start loop # @@ -122567,11 +122836,11 @@ def _drawCpu(graphStats, xtype, pos, size, delay=False): prefix = "" cpuUsage = graphStats["%scpuUsage" % fname][:lent] - cpuProcUsage = graphStats["%scpuProcUsage" % fname] - cpuProcDelay = graphStats["%scpuProcDelay" % fname] + cpuProcUsage = graphStats.get("%scpuProcUsage" % fname, {}) + cpuProcDelay = graphStats.get("%scpuProcDelay" % fname, {}) blkWait = graphStats["%sblkWait" % fname][:lent] - blkProcUsage = graphStats["%sblkProcUsage" % fname] - gpuUsage = graphStats["%sgpuUsage" % fname] + blkProcUsage = graphStats.get("%sblkProcUsage" % fname, {}) + gpuUsage = graphStats.get("%sgpuUsage" % fname, {}) nrCore = graphStats["%snrCore" % fname] if nrCore: maxCore = max(nrCore) @@ -122629,8 +122898,8 @@ def _drawCpu(graphStats, xtype, pos, size, delay=False): stat, c=gcolor, linestyle="-", - linewidth=0.3, - marker=".", + linewidth=0.3 if bold else 0.1, + marker="." if bold else None, markersize=0.3, path_effects=_getPathEffect(), solid_capstyle="round", @@ -122698,6 +122967,13 @@ def _drawCpu(graphStats, xtype, pos, size, delay=False): (blkWait, "pink", "CPU(%s)+IOWAIT" % conv(maxCore)), (cpuUsage, "red", "CPU(%s)" % conv(maxCore)), ): + # skip useless plot # + if ( + targetUsage == blkWait + and max(blkWait) == min(blkWait) == 0 + ): + continue + # accumulate stats # origTargetUsage = targetUsage[:] if "ACCUMULATE" in SysMgr.environList: @@ -122716,8 +122992,8 @@ def _drawCpu(graphStats, xtype, pos, size, delay=False): targetUsage, c=color, linestyle="-", - linewidth=1, - marker=".", + linewidth=1 if bold else 0.1, + marker="." if bold else None, markersize=1, path_effects=_getPathEffect(), solid_capstyle="round", @@ -122809,8 +123085,8 @@ def _drawCpu(graphStats, xtype, pos, size, delay=False): "-", c="green", linestyle="-.", - linewidth=1, - marker=".", + linewidth=1 if bold else 0.1, + marker="." if bold else None, markersize=1, path_effects=_getPathEffect(), solid_capstyle="round", @@ -123072,13 +123348,14 @@ def _drawCpu(graphStats, xtype, pos, size, delay=False): # set ymin # ymin = long(min(ylist)) - if ymin < 0: - ymin = 0 - elif ymin == 0: - try: - ax.set_ylim(bottom=0) - except: - pass + if not diffOpt: + if ymin < 0: + ymin = 0 + elif ymin == 0: + try: + ax.set_ylim(bottom=0) + except: + pass # update yticks # if stacked: @@ -123091,7 +123368,13 @@ def _drawCpu(graphStats, xtype, pos, size, delay=False): if ymax < 100: ymax = 100 margin = True - TaskAnalyzer.drawYticks(ax, ymax, margin=margin) + TaskAnalyzer.drawYticks( + ax, + ymax, + ymin=ymin if diffOpt else -1, + margin=margin, + negYmin=diffOpt, + ) # add % unit to each value # try: @@ -123146,7 +123429,7 @@ def _drawPrio(graphStats, xtype, pos, size): lent = len(timeline) # get stacked plot option # - stacked = "STACKEDPLOT" in SysMgr.environList + stacked = "STACKEDPLOT" in SysMgr.environList and not diffOpt stackedStats = [] stackedTexts = [] @@ -123333,13 +123616,14 @@ def _drawPrio(graphStats, xtype, pos, size): # set ymin # ymin = long(min(ylist)) - if ymin < 0: - ymin = 0 - elif ymin == 0: - try: - ax.set_ylim(bottom=0) - except: - pass + if not diffOpt: + if ymin < 0: + ymin = 0 + elif ymin == 0: + try: + ax.set_ylim(bottom=0) + except: + pass # update yticks # if stacked: @@ -123352,7 +123636,14 @@ def _drawPrio(graphStats, xtype, pos, size): else: ymax = 140 margin = True - TaskAnalyzer.drawYticks(ax, ymax, unit=14, margin=margin) + TaskAnalyzer.drawYticks( + ax, + ymax, + unit=14, + ymin=ymin if diffOpt else -1, + margin=margin, + negYmin=diffOpt, + ) # draw name # TaskAnalyzer.drawName(ax, "Prio") @@ -123532,7 +123823,7 @@ def __drawSystemIo( convSize = UtilMgr.convSize2Unit # get stacked plot option # - stacked = "STACKEDPLOT" in SysMgr.environList + stacked = "STACKEDPLOT" in SysMgr.environList and not diffOpt stackedStats = [] stackedTexts = [] @@ -123586,13 +123877,13 @@ def __drawSystemIo( blkRead = graphStats["%sblkRead" % fname][:lent] blkWrite = graphStats["%sblkWrite" % fname][:lent] - blkProcUsage = graphStats["%sblkProcUsage" % fname] + blkProcUsage = graphStats.get("%sblkProcUsage" % fname, {}) netRead = graphStats["%snetRead" % fname][:lent] netWrite = graphStats["%snetWrite" % fname][:lent] reclaimBg = graphStats["%sreclaimBg" % fname][:lent] reclaimDr = graphStats["%sreclaimDr" % fname][:lent] - storageUsage = graphStats["%sstorageUsage" % fname] - networkUsage = graphStats["%snetworkUsage" % fname] + storageUsage = graphStats.get("%sstorageUsage" % fname, {}) + networkUsage = graphStats.get("%snetworkUsage" % fname, {}) if isVisibleTotal: for color, stats, name, lname in ( @@ -123941,7 +124232,7 @@ def __drawSystemIo( margin = False else: margin = True - TaskAnalyzer.drawYticks(ax, ymax, margin=margin) + TaskAnalyzer.drawYticks(ax, ymax, margin=margin, negYmin=diffOpt) # update xticks # xticks(fontsize=self.xfsize) @@ -124111,7 +124402,7 @@ def __drawSystemMem(statList, color, ymax, name): tcnt = 0 # get stacked plot option # - stacked = "STACKEDPLOT" in SysMgr.environList + stacked = "STACKEDPLOT" in SysMgr.environList and not diffOpt stackedStats = [] stackedTexts = [] @@ -124129,6 +124420,7 @@ def __drawSystemMem(statList, color, ymax, name): fname = "" prefix = "" + # get stats # totalRam = graphStats["%stotalRam" % fname] totalSwap = graphStats["%stotalSwap" % fname] @@ -124157,7 +124449,7 @@ def __drawSystemMem(statList, color, ymax, name): if "NOMEMPROCPLOT" in SysMgr.environList: memProc = {} else: - memProc = graphStats["%smemProcUsage" % fname] + memProc = graphStats.get("%smemProcUsage" % fname, {}) if "NOMEMSWAPPLOT" in SysMgr.environList: swapUsage = [0] @@ -124284,7 +124576,7 @@ def __drawMemPlots(ymax, stacked=False): if SysMgr.vssEnable: # get max vss condition # maxVssCond = UtilMgr.getEnvironNum( - "MAXVSSCOND", False, 0, False, True + "MAXVSSCOND", False, -(SysMgr.maxSize), False, True ) for key, item in sorted( @@ -124527,7 +124819,7 @@ def __drawMemPlots(ymax, stacked=False): elif SysMgr.rssEnable or SysMgr.pssEnable or SysMgr.ussEnable: # get max rss condition # maxRssCond = UtilMgr.getEnvironNum( - "MAXRSSCOND", False, 0, False, True + "MAXRSSCOND", False, -(SysMgr.maxSize), False, True ) if stacked: @@ -124681,7 +124973,7 @@ def __drawMemPlots(ymax, stacked=False): else: isSysplot = True if not "MEMSYSPLOT" in SysMgr.environList: - ymax = __drawMemPlots(ymax, stacked=True) + ymax = __drawMemPlots(ymax, stacked=not diffOpt) """ ylabel('MEMORY', fontsize=5) @@ -124702,13 +124994,14 @@ def __drawMemPlots(ymax, stacked=False): # set ymin # ymin = long(min(ylist)) - if ymin < 0: - ymin = 0 - elif ymin == 0: - try: - ax.set_ylim(bottom=0) - except: - pass + if not diffOpt: + if ymin < 0: + ymin = 0 + elif ymin == 0: + try: + ax.set_ylim(bottom=0) + except: + pass # update yticks # if stacked: @@ -124722,7 +125015,13 @@ def __drawMemPlots(ymax, stacked=False): ymax = 100 margin = True - TaskAnalyzer.drawYticks(ax, ymax, margin=margin) + TaskAnalyzer.drawYticks( + ax, + ymax, + ymin=ymin if diffOpt else -1, + margin=margin, + negYmin=diffOpt, + ) try: # ax.get_xaxis().set_visible(False)