process-job-reports

#!/usr/bin/env python3

# Process job reports and create summary file with performance information.
from argparse import ArgumentParser
from os.path import exists, join
from xml.sax import parseString, ContentHandler
import re
import datetime, time
from cmsutils import getRelValsFiles, readRelValFile

# Schema of the output
class JobReportHandler(ContentHandler):
  def __init__(self, what):
    self.counters = dict((k, 0) for k in what)

  def startElement(self, name, attrs):
    if name != "Metric":
      return
    if not attrs["Name"] in self.counters:
      return
    if "nan" in attrs["Value"]:
      return
    self.counters[attrs["Name"]] = float(attrs["Value"])

if __name__ == "__main__":
  parser = ArgumentParser()
  parser.add_argument("input", nargs="+", help="input file")
  args = parser.parse_args()
  # We start with empty workflows
  results = {}

  printHeader = True
  for inFile in args.input:
    inFiles, zipFile = getRelValsFiles(inFile, "*/JobReport*.xml", ".*/JobReport([0-9]).xml")
    table = {}
    for l in inFiles:
      m = re.match("(.+/|)([^/]+)/JobReport([0-9]).xml", l)
      if not m: continue
      xpath, workflow, step = m.groups()
      handler = JobReportHandler(what=["PeakValueRss", "PeakValueVsize", "TotalJobCPU","TotalJobTime"])
      try:
        parseString(readRelValFile(l,zipFile), handler)
      except:
        continue
      for counter, value in handler.counters.items():
        workflowId = workflow.split("_",1)[0]
        if not workflowId in table:
          table[workflowId] = {}
        table[workflowId][counter + str(step)] = value
      
    if printHeader:
      header = sorted(set([k for (w, r) in table.items() for k in r ]))
      print ("Timestamp", end=" ")
      print ("WorkflowId", end=" ")
      for h in header:
        print (h,end=" ")
      print ("")
      printHeader = False

    inDate = re.match(".*CMSSW_.*_(20\d\d-\d\d-\d\d-\d\d\d\d)/.*", inFile).group(1)
    timestamp = time.mktime(datetime.datetime.strptime(inDate, "%Y-%m-%d-%H%M").timetuple())
    for k in sorted(table):
      v = table[k] 
      print (timestamp,end=" ")
      print (k,end=" ")
      for h in header:
        if h in v:
          # We do not care about showing less than 1MB changes.
          # Also we round to 1/100s the ellapsed time.
          if h.startswith("PeakValue"):
            print (int(v[h]/10),end=" ")
          else:
            print (int(v[h]*10),end=" ")
        else:
          print ("NA",end=" ")
      print ("")