Merge pull request jkguiang#14 from jrotter2/debugBE

Stable Python3 Version

Dockerfile

@@ -6,11 +6,19 @@ RUN yum update -y && yum install -y \
       httpd \
       npm \
       php \
-      python2-pip \
-      root-python
+      python3-pip  
+
+
+RUN echo "alias python=python3" >>~/.bashrc
+
+RUN yum update -y && yum install -y \
+      epel-release \
+      root \
+      python3-root
+
 
 COPY requirements.txt /code/requirements.txt
-RUN pip install -r /code/requirements.txt
+RUN pip3 install -r /code/requirements.txt
 
 RUN mkdir /db /run/secrets
 RUN chown -R apache:apache /db /var/www /run/secrets
@@ -25,7 +33,6 @@ ENV ADQM_DB /db/
 ENV ADQM_PUBLIC /var/www/
 ENV ADQM_CONFIG /var/www/public/config/
 ENV ADQM_PLUGINS /var/www/cgi-bin/plugins/
-ENV ADQM_PICKLES /var/www/cgi-bin/pickle_jar/
 
 WORKDIR /webapp
 COPY webapp/package.json /webapp/package.json
@@ -40,7 +47,6 @@ COPY index.py /var/www/cgi-bin/index.py
 COPY autodqm /var/www/cgi-bin/autodqm
 COPY autoref /var/www/cgi-bin/autoref
 COPY plugins /var/www/cgi-bin/plugins
-COPY pickle_jar /var/www/cgi-bin/pickle_jar
 COPY config /var/www/public/config
 
 CMD ["/usr/sbin/httpd","-D","FOREGROUND"]

README.md

@@ -84,6 +84,16 @@ sudo systemctl disable yum-autoupdate.service
 
 The `./run-offline.py` script can retrieve run data files and process them without needing a web server. Run `./run-offline.py --help` for all the options.
 
+To run `./run-offline.py`:
+```sh
+python3 run-offline.py [SUBSYSTEM] [SERIES] [SAMPLE] [DATA_RUN] [REF_RUN] --sslcert [your-public-cert] --sslkey [your-private-key]
+```
+
+As an example command, the following compares runs 325175 and 319993 for Run2018 DoubleMuon data from the EMTF subsystem:
+```sh
+python3 run-offline.py EMTF Run2018 DoubleMuon 325175 319993 --sslcert your-public-cert --sslkey your-private-key
+```
+
 1. Supply certificate files to the environment variables below. Alternatively, the default uses the files produced when running voms-proxy-init, so that may work instead.
 2. Use `./run-offline.py` to process data with AutoDQM
 

autodqm/cfg.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 
 import json

autodqm/compare_hists.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 
 import os

autodqm/dqm.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 
 import base64

autodqm/histpair.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 
 import json

autodqm/plugin_results.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 
 

autoref/rhapi.py

@@ -1,4 +1,5 @@
-import urllib2
+import requests
+import urllib.parse
 import re
 import json
 import sys
@@ -72,10 +73,10 @@ def dprint(self, *args):
         Print debug information
         """
         if self.debug: 
-            print "RhApi:",
+            print("RhApi:", end=' ')
             for arg in args:
-                print arg, 
-            print
+                print(arg, end=' ') 
+            print()
 
     def get(self, parts, data = None, headers = None, params = None, verbose = False, cols = False):
         """
@@ -90,8 +91,8 @@ def get(self, parts, data = None, headers = None, params = None, verbose = False
         # Constructing request path
         #
 
-        callurl = self.url + "/".join(urllib2.quote(str(p)) for p in parts)
-        callurl = callurl + "?" + "&".join(p + "=" + urllib2.quote(str(params[p])) for p in params.keys())
+        callurl = self.url + "/".join(urllib.parse.quote(str(p)) for p in parts)
+        callurl = callurl + "?" + "&".join(p + "=" + urllib.parse.quote(str(params[p])) for p in list(params.keys()))
 
         sdata = None
         if data != None:
@@ -103,30 +104,20 @@ def get(self, parts, data = None, headers = None, params = None, verbose = False
 
         self.dprint(callurl, "with payload", sdata, "and headers", headers)
 
-        req = urllib2.Request(url = callurl, data = data)
-        if headers != None:
-            for h in headers.keys():
-                req.add_header(h, headers[h])
+        resp = None
+        if(data == None):
+            resp = requests.get(callurl, headers=headers)
+        else:
+            resp = requests.post(callurl, headers=headers, data=data)
 
-        resp = urllib2.urlopen(req)
+        if self.debug:
+            self.dprint("Response", resp.status, " ".join(resp.text).split("\r\n"))
 
-        has_getcode = "getcode" in dir(resp)
-        if self.debug: 
-            if has_getcode:
-                self.dprint("Response", resp.getcode(), " ".join(str(resp.info()).split("\r\n")))
-            else:
-                self.dprint("Response", " ".join(str(resp.info()).split("\r\n")))
-
-        if not has_getcode or resp.getcode() == 200:
-            rdata = resp.read()
-            if re.search("json", resp.info().gettype()):
-                try:
-                    return json.loads(rdata)
-                except TypeError, e:
-                    self.dprint(e)
-                    return rdata
-            else:
-                return rdata
+        if resp.status_code == 200:
+            try:
+                return resp.json()
+            except:
+                return resp.text
 
     def info(self, verbose = False):
         """
@@ -138,15 +129,15 @@ def folders(self, verbose = False):
         """
         Get list of folders
         """
-        return self.get(["tables"], verbose = verbose).keys()
+        return list(self.get(["tables"], verbose = verbose).keys())
 
     def tables(self, folder, verbose = False):
         """
         Get tables for folder or all
         """
         raw = self.get(["tables"], verbose = verbose)
         d = []
-        for t in raw[folder].keys(): 
+        for t in list(raw[folder].keys()): 
             d.append(t)
         return d
 
@@ -394,12 +385,12 @@ def run(self):
 
                 if options.count:
 
-                    print api.count(api.qid(arg), params = params, verbose = options.verbose)
+                    print(api.count(api.qid(arg), params = params, verbose = options.verbose))
 
                 elif options.metadata:
 
                     qid = api.qid(arg)
-                    print self.pprint(api.query(qid, verbose = options.verbose))
+                    print(self.pprint(api.query(qid, verbose = options.verbose)))
 
                 else:
 
@@ -414,78 +405,77 @@ def run(self):
 
                         if options.format == 'csv':
                             try:
-                                print api.csv(arg, params = params, pagesize = options.size, page = options.page, verbose = options.verbose)
-                            except RhApiRowLimitError, e:
+                                print(api.csv(arg, params = params, pagesize = options.size, page = options.page, verbose = options.verbose))
+                            except RhApiRowLimitError as e:
                                 if options.all:
                                     page = 0
                                     while (page * e.rowsLimit) < e.count:
                                         page = page + 1
                                         res = api.csv(arg, params = params, pagesize = e.rowsLimit, page = page, verbose = options.verbose)
                                         if page == 1:
-                                            print res,
+                                            print(res, end=' ')
                                         else:
-                                            print '\n'.join(res.split('\n')[1:]),
+                                            print('\n'.join(res.split('\n')[1:]), end=' ')
                                 else:
                                     raise e
 
                         if options.format == 'xml':
                             try:
-                                print api.xml(arg, params = params, pagesize = options.size, page = options.page, verbose = options.verbose)
-                            except RhApiRowLimitError, e:
+                                print(api.xml(arg, params = params, pagesize = options.size, page = options.page, verbose = options.verbose))
+                            except RhApiRowLimitError as e:
                                 if options.all:
                                     page = 0
-                                    print '<?xml version="1.0" encoding="UTF-8" standalone="no"?><data>', 
+                                    print('<?xml version="1.0" encoding="UTF-8" standalone="no"?><data>', end=' ') 
                                     while (page * e.rowsLimit) < e.count:
                                         page = page + 1
                                         res = api.xml(arg, params = params, pagesize = e.rowsLimit, page = page, verbose = options.verbose)
                                         root = minidom.parseString(res).documentElement
                                         for row in root.getElementsByTagName('row'):
-                                            print row.toxml(),
-                                    print '</data>'
+                                            print(row.toxml(), end=' ')
+                                    print('</data>')
                                 else:
                                     raise e
 
                         if options.format in ['json','json2']:
                             try:
                                 if options.format == 'json':
-                                    print api.json(arg, params = params, pagesize = options.size, page = options.page, verbose = options.verbose, cols = options.cols)
+                                    print(api.json(arg, params = params, pagesize = options.size, page = options.page, verbose = options.verbose, cols = options.cols))
                                     #print_json = api.json(arg, params=params, pagesize=options.size, page=options.page, verbose=options.verbose, cols=options.cols)
                                     #print (json.dumps(print_json, sort_keys=True, indent=4, separators=(',', ': ')))
                                 else:
-                                    print api.json2(arg, params = params, pagesize = options.size, page = options.page, verbose = options.verbose, cols = options.cols)
+                                    print(api.json2(arg, params = params, pagesize = options.size, page = options.page, verbose = options.verbose, cols = options.cols))
                                     #print_json = api.json(arg, params=params, pagesize=options.size, page=options.page, verbose=options.verbose, cols=options.cols)
                                     #print (json.dumps(print_json, sort_keys=True, indent=4, separators=(',', ': ')))
-                            except RhApiRowLimitError, e:
+                            except RhApiRowLimitError as e:
                                 if options.all:
                                     page = 0
-                                    print '{"data": [', 
+                                    print('{"data": [', end=' ') 
                                     while (page * e.rowsLimit) < e.count:
                                         page = page + 1
                                         res = api.json(arg, params = params, pagesize = e.rowsLimit, page = page, verbose = options.verbose)
                                         comma = ','
                                         if page == 1: comma = ''
                                         for d in res['data']:
-                                            print comma, d,
+                                            print(comma, d, end=' ')
                                             comma = ','
-                                    print "]}"
+                                    print("]}")
                                 else:
                                     raise e
 
                 return 0
 
             self.parser.error('Command %s not understood' % arg)
 
-        except RhApiRowLimitError, e:
-
-            print "ERROR: %s\nDetails: %s, consider --all option" % (type(e).__name__, e)
+        except RhApiRowLimitError as e:
 
-        except urllib2.HTTPError, e:
-	    reason = e.reason if hasattr(e, 'reason') else '%d %s' % (e.code, e.msg)
-	    print "ERROR: %s\nDetails: %s" % (reason, e.read())
-
-        except Exception, e:
+            print("ERROR: %s\nDetails: %s, consider --all option" % (type(e).__name__, e))
+
+        except urllib.exception.HTTPError as e:
+            reason = e.reason if hasattr(e, 'reason') else '%d %s' % (e.code, e.msg)
+            print("ERROR: %s\nDetails: %s" % (reason, e.read()))
 
-            print "ERROR: %s\nDetails: %s" % (type(e).__name__, e)
+        except Exception as e:
+            print("ERROR: %s\nDetails: %s" % (type(e).__name__, e))
 
 if __name__ == '__main__':
 

autoref/sql.py

@@ -1,9 +1,9 @@
 # Imports
 # Run Registry API
-from rhapi import DEFAULT_URL, RhApi
+from .rhapi import DEFAULT_URL, RhApi
 
 # Script for getting reference run qualities
-import ref
+from . import ref
 
 def fetch_refs(config, data_run, ref_runs):
     # Handle non-configured subsystems
@@ -55,7 +55,7 @@ def retrieve(max_run=320008, min_run=316766, folder="runreg_csc", table="dataset
     data = {}
     if type(ref_runs) == dict:
         dqm = ref_runs
-        ref_runs = ref_runs.keys()
+        ref_runs = list(ref_runs.keys())
     skipped = 0
     it = 0
     while True:

config/CSC.json

@@ -22,13 +22,9 @@
     },
     {
       "always_show": true,
-      "comparators": ["pca", "ks_test"],
-      "jar_dir": "CSC",
       "path": "recHits/hRHnrechits"
     },
     {
-      "comparators": ["pca", "ks_test"],
-      "jar_dir": "CSC",
       "path": "Digis/hWireTBin_*"
     },
     {
@@ -44,19 +40,13 @@
     },
     {
       "always_show": true,
-      "comparators": ["pca", "ks_test"],
-      "jar_dir": "CSC",
       "path": "Segments/hSnSegments"
     },
     {
-      "comparators": ["pca", "ks_test"],
-      "jar_dir": "CSC",
       "always_show": true,
       "path": "Segments/hSnhits"
     },
     {
-      "comparators": ["pca", "ks_test"],
-      "jar_dir": "CSC",
       "path": "Segments/hSTimeCombined*"
     },
     {
@@ -69,13 +59,9 @@
       "path": "Segments/hSChiSq*"
     },
     {
-      "comparators": ["pca", "ks_test"],
-      "jar_dir": "CSC",
       "path": "Segments/hSGlobalPhi"
     },
     {
-      "comparators": ["pca", "ks_test"],
-      "jar_dir": "CSC",
       "path": "Segments/hSGlobalTheta"
     }
   ]

index.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 
 import autodqm.cfg
@@ -160,12 +160,12 @@ class ServerError(error):
     cgi_req = cgi.FieldStorage()
 
     req = {}
-    for k in cgi_req.keys():
+    for k in list(cgi_req.keys()):
         req[str(k)] = str(cgi_req[k].value)
 
     res = handle_request(req)
 
     print("Content-type: application/json")
     print("Access-Control-Allow-Origin: *")
     print("")
-    print(json.dumps(res))
+    print((json.dumps(res)))