Merge pull request #5295 from stephenchengCloud/private/stephenche/CP…

…-45985

Update to python3: hfx_filename, perfmon, static-vdis, xe-scsi-dev-map

scripts/hfx_filename

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 # Copyright (c) 2015 Citrix, Inc.
 #
@@ -14,8 +14,8 @@
 # ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 # OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 
-from __future__ import print_function
-import sys, os, socket, urllib2, urlparse, XenAPI, traceback, xmlrpclib
+
+import sys, socket, urllib.request, XenAPI
 
 db_url = "/remote_db_access"
 
@@ -76,7 +76,6 @@ def read_field(session_id, table, fld, rf):
     return response
 
 if __name__ == "__main__":
-    import XenAPI
     xapi = XenAPI.xapi_local()
     xapi.xenapi.login_with_password('root', '')
     session_id = xapi._session

scripts/perfmon

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 #
 # perfmon - a daemon for monitoring performance of the host on which it is run 
 # and of all the local VMs, and for generating events based on configurable 
@@ -29,13 +29,13 @@
 #
 # The "cf" CGI param specfies the row.  (All rows are returned if it's missing.)
 
-from __future__ import print_function
+
 import sys
 import os
 import getopt
 import traceback
 import XenAPI
-import urllib
+import urllib.request
 from xml import sax            # used to parse rrd_updates because this may be large and sax is more efficient
 from xml.dom import minidom    # used to parse other-config:perfmon.  Efficiency is less important than reliability here
 from xml.parsers.expat import ExpatError
@@ -46,7 +46,7 @@ import syslog
 import socket
 import gc
 import signal
-import commands
+import subprocess
 
 def print_debug(string):
     if debug: 
@@ -118,7 +118,7 @@ class ObjectReport:
     def get_uuid(self):
         return self.uuid
     def get_var_names(self):
-        return self.vars.keys()
+        return list(self.vars.keys())
     def get_value(self, var_name, row):
         try:
             return (self.vars[var_name])[row]
@@ -312,9 +312,7 @@ class RRDUpdates:
         paramstr = "&".join(["%s=%s"  % (k,params[k]) for k in params])
         print_debug("Calling http://localhost/rrd_updates?%s" % paramstr)
 
-        # this is better than urllib.urlopen() as it raises an Exception on http 401 'Unauthorised' error 
-        # rather than drop into interactive mode
-        sock = urllib.URLopener().open("http://localhost/rrd_updates?%s" % paramstr)
+        sock = urllib.request.urlopen("http://localhost/rrd_updates?%s" % paramstr)
         xmlsource = sock.read()
         sock.close()
 
@@ -354,8 +352,8 @@ def average(mylist):
 
 def get_percent_log_fs_usage(ignored):
     "Get the percent usage of the host filesystem for logs partition. Input list is ignored and should be empty"
-    fs_output = commands.getoutput('df /etc/passwd')
-    log_fs_output = commands.getoutput('df /var/log')
+    fs_output = subprocess.getoutput('df /etc/passwd')
+    log_fs_output = subprocess.getoutput('df /var/log')
     fs_output = ' '.join(fs_output.splitlines()[1:])
     log_fs_output = ' '.join(log_fs_output.splitlines()[1:])
     # Get the percent usage only when there is a separate logs partition
@@ -369,7 +367,7 @@ def get_percent_log_fs_usage(ignored):
 def get_percent_fs_usage(ignored):
     "Get the percent usage of the host filesystem.  Input list is ignored and should be empty"
     # this file is on the filesystem of interest in both OEM and Retail
-    output = commands.getoutput('df /etc/passwd')
+    output = subprocess.getoutput('df /etc/passwd')
     output = ' '.join(output.splitlines()[1:]) # remove header line and rewrap on single line
     percentage = output.split()[4]
     # remove % character and convert to float
@@ -639,10 +637,10 @@ class ObjectMonitor:
 
         for var in self.get_active_variables():
             # find the subset of the params returned for this object that we need to consolidate into var
-            params_to_consolidate = filter(var.rrd_regex.match, params_in_obj_report)
+            params_to_consolidate = list(filter(var.rrd_regex.match, params_in_obj_report))
             for row in range(num_rows):
                 # Get the values to consolidate
-                values_to_consolidate = map(lambda param: obj_report.get_value(param, row), params_to_consolidate)
+                values_to_consolidate = [obj_report.get_value(param, row) for param in params_to_consolidate]
                 # Consolidate them
                 value = var.consolidation_fn(values_to_consolidate)
                 # Pass result on to the variable object - this may result in an alarm being generated
@@ -941,7 +939,7 @@ def update_all_xmlconfigs(session):
     for recs in (all_host_recs, all_vm_recs, all_sr_recs):
         all_otherconfigs.update([
                 (recs[ref]['uuid'], recs[ref]['other_config'])
-                for ref in recs.keys()
+                for ref in recs
                 ])
 
     # rebuild dictionary mapping uuids to xmlconfigs
@@ -1069,13 +1067,13 @@ def main():
             vm_uuid_list = rrd_updates.get_uuid_list_by_objtype('vm')
 
             # Remove any monitors for VMs no longer listed in rrd_updates page
-            for uuid in vm_mon_lookup.keys():
+            for uuid in vm_mon_lookup:
                 if uuid not in vm_uuid_list:
                     vm_mon_lookup.pop(uuid)
 
             # Create monitors for VMs that have just appeared in rrd_updates page
             for uuid in vm_uuid_list:
-                if uuid not in vm_mon_lookup.keys():
+                if uuid not in vm_mon_lookup:
                     vm_mon_lookup[uuid] = VMMonitor(uuid)
                 else:
                     # check if the config has changed, e.g. by XenCenter
@@ -1105,12 +1103,12 @@ def main():
             print_debug("sr_uuid_list = %s" % sr_uuid_list)
 
             # Remove monitors for SRs no longer listed in the rrd_updates page
-            for uuid in sr_mon_lookup.keys():
+            for uuid in sr_mon_lookup:
                 if uuid not in sr_uuid_list:
                     sr_mon_lookup.pop(uuid)
             # Create monitors for SRs that have just appeared in rrd_updates page
             for uuid in sr_uuid_list:
-                if uuid not in sr_mon_lookup.keys():
+                if uuid not in sr_mon_lookup:
                     sr_mon_lookup[uuid] = SRMonitor(uuid)
                 else:
                     sr_mon_lookup[uuid].refresh_config()

scripts/static-vdis

@@ -1,12 +1,12 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 # Common functions for managing statically-attached (ie onboot, without xapi) VDIs
 
-from __future__ import print_function
-import sys, os, subprocess, json, urlparse, itertools
+
+import sys, os, subprocess, json, urllib.parse
 import os.path
 import time
-import XenAPI, inventory, xmlrpclib
+import XenAPI, inventory, xmlrpc.client
 
 main_dir = "/etc/xensource/static-vdis"
 
@@ -22,27 +22,21 @@ def call_volume_plugin(name, command, args):
         cmd_args.extend(args)
     # on Python >= 3.3 a timout can be set, not on 2.7
     # when porting please add a timeout
-    output = subprocess.check_output(cmd_args)
+    output = subprocess.check_output(cmd_args, universal_newlines=True)
     return json.loads(output)
 
 def call_datapath_plugin(name, command, args):
     args = [ xapi_storage_script + "/datapath/" + name + "/" + command, "static-vdis" ] + args
-    output = subprocess.check_output(args)
+    output = subprocess.check_output(args, universal_newlines=True)
     return json.loads(output)
-
+        
 def read_whole_file(filename):
-    f = open(filename)
-    try:
-        return reduce(lambda x, y: x + y.decode('utf8'), f.readlines(), "").strip()
-    finally:
-        f.close()
-
+    with open(filename, 'r', encoding='utf-8') as f:
+        return ''.join(f.readlines()).strip()
+
 def write_whole_file(filename, contents):
-    f = open(filename, "w")
-    try:
-        f.write(contents.encode('utf8'))
-    finally:
-        f.close()
+    with open(filename, "w", encoding='utf-8') as f:
+        f.write(contents)
 
 def load(name):
     """Return a dictionary describing a single static VDI"""
@@ -70,7 +64,7 @@ def wait_for_corosync_quorum():
     cmd_args = ['xcli', 'diagnostics', '--json', 'static-vdis']
     quorate = False
     while not quorate:
-        output = subprocess.check_output(cmd_args)
+        output = subprocess.check_output(cmd_args, universal_newlines=True)
         output_map = json.loads(output)
         quorate = output_map.get('is_quorate')
         if not quorate:
@@ -85,17 +79,17 @@ def check_clusterstack(ty):
 def sr_attach(ty, device_config):
     check_clusterstack(ty)
 
-    args = [arg for (k,v) in device_config.iteritems()
+    args = [arg for (k,v) in device_config.items()
             for arg in ["--configuration", k, v]]
     return call_volume_plugin(ty, "SR.attach", args)
 
-def list():
+def list_vdis():
     all = []
     try:
         all = os.listdir(main_dir)
     except:
         pass
-    return map(load, all)
+    return list(map(load, all))
 
 def fresh_name():
     all = []
@@ -131,7 +125,7 @@ def to_string_list(d):
     return l
 
 def add(session, vdi_uuid, reason):
-    for existing in list():
+    for existing in list_vdis():
         if existing['vdi-uuid'] == vdi_uuid:
             if existing['delete-next-boot'] == "True":
                 # Undo the 'delete-next-boot' flag to reinstitute
@@ -159,7 +153,7 @@ def add(session, vdi_uuid, reason):
 
     sm = None
     all_sm = session.xenapi.SM.get_all_records()
-    for sm_ref in all_sm.keys():
+    for sm_ref in all_sm:
         if all_sm[sm_ref]['type'] == ty:
             sm = all_sm[sm_ref]
             break
@@ -195,12 +189,12 @@ def add(session, vdi_uuid, reason):
     fresh = fresh_name()
     path = main_dir + "/" + fresh
     os.mkdir(path)
-    for key in data.keys():
+    for key in data:
         write_whole_file(path + "/" + key, data[key])
 
 def delete(vdi_uuid):
     found = False
-    for existing in list():
+    for existing in list_vdis():
         if existing['vdi-uuid'] == vdi_uuid:
             found = True
             path = main_dir + "/" + existing['id']
@@ -215,7 +209,7 @@ def delete(vdi_uuid):
 # Copied by util.py
 def doexec(args, inputtext=None):
     """Execute a subprocess, then return its return code, stdout and stderr"""
-    proc = subprocess.Popen(args,stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.PIPE,close_fds=True)
+    proc = subprocess.Popen(args,stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.PIPE,close_fds=True, universal_newlines=True)
     (stdout,stderr) = proc.communicate(inputtext)
     rc = proc.returncode
     return (rc,stdout,stderr)
@@ -227,41 +221,42 @@ def connect_smapiv1_nbd(params_nbd):
 
 
 def call_backend_attach(driver, config):
-    args = map(lambda arg: arg.encode('utf8'), [driver, config])
+    args = [arg.encode('utf-8') for arg in [driver, config]]
     xml = doexec(args)
     if xml[0] != 0:
         raise Exception("SM_BACKEND_FAILURE(%d, %s, %s)" % xml)
-    xmlrpc = xmlrpclib.loads(xml[1])
+    xml_rpc = xmlrpc.client.loads(xml[1])
 
-    if 'params_nbd' in xmlrpc[0][0]:
+    if 'params_nbd' in xml_rpc[0][0]:
         # Prefer NBD if available
-        return connect_smapiv1_nbd(xmlrpc[0][0]['params_nbd'])
+        return connect_smapiv1_nbd(xml_rpc[0][0]['params_nbd'])
 
     try:
-    	path = xmlrpc[0][0]['params']
+        path = xml_rpc[0][0]['params']
     except:
-	path = xmlrpc[0][0]
+        path = xml_rpc[0][0]
     return path
 
 def call_backend_detach(driver, config):
-    params = xmlrpclib.loads(config)[0][0]
+    params = xmlrpc.client.loads(config)[0][0]
     params['command'] = 'vdi_detach_from_config'
-    config = xmlrpclib.dumps(tuple([params]), params['command'])
+    config = xmlrpc.client.dumps(tuple([params]), params['command'])
     xml = doexec([ driver, config ])
     if xml[0] != 0:
         raise Exception("SM_BACKEND_FAILURE(%d, %s, %s)" % xml)
-    xmlrpc = xmlrpclib.loads(xml[1])
+    xml_rpc = xmlrpc.client.loads(xml[1])
     try:
-        res = xmlrpc[0][0]['params']
+        res = xml_rpc[0][0]['params']
     except:
-        res = xmlrpc[0][0]
+        res = xml_rpc[0][0]
     return res
 
 def connect_nbd(path, exportname):
     return subprocess.check_output(
         ['/opt/xensource/libexec/nbd_client_manager.py', 'connect',
          '--path', path,
-         '--exportname', exportname]).strip()
+         '--exportname', exportname], 
+        universal_newlines=True).strip()
 
 def disconnect_nbd_device(nbd_device):
     subprocess.check_call(
@@ -283,7 +278,7 @@ def parse_nbd_uri(uri):
 
 def attach(vdi_uuid):
     found = False
-    for existing in list():
+    for existing in list_vdis():
         if existing['vdi-uuid'] == vdi_uuid:
             found = True
             if not('path' in existing):
@@ -305,7 +300,7 @@ def attach(vdi_uuid):
                     vol_key = read_whole_file(d + "/volume-key")
                     vol_uri = read_whole_file(d + "/volume-uri")
                     multipath = json.loads(read_whole_file(d + "/multipath"))
-                    scheme = urlparse.urlparse(vol_uri).scheme
+                    scheme = urllib.parse.urlparse(vol_uri).scheme
                     # Set the multipath flag if required
                     if multipath:
                         with open(MULTIPATH_FLAG, 'a'):
@@ -332,7 +327,7 @@ def attach(vdi_uuid):
 
 def detach(vdi_uuid):
     found = False
-    for existing in list():
+    for existing in list_vdis():
         if existing['vdi-uuid'] == vdi_uuid:
             if not ('disk' in existing):
                 return
@@ -350,7 +345,7 @@ def detach(vdi_uuid):
                 volume_plugin = read_whole_file(d + "/volume-plugin")
                 vol_key = read_whole_file(d + "/volume-key")
                 vol_uri = read_whole_file(d + "/volume-uri")
-                scheme = urlparse.urlparse(vol_uri).scheme
+                scheme = urllib.parse.urlparse(vol_uri).scheme
                 call_datapath_plugin(scheme, "Datapath.deactivate", [ vol_uri, "0" ])
                 call_datapath_plugin(scheme, "Datapath.detach", [ vol_uri, "0" ])
             os.unlink(d + "/disk")
@@ -373,7 +368,7 @@ if  __name__ == "__main__":
         usage()
 
     if sys.argv[1] == "list" and len(sys.argv) == 2:
-        for i in list ():
+        for i in list_vdis():
             for line in to_string_list(i):
                 print(line)
             print()