Start adding python 3 support to pinball

pinball/config/utils.py

@@ -124,7 +124,7 @@ def timestamp_to_str(timestamp):
     """
     if not timestamp:
         return ''
-    if timestamp == sys.maxint:
+    if timestamp == sys.maxsize:
         # sys.maxint represents infinity.
         return 'inf'
     utc = pytz.timezone('UTC')
@@ -162,7 +162,7 @@ def get_unique_name():
                                      thread_name,
                                      threading.current_thread().ident,
                                      int(time.time() * 1000),
-                                     int(random.random() * sys.maxint))
+                                     int(random.random() * sys.maxsize))
 
 
 def token_data_to_str(token_data):
@@ -175,7 +175,7 @@ def token_data_to_str(token_data):
 def token_to_str(token):
     data_str = token_data_to_str(token.data)
     if token.expirationTime:
-        if token.expirationTime == sys.maxint:
+        if token.expirationTime == sys.maxsize:
             expiration = 'inf (%d)' % token.expirationTime
         else:
             expiration = timestamp_to_str(token.expirationTime)

pinball/master/blessed_version.py

@@ -56,7 +56,7 @@ def __init__(self, name=None, owner=None):
             now = BlessedVersion._get_timestamp_millis()
             data_str = ('blessed version created at %s' %
                         timestamp_to_str(now / 1000))
-            Token.__init__(self, now, name, owner, sys.maxint, 0, data_str)
+            Token.__init__(self, now, name, owner, sys.maxsize, 0, data_str)
         else:
             Token.__init__(self)
 

pinball/run_pinball.py

@@ -13,6 +13,7 @@
 # limitations under the License.
 
 """Command line tool to start token master and workflow workers."""
+from __future__ import print_function
 import argparse
 import gc
 import guppy
@@ -114,8 +115,7 @@ def _run_worker(factory, emailer, store=None):
                 LOG.exception('')
                 LOG.warn("worker thread throws due to: %s, retrying ...", str(ex))
             else:
-                print >> sys.stderr,\
-                    "worker thread throws due to: %s, retrying ..." % str(ex)
+                print("worker thread throws due to: %s, retrying ..." % str(ex), file=sys.stderr)
 
 
 def _wait_for_threads(threads):

pinball/tools/base.py

@@ -13,6 +13,7 @@
 # limitations under the License.
 
 """Constructs shared by multiple tools."""
+from __future__ import print_function
 import abc
 
 from pinball.config.utils import PinballException
@@ -69,7 +70,7 @@ def confirm(prompt='Confirm'):
         if not ans:
             return False
         if ans not in ['y', 'Y', 'n', 'N']:
-            print 'please enter y or n.'
+            print('please enter y or n.')
             continue
         if ans == 'y' or ans == 'Y':
             return True

pinball/tools/pinball_util.py

@@ -19,6 +19,7 @@
 
 TODO(mao): Make sure proper django setting when interact with store.
 """
+from __future__ import print_function
 import argparse
 import sys
 
@@ -146,9 +147,9 @@ def execute(self, client, store):
         if not tokens:
             output += 'no tokens found\n'
         else:
-            print 'removing:'
+            print('removing:')
             for token in tokens:
-                print '\t%s' % token.name
+                print('\t%s' % token.name)
             if self._force or confirm('remove %d tokens' % len(tokens)):
                 request = ModifyRequest(deletes=tokens)
                 client.modify(request)
@@ -285,7 +286,7 @@ def main():
     command.prepare(options)
     factory = Factory(master_hostname=options.host, master_port=options.port)
     client = factory.get_client()
-    print command.execute(client, None)
+    print(command.execute(client, None))
 
 
 if __name__ == '__main__':

pinball/tools/workflow_util.py

@@ -13,6 +13,7 @@
 # limitations under the License.
 
 """Command line tool to monitor and control workflows."""
+from __future__ import print_function
 import abc
 import argparse
 import datetime
@@ -179,7 +180,7 @@ def _own_tokens(tokens):
             timestamp = datetime.datetime.now().strftime("%Y-%m-%d %H:%M")
             token.owner = 'workflow_util user %s machine %s time %s' % (
                 getpass.getuser(), socket.gethostname(), timestamp)
-            token.expirationTime = sys.maxint
+            token.expirationTime = sys.maxsize
 
     def execute(self, client, store):
         instance_tokens = _get_all_tokens(self._workflow,
@@ -1138,12 +1139,12 @@ def execute(self, client, store):
         if not tokens_to_delete:
             output += 'no tokens need to be cleaned up\n'
         else:
-            print 'removing tokens:'
+            print('removing tokens:')
             for token in tokens_to_delete:
-                print '\t%s' % token.name
-            print 'removing directories:'
+                print('\t%s' % token.name)
+            print('removing directories:')
             for directory in directories_to_delete:
-                print '\t%s' % directory
+                print('\t%s' % directory)
             message = 'remove %d tokens and %d directories' % (
                 len(tokens_to_delete), len(directories_to_delete))
             if self._force or confirm(message):
@@ -1278,7 +1279,7 @@ def main():
     if hasattr(PinballConfig, 'MASTER_NAME') and PinballConfig.MASTER_NAME:
         master_name(PinballConfig.MASTER_NAME)
 
-    print run_command(options)
+    print(run_command(options))
 
 
 if __name__ == '__main__':

pinball/ui/data_builder.py

@@ -198,7 +198,7 @@ def _job_data_from_job_token(job_token, instance_start_time,
 
     @staticmethod
     def _jobs_data_from_job_tokens(job_tokens):
-        instance_start_time = sys.maxint
+        instance_start_time = sys.maxsize
         instance_end_time = 0
         now = time.time()
         for job_token in job_tokens:
@@ -209,7 +209,7 @@ def _jobs_data_from_job_tokens(job_tokens):
                 end_time = (job.history[-1].end_time
                             if job.history[-1].end_time else now)
                 instance_end_time = max(instance_end_time, end_time)
-        if instance_start_time == sys.maxint:
+        if instance_start_time == sys.maxsize:
             instance_start_time = now
         if instance_end_time == 0:
             instance_end_time = now
@@ -297,7 +297,7 @@ def _instance_data_from_job_tokens(self, job_tokens):
                     start_time = first_execution_record.start_time
                 last_execution_record = job.history[-1]
                 if not last_execution_record.end_time:
-                    end_time = sys.maxint
+                    end_time = sys.maxsize
                 else:
                     if last_execution_record.end_time > end_time:
                         end_time = last_execution_record.end_time
@@ -339,7 +339,7 @@ def _instance_data_from_job_tokens(self, job_tokens):
                 timestamp = abort_signal.attributes.get(Signal.TIMESTAMP_ATTR)
                 start_time = timestamp
                 end_time = timestamp
-        elif (end_time == 0 or end_time == sys.maxint or
+        elif (end_time == 0 or end_time == sys.maxsize or
               (is_active and not is_scheduled_for_archive)):
             status = Status.RUNNING
             end_time = None
@@ -411,7 +411,7 @@ def _workflow_data_from_instances_data(instances):
             elif wf_end_time is not None:
                 # reuse start time if we lost the actual end time
                 this_end_time = instance.start_time \
-                    if instance.end_time == sys.maxint else instance.end_time
+                    if instance.end_time == sys.maxsize else instance.end_time
                 if this_end_time > wf_end_time:
                     wf_start_time = instance.start_time
                     wf_end_time = this_end_time

pinball/workflow/log_saver.py

@@ -156,7 +156,7 @@ def __init__(self, file_path):
             PinballConfig.LOCAL_LOGS_DIR_PREFIX)
         self._local_file_log_saver = FileLogSaver(local_file_path)
         self._last_remote_upload_time = time.time()
-        self._pending_bytes = 0L
+        self._pending_bytes = 0
         self._s3_key = None
 
     def open(self, mode=None):
@@ -184,7 +184,7 @@ def close(self):
         if os.path.exists(self._local_file_log_saver._file_path):
             try:
                 os.remove(self._local_file_log_saver._file_path)
-            except OSError, e:
+            except OSError as e:
                 LOG.warn('deletion failed due to: %s', e)
 
     def _check_s3_upload_condition(self):
@@ -222,7 +222,7 @@ def write(self, content_str):
         if self._check_s3_upload_condition():
             self._sync_to_s3()
             self._last_remote_upload_time = time.time()
-            self._pending_bytes = 0L
+            self._pending_bytes = 0
 
     def read(self):
         """Read from a s3 file."""

pinball_ext/common/decorators.py

@@ -13,6 +13,7 @@
 # limitations under the License.
 
 """Various decorator classes."""
+from __future__ import print_function
 
 import sys
 import time
@@ -31,7 +32,7 @@
 
 
 def retry(ExceptionToCheck, tries=4, delay=3, backoff=2, logger=LOG,
-          sleep_func=time.sleep, max_delay=sys.maxint):
+          sleep_func=time.sleep, max_delay=sys.maxsize):
     """Retry calling the decorated function using an exponential backoff.
 
     http://www.saltycrane.com/blog/2009/11/trying-out-retry-decorator-python/
@@ -64,16 +65,16 @@ def f_retry(*args, **kwargs):
             while mtries > 1:
                 try:
                     return f(*args, **kwargs)
-                except ExceptionToCheck, e:
+                except ExceptionToCheck as e:
                     if logger:
                         # Don't evaluate the message string as logger module
                         # uses lazy evaluation: assembles the string only if it
                         # is going to be logged at the current logging level.
                         logger.warning(
                             "%s, Retrying in %d seconds...", e, mdelay)
                     else:
-                        print "%s, Retrying in %d seconds..." % (
-                            str(e), mdelay)
+                        print("%s, Retrying in %d seconds..." % (
+                            str(e), mdelay))
                     sleep_func(mdelay)
                     mtries -= 1
                     mdelay *= backoff

pinball_ext/common/shell_utils.py

@@ -204,15 +204,15 @@ def close(self):
 
     def next(self):
         try:
-            return self.popen_obj.stdout.next()
+            return next(self.popen_obj.stdout)
         except StopIteration as e:
             self.popen_obj.wait()
             self._verify_success()
             raise e
 
     def __iter__(self):
         while True:
-            yield self.next()
+            yield next(self)
 
     def read(self, bytes_to_read=None):
         if bytes_to_read:

pinball_ext/examples/jobs.py

@@ -13,6 +13,7 @@
 # limitations under the License.
 
 """Defines example jobs."""
+from __future__ import print_function
 
 import datetime
 import os
@@ -28,16 +29,16 @@
 class ExamplePythonJob(PythonJob):
 
     def _setup(self):
-        print 'Do some setup in example python job!'
+        print('Do some setup in example python job!')
 
     def _execute(self):
-        print 'Current time is %s' % str(datetime.datetime.now())
+        print('Current time is %s' % str(datetime.datetime.now()))
 
 
 class ExampleCommandJob(CommandLineJob):
 
     def _setup(self):
-        print 'Do not need to add args for this command!'
+        print('Do not need to add args for this command!')
         self.arguments = ''
 
     def _get_command(self):

pinball_ext/job/basic_jobs.py

@@ -1,3 +1,4 @@
+from __future__ import print_function
 # Copyright 2015, Pinterest, Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -127,7 +128,7 @@ def runjob(self, dry_run=False):
             self._setup()
             self._set_output_dirs()
             if dry_run:
-                print self
+                print(self)
                 return
             if not self._skip_execution():
                 self._execute()

requirements.txt

@@ -5,7 +5,7 @@ django-sslify==0.2.7
 google_compute_engine==2.8.13
 guppy==0.1.10
 mock==0.8.0
-MySQL-python==1.2.3
+mysqlclient
 nose==1.3.4
 oauth2client==1.5.2
 pycryptodomex==3.7.2

tests/pinball/master/master_handler_test.py

@@ -81,7 +81,7 @@ def test_query_and_own(self):
         query.maxTokens = 10
         request = QueryAndOwnRequest()
         request.owner = 'some_owner'
-        request.expirationTime = sys.maxint
+        request.expirationTime = sys.maxsize
         request.query = query
         handler = MasterHandler(EphemeralStore())
         response = handler.query_and_own(request)

tests/pinball/master/transaction_test.py

@@ -281,13 +281,13 @@ def test_query_and_own(self):
         some_token.expirationTime = 10  # in the past
         some_token = self._trie['/some_dir/some_token_1']
         some_token.owner = 'some_owner'
-        some_token.expirationTime = sys.maxint  # in the future
+        some_token.expirationTime = sys.maxsize  # in the future
         some_query = Query()
         some_query.namePrefix = ''
         some_query.maxTokens = 200
         request = QueryAndOwnRequest()
         request.owner = 'some_other_owner'
-        request.expirationTime = sys.maxint
+        request.expirationTime = sys.maxsize
         request.query = some_query
         transaction = QueryAndOwnTransaction()
         transaction.prepare(request)
@@ -300,4 +300,4 @@ def test_query_and_own(self):
         self.assertEqual(len(self._trie) - 2, len(response.tokens))
         for token in response.tokens:
             self.assertEquals('some_other_owner', token.owner)
-            self.assertEquals(sys.maxint, token.expirationTime)
+            self.assertEquals(sys.maxsize, token.expirationTime)