From 53c2abbd8b7ff92093072f6aae48f40009277aba Mon Sep 17 00:00:00 2001 From: Joe Gordon Date: Fri, 24 May 2019 14:24:28 -0700 Subject: [PATCH 1/2] Start adding python 3 support to pinball Run the safe futurize -1 fixers on all .py files --- pinball/config/utils.py | 6 +++--- pinball/master/blessed_version.py | 2 +- pinball/run_pinball.py | 4 ++-- pinball/tools/base.py | 3 ++- pinball/tools/pinball_util.py | 7 ++++--- pinball/tools/workflow_util.py | 13 +++++++------ pinball/ui/data_builder.py | 10 +++++----- pinball/workflow/log_saver.py | 6 +++--- pinball_ext/common/decorators.py | 9 +++++---- pinball_ext/common/shell_utils.py | 4 ++-- pinball_ext/examples/jobs.py | 7 ++++--- pinball_ext/job/basic_jobs.py | 3 ++- tests/pinball/master/master_handler_test.py | 2 +- tests/pinball/master/transaction_test.py | 6 +++--- tests/pinball/tools/workflow_util_test.py | 12 ++++++------ tests/pinball/ui/data_builder_test.py | 4 ++-- tests/pinball_ext/executor/qubole_executor_test.py | 1 + tests/pinball_ext/job/job_runner_test.py | 1 + tutorial/example_repo/jobs/command_line_jobs.py | 3 ++- tutorial/example_repo/jobs/hive_jobs.py | 5 +++-- tutorial/example_repo/jobs/python_jobs.py | 9 +++++---- 21 files changed, 64 insertions(+), 53 deletions(-) diff --git a/pinball/config/utils.py b/pinball/config/utils.py index 41fd183..09ebe78 100644 --- a/pinball/config/utils.py +++ b/pinball/config/utils.py @@ -124,7 +124,7 @@ def timestamp_to_str(timestamp): """ if not timestamp: return '' - if timestamp == sys.maxint: + if timestamp == sys.maxsize: # sys.maxint represents infinity. return 'inf' utc = pytz.timezone('UTC') @@ -162,7 +162,7 @@ def get_unique_name(): thread_name, threading.current_thread().ident, int(time.time() * 1000), - int(random.random() * sys.maxint)) + int(random.random() * sys.maxsize)) def token_data_to_str(token_data): @@ -175,7 +175,7 @@ def token_data_to_str(token_data): def token_to_str(token): data_str = token_data_to_str(token.data) if token.expirationTime: - if token.expirationTime == sys.maxint: + if token.expirationTime == sys.maxsize: expiration = 'inf (%d)' % token.expirationTime else: expiration = timestamp_to_str(token.expirationTime) diff --git a/pinball/master/blessed_version.py b/pinball/master/blessed_version.py index b0f5a80..c085a91 100644 --- a/pinball/master/blessed_version.py +++ b/pinball/master/blessed_version.py @@ -56,7 +56,7 @@ def __init__(self, name=None, owner=None): now = BlessedVersion._get_timestamp_millis() data_str = ('blessed version created at %s' % timestamp_to_str(now / 1000)) - Token.__init__(self, now, name, owner, sys.maxint, 0, data_str) + Token.__init__(self, now, name, owner, sys.maxsize, 0, data_str) else: Token.__init__(self) diff --git a/pinball/run_pinball.py b/pinball/run_pinball.py index 492583e..aa590c6 100644 --- a/pinball/run_pinball.py +++ b/pinball/run_pinball.py @@ -13,6 +13,7 @@ # limitations under the License. """Command line tool to start token master and workflow workers.""" +from __future__ import print_function import argparse import gc import guppy @@ -114,8 +115,7 @@ def _run_worker(factory, emailer, store=None): LOG.exception('') LOG.warn("worker thread throws due to: %s, retrying ...", str(ex)) else: - print >> sys.stderr,\ - "worker thread throws due to: %s, retrying ..." % str(ex) + print("worker thread throws due to: %s, retrying ..." % str(ex), file=sys.stderr) def _wait_for_threads(threads): diff --git a/pinball/tools/base.py b/pinball/tools/base.py index 0279f98..80366cb 100644 --- a/pinball/tools/base.py +++ b/pinball/tools/base.py @@ -13,6 +13,7 @@ # limitations under the License. """Constructs shared by multiple tools.""" +from __future__ import print_function import abc from pinball.config.utils import PinballException @@ -69,7 +70,7 @@ def confirm(prompt='Confirm'): if not ans: return False if ans not in ['y', 'Y', 'n', 'N']: - print 'please enter y or n.' + print('please enter y or n.') continue if ans == 'y' or ans == 'Y': return True diff --git a/pinball/tools/pinball_util.py b/pinball/tools/pinball_util.py index e8fbb5d..743cc7b 100644 --- a/pinball/tools/pinball_util.py +++ b/pinball/tools/pinball_util.py @@ -19,6 +19,7 @@ TODO(mao): Make sure proper django setting when interact with store. """ +from __future__ import print_function import argparse import sys @@ -146,9 +147,9 @@ def execute(self, client, store): if not tokens: output += 'no tokens found\n' else: - print 'removing:' + print('removing:') for token in tokens: - print '\t%s' % token.name + print('\t%s' % token.name) if self._force or confirm('remove %d tokens' % len(tokens)): request = ModifyRequest(deletes=tokens) client.modify(request) @@ -285,7 +286,7 @@ def main(): command.prepare(options) factory = Factory(master_hostname=options.host, master_port=options.port) client = factory.get_client() - print command.execute(client, None) + print(command.execute(client, None)) if __name__ == '__main__': diff --git a/pinball/tools/workflow_util.py b/pinball/tools/workflow_util.py index 4f07265..85af893 100644 --- a/pinball/tools/workflow_util.py +++ b/pinball/tools/workflow_util.py @@ -13,6 +13,7 @@ # limitations under the License. """Command line tool to monitor and control workflows.""" +from __future__ import print_function import abc import argparse import datetime @@ -179,7 +180,7 @@ def _own_tokens(tokens): timestamp = datetime.datetime.now().strftime("%Y-%m-%d %H:%M") token.owner = 'workflow_util user %s machine %s time %s' % ( getpass.getuser(), socket.gethostname(), timestamp) - token.expirationTime = sys.maxint + token.expirationTime = sys.maxsize def execute(self, client, store): instance_tokens = _get_all_tokens(self._workflow, @@ -1138,12 +1139,12 @@ def execute(self, client, store): if not tokens_to_delete: output += 'no tokens need to be cleaned up\n' else: - print 'removing tokens:' + print('removing tokens:') for token in tokens_to_delete: - print '\t%s' % token.name - print 'removing directories:' + print('\t%s' % token.name) + print('removing directories:') for directory in directories_to_delete: - print '\t%s' % directory + print('\t%s' % directory) message = 'remove %d tokens and %d directories' % ( len(tokens_to_delete), len(directories_to_delete)) if self._force or confirm(message): @@ -1278,7 +1279,7 @@ def main(): if hasattr(PinballConfig, 'MASTER_NAME') and PinballConfig.MASTER_NAME: master_name(PinballConfig.MASTER_NAME) - print run_command(options) + print(run_command(options)) if __name__ == '__main__': diff --git a/pinball/ui/data_builder.py b/pinball/ui/data_builder.py index 791b3a9..5a7476a 100644 --- a/pinball/ui/data_builder.py +++ b/pinball/ui/data_builder.py @@ -198,7 +198,7 @@ def _job_data_from_job_token(job_token, instance_start_time, @staticmethod def _jobs_data_from_job_tokens(job_tokens): - instance_start_time = sys.maxint + instance_start_time = sys.maxsize instance_end_time = 0 now = time.time() for job_token in job_tokens: @@ -209,7 +209,7 @@ def _jobs_data_from_job_tokens(job_tokens): end_time = (job.history[-1].end_time if job.history[-1].end_time else now) instance_end_time = max(instance_end_time, end_time) - if instance_start_time == sys.maxint: + if instance_start_time == sys.maxsize: instance_start_time = now if instance_end_time == 0: instance_end_time = now @@ -297,7 +297,7 @@ def _instance_data_from_job_tokens(self, job_tokens): start_time = first_execution_record.start_time last_execution_record = job.history[-1] if not last_execution_record.end_time: - end_time = sys.maxint + end_time = sys.maxsize else: if last_execution_record.end_time > end_time: end_time = last_execution_record.end_time @@ -339,7 +339,7 @@ def _instance_data_from_job_tokens(self, job_tokens): timestamp = abort_signal.attributes.get(Signal.TIMESTAMP_ATTR) start_time = timestamp end_time = timestamp - elif (end_time == 0 or end_time == sys.maxint or + elif (end_time == 0 or end_time == sys.maxsize or (is_active and not is_scheduled_for_archive)): status = Status.RUNNING end_time = None @@ -411,7 +411,7 @@ def _workflow_data_from_instances_data(instances): elif wf_end_time is not None: # reuse start time if we lost the actual end time this_end_time = instance.start_time \ - if instance.end_time == sys.maxint else instance.end_time + if instance.end_time == sys.maxsize else instance.end_time if this_end_time > wf_end_time: wf_start_time = instance.start_time wf_end_time = this_end_time diff --git a/pinball/workflow/log_saver.py b/pinball/workflow/log_saver.py index 1c1e7c7..6576837 100644 --- a/pinball/workflow/log_saver.py +++ b/pinball/workflow/log_saver.py @@ -156,7 +156,7 @@ def __init__(self, file_path): PinballConfig.LOCAL_LOGS_DIR_PREFIX) self._local_file_log_saver = FileLogSaver(local_file_path) self._last_remote_upload_time = time.time() - self._pending_bytes = 0L + self._pending_bytes = 0 self._s3_key = None def open(self, mode=None): @@ -184,7 +184,7 @@ def close(self): if os.path.exists(self._local_file_log_saver._file_path): try: os.remove(self._local_file_log_saver._file_path) - except OSError, e: + except OSError as e: LOG.warn('deletion failed due to: %s', e) def _check_s3_upload_condition(self): @@ -222,7 +222,7 @@ def write(self, content_str): if self._check_s3_upload_condition(): self._sync_to_s3() self._last_remote_upload_time = time.time() - self._pending_bytes = 0L + self._pending_bytes = 0 def read(self): """Read from a s3 file.""" diff --git a/pinball_ext/common/decorators.py b/pinball_ext/common/decorators.py index f80bd6e..272cc46 100644 --- a/pinball_ext/common/decorators.py +++ b/pinball_ext/common/decorators.py @@ -13,6 +13,7 @@ # limitations under the License. """Various decorator classes.""" +from __future__ import print_function import sys import time @@ -31,7 +32,7 @@ def retry(ExceptionToCheck, tries=4, delay=3, backoff=2, logger=LOG, - sleep_func=time.sleep, max_delay=sys.maxint): + sleep_func=time.sleep, max_delay=sys.maxsize): """Retry calling the decorated function using an exponential backoff. http://www.saltycrane.com/blog/2009/11/trying-out-retry-decorator-python/ @@ -64,7 +65,7 @@ def f_retry(*args, **kwargs): while mtries > 1: try: return f(*args, **kwargs) - except ExceptionToCheck, e: + except ExceptionToCheck as e: if logger: # Don't evaluate the message string as logger module # uses lazy evaluation: assembles the string only if it @@ -72,8 +73,8 @@ def f_retry(*args, **kwargs): logger.warning( "%s, Retrying in %d seconds...", e, mdelay) else: - print "%s, Retrying in %d seconds..." % ( - str(e), mdelay) + print("%s, Retrying in %d seconds..." % ( + str(e), mdelay)) sleep_func(mdelay) mtries -= 1 mdelay *= backoff diff --git a/pinball_ext/common/shell_utils.py b/pinball_ext/common/shell_utils.py index 134594c..e1d8721 100644 --- a/pinball_ext/common/shell_utils.py +++ b/pinball_ext/common/shell_utils.py @@ -204,7 +204,7 @@ def close(self): def next(self): try: - return self.popen_obj.stdout.next() + return next(self.popen_obj.stdout) except StopIteration as e: self.popen_obj.wait() self._verify_success() @@ -212,7 +212,7 @@ def next(self): def __iter__(self): while True: - yield self.next() + yield next(self) def read(self, bytes_to_read=None): if bytes_to_read: diff --git a/pinball_ext/examples/jobs.py b/pinball_ext/examples/jobs.py index 35bc313..6af596c 100644 --- a/pinball_ext/examples/jobs.py +++ b/pinball_ext/examples/jobs.py @@ -13,6 +13,7 @@ # limitations under the License. """Defines example jobs.""" +from __future__ import print_function import datetime import os @@ -28,16 +29,16 @@ class ExamplePythonJob(PythonJob): def _setup(self): - print 'Do some setup in example python job!' + print('Do some setup in example python job!') def _execute(self): - print 'Current time is %s' % str(datetime.datetime.now()) + print('Current time is %s' % str(datetime.datetime.now())) class ExampleCommandJob(CommandLineJob): def _setup(self): - print 'Do not need to add args for this command!' + print('Do not need to add args for this command!') self.arguments = '' def _get_command(self): diff --git a/pinball_ext/job/basic_jobs.py b/pinball_ext/job/basic_jobs.py index 4eb4960..124c5b0 100644 --- a/pinball_ext/job/basic_jobs.py +++ b/pinball_ext/job/basic_jobs.py @@ -1,3 +1,4 @@ +from __future__ import print_function # Copyright 2015, Pinterest, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -127,7 +128,7 @@ def runjob(self, dry_run=False): self._setup() self._set_output_dirs() if dry_run: - print self + print(self) return if not self._skip_execution(): self._execute() diff --git a/tests/pinball/master/master_handler_test.py b/tests/pinball/master/master_handler_test.py index 20fd89e..d2f783c 100644 --- a/tests/pinball/master/master_handler_test.py +++ b/tests/pinball/master/master_handler_test.py @@ -81,7 +81,7 @@ def test_query_and_own(self): query.maxTokens = 10 request = QueryAndOwnRequest() request.owner = 'some_owner' - request.expirationTime = sys.maxint + request.expirationTime = sys.maxsize request.query = query handler = MasterHandler(EphemeralStore()) response = handler.query_and_own(request) diff --git a/tests/pinball/master/transaction_test.py b/tests/pinball/master/transaction_test.py index 08848d6..68e4d04 100644 --- a/tests/pinball/master/transaction_test.py +++ b/tests/pinball/master/transaction_test.py @@ -281,13 +281,13 @@ def test_query_and_own(self): some_token.expirationTime = 10 # in the past some_token = self._trie['/some_dir/some_token_1'] some_token.owner = 'some_owner' - some_token.expirationTime = sys.maxint # in the future + some_token.expirationTime = sys.maxsize # in the future some_query = Query() some_query.namePrefix = '' some_query.maxTokens = 200 request = QueryAndOwnRequest() request.owner = 'some_other_owner' - request.expirationTime = sys.maxint + request.expirationTime = sys.maxsize request.query = some_query transaction = QueryAndOwnTransaction() transaction.prepare(request) @@ -300,4 +300,4 @@ def test_query_and_own(self): self.assertEqual(len(self._trie) - 2, len(response.tokens)) for token in response.tokens: self.assertEquals('some_other_owner', token.owner) - self.assertEquals(sys.maxint, token.expirationTime) + self.assertEquals(sys.maxsize, token.expirationTime) diff --git a/tests/pinball/tools/workflow_util_test.py b/tests/pinball/tools/workflow_util_test.py index dfa5185..ba6df93 100644 --- a/tests/pinball/tools/workflow_util_test.py +++ b/tests/pinball/tools/workflow_util_test.py @@ -228,7 +228,7 @@ def test_pause_workflow(self): self.assertEqual(owned_token.name, original_token.name) self.assertEqual(owned_token.version, original_token.version) self.assertTrue(owned_token.owner.startswith('workflow_util')) - self.assertEqual(sys.maxint, owned_token.expirationTime) + self.assertEqual(sys.maxsize, owned_token.expirationTime) self.assertEqual('claimed 1 token(s) in 1 tries\n', output) @@ -262,7 +262,7 @@ def test_resume_workflow(self): name='/workflow/some_workflow/123/job/waiting/some_job', data='some_data', owner='some_owner', - expirationTime=sys.maxint) + expirationTime=sys.maxsize) query_response = QueryResponse(tokens=[[token]]) client = mock.Mock() client.query.return_value = query_response @@ -879,7 +879,7 @@ def load_path(params): name='/schedule/workflow/some_workflow', data=pickle.dumps(schedule), owner='some_owner', - expirationTime=sys.maxint) + expirationTime=sys.maxsize) new_token = copy.copy(old_token) schedule.recurrence_seconds = 100 new_token.data = pickle.dumps(schedule) @@ -927,7 +927,7 @@ def load_path(params): name='/schedule/workflow/some_workflow', data=pickle.dumps(schedule), owner='some_owner', - expirationTime=sys.maxint) + expirationTime=sys.maxsize) new_token = copy.copy(old_token) schedule.recurrence_seconds = 100 new_token.data = pickle.dumps(schedule) @@ -937,7 +937,7 @@ def load_path(params): name='/schedule/workflow/some_other_workflow', data=pickle.dumps(schedule), owner='some_owner', - expirationTime=sys.maxint) + expirationTime=sys.maxsize) other_new_token = copy.copy(other_old_token) other_new_token.data = pickle.dumps(schedule) @@ -998,7 +998,7 @@ def test_reschedule_workflow(self): name='/schedule/workflow/some_workflow', data='some_data', owner='some_owner', - expirationTime=sys.maxint) + expirationTime=sys.maxsize) query_response = QueryResponse(tokens=[[token]]) client = mock.Mock() client.query.return_value = query_response diff --git a/tests/pinball/ui/data_builder_test.py b/tests/pinball/ui/data_builder_test.py index b21ab79..bcbfe6d 100644 --- a/tests/pinball/ui/data_builder_test.py +++ b/tests/pinball/ui/data_builder_test.py @@ -311,7 +311,7 @@ def test_workflow_data_from_instances_data1(self): # at sys.maxint time def test_workflow_data_from_instances_data2(self): wf_instance_list = [ - WorkflowInstanceData('wf', '22346', Status.ABORTED, 12355, sys.maxint), + WorkflowInstanceData('wf', '22346', Status.ABORTED, 12355, sys.maxsize), WorkflowInstanceData('wf', '22347', Status.SUCCESS, 12365, 12390), WorkflowInstanceData('wf', '22345', Status.RUNNING, 12345, None), ] @@ -346,7 +346,7 @@ def test_workflow_data_from_instances_data4(self): wf_instance_list = [ WorkflowInstanceData('wf', '22346', Status.ABORTED, 12345, 12392), WorkflowInstanceData('wf', '22347', Status.SUCCESS, 12346, 12393), - WorkflowInstanceData('wf', '22345', Status.FAILURE, 12391, sys.maxint), + WorkflowInstanceData('wf', '22345', Status.FAILURE, 12391, sys.maxsize), ] wf_data = DataBuilder._workflow_data_from_instances_data( wf_instance_list) diff --git a/tests/pinball_ext/executor/qubole_executor_test.py b/tests/pinball_ext/executor/qubole_executor_test.py index 77db356..cbb9006 100644 --- a/tests/pinball_ext/executor/qubole_executor_test.py +++ b/tests/pinball_ext/executor/qubole_executor_test.py @@ -1,3 +1,4 @@ +from __future__ import print_function # Copyright 2015, Pinterest, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/tests/pinball_ext/job/job_runner_test.py b/tests/pinball_ext/job/job_runner_test.py index 3db85ab..ae837c0 100644 --- a/tests/pinball_ext/job/job_runner_test.py +++ b/tests/pinball_ext/job/job_runner_test.py @@ -1,3 +1,4 @@ +from __future__ import print_function # Copyright 2015, Pinterest, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/tutorial/example_repo/jobs/command_line_jobs.py b/tutorial/example_repo/jobs/command_line_jobs.py index 3531492..e574374 100644 --- a/tutorial/example_repo/jobs/command_line_jobs.py +++ b/tutorial/example_repo/jobs/command_line_jobs.py @@ -1,3 +1,4 @@ +from __future__ import print_function # Copyright 2015, Pinterest, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -18,7 +19,7 @@ class ExampleCommandJob(CommandLineJob): def _setup(self): - print 'Do not need to add args for this command!' + print('Do not need to add args for this command!') self.arguments = '' def _get_command(self): diff --git a/tutorial/example_repo/jobs/hive_jobs.py b/tutorial/example_repo/jobs/hive_jobs.py index a3f864b..ede215f 100644 --- a/tutorial/example_repo/jobs/hive_jobs.py +++ b/tutorial/example_repo/jobs/hive_jobs.py @@ -1,3 +1,4 @@ +from __future__ import print_function # Copyright 2015, Pinterest, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -30,5 +31,5 @@ class RandomUsersHiveJob(HiveJob): def _complete(self): super(RandomUsersHiveJob, self)._complete() - print "job stdout:\n" - print self._job_output + print("job stdout:\n") + print(self._job_output) diff --git a/tutorial/example_repo/jobs/python_jobs.py b/tutorial/example_repo/jobs/python_jobs.py index c225e79..88bd2a0 100644 --- a/tutorial/example_repo/jobs/python_jobs.py +++ b/tutorial/example_repo/jobs/python_jobs.py @@ -1,3 +1,4 @@ +from __future__ import print_function # Copyright 2015, Pinterest, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -20,17 +21,17 @@ class ExamplePythonJob(PythonJob): def _setup(self): - print 'Do some setup in example python job!' + print('Do some setup in example python job!') def _execute(self): - print 'Current time is %s' % str(datetime.datetime.now()) + print('Current time is %s' % str(datetime.datetime.now())) class ExamplePinballMagicPythonJob(PythonJob): def _setup(self): - print 'Do some setup in example python job!' + print('Do some setup in example python job!') def _execute(self): # a_python_key=a_python_value pair will be passed to # the downstream job as event attribute. - print 'PINBALL:EVENT_ATTR:a_python_key=a_python_value' + print('PINBALL:EVENT_ATTR:a_python_key=a_python_value') From c029d38512574775d104ea4e5e4e7d63a3ad65dc Mon Sep 17 00:00:00 2001 From: Joe Gordon Date: Fri, 24 May 2019 14:27:12 -0700 Subject: [PATCH 2/2] Start adding python 3 support to pinball Summary: Run the safe futurize -1 fixers on all .py files Use mysqlclient instead of MySQL-python: 'mysqlclient is a fork of MySQL-python. It adds Python 3 support and fixed many bugs.' https://pypi.python.org/pypi/mysqlclient https://phabricator.pinadmin.com/D246244 Reviewers: #python3, jon, ahaidrey --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 6f704cd..e1d51c3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,7 +5,7 @@ django-sslify==0.2.7 google_compute_engine==2.8.13 guppy==0.1.10 mock==0.8.0 -MySQL-python==1.2.3 +mysqlclient nose==1.3.4 oauth2client==1.5.2 pycryptodomex==3.7.2