From 81ec3e54faa80d374d25948bb1edb6e0b3fb6aba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Lombra=C3=B1a=20Gonz=C3=A1lez?= Date: Thu, 22 Feb 2018 17:08:39 +0100 Subject: [PATCH 1/3] Closes #52 --- helpers.py | 23 ++++++++++++----------- test/test_helpers.py | 9 +++++++-- 2 files changed, 19 insertions(+), 13 deletions(-) diff --git a/helpers.py b/helpers.py index b9efe1d..39ec949 100644 --- a/helpers.py +++ b/helpers.py @@ -41,6 +41,7 @@ import logging from watchdog.observers import Observer from watchdog.events import PatternMatchingEventHandler +import calendar __all__ = ['find_project_by_short_name', 'check_api_error', @@ -206,8 +207,8 @@ def _add_tasks(config, tasks_file, tasks_type, priority, redundancy): # Check if for the data we have to auto-throttle task creation sleep, msg = enable_auto_throttling(config, data) # If true, warn user - if sleep: # pragma: no cover - click.secho(msg, fg='yellow') + # if sleep: # pragma: no cover + # click.secho(msg, fg='yellow') # Show progress bar with click.progressbar(data, label="Adding Tasks") as pgbar: for d in pgbar: @@ -216,6 +217,8 @@ def _add_tasks(config, tasks_file, tasks_type, priority, redundancy): info=task_info, n_answers=redundancy, priority_0=priority) + + sleep, msg = enable_auto_throttling(config, data) check_api_error(response) # If auto-throttling enabled, sleep for sleep seconds if sleep: # pragma: no cover @@ -428,7 +431,6 @@ def enable_auto_throttling(config, data, limit=299, endpoint='/api/task'): "allowed by the server are requested." # Get header from server endpoint = config.server + endpoint - print requests.head headers = requests.head(endpoint).headers # Get limit server_limit = int(headers.get('X-RateLimit-Remaining', 0)) @@ -436,15 +438,14 @@ def enable_auto_throttling(config, data, limit=299, endpoint='/api/task'): # Get reset time reset_epoch = int(headers.get('X-RateLimit-Reset', 0)) reset_time = datetime.datetime(1970, 1, 1) + \ - datetime.timedelta(reset_epoch) + datetime.timedelta(seconds=reset_epoch) # Compute sleep time - remaining_time = (datetime.datetime.utcnow() - reset_time).seconds - remaining_time = max(remaining_time, 0) or (15 * 60) - sleep = float(remaining_time) / limit - # Check if auto-throttling must be enabled - msg = 'Warning: %s tasks to create.' \ - ' Auto-throttling enabled!' % len(data) - if len(data) > limit: + sleep = (reset_epoch - + calendar.timegm(datetime.datetime.utcnow().utctimetuple())) + msg = 'Warning: %s remaining hits to the endpoint.' \ + ' Auto-throttling enabled!' % limit + # If we have less than 10 hits on the endpoint, sleep + if limit <= 10: return (sleep, msg) else: return 0, None diff --git a/test/test_helpers.py b/test/test_helpers.py index f8d84f5..fcf1feb 100644 --- a/test/test_helpers.py +++ b/test/test_helpers.py @@ -7,6 +7,8 @@ from nose.tools import assert_raises from requests import exceptions from pbsexceptions import * +import calendar +import datetime class TestHelpers(TestDefault): @@ -114,12 +116,15 @@ def test_enable_auto_throttling(self, mock): mock.return_value = MagicMock(['headers']) config = MagicMock(['server']) - mock.return_value.headers = {'X-RateLimit-Remaining': 9} + now = calendar.timegm(datetime.datetime.utcnow().utctimetuple()) + 10 + + mock.return_value.headers = {'X-RateLimit-Remaining': 9, + 'X-RateLimit-Reset': now} sleep, msg = enable_auto_throttling(config, range(10)) assert sleep > 0, "Throttling should be enabled" assert msg is not None, "Throttling should be enabled" - mock.return_value.headers = {'X-RateLimit-Remaining': 10} + mock.return_value.headers = {'X-RateLimit-Remaining': 11} sleep, msg = enable_auto_throttling(config, range(10)) assert sleep == 0, "Throttling should not be enabled" assert msg is None, "Throttling should not be enabled" From f73d85c7a696ee982efcd03e34a9c42ea1271618 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Lombra=C3=B1a=20Gonz=C3=A1lez?= Date: Thu, 22 Feb 2018 17:09:44 +0100 Subject: [PATCH 2/3] Fixes. Increase version. --- helpers.py | 2 -- setup.py | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/helpers.py b/helpers.py index 39ec949..dd435f7 100644 --- a/helpers.py +++ b/helpers.py @@ -437,8 +437,6 @@ def enable_auto_throttling(config, data, limit=299, endpoint='/api/task'): limit = server_limit or limit # Get reset time reset_epoch = int(headers.get('X-RateLimit-Reset', 0)) - reset_time = datetime.datetime(1970, 1, 1) + \ - datetime.timedelta(seconds=reset_epoch) # Compute sleep time sleep = (reset_epoch - calendar.timegm(datetime.datetime.utcnow().utctimetuple())) diff --git a/setup.py b/setup.py index 22fecd0..345d150 100644 --- a/setup.py +++ b/setup.py @@ -15,7 +15,7 @@ setup( name="pybossa-pbs", - version="2.4.5", + version="2.4.6", author="Scifabric LTD", author_email="info@scifabric.com", description="PYBOSSA command line client", From 3b2fe7a376d18bcfbeab2cc0c155a370d7b4aee9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Lombra=C3=B1a=20Gonz=C3=A1lez?= Date: Thu, 22 Feb 2018 17:17:49 +0100 Subject: [PATCH 3/3] Fixes. --- helpers.py | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/helpers.py b/helpers.py index dd435f7..b7615bf 100644 --- a/helpers.py +++ b/helpers.py @@ -204,8 +204,6 @@ def _add_tasks(config, tasks_file, tasks_type, priority, redundancy): if len(data) == 0: return ("Unknown format for the tasks file. Use json, csv, po or " "properties.") - # Check if for the data we have to auto-throttle task creation - sleep, msg = enable_auto_throttling(config, data) # If true, warn user # if sleep: # pragma: no cover # click.secho(msg, fg='yellow') @@ -218,6 +216,7 @@ def _add_tasks(config, tasks_file, tasks_type, priority, redundancy): n_answers=redundancy, priority_0=priority) + # Check if for the data we have to auto-throttle task creation sleep, msg = enable_auto_throttling(config, data) check_api_error(response) # If auto-throttling enabled, sleep for sleep seconds @@ -241,13 +240,6 @@ def _add_helpingmaterials(config, helping_file, helping_type): if len(data) == 0: return ("Unknown format for the tasks file. Use json, csv, po or " "properties.") - # Check if for the data we have to auto-throttle task creation - print enable_auto_throttling - sleep, msg = enable_auto_throttling(config, data, - endpoint='/api/helpinmaterial') - # If true, warn user - if sleep: # pragma: no cover - click.secho(msg, fg='yellow') # Show progress bar with click.progressbar(data, label="Adding Helping Materials") as pgbar: for d in pgbar: @@ -268,6 +260,12 @@ def _add_helpingmaterials(config, helping_file, helping_type): response = config.pbclient.create_helpingmaterial(project_id=project.id, info=helping_info) check_api_error(response) + # Check if for the data we have to auto-throttle task creation + sleep, msg = enable_auto_throttling(config, data, + endpoint='/api/helpinmaterial') + # If true, warn user + if sleep: # pragma: no cover + click.secho(msg, fg='yellow') # If auto-throttling enabled, sleep for sleep seconds if sleep: # pragma: no cover time.sleep(sleep) @@ -327,17 +325,14 @@ def _update_tasks_redundancy(config, task_id, redundancy, limit=300, offset=0): limit = limit offset = offset tasks = config.pbclient.get_tasks(project.id, limit, offset) - # Check if for the data we have to auto-throttle task update - sleep, msg = enable_auto_throttling(config, tasks) - # If true, warn user - if sleep: # pragma: no cover - click.secho(msg, fg='yellow') with click.progressbar(tasks, label="Updating Tasks") as pgbar: while len(tasks) > 0: for t in pgbar: t.n_answers = redundancy response = config.pbclient.update_task(t) check_api_error(response) + # Check if for the data we have to auto-throttle task update + sleep, msg = enable_auto_throttling(config, tasks) # If auto-throttling enabled, sleep for sleep seconds if sleep: # pragma: no cover time.sleep(sleep)