diff --git a/README.md b/README.md index 293add8..df95124 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ progress bar, delete them and update the project templates Requirements ============ -[PYBOSSA server](http://pybossa.com) >= 1.2.0. +[PYBOSSA server](http://pybossa.com) >= 2.3.7. Installation ============ @@ -33,7 +33,7 @@ If you want to hack on the code, just install it but with the **--editable** flag after cloning the repository: ``` - git clone https://github.com/PyBossa/pbs.git + git clone https://github.com/Scifabric/pbs.git cd pbs virtualenv env source env/bin/activate @@ -268,6 +268,41 @@ options, please check the **--help** command: pbs delete_tasks --help ``` +## Adding helping materials to a project + +Adding helping materials is very simple. You can have your materials in three formats: + + * JSON + * Excel (xlsx from 2010. It imports the first sheet) + * CSV + +Therefore, adding helping materials to your project is as simple as this command: + +```bash + pbs add_helpingmaterials + --helping-materials-lfile file.xlsx --helping-type xlsx +``` + +If you want to see all the available +options, please check the **--help** command: + +**NOTE**: By default PYBOSSA servers use a rate limit for avoiding abuse of the +API. For this reason, you can only do usually 300 requests per every 15 +minutes. If you are going to add more than 300 tasks, pbs will detect it and +warn you, auto-enabling the throttling for you to respect the limits. + + +*NOTE*: PYBOSSA helping materials allows you to upload media files like videos, +images, or sounds to support your project tutorials. The command line pbs will check +for a column in your file with the name *file_path* so it can upload it first into +the server. Please, be sure that the file (or files) path is reachable from the +helping materials file. + +```bash + pbs add_helpingmaterials --help +``` + + # Documentation You have more documentation, with real examples at @@ -279,6 +314,6 @@ in the site. # Copyright / License -Copyright (C) 2015 [SciFabric LTD](http://scifabric.com). +Copyright (C) 2017 [Scifabric LTD](http://scifabric.com). License: see LICENSE file. diff --git a/helpers.py b/helpers.py index c5ab989..b7c9f24 100644 --- a/helpers.py +++ b/helpers.py @@ -47,7 +47,8 @@ '_delete_tasks', 'enable_auto_throttling', '_update_tasks_redundancy', '_update_project_watch', 'PbsHandler', - '_update_task_presenter_bundle_js', 'row_empty'] + '_update_task_presenter_bundle_js', 'row_empty', + '_add_helpingmaterials', 'create_helping_material_info'] def _create_project(config): @@ -225,6 +226,53 @@ def _add_tasks(config, tasks_file, tasks_type, priority, redundancy): raise +def _add_helpingmaterials(config, helping_file, helping_type): + """Add helping materials to a project.""" + try: + project = find_project_by_short_name(config.project['short_name'], + config.pbclient, + config.all) + data = _load_data(helping_file, helping_type) + if len(data) == 0: + return ("Unknown format for the tasks file. Use json, csv, po or " + "properties.") + # Check if for the data we have to auto-throttle task creation + sleep, msg = enable_auto_throttling(data) + # If true, warn user + if sleep: # pragma: no cover + click.secho(msg, fg='yellow') + # Show progress bar + with click.progressbar(data, label="Adding Helping Materials") as pgbar: + for d in pgbar: + helping_info, file_path = create_helping_material_info(d) + if file_path: + # Create first the media object + hm = config.pbclient.create_helpingmaterial(project_id=project.id, + info=helping_info, + file_path=file_path) + check_api_error(hm) + + z = hm.info.copy() + z.update(helping_info) + hm.info = z + response = config.pbclient.update_helping_material(hm) + check_api_error(response) + else: + response = config.pbclient.create_helping_material(project_id=project.id, + info=helping_info) + check_api_error(response) + # If auto-throttling enabled, sleep for 3 seconds + if sleep: # pragma: no cover + time.sleep(3) + return ("%s helping materials added to project: %s" % (len(data), + config.project['short_name'])) + except exceptions.ConnectionError: + return ("Connection Error! The server %s is not responding" % config.server) + except (ProjectNotFound, TaskNotFound): + raise + + + def _delete_tasks(config, task_id, limit=100, offset=0): """Delete tasks from a project.""" try: @@ -344,6 +392,20 @@ def create_task_info(task): return task_info +def create_helping_material_info(helping): + """Create helping_material_info field.""" + helping_info = None + file_path = None + if helping.get('info'): + helping_info = helping['info'] + else: + helping_info = helping + if helping_info.get('file_path'): + file_path = helping_info.get('file_path') + del helping_info['file_path'] + return helping_info, file_path + + def enable_auto_throttling(data, limit=299): """Return True if more than 300 tasks have to be created.""" msg = 'Warning: %s tasks to create.' \ diff --git a/pbs.py b/pbs.py index 3698756..1378b6d 100644 --- a/pbs.py +++ b/pbs.py @@ -164,6 +164,20 @@ def add_tasks(config, tasks_file, tasks_type, priority, redundancy): res = _add_tasks(config, tasks_file, tasks_type, priority, redundancy) click.echo(res) + +@cli.command() +@click.option('--helping-materials-file', help='File with helping materials', + default='helping.materials', type=click.File('r')) +@click.option('--helping-type', help='Tasks type: JSON|CSV|XLSX|XLSM|XLTX|XLTM', + default=None, type=click.Choice(['json', 'csv', 'xlsx', 'xlsm', + 'xltx', 'xltm'])) +@pass_config +def add_helpingmaterials(config, helping_materials_file, helping_type): + """Add helping materials to a project.""" + res = _add_helpingmaterials(config, helping_materials_file, helping_type) + click.echo(res) + + @cli.command() @click.option('--task-id', help='Task ID to delete from project', default=None) @pass_config diff --git a/setup.py b/setup.py index e393fcf..22249fa 100644 --- a/setup.py +++ b/setup.py @@ -15,7 +15,7 @@ setup( name="pybossa-pbs", - version="2.4.0", + version="2.4.1", author="Scifabric LTD", author_email="info@scifabric.com", description="PYBOSSA command line client", @@ -29,7 +29,7 @@ 'Operating System :: OS Independent', 'Programming Language :: Python',], py_modules=['pbs', 'helpers', 'pbsexceptions'], - install_requires=['Click>=2.3, <2.4', 'pybossa-client>=1.0.4, <1.0.5', 'requests', 'nose', 'mock', 'coverage', + install_requires=['Click>=2.3, <2.4', 'pybossa-client>=1.2.0, <1.2.1', 'requests', 'nose', 'mock', 'coverage', 'rednose', 'pypandoc', 'simplejson', 'jsonschema', 'polib', 'watchdog', 'openpyxl'], entry_points=''' [console_scripts] diff --git a/test/test_pbs_add_helping_materials.py b/test/test_pbs_add_helping_materials.py new file mode 100644 index 0000000..cc98bb8 --- /dev/null +++ b/test/test_pbs_add_helping_materials.py @@ -0,0 +1,336 @@ +import json +from helpers import * +from default import TestDefault +from mock import patch, MagicMock +from nose.tools import assert_raises +from requests import exceptions +from pbsexceptions import * +from openpyxl import Workbook + +class TestPbsAddHelpingMaterial(TestDefault): + + """Test class for pbs add helping materials commands.""" + + @patch('helpers.find_project_by_short_name') + def test_add_helping_materials_json_with_info(self, find_mock): + """Test add_helpingmaterials json with info field works.""" + project = MagicMock() + project.name = 'name' + project.short_name = 'short_name' + project.description = 'description' + project.info = dict() + + find_mock.return_value = project + + helpingmaterials = MagicMock() + helpingmaterials.read.return_value = json.dumps([{'info': {'key': 'value'}}]) + + pbclient = MagicMock() + pbclient.create_helpingmaterial.return_value = {'id': 1, 'info': {'key': 'value'}} + self.config.pbclient = pbclient + res = _add_helpingmaterials(self.config, helpingmaterials, 'json') + assert res == '1 helping materials added to project: short_name', res + + @patch('helpers.find_project_by_short_name') + def test_add_helping_materials_json_from_filextension(self, find_mock): + """Test add_helpingmaterials json without specifying file extension works.""" + project = MagicMock() + project.name = 'name' + project.short_name = 'short_name' + project.description = 'description' + project.info = dict() + + find_mock.return_value = project + + helpingmaterials = MagicMock() + helpingmaterials.name = 'helpingmaterials.json' + helpingmaterials.read.return_value = json.dumps([{'info': {'key': 'value'}}]) + + pbclient = MagicMock() + pbclient.create_helping_material.return_value = {'id': 1, 'info': {'key': 'value'}} + self.config.pbclient = pbclient + res = _add_helpingmaterials(self.config, helpingmaterials, None) + assert res == '1 helping materials added to project: short_name', res + + @patch('helpers.find_project_by_short_name') + def test_add_helping_materials_csv_with_info(self, find_mock): + """Test add_helpingmaterials csv with info field works.""" + project = MagicMock() + project.name = 'name' + project.short_name = 'short_name' + project.description = 'description' + project.info = dict() + + find_mock.return_value = project + + helpingmaterials = MagicMock() + helpingmaterials.read.return_value = "info, value\n, %s, 2" % json.dumps({'key':'value'}) + + pbclient = MagicMock() + pbclient.create_helping_material.return_value = {'id': 1, 'info': {'key': 'value'}} + self.config.pbclient = pbclient + res = _add_helpingmaterials(self.config, helpingmaterials, 'csv') + assert res == '1 helping materials added to project: short_name', res + + @patch('helpers.openpyxl.load_workbook') + @patch('helpers.find_project_by_short_name') + def test_add_helping_materials_excel_with_info(self, find_mock, workbook_mock): + """Test add_helpingmaterials excel with info field works.""" + project = MagicMock() + project.name = 'name' + project.short_name = 'short_name' + project.description = 'description' + project.info = dict() + project.id = 1 + + wb = Workbook() + ws = wb.active + + headers = ['Column Name', 'foo'] + ws.append(headers) + for row in range(2, 10): + ws.append(['value', 'bar']) + + ws.append([None, None]) + ws.append([None, None]) + + find_mock.return_value = project + + helpingmaterials = MagicMock() + helpingmaterials.read.return_value = wb + + workbook_mock.return_value = wb + + pbclient = MagicMock() + self.config.pbclient = pbclient + res = _add_helpingmaterials(self.config, helpingmaterials, 'xlsx') + self.config.pbclient.create_helping_material.assert_called_with(project_id=find_mock().id, + info={u'column_name': u'value', + u'foo': u'bar'}) + assert res == '8 helping materials added to project: short_name', res + + @patch('helpers.find_project_by_short_name') + def test_add_helping_materials_csv_from_filextension(self, find_mock): + """Test add_helpingmaterials csv without specifying file extension works.""" + project = MagicMock() + project.name = 'name' + project.short_name = 'short_name' + project.description = 'description' + project.info = dict() + + find_mock.return_value = project + + helpingmaterials = MagicMock() + helpingmaterials.name = 'helpingmaterials.csv' + helpingmaterials.read.return_value = "info, value\n, %s, 2" % json.dumps({'key':'value'}) + + pbclient = MagicMock() + pbclient.create_helping_material.return_value = {'id': 1, 'info': {'key': 'value'}} + self.config.pbclient = pbclient + res = _add_helpingmaterials(self.config, helpingmaterials, None) + assert res == '1 helping materials added to project: short_name', res + + @patch('helpers.find_project_by_short_name') + def test_add_helping_materials_json_without_info(self, find_mock): + """Test add_heping_materials json without info field works.""" + project = MagicMock() + project.name = 'name' + project.short_name = 'short_name' + project.description = 'description' + project.info = dict() + + find_mock.return_value = project + + helpingmaterials = MagicMock() + helpingmaterials.read.return_value = json.dumps([{'key': 'value'}]) + + pbclient = MagicMock() + pbclient.create_helping_material.return_value = {'id': 1, 'info': {'key': 'value'}} + self.config.pbclient = pbclient + res = _add_helpingmaterials(self.config, helpingmaterials, 'json') + assert res == '1 helping materials added to project: short_name', res + + @patch('helpers.find_project_by_short_name') + def test_add_helping_materials_csv_without_info(self, find_mock): + """Test add_helpingmaterials csv without info field works.""" + project = MagicMock() + project.name = 'name' + project.short_name = 'short_name' + project.description = 'description' + project.info = dict() + + find_mock.return_value = project + + helpingmaterials = MagicMock() + helpingmaterials.read.return_value = "key, value\n, 1, 2" + + pbclient = MagicMock() + pbclient.create_helping_material.return_value = {'id': 1, 'info': {'key': 'value'}} + self.config.pbclient = pbclient + res = _add_helpingmaterials(self.config, helpingmaterials, 'csv') + assert res == '1 helping materials added to project: short_name', res + + @patch('helpers.find_project_by_short_name') + def test_add_helping_materials_unknow_type_from_filextension(self, find_mock): + """Test add_helpingmaterials with unknown type from file extension works.""" + project = MagicMock() + project.name = 'name' + project.short_name = 'short_name' + project.description = 'description' + project.info = dict() + + find_mock.return_value = project + + helpingmaterials = MagicMock() + helpingmaterials.name = 'helping.doc' + helpingmaterials.read.return_value = "key, value\n, 1, 2" + + pbclient = MagicMock() + pbclient.create_helping_material.return_value = {'id': 1, 'info': {'key': 'value'}} + self.config.pbclient = pbclient + res = _add_helpingmaterials(self.config, helpingmaterials, None) + assert res == ("Unknown format for the tasks file. Use json, csv, po or " + "properties."), res + + @patch('helpers.find_project_by_short_name') + def test_add_helping_materials_unknow_type(self, find_mock): + """Test add_helpingmaterials with unknown type works.""" + project = MagicMock() + project.name = 'name' + project.short_name = 'short_name' + project.description = 'description' + project.info = dict() + + find_mock.return_value = project + + helpingmaterials = MagicMock() + helpingmaterials.read.return_value = "key, value\n, 1, 2" + + pbclient = MagicMock() + pbclient.create_helping_material.return_value = {'id': 1, 'info': {'key': 'value'}} + self.config.pbclient = pbclient + res = _add_helpingmaterials(self.config, helpingmaterials, 'doc') + assert res == ("Unknown format for the tasks file. Use json, csv, po or " + "properties."), res + + @patch('helpers.find_project_by_short_name') + def test_add_helping_materials_csv_connection_error(self, find_mock): + """Test add_helpingmaterials csv connection error works.""" + project = MagicMock() + project.name = 'name' + project.short_name = 'short_name' + project.description = 'description' + project.info = dict() + + find_mock.return_value = project + + helpingmaterials = MagicMock() + helpingmaterials.read.return_value = "key, value\n, 1, 2" + + pbclient = MagicMock() + pbclient.create_helping_material.side_effect = exceptions.ConnectionError + self.config.pbclient = pbclient + res = _add_helpingmaterials(self.config, helpingmaterials, 'csv') + assert res == "Connection Error! The server http://server is not responding", res + + @patch('helpers.find_project_by_short_name') + def test_add_helping_material_json_connection_error(self, find_mock): + """Test add_helpingmaterials json connection error works.""" + project = MagicMock() + project.name = 'name' + project.short_name = 'short_name' + project.description = 'description' + project.info = dict() + + find_mock.return_value = project + + tasks = MagicMock() + tasks.read.return_value = json.dumps([{'key': 'value'}]) + + pbclient = MagicMock() + pbclient.create_helping_material.side_effect = exceptions.ConnectionError + self.config.pbclient = pbclient + res = _add_helpingmaterials(self.config, tasks, 'json') + assert res == "Connection Error! The server http://server is not responding", res + + @patch('helpers.find_project_by_short_name') + def test_add_helpingmaterial_another_error(self, find_mock): + """Test add_tasks another error works.""" + project = MagicMock() + project.name = 'name' + project.short_name = 'short_name' + project.description = 'description' + project.info = dict() + + find_mock.return_value = project + + tasks = MagicMock() + tasks.read.return_value = json.dumps([{'key': 'value'}]) + + pbclient = MagicMock() + pbclient.create_helping_material.return_value = self.error + self.config.pbclient = pbclient + assert_raises(ProjectNotFound, _add_helpingmaterials, self.config, + tasks, 'json') + + def test_create_helping_material_info(self): + """Test create_helping_material_info method works.""" + data = dict(info=dict(foo=1)) + helping_info, file_path = create_helping_material_info(data) + assert helping_info == dict(foo=1) + assert file_path is None + + data = dict(foo=1) + helping_info, file_path = create_helping_material_info(data) + assert helping_info == dict(foo=1) + assert file_path is None + + data = dict(foo=1, file_path='file') + helping_info, file_path = create_helping_material_info(data) + assert helping_info == dict(foo=1) + assert file_path == 'file' + + @patch('helpers.openpyxl.load_workbook') + @patch('helpers.find_project_by_short_name') + def test_add_helping_materials_excel_with_file(self, find_mock, workbook_mock): + """Test add_helpingmaterials excel with file_path field works.""" + project = MagicMock() + project.name = 'name' + project.short_name = 'short_name' + project.description = 'description' + project.info = dict() + project.id = 1 + + wb = Workbook() + ws = wb.active + + headers = ['Column Name', 'foo', 'file Path'] + ws.append(headers) + for row in range(2, 10): + ws.append(['value', 'bar', '/tmp/file.jpg']) + + ws.append([None, None, None]) + ws.append([None, None, None]) + + find_mock.return_value = project + + helpingmaterials = MagicMock() + helpingmaterials.read.return_value = wb + + workbook_mock.return_value = wb + + pbclient = MagicMock() + hm = MagicMock() + hm.info = {'column_name': 'value', 'foo': 'bar'} + hm.id = 1 + pbclient.create_helpingmaterial.return_value = hm + self.config.pbclient = pbclient + res = _add_helpingmaterials(self.config, helpingmaterials, 'xlsx') + self.config.pbclient.create_helpingmaterial.assert_called_with(project_id=find_mock().id, + file_path='/tmp/file.jpg', + info={u'column_name': u'value', + u'foo': u'bar'}) + self.config.pbclient.update_helping_material.assert_called_with(hm) + + assert res == '8 helping materials added to project: short_name', res +