diff --git a/.gitignore b/.gitignore index 8ce4042..e4fb736 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +*.tar.gz *.pyc *.csv *.sqlite \ No newline at end of file diff --git a/circle.yml b/circle.yml new file mode 100644 index 0000000..42032e2 --- /dev/null +++ b/circle.yml @@ -0,0 +1,24 @@ +machine: + python: + version: '2.7.10' + php: + version: '5.5.11' + environment: + DATABASE_URL: mysql://ubuntu:@127.0.0.1:3306/circle_test + COMPOSER_PATH: $HOME/.config/composer/vendor/bin + PATH: $COMPOSER_PATH:$PATH + DKAN_URI: http://127.0.0.1:8888 +checkout: + post: + - rm -rf ~/.composer +dependencies: + pre: + - 'pip install -r requirements.txt' + override: + - 'bash dkan-archive-init.sh --deps --build=$DATABASE_URL' + - 'cd ../dkan/ && ahoy drush --yes runserver :8888': + background: true + - sleep 10 +test: + override: + - 'nosetests --verbose' diff --git a/dkan-archive-init.sh b/dkan-archive-init.sh new file mode 100644 index 0000000..3ff570a --- /dev/null +++ b/dkan-archive-init.sh @@ -0,0 +1,16 @@ + +# DKAN branch or tag to use. +DKAN_VERSION="7.x-1.x" + +# Try to grab archived dkan to speed up bootstrap. +URL="https://s3-us-west-2.amazonaws.com/nucivic-data-dkan-archives/dkan-$DKAN_VERSION.tar.gz" +wget -q -c "$URL" +mv dkan-$DKAN_VERSION.tar.gz ../ +cd .. +tar -xzf dkan-$DKAN_VERSION.tar.gz +rm -rf dkan/docroot/sites/default/settings.php +cd dkan +bash dkan/dkan-init.sh dkan --skip-init --deps +ahoy drush "-y --verbose si minimal --sites-subdir=default --account-pass='admin' --db-url=$DATABASE_URL install_configure_form.update_status_module=\"'array\(FALSE,FALSE\)'\"" +ahoy drush sql-drop -y && ahoy dkan sqlc < backups/last_install.sql && \ +echo "Installed dkan from backup" diff --git a/dkan/client.py b/dkan/client.py index 1e722ae..0efb8ea 100644 --- a/dkan/client.py +++ b/dkan/client.py @@ -29,9 +29,17 @@ def login(self, user, password): token = self.post(uri) self.headers['X-CSRF-Token'] = token.text - def node(self, node_id, action='retrieve', **kwargs): - uri = os.path.join(self.dkan, 'api/dataset/node/%s' % node_id) + def node(self, action='index', **kwargs): + uri = os.path.join(self.dkan, 'api/dataset/node') + if action not in ['index', 'create']: + node_id = kwargs.get('node_id', False) + if node_id: + del kwargs['node_id'] + uri = os.path.join(uri, '%s' % node_id) + else: + raise ValueError('For action type %s you need to specify a node_id' % action) action_map = { + 'index': self.get, 'retrieve': self.get, 'update': self.put, 'create': self.post, @@ -39,15 +47,15 @@ def node(self, node_id, action='retrieve', **kwargs): } if not action in action_map.keys(): raise ValueError('action parameter should be one of the following: %s' % ', '.join(action_map.keys())) - if action == 'retrieve': - return action_map[action](uri) - else: + if action not in ['index', 'retrieve']: kwargs['headers'] = self.headers.copy() kwargs['headers']['Content-Type'] = 'application/json' - return action_map[action](uri, **kwargs) + if 'data' in kwargs.keys(): + kwargs['data'] = json.dumps(kwargs['data']) + return action_map[action](uri, **kwargs) - def get(self, uri): - return self.request(uri, 'GET') + def get(self, uri, **kwargs): + return self.request(uri, 'GET', **kwargs) def post(self, uri, **kwargs): return self.request(uri, 'POST', **kwargs) @@ -69,11 +77,11 @@ def request(self, uri, rtype, **kwargs): s = requests.Session() return s.send(prepared) - def attach_file_to_node(self, file, node_id, update=0): + def attach_file_to_node(self, file, node_id, field, update=0): uri = os.path.join(self.dkan, 'api/dataset/node/%s/attach_file' % node_id) headers = self.headers.copy() data = { - 'field_name': 'field_upload', + 'field_name': field, 'attach': update, } files = { diff --git a/dkan/tests.py b/dkan/tests.py index d22abfa..da4861a 100644 --- a/dkan/tests.py +++ b/dkan/tests.py @@ -1,47 +1,178 @@ -import unittest +import os, unittest from client import DatasetAPI - class TestDatasetAPI(unittest.TestCase): def setUp(self): - # Instantiate client and login - pass + uri = os.environ.get('DKAN_URI', False) + user = os.environ.get('DKAN_USER', 'admin') + password = os.environ.get('DKAN_PASSWORD', 'admin') + # Fail if DKAN_URI is not set + if not uri: + self.skipTest(TestDatasetAPI) + self.api = DatasetAPI(uri, user, password) + self.cleanup = [] + + def tearDown(self): + self.cleanup = list(set(self.cleanup)) + for node in self.cleanup: + delete = self.api.node('delete', node_id=node) + self.assertEqual(delete.status_code, 200) + self.cleanup = [] + + def _get_nodes_of_type(self, node_type): + payload = { + 'parameters[type]': node_type + } + r = self.api.node(params=payload) + self.assertEqual(r.status_code, 200) + return r.json() + + def _get_nodes_types(self, nodes): + types = [dataset['type'] for dataset in nodes] + types = list(set(types)) + return types + + def _test_list_nodes(self, node_type): + nodes = self._get_nodes_of_type(node_type) + types = self._get_nodes_types(nodes) + # Test for dataset type + self.assertEqual(types[0], node_type) + self.assertEqual(len(types), 1) def test_list_datasets(self): - # List existing datasets - pass + self._test_list_nodes('dataset') def test_list_resources(self): - # List existing resources - pass + self._test_list_nodes('resource') + + def _test_create_node(self, data): + node_type = data['type'] + nodes = self._get_nodes_of_type(node_type) + original_count = len(nodes) + # Create Node + node = self.api.node('create', data=data) + self.assertEqual(node.status_code, 200) + node = node.json() + self.cleanup.append(node['nid']) + # Verify count + nodes = self._get_nodes_of_type(node_type) + new_count = len(nodes) + self.assertEqual(new_count, original_count + 1) + return node def test_create_dataset(self): - # Create Dataset - pass + data = { + 'title': 'Test Dataset', + 'type': 'dataset', + 'body': [ + { + 'value': "Body for test Dataset", + }, + ] + } + self._test_create_node(data) def test_create_resource(self): - # Create Resource - pass + data = { + 'title': 'Test Resource', + 'type': 'resource', + 'body': [ + { + 'value': "Body for test Resource", + }, + ] + } + self._test_create_node(data) + + def _test_update_node(self, data): + original_node = self._test_create_node(data) + original_node = self.api.node('retrieve', node_id=original_node['nid']).json() + update = { + 'title': '%s updated' % data['title'], + } + self.assertTrue('nid' in original_node.keys()) + node = self.api.node('update', node_id=original_node['nid'], data=update) + self.assertEqual(node.status_code, 200) + node = self.api.node('retrieve', node_id=original_node['nid']).json() + self.assertEqual('%s updated' % original_node['title'], node['title']) def test_update_dataset(self): - # Update Dataset - pass + data = { + 'title': 'Test Dataset to be updated', + 'type': 'dataset', + 'body': [ + { + 'value': "Body for test Dataset", + }, + ] + } + self._test_update_node(data) def test_update_resource(self): - # Update Resource - pass + data = { + 'title': 'Test Resource to be updated', + 'type': 'resource', + 'body': [ + { + 'value': "Body for test Resource", + }, + ] + } + self._test_update_node(data) def test_attach_file_to_resource(self): - # Attach a test file to an existing resource - pass + csv = os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', + 'examples', 'data', 'tension_sample_data.csv') + data = { + 'title': 'Test Resource to receive a file', + 'type': 'resource', + 'body': [ + { + 'value': "Body for test Resource", + }, + ] + } + resource = self._test_create_node(data) + resource = self.api.node('retrieve', node_id=resource['nid']).json() + r = self.api.attach_file_to_node(csv, resource['nid'], 'field_upload') + self.assertEqual(r.status_code, 200) + resource = self.api.node('retrieve', node_id=resource['nid']).json() + filename = resource['field_upload']['und'][0]['filename'] + self.assertTrue(filename=='tension_sample_data.csv') + + def _test_delete_node(self, data): + count = len(self._get_nodes_of_type(data['type'])) + node = self._test_create_node(data) + self.assertTrue('nid' in node.keys()) + delete = self.api.node('delete', node_id=node['nid']) + self.assertEqual(delete.status_code, 200) + self.cleanup.remove(node['nid']) + new_count = len(self._get_nodes_of_type(data['type'])) + self.assertEqual(count, new_count) def test_delete_dataset(self): - # Delete Dataset - pass + data = { + 'title': 'Test Dataset to be deleted', + 'type': 'dataset', + 'body': [ + { + 'value': "Body for test Dataset", + }, + ] + } + self._test_delete_node(data) def test_delete_resource(self): - # Delete Resource - pass + data = { + 'title': 'Test Resource to be deleted', + 'type': 'resource', + 'body': [ + { + 'value': "Body for test Resource", + }, + ] + } + self._test_delete_node(data) if __name__ == '__main__': unittest.main() diff --git a/examples/attach_file_to_node.py b/examples/attach_file_to_node.py index b78576a..1b05acc 100644 --- a/examples/attach_file_to_node.py +++ b/examples/attach_file_to_node.py @@ -1,20 +1,26 @@ +import sys +sys.path.append('..') + import os import json from dkan.client import DatasetAPI -DKAN_URI = 'http://docker:32782' -RESOURCE_ID = 5 -CSV = os.path.join('/Users/teofilosibileau/Downloads', 'CSV.csv') - -# Make the api authenticate properly -api = DatasetAPI( - DKAN_URI, - 'admin', - 'admin', - True -) +uri = os.environ.get('DKAN_URI', False) +user = os.environ.get('DKAN_USER', 'admin') +password = os.environ.get('DKAN_PASSWORD', 'admin') -# Attach the file to the resource node -r = api.attach_file_to_node(CSV, RESOURCE_ID) -print r.status_code -print r. text \ No newline at end of file +if uri: + api = DatasetAPI(uri, user, password, True) + payload = {'parameters[type]': 'resource'} + nodes = api.node(params=payload).json() + resource = nodes[0] + print resource + csv = os.path.join('.', 'data', 'tension_sample_data.csv') + # Attach the file to the resource node + r = api.attach_file_to_node(csv, resource['nid'], 'field_upload') + print r.status_code + print r. text + resource = api.node('retrieve', node_id=resource['nid']) + print resource.json()['field_upload'] +else: + print 'Please Set the dkan URL as an ENVIRONMENT variable' \ No newline at end of file diff --git a/examples/create_node.py b/examples/create_node.py new file mode 100644 index 0000000..976c926 --- /dev/null +++ b/examples/create_node.py @@ -0,0 +1,22 @@ +import sys +sys.path.append('..') + +import os +import json +from dkan.client import DatasetAPI + +uri = os.environ.get('DKAN_URI', False) +user = os.environ.get('DKAN_USER', 'admin') +password = os.environ.get('DKAN_PASSWORD', 'admin') + +if uri: + api = DatasetAPI(uri, user, password, True) + data = { + 'title': 'Test Dataset', + 'type': 'dataset' + } + dataset = api.node('create', data=data) + print dataset.status_code + print dataset.text +else: + print 'Please Set the dkan URL as an ENVIRONMENT variable' \ No newline at end of file diff --git a/examples/data/tension_sample_data.csv b/examples/data/tension_sample_data.csv new file mode 100644 index 0000000..f6eb2a1 --- /dev/null +++ b/examples/data/tension_sample_data.csv @@ -0,0 +1,4 @@ +tension,current,timestamp +220,10,2016-05-27T19:56:41.870Z +50,15,2016-05-27T19:51:21.794Z +230,10,2016-05-27T19:56:41.870Z \ No newline at end of file diff --git a/examples/delete_node.py b/examples/delete_node.py new file mode 100644 index 0000000..2fb3e28 --- /dev/null +++ b/examples/delete_node.py @@ -0,0 +1,26 @@ +import sys +sys.path.append('..') + +import os +import json +from dkan.client import DatasetAPI + +uri = os.environ.get('DKAN_URI', False) +user = os.environ.get('DKAN_USER', 'admin') +password = os.environ.get('DKAN_PASSWORD', 'admin') + +if uri: + api = DatasetAPI(uri, user, password, True) + data = { + 'title': 'Test Dataset', + 'type': 'dataset' + } + dataset = api.node('create', data=data) + print dataset.status_code + print dataset.text + nid = dataset.json()['nid'] + op = api.node('delete', node_id=nid) + print op.status_code + print op.text +else: + print 'Please Set the dkan URL as an ENVIRONMENT variable' \ No newline at end of file diff --git a/examples/list_nodes.py b/examples/list_nodes.py new file mode 100644 index 0000000..4ae523b --- /dev/null +++ b/examples/list_nodes.py @@ -0,0 +1,21 @@ +import sys +sys.path.append('..') + +import os +import json +from dkan.client import DatasetAPI + +uri = os.environ.get('DKAN_URI', False) +user = os.environ.get('DKAN_USER', 'admin') +password = os.environ.get('DKAN_PASSWORD', 'admin') + +# Make the api authenticate properly +api = DatasetAPI(uri, user, password, True) + +# List datasets +params = { + 'parameters[type]': 'dataset', +} +r = api.node(params=params) + +print r.json() diff --git a/examples/update_node.py b/examples/update_node.py new file mode 100644 index 0000000..c24b528 --- /dev/null +++ b/examples/update_node.py @@ -0,0 +1,29 @@ +import sys +sys.path.append('..') + +import os +import json +from dkan.client import DatasetAPI + +uri = os.environ.get('DKAN_URI', False) +user = os.environ.get('DKAN_USER', 'admin') +password = os.environ.get('DKAN_PASSWORD', 'admin') + +if uri: + api = DatasetAPI(uri, user, password, True) + data = { + 'title': 'Test Dataset', + 'type': 'dataset' + } + dataset = api.node('create', data=data) + print dataset.status_code + print dataset.text + dataset = dataset.json() + update = { + 'title': 'Test Dataset Updated', + } + r = api.node('update', node_id=dataset['nid'], data=update) + print r.status_code + print r.text +else: + print 'Please Set the dkan URL as an ENVIRONMENT variable' \ No newline at end of file