diff --git a/ckanext/right_time_context/controller.py b/ckanext/right_time_context/controller.py index a599316..99bc607 100644 --- a/ckanext/right_time_context/controller.py +++ b/ckanext/right_time_context/controller.py @@ -21,7 +21,8 @@ # along with Orion Context Broker. If not, see http://www.gnu.org/licenses/. import json -from logging import getLogger +import logging + import os import urlparse @@ -31,180 +32,13 @@ import requests import six -from .plugin import NGSI_REG_FORMAT - -log = getLogger(__name__) +import ckanext.right_time_context.utils as utils -CHUNK_SIZE = 512 +log = logging.getLogger(__name__) class ProxyNGSIController(base.BaseController): - def _proxy_query_resource(self, resource, parsed_url, headers, verify=True): - - if parsed_url.path.lower().find('/v1/querycontext') != -1: - if resource.get("payload", "").strip() == "": - details = 'Please add a payload to complete the query.' - base.abort(409, detail=details) - - try: - json.loads(resource['payload']) - except json.JSONDecodeError: - details = "Payload field doesn't contain valid JSON data." - base.abort(409, detail=details) - - headers['Content-Type'] = "application/json" - r = requests.post(resource['url'], headers=headers, data=resource["payload"], stream=True, verify=verify) - - else: - r = requests.get(resource['url'], headers=headers, stream=True, verify=verify) - - return r - - def _proxy_registration_resource(self, resource, parsed_url, headers, verify=True): - path = parsed_url.path - - if path.endswith('/'): - path = path[:-1] - - path = path + '/v2/op/query' - attrs = [] - - if 'attrs_str' in resource and len(resource['attrs_str']): - attrs = resource['attrs_str'].split(',') - body = { - 'entities': [], - 'attrs': attrs - } - - # Include entity information - for entity in resource['entity']: - query_entity = { - 'type': entity['value'] - } - if 'isPattern' in entity and entity['isPattern'] == 'on': - query_entity['idPattern'] = entity['id'] - else: - query_entity['id'] = entity['id'] - - body['entities'].append(query_entity) - - # Parse expression to include georel information - if 'expression' in resource and len(resource['expression']): - # Separate expresion query strings - supported_expressions = ['georel', 'geometry', 'coords'] - parsed_expression = resource['expression'].split('&') - - expression = {} - for exp in parsed_expression: - parsed_exp = exp.split('=') - - if len(parsed_exp) != 2 or not parsed_exp[0] in supported_expressions: - base.abort(422, detail='The expression is not a valid one for NGSI Registration, only georel, geometry, and coords is supported') - else: - expression[parsed_exp[0]] = parsed_exp[1] - - body['expression'] = expression - - headers['Content-Type'] = 'application/json' - url = urlparse.urljoin(parsed_url.scheme + '://' + parsed_url.netloc, path) - response = requests.post(url, headers=headers, json=body, stream=True, verify=verify) - - return response - - def process_auth_credentials(self, resource, headers): - auth_method = resource.get('auth_type', 'none') - - if auth_method == "oauth2": - token = toolkit.c.usertoken['access_token'] - headers['Authorization'] = "Bearer %s" % token - elif auth_method == "x-auth-token-fiware": - # Deprecated method, Including OAuth2 token retrieved from the IdM - # on the Open Stack X-Auth-Token header - token = toolkit.c.usertoken['access_token'] - headers['X-Auth-Token'] = token - def proxy_ngsi_resource(self, resource_id): - # Chunked proxy for ngsi resources. - context = {'model': base.model, 'session': base.model.Session, 'user': base.c.user or base.c.author} - - log.info('Proxify resource {id}'.format(id=resource_id)) - resource = logic.get_action('resource_show')(context, {'id': resource_id}) - - headers = { - 'Accept': 'application/json' - } - - resource.setdefault('auth_type', 'none') - self.process_auth_credentials(resource, headers) - - if resource.get('tenant', '') != '': - headers['FIWARE-Service'] = resource['tenant'] - if resource.get('service_path', '') != '': - headers['FIWARE-ServicePath'] = resource['service_path'] - - url = resource['url'] - parsed_url = urlparse.urlsplit(url) - - if parsed_url.scheme not in ("http", "https") or not parsed_url.netloc: - base.abort(409, detail='Invalid URL.') - - # Process verify configuration - verify_conf = os.environ.get('CKAN_RIGHT_TIME_CONTEXT_VERIFY_REQUESTS', toolkit.config.get('ckan.right_time_context.verify_requests')) - if verify_conf is None or (isinstance(verify_conf, six.string_types) and verify_conf.strip() == ""): - verify_conf = os.environ.get('CKAN_VERIFY_REQUESTS', toolkit.config.get('ckan.verify_requests')) - - if isinstance(verify_conf, six.string_types) and verify_conf.strip() != "": - compare_env = verify_conf.lower().strip() - if compare_env in ("true", "1", "on"): - verify = True - elif compare_env in ("false", "0", "off"): - verify = False - else: - verify = verify_conf - elif isinstance(verify_conf, bool): - verify = verify_conf - else: - verify = True - - # Make the request to the server - try: - if resource['format'].lower() == NGSI_REG_FORMAT: - r = self._proxy_registration_resource(resource, parsed_url, headers, verify=verify) - else: - r = self._proxy_query_resource(resource, parsed_url, headers, verify=verify) - - except requests.HTTPError: - details = 'Could not proxy ngsi_resource. We are working to resolve this issue as quickly as possible' - base.abort(409, detail=details) - except requests.ConnectionError: - details = 'Could not proxy ngsi_resource because a connection error occurred.' - base.abort(502, detail=details) - except requests.Timeout: - details = 'Could not proxy ngsi_resource because the connection timed out.' - base.abort(504, detail=details) - - if r.status_code == 401: - if resource.get('auth_type', 'none') != 'none': - details = 'ERROR 401 token expired. Retrieving new token, reload please.' - log.info(details) - toolkit.c.usertoken_refresh() - base.abort(409, detail=details) - elif resource.get('auth_type', 'none') == 'none': - details = 'Authentication requested by server, please check resource configuration.' - log.info(details) - base.abort(409, detail=details) - - elif r.status_code == 400: - response = r.json() - details = response['description'] - log.info(details) - base.abort(422, detail=details) - - else: - r.raise_for_status() - base.response.content_type = r.headers['content-type'] - base.response.charset = r.encoding + utils.proxy_ngsi_resource(resource_id) - for chunk in r.iter_content(chunk_size=CHUNK_SIZE): - base.response.body_file.write(chunk) diff --git a/ckanext/right_time_context/fanstatic/view_ngsi.min.js b/ckanext/right_time_context/fanstatic/view_ngsi.min.js deleted file mode 100644 index 4396fea..0000000 --- a/ckanext/right_time_context/fanstatic/view_ngsi.min.js +++ /dev/null @@ -1,35 +0,0 @@ -ckan.module('right_time_context',function(jQuery,_){ - return{ - options:{ - i18n:{error:_('An error occurred: %(text)s %(error)s')}, - parameters:{contentType:'application/json', - dataType:'json', - dataConverter:function(data){return JSON.stringify(data,null,2);}, - language:'json',type:'GET'} - }, - initialize:function(){ - var self=this; - var p; - p=this.options.parameters; - if(typeof(view_enable) == 'undefined'){ - view_enable = []; - view_enable[0] = true; - resource_url = preload_resource['url'] - } - if(view_enable[0]){ - jQuery.ajax(resource_url,{type:p.type,contentType:p.contentType,dataType:p.dataType,success:function(data,textStatus,jqXHR){ - data=p.dataConverter?p.dataConverter(data):data; - var highlighted; - if(p.language){highlighted=hljs.highlight(p.language,data,true).value;} - else{highlighted='
'+data+'
';} - self.el.html(highlighted); - }, - error:function(jqXHR,textStatus,errorThrown){ - if(textStatus=='error'&&jqXHR.responseText.length){self.el.html(jqXHR.responseText);} - else{self.el.html(self.i18n('error',{text:textStatus,error:errorThrown}));}}}); - } - else{ - self.el.html("\n\n\n"+view_enable[1]+"\n\n\n"); - } - - }};}); diff --git a/ckanext/right_time_context/fanstatic/webassets.yml b/ckanext/right_time_context/fanstatic/webassets.yml new file mode 100644 index 0000000..a0782a1 --- /dev/null +++ b/ckanext/right_time_context/fanstatic/webassets.yml @@ -0,0 +1,19 @@ +main-css: + output: ckanext-right_time_context/%(version)s_ngsi_icons.css + contents: + - vendor/bootstrap/css/bootstrap-responsive.css + - styles/github.css + - css/ngsi.css + - css/ngsi_icons.css + +main: + output: ckanext-right_time_context/%(version)s_ngsi_icons.js + extra: + preload: + - base/main + - ckanext-right_time_context/main-css + contents: + - vendor/highlight/highlight.pack.js + - vendor/dygraph/dygraph-combined.js + - view_ngsi.js + diff --git a/ckanext/right_time_context/plugin.py b/ckanext/right_time_context/plugin/__init__.py similarity index 82% rename from ckanext/right_time_context/plugin.py rename to ckanext/right_time_context/plugin/__init__.py index 5c65749..7994ac1 100644 --- a/ckanext/right_time_context/plugin.py +++ b/ckanext/right_time_context/plugin/__init__.py @@ -23,38 +23,35 @@ import logging from ckan.common import _, json -import ckan.plugins as p +from ckan import plugins as p import ckan.lib.helpers as h +from ckan.plugins import toolkit log = logging.getLogger(__name__) - +import ckanext.right_time_context.utils as utils NGSI_FORMAT = 'fiware-ngsi' NGSI_REG_FORMAT = 'fiware-ngsi-registry' +if toolkit.check_ckan_version("2.10.1"): + from ckanext.right_time_context.plugin.flask_plugin import MixinPlugin +else: + from ckanext.right_time_context.plugin.pylons_plugin import MixinPlugin + def check_query(resource): parsedurl = resource['url'].lower() - return parsedurl.find('/v2/entities') != -1 or parsedurl.find('/v1/querycontext') != -1 or parsedurl.find('/v1/contextentities/') != -1 + return parsedurl.find('/entities') != -1 or parsedurl.find('/querycontext') != -1 or parsedurl.find('/contextentities/') != -1 -class NgsiView(p.SingletonPlugin): +class NgsiView(MixinPlugin, p.SingletonPlugin): - p.implements(p.IRoutes, inherit=True) p.implements(p.IConfigurer, inherit=True) p.implements(p.IConfigurable, inherit=True) p.implements(p.IResourceView, inherit=True) p.implements(p.IResourceController, inherit=True) p.implements(p.ITemplateHelpers) - def before_map(self, m): - m.connect( - '/dataset/{id}/resource/{resource_id}/ngsiproxy', - controller='ckanext.right_time_context.controller:ProxyNGSIController', - action='proxy_ngsi_resource' - ) - return m - def get_helpers(self): def get_available_auth_methods(): @@ -74,24 +71,17 @@ def get_available_auth_methods(): "right_time_context_get_available_auth_methods": get_available_auth_methods, } - def get_proxified_ngsi_url(self, data_dict): - url = h.url_for( - action='proxy_ngsi_resource', - controller='ckanext.right_time_context.controller:ProxyNGSIController', - id=data_dict['package']['name'], - resource_id=data_dict['resource']['id'] - ) - log.info('Proxified url is {0}'.format(url)) - return url def configure(self, config): self.proxy_is_enabled = p.plugin_loaded('resource_proxy') self.oauth2_is_enabled = p.plugin_loaded('oauth2') def update_config(self, config): - p.toolkit.add_template_directory(config, 'templates') - p.toolkit.add_resource('fanstatic', 'right_time_context') - p.toolkit.add_public_directory(config, 'public') + p.toolkit.add_template_directory(config, '../templates') + p.toolkit.add_resource('../fanstatic', 'ckanext-right_time_context') + p.toolkit.add_public_directory(config, '../public') + + def info(self): return { @@ -132,7 +122,7 @@ def setup_template_variables(self, context, data_dict): h.flash_error(f_details, allow_html=False) view_enable = [False, details] elif format_lower == NGSI_FORMAT and not check_query(resource): - details = "

This is not a ContextBroker query, please check Orion Context Broker documentation


" + #details = "

This is not a ContextBroker query, please check Orion Context Broker documentation


" f_details = "This is not a ContextBroker query, please check Orion Context Broker documentation." h.flash_error(f_details, allow_html=False) view_enable = [False, details] @@ -143,7 +133,7 @@ def setup_template_variables(self, context, data_dict): view_enable = [False, details] else: # All checks passed - url = self.get_proxified_ngsi_url(data_dict) + url = utils.get_proxified_ngsi_url(data_dict) data_dict['resource']['url'] = url view_enable = [True, 'OK'] @@ -209,20 +199,37 @@ def remove_serialized(prefix): return serialized_resource + # CKAN < 2.10 hooks def before_create(self, context, resource): - return self._serialize_resource(resource) + return self.before_resource_create(context, resource) def after_create(self, context, resource): + return self.after_resource_create(context, resource) + + def before_update(self, context, current, resource): + return self.before_resource_update(context, current, resource) + + def after_update(self, context, resource): + return self.after_resource_update(context, resource) + + def before_show(self, resource): + return self.before_resource_show(resource) + + # CKAN >= 2.10 hooks + def before_resource_create(self, context, resource): + return self._serialize_resource(resource) + + def after_resource_create(self, context, resource): # Create entry in the NGSI registry pass - def before_update(self, context, current, resource): + def before_resource_update(self, context, current, resource): return self._serialize_resource(resource) - def after_update(self, context, resource): + def after_resource_update(self, context, resource): pass - def before_show(self, resource): + def before_resource_show(self, resource): # Deserialize resource information entities = [] @@ -242,3 +249,4 @@ def deserilize_handler(prefix): resource['entity'] = entities return resource + diff --git a/ckanext/right_time_context/plugin/flask_plugin.py b/ckanext/right_time_context/plugin/flask_plugin.py new file mode 100644 index 0000000..ab5d4fd --- /dev/null +++ b/ckanext/right_time_context/plugin/flask_plugin.py @@ -0,0 +1,15 @@ +# -*- coding: utf-8 -*- + +from ckan import plugins as p + +from ckanext.right_time_context.views import get_blueprints + + +class MixinPlugin(p.SingletonPlugin): + p.implements(p.IBlueprint) + + # IBlueprint + + def get_blueprint(self): + return get_blueprints() + diff --git a/ckanext/right_time_context/plugin/pylons_plugin.py b/ckanext/right_time_context/plugin/pylons_plugin.py new file mode 100644 index 0000000..9235038 --- /dev/null +++ b/ckanext/right_time_context/plugin/pylons_plugin.py @@ -0,0 +1,16 @@ +# -*- coding: utf-8 -*- + +from ckan import plugins as p + + +class MixinPlugin(p.SingletonPlugin): + + p.implements(p.IRoutes, inherit=True) + + def before_map(self, m): + m.connect( + '/dataset/{id}/resource/{resource_id}/ngsiproxy', + controller='ckanext.right_time_context.controller:ProxyNGSIController', + action='proxy_ngsi_resource' + ) + return m diff --git a/ckanext/right_time_context/templates/base.html b/ckanext/right_time_context/templates/base.html index d3deaf8..2605207 100644 --- a/ckanext/right_time_context/templates/base.html +++ b/ckanext/right_time_context/templates/base.html @@ -2,5 +2,7 @@ {% block styles %} {{ super() }} - {% resource 'right_time_context/css/ngsi_icons.css' %} + {% set type = 'asset' if h.ckan_version().split('.')[1] | int >= 9 else 'resource' %} + {% include 'package/snippets/main_' ~ type ~ '.html' %} {% endblock %} + diff --git a/ckanext/right_time_context/templates/ngsi.html b/ckanext/right_time_context/templates/ngsi.html index 92cd492..ff4bd9e 100644 --- a/ckanext/right_time_context/templates/ngsi.html +++ b/ckanext/right_time_context/templates/ngsi.html @@ -1,4 +1,4 @@ -{% extends 'dataviewer/base.html' %} +{% extends 'base.html' %} {% block page %}
@@ -18,12 +18,5 @@ {% endblock %} -{% resource 'right_time_context/vendor/highlight/highlight.pack.js' %} -{% resource 'right_time_context/vendor/bootstrap/js/bootstrap.js' %} -{% resource 'right_time_context/vendor/dygraph/dygraph-combined.js' %} -{% resource 'right_time_context/view_ngsi.js' %} -{% resource 'right_time_context/vendor/bootstrap/css/bootstrap-responsive.css' %} -{% resource 'right_time_context/vendor/bootstrap/css/bootstrap.css' %} -{% resource 'right_time_context/css/ngsi.css' %} - {% endblock %} + diff --git a/ckanext/right_time_context/templates/package/resource_read.html b/ckanext/right_time_context/templates/package/resource_read.html index aeade8f..2e4cd4b 100644 --- a/ckanext/right_time_context/templates/package/resource_read.html +++ b/ckanext/right_time_context/templates/package/resource_read.html @@ -2,10 +2,12 @@ {% block resource_actions_inner %} {% if h.check_access('package_update', {'id':pkg.id }) %} -
  • {% link_for _('Manage'), controller='package', action='resource_edit', id=pkg.name, resource_id=res.id, class_='btn', icon='wrench' %}
    • +
  • {% link_for _('Edit resource'), named_route=pkg.type ~ '_resource.edit', id=pkg.name, resource_id=res.id, class_='btn btn-default', icon='pencil' %}
    • + {% block action_manage_inner %}{% endblock %} +
  • {% link_for _('Views'), named_route=pkg.type ~ '_resource.views', id=pkg.name, resource_id=res.id, class_='btn btn-default', icon='chart-bar' %} {% endif %} {% if res.url and h.is_url(res.url) %} - {% if res.url.lower().find('/querycontext')== -1 and res.url.lower().find('/contextentities/')== -1 and res.url.lower().find('/entities/')== -1 and res.format.lower().find('fiware-ngsi-registry') == -1 %} + {% if res.url.lower().find('/querycontext')== -1 and res.url.lower().find('/contextentities/')== -1 and res.url.lower().find('/entities/')== -1 and res.format.lower().find('fiware-ngsi-registry') == -1 and res.format.lower().find('fiware-ngsi') == -1 %}
  • {% if res.resource_type in ('listing', 'service') %} @@ -40,3 +42,4 @@

    {{ _('URL:') }} {{ res.url }}

    {% endif %} {% endblock %} + diff --git a/ckanext/right_time_context/templates/package/snippets/main_asset.html b/ckanext/right_time_context/templates/package/snippets/main_asset.html new file mode 100644 index 0000000..0156dc4 --- /dev/null +++ b/ckanext/right_time_context/templates/package/snippets/main_asset.html @@ -0,0 +1 @@ +{% asset 'ckanext-right_time_context/main' %} diff --git a/ckanext/right_time_context/templates/package/snippets/main_resource.html b/ckanext/right_time_context/templates/package/snippets/main_resource.html new file mode 100644 index 0000000..2668400 --- /dev/null +++ b/ckanext/right_time_context/templates/package/snippets/main_resource.html @@ -0,0 +1 @@ +{% resource 'ckanext-right_time_context/main' %} diff --git a/ckanext/right_time_context/templates/package/snippets/resource_form.html b/ckanext/right_time_context/templates/package/snippets/resource_form.html index b93fa48..a4d7d18 100644 --- a/ckanext/right_time_context/templates/package/snippets/resource_form.html +++ b/ckanext/right_time_context/templates/package/snippets/resource_form.html @@ -119,9 +119,9 @@ }; const updatePayloadField = function updatePayloadField() { - let url, field = $("#field-image-url"); + let url, field = $("#field-resource-url"); - if ($("#field-format").val() !== "fiware-ngsi" || field.val().trim() === "") { + if ($("#field-format").val() !== "fiware-ngsi") { $('.ngsiview-v1').addClass('hidden'); return; } @@ -155,9 +155,8 @@ if (e.val === "fiware-ngsi" || e.val === 'fiware-ngsi-registry') { $('.ngsiview-input').removeClass('hidden'); // Support the datastore plugin - $("#field-image-url").css("padding-right", "6px"); + $("#field-resource-url").css("padding-right", "6px"); $('.btn-remove-url').addClass('hidden'); - $("#field-image-upload + a + a")[0].click(); if (e.val === 'fiware-ngsi-registry') { // Activate registry specific fields @@ -168,7 +167,7 @@ } else { $('.ngsiview-input').addClass('hidden'); // Support the datastore plugin - $("#field-image-url").css("padding-right", ""); + $("#field-resource-url").css("padding-right", ""); $('.btn-remove-url').removeClass('hidden'); } }); @@ -185,16 +184,17 @@ // Support the datastore plugin setTimeout(function () { updatePayloadField(); - $("#field-image-url").css("padding-right", "6px"); + $("#field-resource-url").css("padding-right", "6px"); $('.btn-remove-url').addClass('hidden'); - $("#field-image-upload + a + a")[0].click(); + $("#field-resource-upload + a + a")[0].click(); }, 200); } setTimeout(function () { - $("#field-image-url").on("input", updatePayloadField); + $("#field-resource-url").on("input", updatePayloadField); }, 200); }); {% endblock basic_fields %} + diff --git a/ckanext/right_time_context/templates/package/snippets/resource_item.html b/ckanext/right_time_context/templates/package/snippets/resource_item.html index 457533f..fbcc132 100644 --- a/ckanext/right_time_context/templates/package/snippets/resource_item.html +++ b/ckanext/right_time_context/templates/package/snippets/resource_item.html @@ -1,25 +1,27 @@ {% ckan_extends %} {% block resource_item_explore_links %} +{% block explore_view %}
  • - + {% if res.has_views %} - + {{ _('Preview') }} {% else %} - + {{ _('More information') }} {% endif %}
    • + {% endblock explore_view %} {% if res.url and h.is_url(res.url) %} - {% if res.url.lower().find('/querycontext')== -1 and res.url.lower().find('/contextentities/')== -1 and res.url.lower().find('/entities/')== -1 and res.format.lower().find('fiware-ngsi-registry') == -1 %} + {% if res.url.lower().find('/querycontext')== -1 and res.url.lower().find('/contextentities/')== -1 and res.url.lower().find('/entities/')== -1 and res.format.lower().find('fiware-ngsi-registry') == -1 and res.format.lower().find('fiware-ngsi') == -1 %}
  • - + {% if res.has_views %} - + {{ _('Download') }} {% else %} - + {{ _('Go to resource') }} {% endif %} @@ -27,11 +29,9 @@ {% endif %} {% endif %} {% if can_edit %} -
  • - - - {{ _('Edit') }} - -
    • +
  • {% link_for _('Edit resource'), named_route=pkg.type ~ '_resource.edit', id=pkg.name, resource_id=res.id, class_='dropdown-item', icon='pencil' %}
    • + {% block resource_item_explore_inner scoped %}{% endblock %} +
  • {% link_for _('Views'), named_route=pkg.type ~ '_resource.views', id=pkg.name, resource_id=res.id, class_='dropdown-item', icon='chart-bar' %}
    • {% endif %} {% endblock %} + diff --git a/ckanext/right_time_context/tests/test_plugin.py b/ckanext/right_time_context/tests/test_plugin.py index bb40cf6..131f3dd 100644 --- a/ckanext/right_time_context/tests/test_plugin.py +++ b/ckanext/right_time_context/tests/test_plugin.py @@ -20,7 +20,11 @@ import json import unittest -from mock import ANY, DEFAULT, patch +try: + from unittest.mock import ANY, DEFAULT, patch +except ImportError: + from mock import ANY, DEFAULT, patch + from parameterized import parameterized from ckan.plugins.toolkit import ValidationError diff --git a/ckanext/right_time_context/tests/test_controller.py b/ckanext/right_time_context/tests/test_utils.py similarity index 84% rename from ckanext/right_time_context/tests/test_controller.py rename to ckanext/right_time_context/tests/test_utils.py index 7bd38e1..dc7ebe6 100644 --- a/ckanext/right_time_context/tests/test_controller.py +++ b/ckanext/right_time_context/tests/test_utils.py @@ -18,11 +18,13 @@ # along with ckanext-right_time_context. If not, see http://www.gnu.org/licenses/. import unittest - -from mock import ANY, DEFAULT, patch +try: + from unittest.mock import ANY, DEFAULT, patch +except ImportError: + from mock import ANY, DEFAULT, patch from parameterized import parameterized -from ckanext.right_time_context.controller import ProxyNGSIController +import ckanext.right_time_context.utils as utils class NgsiViewControllerTestCase(unittest.TestCase): @@ -48,7 +50,6 @@ class NgsiViewControllerTestCase(unittest.TestCase): @classmethod def setUpClass(cls): super(NgsiViewControllerTestCase, cls).setUpClass() - cls.controller = ProxyNGSIController() def _mock_response(self, req_method): body = '{"json": "body"}' @@ -67,7 +68,7 @@ def _mock_response(self, req_method): ({"service_path": "/a", 'format': 'fiware-ngsi'}, {"FIWARE-ServicePath": "/a"}), ({"tenant": "a", "service_path": "/a,/b", 'format': 'fiware-ngsi'}, {"FIWARE-Service": "a", "FIWARE-ServicePath": "/a,/b"}), ]) - @patch.multiple("ckanext.right_time_context.controller", base=DEFAULT, logic=DEFAULT, requests=DEFAULT, toolkit=DEFAULT, os=DEFAULT) + @patch.multiple("ckanext.right_time_context.utils", base=DEFAULT, logic=DEFAULT, requests=DEFAULT, toolkit=DEFAULT, os=DEFAULT) def test_basic_request(self, resource, headers, base, logic, requests, toolkit, os): resource['url'] = "http://cb.example.org/v2/entites" logic.get_action('resource_show').return_value = resource @@ -85,7 +86,7 @@ def test_basic_request(self, resource, headers, base, logic, requests, toolkit, } expected_headers.update(headers) - self.controller.proxy_ngsi_resource("resource_id") + utils.proxy_ngsi_resource("resource_id") requests.get.assert_called_with(resource['url'], headers=expected_headers, stream=True, verify=True) base.response.body_file.write.assert_called_with(body) @@ -103,7 +104,7 @@ def test_basic_request(self, resource, headers, base, logic, requests, toolkit, 'attrs': [], }, {}) ]) - @patch.multiple("ckanext.right_time_context.controller", base=DEFAULT, logic=DEFAULT, requests=DEFAULT, toolkit=DEFAULT, os=DEFAULT) + @patch.multiple("ckanext.right_time_context.utils", base=DEFAULT, logic=DEFAULT, requests=DEFAULT, toolkit=DEFAULT, os=DEFAULT) def test_registration_request(self, resource, query, headers, base, logic, requests, toolkit, os): logic.get_action('resource_show').return_value = resource response, body = self._mock_response(requests.post()) @@ -121,13 +122,13 @@ def test_registration_request(self, resource, query, headers, base, logic, reque } expected_headers.update(headers) - self.controller.proxy_ngsi_resource("resource_id") + utils.proxy_ngsi_resource("resource_id") url = resource['url'] + '/v2/op/query' requests.post.assert_called_with(url, headers=expected_headers, json=query, stream=True, verify=True) base.response.body_file.write.assert_called_with(body) - @patch.multiple("ckanext.right_time_context.controller", base=DEFAULT, logic=DEFAULT, requests=DEFAULT, toolkit=DEFAULT, os=DEFAULT) + @patch.multiple("ckanext.right_time_context.utils", base=DEFAULT, logic=DEFAULT, requests=DEFAULT, toolkit=DEFAULT, os=DEFAULT) def test_invalid_expression(self, base, logic, requests, toolkit, os): resource = { 'format': 'fiware-ngsi-registry', @@ -146,10 +147,10 @@ def test_invalid_expression(self, base, logic, requests, toolkit, os): logic.get_action('resource_show').return_value = resource - self.controller.proxy_ngsi_resource("resource_id") + utils.proxy_ngsi_resource("resource_id") base.abort.assert_called_with(422, detail='The expression is not a valid one for NGSI Registration, only georel, geometry, and coords is supported') - @patch.multiple("ckanext.right_time_context.controller", base=DEFAULT, logic=DEFAULT, requests=DEFAULT, toolkit=DEFAULT, os=DEFAULT) + @patch.multiple("ckanext.right_time_context.utils", base=DEFAULT, logic=DEFAULT, requests=DEFAULT, toolkit=DEFAULT, os=DEFAULT) def test_invalid_reg_query(self, base, logic, requests, toolkit, os): logic.get_action('resource_show').return_value = self.REGISTRY_RESOURCE response, body = self._mock_response(requests.post()) @@ -161,7 +162,7 @@ def test_invalid_reg_query(self, base, logic, requests, toolkit, os): response.status_code = 400 response.json.return_value = {'description': err_msg} - self.controller.proxy_ngsi_resource("resource_id") + utils.proxy_ngsi_resource("resource_id") base.abort.assert_called_with(422, detail=err_msg) @parameterized.expand([ @@ -171,7 +172,7 @@ def test_invalid_reg_query(self, base, logic, requests, toolkit, os): ("ftp://example.com",), ("tatata:///da",), ]) - @patch.multiple("ckanext.right_time_context.controller", base=DEFAULT, logic=DEFAULT, requests=DEFAULT, toolkit=DEFAULT, os=DEFAULT) + @patch.multiple("ckanext.right_time_context.utils", base=DEFAULT, logic=DEFAULT, requests=DEFAULT, toolkit=DEFAULT, os=DEFAULT) def test_invalid_url_request(self, url, base, logic, requests, toolkit, os=DEFAULT): resource = { 'url': url, @@ -180,7 +181,7 @@ def test_invalid_url_request(self, url, base, logic, requests, toolkit, os=DEFAU base.abort.side_effect = TypeError with self.assertRaises(TypeError): - self.controller.proxy_ngsi_resource("resource_id") + utils.proxy_ngsi_resource("resource_id") base.abort.assert_called_with(409, detail=ANY) requests.get.assert_not_called() @@ -189,7 +190,7 @@ def test_invalid_url_request(self, url, base, logic, requests, toolkit, os=DEFAU (True,), (False,), ]) - @patch.multiple("ckanext.right_time_context.controller", base=DEFAULT, logic=DEFAULT, requests=DEFAULT, toolkit=DEFAULT, os=DEFAULT) + @patch.multiple("ckanext.right_time_context.utils", base=DEFAULT, logic=DEFAULT, requests=DEFAULT, toolkit=DEFAULT, os=DEFAULT) def test_auth_required_request(self, auth_configured, base, logic, requests, toolkit, os): resource = { 'url': "http://cb.example.org/v2/entites", @@ -206,7 +207,7 @@ def test_auth_required_request(self, auth_configured, base, logic, requests, too } with self.assertRaises(TypeError): - self.controller.proxy_ngsi_resource("resource_id") + utils.proxy_ngsi_resource("resource_id") base.abort.assert_called_once_with(409, detail=ANY) requests.get.assert_called_once_with(resource['url'], headers=ANY, stream=True, verify=True) @@ -220,7 +221,7 @@ def test_auth_required_request(self, auth_configured, base, logic, requests, too ("ConnectionError", 502), ("Timeout", 504), ]) - @patch.multiple("ckanext.right_time_context.controller", base=DEFAULT, logic=DEFAULT, requests=DEFAULT, toolkit=DEFAULT, os=DEFAULT) + @patch.multiple("ckanext.right_time_context.utils", base=DEFAULT, logic=DEFAULT, requests=DEFAULT, toolkit=DEFAULT, os=DEFAULT) def test_auth_required_request(self, exception, status_code, base, logic, requests, toolkit, os): resource = { 'url': "http://cb.example.org/v2/entites", @@ -232,7 +233,7 @@ def test_auth_required_request(self, exception, status_code, base, logic, reques base.abort.side_effect = TypeError with self.assertRaises(TypeError): - self.controller.proxy_ngsi_resource("resource_id") + utils.proxy_ngsi_resource("resource_id") base.abort.assert_called_once_with(status_code, detail=ANY) requests.get.assert_called_once_with(resource['url'], headers=ANY, stream=True, verify=True) @@ -256,7 +257,7 @@ def test_auth_required_request(self, exception, status_code, base, logic, reques ({"CKAN_RIGHT_TIME_CONTEXT_VERIFY_REQUESTS": " "}, {"ckan.verify_requests": False}, False), ({"CKAN_VERIFY_REQUESTS": "/path/A/b"}, {"ckan.verify_requests": "path/2"}, "/path/A/b"), ]) - @patch.multiple("ckanext.right_time_context.controller", base=DEFAULT, logic=DEFAULT, requests=DEFAULT, toolkit=DEFAULT, os=DEFAULT) + @patch.multiple("ckanext.right_time_context.utils", base=DEFAULT, logic=DEFAULT, requests=DEFAULT, toolkit=DEFAULT, os=DEFAULT) def test_verify_requests(self, env, config, expected_value, base, logic, requests, toolkit, os): logic.get_action('resource_show').return_value = { 'url': "https://cb.example.org/v2/entites", @@ -265,6 +266,8 @@ def test_verify_requests(self, env, config, expected_value, base, logic, request os.environ = env toolkit.config = config - with patch.object(self.controller, '_proxy_query_resource') as query_mock: - self.controller.proxy_ngsi_resource("resource_id") + with patch.object(utils, 'proxy_query_resource') as query_mock: + utils.proxy_ngsi_resource("resource_id") query_mock.assert_called_once_with(ANY, ANY, ANY, verify=expected_value) + + diff --git a/ckanext/right_time_context/utils.py b/ckanext/right_time_context/utils.py new file mode 100644 index 0000000..75f7930 --- /dev/null +++ b/ckanext/right_time_context/utils.py @@ -0,0 +1,218 @@ +# -*- coding: utf-8 -*- + +import json +import logging +from six.moves.urllib.parse import urlsplit, urljoin +import os + +import requests +import six + +import ckan.lib.base as base +import ckan.logic as logic + +from ckan.plugins import toolkit + + +log = logging.getLogger(__name__) + +CHUNK_SIZE = 512 + +NGSI_REG_FORMAT = 'fiware-ngsi-registry' + + +def proxy_query_resource(resource, parsed_url, headers, verify=True): + if parsed_url.path.lower().find('/querycontext') != -1: + if resource.get("payload", "").strip() == "": + details = 'Please add a payload to complete the query.' + base.abort(409, detail=details) + + try: + json.loads(resource['payload']) + except json.JSONDecodeError: + details = "Payload field doesn't contain valid JSON data." + base.abort(409, detail=details) + + headers['Content-Type'] = "application/json" + r = requests.post(resource['url'], headers=headers, data=resource["payload"], stream=True, verify=verify) + + else: + r = requests.get(resource['url'], headers=headers, stream=True, verify=verify) + + return r + + +def proxy_registration_resource(resource, parsed_url, headers, verify=True): + path = parsed_url.path + + if path.endswith('/'): + path = path[:-1] + path = path + '/v2/op/query' + attrs = [] + + if 'attrs_str' in resource and len(resource['attrs_str']): + attrs = resource['attrs_str'].split(',') + body = { + 'entities': [], + 'attrs': attrs + } + + # Include entity information + for entity in resource['entity']: + query_entity = { + 'type': entity['value'] + } + if 'isPattern' in entity and entity['isPattern'] == 'on': + query_entity['idPattern'] = entity['id'] + else: + query_entity['id'] = entity['id'] + + body['entities'].append(query_entity) + + # Parse expression to include georel information + if 'expression' in resource and len(resource['expression']): + # Separate expresion query strings + supported_expressions = ['georel', 'geometry', 'coords'] + parsed_expression = resource['expression'].split('&') + + expression = {} + for exp in parsed_expression: + parsed_exp = exp.split('=') + + if len(parsed_exp) != 2 or not parsed_exp[0] in supported_expressions: + base.abort(422, detail='The expression is not a valid one for NGSI Registration, only georel, geometry, and coords is supported') + else: + expression[parsed_exp[0]] = parsed_exp[1] + + body['expression'] = expression + + headers['Content-Type'] = 'application/json' + url = urljoin(parsed_url.scheme + '://' + parsed_url.netloc, path) + log.info('Url: {}'.format(url)) + response = requests.post(url, headers=headers, json=body, stream=True, verify=verify) + + return response + + +def process_auth_credentials(resource, headers): + auth_method = resource.get('auth_type', 'none') + + if auth_method == "oauth2": + token = toolkit.c.usertoken['access_token'] + headers['Authorization'] = "Bearer %s" % token + elif auth_method == "x-auth-token-fiware": + # Deprecated method, Including OAuth2 token retrieved from the IdM + # on the Open Stack X-Auth-Token header + token = toolkit.c.usertoken['access_token'] + headers['X-Auth-Token'] = token + + +def proxy_ngsi_resource(resource_id): + # Chunked proxy for ngsi resources. + context = {'user': toolkit.c.user or toolkit.c.author} + + log.info('Proxify resource {id}'.format(id=resource_id)) + resource = logic.get_action('resource_show')(context, {'id': resource_id}) + + headers = { + 'Accept': 'application/json' + } + + resource.setdefault('auth_type', 'none') + process_auth_credentials(resource, headers) + + if resource.get('tenant', '') != '': + headers['FIWARE-Service'] = resource['tenant'] + if resource.get('service_path', '') != '': + headers['FIWARE-ServicePath'] = resource['service_path'] + + url = resource['url'] + parsed_url = urlsplit(url) + + if parsed_url.scheme not in ("http", "https") or not parsed_url.netloc: + base.abort(409, detail='Invalid URL.') + + # Process verify configuration + verify_conf = os.environ.get('CKAN_RIGHT_TIME_CONTEXT_VERIFY_REQUESTS', + toolkit.config.get('ckan.right_time_context.verify_requests')) + if verify_conf is None or (isinstance(verify_conf, six.string_types) and verify_conf.strip() == ""): + verify_conf = os.environ.get('CKAN_VERIFY_REQUESTS', toolkit.config.get('ckan.verify_requests')) + + if isinstance(verify_conf, six.string_types) and verify_conf.strip() != "": + compare_env = verify_conf.lower().strip() + if compare_env in ("true", "1", "on"): + verify = True + elif compare_env in ("false", "0", "off"): + verify = False + else: + verify = verify_conf + elif isinstance(verify_conf, bool): + verify = verify_conf + else: + verify = True + + # Make the request to the server + try: + if resource['format'].lower() == NGSI_REG_FORMAT: + r = proxy_registration_resource(resource, parsed_url, headers, verify=verify) + else: + r = proxy_query_resource(resource, parsed_url, headers, verify=verify) + + except requests.HTTPError: + details = 'Could not proxy ngsi_resource. We are working to resolve this issue as quickly as possible' + base.abort(409, detail=details) + except requests.ConnectionError: + details = 'Could not proxy ngsi_resource because a connection error occurred.' + base.abort(502, detail=details) + except requests.Timeout: + details = 'Could not proxy ngsi_resource because the connection timed out.' + base.abort(504, detail=details) + + if r.status_code == 401: + if resource.get('auth_type', 'none') != 'none': + details = 'ERROR 401 token expired. Retrieving new token, reload please.' + log.info(details) + toolkit.c.usertoken_refresh() + base.abort(409, detail=details) + elif resource.get('auth_type', 'none') == 'none': + details = 'Authentication requested by server, please check resource configuration.' + log.info(details) + base.abort(409, detail=details) + + elif r.status_code == 400: + response = r.json() + details = response['description'] + log.info(details) + base.abort(422, detail=details) + + else: + r.raise_for_status() + if toolkit.check_ckan_version("2.10.1"): + from flask import make_response + response = make_response() + else: + response = toolkit.response + response.content_type = r.headers['content-type'] + response.charset = r.encoding + #base.response.content_type = r.headers['content-type'] + #base.response.charset = r.encoding + + for chunk in r.iter_content(chunk_size=CHUNK_SIZE): + if toolkit.check_ckan_version("2.10.1"): + response.data += chunk + else: + response.body_file.write(chunk) + #base.response.body_file.write(chunk) + return response + +def get_proxified_ngsi_url(data_dict): + log.info('in plugin utils get_proxified_ngsi_url data_dict {0}'.format(data_dict)) + url = toolkit.url_for( + action='proxy_ngsi', + controller='ngsiproxy', + id=data_dict['package']['name'], + resource_id=data_dict['resource']['id'] + ) + log.info('Proxified url is {0}'.format(url)) + return url + diff --git a/ckanext/right_time_context/views.py b/ckanext/right_time_context/views.py new file mode 100644 index 0000000..052f802 --- /dev/null +++ b/ckanext/right_time_context/views.py @@ -0,0 +1,29 @@ +# -*- coding: utf-8 -*- + +import logging + +from flask import Blueprint + +import ckan.lib.base as base + +from ckan import plugins as p +from ckan.plugins import toolkit + +import ckanext.right_time_context.utils as utils + +log = logging.getLogger(__name__) +ngsiproxy = Blueprint("ngsiproxy", __name__) + + +def proxy_ngsi(id, resource_id): + return utils.proxy_ngsi_resource(resource_id) + + +def get_blueprints(): + return [ngsiproxy] + + +ngsiproxy.add_url_rule( + "/dataset//resource//ngsiproxy", + view_func=proxy_ngsi, +) diff --git a/setup.py b/setup.py index 5b55c7f..a79e00e 100644 --- a/setup.py +++ b/setup.py @@ -37,6 +37,8 @@ def read(fname): 'License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)', 'Programming Language :: Python :: 2', 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.10', ], packages=find_packages(exclude=['contrib', 'docs', 'tests*']), install_requires=[],