From db19a92eb31c393628a613ec9d2d88e6f8b26003 Mon Sep 17 00:00:00 2001 From: Paolo Tormene Date: Wed, 15 Jan 2020 16:10:08 +0100 Subject: [PATCH 01/33] Enable setting profiles to connect to instances of the OQ-GeoViewer and check that authentication works --- svir/dialogs/connection_profile_dialog.py | 36 +- svir/dialogs/settings_dialog.py | 163 ++++-- svir/ui/ui_settings.ui | 668 ++++++++++++---------- svir/utilities/shared.py | 4 + svir/utilities/utils.py | 45 +- 5 files changed, 537 insertions(+), 379 deletions(-) diff --git a/svir/dialogs/connection_profile_dialog.py b/svir/dialogs/connection_profile_dialog.py index 9affe7d20..138de3184 100644 --- a/svir/dialogs/connection_profile_dialog.py +++ b/svir/dialogs/connection_profile_dialog.py @@ -29,6 +29,7 @@ from svir.utilities.utils import get_ui_class from svir.utilities.shared import ( DEFAULT_PLATFORM_PROFILES, + DEFAULT_GEOVIEWER_PROFILES, DEFAULT_ENGINE_PROFILES, ) @@ -38,24 +39,23 @@ class ConnectionProfileDialog(QDialog, FORM_CLASS): """ Dialog used to create/edit a connection profile to let the plugin interact - with the OpenQuake Platform or the OpenQuake Engine + with the OpenQuake Platform, the OpenQuake Engine, or the OpenQuake + GeoViewer """ - def __init__(self, platform_or_engine, profile_name='', parent=None): + def __init__(self, server, profile_name='', parent=None): QDialog.__init__(self, parent) - assert platform_or_engine in ('platform', 'engine'), platform_or_engine + assert server in ('platform', 'engine', 'geoviewer'), server # Set up the user interface from Designer. self.setupUi(self) - self.platform_or_engine = platform_or_engine + self.server = server self.initial_profile_name = profile_name if self.initial_profile_name: profiles = json.loads( QSettings().value( - 'irmt/%s_profiles' % self.platform_or_engine, - (DEFAULT_PLATFORM_PROFILES - if self.platform_or_engine == 'platform' - else DEFAULT_ENGINE_PROFILES))) + 'irmt/%s_profiles' % self.server, + self.get_default_profiles())) profile = profiles[self.initial_profile_name] self.profile_name_edt.setText(self.initial_profile_name) self.username_edt.setText(profile['username']) @@ -64,6 +64,15 @@ def __init__(self, platform_or_engine, profile_name='', parent=None): self.ok_button = self.buttonBox.button(QDialogButtonBox.Ok) self.ok_button.setEnabled(self.initial_profile_name != '') + def get_default_profiles(self): + if self.server == 'platform': + default_profiles = DEFAULT_PLATFORM_PROFILES + elif self.server == 'geoviewer': + default_profiles = DEFAULT_GEOVIEWER_PROFILES + else: # engine + default_profiles = DEFAULT_ENGINE_PROFILES + return default_profiles + @pyqtSlot(str) def on_profile_name_edt_textEdited(self, profile_name): self.ok_button.setEnabled(profile_name != '') @@ -72,20 +81,17 @@ def accept(self): # if the (stripped) hostname ends with '/', remove it hostname = self.hostname_edt.text().strip().rstrip('/') # if the (stripped) engine hostname ends with '/engine/', remove it - if self.platform_or_engine == 'engine': + if self.server == 'engine': hostname = ( hostname[:-7] if hostname.endswith('/engine') else hostname) edited_profile_name = self.profile_name_edt.text() mySettings = QSettings() mySettings.setValue( - 'irmt/current_%s_profile' % self.platform_or_engine, + 'irmt/current_%s_profile' % self.server, edited_profile_name) profiles = json.loads( mySettings.value( - 'irmt/%s_profiles' % self.platform_or_engine, - (DEFAULT_PLATFORM_PROFILES - if self.platform_or_engine == 'platform' - else DEFAULT_ENGINE_PROFILES))) + 'irmt/%s_profiles' % self.server, self.get_default_profiles())) profiles[edited_profile_name] = { 'username': self.username_edt.text(), 'password': self.password_edt.text(), @@ -94,6 +100,6 @@ def accept(self): if self.initial_profile_name in profiles: del profiles[self.initial_profile_name] mySettings.setValue( - 'irmt/%s_profiles' % self.platform_or_engine, + 'irmt/%s_profiles' % self.server, json.dumps(profiles)) super(ConnectionProfileDialog, self).accept() diff --git a/svir/dialogs/settings_dialog.py b/svir/dialogs/settings_dialog.py index 72da78b5a..b7b96b873 100644 --- a/svir/dialogs/settings_dialog.py +++ b/svir/dialogs/settings_dialog.py @@ -37,6 +37,7 @@ get_ui_class, get_style, platform_login, + geoviewer_login, engine_login, log_msg, WaitCursorManager, @@ -46,6 +47,7 @@ PLATFORM_REGISTRATION_URL, DEFAULT_SETTINGS, DEFAULT_PLATFORM_PROFILES, + DEFAULT_GEOVIEWER_PROFILES, DEFAULT_ENGINE_PROFILES, LOG_LEVELS, ) @@ -104,6 +106,7 @@ def restore_state(self, restore_defaults=False): mySettings = QSettings() self.refresh_profile_cbxs('platform', restore_defaults) + self.refresh_profile_cbxs('geoviewer', restore_defaults) self.refresh_profile_cbxs('engine', restore_defaults) developer_mode = (DEFAULT_SETTINGS['developer_mode'] @@ -141,36 +144,42 @@ def restore_state(self, restore_defaults=False): self.developer_mode_ckb.setChecked(developer_mode) self.enable_experimental_ckb.setChecked(experimental_enabled) - def refresh_profile_cbxs(self, platform_or_engine, restore_defaults=False): - assert platform_or_engine in ('platform', 'engine'), platform_or_engine - if platform_or_engine == 'platform': + def refresh_profile_cbxs(self, server, restore_defaults=False): + assert server in ('platform', 'engine', 'geoviewer'), server + if server == 'platform': self.platform_profile_cbx.blockSignals(True) self.platform_profile_cbx.clear() self.platform_profile_cbx.blockSignals(False) + default_profiles = DEFAULT_PLATFORM_PROFILES + elif server == 'geoviewer': + self.geoviewer_profile_cbx.blockSignals(True) + self.geoviewer_profile_cbx.clear() + self.geoviewer_profile_cbx.blockSignals(False) + default_profiles = DEFAULT_GEOVIEWER_PROFILES else: # 'engine' self.engine_profile_cbx.blockSignals(True) self.engine_profile_cbx.clear() self.engine_profile_cbx.blockSignals(False) + default_profiles = DEFAULT_ENGINE_PROFILES mySettings = QSettings() if restore_defaults: - profiles = json.loads( - DEFAULT_PLATFORM_PROFILES if platform_or_engine == 'platform' - else DEFAULT_ENGINE_PROFILES) + profiles = json.loads(default_profiles) cur_profile = list(profiles.keys())[0] else: profiles = json.loads( mySettings.value( - 'irmt/%s_profiles' % platform_or_engine, - (DEFAULT_PLATFORM_PROFILES - if platform_or_engine == 'platform' - else DEFAULT_ENGINE_PROFILES))) + 'irmt/%s_profiles' % server, default_profiles)) cur_profile = mySettings.value( - 'irmt/current_%s_profile' % platform_or_engine) + 'irmt/current_%s_profile' % server) for profile in sorted(profiles, key=str.lower): - if platform_or_engine == 'platform': + if server == 'platform': self.platform_profile_cbx.blockSignals(True) self.platform_profile_cbx.addItem(profile) self.platform_profile_cbx.blockSignals(False) + elif server == 'geoviewer': + self.geoviewer_profile_cbx.blockSignals(True) + self.geoviewer_profile_cbx.addItem(profile) + self.geoviewer_profile_cbx.blockSignals(False) else: # engine self.engine_profile_cbx.blockSignals(True) self.engine_profile_cbx.addItem(profile) @@ -178,18 +187,21 @@ def refresh_profile_cbxs(self, platform_or_engine, restore_defaults=False): if cur_profile is None: cur_profile = list(profiles.keys())[0] mySettings.setValue( - 'irmt/current_%s_profile' % platform_or_engine, + 'irmt/current_%s_profile' % server, cur_profile) - if platform_or_engine == 'platform': + if server == 'platform': self.platform_profile_cbx.setCurrentIndex( self.platform_profile_cbx.findText(cur_profile)) - else: # engine - self.engine_profile_cbx.setCurrentIndex( - self.engine_profile_cbx.findText(cur_profile)) - if platform_or_engine == 'platform': self.pla_remove_btn.setEnabled( self.platform_profile_cbx.count() > 1) + elif server == 'geoviewer': + self.geoviewer_profile_cbx.setCurrentIndex( + self.geoviewer_profile_cbx.findText(cur_profile)) + self.gv_remove_btn.setEnabled( + self.geoviewer_profile_cbx.count() > 1) else: # engine + self.engine_profile_cbx.setCurrentIndex( + self.engine_profile_cbx.findText(cur_profile)) self.eng_remove_btn.setEnabled( self.engine_profile_cbx.count() > 1) @@ -211,11 +223,15 @@ def save_state(self): self.log_level_cbx.itemData(self.log_level_cbx.currentIndex())) cur_pla_profile = self.platform_profile_cbx.currentText() + cur_gv_profile = self.geoviewer_profile_cbx.currentText() cur_eng_profile = self.engine_profile_cbx.currentText() platform_profiles = json.loads(mySettings.value( 'irmt/platform_profiles', DEFAULT_PLATFORM_PROFILES)) platform_profile = platform_profiles[cur_pla_profile] + geoviewer_profiles = json.loads(mySettings.value( + 'irmt/geoviewer_profiles', DEFAULT_GEOVIEWER_PROFILES)) + geoviewer_profile = geoviewer_profiles[cur_gv_profile] engine_profiles = json.loads(mySettings.value( 'irmt/engine_profiles', DEFAULT_ENGINE_PROFILES)) engine_profile = engine_profiles[cur_eng_profile] @@ -226,6 +242,12 @@ def save_state(self): platform_profile['username']) mySettings.setValue('irmt/platform_password', platform_profile['password']) + mySettings.setValue('irmt/geoviewer_hostname', + geoviewer_profile['hostname']) + mySettings.setValue('irmt/geoviewer_username', + geoviewer_profile['username']) + mySettings.setValue('irmt/geoviewer_password', + geoviewer_profile['password']) mySettings.setValue('irmt/engine_hostname', engine_profile['hostname']) mySettings.setValue('irmt/engine_username', @@ -285,52 +307,80 @@ def on_platform_profile_cbx_currentIndexChanged(self, idx): profile = self.platform_profile_cbx.itemText(idx) QSettings().setValue('irmt/current_platform_profile', profile) + @pyqtSlot(int) + def on_geoviewer_profile_cbx_currentIndexChanged(self, idx): + profile = self.geoviewer_profile_cbx.itemText(idx) + QSettings().setValue('irmt/current_geoviewer_profile', profile) + @pyqtSlot(int) def on_engine_profile_cbx_currentIndexChanged(self, idx): profile = self.engine_profile_cbx.itemText(idx) QSettings().setValue('irmt/current_engine_profile', profile) - @pyqtSlot() - def on_pla_edit_btn_clicked(self): - profile_name = self.platform_profile_cbx.currentText() + def edit_profile(self, profile_name, server): self.profile_dlg = ConnectionProfileDialog( - 'platform', profile_name, parent=self) + server, profile_name, parent=self) + if self.profile_dlg.exec_(): + self.refresh_profile_cbxs(server) + + def new_profile(self, server): + self.profile_dlg = ConnectionProfileDialog(server, parent=self) if self.profile_dlg.exec_(): - self.refresh_profile_cbxs('platform') + self.refresh_profile_cbxs(server) + + def test_profile(self, profile_name, server): + with WaitCursorManager('Logging in...', self.message_bar): + self._attempt_login(server, profile_name) + + @pyqtSlot() + def on_pla_edit_btn_clicked(self): + self.edit_profile( + self.platform_profile_cbx.currentText(), 'platform') + + @pyqtSlot() + def on_gv_edit_btn_clicked(self): + self.edit_profile( + self.geoviewer_profile_cbx.currentText(), 'geoviewer') @pyqtSlot() def on_eng_edit_btn_clicked(self): - profile_name = self.engine_profile_cbx.currentText() - self.profile_dlg = ConnectionProfileDialog( - 'engine', profile_name, parent=self) - if self.profile_dlg.exec_(): - self.refresh_profile_cbxs('engine') + self.edit_profile( + self.engine_profile_cbx.currentText(), 'engine') @pyqtSlot() def on_pla_new_btn_clicked(self): - self.profile_dlg = ConnectionProfileDialog( - 'platform', parent=self) - if self.profile_dlg.exec_(): - self.refresh_profile_cbxs('platform') + self.new_profile('platform') + + @pyqtSlot() + def on_gv_new_btn_clicked(self): + self.new_profile('geoviewer') + + @pyqtSlot() + def on_eng_new_btn_clicked(self): + self.new_profile('engine') @pyqtSlot() def on_pla_test_btn_clicked(self): - server = 'platform' - profile_name = self.platform_profile_cbx.currentText() - with WaitCursorManager('Logging in...', self.message_bar): - self._attempt_login(server, profile_name) + self.test_profile( + self.platform_profile_cbx.currentText(), 'platform') + + @pyqtSlot() + def on_gv_test_btn_clicked(self): + self.test_profile( + self.geoviewer_profile_cbx.currentText(), 'geoviewer') @pyqtSlot() def on_eng_test_btn_clicked(self): - server = 'engine' - profile_name = self.engine_profile_cbx.currentText() - with WaitCursorManager('Logging in...', self.message_bar): - self._attempt_login(server, profile_name) + self.test_profile( + self.engine_profile_cbx.currentText(), 'engine') def _attempt_login(self, server, profile_name): if server == 'platform': default_profiles = DEFAULT_PLATFORM_PROFILES login_func = platform_login + elif server == 'geoviewer': + default_profiles = DEFAULT_GEOVIEWER_PROFILES + login_func = geoviewer_login elif server == 'engine': default_profiles = DEFAULT_ENGINE_PROFILES login_func = engine_login @@ -368,23 +418,20 @@ def _attempt_login(self, server, profile_name): msg = 'Able to connect' log_msg(msg, level='S', message_bar=self.message_bar, duration=3) - @pyqtSlot() - def on_eng_new_btn_clicked(self): - self.profile_dlg = ConnectionProfileDialog( - 'engine', parent=self) - if self.profile_dlg.exec_(): - self.refresh_profile_cbxs('engine') - @pyqtSlot() def on_pla_remove_btn_clicked(self): self.remove_selected_profile('platform') + @pyqtSlot() + def on_gv_remove_btn_clicked(self): + self.remove_selected_profile('geoviewer') + @pyqtSlot() def on_eng_remove_btn_clicked(self): self.remove_selected_profile('engine') - def remove_selected_profile(self, platform_or_engine): - assert platform_or_engine in ('platform', 'engine'), platform_or_engine + def remove_selected_profile(self, server): + assert server in ('platform', 'geoviewer', 'engine'), server if QMessageBox.question( self, 'Remove connection profile', @@ -393,19 +440,21 @@ def remove_selected_profile(self, platform_or_engine): QMessageBox.Yes | QMessageBox.No) != QMessageBox.Yes: return profiles = json.loads( - QSettings().value('irmt/%s_profiles' % platform_or_engine)) - if platform_or_engine == 'platform': + QSettings().value('irmt/%s_profiles' % server)) + if server == 'platform': cur_profile = self.platform_profile_cbx.currentText() + elif server == 'geoviewer': + cur_profile = self.geoviewer_profile_cbx.currentText() else: # engine cur_profile = self.engine_profile_cbx.currentText() del profiles[cur_profile] - QSettings().remove('irmt/current_%s_profile' % platform_or_engine) - self.save_profiles(platform_or_engine, profiles) - self.refresh_profile_cbxs(platform_or_engine) + QSettings().remove('irmt/current_%s_profile' % server) + self.save_profiles(server, profiles) + self.refresh_profile_cbxs(server) - def save_profiles(self, platform_or_engine, profiles): - assert platform_or_engine in ('platform', 'engine'), platform_or_engine - QSettings().setValue('irmt/%s_profiles' % platform_or_engine, + def save_profiles(self, server, profiles): + assert server in ('platform', 'geoviewer', 'engine'), server + QSettings().setValue('irmt/%s_profiles' % server, json.dumps(profiles)) def select_color(self, button): diff --git a/svir/ui/ui_settings.ui b/svir/ui/ui_settings.ui index e8fab1919..dd427f222 100644 --- a/svir/ui/ui_settings.ui +++ b/svir/ui/ui_settings.ui @@ -6,8 +6,8 @@ 0 0 - 451 - 753 + 460 + 857 @@ -19,313 +19,379 @@ - - - OpenQuake Platform connection profile - - - - QFormLayout::AllNonFixedFieldsGrow - - - - - - - - 0 - - - - - Test connection - - - - - - - Remove - - - - - - - New - - - - - - - Edit - - - - - - - - - - - - Link to the OQ-Platform registration - - - Qt::AlignCenter - - + + true - - Qt::TextBrowserInteraction - - - - - - - OpenQuake Engine connection profile - - - - QFormLayout::AllNonFixedFieldsGrow + + + + 0 + 0 + 440 + 804 + - - - - - - - 0 - - - - - Test connection + + + + + OpenQuake GeoViewer connection profile + + + + + + + + + 0 + + + + + Test connection + + + + + + + Remove + + + + + + + New + + + + + + + Edit + + + + + + + + + + + + OpenQuake Platform connection profile + + + + QFormLayout::AllNonFixedFieldsGrow - - - - - - Remove + + + + + + + 0 + + + + + Test connection + + + + + + + Remove + + + + + + + New + + + + + + + Edit + + + + + + + + + + + + Link to the OQ-Platform registration + + + Qt::AlignCenter + + + true + + + Qt::TextBrowserInteraction + + + + + + + OpenQuake Engine connection profile + + + + QFormLayout::AllNonFixedFieldsGrow - - - - - - New - - - - - - - Edit - - - - - - - - - - - - Rule-based classification settings - - - - QFormLayout::AllNonFixedFieldsGrow - - - - - Color from - - - - - - - - 100 - 16777215 - - - - false - - - - - - - - - - Color to - - - - - - - - 100 - 16777215 - - - - false - - - - - - false - - - false - - - - - - - Mode - - - - - - - - - - Classes - - - - - - - 1 - - - 10 - - - - - - - - - - - 0 - 0 - - - - Project-related settings - - - - - - Restyle layer after recalculating composite indices - - - true - - - - - - - - - - - 0 - 0 - - - - Advanced settings - - - - - - Enable experimental features (requires restart) - - - - - - - It is recommended to keep it unchecked - - - Developer mode (requires restart) - - - - - - - 0 - - - - - Log level + + + + + + + 0 + + + + + Test connection + + + + + + + Remove + + + + + + + New + + + + + + + Edit + + + + + + + + + + + + Rule-based classification settings + + + + QFormLayout::AllNonFixedFieldsGrow - - - - - - - - + + + + Color from + + + + + + + + 100 + 16777215 + + + + false + + + + + + + + + + Color to + + + + + + + + 100 + 16777215 + + + + false + + + + + + false + + + false + + + + + + + Mode + + + + + + + + + + Classes + + + + + + + 1 + + + 10 + + + + + + + + + + + 0 + 0 + + + + Project-related settings + + + + + + Restyle layer after recalculating composite indices + + + true + + + + + + + + + + + 0 + 0 + + + + Advanced settings + + + + + + Enable experimental features (requires restart) + + + + + + + It is recommended to keep it unchecked + + + Developer mode (requires restart) + + + + + + + 0 + + + + + Log level + + + + + + + + + + + + + + + 0 + + + + + Restore default settings + + + + + + + Qt::Horizontal + + + + 40 + 20 + + + + + + + + - - - - 0 - - - - - Restore default settings - - - - - - - Qt::Horizontal - - - - 40 - 20 - - - - - - diff --git a/svir/utilities/shared.py b/svir/utilities/shared.py index 9fb72a530..a329ca974 100644 --- a/svir/utilities/shared.py +++ b/svir/utilities/shared.py @@ -271,6 +271,10 @@ def __repr__(self): '{"OpenQuake Platform": {' '"username": "", "password": "",' '"hostname": "https://platform.openquake.org"}}') +DEFAULT_GEOVIEWER_PROFILES = ( + '{"Local OpenQuake GeoViewer": {' + '"username": "", "password": "",' + '"hostname": "http://localhost:8000"}}') DEFAULT_ENGINE_PROFILES = ( '{"Local OpenQuake Engine Server": {' '"username": "", "password": "",' diff --git a/svir/utilities/utils.py b/svir/utilities/utils.py index 302e705ae..696937081 100644 --- a/svir/utilities/utils.py +++ b/svir/utilities/utils.py @@ -37,6 +37,7 @@ from copy import deepcopy from time import time from pprint import pformat +from requests.auth import HTTPBasicAuth from qgis.core import ( QgsProject, QgsMessageLog, @@ -67,6 +68,7 @@ DEBUG, DEFAULT_SETTINGS, DEFAULT_PLATFORM_PROFILES, + DEFAULT_GEOVIEWER_PROFILES, DEFAULT_ENGINE_PROFILES, ) @@ -521,6 +523,33 @@ def platform_login(host, username, password, session): _login(login_url, username, password, session) +def geoviewer_login(host, username, password, session): + """ + Logs in a session to a geoviewer + + :param host: The host url + :type host: str + :param username: The username + :type username: str + :param password: The password + :type password: str + :param session: The session to be autenticated + :type session: Session + """ + login_url = host + '/api/authenticate/' + try: + session_resp = session.post(login_url, + auth=HTTPBasicAuth(username, password), + timeout=10) + except Exception: + msg = "Unable to login. %s" % traceback.format_exc() + raise SvNetworkError(msg) + if session_resp.status_code != 200: # 200 means successful:OK + error_message = ('Unable to login: %s' % + session_resp.text) + raise SvNetworkError(error_message) + + def engine_login(host, username, password, session): """ Logs in a session to a engine server @@ -1105,21 +1134,25 @@ def warn_missing_package(package_name, message_bar=None): def get_credentials(server): """ - Get from the QSettings the credentials to access the OpenQuake Engine - or the OpenQuake Platform. + Get from the QSettings the credentials to access the OpenQuake Engine, + the OpenQuake Platform or the OpenQuake GeoViewer. If those settings are not found, use defaults instead. - :param server: it can be either 'platform' or 'engine' + :param server: it can be either 'platform', 'geoviewer' or 'engine' :returns: tuple (hostname, username, password) """ qs = QSettings() + if server == 'platform': + default_profiles = DEFAULT_PLATFORM_PROFILES + elif server == 'geoviewer': + default_profiles = DEFAULT_GEOVIEWER_PROFILES + elif server == 'engine': + default_profiles = DEFAULT_ENGINE_PROFILES default_profiles = json.loads( qs.value( - 'irmt/%s_profiles', - (DEFAULT_PLATFORM_PROFILES if server == 'platform' - else DEFAULT_ENGINE_PROFILES))) + 'irmt/%s_profiles', default_profiles)) default_profile = default_profiles[list(default_profiles.keys())[0]] hostname = qs.value('irmt/%s_hostname' % server, default_profile['hostname']) From 5c9ed0ae52af21653171b885eed40b850c857d3d Mon Sep 17 00:00:00 2001 From: Paolo Tormene Date: Thu, 16 Jan 2020 14:21:24 +0100 Subject: [PATCH 02/33] Set session.auth with username and password --- svir/utilities/utils.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/svir/utilities/utils.py b/svir/utilities/utils.py index 696937081..451c08ba4 100644 --- a/svir/utilities/utils.py +++ b/svir/utilities/utils.py @@ -37,7 +37,6 @@ from copy import deepcopy from time import time from pprint import pformat -from requests.auth import HTTPBasicAuth from qgis.core import ( QgsProject, QgsMessageLog, @@ -536,10 +535,10 @@ def geoviewer_login(host, username, password, session): :param session: The session to be autenticated :type session: Session """ + session.auth = (username, password) login_url = host + '/api/authenticate/' try: session_resp = session.post(login_url, - auth=HTTPBasicAuth(username, password), timeout=10) except Exception: msg = "Unable to login. %s" % traceback.format_exc() From e75a24716e5559adf865dbdec3076611f8e9da54 Mon Sep 17 00:00:00 2001 From: Paolo Tormene Date: Thu, 16 Jan 2020 14:21:53 +0100 Subject: [PATCH 03/33] Add a menu group and item to import a geoviwer project (WIP: just retrieving a list of projects) --- svir/irmt.py | 53 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/svir/irmt.py b/svir/irmt.py index a0037b9e1..1040d6642 100644 --- a/svir/irmt.py +++ b/svir/irmt.py @@ -30,9 +30,12 @@ import fileinput import re import processing +import json +import traceback from copy import deepcopy from math import floor, ceil +from requests import Session from qgis.core import ( QgsVectorLayer, QgsMapLayer, @@ -95,12 +98,14 @@ get_checksum, warn_missing_package, import_layer_from_csv, + geoviewer_login, ) from svir.utilities.shared import (DEBUG, PROJECT_TEMPLATE, THEME_TEMPLATE, INDICATOR_TEMPLATE, OQ_XMARKER_TYPES, + DEFAULT_GEOVIEWER_PROFILES, OPERATORS_DICT) from svir.ui.tool_button_with_help_link import QToolButtonWithHelpLink from svir.processing_provider.provider import Provider @@ -202,6 +207,15 @@ def initGui(self): self.menu_action = menu_bar.insertMenu( self.iface.firstRightStandardMenu().menuAction(), self.menu) + # Action to list OQ GeoViewer projects + self.add_menu_item("import_geoviewer_project", + ":/plugins/irmt/load_layer.svg", + u"Import project from the OpenQuake &GeoViewer", + self.import_geoviewer_project, + enable=True, + add_to_layer_actions=False, + submenu='OQ GeoViewer') + # Action to activate the modal dialog to import socioeconomic # data from the platform self.add_menu_item("import_sv_variables", @@ -798,6 +812,45 @@ def _data_download_successful( layer.setFieldAlias( field_idx, 'Country name') + def import_geoviewer_project(self): + """ + FIXME + """ + mySettings = QSettings() + profiles = json.loads(mySettings.value( + 'irmt/geoviewer_profiles', DEFAULT_GEOVIEWER_PROFILES)) + # FIXME: make a utility function to retrieve credentials from settings + profile = profiles['Local OpenQuake GeoViewer'] + session = Session() + hostname, username, password = (profile['hostname'], + profile['username'], + profile['password']) + session.auth = (username, password) + try: + geoviewer_login(hostname, username, password, session) + except Exception as exc: + err_msg = "Unable to connect (see Log Message Panel for details)" + log_msg(err_msg, level='C', message_bar=self.iface.messageBar(), + exception=exc) + else: + msg = 'Able to connect' + log_msg(msg, level='S', message_bar=self.iface.messageBar(), + duration=3) + project_list_url = hostname + '/api/project_list/' + try: + resp = session.get(project_list_url, timeout=10) + except Exception: + msg = "Unable to retrieve the list of projects.\n%s" % ( + traceback.format_exc()) + raise SvNetworkError(msg) + if resp.status_code != 200: # 200 means successful:OK + error_message = ('Unable to retrieve the list of projects: %s' % + resp.text) + raise SvNetworkError(error_message) + project_list = json.loads(resp.text) + # TODO: open a dialog with a list of projects showing chosen properties + # TODO: download the selected project + def download_layer(self): """ Open dialog to select one of the integrated risk projects available on From 3b1c6ac8ecdadc6bf1024ab4f80ce45af5ea2d37 Mon Sep 17 00:00:00 2001 From: Paolo Tormene Date: Fri, 24 Jan 2020 14:58:48 +0100 Subject: [PATCH 04/33] Add geometry validation as first step to upload project to geoviewer --- svir/dialogs/upload_settings_dialog.py | 13 +----- svir/irmt.py | 61 ++++++++++---------------- svir/utilities/shared.py | 11 +++++ 3 files changed, 34 insertions(+), 51 deletions(-) diff --git a/svir/dialogs/upload_settings_dialog.py b/svir/dialogs/upload_settings_dialog.py index e47998e94..98a01f7eb 100644 --- a/svir/dialogs/upload_settings_dialog.py +++ b/svir/dialogs/upload_settings_dialog.py @@ -34,7 +34,7 @@ from svir.calculations.process_layer import ProcessLayer from svir.utilities.shared import (IRMT_PLUGIN_VERSION, SUPPLEMENTAL_INFORMATION_VERSION, - DEBUG, + DEBUG, LICENSES, DEFAULT_LICENSE, ) from svir.utilities.utils import (reload_attrib_cbx, tr, @@ -49,17 +49,6 @@ get_credentials, ) -LICENSES = ( - ('CC0', 'http://creativecommons.org/about/cc0'), - ('CC BY 3.0 ', 'http://creativecommons.org/licenses/by/3.0/'), - ('CC BY-SA 3.0', 'http://creativecommons.org/licenses/by-sa/3.0/'), - ('CC BY-NC-SA 3.0', 'http://creativecommons.org/licenses/by-nc-sa/3.0/'), - ('CC BY 4.0', 'https://creativecommons.org/licenses/by/4.0/'), - ('CC BY-SA 4.0', 'https://creativecommons.org/licenses/by-sa/4.0/'), - ('CC BY-NC-SA 4.0', 'https://creativecommons.org/licenses/by-nc-sa/4.0/') -) -DEFAULT_LICENSE = LICENSES[5] # CC BY-SA 4.0 - FORM_CLASS = get_ui_class('ui_upload_settings.ui') diff --git a/svir/irmt.py b/svir/irmt.py index 1040d6642..64dad018d 100644 --- a/svir/irmt.py +++ b/svir/irmt.py @@ -30,12 +30,9 @@ import fileinput import re import processing -import json -import traceback from copy import deepcopy from math import floor, ceil -from requests import Session from qgis.core import ( QgsVectorLayer, QgsMapLayer, @@ -64,6 +61,8 @@ from svir.dialogs.viewer_dock import ViewerDock from svir.utilities.import_sv_data import get_loggedin_downloader from svir.dialogs.download_layer_dialog import DownloadLayerDialog +from svir.dialogs.import_gv_proj_dialog import ImportGvProjDialog +from svir.dialogs.upload_gv_proj_dialog import UploadGvProjDialog from svir.dialogs.projects_manager_dialog import ProjectsManagerDialog from svir.dialogs.select_sv_variables_dialog import SelectSvVariablesDialog from svir.dialogs.settings_dialog import SettingsDialog @@ -98,14 +97,12 @@ get_checksum, warn_missing_package, import_layer_from_csv, - geoviewer_login, ) from svir.utilities.shared import (DEBUG, PROJECT_TEMPLATE, THEME_TEMPLATE, INDICATOR_TEMPLATE, OQ_XMARKER_TYPES, - DEFAULT_GEOVIEWER_PROFILES, OPERATORS_DICT) from svir.ui.tool_button_with_help_link import QToolButtonWithHelpLink from svir.processing_provider.provider import Provider @@ -216,6 +213,15 @@ def initGui(self): add_to_layer_actions=False, submenu='OQ GeoViewer') + # Action to upload a project to the OQ GeoViewer + self.add_menu_item("upload_project_to_geoviewer", + ":/plugins/irmt/upload.svg", + u"Upload project to the OpenQuake &GeoViewer", + self.upload_project_to_geoviewer, + enable=True, + add_to_layer_actions=False, + submenu='OQ GeoViewer') + # Action to activate the modal dialog to import socioeconomic # data from the platform self.add_menu_item("import_sv_variables", @@ -816,40 +822,17 @@ def import_geoviewer_project(self): """ FIXME """ - mySettings = QSettings() - profiles = json.loads(mySettings.value( - 'irmt/geoviewer_profiles', DEFAULT_GEOVIEWER_PROFILES)) - # FIXME: make a utility function to retrieve credentials from settings - profile = profiles['Local OpenQuake GeoViewer'] - session = Session() - hostname, username, password = (profile['hostname'], - profile['username'], - profile['password']) - session.auth = (username, password) - try: - geoviewer_login(hostname, username, password, session) - except Exception as exc: - err_msg = "Unable to connect (see Log Message Panel for details)" - log_msg(err_msg, level='C', message_bar=self.iface.messageBar(), - exception=exc) - else: - msg = 'Able to connect' - log_msg(msg, level='S', message_bar=self.iface.messageBar(), - duration=3) - project_list_url = hostname + '/api/project_list/' - try: - resp = session.get(project_list_url, timeout=10) - except Exception: - msg = "Unable to retrieve the list of projects.\n%s" % ( - traceback.format_exc()) - raise SvNetworkError(msg) - if resp.status_code != 200: # 200 means successful:OK - error_message = ('Unable to retrieve the list of projects: %s' % - resp.text) - raise SvNetworkError(error_message) - project_list = json.loads(resp.text) - # TODO: open a dialog with a list of projects showing chosen properties - # TODO: download the selected project + self.download_gv_proj_dlg = ImportGvProjDialog( + self.iface.messageBar()) + self.download_gv_proj_dlg.exec_() + + def upload_project_to_geoviewer(self): + """ + FIXME + """ + self.upload_gv_proj_dlg = UploadGvProjDialog( + self.iface.messageBar()) + self.upload_gv_proj_dlg.exec_() def download_layer(self): """ diff --git a/svir/utilities/shared.py b/svir/utilities/shared.py index a329ca974..a58cb4218 100644 --- a/svir/utilities/shared.py +++ b/svir/utilities/shared.py @@ -311,3 +311,14 @@ def __repr__(self): } GEOM_FIELDNAMES = ('geom', 'the_geom', 'geometry', 'wkt') + +LICENSES = ( + ('CC0', 'http://creativecommons.org/about/cc0'), + ('CC BY 3.0 ', 'http://creativecommons.org/licenses/by/3.0/'), + ('CC BY-SA 3.0', 'http://creativecommons.org/licenses/by-sa/3.0/'), + ('CC BY-NC-SA 3.0', 'http://creativecommons.org/licenses/by-nc-sa/3.0/'), + ('CC BY 4.0', 'https://creativecommons.org/licenses/by/4.0/'), + ('CC BY-SA 4.0', 'https://creativecommons.org/licenses/by-sa/4.0/'), + ('CC BY-NC-SA 4.0', 'https://creativecommons.org/licenses/by-nc-sa/4.0/') +) +DEFAULT_LICENSE = LICENSES[5] # CC BY-SA 4.0 From 1973795a95ec6e99e6c4ecb750657b593900b373 Mon Sep 17 00:00:00 2001 From: Paolo Tormene Date: Fri, 24 Jan 2020 15:16:12 +0100 Subject: [PATCH 05/33] Add missing files to import layers from geoviewer --- svir/dialogs/import_gv_proj_dialog.py | 96 +++++++++++++++++++++++ svir/ui/ui_import_gv_proj.ui | 106 ++++++++++++++++++++++++++ 2 files changed, 202 insertions(+) create mode 100644 svir/dialogs/import_gv_proj_dialog.py create mode 100644 svir/ui/ui_import_gv_proj.ui diff --git a/svir/dialogs/import_gv_proj_dialog.py b/svir/dialogs/import_gv_proj_dialog.py new file mode 100644 index 000000000..ee794c094 --- /dev/null +++ b/svir/dialogs/import_gv_proj_dialog.py @@ -0,0 +1,96 @@ +# -*- coding: utf-8 -*- +# /*************************************************************************** +# Irmt +# A QGIS plugin +# OpenQuake Integrated Risk Modelling Toolkit +# ------------------- +# begin : 2020-01-23 +# copyright : (C) 2020 by GEM Foundation +# email : devops@openquake.org +# ***************************************************************************/ +# +# OpenQuake is free software: you can redistribute it and/or modify it +# under the terms of the GNU Affero General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# OpenQuake is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with OpenQuake. If not, see . + +import json +import traceback +from requests import Session +from qgis.PyQt.QtCore import QSettings +from qgis.PyQt.QtWidgets import QDialog, QMessageBox, QTableWidgetItem +from svir.utilities.utils import get_ui_class, log_msg, geoviewer_login +from svir.utilities.shared import DEFAULT_GEOVIEWER_PROFILES + + +FORM_CLASS = get_ui_class('ui_import_gv_proj.ui') + + +class ImportGvProjDialog(QDialog, FORM_CLASS): + """ + # TODO: open a dialog with a list of projects showing chosen properties + # TODO: download the selected project + """ + def __init__(self, message_bar): + self.message_bar = message_bar + QDialog.__init__(self) + # Set up the user interface from Designer. + self.setupUi(self) + project_list = self.get_project_list() + if not project_list: + return + # QMessageBox.information( + # self, "Info", json.dumps(project_list, indent=4), + # QMessageBox.Ok) + self.show_project_list(project_list) + + def show_project_list(self, project_list): + fields_to_display = ('name', ) # FIXME + self.list_of_projects_tbl.setRowCount(len(project_list)) + self.list_of_projects_tbl.setColumnCount(len(fields_to_display)) + for row, project in enumerate(project_list): + if not project['fields']['downloadable']: + continue + for col, field in enumerate(fields_to_display): + item = QTableWidgetItem(project['fields'][field]) + self.list_of_projects_tbl.setItem(row, col, item) + + def get_project_list(self): + mySettings = QSettings() + profiles = json.loads(mySettings.value( + 'irmt/geoviewer_profiles', DEFAULT_GEOVIEWER_PROFILES)) + # FIXME: make a utility function to retrieve credentials from settings + profile = profiles['Local OpenQuake GeoViewer'] + session = Session() + hostname, username, password = (profile['hostname'], + profile['username'], + profile['password']) + session.auth = (username, password) + try: + geoviewer_login(hostname, username, password, session) + except Exception as exc: + err_msg = "Unable to connect (see Log Message Panel for details)" + log_msg(err_msg, level='C', message_bar=self.message_bar, + exception=exc) + return + project_list_url = hostname + '/api/project_list/' + try: + resp = session.get(project_list_url, timeout=10) + except Exception: + msg = "Unable to retrieve the list of projects.\n%s" % ( + traceback.format_exc()) + raise RuntimeError(msg) + if resp.status_code != 200: # 200 means successful:OK + error_message = ('Unable to retrieve the list of projects: %s' % + resp.text) + raise RuntimeError(error_message) + project_list = json.loads(resp.text) + return project_list diff --git a/svir/ui/ui_import_gv_proj.ui b/svir/ui/ui_import_gv_proj.ui new file mode 100644 index 000000000..2c1450acb --- /dev/null +++ b/svir/ui/ui_import_gv_proj.ui @@ -0,0 +1,106 @@ + + + DriveEngineServerDialog + + + + 0 + 0 + 1079 + 650 + + + + Drive the OpenQuake Engine + + + + ../resources/oq_icon.svg../resources/oq_icon.svg + + + + + + Qt::Vertical + + + + + + + 0 + + + + + + 75 + false + true + + + + OpenQuake GeoViewer Projects + + + + + + + + + QAbstractItemView::NoEditTriggers + + + QAbstractItemView::SingleSelection + + + QAbstractItemView::SelectRows + + + 0 + + + 0 + + + false + + + + + + + + + + + 0 + + + + + Qt::Horizontal + + + + 40 + 20 + + + + + + + + <html><head/><body><p><img src=":/plugins/irmt/oq_logo.svg.png"/></p></body></html> + + + + + + + + + + From 0488f161eb757f7b533099242172bf2b432075b7 Mon Sep 17 00:00:00 2001 From: Paolo Tormene Date: Fri, 24 Jan 2020 15:17:29 +0100 Subject: [PATCH 06/33] Add missing files to upload project to geoviewer --- svir/dialogs/upload_gv_proj_dialog.py | 176 ++++++++++++++++++++++++++ svir/ui/ui_upload_gv_proj.ui | 84 ++++++++++++ 2 files changed, 260 insertions(+) create mode 100644 svir/dialogs/upload_gv_proj_dialog.py create mode 100644 svir/ui/ui_upload_gv_proj.ui diff --git a/svir/dialogs/upload_gv_proj_dialog.py b/svir/dialogs/upload_gv_proj_dialog.py new file mode 100644 index 000000000..dd6530794 --- /dev/null +++ b/svir/dialogs/upload_gv_proj_dialog.py @@ -0,0 +1,176 @@ +# -*- coding: utf-8 -*- +# /*************************************************************************** +# Irmt +# A QGIS plugin +# OpenQuake Integrated Risk Modelling Toolkit +# ------------------- +# begin : 2020-01-23 +# copyright : (C) 2020 by GEM Foundation +# email : devops@openquake.org +# ***************************************************************************/ +# +# OpenQuake is free software: you can redistribute it and/or modify it +# under the terms of the GNU Affero General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# OpenQuake is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with OpenQuake. If not, see . + +import json +import traceback +from requests import Session +from qgis.core import ( + QgsApplication, QgsProcessingContext, QgsProcessingFeedback, QgsProject, + QgsProcessingUtils, QgsField, QgsFields, QgsFeature, QgsGeometry, + QgsWkbTypes, QgsMemoryProviderUtils, QgsCoordinateReferenceSystem) +from qgis.PyQt.QtCore import QSettings, QVariant +from qgis.PyQt.QtWidgets import ( + QDialog, QDialogButtonBox) +from processing.gui.AlgorithmExecutor import execute +from svir.utilities.utils import get_ui_class, log_msg, geoviewer_login +from svir.utilities.shared import ( + DEFAULT_GEOVIEWER_PROFILES, LICENSES, DEFAULT_LICENSE) + + +FORM_CLASS = get_ui_class('ui_upload_gv_proj.ui') + + +class UploadGvProjDialog(QDialog, FORM_CLASS): + """ + Workflow: + 1. check that all geometries are valid for all layers + 2. check that a valid CRS has been set for the project + 3. let the user pick a license + 4. consolidate layers into .gpkg files and project into a .qgs + 5. use "api/project/upload" to upload the consolidated project + """ + def __init__(self, message_bar): + self.message_bar = message_bar + QDialog.__init__(self) + # Set up the user interface from Designer. + self.setupUi(self) + self.ok_button = self.buttonBox.button(QDialogButtonBox.Ok) + self.ok_button.setEnabled(False) + self.proj_name_le.textEdited.connect(self.set_ok_btn_status) + # self.add_layer_with_invalid_geometries() # useful to test validity + self.check_geometries() + self.check_crs() + self.populate_license_cbx() + if not self.consolidate(): + self.reject() + return + self.upload_to_geoviewer() + + def set_ok_btn_status(self, proj_name): + self.ok_button.setEnabled(bool(proj_name)) + + def populate_license_cbx(self): + for license_name, license_link in LICENSES: + self.license_cbx.addItem(license_name, license_link) + self.license_cbx.setCurrentIndex( + self.license_cbx.findText(DEFAULT_LICENSE[0])) + + def add_layer_with_invalid_geometries(self): + # NOTE: userful to test the validation + fields = QgsFields() + layer_wkb_name = 'Polygon' + wkb_type = getattr(QgsWkbTypes, layer_wkb_name) + fields.append(QgsField('int_f', QVariant.Int)) + polygon_layer = QgsMemoryProviderUtils.createMemoryLayer( + '%s_layer' % layer_wkb_name, fields, wkb_type, + QgsCoordinateReferenceSystem(4326)) + polygon_layer.startEditing() + f = QgsFeature(polygon_layer.fields()) + f.setAttributes([1]) + # Flake! + f.setGeometry(QgsGeometry.fromWkt( + 'POLYGON ((0 0, 2 2, 0 2, 2 0, 0 0))')) + f2 = QgsFeature(polygon_layer.fields()) + f2.setAttributes([1]) + f2.setGeometry(QgsGeometry.fromWkt( + 'POLYGON((1.1 1.1, 1.1 2.1, 2.1 2.1, 2.1 1.1, 1.1 1.1))')) + polygon_layer.addFeatures([f, f2]) + polygon_layer.commitChanges() + polygon_layer.rollBack() + QgsProject.instance().addMapLayers([polygon_layer]) + + def check_geometries(self): + layers = list(QgsProject.instance().mapLayers().values()) + registry = QgsApplication.instance().processingRegistry() + feedback = MessageBarFeedback(self.message_bar) + # feedback = ConsoleFeedBack() + context = QgsProcessingContext() + context.setProject(QgsProject.instance()) + parameters = {} + parameters['VALID_OUTPUT'] = 'memory:' + parameters['INVALID_OUTPUT'] = 'memory:' + parameters['ERROR_OUTPUT'] = 'memory:' + # QGIS method + # parameters['METHOD'] = 1 + # GEOS method + parameters['METHOD'] = 2 + alg = registry.createAlgorithmById('qgis:checkvalidity') + for layer in layers: + parameters['INPUT_LAYER'] = layer.id() + ok, results = execute( + alg, parameters, context=context, feedback=feedback) + invalid_layer = QgsProcessingUtils.mapLayerFromString( + results['INVALID_OUTPUT'], context) + if invalid_layer.featureCount(): + feedback.reportError( + "Layer '%s' contains features with invalid geometries." + " Please run Vector -> Geometry Tools -> Check Validity" + " for further information" % layer.name()) + feedback.pushInfo( + 'All features in all layers in the project are valid') + + def check_crs(self): + layers = list(QgsProject.instance().mapLayers().values()) + for layer in layers: + if not layer.crs().isValid(): + msg = ("Layer '%s' does not have a valid coordinate" + " reference system" % layer.name()) + log_msg(msg, level='C', message_bar=self.message_bar) + + def consolidate(self): + pass + + def upload_to_geoviewer(self): + pass + + +class ConsoleFeedBack(QgsProcessingFeedback): + + def reportError(self, error, fatalError=False): + print(error) + + +class MessageBarFeedback(QgsProcessingFeedback): + + def __init__(self, message_bar): + self.message_bar = message_bar + super().__init__() + + def pushCommandInfo(self, info): + self.message_bar.pushInfo('Info', info) + + def pushConsoleInfo(self, info): + self.message_bar.pushCritical('Info', info) + + def pushDebugInfo(self, info): + self.message_bar.pushCritical('Info', info) + + def pushInfo(self, info): + self.message_bar.pushInfo('Info', info) + + def reportError(self, error, fatalError=False): + self.message_bar.pushCritical('Error', error) + + def setProgressText(self, text): + self.message_bar.pushInfo('Info', text) diff --git a/svir/ui/ui_upload_gv_proj.ui b/svir/ui/ui_upload_gv_proj.ui new file mode 100644 index 000000000..73ac4c308 --- /dev/null +++ b/svir/ui/ui_upload_gv_proj.ui @@ -0,0 +1,84 @@ + + + Dialog + + + + 0 + 0 + 400 + 151 + + + + Dialog + + + + + + Project name + + + + + + + + + + License + + + + + + + + + + Qt::Horizontal + + + QDialogButtonBox::Cancel|QDialogButtonBox::Ok + + + + + + + + + buttonBox + accepted() + Dialog + accept() + + + 248 + 254 + + + 157 + 274 + + + + + buttonBox + rejected() + Dialog + reject() + + + 316 + 260 + + + 286 + 274 + + + + + From fa8ca7fde5a34c9893873fe9d7fb549e60b3d4ed Mon Sep 17 00:00:00 2001 From: Paolo Tormene Date: Fri, 24 Jan 2020 17:00:19 +0100 Subject: [PATCH 07/33] Add consolidation workflow --- svir/dialogs/upload_gv_proj_dialog.py | 101 ++++++++++- svir/tasks/consolidate_task.py | 249 ++++++++++++++++++++++++++ svir/ui/ui_upload_gv_proj.ui | 2 +- 3 files changed, 343 insertions(+), 9 deletions(-) create mode 100644 svir/tasks/consolidate_task.py diff --git a/svir/dialogs/upload_gv_proj_dialog.py b/svir/dialogs/upload_gv_proj_dialog.py index dd6530794..92132ca1f 100644 --- a/svir/dialogs/upload_gv_proj_dialog.py +++ b/svir/dialogs/upload_gv_proj_dialog.py @@ -22,18 +22,23 @@ # You should have received a copy of the GNU Affero General Public License # along with OpenQuake. If not, see . +import os +import re import json import traceback +import tempfile from requests import Session from qgis.core import ( QgsApplication, QgsProcessingContext, QgsProcessingFeedback, QgsProject, QgsProcessingUtils, QgsField, QgsFields, QgsFeature, QgsGeometry, - QgsWkbTypes, QgsMemoryProviderUtils, QgsCoordinateReferenceSystem) -from qgis.PyQt.QtCore import QSettings, QVariant + QgsWkbTypes, QgsMemoryProviderUtils, QgsCoordinateReferenceSystem, + QgsTask) +from qgis.PyQt.QtCore import QSettings, QVariant, QDir, QFile from qgis.PyQt.QtWidgets import ( QDialog, QDialogButtonBox) from processing.gui.AlgorithmExecutor import execute -from svir.utilities.utils import get_ui_class, log_msg, geoviewer_login +from svir.tasks.consolidate_task import ConsolidateTask +from svir.utilities.utils import get_ui_class, log_msg, geoviewer_login, tr from svir.utilities.shared import ( DEFAULT_GEOVIEWER_PROFILES, LICENSES, DEFAULT_LICENSE) @@ -62,10 +67,6 @@ def __init__(self, message_bar): self.check_geometries() self.check_crs() self.populate_license_cbx() - if not self.consolidate(): - self.reject() - return - self.upload_to_geoviewer() def set_ok_btn_status(self, proj_name): self.ok_button.setEnabled(bool(proj_name)) @@ -138,8 +139,78 @@ def check_crs(self): " reference system" % layer.name()) log_msg(msg, level='C', message_bar=self.message_bar) + def accept(self): + self.consolidate() + self.upload_to_geoviewer() + super().accept() + def consolidate(self): - pass + project_name = self.proj_name_le.text() + if project_name.endswith('.qgs'): + project_name = project_name[:-4] + if not project_name: + msg = tr("Please specify the project name") + log_msg(msg, level='C', message_bar=self.message_bar) + return + + outputDir = tempfile.mkdtemp() + outputDir = os.path.join(outputDir, + get_valid_filename(project_name)) + + # create main directory if not exists + d = QDir(outputDir) + if not d.exists(): + if not d.mkpath("."): + msg = tr("Can't create directory to store the project.") + log_msg(msg, level='C', message_bar=self.message_bar) + return + + # create directory for layers if not exists + if not d.mkdir("layers"): + msg = tr("Can't create directory for layers.") + log_msg(msg, level='C', message_bar=self.message_bar) + return + + # copy project file + projectFile = QgsProject.instance().fileName() + try: + if projectFile: + f = QFile(projectFile) + newProjectFile = os.path.join(outputDir, + '%s.qgs' % project_name) + f.copy(newProjectFile) + else: + newProjectFile = os.path.join( + outputDir, '%s.qgs' % project_name) + p = QgsProject.instance() + p.write(newProjectFile) + except Exception as exc: + log_msg(str(exc), level='C', + message_bar=self.message_bar, + exception=exc) + return + + # start consolidate task that does all real work + self.consolidateTask = ConsolidateTask( + 'Consolidation', QgsTask.CanCancel, outputDir, newProjectFile, + True) + self.consolidateTask.begun.connect(self.on_consolidation_begun) + self.consolidateTask.taskCompleted.connect( + lambda: self.on_consolidation_completed( + newProjectFile)) + + QgsApplication.taskManager().addTask(self.consolidateTask) + super().accept() + + def on_consolidation_begun(self): + log_msg("Consolidation started.", level='I', duration=4, + message_bar=self.message_bar) + + def on_consolidation_completed(self, project_file): + zipped_project = "%s.zip" % os.path.splitext(project_file)[0] + log_msg("Uploading '%s' to geoviewer" % zipped_project, level='I', + message_bar=self.message_bar) + self.upload_to_geoviewer() def upload_to_geoviewer(self): pass @@ -174,3 +245,17 @@ def reportError(self, error, fatalError=False): def setProgressText(self, text): self.message_bar.pushInfo('Info', text) + + +# from https://github.com/django/django/blob/master/django/utils/text.py#L223 +def get_valid_filename(s): + """ + Return the given string converted to a string that can be used for a clean + filename. Remove leading and trailing spaces; convert other spaces to + underscores; and remove anything that is not an alphanumeric, dash, + underscore, or dot. + >>> get_valid_filename("john's portrait in 2004.jpg") + 'johns_portrait_in_2004.jpg' + """ + s = str(s).strip().replace(' ', '_') # FIXME: str + return re.sub(r'(?u)[^-\w.]', '', s) diff --git a/svir/tasks/consolidate_task.py b/svir/tasks/consolidate_task.py new file mode 100644 index 000000000..fa06b5f8e --- /dev/null +++ b/svir/tasks/consolidate_task.py @@ -0,0 +1,249 @@ +# -*- coding: utf-8 -*- +# vim: tabstop=4 shiftwidth=4 softtabstop=4 +# +# Copyright (C) 2017-2018 GEM Foundation +# +# OpenQuake is free software: you can redistribute it and/or modify it +# under the terms of the GNU Affero General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# OpenQuake is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with OpenQuake. If not, see . + +# This plugin was forked from https://github.com/alexbruy/qconsolidate +# by Alexander Bruy (alexander.bruy@gmail.com), +# starting from commit 6f27b0b14b925a25c75ea79aea62a0e3d51e30e3. + +import os +import zipfile + +from qgis.PyQt.QtCore import ( + QIODevice, + QTextStream, + QFile, + ) +from qgis.PyQt.QtXml import QDomDocument + +from qgis.core import ( + QgsMapLayer, + QgsVectorFileWriter, + QgsProject, + QgsTask, + ) +from qgis.utils import iface + +from osgeo import gdal +from shutil import copyfile +from svir.utilities.utils import log_msg + + +class TaskCanceled(Exception): + pass + + +class ConsolidateTask(QgsTask): + + def __init__(self, description, flags, outputDir, projectFile, saveToZip): + super().__init__(description, flags) + self.outputDir = outputDir + self.layersDir = outputDir + "/layers" + self.projectFile = projectFile + self.saveToZip = saveToZip + self.progressMax = None + self.setDependentLayers(QgsProject.instance().mapLayers().values()) + + def run(self): + try: + self.consolidate() + except Exception as exc: + self.exception = exc + return False + else: + return True + + def finished(self, success): + if success: + msg = 'Consolidation to "%s" complete.' % self.outputDir + log_msg(msg, level='S', message_bar=iface.messageBar()) + else: + if self.exception is not None: + if isinstance(self.exception, TaskCanceled): + level = 'W' + else: + level = 'C' + log_msg(str(self.exception), level=level, + message_bar=iface.messageBar(), + exception=self.exception) + + def consolidate(self): + gdal.AllRegister() + + # read project + doc = self.loadProject() + root = doc.documentElement() + + # ensure that relative path used + e = root.firstChildElement("properties") + (e.firstChildElement("Paths").firstChild() + .firstChild().setNodeValue("false")) + + # get layers section in project + e = root.firstChildElement("projectlayers") + + # process layers + layers = QgsProject.instance().mapLayers() + self.progressMax = len(layers) + self.setProgress(1.0) + + # keep full paths of exported layer files (used to zip files) + outFiles = [self.projectFile] + if self.isCanceled(): + raise TaskCanceled('Consolidation canceled') + + for i, layer in enumerate(layers.values()): + if not layer.isValid(): + raise TypeError("Layer %s is invalid" % layer.name()) + lType = layer.type() + lProviderType = layer.providerType() + lName = layer.name() + lUri = layer.dataProvider().dataSourceUri() + if lType == QgsMapLayer.VectorLayer: + # Always convert to GeoPackage + outFile = self.convertGenericVectorLayer( + e, layer, lName) + outFiles.append(outFile) + elif lType == QgsMapLayer.RasterLayer: + # FIXME: should we convert also this to GeoPackage? + if lProviderType == 'gdal': + if self.checkGdalWms(lUri): + outFile = self.copyXmlRasterLayer( + e, layer, lName) + outFiles.append(outFile) + else: + raise TypeError('Layer %s (type %s) is not supported' + % (lName, lType)) + self.setProgress(i / self.progressMax * 100) + if self.isCanceled(): + raise TaskCanceled('Consolidation canceled') + + # save updated project + self.saveProject(doc) + + if self.saveToZip: + self.progressMax = len(outFiles) + self.setProgress(1.0) + # strip .qgs from the project name + self.zipfiles(outFiles, self.projectFile[:-4]) + + return True + + def loadProject(self): + f = QFile(self.projectFile) + if not f.open(QIODevice.ReadOnly | QIODevice.Text): + msg = self.tr("Cannot read file %s:\n%s.") % (self.projectFile, + f.errorString()) + raise IOError(msg) + + doc = QDomDocument() + setOk, errorString, errorLine, errorColumn = doc.setContent(f, True) + if not setOk: + msg = (self.tr("Parse error at line %d, column %d:\n%s") + % (errorLine, errorColumn, errorString)) + raise SyntaxError(msg) + + f.close() + return doc + + def saveProject(self, doc): + f = QFile(self.projectFile) + if not f.open(QIODevice.WriteOnly | QIODevice.Text): + msg = self.tr("Cannot write file %s:\n%s.") % (self.projectFile, + f.errorString()) + raise IOError(msg) + + out = QTextStream(f) + doc.save(out, 4) + f.close() + + def zipfiles(self, file_paths, archive): + """ + Build a zip archive from the given file names. + :param file_paths: list of path names + :param archive: path of the archive + """ + if self.isCanceled(): + raise TaskCanceled('Consolidation canceled') + archive = "%s.zip" % archive + prefix = len( + os.path.commonprefix([os.path.dirname(f) for f in file_paths])) + with zipfile.ZipFile( + archive, 'w', zipfile.ZIP_DEFLATED, allowZip64=True) as z: + for i, f in enumerate(file_paths): + z.write(f, f[prefix:]) + self.setProgress(i / self.progressMax * 100) + if self.isCanceled(): + raise TaskCanceled('Consolidation canceled') + + def copyXmlRasterLayer(self, layerElement, vLayer, layerName): + outFile = "%s/%s.xml" % (self.layersDir, layerName) + try: + copyfile(vLayer.dataProvider().dataSourceUri(), outFile) + except IOError: + msg = self.tr("Cannot copy layer %s") % layerName + raise IOError(msg) + + # update project + layerNode = self.findLayerInProject(layerElement, layerName) + tmpNode = layerNode.firstChildElement("datasource") + p = "./layers/%s.xml" % layerName + tmpNode.firstChild().setNodeValue(p) + tmpNode = layerNode.firstChildElement("provider") + tmpNode.firstChild().setNodeValue("gdal") + return outFile + + def convertGenericVectorLayer(self, layerElement, vLayer, layerName): + crs = vLayer.crs() + enc = vLayer.dataProvider().encoding() + outFile = "%s/%s.gpkg" % (self.layersDir, layerName) + + # TODO: If it's already a geopackage, we chould just copy it instead of + # converting it + # (if vLayer.dataProvider().storageType() == 'GPKG':) + + error, error_msg = QgsVectorFileWriter.writeAsVectorFormat( + vLayer, outFile, enc, crs, 'GPKG') + if error != QgsVectorFileWriter.NoError: + msg = self.tr("Cannot copy layer %s: %s") % (layerName, error_msg) + raise IOError(msg) + + # update project + layerNode = self.findLayerInProject(layerElement, layerName) + tmpNode = layerNode.firstChildElement("datasource") + p = "./layers/%s.gpkg" % layerName + tmpNode.firstChild().setNodeValue(p) + tmpNode = layerNode.firstChildElement("provider") + tmpNode.setAttribute("encoding", enc) + tmpNode.firstChild().setNodeValue("ogr") + return outFile + + def findLayerInProject(self, layerElement, layerName): + child = layerElement.firstChildElement() + while not child.isNull(): + nm = child.firstChildElement("layername") + if nm.text() == layerName: + return child + child = child.nextSiblingElement() + return None + + def checkGdalWms(self, layer): + ds = gdal.Open(layer, gdal.GA_ReadOnly) + isGdalWms = True if ds.GetDriver().ShortName == "WMS" else False + del ds + + return isGdalWms diff --git a/svir/ui/ui_upload_gv_proj.ui b/svir/ui/ui_upload_gv_proj.ui index 73ac4c308..ecb531c30 100644 --- a/svir/ui/ui_upload_gv_proj.ui +++ b/svir/ui/ui_upload_gv_proj.ui @@ -11,7 +11,7 @@ - Dialog + Upload project to the OpenQuake GeoViewer From a817873f45fe67efa4e57d08c8e073d6cd16d891 Mon Sep 17 00:00:00 2001 From: Paolo Tormene Date: Mon, 27 Jan 2020 10:05:49 +0100 Subject: [PATCH 08/33] Add layer with invalid feats to tree; fix feedback --- svir/dialogs/upload_gv_proj_dialog.py | 21 ++++++++++++--------- svir/tasks/consolidate_task.py | 2 +- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/svir/dialogs/upload_gv_proj_dialog.py b/svir/dialogs/upload_gv_proj_dialog.py index 92132ca1f..fc86090a8 100644 --- a/svir/dialogs/upload_gv_proj_dialog.py +++ b/svir/dialogs/upload_gv_proj_dialog.py @@ -117,6 +117,7 @@ def check_geometries(self): # GEOS method parameters['METHOD'] = 2 alg = registry.createAlgorithmById('qgis:checkvalidity') + invalid_features_found = False for layer in layers: parameters['INPUT_LAYER'] = layer.id() ok, results = execute( @@ -126,10 +127,13 @@ def check_geometries(self): if invalid_layer.featureCount(): feedback.reportError( "Layer '%s' contains features with invalid geometries." - " Please run Vector -> Geometry Tools -> Check Validity" - " for further information" % layer.name()) - feedback.pushInfo( - 'All features in all layers in the project are valid') + " A layer containing these invalid geometries was added" + " to the project." % layer.name()) + QgsProject.instance().addMapLayer(invalid_layer) + invalid_features_found = True + if not invalid_features_found: + feedback.pushInfo( + 'All features in all layers in the project are valid') def check_crs(self): layers = list(QgsProject.instance().mapLayers().values()) @@ -140,9 +144,9 @@ def check_crs(self): log_msg(msg, level='C', message_bar=self.message_bar) def accept(self): + super().accept() self.consolidate() self.upload_to_geoviewer() - super().accept() def consolidate(self): project_name = self.proj_name_le.text() @@ -203,13 +207,12 @@ def consolidate(self): super().accept() def on_consolidation_begun(self): - log_msg("Consolidation started.", level='I', duration=4, - message_bar=self.message_bar) + log_msg("Consolidation started.", level='I', duration=4) def on_consolidation_completed(self, project_file): zipped_project = "%s.zip" % os.path.splitext(project_file)[0] - log_msg("Uploading '%s' to geoviewer" % zipped_project, level='I', - message_bar=self.message_bar) + log_msg("The project was consolidated and saved to '%s'" + % zipped_project, level='S') self.upload_to_geoviewer() def upload_to_geoviewer(self): diff --git a/svir/tasks/consolidate_task.py b/svir/tasks/consolidate_task.py index fa06b5f8e..881513ecb 100644 --- a/svir/tasks/consolidate_task.py +++ b/svir/tasks/consolidate_task.py @@ -70,7 +70,7 @@ def run(self): def finished(self, success): if success: msg = 'Consolidation to "%s" complete.' % self.outputDir - log_msg(msg, level='S', message_bar=iface.messageBar()) + log_msg(msg, level='S') else: if self.exception is not None: if isinstance(self.exception, TaskCanceled): From c53d6d2407a097e83e7dc77043c3743280d72683 Mon Sep 17 00:00:00 2001 From: Paolo Tormene Date: Mon, 27 Jan 2020 11:17:32 +0100 Subject: [PATCH 09/33] Add authentication and call to the upload api --- svir/dialogs/upload_gv_proj_dialog.py | 31 +++++++++++++++++++++------ 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/svir/dialogs/upload_gv_proj_dialog.py b/svir/dialogs/upload_gv_proj_dialog.py index fc86090a8..9ce5898a7 100644 --- a/svir/dialogs/upload_gv_proj_dialog.py +++ b/svir/dialogs/upload_gv_proj_dialog.py @@ -38,7 +38,8 @@ QDialog, QDialogButtonBox) from processing.gui.AlgorithmExecutor import execute from svir.tasks.consolidate_task import ConsolidateTask -from svir.utilities.utils import get_ui_class, log_msg, geoviewer_login, tr +from svir.utilities.utils import ( + get_ui_class, log_msg, get_credentials, tr, geoviewer_login) from svir.utilities.shared import ( DEFAULT_GEOVIEWER_PROFILES, LICENSES, DEFAULT_LICENSE) @@ -67,6 +68,12 @@ def __init__(self, message_bar): self.check_geometries() self.check_crs() self.populate_license_cbx() + self.session = Session() + self.authenticate() + + def authenticate(self): + self.hostname, username, password = get_credentials('geoviewer') + geoviewer_login(self.hostname, username, password, self.session) def set_ok_btn_status(self, proj_name): self.ok_button.setEnabled(bool(proj_name)) @@ -103,6 +110,10 @@ def add_layer_with_invalid_geometries(self): def check_geometries(self): layers = list(QgsProject.instance().mapLayers().values()) + if len(layers) == 0: + log_msg("The project has no layers", + level='C', message_bar=self.message_bar) + return registry = QgsApplication.instance().processingRegistry() feedback = MessageBarFeedback(self.message_bar) # feedback = ConsoleFeedBack() @@ -146,7 +157,6 @@ def check_crs(self): def accept(self): super().accept() self.consolidate() - self.upload_to_geoviewer() def consolidate(self): project_name = self.proj_name_le.text() @@ -213,10 +223,19 @@ def on_consolidation_completed(self, project_file): zipped_project = "%s.zip" % os.path.splitext(project_file)[0] log_msg("The project was consolidated and saved to '%s'" % zipped_project, level='S') - self.upload_to_geoviewer() - - def upload_to_geoviewer(self): - pass + self.upload_to_geoviewer(zipped_project) + + def upload_to_geoviewer(self, zipped_project): + # FIXME: add license (to data?) + files = {'file': open(zipped_project, 'rb')} + r = self.session.post( + self.hostname + '/api/project/upload', files=files) + if r.ok: + msg = ("The project was successfully uploaded to the" + " OpenQuake GeoViewer") + log_msg(msg, level='S', message_bar=self.message_bar) + else: + log_msg(r.reason, level='C', message_bar=self.message_bar) class ConsoleFeedBack(QgsProcessingFeedback): From 6eb22077f752f08ba6ca3bb47e096d046d621219 Mon Sep 17 00:00:00 2001 From: Paolo Tormene Date: Tue, 28 Jan 2020 11:15:40 +0100 Subject: [PATCH 10/33] Check geometries only for vector layers --- svir/dialogs/upload_gv_proj_dialog.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/svir/dialogs/upload_gv_proj_dialog.py b/svir/dialogs/upload_gv_proj_dialog.py index 9ce5898a7..599cad529 100644 --- a/svir/dialogs/upload_gv_proj_dialog.py +++ b/svir/dialogs/upload_gv_proj_dialog.py @@ -32,7 +32,7 @@ QgsApplication, QgsProcessingContext, QgsProcessingFeedback, QgsProject, QgsProcessingUtils, QgsField, QgsFields, QgsFeature, QgsGeometry, QgsWkbTypes, QgsMemoryProviderUtils, QgsCoordinateReferenceSystem, - QgsTask) + QgsTask, QgsMapLayerType) from qgis.PyQt.QtCore import QSettings, QVariant, QDir, QFile from qgis.PyQt.QtWidgets import ( QDialog, QDialogButtonBox) @@ -130,6 +130,8 @@ def check_geometries(self): alg = registry.createAlgorithmById('qgis:checkvalidity') invalid_features_found = False for layer in layers: + if layer.type() != QgsMapLayerType.VectorLayer: + continue parameters['INPUT_LAYER'] = layer.id() ok, results = execute( alg, parameters, context=context, feedback=feedback) From 727f7518b520732fcbf2f58b881f0b3c46fda52f Mon Sep 17 00:00:00 2001 From: Paolo Tormene Date: Tue, 28 Jan 2020 11:16:38 +0100 Subject: [PATCH 11/33] While uploading to geoviewer, add lincense to the POST data --- svir/dialogs/upload_gv_proj_dialog.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/svir/dialogs/upload_gv_proj_dialog.py b/svir/dialogs/upload_gv_proj_dialog.py index 599cad529..acebb8f5d 100644 --- a/svir/dialogs/upload_gv_proj_dialog.py +++ b/svir/dialogs/upload_gv_proj_dialog.py @@ -228,10 +228,10 @@ def on_consolidation_completed(self, project_file): self.upload_to_geoviewer(zipped_project) def upload_to_geoviewer(self, zipped_project): - # FIXME: add license (to data?) + data = {'license': self.license_cbx.currentText()} files = {'file': open(zipped_project, 'rb')} r = self.session.post( - self.hostname + '/api/project/upload', files=files) + self.hostname + '/api/project/upload', data=data, files=files) if r.ok: msg = ("The project was successfully uploaded to the" " OpenQuake GeoViewer") From efdfa0b85b9098a7f97359901f5772ae729595f1 Mon Sep 17 00:00:00 2001 From: Paolo Tormene Date: Tue, 28 Jan 2020 11:55:24 +0100 Subject: [PATCH 12/33] Display an error message if the project does not have a valid crs --- svir/dialogs/upload_gv_proj_dialog.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/svir/dialogs/upload_gv_proj_dialog.py b/svir/dialogs/upload_gv_proj_dialog.py index acebb8f5d..98e4bebf3 100644 --- a/svir/dialogs/upload_gv_proj_dialog.py +++ b/svir/dialogs/upload_gv_proj_dialog.py @@ -155,6 +155,10 @@ def check_crs(self): msg = ("Layer '%s' does not have a valid coordinate" " reference system" % layer.name()) log_msg(msg, level='C', message_bar=self.message_bar) + if not QgsProject.instance().crs().isValid(): + msg = ("The current project does not have a valid coordinate" + " reference system") + log_msg(msg, level='C', message_bar=self.message_bar) def accept(self): super().accept() From a46c40b81ac53f128c5d8aa09db62f2ec76181d9 Mon Sep 17 00:00:00 2001 From: Paolo Tormene Date: Tue, 28 Jan 2020 12:12:32 +0100 Subject: [PATCH 13/33] Display a warning if a basemap is not in a group named basemaps or similar --- svir/dialogs/upload_gv_proj_dialog.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/svir/dialogs/upload_gv_proj_dialog.py b/svir/dialogs/upload_gv_proj_dialog.py index 98e4bebf3..4d4f56738 100644 --- a/svir/dialogs/upload_gv_proj_dialog.py +++ b/svir/dialogs/upload_gv_proj_dialog.py @@ -131,6 +131,18 @@ def check_geometries(self): invalid_features_found = False for layer in layers: if layer.type() != QgsMapLayerType.VectorLayer: + # If it is not in a group for basemaps, give a warning + root = QgsProject.instance().layerTreeRoot() + tree_layer = root.findLayer(layer.id()) + assert tree_layer + layer_parent = tree_layer.parent() + if (not layer_parent or + layer_parent.name().lower().strip().replace( + ' ', '') not in ('basemap', 'basemaps')): + msg = ("Layer %s looks like a basemap, and it should" + " probably be added to a group called" + " 'Basemaps'" % layer.name()) + log_msg(msg, level='W', message_bar=self.message_bar) continue parameters['INPUT_LAYER'] = layer.id() ok, results = execute( @@ -247,7 +259,7 @@ def upload_to_geoviewer(self, zipped_project): class ConsoleFeedBack(QgsProcessingFeedback): def reportError(self, error, fatalError=False): - print(error) + print(error) class MessageBarFeedback(QgsProcessingFeedback): From 64c46d1b40799683d25be9975c44257534707d45 Mon Sep 17 00:00:00 2001 From: Paolo Tormene Date: Wed, 29 Jan 2020 09:33:48 +0100 Subject: [PATCH 14/33] Revert "While uploading to geoviewer, add lincense to the POST data" This reverts commit 727f7518b520732fcbf2f58b881f0b3c46fda52f. --- svir/dialogs/upload_gv_proj_dialog.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/svir/dialogs/upload_gv_proj_dialog.py b/svir/dialogs/upload_gv_proj_dialog.py index 4d4f56738..18ebf2cc7 100644 --- a/svir/dialogs/upload_gv_proj_dialog.py +++ b/svir/dialogs/upload_gv_proj_dialog.py @@ -244,10 +244,10 @@ def on_consolidation_completed(self, project_file): self.upload_to_geoviewer(zipped_project) def upload_to_geoviewer(self, zipped_project): - data = {'license': self.license_cbx.currentText()} + # FIXME: add license (to data?) files = {'file': open(zipped_project, 'rb')} r = self.session.post( - self.hostname + '/api/project/upload', data=data, files=files) + self.hostname + '/api/project/upload', files=files) if r.ok: msg = ("The project was successfully uploaded to the" " OpenQuake GeoViewer") From 651acfdefe5ebc2e9ebbfb2363be6836ad69fa54 Mon Sep 17 00:00:00 2001 From: Paolo Tormene Date: Wed, 29 Jan 2020 09:34:45 +0100 Subject: [PATCH 15/33] Improve a comment --- svir/dialogs/upload_gv_proj_dialog.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/svir/dialogs/upload_gv_proj_dialog.py b/svir/dialogs/upload_gv_proj_dialog.py index 18ebf2cc7..1419f4ebd 100644 --- a/svir/dialogs/upload_gv_proj_dialog.py +++ b/svir/dialogs/upload_gv_proj_dialog.py @@ -244,7 +244,8 @@ def on_consolidation_completed(self, project_file): self.upload_to_geoviewer(zipped_project) def upload_to_geoviewer(self, zipped_project): - # FIXME: add license (to data?) + # FIXME: probably the license should be added to the project properties + # and it should be read GeoViewer-side through an api files = {'file': open(zipped_project, 'rb')} r = self.session.post( self.hostname + '/api/project/upload', files=files) From bce29edb7afcb082a9f8a77305ecacd608e6e2a8 Mon Sep 17 00:00:00 2001 From: Paolo Tormene Date: Wed, 29 Jan 2020 16:38:38 +0100 Subject: [PATCH 16/33] Add warnings if project owsServiceCapabilities or wmsExtent are disabled --- svir/dialogs/upload_gv_proj_dialog.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/svir/dialogs/upload_gv_proj_dialog.py b/svir/dialogs/upload_gv_proj_dialog.py index 1419f4ebd..eb8afe2fb 100644 --- a/svir/dialogs/upload_gv_proj_dialog.py +++ b/svir/dialogs/upload_gv_proj_dialog.py @@ -28,6 +28,7 @@ import traceback import tempfile from requests import Session +from qgis.server import QgsServerProjectUtils from qgis.core import ( QgsApplication, QgsProcessingContext, QgsProcessingFeedback, QgsProject, QgsProcessingUtils, QgsField, QgsFields, QgsFeature, QgsGeometry, @@ -65,6 +66,7 @@ def __init__(self, message_bar): self.ok_button.setEnabled(False) self.proj_name_le.textEdited.connect(self.set_ok_btn_status) # self.add_layer_with_invalid_geometries() # useful to test validity + self.check_capabilities() self.check_geometries() self.check_crs() self.populate_license_cbx() @@ -108,6 +110,15 @@ def add_layer_with_invalid_geometries(self): polygon_layer.rollBack() QgsProject.instance().addMapLayers([polygon_layer]) + def check_capabilities(self): + p = QgsProject.instance() + if not QgsServerProjectUtils.owsServiceCapabilities(p): + log_msg("Project capabilities are disabled", level='W', + message_bar=self.message_bar) + if QgsServerProjectUtils.wmsExtent(p).isEmpty(): + log_msg("Project extent is not advertised", level='W', + message_bar=self.message_bar) + def check_geometries(self): layers = list(QgsProject.instance().mapLayers().values()) if len(layers) == 0: @@ -128,7 +139,6 @@ def check_geometries(self): # GEOS method parameters['METHOD'] = 2 alg = registry.createAlgorithmById('qgis:checkvalidity') - invalid_features_found = False for layer in layers: if layer.type() != QgsMapLayerType.VectorLayer: # If it is not in a group for basemaps, give a warning @@ -155,10 +165,6 @@ def check_geometries(self): " A layer containing these invalid geometries was added" " to the project." % layer.name()) QgsProject.instance().addMapLayer(invalid_layer) - invalid_features_found = True - if not invalid_features_found: - feedback.pushInfo( - 'All features in all layers in the project are valid') def check_crs(self): layers = list(QgsProject.instance().mapLayers().values()) From 64ef88388b265caea16a3dd56f533f34bdc75a1d Mon Sep 17 00:00:00 2001 From: Paolo Tormene Date: Wed, 29 Jan 2020 17:29:47 +0100 Subject: [PATCH 17/33] Assign parent to UploadGvProjDialog; add comment about retrieving licenses --- svir/dialogs/upload_gv_proj_dialog.py | 18 ++++++++++++------ svir/irmt.py | 2 +- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/svir/dialogs/upload_gv_proj_dialog.py b/svir/dialogs/upload_gv_proj_dialog.py index eb8afe2fb..fafedb010 100644 --- a/svir/dialogs/upload_gv_proj_dialog.py +++ b/svir/dialogs/upload_gv_proj_dialog.py @@ -57,21 +57,21 @@ class UploadGvProjDialog(QDialog, FORM_CLASS): 4. consolidate layers into .gpkg files and project into a .qgs 5. use "api/project/upload" to upload the consolidated project """ - def __init__(self, message_bar): + def __init__(self, message_bar, parent=None): self.message_bar = message_bar - QDialog.__init__(self) + super().__init__(parent) # Set up the user interface from Designer. self.setupUi(self) self.ok_button = self.buttonBox.button(QDialogButtonBox.Ok) self.ok_button.setEnabled(False) self.proj_name_le.textEdited.connect(self.set_ok_btn_status) # self.add_layer_with_invalid_geometries() # useful to test validity + self.populate_license_cbx() self.check_capabilities() self.check_geometries() self.check_crs() - self.populate_license_cbx() self.session = Session() - self.authenticate() + # self.authenticate() # FIXME def authenticate(self): self.hostname, username, password = get_credentials('geoviewer') @@ -81,6 +81,8 @@ def set_ok_btn_status(self, proj_name): self.ok_button.setEnabled(bool(proj_name)) def populate_license_cbx(self): + # FIXME: licenses should be retrieved from those available in the + # GeoViewer for license_name, license_link in LICENSES: self.license_cbx.addItem(license_name, license_link) self.license_cbx.setCurrentIndex( @@ -113,11 +115,15 @@ def add_layer_with_invalid_geometries(self): def check_capabilities(self): p = QgsProject.instance() if not QgsServerProjectUtils.owsServiceCapabilities(p): - log_msg("Project capabilities are disabled", level='W', + log_msg("Project capabilities are disabled", level='C', message_bar=self.message_bar) + self.reject() + return if QgsServerProjectUtils.wmsExtent(p).isEmpty(): - log_msg("Project extent is not advertised", level='W', + log_msg("Project extent is not advertised", level='C', message_bar=self.message_bar) + self.reject() + return def check_geometries(self): layers = list(QgsProject.instance().mapLayers().values()) diff --git a/svir/irmt.py b/svir/irmt.py index 64dad018d..fa309cd7c 100644 --- a/svir/irmt.py +++ b/svir/irmt.py @@ -831,7 +831,7 @@ def upload_project_to_geoviewer(self): FIXME """ self.upload_gv_proj_dlg = UploadGvProjDialog( - self.iface.messageBar()) + self.iface.messageBar(), parent=self.iface.mainWindow()) self.upload_gv_proj_dlg.exec_() def download_layer(self): From 5648650092ec5f1770fbe9135bfdf2a0a28e0a5c Mon Sep 17 00:00:00 2001 From: Paolo Tormene Date: Thu, 27 Aug 2020 16:02:25 +0200 Subject: [PATCH 18/33] Add warnings if "Add geometry to feature response" or "Segmentize feature info geometry" are disabled --- svir/dialogs/upload_gv_proj_dialog.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/svir/dialogs/upload_gv_proj_dialog.py b/svir/dialogs/upload_gv_proj_dialog.py index fafedb010..4fa9dc9e0 100644 --- a/svir/dialogs/upload_gv_proj_dialog.py +++ b/svir/dialogs/upload_gv_proj_dialog.py @@ -114,6 +114,17 @@ def add_layer_with_invalid_geometries(self): def check_capabilities(self): p = QgsProject.instance() + add_geoms_to_feat_resp = p.readBoolEntry("WMSAddWktGeometry", "/")[0] + if not add_geoms_to_feat_resp: + msg = ('Project QGIS Server property "Add geometry to feature' + ' response" is disabled') + log_msg(msg, level='W', + message_bar=self.message_bar) + if not QgsServerProjectUtils.wmsFeatureInfoSegmentizeWktGeometry(p): + msg = ('Project QGIS Server property "Segmentize feature info' + ' geometry" is disabled') + log_msg(msg, level='W', + message_bar=self.message_bar) if not QgsServerProjectUtils.owsServiceCapabilities(p): log_msg("Project capabilities are disabled", level='C', message_bar=self.message_bar) From af8f7478ca51d7d6dc558c051cd6c302dffc9397 Mon Sep 17 00:00:00 2001 From: Paolo Tormene Date: Fri, 28 Aug 2020 14:21:28 +0200 Subject: [PATCH 19/33] Populate project kind cbx; check if proj name is already taken --- svir/dialogs/upload_gv_proj_dialog.py | 38 +++++++++++++++++++++++---- svir/ui/ui_upload_gv_proj.ui | 19 +++++++++++++- svir/utilities/shared.py | 7 +++++ 3 files changed, 58 insertions(+), 6 deletions(-) diff --git a/svir/dialogs/upload_gv_proj_dialog.py b/svir/dialogs/upload_gv_proj_dialog.py index 4fa9dc9e0..73ad50c38 100644 --- a/svir/dialogs/upload_gv_proj_dialog.py +++ b/svir/dialogs/upload_gv_proj_dialog.py @@ -42,7 +42,8 @@ from svir.utilities.utils import ( get_ui_class, log_msg, get_credentials, tr, geoviewer_login) from svir.utilities.shared import ( - DEFAULT_GEOVIEWER_PROFILES, LICENSES, DEFAULT_LICENSE) + DEFAULT_GEOVIEWER_PROFILES, LICENSES, DEFAULT_LICENSE, PROJECT_KINDS, + DEFAULT_PROJECT_KIND) FORM_CLASS = get_ui_class('ui_upload_gv_proj.ui') @@ -67,11 +68,13 @@ def __init__(self, message_bar, parent=None): self.proj_name_le.textEdited.connect(self.set_ok_btn_status) # self.add_layer_with_invalid_geometries() # useful to test validity self.populate_license_cbx() - self.check_capabilities() + self.populate_kind_cbx() + self.check_project_properties() self.check_geometries() self.check_crs() self.session = Session() - # self.authenticate() # FIXME + self.authenticate() + self.get_geoviewer_project_list() def authenticate(self): self.hostname, username, password = get_credentials('geoviewer') @@ -88,6 +91,14 @@ def populate_license_cbx(self): self.license_cbx.setCurrentIndex( self.license_cbx.findText(DEFAULT_LICENSE[0])) + def populate_kind_cbx(self): + # FIXME: kinds should be retrieved from those available in the + # GeoViewer + for kind_name, kind_text in PROJECT_KINDS: + self.project_kind_cbx.addItem(kind_text, kind_name) + self.project_kind_cbx.setCurrentIndex( + self.project_kind_cbx.findData(DEFAULT_PROJECT_KIND)) + def add_layer_with_invalid_geometries(self): # NOTE: userful to test the validation fields = QgsFields() @@ -112,7 +123,7 @@ def add_layer_with_invalid_geometries(self): polygon_layer.rollBack() QgsProject.instance().addMapLayers([polygon_layer]) - def check_capabilities(self): + def check_project_properties(self): p = QgsProject.instance() add_geoms_to_feat_resp = p.readBoolEntry("WMSAddWktGeometry", "/")[0] if not add_geoms_to_feat_resp: @@ -196,6 +207,11 @@ def check_crs(self): log_msg(msg, level='C', message_bar=self.message_bar) def accept(self): + proj_name = self.proj_name_le.text() + if proj_name in self.proj_names: + msg = ("A project named %s already exists" % proj_name) + log_msg(msg, level='C', message_bar=self.message_bar) + return super().accept() self.consolidate() @@ -255,7 +271,7 @@ def consolidate(self): newProjectFile)) QgsApplication.taskManager().addTask(self.consolidateTask) - super().accept() + # super().accept() def on_consolidation_begun(self): log_msg("Consolidation started.", level='I', duration=4) @@ -266,9 +282,21 @@ def on_consolidation_completed(self, project_file): % zipped_project, level='S') self.upload_to_geoviewer(zipped_project) + def get_geoviewer_project_list(self): + r = self.session.get(self.hostname + '/api/project_list/') + if r.ok: + content = json.loads(r.text) + self.proj_names = [proj['fields']['name'] for proj in content] + msg = ("Projects available on the OpenQuake GeoViewer: %s" + % self.proj_names) + log_msg(msg, level='S', message_bar=self.message_bar) + else: + log_msg(r.reason, level='C', message_bar=self.message_bar) + def upload_to_geoviewer(self, zipped_project): # FIXME: probably the license should be added to the project properties # and it should be read GeoViewer-side through an api + # FIXME: same as above for the project kind files = {'file': open(zipped_project, 'rb')} r = self.session.post( self.hostname + '/api/project/upload', files=files) diff --git a/svir/ui/ui_upload_gv_proj.ui b/svir/ui/ui_upload_gv_proj.ui index ecb531c30..ff2fc52d9 100644 --- a/svir/ui/ui_upload_gv_proj.ui +++ b/svir/ui/ui_upload_gv_proj.ui @@ -7,7 +7,7 @@ 0 0 400 - 151 + 234 @@ -24,6 +24,23 @@ + + + + Project kind + + + + + + + + + + Automatically create a map associated to the project + + + diff --git a/svir/utilities/shared.py b/svir/utilities/shared.py index a3da65a54..5db19c0e9 100644 --- a/svir/utilities/shared.py +++ b/svir/utilities/shared.py @@ -328,3 +328,10 @@ def __repr__(self): ('CC BY-NC-SA 4.0', 'https://creativecommons.org/licenses/by-nc-sa/4.0/') ) DEFAULT_LICENSE = LICENSES[5] # CC BY-SA 4.0 + +PROJECT_KINDS = ( + ('normal', 'Normal'), + ('gem_explorer', 'GEM Risk Explorer'), + ('disag', 'Disaggregation') +) +DEFAULT_PROJECT_KIND = 'normal' From c6182bd4f704f367200a68d486885374a8f167ef Mon Sep 17 00:00:00 2001 From: Paolo Tormene Date: Fri, 28 Aug 2020 15:13:56 +0200 Subject: [PATCH 20/33] Add logo to ui_upload_gv_proj.ui --- svir/ui/ui_import_gv_proj.ui | 2 +- svir/ui/ui_upload_gv_proj.ui | 28 +++++++++++++++++++++------- 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/svir/ui/ui_import_gv_proj.ui b/svir/ui/ui_import_gv_proj.ui index 2c1450acb..8b2c45d7c 100644 --- a/svir/ui/ui_import_gv_proj.ui +++ b/svir/ui/ui_import_gv_proj.ui @@ -6,7 +6,7 @@ 0 0 - 1079 + 763 650 diff --git a/svir/ui/ui_upload_gv_proj.ui b/svir/ui/ui_upload_gv_proj.ui index ff2fc52d9..8682ff5eb 100644 --- a/svir/ui/ui_upload_gv_proj.ui +++ b/svir/ui/ui_upload_gv_proj.ui @@ -52,14 +52,28 @@ - - - Qt::Horizontal + + + 0 - - QDialogButtonBox::Cancel|QDialogButtonBox::Ok - - + + + + <html><head/><body><p><img src=":/plugins/irmt/oq_logo.svg.png"/></p></body></html> + + + + + + + Qt::Horizontal + + + QDialogButtonBox::Cancel|QDialogButtonBox::Ok + + + + From 3aace1744162d790cbb0f497acad9a181598ecfa Mon Sep 17 00:00:00 2001 From: Paolo Tormene Date: Fri, 28 Aug 2020 17:07:39 +0200 Subject: [PATCH 21/33] Add buttons to download projects from GeoViewer --- svir/dialogs/import_gv_proj_dialog.py | 51 +++++++++++++-------------- svir/ui/ui_import_gv_proj.ui | 6 ++-- 2 files changed, 28 insertions(+), 29 deletions(-) diff --git a/svir/dialogs/import_gv_proj_dialog.py b/svir/dialogs/import_gv_proj_dialog.py index ee794c094..ed965bf0f 100644 --- a/svir/dialogs/import_gv_proj_dialog.py +++ b/svir/dialogs/import_gv_proj_dialog.py @@ -26,11 +26,12 @@ import traceback from requests import Session from qgis.PyQt.QtCore import QSettings -from qgis.PyQt.QtWidgets import QDialog, QMessageBox, QTableWidgetItem -from svir.utilities.utils import get_ui_class, log_msg, geoviewer_login +from qgis.PyQt.QtWidgets import ( + QDialog, QMessageBox, QTableWidgetItem, QPushButton) +from svir.utilities.utils import ( + get_ui_class, log_msg, geoviewer_login, get_credentials) from svir.utilities.shared import DEFAULT_GEOVIEWER_PROFILES - FORM_CLASS = get_ui_class('ui_import_gv_proj.ui') @@ -44,6 +45,8 @@ def __init__(self, message_bar): QDialog.__init__(self) # Set up the user interface from Designer. self.setupUi(self) + self.session = Session() + self.authenticate() project_list = self.get_project_list() if not project_list: return @@ -52,38 +55,34 @@ def __init__(self, message_bar): # QMessageBox.Ok) self.show_project_list(project_list) + def authenticate(self): + self.hostname, username, password = get_credentials('geoviewer') + geoviewer_login(self.hostname, username, password, self.session) + def show_project_list(self, project_list): - fields_to_display = ('name', ) # FIXME + fields_to_display = ('name', 'kind') # FIXME self.list_of_projects_tbl.setRowCount(len(project_list)) - self.list_of_projects_tbl.setColumnCount(len(fields_to_display)) + self.list_of_projects_tbl.setColumnCount(len(fields_to_display) + 1) for row, project in enumerate(project_list): - if not project['fields']['downloadable']: - continue for col, field in enumerate(fields_to_display): item = QTableWidgetItem(project['fields'][field]) self.list_of_projects_tbl.setItem(row, col, item) + if project['fields']['downloadable']: + button = QPushButton('Download') + self.list_of_projects_tbl.setCellWidget( + row, len(fields_to_display), button) + button.clicked.connect( + lambda checked=False, name=project['fields']['name']: + self.on_download_btn_clicked(name)) + + def on_download_btn_clicked(self, proj_name): + msg = 'Downloading project %s' % proj_name + log_msg(msg, level='S', message_bar=self.message_bar) def get_project_list(self): - mySettings = QSettings() - profiles = json.loads(mySettings.value( - 'irmt/geoviewer_profiles', DEFAULT_GEOVIEWER_PROFILES)) - # FIXME: make a utility function to retrieve credentials from settings - profile = profiles['Local OpenQuake GeoViewer'] - session = Session() - hostname, username, password = (profile['hostname'], - profile['username'], - profile['password']) - session.auth = (username, password) - try: - geoviewer_login(hostname, username, password, session) - except Exception as exc: - err_msg = "Unable to connect (see Log Message Panel for details)" - log_msg(err_msg, level='C', message_bar=self.message_bar, - exception=exc) - return - project_list_url = hostname + '/api/project_list/' + project_list_url = self.hostname + '/api/project_list/' try: - resp = session.get(project_list_url, timeout=10) + resp = self.session.get(project_list_url, timeout=10) except Exception: msg = "Unable to retrieve the list of projects.\n%s" % ( traceback.format_exc()) diff --git a/svir/ui/ui_import_gv_proj.ui b/svir/ui/ui_import_gv_proj.ui index 8b2c45d7c..b43046547 100644 --- a/svir/ui/ui_import_gv_proj.ui +++ b/svir/ui/ui_import_gv_proj.ui @@ -1,7 +1,7 @@ - DriveEngineServerDialog - + DownloadGeoviewerProjectDialog + 0 @@ -11,7 +11,7 @@ - Drive the OpenQuake Engine + Download OQ-GeoViewer projects From f07d4111318eca7a0d499b72c4e4d842f47fb318 Mon Sep 17 00:00:00 2001 From: Paolo Tormene Date: Fri, 4 Sep 2020 10:23:18 +0200 Subject: [PATCH 22/33] Fix setting project kind and set some timeouts to requests --- svir/dialogs/upload_gv_proj_dialog.py | 42 +++++++++++++++------------ 1 file changed, 23 insertions(+), 19 deletions(-) diff --git a/svir/dialogs/upload_gv_proj_dialog.py b/svir/dialogs/upload_gv_proj_dialog.py index 73ad50c38..4182a402e 100644 --- a/svir/dialogs/upload_gv_proj_dialog.py +++ b/svir/dialogs/upload_gv_proj_dialog.py @@ -53,10 +53,13 @@ class UploadGvProjDialog(QDialog, FORM_CLASS): """ Workflow: 1. check that all geometries are valid for all layers + (NOTE: some layers might contain no geometries, but they could be used as + data sources, joined to layers containing geometries) 2. check that a valid CRS has been set for the project - 3. let the user pick a license - 4. consolidate layers into .gpkg files and project into a .qgs - 5. use "api/project/upload" to upload the consolidated project + 3. let the user pick a type for the project + 4. let the user pick a license + 5. consolidate layers into .gpkg files and project into a .qgs + 6. use "api/project/upload" to upload the consolidated project """ def __init__(self, message_bar, parent=None): self.message_bar = message_bar @@ -185,14 +188,14 @@ def check_geometries(self): parameters['INPUT_LAYER'] = layer.id() ok, results = execute( alg, parameters, context=context, feedback=feedback) - invalid_layer = QgsProcessingUtils.mapLayerFromString( + invalid_geoms_layer = QgsProcessingUtils.mapLayerFromString( results['INVALID_OUTPUT'], context) - if invalid_layer.featureCount(): + if invalid_geoms_layer.featureCount(): feedback.reportError( "Layer '%s' contains features with invalid geometries." " A layer containing these invalid geometries was added" " to the project." % layer.name()) - QgsProject.instance().addMapLayer(invalid_layer) + QgsProject.instance().addMapLayer(invalid_geoms_layer) def check_crs(self): layers = list(QgsProject.instance().mapLayers().values()) @@ -200,22 +203,13 @@ def check_crs(self): if not layer.crs().isValid(): msg = ("Layer '%s' does not have a valid coordinate" " reference system" % layer.name()) - log_msg(msg, level='C', message_bar=self.message_bar) + log_msg(msg, level='W', message_bar=self.message_bar) if not QgsProject.instance().crs().isValid(): msg = ("The current project does not have a valid coordinate" " reference system") log_msg(msg, level='C', message_bar=self.message_bar) def accept(self): - proj_name = self.proj_name_le.text() - if proj_name in self.proj_names: - msg = ("A project named %s already exists" % proj_name) - log_msg(msg, level='C', message_bar=self.message_bar) - return - super().accept() - self.consolidate() - - def consolidate(self): project_name = self.proj_name_le.text() if project_name.endswith('.qgs'): project_name = project_name[:-4] @@ -223,7 +217,14 @@ def consolidate(self): msg = tr("Please specify the project name") log_msg(msg, level='C', message_bar=self.message_bar) return + if project_name in self.proj_names: + msg = ("A project named %s already exists" % project_name) + log_msg(msg, level='C', message_bar=self.message_bar) + return + super().accept() + self.consolidate(project_name) + def consolidate(self, project_name): outputDir = tempfile.mkdtemp() outputDir = os.path.join(outputDir, get_valid_filename(project_name)) @@ -283,7 +284,7 @@ def on_consolidation_completed(self, project_file): self.upload_to_geoviewer(zipped_project) def get_geoviewer_project_list(self): - r = self.session.get(self.hostname + '/api/project_list/') + r = self.session.get(self.hostname + '/api/project_list/', timeout=10) if r.ok: content = json.loads(r.text) self.proj_names = [proj['fields']['name'] for proj in content] @@ -294,12 +295,15 @@ def get_geoviewer_project_list(self): log_msg(r.reason, level='C', message_bar=self.message_bar) def upload_to_geoviewer(self, zipped_project): + project_license = self.license_cbx.currentText() # FIXME: probably the license should be added to the project properties # and it should be read GeoViewer-side through an api - # FIXME: same as above for the project kind + project_kind = self.project_kind_cbx.currentData() files = {'file': open(zipped_project, 'rb')} + data = {'license': project_license, 'kind': project_kind} r = self.session.post( - self.hostname + '/api/project/upload', files=files) + self.hostname + '/api/project/upload', files=files, data=data, + timeout=20) if r.ok: msg = ("The project was successfully uploaded to the" " OpenQuake GeoViewer") From dea87ed70ca041230a9b1ac4f95c72995172426f Mon Sep 17 00:00:00 2001 From: Paolo Tormene Date: Fri, 4 Sep 2020 17:20:57 +0200 Subject: [PATCH 23/33] Add checkbod to automatically create a map out of the uploaded GeoViewer project --- svir/dialogs/upload_gv_proj_dialog.py | 4 +++- svir/ui/ui_upload_gv_proj.ui | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/svir/dialogs/upload_gv_proj_dialog.py b/svir/dialogs/upload_gv_proj_dialog.py index 4182a402e..e1fc42d4e 100644 --- a/svir/dialogs/upload_gv_proj_dialog.py +++ b/svir/dialogs/upload_gv_proj_dialog.py @@ -299,8 +299,10 @@ def upload_to_geoviewer(self, zipped_project): # FIXME: probably the license should be added to the project properties # and it should be read GeoViewer-side through an api project_kind = self.project_kind_cbx.currentData() + auto_create_map = self.auto_create_map_ckb.isChecked() files = {'file': open(zipped_project, 'rb')} - data = {'license': project_license, 'kind': project_kind} + data = {'license': project_license, 'kind': project_kind, + 'auto_create_map': auto_create_map} r = self.session.post( self.hostname + '/api/project/upload', files=files, data=data, timeout=20) diff --git a/svir/ui/ui_upload_gv_proj.ui b/svir/ui/ui_upload_gv_proj.ui index 8682ff5eb..138e5c3be 100644 --- a/svir/ui/ui_upload_gv_proj.ui +++ b/svir/ui/ui_upload_gv_proj.ui @@ -7,7 +7,7 @@ 0 0 400 - 234 + 236 @@ -35,7 +35,7 @@ - + Automatically create a map associated to the project From b8d6f6ff60c598abbb18ada9f536c10270598971 Mon Sep 17 00:00:00 2001 From: Paolo Tormene Date: Mon, 11 Jan 2021 14:46:56 +0100 Subject: [PATCH 24/33] Fixed downloading procedure and minor things --- svir/dialogs/import_gv_map_dialog.py | 138 ++++++++++++++++++ svir/dialogs/import_gv_proj_dialog.py | 95 ------------ svir/dialogs/upload_gv_proj_dialog.py | 7 +- svir/irmt.py | 22 +-- ..._import_gv_proj.ui => ui_import_gv_map.ui} | 12 +- 5 files changed, 160 insertions(+), 114 deletions(-) create mode 100644 svir/dialogs/import_gv_map_dialog.py delete mode 100644 svir/dialogs/import_gv_proj_dialog.py rename svir/ui/{ui_import_gv_proj.ui => ui_import_gv_map.ui} (88%) diff --git a/svir/dialogs/import_gv_map_dialog.py b/svir/dialogs/import_gv_map_dialog.py new file mode 100644 index 000000000..3ca578af8 --- /dev/null +++ b/svir/dialogs/import_gv_map_dialog.py @@ -0,0 +1,138 @@ +# -*- coding: utf-8 -*- +# /*************************************************************************** +# Irmt +# A QGIS plugin +# OpenQuake Integrated Risk Modelling Toolkit +# ------------------- +# begin : 2020-01-23 +# copyright : (C) 2020 by GEM Foundation +# email : devops@openquake.org +# ***************************************************************************/ +# +# OpenQuake is free software: you can redistribute it and/or modify it +# under the terms of the GNU Affero General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# OpenQuake is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with OpenQuake. If not, see . + +import json +import traceback +import tempfile +import requests +import zipfile +import os +from requests import Session +from qgis.core import QgsProject +from qgis.PyQt.QtCore import QSettings +from qgis.PyQt.QtWidgets import ( + QDialog, QMessageBox, QTableWidgetItem, QPushButton) +from svir.utilities.utils import ( + get_ui_class, log_msg, geoviewer_login, get_credentials) +from svir.utilities.shared import DEFAULT_GEOVIEWER_PROFILES + +BUTTON_WIDTH = 75 + +FORM_CLASS = get_ui_class('ui_import_gv_map.ui') + + +class ImportGvMapDialog(QDialog, FORM_CLASS): + """ + Dialog listing maps available on a connected OqGeoviewer, showing map + properties and a button to download the map as a QGIS project + """ + def __init__(self, message_bar): + self.message_bar = message_bar + QDialog.__init__(self) + # Set up the user interface from Designer. + self.setupUi(self) + self.session = Session() + self.authenticate() + map_list = self.get_map_list() + if not map_list: + self.reject() + return + self.show_map_list(map_list) + + def authenticate(self): + self.hostname, username, password = get_credentials('geoviewer') + geoviewer_login(self.hostname, username, password, self.session) + + def show_map_list(self, map_list): + if not map_list: + return + fields_to_display = map_list[0]['fields'] + self.list_of_maps_tbl.setRowCount(len(map_list)) + self.list_of_maps_tbl.setColumnCount(len(fields_to_display) + 1) + for row, map in enumerate(map_list): + for col, field in enumerate(fields_to_display): + item = QTableWidgetItem(str(map['fields'][field])) + self.list_of_maps_tbl.setItem(row, col, item) + self.list_of_maps_tbl.setHorizontalHeaderItem( + col, QTableWidgetItem(field)) + # if True: # map['fields']['downloadable']: + button = QPushButton('Download') + self.list_of_maps_tbl.setCellWidget( + row, len(fields_to_display), button) + self.list_of_maps_tbl.setColumnWidth(col, BUTTON_WIDTH) + button.clicked.connect( + lambda checked=False, + map_name=map['fields']['name'], + map_slug=map['fields']['slug']: + self.on_download_btn_clicked(map_name, map_slug)) + + def download_url(self, url, save_path, chunk_size=128): + r = requests.get(url, stream=True) + if not r.ok: + msg = 'Unable to download the selected map: %s' % r.reason + log_msg(msg, level='C', message_bar=self.message_bar) + self.reject() + return False + with open(save_path, 'wb') as fd: + for chunk in r.iter_content(chunk_size=chunk_size): + fd.write(chunk) + return True + + def on_download_btn_clicked(self, map_name, map_slug): + msg = 'Downloading map %s (%s)' % (map_name, map_slug) + log_msg(msg, level='S', message_bar=self.message_bar) + map_download_url = '%s/map/%s/download' % (self.hostname, map_slug) + zip_file = tempfile.NamedTemporaryFile(suffix='.zip') + zip_file.close() + if not self.download_url(map_download_url, zip_file.name): + return + dirpath = tempfile.mkdtemp() + with zipfile.ZipFile(zip_file.name, 'r') as zip_ref: + zip_ref.extractall(dirpath) + log_msg('The project was downloaded into the folder: %s' % dirpath, + level='S') + for filename in os.listdir(dirpath): + if filename.endswith('.qgs'): + qgsfilepath = os.path.join(dirpath, filename) + project = QgsProject.instance() + project.read(qgsfilepath) + break + + def get_map_list(self): + map_list_url = self.hostname + '/api/map_list/' + try: + resp = self.session.get(map_list_url, timeout=10) + except Exception: + msg = "Unable to retrieve the list of maps.\n%s" % ( + traceback.format_exc()) + log_msg(msg, level='C', message_bar=self.messageBar()) + self.reject() + return + if resp.status_code != 200: # 200 means successful:OK + msg = ('Unable to retrieve the list of maps: %s' % resp.text) + log_msg(msg, level='C', message_bar=self.messageBar()) + self.reject() + return + map_list = json.loads(resp.text) + return map_list diff --git a/svir/dialogs/import_gv_proj_dialog.py b/svir/dialogs/import_gv_proj_dialog.py deleted file mode 100644 index ed965bf0f..000000000 --- a/svir/dialogs/import_gv_proj_dialog.py +++ /dev/null @@ -1,95 +0,0 @@ -# -*- coding: utf-8 -*- -# /*************************************************************************** -# Irmt -# A QGIS plugin -# OpenQuake Integrated Risk Modelling Toolkit -# ------------------- -# begin : 2020-01-23 -# copyright : (C) 2020 by GEM Foundation -# email : devops@openquake.org -# ***************************************************************************/ -# -# OpenQuake is free software: you can redistribute it and/or modify it -# under the terms of the GNU Affero General Public License as published -# by the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# OpenQuake is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with OpenQuake. If not, see . - -import json -import traceback -from requests import Session -from qgis.PyQt.QtCore import QSettings -from qgis.PyQt.QtWidgets import ( - QDialog, QMessageBox, QTableWidgetItem, QPushButton) -from svir.utilities.utils import ( - get_ui_class, log_msg, geoviewer_login, get_credentials) -from svir.utilities.shared import DEFAULT_GEOVIEWER_PROFILES - -FORM_CLASS = get_ui_class('ui_import_gv_proj.ui') - - -class ImportGvProjDialog(QDialog, FORM_CLASS): - """ - # TODO: open a dialog with a list of projects showing chosen properties - # TODO: download the selected project - """ - def __init__(self, message_bar): - self.message_bar = message_bar - QDialog.__init__(self) - # Set up the user interface from Designer. - self.setupUi(self) - self.session = Session() - self.authenticate() - project_list = self.get_project_list() - if not project_list: - return - # QMessageBox.information( - # self, "Info", json.dumps(project_list, indent=4), - # QMessageBox.Ok) - self.show_project_list(project_list) - - def authenticate(self): - self.hostname, username, password = get_credentials('geoviewer') - geoviewer_login(self.hostname, username, password, self.session) - - def show_project_list(self, project_list): - fields_to_display = ('name', 'kind') # FIXME - self.list_of_projects_tbl.setRowCount(len(project_list)) - self.list_of_projects_tbl.setColumnCount(len(fields_to_display) + 1) - for row, project in enumerate(project_list): - for col, field in enumerate(fields_to_display): - item = QTableWidgetItem(project['fields'][field]) - self.list_of_projects_tbl.setItem(row, col, item) - if project['fields']['downloadable']: - button = QPushButton('Download') - self.list_of_projects_tbl.setCellWidget( - row, len(fields_to_display), button) - button.clicked.connect( - lambda checked=False, name=project['fields']['name']: - self.on_download_btn_clicked(name)) - - def on_download_btn_clicked(self, proj_name): - msg = 'Downloading project %s' % proj_name - log_msg(msg, level='S', message_bar=self.message_bar) - - def get_project_list(self): - project_list_url = self.hostname + '/api/project_list/' - try: - resp = self.session.get(project_list_url, timeout=10) - except Exception: - msg = "Unable to retrieve the list of projects.\n%s" % ( - traceback.format_exc()) - raise RuntimeError(msg) - if resp.status_code != 200: # 200 means successful:OK - error_message = ('Unable to retrieve the list of projects: %s' % - resp.text) - raise RuntimeError(error_message) - project_list = json.loads(resp.text) - return project_list diff --git a/svir/dialogs/upload_gv_proj_dialog.py b/svir/dialogs/upload_gv_proj_dialog.py index e1fc42d4e..98cfedd04 100644 --- a/svir/dialogs/upload_gv_proj_dialog.py +++ b/svir/dialogs/upload_gv_proj_dialog.py @@ -25,7 +25,6 @@ import os import re import json -import traceback import tempfile from requests import Session from qgis.server import QgsServerProjectUtils @@ -34,7 +33,7 @@ QgsProcessingUtils, QgsField, QgsFields, QgsFeature, QgsGeometry, QgsWkbTypes, QgsMemoryProviderUtils, QgsCoordinateReferenceSystem, QgsTask, QgsMapLayerType) -from qgis.PyQt.QtCore import QSettings, QVariant, QDir, QFile +from qgis.PyQt.QtCore import QVariant, QDir, QFile from qgis.PyQt.QtWidgets import ( QDialog, QDialogButtonBox) from processing.gui.AlgorithmExecutor import execute @@ -42,7 +41,7 @@ from svir.utilities.utils import ( get_ui_class, log_msg, get_credentials, tr, geoviewer_login) from svir.utilities.shared import ( - DEFAULT_GEOVIEWER_PROFILES, LICENSES, DEFAULT_LICENSE, PROJECT_KINDS, + LICENSES, DEFAULT_LICENSE, PROJECT_KINDS, DEFAULT_PROJECT_KIND) @@ -185,6 +184,8 @@ def check_geometries(self): " 'Basemaps'" % layer.name()) log_msg(msg, level='W', message_bar=self.message_bar) continue + # FIXME: we could add a check to avoid warnings for layers that + # have no geometries (e.g. those used as data sources for joins) parameters['INPUT_LAYER'] = layer.id() ok, results = execute( alg, parameters, context=context, feedback=feedback) diff --git a/svir/irmt.py b/svir/irmt.py index fa309cd7c..dd9364a05 100644 --- a/svir/irmt.py +++ b/svir/irmt.py @@ -61,7 +61,7 @@ from svir.dialogs.viewer_dock import ViewerDock from svir.utilities.import_sv_data import get_loggedin_downloader from svir.dialogs.download_layer_dialog import DownloadLayerDialog -from svir.dialogs.import_gv_proj_dialog import ImportGvProjDialog +from svir.dialogs.import_gv_map_dialog import ImportGvMapDialog from svir.dialogs.upload_gv_proj_dialog import UploadGvProjDialog from svir.dialogs.projects_manager_dialog import ProjectsManagerDialog from svir.dialogs.select_sv_variables_dialog import SelectSvVariablesDialog @@ -204,11 +204,13 @@ def initGui(self): self.menu_action = menu_bar.insertMenu( self.iface.firstRightStandardMenu().menuAction(), self.menu) - # Action to list OQ GeoViewer projects - self.add_menu_item("import_geoviewer_project", + # NOTE: we could either list maps and use the existing api to download + # maps or add a GeoViewer api to download projects + # Action to list OQ GeoViewer maps + self.add_menu_item("import_geoviewer_map", ":/plugins/irmt/load_layer.svg", - u"Import project from the OpenQuake &GeoViewer", - self.import_geoviewer_project, + u"Import map from the OpenQuake &GeoViewer", + self.import_geoviewer_map, enable=True, add_to_layer_actions=False, submenu='OQ GeoViewer') @@ -818,17 +820,17 @@ def _data_download_successful( layer.setFieldAlias( field_idx, 'Country name') - def import_geoviewer_project(self): + def import_geoviewer_map(self): """ - FIXME + Open dialog to import maps from a connected OQ-GeoViewer """ - self.download_gv_proj_dlg = ImportGvProjDialog( + self.download_gv_map_dlg = ImportGvMapDialog( self.iface.messageBar()) - self.download_gv_proj_dlg.exec_() + self.download_gv_map_dlg.exec_() def upload_project_to_geoviewer(self): """ - FIXME + Open dialog to upload the current project to a connected OQ-GeoViewer """ self.upload_gv_proj_dlg = UploadGvProjDialog( self.iface.messageBar(), parent=self.iface.mainWindow()) diff --git a/svir/ui/ui_import_gv_proj.ui b/svir/ui/ui_import_gv_map.ui similarity index 88% rename from svir/ui/ui_import_gv_proj.ui rename to svir/ui/ui_import_gv_map.ui index b43046547..0d38c56b1 100644 --- a/svir/ui/ui_import_gv_proj.ui +++ b/svir/ui/ui_import_gv_map.ui @@ -1,7 +1,7 @@ - DownloadGeoviewerProjectDialog - + DownloadGeoviewerMapDialog + 0 @@ -11,7 +11,7 @@ - Download OQ-GeoViewer projects + Download OQ-GeoViewer map @@ -31,7 +31,7 @@ 0 - + 75 @@ -40,14 +40,14 @@ - OpenQuake GeoViewer Projects + OpenQuake GeoViewer Maps - + QAbstractItemView::NoEditTriggers From eb0886590a94726ce98b8e18bb1bc9e148637125 Mon Sep 17 00:00:00 2001 From: Paolo Tormene Date: Mon, 11 Jan 2021 15:12:32 +0100 Subject: [PATCH 25/33] Fix indentation of code generating download buttons --- svir/dialogs/import_gv_map_dialog.py | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/svir/dialogs/import_gv_map_dialog.py b/svir/dialogs/import_gv_map_dialog.py index 3ca578af8..83b0b7621 100644 --- a/svir/dialogs/import_gv_map_dialog.py +++ b/svir/dialogs/import_gv_map_dialog.py @@ -30,12 +30,10 @@ import os from requests import Session from qgis.core import QgsProject -from qgis.PyQt.QtCore import QSettings from qgis.PyQt.QtWidgets import ( - QDialog, QMessageBox, QTableWidgetItem, QPushButton) + QDialog, QTableWidgetItem, QPushButton) from svir.utilities.utils import ( get_ui_class, log_msg, geoviewer_login, get_credentials) -from svir.utilities.shared import DEFAULT_GEOVIEWER_PROFILES BUTTON_WIDTH = 75 @@ -76,16 +74,16 @@ def show_map_list(self, map_list): self.list_of_maps_tbl.setItem(row, col, item) self.list_of_maps_tbl.setHorizontalHeaderItem( col, QTableWidgetItem(field)) - # if True: # map['fields']['downloadable']: - button = QPushButton('Download') - self.list_of_maps_tbl.setCellWidget( - row, len(fields_to_display), button) - self.list_of_maps_tbl.setColumnWidth(col, BUTTON_WIDTH) - button.clicked.connect( - lambda checked=False, - map_name=map['fields']['name'], - map_slug=map['fields']['slug']: - self.on_download_btn_clicked(map_name, map_slug)) + # if map['fields']['downloadable']: + button = QPushButton('Download') + self.list_of_maps_tbl.setCellWidget( + row, len(fields_to_display), button) + self.list_of_maps_tbl.setColumnWidth(col, BUTTON_WIDTH) + button.clicked.connect( + lambda checked=False, + map_name=map['fields']['name'], + map_slug=map['fields']['slug']: + self.on_download_btn_clicked(map_name, map_slug)) def download_url(self, url, save_path, chunk_size=128): r = requests.get(url, stream=True) From a45f4eedfaf8867b0611a2c6269109a99f7d268d Mon Sep 17 00:00:00 2001 From: Paolo Tormene Date: Mon, 11 Jan 2021 17:21:12 +0100 Subject: [PATCH 26/33] In list of GeoViewer maps, put Download button first, then name, then other fields --- svir/dialogs/import_gv_map_dialog.py | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/svir/dialogs/import_gv_map_dialog.py b/svir/dialogs/import_gv_map_dialog.py index 83b0b7621..9b6d42416 100644 --- a/svir/dialogs/import_gv_map_dialog.py +++ b/svir/dialogs/import_gv_map_dialog.py @@ -65,25 +65,32 @@ def authenticate(self): def show_map_list(self, map_list): if not map_list: return - fields_to_display = map_list[0]['fields'] + fields_to_display = list(map_list[0]['fields']) + name_idx = fields_to_display.index('name') + fields_to_display.pop(name_idx) + fields_to_display.insert(0, 'name') + self.list_of_maps_tbl.setRowCount(len(map_list)) self.list_of_maps_tbl.setColumnCount(len(fields_to_display) + 1) for row, map in enumerate(map_list): - for col, field in enumerate(fields_to_display): - item = QTableWidgetItem(str(map['fields'][field])) - self.list_of_maps_tbl.setItem(row, col, item) - self.list_of_maps_tbl.setHorizontalHeaderItem( - col, QTableWidgetItem(field)) + # FIXME # if map['fields']['downloadable']: button = QPushButton('Download') self.list_of_maps_tbl.setCellWidget( - row, len(fields_to_display), button) - self.list_of_maps_tbl.setColumnWidth(col, BUTTON_WIDTH) + row, 0, button) + # self.list_of_maps_tbl.setColumnWidth(0, BUTTON_WIDTH) + self.list_of_maps_tbl.setHorizontalHeaderItem( + 0, QTableWidgetItem('')) button.clicked.connect( lambda checked=False, map_name=map['fields']['name'], map_slug=map['fields']['slug']: self.on_download_btn_clicked(map_name, map_slug)) + for col, field in enumerate(fields_to_display, start=1): + item = QTableWidgetItem(str(map['fields'][field])) + self.list_of_maps_tbl.setItem(row, col, item) + self.list_of_maps_tbl.setHorizontalHeaderItem( + col, QTableWidgetItem(field)) def download_url(self, url, save_path, chunk_size=128): r = requests.get(url, stream=True) From bcdcacb5af02bec9f19f92e5b1525ee6d56b161b Mon Sep 17 00:00:00 2001 From: Paolo Tormene Date: Mon, 11 Jan 2021 17:22:16 +0100 Subject: [PATCH 27/33] Add message boxes to automatically set up missing properties before uploading to GeoViewer --- svir/dialogs/upload_gv_proj_dialog.py | 59 +++++++++++++++++++-------- 1 file changed, 41 insertions(+), 18 deletions(-) diff --git a/svir/dialogs/upload_gv_proj_dialog.py b/svir/dialogs/upload_gv_proj_dialog.py index 98cfedd04..a990af90f 100644 --- a/svir/dialogs/upload_gv_proj_dialog.py +++ b/svir/dialogs/upload_gv_proj_dialog.py @@ -35,7 +35,7 @@ QgsTask, QgsMapLayerType) from qgis.PyQt.QtCore import QVariant, QDir, QFile from qgis.PyQt.QtWidgets import ( - QDialog, QDialogButtonBox) + QDialog, QDialogButtonBox, QMessageBox) from processing.gui.AlgorithmExecutor import execute from svir.tasks.consolidate_task import ConsolidateTask from svir.utilities.utils import ( @@ -129,25 +129,48 @@ def check_project_properties(self): p = QgsProject.instance() add_geoms_to_feat_resp = p.readBoolEntry("WMSAddWktGeometry", "/")[0] if not add_geoms_to_feat_resp: - msg = ('Project QGIS Server property "Add geometry to feature' - ' response" is disabled') - log_msg(msg, level='W', - message_bar=self.message_bar) - if not QgsServerProjectUtils.wmsFeatureInfoSegmentizeWktGeometry(p): - msg = ('Project QGIS Server property "Segmentize feature info' - ' geometry" is disabled') - log_msg(msg, level='W', - message_bar=self.message_bar) + msg = 'Would you like to add geometry to GetFeatureInfo response?' + ret = QMessageBox.question( + self, '', msg, QMessageBox.Yes, QMessageBox.No) + if ret == QMessageBox.Yes: + p.writeEntryBool("WMSAddWktGeometry", "/", True) + + # FIXME: this returns False even if the precision is set to 1 + geom_precision = p.readEntry("WMSPrecision", "/")[0] + if geom_precision != 1: + msg = ('Would you like to set GetFeatureInfo geometry precision ' + 'to 1 decimal place?') + ret = QMessageBox.question( + self, '', msg, QMessageBox.Yes, QMessageBox.No) + if ret == QMessageBox.Yes: + p.writeEntry("WMSPrecision", "/", 1) + # if not QgsServerProjectUtils.wmsFeatureInfoSegmentizeWktGeometry(p): + # msg = ('Project QGIS Server property "Segmentize feature info' + # ' geometry" is disabled') + # log_msg(msg, level='W', + # message_bar=self.message_bar) if not QgsServerProjectUtils.owsServiceCapabilities(p): - log_msg("Project capabilities are disabled", level='C', - message_bar=self.message_bar) - self.reject() - return + msg = ('Would you like to enable service capabilities?') + ret = QMessageBox.question( + self, '', msg, QMessageBox.Yes, QMessageBox.No) + if ret == QMessageBox.Yes: + p.writeEntryBool("WMSServiceCapabilities", "/", True) + # log_msg("Project capabilities are disabled", level='C', + # message_bar=self.message_bar) + # self.reject() + # return + + # FIXME: we should set the extent to the current one if QgsServerProjectUtils.wmsExtent(p).isEmpty(): - log_msg("Project extent is not advertised", level='C', - message_bar=self.message_bar) - self.reject() - return + msg = ('Would you like to advertise project extent?') + ret = QMessageBox.question( + self, '', msg, QMessageBox.Yes, QMessageBox.No) + if ret == QMessageBox.Yes: + p.writeEntry("WMSExtent", "/", True) + # log_msg("Project extent is not advertised", level='C', + # message_bar=self.message_bar) + # self.reject() + # return def check_geometries(self): layers = list(QgsProject.instance().mapLayers().values()) From 184d85546a34227e3a8098000089f40de7bf6269 Mon Sep 17 00:00:00 2001 From: Paolo Tormene Date: Tue, 12 Jan 2021 10:03:36 +0100 Subject: [PATCH 28/33] Do not give warnings about geometries for csv-based layers (tables) --- svir/dialogs/upload_gv_proj_dialog.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/svir/dialogs/upload_gv_proj_dialog.py b/svir/dialogs/upload_gv_proj_dialog.py index a990af90f..c597c299b 100644 --- a/svir/dialogs/upload_gv_proj_dialog.py +++ b/svir/dialogs/upload_gv_proj_dialog.py @@ -207,8 +207,8 @@ def check_geometries(self): " 'Basemaps'" % layer.name()) log_msg(msg, level='W', message_bar=self.message_bar) continue - # FIXME: we could add a check to avoid warnings for layers that - # have no geometries (e.g. those used as data sources for joins) + if layer.geometryType() == QgsWkbTypes.NullGeometry: + continue parameters['INPUT_LAYER'] = layer.id() ok, results = execute( alg, parameters, context=context, feedback=feedback) @@ -224,6 +224,8 @@ def check_geometries(self): def check_crs(self): layers = list(QgsProject.instance().mapLayers().values()) for layer in layers: + if layer.geometryType() == QgsWkbTypes.NullGeometry: + continue if not layer.crs().isValid(): msg = ("Layer '%s' does not have a valid coordinate" " reference system" % layer.name()) From 7ac9017f5c87c2e5fafaff41bf00b6120df47e7e Mon Sep 17 00:00:00 2001 From: Paolo Tormene Date: Tue, 13 Apr 2021 10:48:07 +0200 Subject: [PATCH 29/33] Retrieve only published maps from geoviewer and display a clear error message when attempting to load a map that is not flagged as downloadable --- svir/dialogs/import_gv_map_dialog.py | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/svir/dialogs/import_gv_map_dialog.py b/svir/dialogs/import_gv_map_dialog.py index 9b6d42416..d22e5aec2 100644 --- a/svir/dialogs/import_gv_map_dialog.py +++ b/svir/dialogs/import_gv_map_dialog.py @@ -52,19 +52,19 @@ def __init__(self, message_bar): self.setupUi(self) self.session = Session() self.authenticate() - map_list = self.get_map_list() - if not map_list: + map_list = self.get_published_map_list() + if map_list: + self.show_map_list(map_list) + else: + log_msg('There are no published maps available', + level='W', message_bar=self.message_bar) self.reject() - return - self.show_map_list(map_list) def authenticate(self): self.hostname, username, password = get_credentials('geoviewer') geoviewer_login(self.hostname, username, password, self.session) def show_map_list(self, map_list): - if not map_list: - return fields_to_display = list(map_list[0]['fields']) name_idx = fields_to_display.index('name') fields_to_display.pop(name_idx) @@ -95,7 +95,12 @@ def show_map_list(self, map_list): def download_url(self, url, save_path, chunk_size=128): r = requests.get(url, stream=True) if not r.ok: - msg = 'Unable to download the selected map: %s' % r.reason + msg = 'Unable to download the selected map.' + if r.reason == 'Forbidden': + msg += (' Most likely, the map is set as published but the' + ' corresponding project is not set as downloadable.') + else: + msg += ' ' + r.reason log_msg(msg, level='C', message_bar=self.message_bar) self.reject() return False @@ -124,7 +129,7 @@ def on_download_btn_clicked(self, map_name, map_slug): project.read(qgsfilepath) break - def get_map_list(self): + def get_published_map_list(self): map_list_url = self.hostname + '/api/map_list/' try: resp = self.session.get(map_list_url, timeout=10) @@ -140,4 +145,6 @@ def get_map_list(self): self.reject() return map_list = json.loads(resp.text) - return map_list + published_map_list = [map for map in map_list + if map['fields']['published']] + return published_map_list From 265a4a70933a71caf900b1bf56bde173a5ad34a3 Mon Sep 17 00:00:00 2001 From: Paolo Tormene Date: Tue, 13 Apr 2021 10:49:16 +0200 Subject: [PATCH 30/33] Check validity of geometries only for vector layers --- svir/dialogs/upload_gv_proj_dialog.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/svir/dialogs/upload_gv_proj_dialog.py b/svir/dialogs/upload_gv_proj_dialog.py index c597c299b..9d514add6 100644 --- a/svir/dialogs/upload_gv_proj_dialog.py +++ b/svir/dialogs/upload_gv_proj_dialog.py @@ -32,7 +32,7 @@ QgsApplication, QgsProcessingContext, QgsProcessingFeedback, QgsProject, QgsProcessingUtils, QgsField, QgsFields, QgsFeature, QgsGeometry, QgsWkbTypes, QgsMemoryProviderUtils, QgsCoordinateReferenceSystem, - QgsTask, QgsMapLayerType) + QgsTask, QgsMapLayerType, QgsVectorLayer) from qgis.PyQt.QtCore import QVariant, QDir, QFile from qgis.PyQt.QtWidgets import ( QDialog, QDialogButtonBox, QMessageBox) @@ -224,7 +224,8 @@ def check_geometries(self): def check_crs(self): layers = list(QgsProject.instance().mapLayers().values()) for layer in layers: - if layer.geometryType() == QgsWkbTypes.NullGeometry: + if (isinstance(layer, QgsVectorLayer) + and layer.geometryType() == QgsWkbTypes.NullGeometry): continue if not layer.crs().isValid(): msg = ("Layer '%s' does not have a valid coordinate" From 0eb436e87963b868721ce8e4406db1cbae39d1aa Mon Sep 17 00:00:00 2001 From: Paolo Tormene Date: Thu, 15 Apr 2021 14:40:35 +0200 Subject: [PATCH 31/33] Save project before consolidating (otherwise it does not change on second upload) --- svir/dialogs/upload_gv_proj_dialog.py | 1 + 1 file changed, 1 insertion(+) diff --git a/svir/dialogs/upload_gv_proj_dialog.py b/svir/dialogs/upload_gv_proj_dialog.py index 9d514add6..a704da670 100644 --- a/svir/dialogs/upload_gv_proj_dialog.py +++ b/svir/dialogs/upload_gv_proj_dialog.py @@ -248,6 +248,7 @@ def accept(self): msg = ("A project named %s already exists" % project_name) log_msg(msg, level='C', message_bar=self.message_bar) return + QgsProject.instance().write() super().accept() self.consolidate(project_name) From ae2a1a138b2399323463c5aaa2eea9d9580cc10b Mon Sep 17 00:00:00 2001 From: Paolo Tormene Date: Fri, 16 Apr 2021 15:45:13 +0200 Subject: [PATCH 32/33] Check auto_create_map by default --- svir/ui/ui_upload_gv_proj.ui | 3 +++ 1 file changed, 3 insertions(+) diff --git a/svir/ui/ui_upload_gv_proj.ui b/svir/ui/ui_upload_gv_proj.ui index 138e5c3be..904729a27 100644 --- a/svir/ui/ui_upload_gv_proj.ui +++ b/svir/ui/ui_upload_gv_proj.ui @@ -39,6 +39,9 @@ Automatically create a map associated to the project + + true + From 8445e5cfe8e52322f5337d7986a2a60fcd987e8a Mon Sep 17 00:00:00 2001 From: Paolo Tormene Date: Wed, 21 Apr 2021 10:05:56 +0200 Subject: [PATCH 33/33] When loading GeoViewer project, add basemaps into a group --- svir/dialogs/import_gv_map_dialog.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/svir/dialogs/import_gv_map_dialog.py b/svir/dialogs/import_gv_map_dialog.py index d22e5aec2..dbfe066f2 100644 --- a/svir/dialogs/import_gv_map_dialog.py +++ b/svir/dialogs/import_gv_map_dialog.py @@ -29,7 +29,7 @@ import zipfile import os from requests import Session -from qgis.core import QgsProject +from qgis.core import QgsProject, QgsRasterLayer from qgis.PyQt.QtWidgets import ( QDialog, QTableWidgetItem, QPushButton) from svir.utilities.utils import ( @@ -122,12 +122,21 @@ def on_download_btn_clicked(self, map_name, map_slug): zip_ref.extractall(dirpath) log_msg('The project was downloaded into the folder: %s' % dirpath, level='S') + qgsfilepath = None for filename in os.listdir(dirpath): if filename.endswith('.qgs'): qgsfilepath = os.path.join(dirpath, filename) - project = QgsProject.instance() - project.read(qgsfilepath) break + if qgsfilepath is None: + raise RuntimeError('No .qgs file was found in %s' % dirpath) + project = QgsProject.instance() + project.read(qgsfilepath) + root = project.layerTreeRoot() + basemaps_group = root.addGroup('Basemaps') + basemaps_group.setIsMutuallyExclusive(True) + for maplayer in project.mapLayers().values(): + if isinstance(maplayer, QgsRasterLayer): + basemaps_group.insertLayer(0, maplayer) def get_published_map_list(self): map_list_url = self.hostname + '/api/map_list/' @@ -146,5 +155,5 @@ def get_published_map_list(self): return map_list = json.loads(resp.text) published_map_list = [map for map in map_list - if map['fields']['published']] + if map['fields']['published']] return published_map_list