From 8291bb87f0dc43e51c2e3aca4536615494d2ae25 Mon Sep 17 00:00:00 2001 From: index-git Date: Mon, 11 Dec 2023 14:39:24 +0100 Subject: [PATCH 01/11] Fix database name in LAYMAN_ROLE_SERVICE_URI for dev and demo --- .env.demo | 2 +- .env.dev | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.env.demo b/.env.demo index 90fb09887..315bda697 100644 --- a/.env.demo +++ b/.env.demo @@ -16,7 +16,7 @@ LAYMAN_AUTHN_MODULES=layman.authn.http_header LAYMAN_AUTHN_HTTP_HEADER_NAME=a0468616f9968eaecdc3377988aba650 GRANT_CREATE_PUBLIC_WORKSPACE=EVERYONE GRANT_PUBLISH_IN_PUBLIC_WORKSPACE=EVERYONE -LAYMAN_ROLE_SERVICE_URI=postgresql://docker:docker@postgresql:5432/layman_test?schema=_role_service +LAYMAN_ROLE_SERVICE_URI=postgresql://docker:docker@postgresql:5432/gis?schema=_role_service # connection parameters to PostgreSQL database LAYMAN_PG_HOST=postgresql diff --git a/.env.dev b/.env.dev index 627780c2a..db7fba29b 100644 --- a/.env.dev +++ b/.env.dev @@ -16,7 +16,7 @@ LAYMAN_AUTHN_MODULES=layman.authn.oauth2,layman.authn.http_header LAYMAN_AUTHN_HTTP_HEADER_NAME=a0468616f9968eaecdc3377988aba650 GRANT_CREATE_PUBLIC_WORKSPACE=EVERYONE GRANT_PUBLISH_IN_PUBLIC_WORKSPACE=EVERYONE -LAYMAN_ROLE_SERVICE_URI=postgresql://docker:docker@postgresql:5432/layman_test?schema=_role_service +LAYMAN_ROLE_SERVICE_URI=postgresql://docker:docker@postgresql:5432/gis?schema=_role_service # connection parameters to PostgreSQL database LAYMAN_PG_HOST=postgresql From 2e95043aaeb8acc3a14fd47532371a209a7886b2 Mon Sep 17 00:00:00 2001 From: index-git Date: Mon, 11 Dec 2023 14:44:49 +0100 Subject: [PATCH 02/11] Create other table for GS role service --- src/layman/upgrade/upgrade_v1_23.py | 7 +++++++ src/layman/upgrade/upgrade_v1_23_test.py | 4 ++++ 2 files changed, 11 insertions(+) diff --git a/src/layman/upgrade/upgrade_v1_23.py b/src/layman/upgrade/upgrade_v1_23.py index 930c2e738..b9fa652bd 100644 --- a/src/layman/upgrade/upgrade_v1_23.py +++ b/src/layman/upgrade/upgrade_v1_23.py @@ -102,3 +102,10 @@ def create_role_service_schema(): from {ROLE_SERVICE_SCHEMA}.admin_user_roles ;""" db_util.run_statement(create_user_roles_view) + + create_other_tables = f""" + create table {ROLE_SERVICE_SCHEMA}.role_props(rolename varchar(64) not null,propname varchar(64) not null, propvalue varchar(2048),primary key (rolename,propname)); + create table {ROLE_SERVICE_SCHEMA}.group_roles(groupname varchar(128) not null, rolename varchar(64) not null, primary key(groupname,rolename)); + create index group_roles_idx on {ROLE_SERVICE_SCHEMA}.group_roles(rolename,groupname); + """ + db_util.run_statement(create_other_tables) diff --git a/src/layman/upgrade/upgrade_v1_23_test.py b/src/layman/upgrade/upgrade_v1_23_test.py index ebafadc48..deb1aef2d 100644 --- a/src/layman/upgrade/upgrade_v1_23_test.py +++ b/src/layman/upgrade/upgrade_v1_23_test.py @@ -128,3 +128,7 @@ def test_create_role_service_schema(): assert result[0] + result[1] + result[2] == result[3] result = db_util.run_query(user_roles_query)[0] assert result[0] + result[1] + result[2] == result[3] + result = db_util.run_query(table_existence_query, ('role_props',))[0][0] + assert result == 1 + result = db_util.run_query(table_existence_query, ('group_roles',))[0][0] + assert result == 1 From 76dced4f9358747cf68d565fa8e60cbe1e5ba096 Mon Sep 17 00:00:00 2001 From: index-git Date: Tue, 12 Dec 2023 13:43:31 +0100 Subject: [PATCH 03/11] Move redact_uri --- src/layman/layer/db/table.py | 9 +++++---- src/layman/layer/util.py | 11 ----------- src/layman/layer/util_test.py | 15 --------------- src/requests_util/url_util.py | 12 ++++++++++++ src/requests_util/url_util_test.py | 18 ++++++++++++++++++ tests/asserts/final/publication/internal.py | 6 ++++-- 6 files changed, 39 insertions(+), 32 deletions(-) create mode 100644 src/requests_util/url_util.py create mode 100644 src/requests_util/url_util_test.py diff --git a/src/layman/layer/db/table.py b/src/layman/layer/db/table.py index ae3814bee..46c1b410c 100644 --- a/src/layman/layer/db/table.py +++ b/src/layman/layer/db/table.py @@ -1,12 +1,13 @@ import logging from psycopg2 import sql +import requests_util.url_util from db import util as db_util from layman import settings, patch_mode, util as layman_util from layman.common import empty_method, empty_method_returns_none, empty_method_returns_dict from layman.http import LaymanError from . import get_internal_table_name -from .. import LAYER_TYPE, util as layer_util +from .. import LAYER_TYPE logger = logging.getLogger(__name__) PATCH_MODE = patch_mode.DELETE_IF_DEPENDANT @@ -35,7 +36,7 @@ def get_layer_info(workspace, layername,): 'schema': table_uri.schema, 'table': table_uri.table, 'geo_column': table_uri.geo_column, - 'external_uri': layer_util.redact_uri(table_uri.db_uri_str), + 'external_uri': requests_util.url_util.redact_uri(table_uri.db_uri_str), 'status': 'NOT_AVAILABLE', 'error': 'Cannot connect to DB.', } @@ -56,13 +57,13 @@ def get_layer_info(workspace, layername,): 'geo_column': table_uri.geo_column, } if layer_info['original_data_source'] == settings.EnumOriginalDataSource.TABLE.value: - result['db']['external_uri'] = layer_util.redact_uri(table_uri.db_uri_str) + result['db']['external_uri'] = requests_util.url_util.redact_uri(table_uri.db_uri_str) elif layer_info['original_data_source'] == settings.EnumOriginalDataSource.TABLE.value: result['db'] = { 'schema': table_uri.schema, 'table': table_uri.table, 'geo_column': table_uri.geo_column, - 'external_uri': layer_util.redact_uri(table_uri.db_uri_str), + 'external_uri': requests_util.url_util.redact_uri(table_uri.db_uri_str), 'status': 'NOT_AVAILABLE', 'error': 'Table does not exist.', } diff --git a/src/layman/layer/util.py b/src/layman/layer/util.py index 8aa2daddd..7e693f6b7 100644 --- a/src/layman/layer/util.py +++ b/src/layman/layer/util.py @@ -443,17 +443,6 @@ def parse_and_validate_external_table_uri_str(external_table_uri_str): return result -def redact_uri(uri): - url_components = parse.urlparse(uri) - if url_components.username or url_components.password: - hostname = f'{url_components.hostname}:{url_components.port}' if url_components.port else url_components.hostname - url_components = url_components._replace( - netloc=f"{url_components.username}@{hostname}", - ) - - return url_components.geturl() - - def set_wfs_wms_status_after_fail(workspace, name): keys = ['wfs', 'wms', 'style', 'geodata_type'] publ_info = layman_util.get_publication_info(workspace, LAYER_TYPE, name, context={'keys': keys}) diff --git a/src/layman/layer/util_test.py b/src/layman/layer/util_test.py index e4faf2066..20d6558b7 100644 --- a/src/layman/layer/util_test.py +++ b/src/layman/layer/util_test.py @@ -497,18 +497,3 @@ def test_validate_external_table_uri_str_parse(external_table_uri_str, scheme): }, } test_util.assert_error(exp_error, exc_info) - - -@pytest.mark.parametrize('uri, exp_redact_uri', [ - ('mysql://docker:docker@postgresql:5432/external_test_db?table=table_name&geo_column=wkb_geometry', - 'mysql://docker@postgresql:5432/external_test_db?table=table_name&geo_column=wkb_geometry'), - ('mysql://docker:@postgresql:5432/external_test_db?table=table_name&geo_column=wkb_geometry', - 'mysql://docker@postgresql:5432/external_test_db?table=table_name&geo_column=wkb_geometry'), - ('mysql://docker@postgresql:5432/external_test_db?table=table_name&geo_column=wkb_geometry', - 'mysql://docker@postgresql:5432/external_test_db?table=table_name&geo_column=wkb_geometry'), - ('mysql://postgresql:5432/external_test_db?table=table_name&geo_column=wkb_geometry', - 'mysql://postgresql:5432/external_test_db?table=table_name&geo_column=wkb_geometry'), -]) -def test_redact_uri(uri, exp_redact_uri): - redact_table_uri = util.redact_uri(uri) - assert redact_table_uri == exp_redact_uri diff --git a/src/requests_util/url_util.py b/src/requests_util/url_util.py new file mode 100644 index 000000000..adb2a1d0d --- /dev/null +++ b/src/requests_util/url_util.py @@ -0,0 +1,12 @@ +from urllib import parse + + +def redact_uri(uri): + url_components = parse.urlparse(uri) + if url_components.username or url_components.password: + hostname = f'{url_components.hostname}:{url_components.port}' if url_components.port else url_components.hostname + url_components = url_components._replace( + netloc=f"{url_components.username}@{hostname}", + ) + + return url_components.geturl() diff --git a/src/requests_util/url_util_test.py b/src/requests_util/url_util_test.py new file mode 100644 index 000000000..090f01101 --- /dev/null +++ b/src/requests_util/url_util_test.py @@ -0,0 +1,18 @@ +import pytest + +import requests_util.url_util + + +@pytest.mark.parametrize('uri, exp_redact_uri', [ + ('mysql://docker:docker@postgresql:5432/external_test_db?table=table_name&geo_column=wkb_geometry', + 'mysql://docker@postgresql:5432/external_test_db?table=table_name&geo_column=wkb_geometry'), + ('mysql://docker:@postgresql:5432/external_test_db?table=table_name&geo_column=wkb_geometry', + 'mysql://docker@postgresql:5432/external_test_db?table=table_name&geo_column=wkb_geometry'), + ('mysql://docker@postgresql:5432/external_test_db?table=table_name&geo_column=wkb_geometry', + 'mysql://docker@postgresql:5432/external_test_db?table=table_name&geo_column=wkb_geometry'), + ('mysql://postgresql:5432/external_test_db?table=table_name&geo_column=wkb_geometry', + 'mysql://postgresql:5432/external_test_db?table=table_name&geo_column=wkb_geometry'), +]) +def test_redact_uri(uri, exp_redact_uri): + redact_table_uri = requests_util.url_util.redact_uri(uri) + assert redact_table_uri == exp_redact_uri diff --git a/tests/asserts/final/publication/internal.py b/tests/asserts/final/publication/internal.py index 984cca07d..8ea9b6b4a 100644 --- a/tests/asserts/final/publication/internal.py +++ b/tests/asserts/final/publication/internal.py @@ -3,11 +3,13 @@ from psycopg2 import sql import pytest + +import requests_util.url_util from db import util as db_util, TableUri from layman import app, util as layman_util, settings, celery from layman.common import bbox as bbox_util from layman.common.prime_db_schema import publications -from layman.layer import LAYER_TYPE, util as layer_util +from layman.layer import LAYER_TYPE from layman.layer.filesystem import gdal, input_file from layman.map import MAP_TYPE, util as map_util from layman.map.filesystem import input_file as map_input_file @@ -223,7 +225,7 @@ def correct_values_in_detail(workspace, publ_type, name, *, exp_publication_deta 'wfs': {'url': f'{settings.LAYMAN_GS_PROXY_BASE_URL}{workspace}/wfs'}, '_table_uri': table_uri, 'db': { - 'external_uri': layer_util.redact_uri(table_uri.db_uri_str), + 'external_uri': requests_util.url_util.redact_uri(table_uri.db_uri_str), 'schema': table_uri.schema, 'table': table_uri.table, 'geo_column': table_uri.geo_column, From 684b860670a72b6c6614822bf540ab5da8aac708 Mon Sep 17 00:00:00 2001 From: index-git Date: Tue, 12 Dec 2023 15:51:48 +0100 Subject: [PATCH 04/11] Enable redact_uri with remove_user --- src/requests_util/url_util.py | 4 ++-- src/requests_util/url_util_test.py | 22 +++++++++++++++------- 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/src/requests_util/url_util.py b/src/requests_util/url_util.py index adb2a1d0d..1233b8611 100644 --- a/src/requests_util/url_util.py +++ b/src/requests_util/url_util.py @@ -1,12 +1,12 @@ from urllib import parse -def redact_uri(uri): +def redact_uri(uri, *, remove_username=False): url_components = parse.urlparse(uri) if url_components.username or url_components.password: hostname = f'{url_components.hostname}:{url_components.port}' if url_components.port else url_components.hostname url_components = url_components._replace( - netloc=f"{url_components.username}@{hostname}", + netloc=f"{url_components.username}@{hostname}" if not remove_username else f"{hostname}", ) return url_components.geturl() diff --git a/src/requests_util/url_util_test.py b/src/requests_util/url_util_test.py index 090f01101..288628fc8 100644 --- a/src/requests_util/url_util_test.py +++ b/src/requests_util/url_util_test.py @@ -3,16 +3,24 @@ import requests_util.url_util -@pytest.mark.parametrize('uri, exp_redact_uri', [ - ('mysql://docker:docker@postgresql:5432/external_test_db?table=table_name&geo_column=wkb_geometry', +@pytest.mark.parametrize('uri, remove_username, exp_redact_uri', [ + ('mysql://docker:docker@postgresql:5432/external_test_db?table=table_name&geo_column=wkb_geometry', False, 'mysql://docker@postgresql:5432/external_test_db?table=table_name&geo_column=wkb_geometry'), - ('mysql://docker:@postgresql:5432/external_test_db?table=table_name&geo_column=wkb_geometry', + ('mysql://docker:@postgresql:5432/external_test_db?table=table_name&geo_column=wkb_geometry', False, 'mysql://docker@postgresql:5432/external_test_db?table=table_name&geo_column=wkb_geometry'), - ('mysql://docker@postgresql:5432/external_test_db?table=table_name&geo_column=wkb_geometry', + ('mysql://docker@postgresql:5432/external_test_db?table=table_name&geo_column=wkb_geometry', False, 'mysql://docker@postgresql:5432/external_test_db?table=table_name&geo_column=wkb_geometry'), - ('mysql://postgresql:5432/external_test_db?table=table_name&geo_column=wkb_geometry', + ('mysql://postgresql:5432/external_test_db?table=table_name&geo_column=wkb_geometry', False, + 'mysql://postgresql:5432/external_test_db?table=table_name&geo_column=wkb_geometry'), + ('mysql://docker:docker@postgresql:5432/external_test_db?table=table_name&geo_column=wkb_geometry', True, + 'mysql://postgresql:5432/external_test_db?table=table_name&geo_column=wkb_geometry'), + ('mysql://docker:@postgresql:5432/external_test_db?table=table_name&geo_column=wkb_geometry', True, + 'mysql://postgresql:5432/external_test_db?table=table_name&geo_column=wkb_geometry'), + ('mysql://docker@postgresql:5432/external_test_db?table=table_name&geo_column=wkb_geometry', True, + 'mysql://postgresql:5432/external_test_db?table=table_name&geo_column=wkb_geometry'), + ('mysql://postgresql:5432/external_test_db?table=table_name&geo_column=wkb_geometry', True, 'mysql://postgresql:5432/external_test_db?table=table_name&geo_column=wkb_geometry'), ]) -def test_redact_uri(uri, exp_redact_uri): - redact_table_uri = requests_util.url_util.redact_uri(uri) +def test_redact_uri(uri, remove_username, exp_redact_uri): + redact_table_uri = requests_util.url_util.redact_uri(uri, remove_username=remove_username) assert redact_table_uri == exp_redact_uri From e28652a2c10975fa1517f7e323759ebc8fcee061 Mon Sep 17 00:00:00 2001 From: index-git Date: Thu, 14 Dec 2023 11:57:34 +0100 Subject: [PATCH 05/11] Ensure JDBC role service internal schema in setup_geoserver.py --- Makefile | 29 +++++++++++----- src/clear_layman_data.py | 18 ---------- src/layman/upgrade/upgrade_v1_23.py | 5 ++- src/layman_settings.py | 1 + src/setup_geoserver.py | 53 +++++++++++++++++++++++++++++ test.sh | 10 +++--- test_separated.sh | 4 +-- test_static.sh | 4 +-- 8 files changed, 88 insertions(+), 36 deletions(-) diff --git a/Makefile b/Makefile index 2c80ed756..9d8a15476 100644 --- a/Makefile +++ b/Makefile @@ -3,24 +3,28 @@ start-demo: mkdir -p layman_data deps/qgis/data docker compose -f docker-compose.deps.demo.yml -f docker-compose.demo.yml build layman layman_client timgen + docker compose -f docker-compose.deps.demo.yml -f docker-compose.demo.yml up -d postgresql docker compose -f docker-compose.deps.demo.yml -f docker-compose.demo.yml run --rm --no-deps -u root layman bash -c "cd src && python3 -B setup_geoserver.py" - docker compose -f docker-compose.deps.demo.yml -f docker-compose.demo.yml up -d --force-recreate postgresql qgis geoserver redis layman celery_worker flower timgen layman_client nginx + docker compose -f docker-compose.deps.demo.yml -f docker-compose.demo.yml up -d --force-recreate qgis geoserver redis layman celery_worker flower timgen layman_client nginx start-demo-full: mkdir -p layman_data deps/qgis/data docker compose -f docker-compose.deps.demo.yml -f docker-compose.demo.yml build layman layman_client timgen + docker compose -f docker-compose.deps.demo.yml -f docker-compose.demo.yml up -d postgresql docker compose -f docker-compose.deps.demo.yml -f docker-compose.demo.yml run --rm --no-deps -u root layman bash -c "cd src && python3 -B setup_geoserver.py" - docker compose -f docker-compose.deps.demo.yml -f docker-compose.demo.yml up -d --force-recreate postgresql qgis geoserver redis layman celery_worker flower timgen layman_client micka nginx + docker compose -f docker-compose.deps.demo.yml -f docker-compose.demo.yml up -d --force-recreate qgis geoserver redis layman celery_worker flower timgen layman_client micka nginx start-demo-only: mkdir -p layman_data deps/qgis/data docker compose -f docker-compose.deps.demo.yml -f docker-compose.demo.yml build layman layman_client timgen + docker compose -f docker-compose.deps.demo.yml -f docker-compose.demo.yml up -d postgresql docker compose -f docker-compose.deps.demo.yml -f docker-compose.demo.yml run --rm --no-deps -u root layman bash -c "cd src && python3 -B setup_geoserver.py" docker compose -f docker-compose.deps.demo.yml -f docker-compose.demo.yml up -d --force-recreate --no-deps layman celery_worker flower timgen layman_client nginx start-demo-full-with-optional-deps: mkdir -p layman_data deps/qgis/data docker compose -f docker-compose.deps.demo.yml -f docker-compose.demo.yml build layman layman_client timgen + docker compose -f docker-compose.deps.demo.yml -f docker-compose.demo.yml up -d postgresql docker compose -f docker-compose.deps.demo.yml -f docker-compose.demo.yml run --rm --no-deps -u root layman bash -c "cd src && python3 -B setup_geoserver.py" docker compose -f docker-compose.deps.demo.yml -f docker-compose.demo.yml up -d --force-recreate @@ -33,15 +37,17 @@ build-demo: upgrade-demo: mkdir -p layman_data deps/qgis/data docker compose -f docker-compose.deps.demo.yml -f docker-compose.demo.yml build layman layman_client timgen + docker compose -f docker-compose.deps.demo.yml -f docker-compose.demo.yml up -d postgresql docker compose -f docker-compose.deps.demo.yml -f docker-compose.demo.yml run --rm --no-deps -u root layman bash -c "cd src && python3 -B setup_geoserver.py" - docker compose -f docker-compose.deps.demo.yml -f docker-compose.demo.yml up -d --force-recreate --no-deps postgresql qgis geoserver redis timgen layman_client nginx + docker compose -f docker-compose.deps.demo.yml -f docker-compose.demo.yml up -d --force-recreate --no-deps qgis geoserver redis timgen layman_client nginx docker compose -f docker-compose.deps.demo.yml -f docker-compose.demo.yml run --rm --no-deps layman bash -c "cd src && python3 layman_flush_redis.py && python3 wait_for_deps.py && python3 standalone_upgrade.py" upgrade-demo-full: mkdir -p layman_data deps/qgis/data docker compose -f docker-compose.deps.demo.yml -f docker-compose.demo.yml build layman layman_client timgen + docker compose -f docker-compose.deps.demo.yml -f docker-compose.demo.yml up -d postgresql docker compose -f docker-compose.deps.demo.yml -f docker-compose.demo.yml run --rm --no-deps -u root layman bash -c "cd src && python3 -B setup_geoserver.py" - docker compose -f docker-compose.deps.demo.yml -f docker-compose.demo.yml up -d --force-recreate --no-deps postgresql qgis geoserver redis timgen layman_client micka nginx + docker compose -f docker-compose.deps.demo.yml -f docker-compose.demo.yml up -d --force-recreate --no-deps qgis geoserver redis timgen layman_client micka nginx docker compose -f docker-compose.deps.demo.yml -f docker-compose.demo.yml run --rm --no-deps layman bash -c "cd src && python3 layman_flush_redis.py && python3 wait_for_deps.py && python3 standalone_upgrade.py" upgrade-after-timeout: @@ -55,6 +61,7 @@ deps-stop: start-dev: mkdir -p layman_data layman_data_test tmp deps/qgis/data + docker compose -f docker-compose.deps.yml -f docker-compose.dev.yml up -d postgresql docker compose -f docker-compose.deps.yml -f docker-compose.dev.yml run --rm --no-deps -u root layman_dev bash -c "cd src && python3 -B setup_geoserver.py" docker compose -f docker-compose.deps.yml -f docker-compose.dev.yml up --force-recreate -d @@ -64,6 +71,7 @@ stop-dev: start-dev-only: mkdir -p layman_data layman_data_test tmp deps/qgis/data docker compose -f docker-compose.deps.yml -f docker-compose.dev.yml rm -fsv layman_dev celery_worker_dev flower timgen layman_client + docker compose -f docker-compose.deps.yml -f docker-compose.dev.yml up -d postgresql docker compose -f docker-compose.deps.yml -f docker-compose.dev.yml run --rm --no-deps -u root layman_dev bash -c "cd src && python3 -B setup_geoserver.py" docker compose -f docker-compose.deps.yml -f docker-compose.dev.yml up -d layman_dev celery_worker_dev flower timgen layman_client @@ -93,8 +101,9 @@ restart-celery-dev: upgrade-dev: mkdir -p layman_data layman_data_test tmp deps/qgis/data + docker compose -f docker-compose.deps.yml -f docker-compose.dev.yml up -d postgresql docker compose -f docker-compose.deps.yml -f docker-compose.dev.yml run --rm --no-deps -u root layman_dev bash -c "cd src && python3 -B setup_geoserver.py" - docker compose -f docker-compose.deps.yml -f docker-compose.dev.yml up -d timgen layman_client postgresql qgis nginx-qgis geoserver redis micka + docker compose -f docker-compose.deps.yml -f docker-compose.dev.yml up -d timgen layman_client qgis nginx-qgis geoserver redis micka docker compose -f docker-compose.deps.yml -f docker-compose.dev.yml run --rm --no-deps layman_dev bash -c "cd src && python3 layman_flush_redis.py && python3 wait_for_deps.py && python3 standalone_upgrade.py" prepare-dirs: @@ -194,7 +203,8 @@ test: docker compose -f docker-compose.deps.yml -f docker-compose.test.yml build layman_client_test ; \ fi; docker compose -f docker-compose.deps.yml -f docker-compose.test.yml rm -f layman_test - docker compose -f docker-compose.deps.yml -f docker-compose.test.yml run --rm --no-deps layman_test bash -c "bash ensure-test-data.sh" + docker compose -f docker-compose.deps.yml -f docker-compose.test.yml up -d postgresql + docker compose -f docker-compose.deps.yml -f docker-compose.test.yml run --rm --no-deps layman_test bash -c "bash ensure-test-data.sh && python3 src/assert_db.py" docker compose -f docker-compose.deps.yml -f docker-compose.test.yml run --rm --no-deps -u root layman_test bash -c "cd src && python3 -B setup_geoserver.py" docker compose -f docker-compose.deps.yml -f docker-compose.test.yml up --force-recreate --no-deps -d celery_worker_test docker compose -f docker-compose.deps.yml -f docker-compose.test.yml run --rm --name layman_test_run_1 layman_test @@ -214,7 +224,8 @@ test-separated: docker compose -f docker-compose.deps.yml -f docker-compose.test.yml build layman_client_test ; \ fi; docker compose -f docker-compose.deps.yml -f docker-compose.test.yml rm -f layman_test - docker compose -f docker-compose.deps.yml -f docker-compose.test.yml run --rm --no-deps layman_test bash -c "bash ensure-test-data.sh" + docker compose -f docker-compose.deps.yml -f docker-compose.test.yml up -d postgresql + docker compose -f docker-compose.deps.yml -f docker-compose.test.yml run --rm --no-deps layman_test bash -c "bash ensure-test-data.sh && python3 src/assert_db.py" docker compose -f docker-compose.deps.yml -f docker-compose.test.yml run --rm --no-deps -u root layman_test bash -c "cd src && python3 -B setup_geoserver.py" docker compose -f docker-compose.deps.yml -f docker-compose.test.yml up --force-recreate --no-deps -d celery_worker_test docker compose -f docker-compose.deps.yml -f docker-compose.test.yml run --rm --name layman_test_run_1 -e "TEST_TYPE=$(test_type)" layman_test bash -c "bash test_separated.sh $(max_fail)" @@ -234,7 +245,8 @@ test-static: docker compose -f docker-compose.deps.yml -f docker-compose.test.yml build layman_client_test ; \ fi; docker compose -f docker-compose.deps.yml -f docker-compose.test.yml rm -f layman_test - docker compose -f docker-compose.deps.yml -f docker-compose.test.yml run --rm --no-deps layman_test bash -c "bash ensure-test-data.sh" + docker compose -f docker-compose.deps.yml -f docker-compose.test.yml up -d postgresql + docker compose -f docker-compose.deps.yml -f docker-compose.test.yml run --rm --no-deps layman_test bash -c "bash ensure-test-data.sh && python3 src/assert_db.py" docker compose -f docker-compose.deps.yml -f docker-compose.test.yml run --rm --no-deps -u root layman_test bash -c "cd src && python3 -B setup_geoserver.py" docker compose -f docker-compose.deps.yml -f docker-compose.test.yml up --force-recreate --no-deps -d celery_worker_test docker compose -f docker-compose.deps.yml -f docker-compose.test.yml run --rm --name layman_test_run_1 layman_test bash -c "bash test_static.sh" @@ -283,6 +295,7 @@ geoserver-exec: docker compose -f docker-compose.deps.yml exec geoserver bash geoserver-ensure-authn: + docker compose -f docker-compose.deps.yml -f docker-compose.dev.yml up -d postgresql docker compose -f docker-compose.deps.yml -f docker-compose.dev.yml run --rm --no-deps -u root layman_dev bash -c "cd src && python3 -B setup_geoserver.py" get-current-user: diff --git a/src/clear_layman_data.py b/src/clear_layman_data.py index bb3d27cab..1f270c052 100644 --- a/src/clear_layman_data.py +++ b/src/clear_layman_data.py @@ -120,24 +120,6 @@ def main(): ) response.raise_for_status() - for role in roles: - response = requests.delete( - urljoin(geoserver.GS_REST_ROLES, f'role/{role}/user/{user}/'), - headers=headers_json, - auth=auth, - timeout=settings.DEFAULT_CONNECTION_TIMEOUT, - ) - response.raise_for_status() - - response = requests.delete( - urljoin(geoserver.GS_REST_ROLES, 'role/' + f"USER_{user.upper()}"), - headers=headers_json, - auth=auth, - timeout=settings.DEFAULT_CONNECTION_TIMEOUT, - ) - if response.status_code != 404: - response.raise_for_status() - response = requests.delete( urljoin(geoserver.GS_REST_USER, user), headers=headers_json, diff --git a/src/layman/upgrade/upgrade_v1_23.py b/src/layman/upgrade/upgrade_v1_23.py index b9fa652bd..1f6c82ff8 100644 --- a/src/layman/upgrade/upgrade_v1_23.py +++ b/src/layman/upgrade/upgrade_v1_23.py @@ -31,7 +31,10 @@ def adjust_db_for_roles(): def create_role_service_schema(): logger.info(f' Create internal role service schema') - create_schema = f"""CREATE SCHEMA IF NOT EXISTS "{ROLE_SERVICE_SCHEMA}" AUTHORIZATION {settings.LAYMAN_PG_USER};""" + drop_temporary_views = f"""drop schema if exists "{ROLE_SERVICE_SCHEMA}" CASCADE;""" + db_util.run_statement(drop_temporary_views) + + create_schema = f"""CREATE SCHEMA "{ROLE_SERVICE_SCHEMA}" AUTHORIZATION {settings.LAYMAN_PG_USER};""" db_util.run_statement(create_schema) create_role_table = f"""create table {ROLE_SERVICE_SCHEMA}.bussiness_roles( diff --git a/src/layman_settings.py b/src/layman_settings.py index 27dbe0593..6938151df 100644 --- a/src/layman_settings.py +++ b/src/layman_settings.py @@ -165,6 +165,7 @@ class EnumWfsWmsStatus(Enum): 'pg_toast', 'information_schema', LAYMAN_PRIME_SCHEMA, + LAYMAN_INTERNAL_ROLE_SERVICE_SCHEMA, ] PG_POSTGIS_SCHEMA = 'public' diff --git a/src/setup_geoserver.py b/src/setup_geoserver.py index 5010e3a37..ad037f02d 100644 --- a/src/setup_geoserver.py +++ b/src/setup_geoserver.py @@ -1,6 +1,8 @@ import logging import sys +import time +from db import util as db_util import geoserver from geoserver import epsg_properties from geoserver import authn @@ -12,6 +14,56 @@ logging.basicConfig(stream=sys.stdout, level=logging.DEBUG) +def wait_for_db(conn_dict): + max_attempts = 60 + attempt = 0 + + while True: + import psycopg2 + try: + with psycopg2.connect(**conn_dict): + pass + logger.info(f" Attempt {attempt}/{max_attempts} successful.") + break + except psycopg2.OperationalError: + if attempt >= max_attempts: + logger.info(f" Reaching max attempts when waiting for DB") + sys.exit(1) + time.sleep(2) + attempt += 1 + + +def ensure_jdbc_role_service_internal_schema(): + db_conn = settings.PG_CONN + internal_service_schema = settings.LAYMAN_INTERNAL_ROLE_SERVICE_SCHEMA + uri_str = settings.PG_URI_STR + + logger.info(f"Ensuring internal JDBC role service schema. db_conn={db_conn}") + + logger.info(f" Waiting for DB") + wait_for_db(db_conn) + + logger.info(f" Checking internal role service DB schema") + schema_query = f'''SELECT COUNT(*) FROM information_schema.schemata WHERE schema_name = '{internal_service_schema}';''' + schema_exists = db_util.run_query(schema_query, uri_str=uri_str)[0][0] + if schema_exists == 0: + logger.info(f" Setting up internal role service DB schema") + statement = f""" + CREATE SCHEMA "{internal_service_schema}" AUTHORIZATION {settings.LAYMAN_PG_USER}; + create view {internal_service_schema}.roles as select 'ADMIN' as name, null as parent + union all select 'GROUP_ADMIN', null + union all select %s, null + ; + create view {internal_service_schema}.role_props as select null::varchar as rolename, null::varchar as propname, null::varchar as propvalue; + create view {internal_service_schema}.user_roles as select %s as username, 'ADMIN' as rolename + union all select %s, %s + union all select %s, 'ADMIN' + ; + create view {internal_service_schema}.group_roles as select null::varchar as groupname, null::varchar as rolename; + """ + db_util.run_statement(statement, data=(settings.LAYMAN_GS_ROLE, settings.LAYMAN_GS_USER, settings.LAYMAN_GS_USER, settings.LAYMAN_GS_ROLE, settings.GEOSERVER_ADMIN_USER, ), uri_str=uri_str) + + def main(): geoserver.ensure_data_dir(settings.GEOSERVER_DATADIR, settings.GEOSERVER_INITIAL_DATADIR, @@ -24,6 +76,7 @@ def main(): settings.LAYMAN_GS_ROLE_SERVICE, settings.LAYMAN_GS_AUTHN_FILTER_NAME_OLD, ) + ensure_jdbc_role_service_internal_schema() epsg_properties.setup_epsg(settings.GEOSERVER_DATADIR, set(settings.LAYMAN_OUTPUT_SRS_LIST)) diff --git a/test.sh b/test.sh index 813eb8ab5..26584794d 100644 --- a/test.sh +++ b/test.sh @@ -9,12 +9,12 @@ rm -rf tmp/artifacts/* if [ "$CI" == "true" ] then - python3 src/assert_db.py && python3 src/wait_for_deps.py && python3 src/clear_layman_data.py && python3 -m pytest -m "not irritating" --timeout=60 -W ignore::DeprecationWarning -xvv src + python3 src/wait_for_deps.py && python3 src/clear_layman_data.py && python3 -m pytest -m "not irritating" --timeout=60 -W ignore::DeprecationWarning -xvv src else - python3 src/assert_db.py && python3 src/wait_for_deps.py && python3 src/clear_layman_data.py && python3 -m pytest -W ignore::DeprecationWarning -xvv src + python3 src/wait_for_deps.py && python3 src/clear_layman_data.py && python3 -m pytest -W ignore::DeprecationWarning -xvv src fi -#python3 src/assert_db.py && python3 src/wait_for_deps.py && python3 src/clear_layman_data.py && python3 -m pytest -W ignore::DeprecationWarning --capture=tee-sys -xvv src/layman/gs_wfs_proxy_test.py -#python3 src/assert_db.py && python3 src/wait_for_deps.py && python3 src/clear_layman_data.py && python3 -m pytest -W ignore::DeprecationWarning -xsvv src/layman/layer/client_test.py -#python3 src/assert_db.py && python3 src/wait_for_deps.py && python3 src/clear_layman_data.py && python3 -m pytest -W ignore::DeprecationWarning -xsvv src/layman/layer/rest_test.py::test_post_layers_complex src/layman/layer/rest_test.py::test_patch_layer_data src/layman/layer/rest_test.py::test_patch_layer_concurrent_and_delete_it +#python3 src/wait_for_deps.py && python3 src/clear_layman_data.py && python3 -m pytest -W ignore::DeprecationWarning --capture=tee-sys -xvv src/layman/gs_wfs_proxy_test.py +#python3 src/wait_for_deps.py && python3 src/clear_layman_data.py && python3 -m pytest -W ignore::DeprecationWarning -xsvv src/layman/layer/client_test.py +#python3 src/wait_for_deps.py && python3 src/clear_layman_data.py && python3 -m pytest -W ignore::DeprecationWarning -xsvv src/layman/layer/rest_test.py::test_post_layers_complex src/layman/layer/rest_test.py::test_patch_layer_data src/layman/layer/rest_test.py::test_patch_layer_concurrent_and_delete_it diff --git a/test_separated.sh b/test_separated.sh index e6f04c94d..7711f9b86 100644 --- a/test_separated.sh +++ b/test_separated.sh @@ -10,5 +10,5 @@ rm -rf tmp/artifacts/* max_fail="$1" if [ -z "$max_fail" ]; then max_fail=1; fi -python3 src/assert_db.py && python3 src/wait_for_deps.py && python3 src/clear_layman_data.py && python3 -m pytest -W ignore::DeprecationWarning --maxfail="$max_fail" -vv --ignore=tests/static_data/ tests -#python3 src/assert_db.py && python3 src/wait_for_deps.py && python3 src/clear_layman_data.py && python3 -m TEST_TYPE=optional pytest -W ignore::DeprecationWarning -sxvv --capture=tee-sys --nocleanup --ignore=tests/static_data/ tests +python3 src/wait_for_deps.py && python3 src/clear_layman_data.py && python3 -m pytest -W ignore::DeprecationWarning --maxfail="$max_fail" -vv --ignore=tests/static_data/ tests +#python3 src/wait_for_deps.py && python3 src/clear_layman_data.py && python3 -m TEST_TYPE=optional pytest -W ignore::DeprecationWarning -sxvv --capture=tee-sys --nocleanup --ignore=tests/static_data/ tests diff --git a/test_static.sh b/test_static.sh index 6e0930627..f92f8c1e9 100644 --- a/test_static.sh +++ b/test_static.sh @@ -7,5 +7,5 @@ bash src/clear-python-cache.sh mkdir -p tmp/artifacts rm -rf tmp/artifacts/* -python3 src/assert_db.py && python3 src/wait_for_deps.py && python3 src/clear_layman_data.py && python3 -m pytest -W ignore::DeprecationWarning -xvv tests/static_data -#python3 src/assert_db.py && python3 src/wait_for_deps.py && python3 src/clear_layman_data.py && python3 -m pytest -W ignore::DeprecationWarning -sxvv --capture=tee-sys --nocleanup tests/dynamic_data/publication_test.py +python3 src/wait_for_deps.py && python3 src/clear_layman_data.py && python3 -m pytest -W ignore::DeprecationWarning -xvv tests/static_data +#python3 src/wait_for_deps.py && python3 src/clear_layman_data.py && python3 -m pytest -W ignore::DeprecationWarning -sxvv --capture=tee-sys --nocleanup tests/dynamic_data/publication_test.py From 125e350efbe7c0ee8fc66f8429bc533ef6b05015 Mon Sep 17 00:00:00 2001 From: index-git Date: Thu, 14 Dec 2023 14:06:26 +0100 Subject: [PATCH 06/11] Default values of GS JDBC role service files --- src/geoserver/jdbc_role_service/config.xml | 15 +++ src/geoserver/jdbc_role_service/rolesddl.xml | 33 ++++++ src/geoserver/jdbc_role_service/rolesdml.xml | 106 +++++++++++++++++++ 3 files changed, 154 insertions(+) create mode 100644 src/geoserver/jdbc_role_service/config.xml create mode 100644 src/geoserver/jdbc_role_service/rolesddl.xml create mode 100644 src/geoserver/jdbc_role_service/rolesdml.xml diff --git a/src/geoserver/jdbc_role_service/config.xml b/src/geoserver/jdbc_role_service/config.xml new file mode 100644 index 000000000..f5b549ac7 --- /dev/null +++ b/src/geoserver/jdbc_role_service/config.xml @@ -0,0 +1,15 @@ + + -52d98abd:18c3f4945bb:-7ffa + layman_role_service + org.geoserver.security.jdbc.JDBCRoleService + rolesddl.xml + rolesdml.xml + false + org.postgresql.Driver + jdbc:postgresql://postgres:5432/gis + docker + crypt1:E/7ccSbjdBTkK3i/PfgCvA== + false + ADMIN + GROUP_ADMIN + diff --git a/src/geoserver/jdbc_role_service/rolesddl.xml b/src/geoserver/jdbc_role_service/rolesddl.xml new file mode 100644 index 000000000..592a2772c --- /dev/null +++ b/src/geoserver/jdbc_role_service/rolesddl.xml @@ -0,0 +1,33 @@ + + + + DDL statements for role database + role_props + + create table roles(name varchar(64) not null,parent varchar(64), primary key(name)) + + + create table role_props(rolename varchar(64) not null,propname varchar(64) not null, propvalue varchar(2048),primary key (rolename,propname)) + + + + create table user_roles(username varchar(128) not null, rolename varchar(64) not null, primary key(username,rolename)) + + + create index user_roles_idx on user_roles(rolename,username) + + + create table group_roles(groupname varchar(128) not null, rolename varchar(64) not null, primary key(groupname,rolename)) + + + create index group_roles_idx on group_roles(rolename,groupname) + + + + + drop table roles + drop table role_props + drop table user_roles + drop table group_roles + + diff --git a/src/geoserver/jdbc_role_service/rolesdml.xml b/src/geoserver/jdbc_role_service/rolesdml.xml new file mode 100644 index 000000000..7ef46b819 --- /dev/null +++ b/src/geoserver/jdbc_role_service/rolesdml.xml @@ -0,0 +1,106 @@ + + + + DML statements for role database + + + select count(*) from roles + + + select name,parent from roles + + + select parent from roles where name = ? + + + insert into roles (name) values (?) + + + + update roles set name=name where name = ? + + + update roles set parent = ? where name = ? + + + update roles set parent = null where parent = ? + + + delete from roles where name = ? + + + delete from roles + + + + + select rolename,propname,propvalue from role_props + + + select propname,propvalue from role_props where rolename = ? + + + select p.rolename,p.propname,p.propvalue from role_props p,user_roles u where u.rolename = p.rolename and u.username = ? + + + select p.rolename,p.propname,p.propvalue from role_props p,group_roles g where g.rolename = p.rolename and g.groupname = ? + + + delete from role_props where rolename=? + + + insert into role_props(rolename,propname,propvalue) values (?,?,?) + + + delete from role_props + + + + + select u.rolename,r.parent from user_roles u ,roles r where r.name=u.rolename and u.username = ? + + + select username from user_roles where rolename = ? + + + insert into user_roles(rolename,username) values (?,?) + + + delete from user_roles where rolename=? and username = ? + + + delete from user_roles where rolename=? + + + delete from user_roles where username = ? + + + delete from user_roles + + + + + + select g.rolename,r.parent from group_roles g,roles r where g.rolename = r.name and g.groupname = ? + + + select groupname from group_roles where rolename = ? + + + insert into group_roles(rolename,groupname) values (?,?) + + + delete from group_roles where rolename=? and groupname = ? + + + delete from group_roles where rolename=? + + + delete from group_roles where groupname = ? + + + delete from group_roles + + + + From 7632da2dcb3f2b927528af121b2b255252e4a206 Mon Sep 17 00:00:00 2001 From: index-git Date: Thu, 14 Dec 2023 14:48:48 +0100 Subject: [PATCH 07/11] Set up GS JDBC role service --- src/geoserver/jdbc_role_service/config.xml | 6 +- src/geoserver/jdbc_role_service/rolesddl.xml | 20 +++---- src/geoserver/jdbc_role_service/rolesdml.xml | 60 ++++++++++---------- src/geoserver/role_service.py | 52 +++++++++++++++++ src/setup_geoserver.py | 8 ++- 5 files changed, 101 insertions(+), 45 deletions(-) create mode 100644 src/geoserver/role_service.py diff --git a/src/geoserver/jdbc_role_service/config.xml b/src/geoserver/jdbc_role_service/config.xml index f5b549ac7..30dcf7e03 100644 --- a/src/geoserver/jdbc_role_service/config.xml +++ b/src/geoserver/jdbc_role_service/config.xml @@ -6,9 +6,9 @@ rolesdml.xml false org.postgresql.Driver - jdbc:postgresql://postgres:5432/gis - docker - crypt1:E/7ccSbjdBTkK3i/PfgCvA== + {connection_string} + {username} + {password} false ADMIN GROUP_ADMIN diff --git a/src/geoserver/jdbc_role_service/rolesddl.xml b/src/geoserver/jdbc_role_service/rolesddl.xml index 592a2772c..dcccf550c 100644 --- a/src/geoserver/jdbc_role_service/rolesddl.xml +++ b/src/geoserver/jdbc_role_service/rolesddl.xml @@ -4,30 +4,30 @@ DDL statements for role database role_props - create table roles(name varchar(64) not null,parent varchar(64), primary key(name)) + create table {schema}.roles(name varchar(64) not null,parent varchar(64), primary key(name)) - create table role_props(rolename varchar(64) not null,propname varchar(64) not null, propvalue varchar(2048),primary key (rolename,propname)) + create table {schema}.role_props(rolename varchar(64) not null,propname varchar(64) not null, propvalue varchar(2048),primary key (rolename,propname)) - create table user_roles(username varchar(128) not null, rolename varchar(64) not null, primary key(username,rolename)) + create table {schema}.user_roles(username varchar(128) not null, rolename varchar(64) not null, primary key(username,rolename)) - create index user_roles_idx on user_roles(rolename,username) + create index {schema}.user_roles_idx on user_roles(rolename,username) - create table group_roles(groupname varchar(128) not null, rolename varchar(64) not null, primary key(groupname,rolename)) + create table {schema}.group_roles(groupname varchar(128) not null, rolename varchar(64) not null, primary key(groupname,rolename)) - create index group_roles_idx on group_roles(rolename,groupname) + create index group_roles_idx on {schema}.group_roles(rolename,groupname) - drop table roles - drop table role_props - drop table user_roles - drop table group_roles + drop table {schema}.roles + drop table {schema}.role_props + drop table {schema}.user_roles + drop table {schema}.group_roles diff --git a/src/geoserver/jdbc_role_service/rolesdml.xml b/src/geoserver/jdbc_role_service/rolesdml.xml index 7ef46b819..c9f1d1b59 100644 --- a/src/geoserver/jdbc_role_service/rolesdml.xml +++ b/src/geoserver/jdbc_role_service/rolesdml.xml @@ -4,102 +4,102 @@ DML statements for role database - select count(*) from roles + select count(*) from {schema}.roles - select name,parent from roles + select name,parent from {schema}.roles - select parent from roles where name = ? + select parent from {schema}.roles where name = ? - insert into roles (name) values (?) + insert into {schema}.roles (name) values (?) - update roles set name=name where name = ? + update {schema}.roles set name=name where name = ? - update roles set parent = ? where name = ? + update {schema}.roles set parent = ? where name = ? - update roles set parent = null where parent = ? + update {schema}.roles set parent = null where parent = ? - delete from roles where name = ? + delete from {schema}.roles where name = ? - delete from roles + delete from {schema}.roles - select rolename,propname,propvalue from role_props + select rolename,propname,propvalue from {schema}.role_props - select propname,propvalue from role_props where rolename = ? + select propname,propvalue from {schema}.role_props where rolename = ? - select p.rolename,p.propname,p.propvalue from role_props p,user_roles u where u.rolename = p.rolename and u.username = ? + select p.rolename,p.propname,p.propvalue from {schema}.role_props p,{schema}.user_roles u where u.rolename = p.rolename and u.username = ? - select p.rolename,p.propname,p.propvalue from role_props p,group_roles g where g.rolename = p.rolename and g.groupname = ? + select p.rolename,p.propname,p.propvalue from {schema}.role_props p,{schema}.group_roles g where g.rolename = p.rolename and g.groupname = ? - delete from role_props where rolename=? + delete from {schema}.role_props where rolename=? - insert into role_props(rolename,propname,propvalue) values (?,?,?) + insert into {schema}.role_props(rolename,propname,propvalue) values (?,?,?) - delete from role_props + delete from {schema}.role_props - select u.rolename,r.parent from user_roles u ,roles r where r.name=u.rolename and u.username = ? + select u.rolename,r.parent from {schema}.user_roles u ,{schema}.roles r where r.name=u.rolename and u.username = ? - select username from user_roles where rolename = ? + select username from {schema}.user_roles where rolename = ? - insert into user_roles(rolename,username) values (?,?) + insert into {schema}.user_roles(rolename,username) values (?,?) - delete from user_roles where rolename=? and username = ? + delete from {schema}.user_roles where rolename=? and username = ? - delete from user_roles where rolename=? + delete from {schema}.user_roles where rolename=? - delete from user_roles where username = ? + delete from {schema}.user_roles where username = ? - delete from user_roles + delete from {schema}.user_roles - select g.rolename,r.parent from group_roles g,roles r where g.rolename = r.name and g.groupname = ? + select g.rolename,r.parent from {schema}.group_roles g,r{schema}.oles r where g.rolename = r.name and g.groupname = ? - select groupname from group_roles where rolename = ? + select groupname from {schema}.group_roles where rolename = ? - insert into group_roles(rolename,groupname) values (?,?) + insert into {schema}.group_roles(rolename,groupname) values (?,?) - delete from group_roles where rolename=? and groupname = ? + delete from {schema}.group_roles where rolename=? and groupname = ? - delete from group_roles where rolename=? + delete from {schema}.group_roles where rolename=? - delete from group_roles where groupname = ? + delete from {schema}.group_roles where groupname = ? - delete from group_roles + delete from {schema}.group_roles diff --git a/src/geoserver/role_service.py b/src/geoserver/role_service.py new file mode 100644 index 000000000..cdf633137 --- /dev/null +++ b/src/geoserver/role_service.py @@ -0,0 +1,52 @@ +import logging +import os +import shutil +import sys +from urllib.parse import urlparse +from xml.sax.saxutils import escape + +from requests_util import url_util + +logger = logging.getLogger(__name__) +logging.basicConfig(stream=sys.stdout, level=logging.DEBUG) + + +ROLE_SERVICE_PATH = 'security/role/' +DIRECTORY = os.path.dirname(os.path.abspath(__file__)) + + +def setup_jdbc_role_service(data_dir, service_url, role_service_name, db_schema): + logger.info(f"Ensuring GeoServer DB role service '{role_service_name}' " + f"for URL: {service_url}.") + + role_service_path = os.path.join(data_dir, ROLE_SERVICE_PATH) + layman_role_service_path = os.path.join(role_service_path, role_service_name) + if os.path.exists(layman_role_service_path): + shutil.rmtree(layman_role_service_path) + source_path = os.path.join(DIRECTORY, 'jdbc_role_service') + os.mkdir(layman_role_service_path) + + parsed_url = urlparse(service_url) + + with open(os.path.join(source_path, 'config.xml'), encoding="utf-8") as file: + config_content = file.read() + with open(os.path.join(layman_role_service_path, 'config.xml'), "w", encoding="utf-8") as file: + file.write(config_content.format( + connection_string=f'jdbc:{escape(url_util.redact_uri(service_url, remove_username=True))}', + username=escape(parsed_url.username), + password=escape(parsed_url.password), + )) + + with open(os.path.join(source_path, 'rolesddl.xml'), encoding="utf-8") as file: + rolesddl_content = file.read() + with open(os.path.join(layman_role_service_path, 'rolesddl.xml'), "w", encoding="utf-8") as file: + file.write(rolesddl_content.format( + schema=escape(db_schema), + )) + + with open(os.path.join(source_path, 'rolesdml.xml'), encoding="utf-8") as file: + rolesdml_content = file.read() + with open(os.path.join(layman_role_service_path, 'rolesdml.xml'), "w", encoding="utf-8") as file: + file.write(rolesdml_content.format( + schema=escape(db_schema), + )) diff --git a/src/setup_geoserver.py b/src/setup_geoserver.py index ad037f02d..b95362103 100644 --- a/src/setup_geoserver.py +++ b/src/setup_geoserver.py @@ -4,8 +4,7 @@ from db import util as db_util import geoserver -from geoserver import epsg_properties -from geoserver import authn +from geoserver import authn, epsg_properties, role_service import layman_settings as settings @@ -77,6 +76,11 @@ def main(): settings.LAYMAN_GS_AUTHN_FILTER_NAME_OLD, ) ensure_jdbc_role_service_internal_schema() + role_service.setup_jdbc_role_service(settings.GEOSERVER_DATADIR, + settings.LAYMAN_ROLE_SERVICE_URI, + 'layman_role_service', + settings.LAYMAN_ROLE_SERVICE_SCHEMA, + ) epsg_properties.setup_epsg(settings.GEOSERVER_DATADIR, set(settings.LAYMAN_OUTPUT_SRS_LIST)) From 89ae38cda935d98b726464f03e46b3a417254906 Mon Sep 17 00:00:00 2001 From: index-git Date: Thu, 14 Dec 2023 15:12:17 +0100 Subject: [PATCH 08/11] Set JDBC role service as primary in GS --- CHANGELOG.md | 3 ++- doc/env-settings.md | 3 --- src/geoserver/role_service.py | 9 +++++++++ src/layman_settings.py | 14 ++++++++------ src/setup_geoserver.py | 5 ++++- 5 files changed, 23 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5e5d8c2df..5541b6c6d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ ``` It was already required in v1.22.3. - Set new environment variable [LAYMAN_ROLE_SERVICE_URI](doc/env-settings.md#LAYMAN_ROLE_SERVICE_URI) +- Stop using environment variable `LAYMAN_GS_ROLE_SERVICE`, it has no effect to Layman anymore. Role service called `layman_role_service` is used now. ### Migrations and checks #### Schema migrations - [#165](https://github.com/LayerManager/layman/issues/165) Add column `role_name` to table `rights` in prime DB schema. Add constraint that exactly one of columns `role_name` and `id_user` is not null. @@ -856,7 +857,7 @@ There is a critical bug in this release, posting new layer breaks Layman: https: - [#62](https://github.com/LayerManager/layman/issues/62) GeoServer [Proxy Base URL](https://docs.geoserver.org/2.21.x/en/user/configuration/globalsettings.html) is now automatically set on Layman's startup according to [LAYMAN_GS_PROXY_BASE_URL](https://github.com/LayerManager/layman/blob/v1.21.1/doc/env-settings.md#LAYMAN_GS_PROXY_BASE_URL). If you do not set the variable, value is calculated as [LAYMAN_CLIENT_PUBLIC_URL](doc/env-settings.md#LAYMAN_CLIENT_PUBLIC_URL)+[LAYMAN_GS_PATH](https://github.com/LayerManager/layman/blob/v1.21.1/doc/env-settings.md#LAYMAN_GS_PATH). If you set it to empty string, no change of Proxy Base URL will be done on GeoServer side. - [#83](https://github.com/LayerManager/layman/issues/89) All layers are created as `GEOMETRY` type, so any other type can be added (for example polygons can be added to points). - [#73](https://github.com/LayerManager/layman/issues/73) Layman users are automatically created on GeoServer (either at start up of Layman or when reserved) with separate role and workspace. Username is the same as in Layman, name of role is `"USER_"+username`, name of workspace is the same as username. Read and write permissions for workspace are set according to Layman's authorization (as of now read-everyone-write-everyone or read-everyone-write-owner). -- New environment variables [LAYMAN_GS_USER_GROUP_SERVICE](doc/env-settings.md#LAYMAN_GS_USER_GROUP_SERVICE) and [LAYMAN_GS_ROLE_SERVICE](doc/env-settings.md#LAYMAN_GS_ROLE_SERVICE) enable to control which user/group and role services are used at GeoServer. Not setting these variables means to use default services. +- New environment variables [LAYMAN_GS_USER_GROUP_SERVICE](doc/env-settings.md#LAYMAN_GS_USER_GROUP_SERVICE) and [LAYMAN_GS_ROLE_SERVICE](https://github.com/LayerManager/layman/blob/v1.22.0/doc/env-settings.md#LAYMAN_GS_ROLE_SERVICE) enable to control which user/group and role services are used at GeoServer. Not setting these variables means to use default services. - [#69](https://github.com/LayerManager/layman/issues/69) Three separate identical settings files (`layman_settings_demo.py`, `layman_settings_dev.py`, `layman_settings_test.py`) were merged into one file `layman_settings.py`. - If username used in REST API request path is not yet reserved, HTTP requests other than POST returns (e.g. GET) HTTP error 404 (Layman code 40). Previously in case of GET request, empty list was returned. - List of GeoServer reserved workspace names was moved from `layman_settings.py` into source code (`src\layman\common\geoserver\__init__.py`) diff --git a/doc/env-settings.md b/doc/env-settings.md index 4736f701e..dcbac4e9a 100644 --- a/doc/env-settings.md +++ b/doc/env-settings.md @@ -176,9 +176,6 @@ Name of [GeoServer role](https://docs.geoserver.org/2.21.x/en/user/security/weba ### LAYMAN_GS_USER_GROUP_SERVICE Name of [user/group service](https://docs.geoserver.org/2.21.x/en/user/security/usergrouprole/usergroupservices.html) used for managing users at GeoServer. If not set (default), the service named `default` is chosen. Usually it's [XML user/group service](https://docs.geoserver.org/2.21.x/en/user/security/usergrouprole/usergroupservices.html#xml-user-group-service). -### LAYMAN_GS_ROLE_SERVICE -Name of [role service](https://docs.geoserver.org/2.21.x/en/user/security/usergrouprole/roleservices.html) used for managing roles and user-role associations at GeoServer. If not set (default), the service named `default` is chosen. Usually it's [XML user/group service](https://docs.geoserver.org/2.21.x/en/user/security/usergrouprole/roleservices.html#xml-role-service). - ### LAYMAN_GS_AUTHN_HTTP_HEADER_ATTRIBUTE Secret value of [GeoServer HTTP authentication request header attribute](https://docs.geoserver.org/2.21.x/en/user/security/tutorials/httpheaderproxy/index.html) used for WFS proxy. Only combination of lowercase characters and numbers must be used for the value. If you change an existing value, you have to change it also in GeoServer GUI manually. diff --git a/src/geoserver/role_service.py b/src/geoserver/role_service.py index cdf633137..e8fc9038f 100644 --- a/src/geoserver/role_service.py +++ b/src/geoserver/role_service.py @@ -6,6 +6,7 @@ from xml.sax.saxutils import escape from requests_util import url_util +from . import authn logger = logging.getLogger(__name__) logging.basicConfig(stream=sys.stdout, level=logging.DEBUG) @@ -50,3 +51,11 @@ def setup_jdbc_role_service(data_dir, service_url, role_service_name, db_schema) file.write(rolesdml_content.format( schema=escape(db_schema), )) + + +def set_primary_role_service(data_dir, role_service_name): + security_xml = authn.get_security(data_dir) + element = security_xml.find('roleServiceName') + element.text = role_service_name + security_path = os.path.join(data_dir, 'security/config.xml') + security_xml.write(security_path) diff --git a/src/layman_settings.py b/src/layman_settings.py index 6938151df..e5408af21 100644 --- a/src/layman_settings.py +++ b/src/layman_settings.py @@ -105,9 +105,16 @@ class EnumWfsWmsStatus(Enum): GEOSERVER_ADMIN_PASSWORD) GEOSERVER_DATADIR = '/geoserver/data_dir' GEOSERVER_INITIAL_DATADIR = '/geoserver/initial_data_dir' -LAYMAN_GS_ROLE_SERVICE = os.getenv('LAYMAN_GS_ROLE_SERVICE', '') or 'default' + +LAYMAN_GS_ROLE_SERVICE = 'layman_role_service' +# Name of schema, where Layman maintains internal GS JDBC Role Service. +LAYMAN_INTERNAL_ROLE_SERVICE_SCHEMA = '_role_service' +LAYMAN_ROLE_SERVICE_URI = os.environ['LAYMAN_ROLE_SERVICE_URI'] +LAYMAN_ROLE_SERVICE_SCHEMA = parse_qs(urlparse(LAYMAN_ROLE_SERVICE_URI).query)['schema'][0] + LAYMAN_GS_USER_GROUP_SERVICE = os.getenv('LAYMAN_GS_USER_GROUP_SERVICE', '') or 'default' + LAYMAN_GS_USER = os.environ['LAYMAN_GS_USER'] LAYMAN_GS_PASSWORD = os.environ['LAYMAN_GS_PASSWORD'] LAYMAN_GS_AUTH = (LAYMAN_GS_USER, LAYMAN_GS_PASSWORD) @@ -224,11 +231,6 @@ class EnumWfsWmsStatus(Enum): if RIGHTS_EVERYONE_ROLE not in GRANT_PUBLISH_IN_PUBLIC_WORKSPACE: assert not GRANT_CREATE_PUBLIC_WORKSPACE.difference(GRANT_PUBLISH_IN_PUBLIC_WORKSPACE) -# Name of schema, where Layman maintains internal GS JDBC Role Service. -LAYMAN_INTERNAL_ROLE_SERVICE_SCHEMA = '_role_service' -LAYMAN_ROLE_SERVICE_URI = os.environ['LAYMAN_ROLE_SERVICE_URI'] -LAYMAN_ROLE_SERVICE_SCHEMA = parse_qs(urlparse(LAYMAN_ROLE_SERVICE_URI).query)['schema'][0] - # UPLOAD_MAX_INACTIVITY_TIME = 10 # 10 seconds UPLOAD_MAX_INACTIVITY_TIME = 5 * 60 # 5 minutes diff --git a/src/setup_geoserver.py b/src/setup_geoserver.py index b95362103..74e963d66 100644 --- a/src/setup_geoserver.py +++ b/src/setup_geoserver.py @@ -78,9 +78,12 @@ def main(): ensure_jdbc_role_service_internal_schema() role_service.setup_jdbc_role_service(settings.GEOSERVER_DATADIR, settings.LAYMAN_ROLE_SERVICE_URI, - 'layman_role_service', + settings.LAYMAN_GS_ROLE_SERVICE, settings.LAYMAN_ROLE_SERVICE_SCHEMA, ) + role_service.set_primary_role_service(settings.GEOSERVER_DATADIR, + settings.LAYMAN_GS_ROLE_SERVICE, + ) epsg_properties.setup_epsg(settings.GEOSERVER_DATADIR, set(settings.LAYMAN_OUTPUT_SRS_LIST)) From 71bf5e622e21fee61db73b21e97a5e366243805d Mon Sep 17 00:00:00 2001 From: index-git Date: Thu, 14 Dec 2023 16:55:16 +0100 Subject: [PATCH 09/11] Remove role and role-user operations on GeoServer --- src/geoserver/geoserver_test.py | 38 --------------- src/geoserver/util.py | 66 -------------------------- src/layman/__init__.py | 3 -- src/layman/layer/geoserver/__init__.py | 8 ---- 4 files changed, 115 deletions(-) diff --git a/src/geoserver/geoserver_test.py b/src/geoserver/geoserver_test.py index a803a0e57..2716a0e52 100644 --- a/src/geoserver/geoserver_test.py +++ b/src/geoserver/geoserver_test.py @@ -24,29 +24,6 @@ def gs_user(): assert gs_util.delete_user(TEST_USER, GS_AUTH) -@pytest.fixture() -def gs_role(): - roles = gs_util.get_roles(GS_AUTH) - assert TEST_ROLE not in roles - assert gs_util.ensure_role(TEST_ROLE, GS_AUTH) - yield TEST_ROLE - assert gs_util.delete_role(TEST_ROLE, GS_AUTH) - - -def test_role_management(): - init_roles = gs_util.get_roles(GS_AUTH) - new_role = TEST_ROLE - assert new_role not in init_roles - assert gs_util.ensure_role(new_role, GS_AUTH) - roles = gs_util.get_roles(GS_AUTH) - assert new_role in roles - assert len(init_roles) + 1 == len(roles) - assert gs_util.delete_role(new_role, GS_AUTH) - roles = gs_util.get_roles(GS_AUTH) - assert new_role not in roles - assert len(init_roles) == len(roles) - - def test_user_management(): init_usernames = gs_util.get_usernames(GS_AUTH) new_user = TEST_USER @@ -62,21 +39,6 @@ def test_user_management(): assert len(init_usernames) == len(usernames) -def test_user_role_management(gs_user, gs_role): - user = gs_user[0] - init_user_roles = gs_util.get_user_roles(user, GS_AUTH) - role = gs_role - assert role not in init_user_roles - assert gs_util.ensure_user_role(user, role, GS_AUTH) - user_roles = gs_util.get_user_roles(user, GS_AUTH) - assert role in user_roles - assert len(init_user_roles) + 1 == len(user_roles) - assert gs_util.delete_user_role(user, role, GS_AUTH) - user_roles = gs_util.get_user_roles(user, GS_AUTH) - assert role not in user_roles - assert len(init_user_roles) == len(user_roles) - - @pytest.mark.parametrize('service', gs_util.SERVICE_TYPES) def test_service_srs_list_management(service): init_service_epsg_codes = gs_util.get_service_srs_list(service, GS_AUTH) diff --git a/src/geoserver/util.py b/src/geoserver/util.py index 805572b54..d0c749c72 100644 --- a/src/geoserver/util.py +++ b/src/geoserver/util.py @@ -78,38 +78,6 @@ def get_roles(auth): return response.json()['roles'] -def ensure_role(role, auth): - roles = get_roles(auth) - role_exists = role in roles - if not role_exists: - logger.info(f"Role {role} does not exist yet, creating.") - response = requests.post( - urljoin(GS_REST_ROLES, 'role/' + role), - headers=headers_json, - auth=auth, - timeout=GS_REST_TIMEOUT, - ) - response.raise_for_status() - else: - logger.info(f"Role {role} already exists") - role_created = not role_exists - return role_created - - -def delete_role(role, auth): - response = requests.delete( - urljoin(GS_REST_ROLES, 'role/' + role), - headers=headers_json, - auth=auth, - timeout=GS_REST_TIMEOUT, - ) - role_not_exists = response.status_code == 404 - if not role_not_exists: - response.raise_for_status() - role_deleted = not role_not_exists - return role_deleted - - def get_usernames(auth): r_url = GS_REST_USERS response = requests.get(r_url, @@ -902,40 +870,6 @@ def get_user_roles(user, auth): return response.json()['roles'] -def ensure_user_role(user, role, auth): - roles = get_user_roles(user, auth) - association_exists = role in roles - if not association_exists: - logger.info(f"Role {role} not associated with user {user} yet, associating.") - r_url = urljoin(GS_REST_ROLES, f'role/{role}/user/{user}/') - response = requests.post( - r_url, - headers=headers_json, - auth=auth, - timeout=GS_REST_TIMEOUT, - ) - response.raise_for_status() - else: - logger.info(f"Role {role} already associated with user {user}") - association_created = not association_exists - return association_created - - -def delete_user_role(user, role, auth): - r_url = urljoin(GS_REST_ROLES, f'role/{role}/user/{user}/') - response = requests.delete( - r_url, - headers=headers_json, - auth=auth, - timeout=GS_REST_TIMEOUT, - ) - association_not_exists = response.status_code == 404 - if not association_not_exists: - response.raise_for_status() - association_deleted = not association_not_exists - return association_deleted - - def get_service_url(service): return { WMS_SERVICE_TYPE: GS_REST_WMS_SETTINGS, diff --git a/src/layman/__init__.py b/src/layman/__init__.py index dc5892070..491403ef1 100644 --- a/src/layman/__init__.py +++ b/src/layman/__init__.py @@ -86,10 +86,7 @@ logger.info(f'Adjusting GeoServer roles') if settings.GEOSERVER_ADMIN_AUTH: - gs_util.ensure_role(settings.LAYMAN_GS_ROLE, settings.GEOSERVER_ADMIN_AUTH) gs_util.ensure_user(settings.LAYMAN_GS_USER, settings.LAYMAN_GS_PASSWORD, settings.GEOSERVER_ADMIN_AUTH) - gs_util.ensure_user_role(settings.LAYMAN_GS_USER, 'ADMIN', settings.GEOSERVER_ADMIN_AUTH) - gs_util.ensure_user_role(settings.LAYMAN_GS_USER, settings.LAYMAN_GS_ROLE, settings.GEOSERVER_ADMIN_AUTH) gs_util.ensure_proxy_base_url(settings.LAYMAN_GS_PROXY_BASE_URL_WITH_PLACEHOLDERS, settings.LAYMAN_GS_AUTH) diff --git a/src/layman/layer/geoserver/__init__.py b/src/layman/layer/geoserver/__init__.py index 225be114a..312112419 100644 --- a/src/layman/layer/geoserver/__init__.py +++ b/src/layman/layer/geoserver/__init__.py @@ -16,19 +16,11 @@ def ensure_whole_user(username, auth=settings.LAYMAN_GS_AUTH): gs_util.ensure_user(username, None, auth) - role = gs_util.username_to_rolename(username) - gs_util.ensure_role(role, auth) - gs_util.ensure_user_role(username, role, auth) - gs_util.ensure_user_role(username, settings.LAYMAN_GS_ROLE, auth) ensure_workspace(username, auth) def delete_whole_user(username, auth=settings.LAYMAN_GS_AUTH): - role = gs_util.username_to_rolename(username) delete_workspace(username, auth) - gs_util.delete_user_role(username, role, auth) - gs_util.delete_user_role(username, settings.LAYMAN_GS_ROLE, auth) - gs_util.delete_role(role, auth) gs_util.delete_user(username, auth) From 8390518c8083638ebff649f09c92ebf683a601aa Mon Sep 17 00:00:00 2001 From: index-git Date: Fri, 15 Dec 2023 14:44:36 +0100 Subject: [PATCH 10/11] Recreate Role Service admin role views in setup_geoserver --- src/layman/__init__.py | 5 ---- src/layman/authz/internal_role_service.py | 24 ---------------- src/layman/upgrade/upgrade_v1_23.py | 26 +++++++++++++++-- src/layman/upgrade/upgrade_v1_23_test.py | 7 ----- src/setup_geoserver.py | 34 +++++++++++++++++++++-- 5 files changed, 56 insertions(+), 40 deletions(-) delete mode 100644 src/layman/authz/internal_role_service.py diff --git a/src/layman/__init__.py b/src/layman/__init__.py index 491403ef1..9ceef916b 100644 --- a/src/layman/__init__.py +++ b/src/layman/__init__.py @@ -125,11 +125,6 @@ from .layer.prime_db_schema.wfs_wms_status import set_after_restart set_after_restart() - logger.info(f'Recreate Role Service admin role views') - from .authz.internal_role_service import ensure_admin_roles - - ensure_admin_roles() - pipe.multi() pipe.set(LAYMAN_DEPS_ADJUSTED_KEY, 'done') pipe.execute() diff --git a/src/layman/authz/internal_role_service.py b/src/layman/authz/internal_role_service.py deleted file mode 100644 index b52bd914d..000000000 --- a/src/layman/authz/internal_role_service.py +++ /dev/null @@ -1,24 +0,0 @@ -from layman import settings -from db import util as db_util - -ROLE_SERVICE_SCHEMA = settings.LAYMAN_INTERNAL_ROLE_SERVICE_SCHEMA - - -def ensure_admin_roles(): - create_admin_roles_view = f"""CREATE OR REPLACE view {ROLE_SERVICE_SCHEMA}.admin_roles - as - select 'ADMIN' as name - UNION ALL - select 'GROUP_ADMIN' - UNION ALL - select %s - ;""" - db_util.run_statement(create_admin_roles_view, (settings.LAYMAN_GS_ROLE, )) - - create_admin_user_roles_view = f"""CREATE OR REPLACE view {ROLE_SERVICE_SCHEMA}.admin_user_roles - as - select %s as username, %s as rolename - UNION ALL - select %s, 'ADMIN' - ;""" - db_util.run_statement(create_admin_user_roles_view, (settings.LAYMAN_GS_USER, settings.LAYMAN_GS_ROLE, settings.LAYMAN_GS_USER)) diff --git a/src/layman/upgrade/upgrade_v1_23.py b/src/layman/upgrade/upgrade_v1_23.py index 1f6c82ff8..643c1bec5 100644 --- a/src/layman/upgrade/upgrade_v1_23.py +++ b/src/layman/upgrade/upgrade_v1_23.py @@ -2,7 +2,6 @@ from db import util as db_util from layman import settings -from layman.authz import internal_role_service logger = logging.getLogger(__name__) DB_SCHEMA = settings.LAYMAN_PRIME_SCHEMA @@ -72,7 +71,30 @@ def create_role_service_schema(): ;""" db_util.run_statement(create_layman_users_user_roles_view) - internal_role_service.ensure_admin_roles() + create_admin_roles_view = f"""CREATE OR REPLACE view {ROLE_SERVICE_SCHEMA}.admin_roles + as + select 'ADMIN' as name + UNION ALL + select 'GROUP_ADMIN' + UNION ALL + select %s + ;""" + db_util.run_statement(create_admin_roles_view, (settings.LAYMAN_GS_ROLE, )) + + create_admin_user_roles_view = f"""CREATE OR REPLACE view {ROLE_SERVICE_SCHEMA}.admin_user_roles + as + select %s as username, %s as rolename + UNION ALL + select %s, 'ADMIN' + UNION ALL + select %s, 'ADMIN' + union all + select w.name as username, + %s as rolename + from {settings.LAYMAN_PRIME_SCHEMA}.users u inner join + {settings.LAYMAN_PRIME_SCHEMA}.workspaces w on w.id = u.id_workspace + ;""" + db_util.run_statement(create_admin_user_roles_view, (settings.LAYMAN_GS_USER, settings.LAYMAN_GS_ROLE, settings.LAYMAN_GS_USER, settings.GEOSERVER_ADMIN_USER, settings.LAYMAN_GS_ROLE, )) create_roles_view = f"""create view {ROLE_SERVICE_SCHEMA}.roles as diff --git a/src/layman/upgrade/upgrade_v1_23_test.py b/src/layman/upgrade/upgrade_v1_23_test.py index deb1aef2d..0c43099e0 100644 --- a/src/layman/upgrade/upgrade_v1_23_test.py +++ b/src/layman/upgrade/upgrade_v1_23_test.py @@ -2,7 +2,6 @@ from db import util as db_util from layman import app, settings -from layman.authz import internal_role_service from layman.common.prime_db_schema import ensure_whole_user from test_tools import process_client from . import upgrade_v1_23 @@ -122,12 +121,6 @@ def test_create_role_service_schema(): assert result[0] + result[1] + result[2] == result[3] result = db_util.run_query(user_roles_query)[0] assert result[0] + result[1] + result[2] == result[3] - - internal_role_service.ensure_admin_roles() - result = db_util.run_query(roles_query)[0] - assert result[0] + result[1] + result[2] == result[3] - result = db_util.run_query(user_roles_query)[0] - assert result[0] + result[1] + result[2] == result[3] result = db_util.run_query(table_existence_query, ('role_props',))[0][0] assert result == 1 result = db_util.run_query(table_existence_query, ('group_roles',))[0][0] diff --git a/src/setup_geoserver.py b/src/setup_geoserver.py index 74e963d66..7b648f865 100644 --- a/src/setup_geoserver.py +++ b/src/setup_geoserver.py @@ -43,8 +43,8 @@ def ensure_jdbc_role_service_internal_schema(): wait_for_db(db_conn) logger.info(f" Checking internal role service DB schema") - schema_query = f'''SELECT COUNT(*) FROM information_schema.schemata WHERE schema_name = '{internal_service_schema}';''' - schema_exists = db_util.run_query(schema_query, uri_str=uri_str)[0][0] + schema_query = f'''SELECT COUNT(*) FROM information_schema.schemata WHERE schema_name = %s;''' + schema_exists = db_util.run_query(schema_query, (internal_service_schema, ), uri_str=uri_str)[0][0] if schema_exists == 0: logger.info(f" Setting up internal role service DB schema") statement = f""" @@ -61,6 +61,36 @@ def ensure_jdbc_role_service_internal_schema(): create view {internal_service_schema}.group_roles as select null::varchar as groupname, null::varchar as rolename; """ db_util.run_statement(statement, data=(settings.LAYMAN_GS_ROLE, settings.LAYMAN_GS_USER, settings.LAYMAN_GS_USER, settings.LAYMAN_GS_ROLE, settings.GEOSERVER_ADMIN_USER, ), uri_str=uri_str) + else: + prime_schema_exists = db_util.run_query(schema_query, (settings.LAYMAN_PRIME_SCHEMA, ), uri_str=uri_str)[0][0] + if prime_schema_exists: + logger.info(f' Recreate Role Service admin role views') + create_admin_roles_view = f"""CREATE OR REPLACE view {internal_service_schema}.admin_roles + as + select 'ADMIN' as name + UNION ALL + select 'GROUP_ADMIN' + UNION ALL + select %s + ;""" + db_util.run_statement(create_admin_roles_view, (settings.LAYMAN_GS_ROLE,), uri_str=uri_str) + + create_admin_user_roles_view = f"""CREATE OR REPLACE view {internal_service_schema}.admin_user_roles + as + select %s as username, %s as rolename + UNION ALL + select %s, 'ADMIN' + UNION ALL + select %s, 'ADMIN' + union all + select w.name as username, + %s as rolename + from {settings.LAYMAN_PRIME_SCHEMA}.users u inner join + {settings.LAYMAN_PRIME_SCHEMA}.workspaces w on w.id = u.id_workspace + ;""" + db_util.run_statement(create_admin_user_roles_view, ( + settings.LAYMAN_GS_USER, settings.LAYMAN_GS_ROLE, settings.LAYMAN_GS_USER, settings.GEOSERVER_ADMIN_USER, + settings.LAYMAN_GS_ROLE,), uri_str=uri_str) def main(): From 80844610d64e5ba8c77ea698cd1aae0fc5880ba7 Mon Sep 17 00:00:00 2001 From: index-git Date: Mon, 18 Dec 2023 11:21:10 +0100 Subject: [PATCH 11/11] Fix requests.readthedocs.io links --- doc/publish-map.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/publish-map.md b/doc/publish-map.md index a85d5ed23..dfb21a832 100644 --- a/doc/publish-map.md +++ b/doc/publish-map.md @@ -13,9 +13,9 @@ In QGIS, you need to implement following steps. First, compose JSON valid against [map-composition schema](https://github.com/hslayers/map-compositions). For Layman, especially `describedBy`, `name`, `title`, `abstract`, `layers`, `projection`, and `extent attributes are important. Each layer must have `className` attribute equal to `HSLayers.Layer.WMS` or `WMS`. Then save the file to Layman using [POST Workspace Maps](rest.md#post-workspace-maps) endpoint. Well-known [requests](https://requests.readthedocs.io/en/latest/) module can be used for sending HTTP requests. See especially -- [More complicated POST requests](https://requests.readthedocs.io/en/latest/user/quickstart/#more-complicated-post-requests) -- [POST a Multipart-Encoded File](https://requests.readthedocs.io/en/latest/user/quickstart/#post-a-multipart-encoded-file) -- [POST Multiple Multipart-Encoded Files](https://requests.readthedocs.io/en/latest/user/advanced/#post-multiple-multipart-encoded-files) +- [More complicated POST requests](https://requests.readthedocs.io/en/latest/user/quickstart.html#more-complicated-post-requests) +- [POST a Multipart-Encoded File](https://requests.readthedocs.io/en/latest/user/quickstart.html#post-a-multipart-encoded-file) +- [POST Multiple Multipart-Encoded Files](https://requests.readthedocs.io/en/latest/user/advanced.html#post-multiple-multipart-encoded-files) In response of [POST Workspace Maps](rest.md#post-workspace-maps) you will obtain - `name` of the map unique within all maps in used [workspace](models.md#workspace)