diff --git a/Makefile b/Makefile index 2c80ed756..3ed27a92b 100644 --- a/Makefile +++ b/Makefile @@ -3,24 +3,28 @@ start-demo: mkdir -p layman_data deps/qgis/data docker compose -f docker-compose.deps.demo.yml -f docker-compose.demo.yml build layman layman_client timgen + docker compose -f docker-compose.deps.demo.yml -f docker-compose.demo.yml up -d postgresql docker compose -f docker-compose.deps.demo.yml -f docker-compose.demo.yml run --rm --no-deps -u root layman bash -c "cd src && python3 -B setup_geoserver.py" docker compose -f docker-compose.deps.demo.yml -f docker-compose.demo.yml up -d --force-recreate postgresql qgis geoserver redis layman celery_worker flower timgen layman_client nginx start-demo-full: mkdir -p layman_data deps/qgis/data docker compose -f docker-compose.deps.demo.yml -f docker-compose.demo.yml build layman layman_client timgen + docker compose -f docker-compose.deps.demo.yml -f docker-compose.demo.yml up -d postgresql docker compose -f docker-compose.deps.demo.yml -f docker-compose.demo.yml run --rm --no-deps -u root layman bash -c "cd src && python3 -B setup_geoserver.py" docker compose -f docker-compose.deps.demo.yml -f docker-compose.demo.yml up -d --force-recreate postgresql qgis geoserver redis layman celery_worker flower timgen layman_client micka nginx start-demo-only: mkdir -p layman_data deps/qgis/data docker compose -f docker-compose.deps.demo.yml -f docker-compose.demo.yml build layman layman_client timgen + docker compose -f docker-compose.deps.demo.yml -f docker-compose.demo.yml up -d postgresql docker compose -f docker-compose.deps.demo.yml -f docker-compose.demo.yml run --rm --no-deps -u root layman bash -c "cd src && python3 -B setup_geoserver.py" docker compose -f docker-compose.deps.demo.yml -f docker-compose.demo.yml up -d --force-recreate --no-deps layman celery_worker flower timgen layman_client nginx start-demo-full-with-optional-deps: mkdir -p layman_data deps/qgis/data docker compose -f docker-compose.deps.demo.yml -f docker-compose.demo.yml build layman layman_client timgen + docker compose -f docker-compose.deps.demo.yml -f docker-compose.demo.yml up -d postgresql docker compose -f docker-compose.deps.demo.yml -f docker-compose.demo.yml run --rm --no-deps -u root layman bash -c "cd src && python3 -B setup_geoserver.py" docker compose -f docker-compose.deps.demo.yml -f docker-compose.demo.yml up -d --force-recreate @@ -33,6 +37,7 @@ build-demo: upgrade-demo: mkdir -p layman_data deps/qgis/data docker compose -f docker-compose.deps.demo.yml -f docker-compose.demo.yml build layman layman_client timgen + docker compose -f docker-compose.deps.demo.yml -f docker-compose.demo.yml up -d postgresql docker compose -f docker-compose.deps.demo.yml -f docker-compose.demo.yml run --rm --no-deps -u root layman bash -c "cd src && python3 -B setup_geoserver.py" docker compose -f docker-compose.deps.demo.yml -f docker-compose.demo.yml up -d --force-recreate --no-deps postgresql qgis geoserver redis timgen layman_client nginx docker compose -f docker-compose.deps.demo.yml -f docker-compose.demo.yml run --rm --no-deps layman bash -c "cd src && python3 layman_flush_redis.py && python3 wait_for_deps.py && python3 standalone_upgrade.py" @@ -40,6 +45,7 @@ upgrade-demo: upgrade-demo-full: mkdir -p layman_data deps/qgis/data docker compose -f docker-compose.deps.demo.yml -f docker-compose.demo.yml build layman layman_client timgen + docker compose -f docker-compose.deps.demo.yml -f docker-compose.demo.yml up -d postgresql docker compose -f docker-compose.deps.demo.yml -f docker-compose.demo.yml run --rm --no-deps -u root layman bash -c "cd src && python3 -B setup_geoserver.py" docker compose -f docker-compose.deps.demo.yml -f docker-compose.demo.yml up -d --force-recreate --no-deps postgresql qgis geoserver redis timgen layman_client micka nginx docker compose -f docker-compose.deps.demo.yml -f docker-compose.demo.yml run --rm --no-deps layman bash -c "cd src && python3 layman_flush_redis.py && python3 wait_for_deps.py && python3 standalone_upgrade.py" @@ -55,6 +61,7 @@ deps-stop: start-dev: mkdir -p layman_data layman_data_test tmp deps/qgis/data + docker compose -f docker-compose.deps.yml -f docker-compose.dev.yml up -d postgresql docker compose -f docker-compose.deps.yml -f docker-compose.dev.yml run --rm --no-deps -u root layman_dev bash -c "cd src && python3 -B setup_geoserver.py" docker compose -f docker-compose.deps.yml -f docker-compose.dev.yml up --force-recreate -d @@ -64,6 +71,7 @@ stop-dev: start-dev-only: mkdir -p layman_data layman_data_test tmp deps/qgis/data docker compose -f docker-compose.deps.yml -f docker-compose.dev.yml rm -fsv layman_dev celery_worker_dev flower timgen layman_client + docker compose -f docker-compose.deps.yml -f docker-compose.dev.yml up -d postgresql docker compose -f docker-compose.deps.yml -f docker-compose.dev.yml run --rm --no-deps -u root layman_dev bash -c "cd src && python3 -B setup_geoserver.py" docker compose -f docker-compose.deps.yml -f docker-compose.dev.yml up -d layman_dev celery_worker_dev flower timgen layman_client @@ -93,6 +101,7 @@ restart-celery-dev: upgrade-dev: mkdir -p layman_data layman_data_test tmp deps/qgis/data + docker compose -f docker-compose.deps.yml -f docker-compose.dev.yml up -d postgresql docker compose -f docker-compose.deps.yml -f docker-compose.dev.yml run --rm --no-deps -u root layman_dev bash -c "cd src && python3 -B setup_geoserver.py" docker compose -f docker-compose.deps.yml -f docker-compose.dev.yml up -d timgen layman_client postgresql qgis nginx-qgis geoserver redis micka docker compose -f docker-compose.deps.yml -f docker-compose.dev.yml run --rm --no-deps layman_dev bash -c "cd src && python3 layman_flush_redis.py && python3 wait_for_deps.py && python3 standalone_upgrade.py" @@ -195,6 +204,7 @@ test: fi; docker compose -f docker-compose.deps.yml -f docker-compose.test.yml rm -f layman_test docker compose -f docker-compose.deps.yml -f docker-compose.test.yml run --rm --no-deps layman_test bash -c "bash ensure-test-data.sh" + docker compose -f docker-compose.deps.yml -f docker-compose.test.yml up -d postgresql docker compose -f docker-compose.deps.yml -f docker-compose.test.yml run --rm --no-deps -u root layman_test bash -c "cd src && python3 -B setup_geoserver.py" docker compose -f docker-compose.deps.yml -f docker-compose.test.yml up --force-recreate --no-deps -d celery_worker_test docker compose -f docker-compose.deps.yml -f docker-compose.test.yml run --rm --name layman_test_run_1 layman_test @@ -215,6 +225,7 @@ test-separated: fi; docker compose -f docker-compose.deps.yml -f docker-compose.test.yml rm -f layman_test docker compose -f docker-compose.deps.yml -f docker-compose.test.yml run --rm --no-deps layman_test bash -c "bash ensure-test-data.sh" + docker compose -f docker-compose.deps.yml -f docker-compose.test.yml up -d postgresql docker compose -f docker-compose.deps.yml -f docker-compose.test.yml run --rm --no-deps -u root layman_test bash -c "cd src && python3 -B setup_geoserver.py" docker compose -f docker-compose.deps.yml -f docker-compose.test.yml up --force-recreate --no-deps -d celery_worker_test docker compose -f docker-compose.deps.yml -f docker-compose.test.yml run --rm --name layman_test_run_1 -e "TEST_TYPE=$(test_type)" layman_test bash -c "bash test_separated.sh $(max_fail)" @@ -235,6 +246,7 @@ test-static: fi; docker compose -f docker-compose.deps.yml -f docker-compose.test.yml rm -f layman_test docker compose -f docker-compose.deps.yml -f docker-compose.test.yml run --rm --no-deps layman_test bash -c "bash ensure-test-data.sh" + docker compose -f docker-compose.deps.yml -f docker-compose.test.yml up -d postgresql docker compose -f docker-compose.deps.yml -f docker-compose.test.yml run --rm --no-deps -u root layman_test bash -c "cd src && python3 -B setup_geoserver.py" docker compose -f docker-compose.deps.yml -f docker-compose.test.yml up --force-recreate --no-deps -d celery_worker_test docker compose -f docker-compose.deps.yml -f docker-compose.test.yml run --rm --name layman_test_run_1 layman_test bash -c "bash test_static.sh" @@ -283,6 +295,7 @@ geoserver-exec: docker compose -f docker-compose.deps.yml exec geoserver bash geoserver-ensure-authn: + docker compose -f docker-compose.deps.yml -f docker-compose.dev.yml up -d postgresql docker compose -f docker-compose.deps.yml -f docker-compose.dev.yml run --rm --no-deps -u root layman_dev bash -c "cd src && python3 -B setup_geoserver.py" get-current-user: diff --git a/src/geoserver/layman_role_service/config.xml b/src/geoserver/layman_role_service/config.xml new file mode 100644 index 000000000..fb228d0b7 --- /dev/null +++ b/src/geoserver/layman_role_service/config.xml @@ -0,0 +1,15 @@ + + -6e75d003:18c584ed8f6:-7fee + layman_role_service + org.geoserver.security.jdbc.JDBCRoleService + rolesddl.xml + rolesdml.xml + false + org.postgresql.Driver + xxx + xxx + xxx + false + ADMIN + GROUP_ADMIN + diff --git a/src/geoserver/layman_role_service/rolesddl.xml b/src/geoserver/layman_role_service/rolesddl.xml new file mode 100644 index 000000000..7c9336b57 --- /dev/null +++ b/src/geoserver/layman_role_service/rolesddl.xml @@ -0,0 +1,33 @@ + + + + DDL statements for role database + role_props + + create table _role_service.roles(name varchar(64) not null,parent varchar(64), primary key(name)) + + + create table _role_service.role_props(rolename varchar(64) not null,propname varchar(64) not null, propvalue varchar(2048),primary key (rolename,propname)) + + + + create table _role_service.user_roles(username varchar(128) not null, rolename varchar(64) not null, primary key(username,rolename)) + + + create index _role_service.user_roles_idx on user_roles(rolename,username) + + + create table _role_service.group_roles(groupname varchar(128) not null, rolename varchar(64) not null, primary key(groupname,rolename)) + + + create index group_roles_idx on _role_service.group_roles(rolename,groupname) + + + + + drop table _role_service.roles + drop table _role_service.role_props + drop table _role_service.user_roles + drop table _role_service.group_roles + + diff --git a/src/geoserver/layman_role_service/rolesdml.xml b/src/geoserver/layman_role_service/rolesdml.xml new file mode 100644 index 000000000..518d46413 --- /dev/null +++ b/src/geoserver/layman_role_service/rolesdml.xml @@ -0,0 +1,106 @@ + + + + DML statements for role database + + + select count(*) from _role_service.roles + + + select name,parent from _role_service.roles + + + select parent from _role_service.roles where name = ? + + + insert into _role_service.roles (name) values (?) + + + + update _role_service.roles set name=name where name = ? + + + update _role_service.roles set parent = ? where name = ? + + + update _role_service.roles set parent = null where parent = ? + + + delete from _role_service.roles where name = ? + + + delete from _role_service.roles + + + + + select rolename,propname,propvalue from _role_service.role_props + + + select propname,propvalue from _role_service.role_props where rolename = ? + + + select p.rolename,p.propname,p.propvalue from _role_service.role_props p,_role_service.user_roles u where u.rolename = p.rolename and u.username = ? + + + select p.rolename,p.propname,p.propvalue from _role_service.role_props p,_role_service.group_roles g where g.rolename = p.rolename and g.groupname = ? + + + delete from _role_service.role_props where rolename=? + + + insert into _role_service.role_props(rolename,propname,propvalue) values (?,?,?) + + + delete from _role_service.role_props + + + + + select u.rolename,r.parent from _role_service.user_roles u ,_role_service.roles r where r.name=u.rolename and u.username = ? + + + select username from _role_service.user_roles where rolename = ? + + + insert into _role_service.user_roles(rolename,username) values (?,?) + + + delete from _role_service.user_roles where rolename=? and username = ? + + + delete from _role_service.user_roles where rolename=? + + + delete from _role_service.user_roles where username = ? + + + delete from _role_service.user_roles + + + + + + select g.rolename,r.parent from _role_service.group_roles g,r_role_service.oles r where g.rolename = r.name and g.groupname = ? + + + select groupname from _role_service.group_roles where rolename = ? + + + insert into _role_service.group_roles(rolename,groupname) values (?,?) + + + delete from _role_service.group_roles where rolename=? and groupname = ? + + + delete from _role_service.group_roles where rolename=? + + + delete from _role_service.group_roles where groupname = ? + + + delete from _role_service.group_roles + + + + diff --git a/src/geoserver/role_service.py b/src/geoserver/role_service.py new file mode 100644 index 000000000..46115c22a --- /dev/null +++ b/src/geoserver/role_service.py @@ -0,0 +1,97 @@ +from distutils.dir_util import copy_tree +import logging +import os +import shutil +import sys +import time +from urllib.parse import urlparse +from lxml import etree as ET + +from db import util as db_util +from requests_util import url_util +from . import authn + +logger = logging.getLogger(__name__) +logging.basicConfig(stream=sys.stdout, level=logging.DEBUG) + + +ROLE_SERVICE_PATH = 'security/role/' +DIRECTORY = os.path.dirname(os.path.abspath(__file__)) + + +def wait_for_db(conn_dict): + max_attempts = 10 + attempt = 0 + + while True: + import psycopg2 + try: + with psycopg2.connect(**conn_dict): + pass + logger.info(f" Attempt {attempt}/{max_attempts} successful.") + break + except psycopg2.OperationalError: + if attempt >= max_attempts: + logger.info(f" Reaching max attempts when waiting for DB") + sys.exit(1) + time.sleep(2) + attempt += 1 + + +def setup_role_service(data_dir, db_conn, uri_str, internal_service_schema, layman_pg_user, layman_gs_user, layman_gs_role, service_url, role_service_name): + logger.info(f"Ensuring GeoServer DB role service '{role_service_name}' " + f"for URL: {service_url}.") + + logger.info(f" Waiting for DB") + wait_for_db(db_conn) + + logger.info(f" Checking internal role service DB schema") + schema_query = f'''SELECT COUNT(*) FROM information_schema.schemata WHERE schema_name = '{internal_service_schema}';''' + schema_exists = db_util.run_query(schema_query, uri_str=uri_str)[0][0] + if schema_exists == 0: + logger.info(f" Setting up internal role service DB schema") + statement = f""" + CREATE SCHEMA "{internal_service_schema}" AUTHORIZATION {layman_pg_user}; + create view {internal_service_schema}.roles as select 'ADMIN' as name, null as parent + union all select 'GROUP_ADMIN', null + union all select 'LAYMAN_ROLE', null + ; + create view {internal_service_schema}.role_props as select null::varchar as rolename, null::varchar as propname, null::varchar as propvalue; + create view {internal_service_schema}.user_roles as select 'layman' as username, 'ADMIN' as rolename + union all select 'layman', 'LAYMAN_ROLE' + union all select 'admin', 'ADMIN' + ; + create view {internal_service_schema}.group_roles as select null::varchar as groupname, null::varchar as rolename; +""" + db_util.run_statement(statement, uri_str=uri_str) + + logger.info(f" Setting up files") + role_service_path = os.path.join(data_dir, ROLE_SERVICE_PATH) + layman_role_service_path = os.path.join(role_service_path, role_service_name) + if os.path.exists(layman_role_service_path): + shutil.rmtree(layman_role_service_path) + source_path = os.path.join(DIRECTORY, role_service_name) + os.mkdir(layman_role_service_path) + copy_tree(source_path, layman_role_service_path) + + role_service_config_path = os.path.join(layman_role_service_path, 'config.xml') + role_service_xml = ET.parse(role_service_config_path) + + parsed_url = urlparse(service_url) + + element = role_service_xml.find('userName') + element.text = parsed_url.username + + element = role_service_xml.find('password') + element.text = parsed_url.password + + element = role_service_xml.find('connectURL') + element.text = f'jdbc:{url_util.redact_uri(service_url, remove_username=True)}' + + role_service_xml.write(role_service_config_path) + + security_xml = authn.get_security(data_dir) + element = security_xml.find('roleServiceName') + element.text = role_service_name + security_path = os.path.join(data_dir, 'security/config.xml') + security_xml.write(security_path) diff --git a/src/layman/__init__.py b/src/layman/__init__.py index a76a05daa..e5c578b7f 100644 --- a/src/layman/__init__.py +++ b/src/layman/__init__.py @@ -86,10 +86,7 @@ logger.info(f'Adjusting GeoServer roles') if settings.GEOSERVER_ADMIN_AUTH: - gs_util.ensure_role(settings.LAYMAN_GS_ROLE, settings.GEOSERVER_ADMIN_AUTH) gs_util.ensure_user(settings.LAYMAN_GS_USER, settings.LAYMAN_GS_PASSWORD, settings.GEOSERVER_ADMIN_AUTH) - gs_util.ensure_user_role(settings.LAYMAN_GS_USER, 'ADMIN', settings.GEOSERVER_ADMIN_AUTH) - gs_util.ensure_user_role(settings.LAYMAN_GS_USER, settings.LAYMAN_GS_ROLE, settings.GEOSERVER_ADMIN_AUTH) gs_util.ensure_proxy_base_url(settings.LAYMAN_GS_PROXY_BASE_URL_WITH_PLACEHOLDERS, settings.LAYMAN_GS_AUTH) diff --git a/src/layman/upgrade/upgrade_v1_23.py b/src/layman/upgrade/upgrade_v1_23.py index 237c11a99..a98db7bd7 100644 --- a/src/layman/upgrade/upgrade_v1_23.py +++ b/src/layman/upgrade/upgrade_v1_23.py @@ -34,6 +34,14 @@ def create_role_service_schema(): create_schema = f"""CREATE SCHEMA IF NOT EXISTS "{ROLE_SERVICE_SCHEMA}" AUTHORIZATION {settings.LAYMAN_PG_USER};""" db_util.run_statement(create_schema) + drop_temporary_views = f""" +drop view {ROLE_SERVICE_SCHEMA}.roles; +drop view {ROLE_SERVICE_SCHEMA}.role_props; +drop view {ROLE_SERVICE_SCHEMA}.user_roles; +drop view {ROLE_SERVICE_SCHEMA}.group_roles; +""" + db_util.run_statement(drop_temporary_views) + create_role_table = f"""create table {ROLE_SERVICE_SCHEMA}.bussiness_roles( id integer GENERATED ALWAYS AS IDENTITY, name varchar(64) not null, diff --git a/src/layman_settings.py b/src/layman_settings.py index 27dbe0593..4ccfaa9dc 100644 --- a/src/layman_settings.py +++ b/src/layman_settings.py @@ -105,7 +105,12 @@ class EnumWfsWmsStatus(Enum): GEOSERVER_ADMIN_PASSWORD) GEOSERVER_DATADIR = '/geoserver/data_dir' GEOSERVER_INITIAL_DATADIR = '/geoserver/initial_data_dir' -LAYMAN_GS_ROLE_SERVICE = os.getenv('LAYMAN_GS_ROLE_SERVICE', '') or 'default' +LAYMAN_GS_ROLE_SERVICE = 'layman_role_service' +# Name of schema, where Layman maintains internal GS JDBC Role Service. +LAYMAN_INTERNAL_ROLE_SERVICE_SCHEMA = '_role_service' +LAYMAN_ROLE_SERVICE_URI = os.environ['LAYMAN_ROLE_SERVICE_URI'] +LAYMAN_ROLE_SERVICE_SCHEMA = parse_qs(urlparse(LAYMAN_ROLE_SERVICE_URI).query)['schema'][0] + LAYMAN_GS_USER_GROUP_SERVICE = os.getenv('LAYMAN_GS_USER_GROUP_SERVICE', '') or 'default' LAYMAN_GS_USER = os.environ['LAYMAN_GS_USER'] @@ -223,11 +228,6 @@ class EnumWfsWmsStatus(Enum): if RIGHTS_EVERYONE_ROLE not in GRANT_PUBLISH_IN_PUBLIC_WORKSPACE: assert not GRANT_CREATE_PUBLIC_WORKSPACE.difference(GRANT_PUBLISH_IN_PUBLIC_WORKSPACE) -# Name of schema, where Layman maintains internal GS JDBC Role Service. -LAYMAN_INTERNAL_ROLE_SERVICE_SCHEMA = '_role_service' -LAYMAN_ROLE_SERVICE_URI = os.environ['LAYMAN_ROLE_SERVICE_URI'] -LAYMAN_ROLE_SERVICE_SCHEMA = parse_qs(urlparse(LAYMAN_ROLE_SERVICE_URI).query)['schema'][0] - # UPLOAD_MAX_INACTIVITY_TIME = 10 # 10 seconds UPLOAD_MAX_INACTIVITY_TIME = 5 * 60 # 5 minutes diff --git a/src/setup_geoserver.py b/src/setup_geoserver.py index 5010e3a37..d7603ea0e 100644 --- a/src/setup_geoserver.py +++ b/src/setup_geoserver.py @@ -2,8 +2,7 @@ import sys import geoserver -from geoserver import epsg_properties -from geoserver import authn +from geoserver import epsg_properties, authn, role_service import layman_settings as settings @@ -26,6 +25,16 @@ def main(): ) epsg_properties.setup_epsg(settings.GEOSERVER_DATADIR, set(settings.LAYMAN_OUTPUT_SRS_LIST)) + role_service.setup_role_service(settings.GEOSERVER_DATADIR, + settings.PG_CONN, + settings.PG_URI_STR, + settings.LAYMAN_INTERNAL_ROLE_SERVICE_SCHEMA, + settings.LAYMAN_PG_USER, + settings.LAYMAN_GS_USER, + settings.LAYMAN_GS_ROLE, + settings.LAYMAN_ROLE_SERVICE_URI, + settings.LAYMAN_GS_ROLE_SERVICE, + ) if __name__ == "__main__":