Skip to content

Commit

Permalink
WIP: Set-up GS JDBC role service
Browse files Browse the repository at this point in the history
  • Loading branch information
index-git committed Dec 14, 2023
1 parent ece2704 commit 0a779ae
Show file tree
Hide file tree
Showing 9 changed files with 289 additions and 11 deletions.
13 changes: 13 additions & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -3,24 +3,28 @@
start-demo:
mkdir -p layman_data deps/qgis/data
docker compose -f docker-compose.deps.demo.yml -f docker-compose.demo.yml build layman layman_client timgen
docker compose -f docker-compose.deps.demo.yml -f docker-compose.demo.yml up -d postgresql
docker compose -f docker-compose.deps.demo.yml -f docker-compose.demo.yml run --rm --no-deps -u root layman bash -c "cd src && python3 -B setup_geoserver.py"
docker compose -f docker-compose.deps.demo.yml -f docker-compose.demo.yml up -d --force-recreate postgresql qgis geoserver redis layman celery_worker flower timgen layman_client nginx

start-demo-full:
mkdir -p layman_data deps/qgis/data
docker compose -f docker-compose.deps.demo.yml -f docker-compose.demo.yml build layman layman_client timgen
docker compose -f docker-compose.deps.demo.yml -f docker-compose.demo.yml up -d postgresql
docker compose -f docker-compose.deps.demo.yml -f docker-compose.demo.yml run --rm --no-deps -u root layman bash -c "cd src && python3 -B setup_geoserver.py"
docker compose -f docker-compose.deps.demo.yml -f docker-compose.demo.yml up -d --force-recreate postgresql qgis geoserver redis layman celery_worker flower timgen layman_client micka nginx

start-demo-only:
mkdir -p layman_data deps/qgis/data
docker compose -f docker-compose.deps.demo.yml -f docker-compose.demo.yml build layman layman_client timgen
docker compose -f docker-compose.deps.demo.yml -f docker-compose.demo.yml up -d postgresql
docker compose -f docker-compose.deps.demo.yml -f docker-compose.demo.yml run --rm --no-deps -u root layman bash -c "cd src && python3 -B setup_geoserver.py"
docker compose -f docker-compose.deps.demo.yml -f docker-compose.demo.yml up -d --force-recreate --no-deps layman celery_worker flower timgen layman_client nginx

start-demo-full-with-optional-deps:
mkdir -p layman_data deps/qgis/data
docker compose -f docker-compose.deps.demo.yml -f docker-compose.demo.yml build layman layman_client timgen
docker compose -f docker-compose.deps.demo.yml -f docker-compose.demo.yml up -d postgresql
docker compose -f docker-compose.deps.demo.yml -f docker-compose.demo.yml run --rm --no-deps -u root layman bash -c "cd src && python3 -B setup_geoserver.py"
docker compose -f docker-compose.deps.demo.yml -f docker-compose.demo.yml up -d --force-recreate

Expand All @@ -33,13 +37,15 @@ build-demo:
upgrade-demo:
mkdir -p layman_data deps/qgis/data
docker compose -f docker-compose.deps.demo.yml -f docker-compose.demo.yml build layman layman_client timgen
docker compose -f docker-compose.deps.demo.yml -f docker-compose.demo.yml up -d postgresql
docker compose -f docker-compose.deps.demo.yml -f docker-compose.demo.yml run --rm --no-deps -u root layman bash -c "cd src && python3 -B setup_geoserver.py"
docker compose -f docker-compose.deps.demo.yml -f docker-compose.demo.yml up -d --force-recreate --no-deps postgresql qgis geoserver redis timgen layman_client nginx
docker compose -f docker-compose.deps.demo.yml -f docker-compose.demo.yml run --rm --no-deps layman bash -c "cd src && python3 layman_flush_redis.py && python3 wait_for_deps.py && python3 standalone_upgrade.py"

upgrade-demo-full:
mkdir -p layman_data deps/qgis/data
docker compose -f docker-compose.deps.demo.yml -f docker-compose.demo.yml build layman layman_client timgen
docker compose -f docker-compose.deps.demo.yml -f docker-compose.demo.yml up -d postgresql
docker compose -f docker-compose.deps.demo.yml -f docker-compose.demo.yml run --rm --no-deps -u root layman bash -c "cd src && python3 -B setup_geoserver.py"
docker compose -f docker-compose.deps.demo.yml -f docker-compose.demo.yml up -d --force-recreate --no-deps postgresql qgis geoserver redis timgen layman_client micka nginx
docker compose -f docker-compose.deps.demo.yml -f docker-compose.demo.yml run --rm --no-deps layman bash -c "cd src && python3 layman_flush_redis.py && python3 wait_for_deps.py && python3 standalone_upgrade.py"
Expand All @@ -55,6 +61,7 @@ deps-stop:

start-dev:
mkdir -p layman_data layman_data_test tmp deps/qgis/data
docker compose -f docker-compose.deps.yml -f docker-compose.dev.yml up -d postgresql
docker compose -f docker-compose.deps.yml -f docker-compose.dev.yml run --rm --no-deps -u root layman_dev bash -c "cd src && python3 -B setup_geoserver.py"
docker compose -f docker-compose.deps.yml -f docker-compose.dev.yml up --force-recreate -d

Expand All @@ -64,6 +71,7 @@ stop-dev:
start-dev-only:
mkdir -p layman_data layman_data_test tmp deps/qgis/data
docker compose -f docker-compose.deps.yml -f docker-compose.dev.yml rm -fsv layman_dev celery_worker_dev flower timgen layman_client
docker compose -f docker-compose.deps.yml -f docker-compose.dev.yml up -d postgresql
docker compose -f docker-compose.deps.yml -f docker-compose.dev.yml run --rm --no-deps -u root layman_dev bash -c "cd src && python3 -B setup_geoserver.py"
docker compose -f docker-compose.deps.yml -f docker-compose.dev.yml up -d layman_dev celery_worker_dev flower timgen layman_client

Expand Down Expand Up @@ -93,6 +101,7 @@ restart-celery-dev:

upgrade-dev:
mkdir -p layman_data layman_data_test tmp deps/qgis/data
docker compose -f docker-compose.deps.yml -f docker-compose.dev.yml up -d postgresql
docker compose -f docker-compose.deps.yml -f docker-compose.dev.yml run --rm --no-deps -u root layman_dev bash -c "cd src && python3 -B setup_geoserver.py"
docker compose -f docker-compose.deps.yml -f docker-compose.dev.yml up -d timgen layman_client postgresql qgis nginx-qgis geoserver redis micka
docker compose -f docker-compose.deps.yml -f docker-compose.dev.yml run --rm --no-deps layman_dev bash -c "cd src && python3 layman_flush_redis.py && python3 wait_for_deps.py && python3 standalone_upgrade.py"
Expand Down Expand Up @@ -195,6 +204,7 @@ test:
fi;
docker compose -f docker-compose.deps.yml -f docker-compose.test.yml rm -f layman_test
docker compose -f docker-compose.deps.yml -f docker-compose.test.yml run --rm --no-deps layman_test bash -c "bash ensure-test-data.sh"
docker compose -f docker-compose.deps.yml -f docker-compose.test.yml up -d postgresql
docker compose -f docker-compose.deps.yml -f docker-compose.test.yml run --rm --no-deps -u root layman_test bash -c "cd src && python3 -B setup_geoserver.py"
docker compose -f docker-compose.deps.yml -f docker-compose.test.yml up --force-recreate --no-deps -d celery_worker_test
docker compose -f docker-compose.deps.yml -f docker-compose.test.yml run --rm --name layman_test_run_1 layman_test
Expand All @@ -215,6 +225,7 @@ test-separated:
fi;
docker compose -f docker-compose.deps.yml -f docker-compose.test.yml rm -f layman_test
docker compose -f docker-compose.deps.yml -f docker-compose.test.yml run --rm --no-deps layman_test bash -c "bash ensure-test-data.sh"
docker compose -f docker-compose.deps.yml -f docker-compose.test.yml up -d postgresql
docker compose -f docker-compose.deps.yml -f docker-compose.test.yml run --rm --no-deps -u root layman_test bash -c "cd src && python3 -B setup_geoserver.py"
docker compose -f docker-compose.deps.yml -f docker-compose.test.yml up --force-recreate --no-deps -d celery_worker_test
docker compose -f docker-compose.deps.yml -f docker-compose.test.yml run --rm --name layman_test_run_1 -e "TEST_TYPE=$(test_type)" layman_test bash -c "bash test_separated.sh $(max_fail)"
Expand All @@ -235,6 +246,7 @@ test-static:
fi;
docker compose -f docker-compose.deps.yml -f docker-compose.test.yml rm -f layman_test
docker compose -f docker-compose.deps.yml -f docker-compose.test.yml run --rm --no-deps layman_test bash -c "bash ensure-test-data.sh"
docker compose -f docker-compose.deps.yml -f docker-compose.test.yml up -d postgresql
docker compose -f docker-compose.deps.yml -f docker-compose.test.yml run --rm --no-deps -u root layman_test bash -c "cd src && python3 -B setup_geoserver.py"
docker compose -f docker-compose.deps.yml -f docker-compose.test.yml up --force-recreate --no-deps -d celery_worker_test
docker compose -f docker-compose.deps.yml -f docker-compose.test.yml run --rm --name layman_test_run_1 layman_test bash -c "bash test_static.sh"
Expand Down Expand Up @@ -283,6 +295,7 @@ geoserver-exec:
docker compose -f docker-compose.deps.yml exec geoserver bash

geoserver-ensure-authn:
docker compose -f docker-compose.deps.yml -f docker-compose.dev.yml up -d postgresql
docker compose -f docker-compose.deps.yml -f docker-compose.dev.yml run --rm --no-deps -u root layman_dev bash -c "cd src && python3 -B setup_geoserver.py"

get-current-user:
Expand Down
15 changes: 15 additions & 0 deletions src/geoserver/layman_role_service/config.xml
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
<jdbcroleservice>
<id>-6e75d003:18c584ed8f6:-7fee</id>
<name>layman_role_service</name>
<className>org.geoserver.security.jdbc.JDBCRoleService</className>
<propertyFileNameDDL>rolesddl.xml</propertyFileNameDDL>
<propertyFileNameDML>rolesdml.xml</propertyFileNameDML>
<jndi>false</jndi>
<driverClassName>org.postgresql.Driver</driverClassName>
<connectURL>xxx</connectURL>
<userName>xxx</userName>
<password>xxx</password>
<creatingTables>false</creatingTables>
<adminRoleName>ADMIN</adminRoleName>
<groupAdminRoleName>GROUP_ADMIN</groupAdminRoleName>
</jdbcroleservice>
33 changes: 33 additions & 0 deletions src/geoserver/layman_role_service/rolesddl.xml
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE properties SYSTEM "http://java.sun.com/dtd/properties.dtd">
<properties>
<comment>DDL statements for role database</comment>
<entry key="check.table">role_props</entry>
<entry key="roles.create">
create table _role_service.roles(name varchar(64) not null,parent varchar(64), primary key(name))
</entry>
<entry key="roleprops.create">
create table _role_service.role_props(rolename varchar(64) not null,propname varchar(64) not null, propvalue varchar(2048),primary key (rolename,propname))
</entry>

<entry key="userroles.create">
create table _role_service.user_roles(username varchar(128) not null, rolename varchar(64) not null, primary key(username,rolename))
</entry>
<entry key="userroles.indexcreate">
create index _role_service.user_roles_idx on user_roles(rolename,username)
</entry>
<entry key="grouproles.create">
create table _role_service.group_roles(groupname varchar(128) not null, rolename varchar(64) not null, primary key(groupname,rolename))
</entry>
<entry key="grouproles.indexcreate">
create index group_roles_idx on _role_service.group_roles(rolename,groupname)
</entry>



<entry key="roles.drop">drop table _role_service.roles</entry>
<entry key="roleprops.drop">drop table _role_service.role_props</entry>
<entry key="userroles.drop">drop table _role_service.user_roles</entry>
<entry key="grouproles.drop">drop table _role_service.group_roles</entry>

</properties>
106 changes: 106 additions & 0 deletions src/geoserver/layman_role_service/rolesdml.xml
Original file line number Diff line number Diff line change
@@ -0,0 +1,106 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE properties SYSTEM "http://java.sun.com/dtd/properties.dtd">
<properties>
<comment>DML statements for role database</comment>

<entry key="roles.count">
select count(*) from _role_service.roles
</entry>
<entry key="roles.all">
select name,parent from _role_service.roles
</entry>
<entry key="roles.keyed">
select parent from _role_service.roles where name = ?
</entry>
<entry key="roles.insert">
insert into _role_service.roles (name) values (?)
</entry>
<!-- nothing to update at the moment, use dummy statement -->
<entry key="roles.update">
update _role_service.roles set name=name where name = ?
</entry>
<entry key="roles.parentUpdate">
update _role_service.roles set parent = ? where name = ?
</entry>
<entry key="roles.deleteParent">
update _role_service.roles set parent = null where parent = ?
</entry>
<entry key="roles.delete">
delete from _role_service.roles where name = ?
</entry>
<entry key="roles.deleteAll">
delete from _role_service.roles
</entry>


<entry key="roleprops.all">
select rolename,propname,propvalue from _role_service.role_props
</entry>
<entry key="roleprops.selectForRole">
select propname,propvalue from _role_service.role_props where rolename = ?
</entry>
<entry key="roleprops.selectForUser">
select p.rolename,p.propname,p.propvalue from _role_service.role_props p,_role_service.user_roles u where u.rolename = p.rolename and u.username = ?
</entry>
<entry key="roleprops.selectForGroup">
select p.rolename,p.propname,p.propvalue from _role_service.role_props p,_role_service.group_roles g where g.rolename = p.rolename and g.groupname = ?
</entry>
<entry key="roleprops.deleteForRole">
delete from _role_service.role_props where rolename=?
</entry>
<entry key="roleprops.insert">
insert into _role_service.role_props(rolename,propname,propvalue) values (?,?,?)
</entry>
<entry key="roleprops.deleteAll">
delete from _role_service.role_props
</entry>


<entry key="userroles.rolesForUser">
select u.rolename,r.parent from _role_service.user_roles u ,_role_service.roles r where r.name=u.rolename and u.username = ?
</entry>
<entry key="userroles.usersForRole">
select username from _role_service.user_roles where rolename = ?
</entry>
<entry key="userroles.insert">
insert into _role_service.user_roles(rolename,username) values (?,?)
</entry>
<entry key="userroles.delete">
delete from _role_service.user_roles where rolename=? and username = ?
</entry>
<entry key="userroles.deleteRole">
delete from _role_service.user_roles where rolename=?
</entry>
<entry key="userroles.deleteUser">
delete from _role_service.user_roles where username = ?
</entry>
<entry key="userroles.deleteAll">
delete from _role_service.user_roles
</entry>



<entry key="grouproles.rolesForGroup">
select g.rolename,r.parent from _role_service.group_roles g,r_role_service.oles r where g.rolename = r.name and g.groupname = ?
</entry>
<entry key="grouproles.groupsForRole">
select groupname from _role_service.group_roles where rolename = ?
</entry>
<entry key="grouproles.insert">
insert into _role_service.group_roles(rolename,groupname) values (?,?)
</entry>
<entry key="grouproles.delete">
delete from _role_service.group_roles where rolename=? and groupname = ?
</entry>
<entry key="grouproles.deleteRole">
delete from _role_service.group_roles where rolename=?
</entry>
<entry key="grouproles.deleteGroup">
delete from _role_service.group_roles where groupname = ?
</entry>
<entry key="grouproles.deleteAll">
delete from _role_service.group_roles
</entry>


</properties>
97 changes: 97 additions & 0 deletions src/geoserver/role_service.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,97 @@
from distutils.dir_util import copy_tree
import logging
import os
import shutil
import sys
import time
from urllib.parse import urlparse
from lxml import etree as ET

from db import util as db_util
from requests_util import url_util
from . import authn

logger = logging.getLogger(__name__)
logging.basicConfig(stream=sys.stdout, level=logging.DEBUG)


ROLE_SERVICE_PATH = 'security/role/'
DIRECTORY = os.path.dirname(os.path.abspath(__file__))


def wait_for_db(conn_dict):
max_attempts = 10
attempt = 0

while True:
import psycopg2
try:
with psycopg2.connect(**conn_dict):
pass
logger.info(f" Attempt {attempt}/{max_attempts} successful.")
break
except psycopg2.OperationalError:
if attempt >= max_attempts:
logger.info(f" Reaching max attempts when waiting for DB")
sys.exit(1)
time.sleep(2)
attempt += 1


def setup_role_service(data_dir, db_conn, uri_str, internal_service_schema, layman_pg_user, layman_gs_user, layman_gs_role, service_url, role_service_name):
logger.info(f"Ensuring GeoServer DB role service '{role_service_name}' "
f"for URL: {service_url}.")

logger.info(f" Waiting for DB")
wait_for_db(db_conn)

logger.info(f" Checking internal role service DB schema")
schema_query = f'''SELECT COUNT(*) FROM information_schema.schemata WHERE schema_name = '{internal_service_schema}';'''
schema_exists = db_util.run_query(schema_query, uri_str=uri_str)[0][0]
if schema_exists == 0:
logger.info(f" Setting up internal role service DB schema")
statement = f"""
CREATE SCHEMA "{internal_service_schema}" AUTHORIZATION {layman_pg_user};
create view {internal_service_schema}.roles as select 'ADMIN' as name, null as parent
union all select 'GROUP_ADMIN', null
union all select 'LAYMAN_ROLE', null
;
create view {internal_service_schema}.role_props as select null::varchar as rolename, null::varchar as propname, null::varchar as propvalue;
create view {internal_service_schema}.user_roles as select 'layman' as username, 'ADMIN' as rolename
union all select 'layman', 'LAYMAN_ROLE'
union all select 'admin', 'ADMIN'
;
create view {internal_service_schema}.group_roles as select null::varchar as groupname, null::varchar as rolename;
"""
db_util.run_statement(statement, uri_str=uri_str)

logger.info(f" Setting up files")
role_service_path = os.path.join(data_dir, ROLE_SERVICE_PATH)
layman_role_service_path = os.path.join(role_service_path, role_service_name)
if os.path.exists(layman_role_service_path):
shutil.rmtree(layman_role_service_path)
source_path = os.path.join(DIRECTORY, role_service_name)
os.mkdir(layman_role_service_path)
copy_tree(source_path, layman_role_service_path)

role_service_config_path = os.path.join(layman_role_service_path, 'config.xml')
role_service_xml = ET.parse(role_service_config_path)

parsed_url = urlparse(service_url)

element = role_service_xml.find('userName')
element.text = parsed_url.username

element = role_service_xml.find('password')
element.text = parsed_url.password

element = role_service_xml.find('connectURL')
element.text = f'jdbc:{url_util.redact_uri(service_url, remove_username=True)}'

role_service_xml.write(role_service_config_path)

security_xml = authn.get_security(data_dir)
element = security_xml.find('roleServiceName')
element.text = role_service_name
security_path = os.path.join(data_dir, 'security/config.xml')
security_xml.write(security_path)
3 changes: 0 additions & 3 deletions src/layman/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -86,10 +86,7 @@
logger.info(f'Adjusting GeoServer roles')

if settings.GEOSERVER_ADMIN_AUTH:
gs_util.ensure_role(settings.LAYMAN_GS_ROLE, settings.GEOSERVER_ADMIN_AUTH)
gs_util.ensure_user(settings.LAYMAN_GS_USER, settings.LAYMAN_GS_PASSWORD, settings.GEOSERVER_ADMIN_AUTH)
gs_util.ensure_user_role(settings.LAYMAN_GS_USER, 'ADMIN', settings.GEOSERVER_ADMIN_AUTH)
gs_util.ensure_user_role(settings.LAYMAN_GS_USER, settings.LAYMAN_GS_ROLE, settings.GEOSERVER_ADMIN_AUTH)

gs_util.ensure_proxy_base_url(settings.LAYMAN_GS_PROXY_BASE_URL_WITH_PLACEHOLDERS, settings.LAYMAN_GS_AUTH)

Expand Down
8 changes: 8 additions & 0 deletions src/layman/upgrade/upgrade_v1_23.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,14 @@ def create_role_service_schema():
create_schema = f"""CREATE SCHEMA IF NOT EXISTS "{ROLE_SERVICE_SCHEMA}" AUTHORIZATION {settings.LAYMAN_PG_USER};"""
db_util.run_statement(create_schema)

drop_temporary_views = f"""
drop view {ROLE_SERVICE_SCHEMA}.roles;
drop view {ROLE_SERVICE_SCHEMA}.role_props;
drop view {ROLE_SERVICE_SCHEMA}.user_roles;
drop view {ROLE_SERVICE_SCHEMA}.group_roles;
"""
db_util.run_statement(drop_temporary_views)

create_role_table = f"""create table {ROLE_SERVICE_SCHEMA}.bussiness_roles(
id integer GENERATED ALWAYS AS IDENTITY,
name varchar(64) not null,
Expand Down
Loading

0 comments on commit 0a779ae

Please sign in to comment.