From 9a70029d63074b802ee3ff904909d3b6c881c13e Mon Sep 17 00:00:00 2001 From: Rub21 Date: Thu, 23 Jan 2025 19:16:46 -0500 Subject: [PATCH 1/9] Make osm_land as it own maplayer - provider --- images/tiler-server/Dockerfile | 3 +- .../tiler-server/config/config.template.toml | 111 +++++++++++++++++- .../tiler-server/config/providers/land.toml | 92 --------------- 3 files changed, 111 insertions(+), 95 deletions(-) delete mode 100644 images/tiler-server/config/providers/land.toml diff --git a/images/tiler-server/Dockerfile b/images/tiler-server/Dockerfile index 7711f1d4..7fd9fcee 100644 --- a/images/tiler-server/Dockerfile +++ b/images/tiler-server/Dockerfile @@ -31,8 +31,7 @@ RUN mkdir /opt/tegola_config RUN cd /opt/ && python build_config.py \ --output=/opt/tegola_config/config.toml \ --provider_names \ - "land,\ - admin_boundaries_lines,\ + "admin_boundaries_lines,\ admin_boundaries.centroids,\ admin_boundaries_maritime,\ place_areas,\ diff --git a/images/tiler-server/config/config.template.toml b/images/tiler-server/config/config.template.toml index ae968845..cb6e3408 100644 --- a/images/tiler-server/config/config.template.toml +++ b/images/tiler-server/config/config.template.toml @@ -30,6 +30,65 @@ uri = "postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POS max_connections = 100 ###### PROVIDERS +#################################### +## OpenHistoricalMap providers +#################################### +[[providers]] +name = "osm_land" +type = "mvt_postgis" +uri = "postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB}" +max_connections = 50 + ###### PROVIDERS + [[providers.layers]] + name = "land_0-2" + geometry_fieldname = "geometry" + geometry_type = "multipolygon" + id_fieldname = "ogc_fid" + sql = "SELECT ST_AsMVTGeom(geometry,!BBOX!) AS geometry, ogc_fid FROM mview_land_0_2 WHERE geometry && !BBOX!" + + [[providers.layers]] + name = "land_3-5" + geometry_fieldname = "geometry" + geometry_type = "multipolygon" + id_fieldname = "ogc_fid" + sql = "SELECT ST_AsMVTGeom(geometry,!BBOX!) AS geometry, ogc_fid FROM mview_land_3_5 WHERE geometry && !BBOX!" + + [[providers.layers]] + name = "land_6-7" + geometry_fieldname = "geometry" + geometry_type = "multipolygon" + id_fieldname = "ogc_fid" + sql = "SELECT ST_AsMVTGeom(geometry,!BBOX!) AS geometry, ogc_fid FROM mview_land_6_7 WHERE geometry && !BBOX!" + + [[providers.layers]] + name = "land_8-9" + geometry_fieldname = "geometry" + geometry_type = "multipolygon" + id_fieldname = "ogc_fid" + sql = "SELECT ST_AsMVTGeom(geometry,!BBOX!) AS geometry, ogc_fid FROM mview_land_8_9 WHERE geometry && !BBOX!" + + [[providers.layers]] + name = "land_10-12" + geometry_fieldname = "geometry" + geometry_type = "multipolygon" + id_fieldname = "ogc_fid" + sql = "SELECT ST_AsMVTGeom(geometry,!BBOX!) AS geometry, ogc_fid FROM mview_land_10_12 WHERE geometry && !BBOX!" + + [[providers.layers]] + name = "land_13-15" + geometry_fieldname = "geometry" + geometry_type = "multipolygon" + id_fieldname = "ogc_fid" + sql = "SELECT ST_AsMVTGeom(geometry,!BBOX!) AS geometry, ogc_fid FROM mview_land_13_15 WHERE geometry && !BBOX!" + + [[providers.layers]] + name = "land_16-20" + geometry_fieldname = "geometry" + geometry_type = "multipolygon" + id_fieldname = "ogc_fid" + sql = "SELECT ST_AsMVTGeom(geometry,!BBOX!) AS geometry, ogc_fid FROM mview_land_16_20 WHERE geometry && !BBOX!" + + #################################### ## Natural Earth (ne) providers #################################### @@ -37,7 +96,7 @@ max_connections = 100 name = "ne" type = "mvt_postgis" uri = "postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB}" -max_connections = 100 +max_connections = 50 ###### Water lands ne [[providers.layers]] name = "ne_water_lakes_0-2" @@ -69,6 +128,56 @@ attribution = "OpenHistoricalMap" center = [-74.275329586789, -12.153492567373, 1.0] ###### MAPS + +#################################### +## OSM Land maps +#################################### +[[maps]] +name = "osm_land" +attribution = "OSM Land" +center = [-74.275329586789, -12.153492567373, 8.0] + [[maps.layers]] + name = "land" + provider_layer = "osm_land.land_0-2" + min_zoom = 0 + max_zoom = 2 + + [[maps.layers]] + name = "land" + provider_layer = "osm_land.land_3-5" + min_zoom = 3 + max_zoom = 5 + + [[maps.layers]] + name = "land" + provider_layer = "osm_land.land_6-7" + min_zoom = 6 + max_zoom = 7 + + [[maps.layers]] + name = "land" + provider_layer = "osm_land.land_8-9" + min_zoom = 8 + max_zoom = 9 + + [[maps.layers]] + name = "land" + provider_layer = "osm_land.land_10-12" + min_zoom = 10 + max_zoom = 12 + + [[maps.layers]] + name = "land" + provider_layer = "osm_land.land_13-15" + min_zoom = 13 + max_zoom = 15 + + [[maps.layers]] + name = "land" + provider_layer = "osm_land.land_16-20" + min_zoom = 16 + max_zoom = 20 + #################################### ## Natural Earth maps #################################### diff --git a/images/tiler-server/config/providers/land.toml b/images/tiler-server/config/providers/land.toml deleted file mode 100644 index ab287312..00000000 --- a/images/tiler-server/config/providers/land.toml +++ /dev/null @@ -1,92 +0,0 @@ -[[providers.layers]] -name = "land_0-2" -geometry_fieldname = "geometry" -geometry_type = "multipolygon" -id_fieldname = "ogc_fid" -sql = "SELECT ST_AsMVTGeom(geometry,!BBOX!) AS geometry, ogc_fid FROM mview_land_0_2 WHERE geometry && !BBOX!" - -[[providers.layers]] -name = "land_3-5" -geometry_fieldname = "geometry" -geometry_type = "multipolygon" -id_fieldname = "ogc_fid" -sql = "SELECT ST_AsMVTGeom(geometry,!BBOX!) AS geometry, ogc_fid FROM mview_land_3_5 WHERE geometry && !BBOX!" - -[[providers.layers]] -name = "land_6-7" -geometry_fieldname = "geometry" -geometry_type = "multipolygon" -id_fieldname = "ogc_fid" -sql = "SELECT ST_AsMVTGeom(geometry,!BBOX!) AS geometry, ogc_fid FROM mview_land_6_7 WHERE geometry && !BBOX!" - -[[providers.layers]] -name = "land_8-9" -geometry_fieldname = "geometry" -geometry_type = "multipolygon" -id_fieldname = "ogc_fid" -sql = "SELECT ST_AsMVTGeom(geometry,!BBOX!) AS geometry, ogc_fid FROM mview_land_8_9 WHERE geometry && !BBOX!" - -[[providers.layers]] -name = "land_10-12" -geometry_fieldname = "geometry" -geometry_type = "multipolygon" -id_fieldname = "ogc_fid" -sql = "SELECT ST_AsMVTGeom(geometry,!BBOX!) AS geometry, ogc_fid FROM mview_land_10_12 WHERE geometry && !BBOX!" - -[[providers.layers]] -name = "land_13-15" -geometry_fieldname = "geometry" -geometry_type = "multipolygon" -id_fieldname = "ogc_fid" -sql = "SELECT ST_AsMVTGeom(geometry,!BBOX!) AS geometry, ogc_fid FROM mview_land_13_15 WHERE geometry && !BBOX!" - -[[providers.layers]] -name = "land_16-20" -geometry_fieldname = "geometry" -geometry_type = "multipolygon" -id_fieldname = "ogc_fid" -sql = "SELECT ST_AsMVTGeom(geometry,!BBOX!) AS geometry, ogc_fid FROM mview_land_16_20 WHERE geometry && !BBOX!" - -#######Maps - -[[maps.layers]] -name = "land" -provider_layer = "osm.land_0-2" -min_zoom = 0 -max_zoom = 2 - -[[maps.layers]] -name = "land" -provider_layer = "osm.land_3-5" -min_zoom = 3 -max_zoom = 5 - -[[maps.layers]] -name = "land" -provider_layer = "osm.land_6-7" -min_zoom = 6 -max_zoom = 7 - -[[maps.layers]] -name = "land" -provider_layer = "osm.land_8-9" -min_zoom = 8 -max_zoom = 9 - -[[maps.layers]] -name = "land" -provider_layer = "osm.land_10-12" -min_zoom = 10 -max_zoom = 12 - -[[maps.layers]] -name = "land" -provider_layer = "osm.land_13-15" -min_zoom = 13 -max_zoom = 15 - -[[maps.layers]] -name = "land" -provider_layer = "osm.land_16-20" -min_zoom = 16 -max_zoom = 20 From e8089e5c7c2da4817951189f5e8bc17f0b0e874c Mon Sep 17 00:00:00 2001 From: Rub21 Date: Thu, 23 Jan 2025 20:27:12 -0500 Subject: [PATCH 2/9] Use tiler-db that host in hetzner - staging --- hetzner/config/postgresql.config | 10 ++++++++++ images/tiler-server/config/config.template.toml | 8 +++----- images/tiler-server/start.sh | 2 +- values.staging.template.yaml | 6 +++--- 4 files changed, 17 insertions(+), 9 deletions(-) diff --git a/hetzner/config/postgresql.config b/hetzner/config/postgresql.config index acf6f78b..e7d02f99 100644 --- a/hetzner/config/postgresql.config +++ b/hetzner/config/postgresql.config @@ -67,6 +67,16 @@ log_min_duration_statement = 20000 # Log queries that take longer than 20 log_statement = 'none' # Disable logging of all statements log_duration = off # Disable duration logging for all queries + +#------------------------------------------------------------------------------ +# CLIENT CONNECTION DEFAULTS +#------------------------------------------------------------------------------ + +statement_timeout = 60000 # Terminate queries running longer than 60 seconds (in milliseconds) +lock_timeout = 60000 # Timeout for acquiring locks (in milliseconds) +idle_in_transaction_session_timeout = 60000 # Close idle transactions after 60 seconds (in milliseconds) + + #------------------------------------------------------------------------------ # CLIENT CONNECTION DEFAULTS #------------------------------------------------------------------------------ diff --git a/images/tiler-server/config/config.template.toml b/images/tiler-server/config/config.template.toml index cb6e3408..3c38a021 100644 --- a/images/tiler-server/config/config.template.toml +++ b/images/tiler-server/config/config.template.toml @@ -1,7 +1,5 @@ [webserver] port = ":${TILER_SERVER_PORT}" -version = "v0.20.0" - [webserver.headers] Access-Control-Allow-Origin = "*" @@ -27,7 +25,7 @@ type = "prometheus" name = "osm" type = "mvt_postgis" uri = "postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB}" -max_connections = 100 +max_connections = 150 ###### PROVIDERS #################################### @@ -37,7 +35,7 @@ max_connections = 100 name = "osm_land" type = "mvt_postgis" uri = "postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB}" -max_connections = 50 +max_connections = 25 ###### PROVIDERS [[providers.layers]] name = "land_0-2" @@ -96,7 +94,7 @@ max_connections = 50 name = "ne" type = "mvt_postgis" uri = "postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB}" -max_connections = 50 +max_connections = 25 ###### Water lands ne [[providers.layers]] name = "ne_water_lakes_0-2" diff --git a/images/tiler-server/start.sh b/images/tiler-server/start.sh index d52d6e72..d36b7cb4 100755 --- a/images/tiler-server/start.sh +++ b/images/tiler-server/start.sh @@ -64,5 +64,5 @@ while [ "$flag" = true ]; do fi # Start Tegola server - TEGOLA_SQL_DEBUG=LAYER_SQL:EXECUTE_SQL tegola serve --config=/opt/tegola_config/config.toml + tegola serve --config=/opt/tegola_config/config.toml done \ No newline at end of file diff --git a/values.staging.template.yaml b/values.staging.template.yaml index 3ca2840f..facc5264 100644 --- a/values.staging.template.yaml +++ b/values.staging.template.yaml @@ -392,10 +392,10 @@ osm-seed: tilerDb: enabled: true useExternalHost: # When we are using useExternalHost.enabled= true other variables are giong to be disable ans use the external host config - enabled: false + enabled: true env: POSTGRES_HOST: {{STAGING_TILER_DB_HOST}} - POSTGRES_DB: tiler_osm_staging # Kuberntes existing db called, tiler-osm + POSTGRES_DB: tiler_osm_production # Kuberntes existing db called, tiler-osm POSTGRES_USER: postgres POSTGRES_PASSWORD: {{STAGING_TILER_DB_PASSWORD}} POSTGRES_PORT: 5432 @@ -487,7 +487,7 @@ osm-seed: # ==================================================================================================== tilerImposm: - enabled: true + enabled: false env: TILER_IMPORT_FROM: osm TILER_IMPORT_PBF_URL: https://s3.amazonaws.com/planet.openhistoricalmap.org/planet/planet-250117_0002.osm.pbf From 523fcbbaa5797a84ad4d14267f299854aae9fdbf Mon Sep 17 00:00:00 2001 From: Rub21 Date: Thu, 23 Jan 2025 20:52:46 -0500 Subject: [PATCH 3/9] Create unique index for material view --- images/tiler-imposm/materialized_views.py | 26 +++++++++++------------ images/tiler-server/utils/exec.sh | 5 +++-- 2 files changed, 15 insertions(+), 16 deletions(-) diff --git a/images/tiler-imposm/materialized_views.py b/images/tiler-imposm/materialized_views.py index da268a96..b58510d3 100644 --- a/images/tiler-imposm/materialized_views.py +++ b/images/tiler-imposm/materialized_views.py @@ -70,24 +70,23 @@ def get_columns_of_table(table_name: str) -> list: columns = [col.strip() for col in result.stdout.strip().split("\n") if col.strip()] return columns - def create_indexes_for_mview(mview_name: str, columns: list): """ - Creates indexes for 'osm_id' (B-Tree) and 'geometry' (GiST) in the specified materialized view, - only if those columns exist. + Creates indexes for the 'osm_id' (B-Tree) and 'geometry' (GiST) columns. """ - # Check if the columns exist + + # Ensure 'osm_id' exists in the columns list if "osm_id" in columns: - create_idx_osm = f"CREATE INDEX idx_{mview_name}_osm_id ON {mview_name} (osm_id);" - logger.info(f"Creating index for osm_id in {mview_name}") - execute_psql_query(create_idx_osm) + create_unique_idx = f"CREATE UNIQUE INDEX IF NOT EXISTS idx_{mview_name}_osm_id ON {mview_name} (osm_id);" + logger.info(f"Creating unique index for osm_id in {mview_name}") + execute_psql_query(create_unique_idx) + else: + logger.warning(f"'osm_id' column not found in {mview_name}, skipping unique index creation.") if "geometry" in columns: - create_idx_geom = ( - f"CREATE INDEX idx_{mview_name}_geom ON {mview_name} USING GIST (geometry);" - ) - logger.info(f"Creating index for geometry in {mview_name}") - execute_psql_query(create_idx_geom) + create_geom_idx = f"CREATE INDEX IF NOT EXISTS idx_{mview_name}_geom ON {mview_name} USING GIST (geometry);" + logger.info(f"Creating GiST index for geometry in {mview_name}") + execute_psql_query(create_geom_idx) def create_materialized_view( @@ -204,12 +203,11 @@ def refresh_all_materialized_views(config_dict: dict): mview_name = mt_view.get("view") if object_exists(mview_name): logger.info(f"Refreshing materialized view: {mview_name}") - query = f"REFRESH MATERIALIZED VIEW {mview_name};" + query = f"REFRESH MATERIALIZED VIEW CONCURRENTLY {mview_name};" execute_psql_query(query) else: logger.warning(f"Materialized view {mview_name} not found. Skipping refresh.") - # ------------------------------------------------------------------------------ # MAIN: LOADS CONFIG ONCE, CREATES/UPDATES VIEWS, THEN REFRESHES THEM IN A LOOP # ------------------------------------------------------------------------------ diff --git a/images/tiler-server/utils/exec.sh b/images/tiler-server/utils/exec.sh index 783d1cce..4f8ec99d 100755 --- a/images/tiler-server/utils/exec.sh +++ b/images/tiler-server/utils/exec.sh @@ -1,4 +1,5 @@ -python query_benchmarking.py \ +python3 query_benchmarking.py \ --toml ./../config/config.toml \ --bbox 1.3149614848125e+07,2.5046885425000004e+06,1.3775786983749997e+07,3.1308606781250015e+06,3857 \ - --zoom 6 \ No newline at end of file + --zoom 6 + \ No newline at end of file From 5eeb16456437fe70fe1febc854e3a39331f4464b Mon Sep 17 00:00:00 2001 From: Rub21 Date: Thu, 23 Jan 2025 22:36:11 -0500 Subject: [PATCH 4/9] Update map-style gitsha --- hetzner/config/postgresql.config | 14 +++++++------- images/web/Dockerfile | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/hetzner/config/postgresql.config b/hetzner/config/postgresql.config index e7d02f99..7f0636dc 100644 --- a/hetzner/config/postgresql.config +++ b/hetzner/config/postgresql.config @@ -3,7 +3,7 @@ #------------------------------------------------------------------------------ listen_addresses = '*' # Allow connections from all addresses -max_connections = 100 # Set max connections to 120 +max_connections = 200 # Set max connections to 120 superuser_reserved_connections = 3 # Reserve for superusers #------------------------------------------------------------------------------ @@ -13,7 +13,7 @@ superuser_reserved_connections = 3 # Reserve for superusers # - Memory Configuration - shared_buffers = 14GB # Allocate 25% of total memory -work_mem = 512MB # Increase per-query memory for sorting, hashing +work_mem = 256MB # Increase per-query memory for sorting, hashing maintenance_work_mem = 2GB # For maintenance operations like VACUUM, ALTER effective_cache_size = 41GB # 75% of total memory for caching @@ -50,7 +50,7 @@ parallel_tuple_cost = 0.1 # Lower to encourage parallelism parallel_setup_cost = 500 # Lower setup cost for parallel processing max_worker_processes = 28 # Utilize CPU cores efficiently max_parallel_workers_per_gather = 8 # Allow parallel processing for queries -max_parallel_workers = 14 # Limit the total parallel workers +max_parallel_workers = 28 # Limit the total parallel workers #------------------------------------------------------------------------------ # LOGGING @@ -64,7 +64,7 @@ log_truncate_on_rotation = on # Overwrite old logs with the same nam log_min_duration_statement = 20000 # Log queries that take longer than 20 seconds (in milliseconds) -log_statement = 'none' # Disable logging of all statements +log_statement = 'ddl' # Disable logging of all statements log_duration = off # Disable duration logging for all queries @@ -72,9 +72,9 @@ log_duration = off # Disable duration logging for all qu # CLIENT CONNECTION DEFAULTS #------------------------------------------------------------------------------ -statement_timeout = 60000 # Terminate queries running longer than 60 seconds (in milliseconds) -lock_timeout = 60000 # Timeout for acquiring locks (in milliseconds) -idle_in_transaction_session_timeout = 60000 # Close idle transactions after 60 seconds (in milliseconds) +statement_timeout = 600000 # Terminate queries running longer than 10 min (in milliseconds) +lock_timeout = 600000 # Timeout for acquiring locks (in milliseconds) +idle_in_transaction_session_timeout = 600000 # Close idle transactions after 10 min (in milliseconds) #------------------------------------------------------------------------------ diff --git a/images/web/Dockerfile b/images/web/Dockerfile index 065de1e1..be53b59a 100644 --- a/images/web/Dockerfile +++ b/images/web/Dockerfile @@ -133,7 +133,7 @@ RUN apache2ctl configtest RUN chown -R www-data: $workdir # Clone Map-styles -ENV OPENHISTORICALMAP_MAP_STYLES_GITSHA=6de6e329569ab6561d1fa6ab2cebb42df7fd17c7 +ENV OPENHISTORICALMAP_MAP_STYLES_GITSHA=34b84bbbd1307698ee1d30a32dc96222b8992de4 RUN git clone --branch staging --depth 1 https://github.com/OpenHistoricalMap/map-styles.git $workdir/public/map-styles RUN cd $workdir/public/map-styles && git fetch --depth 1 origin $OPENHISTORICALMAP_MAP_STYLES_GITSHA && git checkout $OPENHISTORICALMAP_MAP_STYLES_GITSHA RUN rm -rf $workdir/public/map-styles/.git From e6149187bf75a9ef369a3da6beb64a789e6260aa Mon Sep 17 00:00:00 2001 From: Rub21 Date: Thu, 23 Jan 2025 23:19:32 -0500 Subject: [PATCH 5/9] Update map-style gitsha --- images/web/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/images/web/Dockerfile b/images/web/Dockerfile index be53b59a..26c345d7 100644 --- a/images/web/Dockerfile +++ b/images/web/Dockerfile @@ -133,7 +133,7 @@ RUN apache2ctl configtest RUN chown -R www-data: $workdir # Clone Map-styles -ENV OPENHISTORICALMAP_MAP_STYLES_GITSHA=34b84bbbd1307698ee1d30a32dc96222b8992de4 +ENV OPENHISTORICALMAP_MAP_STYLES_GITSHA=57ce4507696d26b2f4ce96dbcc0f7b38ac6a0b16 RUN git clone --branch staging --depth 1 https://github.com/OpenHistoricalMap/map-styles.git $workdir/public/map-styles RUN cd $workdir/public/map-styles && git fetch --depth 1 origin $OPENHISTORICALMAP_MAP_STYLES_GITSHA && git checkout $OPENHISTORICALMAP_MAP_STYLES_GITSHA RUN rm -rf $workdir/public/map-styles/.git From 1998b1e5a8a9189d1244731df404cd41c7b0c2ae Mon Sep 17 00:00:00 2001 From: Rub21 Date: Fri, 24 Jan 2025 15:50:19 -0500 Subject: [PATCH 6/9] Revert using land on osm source --- images/tiler-server/Dockerfile | 3 +- .../tiler-server/config/providers/land.toml | 92 +++++++++++++++++++ images/web/Dockerfile | 2 +- 3 files changed, 95 insertions(+), 2 deletions(-) create mode 100644 images/tiler-server/config/providers/land.toml diff --git a/images/tiler-server/Dockerfile b/images/tiler-server/Dockerfile index 7fd9fcee..7711f1d4 100644 --- a/images/tiler-server/Dockerfile +++ b/images/tiler-server/Dockerfile @@ -31,7 +31,8 @@ RUN mkdir /opt/tegola_config RUN cd /opt/ && python build_config.py \ --output=/opt/tegola_config/config.toml \ --provider_names \ - "admin_boundaries_lines,\ + "land,\ + admin_boundaries_lines,\ admin_boundaries.centroids,\ admin_boundaries_maritime,\ place_areas,\ diff --git a/images/tiler-server/config/providers/land.toml b/images/tiler-server/config/providers/land.toml new file mode 100644 index 00000000..ab287312 --- /dev/null +++ b/images/tiler-server/config/providers/land.toml @@ -0,0 +1,92 @@ +[[providers.layers]] +name = "land_0-2" +geometry_fieldname = "geometry" +geometry_type = "multipolygon" +id_fieldname = "ogc_fid" +sql = "SELECT ST_AsMVTGeom(geometry,!BBOX!) AS geometry, ogc_fid FROM mview_land_0_2 WHERE geometry && !BBOX!" + +[[providers.layers]] +name = "land_3-5" +geometry_fieldname = "geometry" +geometry_type = "multipolygon" +id_fieldname = "ogc_fid" +sql = "SELECT ST_AsMVTGeom(geometry,!BBOX!) AS geometry, ogc_fid FROM mview_land_3_5 WHERE geometry && !BBOX!" + +[[providers.layers]] +name = "land_6-7" +geometry_fieldname = "geometry" +geometry_type = "multipolygon" +id_fieldname = "ogc_fid" +sql = "SELECT ST_AsMVTGeom(geometry,!BBOX!) AS geometry, ogc_fid FROM mview_land_6_7 WHERE geometry && !BBOX!" + +[[providers.layers]] +name = "land_8-9" +geometry_fieldname = "geometry" +geometry_type = "multipolygon" +id_fieldname = "ogc_fid" +sql = "SELECT ST_AsMVTGeom(geometry,!BBOX!) AS geometry, ogc_fid FROM mview_land_8_9 WHERE geometry && !BBOX!" + +[[providers.layers]] +name = "land_10-12" +geometry_fieldname = "geometry" +geometry_type = "multipolygon" +id_fieldname = "ogc_fid" +sql = "SELECT ST_AsMVTGeom(geometry,!BBOX!) AS geometry, ogc_fid FROM mview_land_10_12 WHERE geometry && !BBOX!" + +[[providers.layers]] +name = "land_13-15" +geometry_fieldname = "geometry" +geometry_type = "multipolygon" +id_fieldname = "ogc_fid" +sql = "SELECT ST_AsMVTGeom(geometry,!BBOX!) AS geometry, ogc_fid FROM mview_land_13_15 WHERE geometry && !BBOX!" + +[[providers.layers]] +name = "land_16-20" +geometry_fieldname = "geometry" +geometry_type = "multipolygon" +id_fieldname = "ogc_fid" +sql = "SELECT ST_AsMVTGeom(geometry,!BBOX!) AS geometry, ogc_fid FROM mview_land_16_20 WHERE geometry && !BBOX!" + +#######Maps + +[[maps.layers]] +name = "land" +provider_layer = "osm.land_0-2" +min_zoom = 0 +max_zoom = 2 + +[[maps.layers]] +name = "land" +provider_layer = "osm.land_3-5" +min_zoom = 3 +max_zoom = 5 + +[[maps.layers]] +name = "land" +provider_layer = "osm.land_6-7" +min_zoom = 6 +max_zoom = 7 + +[[maps.layers]] +name = "land" +provider_layer = "osm.land_8-9" +min_zoom = 8 +max_zoom = 9 + +[[maps.layers]] +name = "land" +provider_layer = "osm.land_10-12" +min_zoom = 10 +max_zoom = 12 + +[[maps.layers]] +name = "land" +provider_layer = "osm.land_13-15" +min_zoom = 13 +max_zoom = 15 + +[[maps.layers]] +name = "land" +provider_layer = "osm.land_16-20" +min_zoom = 16 +max_zoom = 20 diff --git a/images/web/Dockerfile b/images/web/Dockerfile index 26c345d7..065de1e1 100644 --- a/images/web/Dockerfile +++ b/images/web/Dockerfile @@ -133,7 +133,7 @@ RUN apache2ctl configtest RUN chown -R www-data: $workdir # Clone Map-styles -ENV OPENHISTORICALMAP_MAP_STYLES_GITSHA=57ce4507696d26b2f4ce96dbcc0f7b38ac6a0b16 +ENV OPENHISTORICALMAP_MAP_STYLES_GITSHA=6de6e329569ab6561d1fa6ab2cebb42df7fd17c7 RUN git clone --branch staging --depth 1 https://github.com/OpenHistoricalMap/map-styles.git $workdir/public/map-styles RUN cd $workdir/public/map-styles && git fetch --depth 1 origin $OPENHISTORICALMAP_MAP_STYLES_GITSHA && git checkout $OPENHISTORICALMAP_MAP_STYLES_GITSHA RUN rm -rf $workdir/public/map-styles/.git From 4e114124caa66fc629b244ef602221a09bfd7616 Mon Sep 17 00:00:00 2001 From: Rub21 Date: Fri, 24 Jan 2025 16:56:44 -0500 Subject: [PATCH 7/9] Update tiler tegola config for ohm/osm source --- .../tiler-server/config/config.template.toml | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/images/tiler-server/config/config.template.toml b/images/tiler-server/config/config.template.toml index 3c38a021..c61b6a3b 100644 --- a/images/tiler-server/config/config.template.toml +++ b/images/tiler-server/config/config.template.toml @@ -38,49 +38,49 @@ uri = "postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POS max_connections = 25 ###### PROVIDERS [[providers.layers]] - name = "land_0-2" + name = "osm_land_0-2" geometry_fieldname = "geometry" geometry_type = "multipolygon" id_fieldname = "ogc_fid" sql = "SELECT ST_AsMVTGeom(geometry,!BBOX!) AS geometry, ogc_fid FROM mview_land_0_2 WHERE geometry && !BBOX!" [[providers.layers]] - name = "land_3-5" + name = "osm_land_3-5" geometry_fieldname = "geometry" geometry_type = "multipolygon" id_fieldname = "ogc_fid" sql = "SELECT ST_AsMVTGeom(geometry,!BBOX!) AS geometry, ogc_fid FROM mview_land_3_5 WHERE geometry && !BBOX!" [[providers.layers]] - name = "land_6-7" + name = "osm_land_6-7" geometry_fieldname = "geometry" geometry_type = "multipolygon" id_fieldname = "ogc_fid" sql = "SELECT ST_AsMVTGeom(geometry,!BBOX!) AS geometry, ogc_fid FROM mview_land_6_7 WHERE geometry && !BBOX!" [[providers.layers]] - name = "land_8-9" + name = "osm_land_8-9" geometry_fieldname = "geometry" geometry_type = "multipolygon" id_fieldname = "ogc_fid" sql = "SELECT ST_AsMVTGeom(geometry,!BBOX!) AS geometry, ogc_fid FROM mview_land_8_9 WHERE geometry && !BBOX!" [[providers.layers]] - name = "land_10-12" + name = "osm_land_10-12" geometry_fieldname = "geometry" geometry_type = "multipolygon" id_fieldname = "ogc_fid" sql = "SELECT ST_AsMVTGeom(geometry,!BBOX!) AS geometry, ogc_fid FROM mview_land_10_12 WHERE geometry && !BBOX!" [[providers.layers]] - name = "land_13-15" + name = "osm_land_13-15" geometry_fieldname = "geometry" geometry_type = "multipolygon" id_fieldname = "ogc_fid" sql = "SELECT ST_AsMVTGeom(geometry,!BBOX!) AS geometry, ogc_fid FROM mview_land_13_15 WHERE geometry && !BBOX!" [[providers.layers]] - name = "land_16-20" + name = "osm_land_16-20" geometry_fieldname = "geometry" geometry_type = "multipolygon" id_fieldname = "ogc_fid" @@ -136,43 +136,43 @@ attribution = "OSM Land" center = [-74.275329586789, -12.153492567373, 8.0] [[maps.layers]] name = "land" - provider_layer = "osm_land.land_0-2" + provider_layer = "osm_land.osm_land_0-2" min_zoom = 0 max_zoom = 2 [[maps.layers]] name = "land" - provider_layer = "osm_land.land_3-5" + provider_layer = "osm_land.osm_land_3-5" min_zoom = 3 max_zoom = 5 [[maps.layers]] name = "land" - provider_layer = "osm_land.land_6-7" + provider_layer = "osm_land.osm_land_6-7" min_zoom = 6 max_zoom = 7 [[maps.layers]] name = "land" - provider_layer = "osm_land.land_8-9" + provider_layer = "osm_land.osm_land_8-9" min_zoom = 8 max_zoom = 9 [[maps.layers]] name = "land" - provider_layer = "osm_land.land_10-12" + provider_layer = "osm_land.osm_land_10-12" min_zoom = 10 max_zoom = 12 [[maps.layers]] name = "land" - provider_layer = "osm_land.land_13-15" + provider_layer = "osm_land.osm_land_13-15" min_zoom = 13 max_zoom = 15 [[maps.layers]] name = "land" - provider_layer = "osm_land.land_16-20" + provider_layer = "osm_land.osm_land_16-20" min_zoom = 16 max_zoom = 20 From 30e5444c7c12420cc39fc276d19c728d6216d929 Mon Sep 17 00:00:00 2001 From: Rub21 Date: Fri, 24 Jan 2025 17:00:04 -0500 Subject: [PATCH 8/9] Remove osm_lad source --- .../tiler-server/config/config.template.toml | 109 ------------------ 1 file changed, 109 deletions(-) diff --git a/images/tiler-server/config/config.template.toml b/images/tiler-server/config/config.template.toml index c61b6a3b..bfc0b961 100644 --- a/images/tiler-server/config/config.template.toml +++ b/images/tiler-server/config/config.template.toml @@ -28,65 +28,6 @@ uri = "postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POS max_connections = 150 ###### PROVIDERS -#################################### -## OpenHistoricalMap providers -#################################### -[[providers]] -name = "osm_land" -type = "mvt_postgis" -uri = "postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB}" -max_connections = 25 - ###### PROVIDERS - [[providers.layers]] - name = "osm_land_0-2" - geometry_fieldname = "geometry" - geometry_type = "multipolygon" - id_fieldname = "ogc_fid" - sql = "SELECT ST_AsMVTGeom(geometry,!BBOX!) AS geometry, ogc_fid FROM mview_land_0_2 WHERE geometry && !BBOX!" - - [[providers.layers]] - name = "osm_land_3-5" - geometry_fieldname = "geometry" - geometry_type = "multipolygon" - id_fieldname = "ogc_fid" - sql = "SELECT ST_AsMVTGeom(geometry,!BBOX!) AS geometry, ogc_fid FROM mview_land_3_5 WHERE geometry && !BBOX!" - - [[providers.layers]] - name = "osm_land_6-7" - geometry_fieldname = "geometry" - geometry_type = "multipolygon" - id_fieldname = "ogc_fid" - sql = "SELECT ST_AsMVTGeom(geometry,!BBOX!) AS geometry, ogc_fid FROM mview_land_6_7 WHERE geometry && !BBOX!" - - [[providers.layers]] - name = "osm_land_8-9" - geometry_fieldname = "geometry" - geometry_type = "multipolygon" - id_fieldname = "ogc_fid" - sql = "SELECT ST_AsMVTGeom(geometry,!BBOX!) AS geometry, ogc_fid FROM mview_land_8_9 WHERE geometry && !BBOX!" - - [[providers.layers]] - name = "osm_land_10-12" - geometry_fieldname = "geometry" - geometry_type = "multipolygon" - id_fieldname = "ogc_fid" - sql = "SELECT ST_AsMVTGeom(geometry,!BBOX!) AS geometry, ogc_fid FROM mview_land_10_12 WHERE geometry && !BBOX!" - - [[providers.layers]] - name = "osm_land_13-15" - geometry_fieldname = "geometry" - geometry_type = "multipolygon" - id_fieldname = "ogc_fid" - sql = "SELECT ST_AsMVTGeom(geometry,!BBOX!) AS geometry, ogc_fid FROM mview_land_13_15 WHERE geometry && !BBOX!" - - [[providers.layers]] - name = "osm_land_16-20" - geometry_fieldname = "geometry" - geometry_type = "multipolygon" - id_fieldname = "ogc_fid" - sql = "SELECT ST_AsMVTGeom(geometry,!BBOX!) AS geometry, ogc_fid FROM mview_land_16_20 WHERE geometry && !BBOX!" - - #################################### ## Natural Earth (ne) providers #################################### @@ -126,56 +67,6 @@ attribution = "OpenHistoricalMap" center = [-74.275329586789, -12.153492567373, 1.0] ###### MAPS - -#################################### -## OSM Land maps -#################################### -[[maps]] -name = "osm_land" -attribution = "OSM Land" -center = [-74.275329586789, -12.153492567373, 8.0] - [[maps.layers]] - name = "land" - provider_layer = "osm_land.osm_land_0-2" - min_zoom = 0 - max_zoom = 2 - - [[maps.layers]] - name = "land" - provider_layer = "osm_land.osm_land_3-5" - min_zoom = 3 - max_zoom = 5 - - [[maps.layers]] - name = "land" - provider_layer = "osm_land.osm_land_6-7" - min_zoom = 6 - max_zoom = 7 - - [[maps.layers]] - name = "land" - provider_layer = "osm_land.osm_land_8-9" - min_zoom = 8 - max_zoom = 9 - - [[maps.layers]] - name = "land" - provider_layer = "osm_land.osm_land_10-12" - min_zoom = 10 - max_zoom = 12 - - [[maps.layers]] - name = "land" - provider_layer = "osm_land.osm_land_13-15" - min_zoom = 13 - max_zoom = 15 - - [[maps.layers]] - name = "land" - provider_layer = "osm_land.osm_land_16-20" - min_zoom = 16 - max_zoom = 20 - #################################### ## Natural Earth maps #################################### From ea4ef15c75d5c15e101e4d72c8d2de31c681d514 Mon Sep 17 00:00:00 2001 From: Rub21 Date: Fri, 24 Jan 2025 17:05:58 -0500 Subject: [PATCH 9/9] Use tiler-db in hetzner for production --- values.production.template.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/values.production.template.yaml b/values.production.template.yaml index fe3382fd..33d41eed 100644 --- a/values.production.template.yaml +++ b/values.production.template.yaml @@ -362,7 +362,7 @@ osm-seed: tilerDb: enabled: true useExternalHost: # When we are using useExternalHost.enabled= true other variables are giong to be disable ans use the external host config - enabled: false + enabled: true nodeSelector: enabled: true label_key: nodegroup_type @@ -455,7 +455,7 @@ osm-seed: # ==================================================================================================== tilerImposm: - enabled: true + enabled: false nodeSelector: enabled: true label_key: nodegroup_type