Merge pull request #466 from OpenHistoricalMap/staging

Use Hetzner tiler-db for production and update Material views

hetzner/config/postgresql.config

@@ -3,7 +3,7 @@
 #------------------------------------------------------------------------------
 
 listen_addresses = '*'                  # Allow connections from all addresses
-max_connections = 100                    # Set max connections to 120
+max_connections = 200                    # Set max connections to 120
 superuser_reserved_connections = 3       # Reserve for superusers
 
 #------------------------------------------------------------------------------
@@ -13,7 +13,7 @@ superuser_reserved_connections = 3       # Reserve for superusers
 # - Memory Configuration -
 
 shared_buffers = 14GB                    # Allocate 25% of total memory
-work_mem = 512MB                           # Increase per-query memory for sorting, hashing
+work_mem = 256MB                           # Increase per-query memory for sorting, hashing
 maintenance_work_mem = 2GB                # For maintenance operations like VACUUM, ALTER
 effective_cache_size = 41GB                # 75% of total memory for caching
 
@@ -50,7 +50,7 @@ parallel_tuple_cost = 0.1                      # Lower to encourage parallelism
 parallel_setup_cost = 500                       # Lower setup cost for parallel processing
 max_worker_processes = 28                      # Utilize CPU cores efficiently
 max_parallel_workers_per_gather = 8            # Allow parallel processing for queries
-max_parallel_workers = 14                       # Limit the total parallel workers
+max_parallel_workers = 28                       # Limit the total parallel workers
 
 #------------------------------------------------------------------------------
 # LOGGING
@@ -64,9 +64,19 @@ log_truncate_on_rotation = on             # Overwrite old logs with the same nam
 
 log_min_duration_statement = 20000        # Log queries that take longer than 20 seconds (in milliseconds)
 
-log_statement = 'none'                    # Disable logging of all statements
+log_statement = 'ddl'                    # Disable logging of all statements
 log_duration = off                         # Disable duration logging for all queries
 
+
+#------------------------------------------------------------------------------
+# CLIENT CONNECTION DEFAULTS
+#------------------------------------------------------------------------------
+
+statement_timeout = 600000                  # Terminate queries running longer than 10 min (in milliseconds)
+lock_timeout = 600000                        # Timeout for acquiring locks (in milliseconds)
+idle_in_transaction_session_timeout = 600000 # Close idle transactions after 10 min (in milliseconds)
+
+
 #------------------------------------------------------------------------------
 # CLIENT CONNECTION DEFAULTS
 #------------------------------------------------------------------------------

images/tiler-imposm/materialized_views.py

@@ -70,24 +70,23 @@ def get_columns_of_table(table_name: str) -> list:
     columns = [col.strip() for col in result.stdout.strip().split("\n") if col.strip()]
     return columns
 
-
 def create_indexes_for_mview(mview_name: str, columns: list):
     """
-    Creates indexes for 'osm_id' (B-Tree) and 'geometry' (GiST) in the specified materialized view,
-    only if those columns exist.
+    Creates indexes for the 'osm_id' (B-Tree) and 'geometry' (GiST) columns.
     """
-    # Check if the columns exist
+
+    # Ensure 'osm_id' exists in the columns list
     if "osm_id" in columns:
-        create_idx_osm = f"CREATE INDEX idx_{mview_name}_osm_id ON {mview_name} (osm_id);"
-        logger.info(f"Creating index for osm_id in {mview_name}")
-        execute_psql_query(create_idx_osm)
+        create_unique_idx = f"CREATE UNIQUE INDEX IF NOT EXISTS idx_{mview_name}_osm_id ON {mview_name} (osm_id);"
+        logger.info(f"Creating unique index for osm_id in {mview_name}")
+        execute_psql_query(create_unique_idx)
+    else:
+        logger.warning(f"'osm_id' column not found in {mview_name}, skipping unique index creation.")
 
     if "geometry" in columns:
-        create_idx_geom = (
-            f"CREATE INDEX idx_{mview_name}_geom ON {mview_name} USING GIST (geometry);"
-        )
-        logger.info(f"Creating index for geometry in {mview_name}")
-        execute_psql_query(create_idx_geom)
+        create_geom_idx = f"CREATE INDEX IF NOT EXISTS idx_{mview_name}_geom ON {mview_name} USING GIST (geometry);"
+        logger.info(f"Creating GiST index for geometry in {mview_name}")
+        execute_psql_query(create_geom_idx)
 
 
 def create_materialized_view(
@@ -204,12 +203,11 @@ def refresh_all_materialized_views(config_dict: dict):
             mview_name = mt_view.get("view")
             if object_exists(mview_name):
                 logger.info(f"Refreshing materialized view: {mview_name}")
-                query = f"REFRESH MATERIALIZED VIEW {mview_name};"
+                query = f"REFRESH MATERIALIZED VIEW CONCURRENTLY {mview_name};"
                 execute_psql_query(query)
             else:
                 logger.warning(f"Materialized view {mview_name} not found. Skipping refresh.")
 
-
 # ------------------------------------------------------------------------------
 #  MAIN: LOADS CONFIG ONCE, CREATES/UPDATES VIEWS, THEN REFRESHES THEM IN A LOOP
 # ------------------------------------------------------------------------------

images/tiler-server/config/config.template.toml

@@ -1,7 +1,5 @@
 [webserver]
 port = ":${TILER_SERVER_PORT}"
-version = "v0.20.0"
-
 	[webserver.headers]
 	Access-Control-Allow-Origin = "*"
 
@@ -27,7 +25,7 @@ type = "prometheus"
 name = "osm"
 type = "mvt_postgis"
 uri = "postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB}"
-max_connections = 100
+max_connections = 150
 	###### PROVIDERS
 
 ####################################
@@ -37,7 +35,7 @@ max_connections = 100
 name = "ne"
 type = "mvt_postgis"
 uri = "postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB}"
-max_connections = 100
+max_connections = 25
 	###### Water lands ne
 	[[providers.layers]]
 	name = "ne_water_lakes_0-2"

images/tiler-server/start.sh

@@ -64,5 +64,5 @@ while [ "$flag" = true ]; do
   fi
 
   # Start Tegola server
-  TEGOLA_SQL_DEBUG=LAYER_SQL:EXECUTE_SQL tegola serve --config=/opt/tegola_config/config.toml
+  tegola serve --config=/opt/tegola_config/config.toml
 done

images/tiler-server/utils/exec.sh

@@ -1,4 +1,5 @@
-python query_benchmarking.py \
+python3 query_benchmarking.py \
     --toml ./../config/config.toml \
     --bbox 1.3149614848125e+07,2.5046885425000004e+06,1.3775786983749997e+07,3.1308606781250015e+06,3857 \
-    --zoom 6
+    --zoom 6
+

values.production.template.yaml

@@ -362,7 +362,7 @@ osm-seed:
   tilerDb:
     enabled: true
     useExternalHost: # When we are using useExternalHost.enabled= true other variables are giong to be disable ans use the external host config
-      enabled: false
+      enabled: true
     nodeSelector:
       enabled: true
       label_key: nodegroup_type
@@ -455,7 +455,7 @@ osm-seed:
   # ====================================================================================================
 
   tilerImposm:
-    enabled: true
+    enabled: false
     nodeSelector:
       enabled: true
       label_key: nodegroup_type

values.staging.template.yaml

@@ -392,10 +392,10 @@ osm-seed:
     tilerDb:
       enabled: true
       useExternalHost: # When we are using useExternalHost.enabled= true other variables are giong to be disable ans use the external host config
-        enabled: false
+        enabled: true
       env:
         POSTGRES_HOST: {{STAGING_TILER_DB_HOST}}
-        POSTGRES_DB: tiler_osm_staging # Kuberntes existing db called,  tiler-osm
+        POSTGRES_DB: tiler_osm_production # Kuberntes existing db called,  tiler-osm
         POSTGRES_USER: postgres
         POSTGRES_PASSWORD: {{STAGING_TILER_DB_PASSWORD}}
         POSTGRES_PORT: 5432
@@ -487,7 +487,7 @@ osm-seed:
     # ====================================================================================================
 
     tilerImposm:
-      enabled: true
+      enabled: false
       env:
         TILER_IMPORT_FROM: osm
         TILER_IMPORT_PBF_URL: https://s3.amazonaws.com/planet.openhistoricalmap.org/planet/planet-250117_0002.osm.pbf