Update node and imposm date funcion

OpenHistoricalMap · Dec 7, 2024 · 4b663ce · 4b663ce
1 parent 45e5613
commit 4b663ce
Show file tree

Hide file tree

Showing 3 changed files with 21 additions and 14 deletions.
diff --git a/images/tiler-imposm/Dockerfile b/images/tiler-imposm/Dockerfile
@@ -41,6 +41,9 @@ RUN git clone https://github.com/omniscale/imposm3.git $GOPATH/src/github.com/om
 WORKDIR $GOPATH/src/github.com/omniscale/imposm3
 RUN git checkout v0.14.0
 
+RUN git clone https://github.com/OpenHistoricalMap/DateFunctions-plpgsql.git /usr/local/datefunctions
+WORKDIR /usr/local/datefunctions
+
 # Apply fixes or patches (if necessary)
 RUN sed -i '/setMaxFileSize/d' cache/ldb_pre_121.go
 RUN go install github.com/omniscale/imposm3/cmd/imposm

diff --git a/images/tiler-imposm/start.sh b/images/tiler-imposm/start.sh
@@ -15,6 +15,8 @@ LIMITFILE="limitFile.geojson"
 BUCKET_IMPOSM_FOLDER=imposm
 INIT_FILE=/mnt/data/init_done
 
+PG_CONNECTION="postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@$POSTGRES_HOST/$POSTGRES_DB"
+
 # tracking file
 TRACKING_FILE="$WORKDIR/uploaded_files.log"
 [ -f "$TRACKING_FILE" ] || touch "$TRACKING_FILE"
@@ -136,7 +138,7 @@ function updateData() {
 function importData() {
     ### Import the PBF  and Natural Earth files to the DB
     echo "Execute the missing functions"
-    psql "postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@$POSTGRES_HOST/$POSTGRES_DB" -f queries/postgis_helpers.sql
+    psql $PG_CONNECTION -f queries/postgis_helpers.sql
 
     if [ "$IMPORT_NATURAL_EARTH" = "true" ]; then
         echo "Importing Natural Earth..."
@@ -170,25 +172,25 @@ function importData() {
         -deployproduction
 
     # These index will help speed up tegola tile generation
-    # psql "postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@$POSTGRES_HOST/$POSTGRES_DB" -f queries/postgis_index.sql
-    psql "postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@$POSTGRES_HOST/$POSTGRES_DB" -f queries/postgis_post_import.sql
+    # psql $PG_CONNECTION -f queries/postgis_index.sql
+    psql $PG_CONNECTION -f queries/postgis_post_import.sql
 
     touch $INIT_FILE
 
     # Update tables
     python update_tables.py
 
     echo "Create Table/Tigger for osm_relation_menbers_routes_merged"
-    # psql "postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@$POSTGRES_HOST/$POSTGRES_DB" -f queries/osm_relation_menbers_routes_table.sql
-    # psql "postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@$POSTGRES_HOST/$POSTGRES_DB" -f queries/osm_relation_menbers_routes_trigger.sql
-    psql "postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@$POSTGRES_HOST/$POSTGRES_DB" -f queries/admin_boundaries_centroids.sql
+    # psql $PG_CONNECTION -f queries/osm_relation_menbers_routes_table.sql
+    # psql $PG_CONNECTION -f queries/osm_relation_menbers_routes_trigger.sql
+    psql $PG_CONNECTION -f queries/admin_boundaries_centroids.sql
 
     # Updata data with minute replication
     updateData
 }
 
 function countTables() {
-    psql "postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@$POSTGRES_HOST/$POSTGRES_DB" -t -c "SELECT count(*) FROM information_schema.tables WHERE table_schema='public';" | xargs
+    psql $PG_CONNECTION -t -c "SELECT count(*) FROM information_schema.tables WHERE table_schema='public';" | xargs
 }
 
 echo "Connecting to $POSTGRES_HOST DB"
@@ -197,6 +199,8 @@ while $flag; do
     pg_isready -h $POSTGRES_HOST -p $POSTGRES_PORT >/dev/null 2>&2 || continue
     # Change flag to false to stop pinging the DB
     flag=false
+    echo "Run date functions"
+    psql $PG_CONNECTION -f /usr/local/datefunctions/datefunctions.sql
     echo "Check number of tables in the database"
     table_count=$(countTables)
     echo "Check if $INIT_FILE exists"

diff --git a/values.staging.template.yaml b/values.staging.template.yaml
@@ -327,7 +327,7 @@ osm-seed:
         nodeSelector:
           enabled: true
           label_key: nodegroup_type
-          label_value: web_large
+          label_value: db_large
       - name: tm-db
         enabled: true
         schedule: '0 1 * * *'
@@ -350,7 +350,7 @@ osm-seed:
         nodeSelector:
           enabled: true
           label_key: nodegroup_type
-          label_value: web_large
+          label_value: db_large
       - name: osmcha-db
         enabled: false
         schedule: '0 1 * * *'
@@ -509,7 +509,7 @@ osm-seed:
       nodeSelector:
         enabled: true
         label_key: nodegroup_type
-        label_value: web_large
+        label_value: db_large
     # ====================================================================================================
     # Variables for tiler-server
     # ====================================================================================================
@@ -519,7 +519,7 @@ osm-seed:
       nodeSelector:
         enabled: true
         label_key: nodegroup_type
-        label_value: web_large
+        label_value: db_large
       replicaCount: 1
       commad: './start.sh'
       serviceAnnotations:
@@ -968,7 +968,7 @@ ohm:
       NAMESPACE: default # Namespace to run the job
       DOCKER_IMAGE: ghcr.io/openhistoricalmap/tiler-server:0.0.1-0.dev.git.1859.h75499f5 # TODO, this should be automatically updated from tiler server image
       SQS_QUEUE_URL: {{STAGING_SQS_QUEUE_URL}}
-      NODEGROUP_TYPE: db_large # Nodegroup type to run the job
+      NODEGROUP_TYPE: web_large # Nodegroup type to run the job
       MAX_ACTIVE_JOBS: 2 # Maximum number of active jobs in high concurrency queue
       DELETE_OLD_JOBS_AGE: 3600 # Age in seconds to delete old jobs
       EXECUTE_PURGE: 'true'
@@ -993,7 +993,7 @@ ohm:
     nodeSelector:
       enabled: true
       label_key: nodegroup_type
-      label_value: db_large
+      label_value: web_large
 
   # Tiler seed by default is giong to seet tiles from 0-5 zoom level
   tilerCacheSeed:
@@ -1016,4 +1016,4 @@ ohm:
     nodeSelector:
       enabled: true
       label_key: nodegroup_type
-      label_value: db_large
+      label_value: web_large