Merge pull request #405 from OpenHistoricalMap/staging

Set zoom levels for Purge and Seed
OpenHistoricalMap · Nov 26, 2024 · b4c3ef2 · b4c3ef2
2 parents 2639c12 + 8d65cd4
commit b4c3ef2
Show file tree

Hide file tree

Showing 6 changed files with 42 additions and 27 deletions.
diff --git a/images/tiler-cache/purge.py b/images/tiler-cache/purge.py
@@ -19,13 +19,16 @@
 REGION_NAME = os.getenv("REGION_NAME", "us-east-1")
 DOCKER_IMAGE = os.getenv(
     "DOCKER_IMAGE",
-    "ghcr.io/openhistoricalmap/tiler-server:0.0.1-0.dev.git.1734.h5b4d15d",
+    "ghcr.io/openhistoricalmap/tiler-server:0.0.1-0.dev.git.1780.h62561a8",
 )
 NODEGROUP_TYPE = os.getenv("NODEGROUP_TYPE", "job_large")
 MAX_ACTIVE_JOBS = int(os.getenv("MAX_ACTIVE_JOBS", 2))
 DELETE_OLD_JOBS_AGE = int(os.getenv("DELETE_OLD_JOBS_AGE", 86400))  # default 1 day
-MIN_ZOOM = os.getenv("MIN_ZOOM", 8)
-MAX_ZOOM = os.getenv("MAX_ZOOM", 16)
+PURGE_MIN_ZOOM = os.getenv("PURGE_MIN_ZOOM", 8)
+PURGE_MAX_ZOOM = os.getenv("PURGE_MAX_ZOOM", 20)
+SEED_MIN_ZOOM = os.getenv("SEED_MIN_ZOOM", 8)
+SEED_MAX_ZOOM = os.getenv("SEED_MAX_ZOOM", 14)
+
 JOB_NAME_PREFIX = f"{ENVIRONMENT}-tiler-cache-purge-seed"
 POSTGRES_HOST = os.getenv("POSTGRES_HOST", "localhost")
 POSTGRES_PORT = int(os.getenv("POSTGRES_PORT", 5432))
@@ -90,8 +93,10 @@ def create_kubernetes_job(file_url, file_name):
                             "envFrom": [{"configMapRef": {"name": config_map_name}}],
                             "env": [
                                 {"name": "IMPOSM_EXPIRED_FILE", "value": file_url},
-                                {"name": "MIN_ZOOM", "value": str(MIN_ZOOM)},
-                                {"name": "MAX_ZOOM", "value": str(MAX_ZOOM)},
+                                {"name": "PURGE_MIN_ZOOM", "value": str(PURGE_MIN_ZOOM)},
+                                {"name": "PURGE_MAX_ZOOM", "value": str(PURGE_MAX_ZOOM)},
+                                {"name": "SEED_MIN_ZOOM", "value": str(SEED_MIN_ZOOM)},
+                                {"name": "SEED_MAX_ZOOM", "value": str(SEED_MAX_ZOOM)},
                             ],
                         }
                     ],

diff --git a/images/tiler-cache/seed.py b/images/tiler-cache/seed.py
@@ -25,7 +25,7 @@
 @click.option(
     "--zoom-levels",
     help="Comma-separated list of zoom levels",
-    default="6,7,8,9",
+    default="6,7",
 )
 @click.option(
     "--concurrency",

diff --git a/images/tiler-server/purge_and_seed.sh b/images/tiler-server/purge_and_seed.sh
@@ -1,8 +1,10 @@
 #!/bin/bash
 set -e
 
-MIN_ZOOM=${MIN_ZOOM:-9}
-MAX_ZOOM=${MAX_ZOOM:-16}
+PURGE_MIN_ZOOM=${PURGE_MIN_ZOOM:-8}
+PURGE_MAX_ZOOM=${PURGE_MAX_ZOOM:-20}
+SEED_MIN_ZOOM=${SEED_MIN_ZOOM:-8}
+SEED_MAX_ZOOM=${SEED_MAX_ZOOM:-14}
 
 file_name=$(basename "$IMPOSM_EXPIRED_FILE")
 aws s3 cp "$IMPOSM_EXPIRED_FILE" "$file_name"
@@ -16,8 +18,8 @@ set -x
 tegola cache purge tile-list "$file_name" \
     --config=/opt/tegola_config/config.toml \
     --format="/zxy" \
-    --min-zoom=$MIN_ZOOM \
-    --max-zoom=$MAX_ZOOM \
+    --min-zoom=$PURGE_MIN_ZOOM \
+    --max-zoom=$PURGE_MAX_ZOOM \
     --map=osm \
     --overwrite=false \
     --concurrency=16
@@ -26,10 +28,10 @@ tegola cache purge tile-list "$file_name" \
 tegola cache seed tile-list "$file_name" \
     --config=/opt/tegola_config/config.toml \
     --map=osm \
-    --min-zoom=$MIN_ZOOM \
-    --max-zoom=$MAX_ZOOM \
+    --min-zoom=$SEED_MIN_ZOOM \
+    --max-zoom=$SEED_MAX_ZOOM \
     --overwrite=true \
     --concurrency=16
 set +x
 
-echo "Script completed successfully."
+echo "Script completed successfully."
diff --git a/ohm/templates/tiler-cache-purge/deployment.yaml b/ohm/templates/tiler-cache-purge/deployment.yaml
@@ -46,10 +46,14 @@ spec:
               value: {{ .Values.ohm.tilerCachePurge.env.MAX_ACTIVE_JOBS | quote }}
             - name: DELETE_OLD_JOBS_AGE
               value: {{ .Values.ohm.tilerCachePurge.env.DELETE_OLD_JOBS_AGE | quote }}
-            - name: MIN_ZOOM
-              value: {{ .Values.ohm.tilerCachePurge.env.MIN_ZOOM | quote }}
-            - name: MAX_ZOOM
-              value: {{ .Values.ohm.tilerCachePurge.env.MAX_ZOOM | quote }}
+            - name: PURGE_MIN_ZOOM
+              value: {{ .Values.ohm.tilerCachePurge.env.PURGE_MIN_ZOOM | quote }}
+            - name: PURGE_MAX_ZOOM
+              value: {{ .Values.ohm.tilerCachePurge.env.PURGE_MAX_ZOOM | quote }}
+            - name: SEED_MIN_ZOOM
+              value: {{ .Values.ohm.tilerCachePurge.env.SEED_MIN_ZOOM | quote }}
+            - name: SEED_MAX_ZOOM
+              value: {{ .Values.ohm.tilerCachePurge.env.SEED_MAX_ZOOM | quote }}
           envFrom:
             - configMapRef:
                 name: {{ .Release.Name }}-tiler-server-cm

diff --git a/values.production.template.yaml b/values.production.template.yaml
@@ -925,13 +925,15 @@ ohm:
     env:
       REGION_NAME: us-east-1
       NAMESPACE: default # Namespace to run the job
-      DOCKER_IMAGE: ghcr.io/openhistoricalmap/tiler-server:0.0.1-0.dev.git.1741.h7dadda7 # TODO, this should be automatically updated from tiler server image
+      DOCKER_IMAGE: ghcr.io/openhistoricalmap/tiler-server:0.0.1-0.dev.git.1780.h62561a8 # TODO, this should be automatically updated from tiler server image
       SQS_QUEUE_URL: {{PRODUCTION_SQS_QUEUE_URL}}
       NODEGROUP_TYPE: web_large # Nodegroup type to run the job
       MAX_ACTIVE_JOBS: 5 # Maximum number of active jobs in high concurrency queue
-      DELETE_OLD_JOBS_AGE: 14400 # 4 hours
-      MIN_ZOOM: 8 # Minimum zoom level to clean and seed cache
-      MAX_ZOOM: 16 # Maximum zoom level to clean and seed cache
+      DELETE_OLD_JOBS_AGE: 3600 # 1 hours
+      PURGE_MIN_ZOOM: 8
+      PURGE_MAX_ZOOM: 20
+      SEED_MIN_ZOOM: 8
+      SEED_MAX_ZOOM: 14
     resources:
       enabled: false
       requests:
@@ -951,7 +953,7 @@ ohm:
     schedule: '0 */24 * * *' # Every 24 hours
     env:
       GEOJSON_URL: https://osmseed-dev.s3.us-east-1.amazonaws.com/tiler/wold-usa-eu.geojson
-      ZOOM_LEVELS: '7,8,9'
+      ZOOM_LEVELS: '7,8'
       CONCURRENCY: 32
       S3_BUCKET: osmseed-dev
       OUTPUT_FILE: /logs/tiler_benchmark.log

diff --git a/values.staging.template.yaml b/values.staging.template.yaml
@@ -961,13 +961,15 @@ ohm:
     env:
       REGION_NAME: us-east-1
       NAMESPACE: default # Namespace to run the job
-      DOCKER_IMAGE: ghcr.io/openhistoricalmap/tiler-server:0.0.1-0.dev.git.1741.h7dadda7 # TODO, this should be automatically updated from tiler server image
+      DOCKER_IMAGE: ghcr.io/openhistoricalmap/tiler-server:0.0.1-0.dev.git.1780.h62561a8 # TODO, this should be automatically updated from tiler server image
       SQS_QUEUE_URL: {{STAGING_SQS_QUEUE_URL}}
       NODEGROUP_TYPE: job_large # Nodegroup type to run the job
       MAX_ACTIVE_JOBS: 2 # Maximum number of active jobs in high concurrency queue
-      DELETE_OLD_JOBS_AGE: 600 # Age in seconds to delete old jobs
-      MIN_ZOOM: 8 # Minimum zoom level to clean and seed cache
-      MAX_ZOOM: 15 # Maximum zoom level to clean and seed cache
+      DELETE_OLD_JOBS_AGE: 3600 # Age in seconds to delete old jobs
+      PURGE_MIN_ZOOM: 8
+      PURGE_MAX_ZOOM: 20
+      SEED_MIN_ZOOM: 8
+      SEED_MAX_ZOOM: 14
     resources:
       enabled: false
       requests:
@@ -987,7 +989,7 @@ ohm:
     schedule: '0 * * * *' # testing every hour
     env:
       GEOJSON_URL: https://osmseed-dev.s3.us-east-1.amazonaws.com/tiler/wold-usa-eu.geojson
-      ZOOM_LEVELS: '7,8,9'
+      ZOOM_LEVELS: '7,8'
       CONCURRENCY: 32
       S3_BUCKET: osmseed-dev
       OUTPUT_FILE: /logs/tiler_benchmark.log