Merge pull request #430 from OpenHistoricalMap/staging

Update purge script and configuration

images/tiler-cache/purge.py

@@ -84,13 +84,75 @@ def get_active_jobs_count():
     logging.info(f"Total active or pending jobs: {active_jobs_count}")
     return active_jobs_count
 
+def get_purge_and_seed_commands():
+    return """
+    set -x
+
+    # Default zoom levels
+    PURGE_MIN_ZOOM=${PURGE_MIN_ZOOM:-8}
+    PURGE_MAX_ZOOM=${PURGE_MAX_ZOOM:-20}
+    PURGE_CONCURRENCY=${PURGE_CONCURRENCY:-16}
+
+    SEED_MIN_ZOOM=${SEED_MIN_ZOOM:-8}
+    SEED_MAX_ZOOM=${SEED_MAX_ZOOM:-14}
+    SEED_CONCURRENCY=${SEED_CONCURRENCY:-16}
+
+    EXECUTE_PURGE=${EXECUTE_PURGE:-true}
+    EXECUTE_SEED=${EXECUTE_SEED:-true}
+
+    # Download file from S3
+    file_name=$(basename "$IMPOSM_EXPIRED_FILE")
+    aws s3 cp "$IMPOSM_EXPIRED_FILE" "$file_name"
+    if [ $? -ne 0 ]; then
+        echo "Error: Failed to download the file from S3."
+        exit 1
+    fi
+
+    # Run Tegola cache purge if enabled
+    if [ "$EXECUTE_PURGE" = "true" ]; then
+        echo "Starting Tegola cache purge..."
+        set -x
+        tegola cache purge tile-list "$file_name" \
+            --config=/opt/tegola_config/config.toml \
+            --format="/zxy" \
+            --min-zoom=$PURGE_MIN_ZOOM \
+            --max-zoom=$PURGE_MAX_ZOOM \
+            --map=osm \
+            --overwrite=false \
+            --concurrency=$PURGE_CONCURRENCY
+        set +x
+    else
+        echo "Skipping Tegola cache purge (EXECUTE_PURGE=false)."
+    fi
+
+    # Run Tegola cache seed if enabled
+    if [ "$EXECUTE_SEED" = "true" ]; then
+        echo "Starting Tegola cache seed..."
+        set -x
+        tegola cache seed tile-list "$file_name" \
+            --config=/opt/tegola_config/config.toml \
+            --map=osm \
+            --min-zoom=$SEED_MIN_ZOOM \
+            --max-zoom=$SEED_MAX_ZOOM \
+            --overwrite=true \
+            --concurrency=$SEED_CONCURRENCY
+        set +x
+    else
+        echo "Skipping Tegola cache seed (EXECUTE_SEED=false)."
+    fi
+
+    echo "Script completed successfully."
+    """
+
 
 def create_kubernetes_job(file_url, file_name):
     """Create a Kubernetes Job to process a file."""
     configmap_tiler_server = f"{ENVIRONMENT}-tiler-server-cm"
     configmap_tiler_db = f"{ENVIRONMENT}-tiler-db-cm"
 
     job_name = f"{JOB_NAME_PREFIX}-{file_name}"
+    shell_commands = get_purge_and_seed_commands()
+
     job_manifest = {
         "apiVersion": "batch/v1",
         "kind": "Job",
@@ -104,7 +166,7 @@ def create_kubernetes_job(file_url, file_name):
                         {
                             "name": "tiler-purge-seed",
                             "image": DOCKER_IMAGE,
-                            "command": ["sh", "./purge_and_seed.sh"],
+                            "command": ["bash", "-c", shell_commands],
                             "envFrom": [{"configMapRef": {"name": configmap_tiler_server}},{"configMapRef": {"name": configmap_tiler_db}}],
                             "env": [
                                 {"name": "IMPOSM_EXPIRED_FILE", "value": file_url},
@@ -122,7 +184,7 @@ def create_kubernetes_job(file_url, file_name):
                     "restartPolicy": "Never",
                 }
             },
-            "backoffLimit": 3,
+            "backoffLimit": 4,
         },
     }
 

images/tiler-cache/utils.py

@@ -201,3 +201,4 @@ def upload_to_s3(local_file, s3_bucket, s3_key):
         logging.info(f"Uploaded {local_file} to {s3_url}.")
     except Exception as e:
         logging.error(f"Error uploading to S3: {e}")
+

images/tiler-server/purge_and_seed.sh

@@ -47,7 +47,7 @@ if [ "$EXECUTE_SEED" = "true" ]; then
         --map=osm \
         --min-zoom=$SEED_MIN_ZOOM \
         --max-zoom=$SEED_MAX_ZOOM \
-        --overwrite=false \
+        --overwrite=true \
         --concurrency=$SEED_CONCURRENCY
     set +x
 else

values.production.template.yaml

@@ -943,7 +943,7 @@ ohm:
       ## Execute seeding
       EXECUTE_SEED: true
       SEED_CONCURRENCY: 64
-      SEED_MIN_ZOOM: 7
+      SEED_MIN_ZOOM: 3
       SEED_MAX_ZOOM: 14
       ## Remove tiles from s3 for zoom levels
       ZOOM_LEVELS_TO_DELETE: 18,19,20
@@ -965,11 +965,11 @@ ohm:
   # Tiler seed by default is giong to seet tiles from 0-6 zoom level
   tilerCacheSeed:
     enabled: true
-    schedule: '0 */6 * * *'
+    schedule: '0 */3 * * *'
     env:
       GEOJSON_URL: https://osmseed-dev.s3.us-east-1.amazonaws.com/tiler/usa-eu-lat-afr.geojson
       ZOOM_LEVELS: '7,8,9,10'
-      CONCURRENCY: 128
+      CONCURRENCY: 200
       S3_BUCKET: osmseed-dev
       OUTPUT_FILE: /logs/tiler_benchmark.log
     resources:

values.staging.template.yaml

@@ -962,7 +962,7 @@ ohm:
     enabled: true
 
   tilerCachePurge:
-    enabled: false
+    enabled: true
     env:
       REGION_NAME: us-east-1
       NAMESPACE: default # Namespace to run the job