diff --git a/images/tiler-cache/purge.py b/images/tiler-cache/purge.py index f2a0e967..764f7927 100644 --- a/images/tiler-cache/purge.py +++ b/images/tiler-cache/purge.py @@ -84,6 +84,66 @@ def get_active_jobs_count(): logging.info(f"Total active or pending jobs: {active_jobs_count}") return active_jobs_count +def get_purge_and_seed_commands(): + return """ + set -x + + # Default zoom levels + PURGE_MIN_ZOOM=${PURGE_MIN_ZOOM:-8} + PURGE_MAX_ZOOM=${PURGE_MAX_ZOOM:-20} + PURGE_CONCURRENCY=${PURGE_CONCURRENCY:-16} + + SEED_MIN_ZOOM=${SEED_MIN_ZOOM:-8} + SEED_MAX_ZOOM=${SEED_MAX_ZOOM:-14} + SEED_CONCURRENCY=${SEED_CONCURRENCY:-16} + + EXECUTE_PURGE=${EXECUTE_PURGE:-true} + EXECUTE_SEED=${EXECUTE_SEED:-true} + + # Download file from S3 + file_name=$(basename "$IMPOSM_EXPIRED_FILE") + aws s3 cp "$IMPOSM_EXPIRED_FILE" "$file_name" + if [ $? -ne 0 ]; then + echo "Error: Failed to download the file from S3." + exit 1 + fi + + # Run Tegola cache purge if enabled + if [ "$EXECUTE_PURGE" = "true" ]; then + echo "Starting Tegola cache purge..." + set -x + tegola cache purge tile-list "$file_name" \ + --config=/opt/tegola_config/config.toml \ + --format="/zxy" \ + --min-zoom=$PURGE_MIN_ZOOM \ + --max-zoom=$PURGE_MAX_ZOOM \ + --map=osm \ + --overwrite=false \ + --concurrency=$PURGE_CONCURRENCY + set +x + else + echo "Skipping Tegola cache purge (EXECUTE_PURGE=false)." + fi + + # Run Tegola cache seed if enabled + if [ "$EXECUTE_SEED" = "true" ]; then + echo "Starting Tegola cache seed..." + set -x + tegola cache seed tile-list "$file_name" \ + --config=/opt/tegola_config/config.toml \ + --map=osm \ + --min-zoom=$SEED_MIN_ZOOM \ + --max-zoom=$SEED_MAX_ZOOM \ + --overwrite=true \ + --concurrency=$SEED_CONCURRENCY + set +x + else + echo "Skipping Tegola cache seed (EXECUTE_SEED=false)." + fi + + echo "Script completed successfully." + """ + def create_kubernetes_job(file_url, file_name): """Create a Kubernetes Job to process a file.""" @@ -91,6 +151,8 @@ def create_kubernetes_job(file_url, file_name): configmap_tiler_db = f"{ENVIRONMENT}-tiler-db-cm" job_name = f"{JOB_NAME_PREFIX}-{file_name}" + shell_commands = get_purge_and_seed_commands() + job_manifest = { "apiVersion": "batch/v1", "kind": "Job", @@ -104,7 +166,7 @@ def create_kubernetes_job(file_url, file_name): { "name": "tiler-purge-seed", "image": DOCKER_IMAGE, - "command": ["sh", "./purge_and_seed.sh"], + "command": ["bash", "-c", shell_commands], "envFrom": [{"configMapRef": {"name": configmap_tiler_server}},{"configMapRef": {"name": configmap_tiler_db}}], "env": [ {"name": "IMPOSM_EXPIRED_FILE", "value": file_url}, @@ -122,7 +184,7 @@ def create_kubernetes_job(file_url, file_name): "restartPolicy": "Never", } }, - "backoffLimit": 3, + "backoffLimit": 4, }, } diff --git a/images/tiler-cache/utils.py b/images/tiler-cache/utils.py index d6ece8f4..4cd324b0 100644 --- a/images/tiler-cache/utils.py +++ b/images/tiler-cache/utils.py @@ -201,3 +201,4 @@ def upload_to_s3(local_file, s3_bucket, s3_key): logging.info(f"Uploaded {local_file} to {s3_url}.") except Exception as e: logging.error(f"Error uploading to S3: {e}") + diff --git a/images/tiler-server/purge_and_seed.sh b/images/tiler-server/purge_and_seed.sh index 14840ae8..830fa375 100755 --- a/images/tiler-server/purge_and_seed.sh +++ b/images/tiler-server/purge_and_seed.sh @@ -47,7 +47,7 @@ if [ "$EXECUTE_SEED" = "true" ]; then --map=osm \ --min-zoom=$SEED_MIN_ZOOM \ --max-zoom=$SEED_MAX_ZOOM \ - --overwrite=false \ + --overwrite=true \ --concurrency=$SEED_CONCURRENCY set +x else diff --git a/values.production.template.yaml b/values.production.template.yaml index acee02b5..4b0ca4c3 100644 --- a/values.production.template.yaml +++ b/values.production.template.yaml @@ -943,7 +943,7 @@ ohm: ## Execute seeding EXECUTE_SEED: true SEED_CONCURRENCY: 64 - SEED_MIN_ZOOM: 7 + SEED_MIN_ZOOM: 3 SEED_MAX_ZOOM: 14 ## Remove tiles from s3 for zoom levels ZOOM_LEVELS_TO_DELETE: 18,19,20 @@ -965,11 +965,11 @@ ohm: # Tiler seed by default is giong to seet tiles from 0-6 zoom level tilerCacheSeed: enabled: true - schedule: '0 */6 * * *' + schedule: '0 */3 * * *' env: GEOJSON_URL: https://osmseed-dev.s3.us-east-1.amazonaws.com/tiler/usa-eu-lat-afr.geojson ZOOM_LEVELS: '7,8,9,10' - CONCURRENCY: 128 + CONCURRENCY: 200 S3_BUCKET: osmseed-dev OUTPUT_FILE: /logs/tiler_benchmark.log resources: diff --git a/values.staging.template.yaml b/values.staging.template.yaml index 5df6bee7..8b99001e 100644 --- a/values.staging.template.yaml +++ b/values.staging.template.yaml @@ -962,7 +962,7 @@ ohm: enabled: true tilerCachePurge: - enabled: false + enabled: true env: REGION_NAME: us-east-1 NAMESPACE: default # Namespace to run the job