diff --git a/images/tiler-cache/purge.py b/images/tiler-cache/purge.py index a1f65d2d..6e695639 100644 --- a/images/tiler-cache/purge.py +++ b/images/tiler-cache/purge.py @@ -19,13 +19,16 @@ REGION_NAME = os.getenv("REGION_NAME", "us-east-1") DOCKER_IMAGE = os.getenv( "DOCKER_IMAGE", - "ghcr.io/openhistoricalmap/tiler-server:0.0.1-0.dev.git.1734.h5b4d15d", + "ghcr.io/openhistoricalmap/tiler-server:0.0.1-0.dev.git.1780.h62561a8", ) NODEGROUP_TYPE = os.getenv("NODEGROUP_TYPE", "job_large") MAX_ACTIVE_JOBS = int(os.getenv("MAX_ACTIVE_JOBS", 2)) DELETE_OLD_JOBS_AGE = int(os.getenv("DELETE_OLD_JOBS_AGE", 86400)) # default 1 day -MIN_ZOOM = os.getenv("MIN_ZOOM", 8) -MAX_ZOOM = os.getenv("MAX_ZOOM", 16) +PURGE_MIN_ZOOM = os.getenv("PURGE_MIN_ZOOM", 8) +PURGE_MAX_ZOOM = os.getenv("PURGE_MAX_ZOOM", 20) +SEED_MIN_ZOOM = os.getenv("SEED_MIN_ZOOM", 8) +SEED_MAX_ZOOM = os.getenv("SEED_MAX_ZOOM", 14) + JOB_NAME_PREFIX = f"{ENVIRONMENT}-tiler-cache-purge-seed" POSTGRES_HOST = os.getenv("POSTGRES_HOST", "localhost") POSTGRES_PORT = int(os.getenv("POSTGRES_PORT", 5432)) @@ -90,8 +93,10 @@ def create_kubernetes_job(file_url, file_name): "envFrom": [{"configMapRef": {"name": config_map_name}}], "env": [ {"name": "IMPOSM_EXPIRED_FILE", "value": file_url}, - {"name": "MIN_ZOOM", "value": str(MIN_ZOOM)}, - {"name": "MAX_ZOOM", "value": str(MAX_ZOOM)}, + {"name": "PURGE_MIN_ZOOM", "value": str(PURGE_MIN_ZOOM)}, + {"name": "PURGE_MAX_ZOOM", "value": str(PURGE_MAX_ZOOM)}, + {"name": "SEED_MIN_ZOOM", "value": str(SEED_MIN_ZOOM)}, + {"name": "SEED_MAX_ZOOM", "value": str(SEED_MAX_ZOOM)}, ], } ], diff --git a/images/tiler-cache/seed.py b/images/tiler-cache/seed.py index 82643073..01107520 100644 --- a/images/tiler-cache/seed.py +++ b/images/tiler-cache/seed.py @@ -25,7 +25,7 @@ @click.option( "--zoom-levels", help="Comma-separated list of zoom levels", - default="6,7,8,9", + default="6,7", ) @click.option( "--concurrency", diff --git a/images/tiler-server/purge_and_seed.sh b/images/tiler-server/purge_and_seed.sh index fac3fba0..1dcd4b70 100755 --- a/images/tiler-server/purge_and_seed.sh +++ b/images/tiler-server/purge_and_seed.sh @@ -1,8 +1,10 @@ #!/bin/bash set -e -MIN_ZOOM=${MIN_ZOOM:-9} -MAX_ZOOM=${MAX_ZOOM:-16} +PURGE_MIN_ZOOM=${PURGE_MIN_ZOOM:-8} +PURGE_MAX_ZOOM=${PURGE_MAX_ZOOM:-20} +SEED_MIN_ZOOM=${SEED_MIN_ZOOM:-8} +SEED_MAX_ZOOM=${SEED_MAX_ZOOM:-14} file_name=$(basename "$IMPOSM_EXPIRED_FILE") aws s3 cp "$IMPOSM_EXPIRED_FILE" "$file_name" @@ -16,8 +18,8 @@ set -x tegola cache purge tile-list "$file_name" \ --config=/opt/tegola_config/config.toml \ --format="/zxy" \ - --min-zoom=$MIN_ZOOM \ - --max-zoom=$MAX_ZOOM \ + --min-zoom=$PURGE_MIN_ZOOM \ + --max-zoom=$PURGE_MAX_ZOOM \ --map=osm \ --overwrite=false \ --concurrency=16 @@ -26,10 +28,10 @@ tegola cache purge tile-list "$file_name" \ tegola cache seed tile-list "$file_name" \ --config=/opt/tegola_config/config.toml \ --map=osm \ - --min-zoom=$MIN_ZOOM \ - --max-zoom=$MAX_ZOOM \ + --min-zoom=$SEED_MIN_ZOOM \ + --max-zoom=$SEED_MAX_ZOOM \ --overwrite=true \ --concurrency=16 set +x -echo "Script completed successfully." \ No newline at end of file +echo "Script completed successfully." diff --git a/ohm/templates/tiler-cache-purge/deployment.yaml b/ohm/templates/tiler-cache-purge/deployment.yaml index 8ecf6f2b..2f114440 100644 --- a/ohm/templates/tiler-cache-purge/deployment.yaml +++ b/ohm/templates/tiler-cache-purge/deployment.yaml @@ -46,10 +46,14 @@ spec: value: {{ .Values.ohm.tilerCachePurge.env.MAX_ACTIVE_JOBS | quote }} - name: DELETE_OLD_JOBS_AGE value: {{ .Values.ohm.tilerCachePurge.env.DELETE_OLD_JOBS_AGE | quote }} - - name: MIN_ZOOM - value: {{ .Values.ohm.tilerCachePurge.env.MIN_ZOOM | quote }} - - name: MAX_ZOOM - value: {{ .Values.ohm.tilerCachePurge.env.MAX_ZOOM | quote }} + - name: PURGE_MIN_ZOOM + value: {{ .Values.ohm.tilerCachePurge.env.PURGE_MIN_ZOOM | quote }} + - name: PURGE_MAX_ZOOM + value: {{ .Values.ohm.tilerCachePurge.env.PURGE_MAX_ZOOM | quote }} + - name: SEED_MIN_ZOOM + value: {{ .Values.ohm.tilerCachePurge.env.SEED_MIN_ZOOM | quote }} + - name: SEED_MAX_ZOOM + value: {{ .Values.ohm.tilerCachePurge.env.SEED_MAX_ZOOM | quote }} envFrom: - configMapRef: name: {{ .Release.Name }}-tiler-server-cm diff --git a/values.production.template.yaml b/values.production.template.yaml index 0e3b4f80..4b9a5b8a 100644 --- a/values.production.template.yaml +++ b/values.production.template.yaml @@ -925,13 +925,15 @@ ohm: env: REGION_NAME: us-east-1 NAMESPACE: default # Namespace to run the job - DOCKER_IMAGE: ghcr.io/openhistoricalmap/tiler-server:0.0.1-0.dev.git.1741.h7dadda7 # TODO, this should be automatically updated from tiler server image + DOCKER_IMAGE: ghcr.io/openhistoricalmap/tiler-server:0.0.1-0.dev.git.1780.h62561a8 # TODO, this should be automatically updated from tiler server image SQS_QUEUE_URL: {{PRODUCTION_SQS_QUEUE_URL}} NODEGROUP_TYPE: web_large # Nodegroup type to run the job MAX_ACTIVE_JOBS: 5 # Maximum number of active jobs in high concurrency queue - DELETE_OLD_JOBS_AGE: 14400 # 4 hours - MIN_ZOOM: 8 # Minimum zoom level to clean and seed cache - MAX_ZOOM: 16 # Maximum zoom level to clean and seed cache + DELETE_OLD_JOBS_AGE: 3600 # 1 hours + PURGE_MIN_ZOOM: 8 + PURGE_MAX_ZOOM: 20 + SEED_MIN_ZOOM: 8 + SEED_MAX_ZOOM: 14 resources: enabled: false requests: @@ -951,7 +953,7 @@ ohm: schedule: '0 */24 * * *' # Every 24 hours env: GEOJSON_URL: https://osmseed-dev.s3.us-east-1.amazonaws.com/tiler/wold-usa-eu.geojson - ZOOM_LEVELS: '7,8,9' + ZOOM_LEVELS: '7,8' CONCURRENCY: 32 S3_BUCKET: osmseed-dev OUTPUT_FILE: /logs/tiler_benchmark.log diff --git a/values.staging.template.yaml b/values.staging.template.yaml index d65c29a2..0cd70f1c 100644 --- a/values.staging.template.yaml +++ b/values.staging.template.yaml @@ -961,13 +961,15 @@ ohm: env: REGION_NAME: us-east-1 NAMESPACE: default # Namespace to run the job - DOCKER_IMAGE: ghcr.io/openhistoricalmap/tiler-server:0.0.1-0.dev.git.1741.h7dadda7 # TODO, this should be automatically updated from tiler server image + DOCKER_IMAGE: ghcr.io/openhistoricalmap/tiler-server:0.0.1-0.dev.git.1780.h62561a8 # TODO, this should be automatically updated from tiler server image SQS_QUEUE_URL: {{STAGING_SQS_QUEUE_URL}} NODEGROUP_TYPE: job_large # Nodegroup type to run the job MAX_ACTIVE_JOBS: 2 # Maximum number of active jobs in high concurrency queue - DELETE_OLD_JOBS_AGE: 600 # Age in seconds to delete old jobs - MIN_ZOOM: 8 # Minimum zoom level to clean and seed cache - MAX_ZOOM: 15 # Maximum zoom level to clean and seed cache + DELETE_OLD_JOBS_AGE: 3600 # Age in seconds to delete old jobs + PURGE_MIN_ZOOM: 8 + PURGE_MAX_ZOOM: 20 + SEED_MIN_ZOOM: 8 + SEED_MAX_ZOOM: 14 resources: enabled: false requests: @@ -987,7 +989,7 @@ ohm: schedule: '0 * * * *' # testing every hour env: GEOJSON_URL: https://osmseed-dev.s3.us-east-1.amazonaws.com/tiler/wold-usa-eu.geojson - ZOOM_LEVELS: '7,8,9' + ZOOM_LEVELS: '7,8' CONCURRENCY: 32 S3_BUCKET: osmseed-dev OUTPUT_FILE: /logs/tiler_benchmark.log