From 4d358c9f89d5466f29608ceeb8b0e77e2b3ebad8 Mon Sep 17 00:00:00 2001 From: Rub21 Date: Tue, 26 Nov 2024 11:30:06 -0500 Subject: [PATCH] Config to enable and disable seed tiler option --- images/tiler-cache/purge.py | 12 ++++++++++++ ohm/templates/tiler-cache-purge/deployment.yaml | 11 +++++++++++ values.production.template.yaml | 2 +- values.staging.template.yaml | 2 +- 4 files changed, 25 insertions(+), 2 deletions(-) diff --git a/images/tiler-cache/purge.py b/images/tiler-cache/purge.py index 6e695639..f79526a0 100644 --- a/images/tiler-cache/purge.py +++ b/images/tiler-cache/purge.py @@ -24,10 +24,18 @@ NODEGROUP_TYPE = os.getenv("NODEGROUP_TYPE", "job_large") MAX_ACTIVE_JOBS = int(os.getenv("MAX_ACTIVE_JOBS", 2)) DELETE_OLD_JOBS_AGE = int(os.getenv("DELETE_OLD_JOBS_AGE", 86400)) # default 1 day + +# Tiler cache purge and seed settings +EXECUTE_PURGE = os.getenv("EXECUTE_PURGE", "true") +EXECUTE_SEED = os.getenv("EXECUTE_SEED", "true") +# zoom PURGE_MIN_ZOOM = os.getenv("PURGE_MIN_ZOOM", 8) PURGE_MAX_ZOOM = os.getenv("PURGE_MAX_ZOOM", 20) SEED_MIN_ZOOM = os.getenv("SEED_MIN_ZOOM", 8) SEED_MAX_ZOOM = os.getenv("SEED_MAX_ZOOM", 14) +## concurrency +SEED_CONCURRENCY = os.getenv("SEED_CONCURRENCY", 16) +PURGE_CONCURRENCY = os.getenv("PURGE_CONCURRENCY", 16) JOB_NAME_PREFIX = f"{ENVIRONMENT}-tiler-cache-purge-seed" POSTGRES_HOST = os.getenv("POSTGRES_HOST", "localhost") @@ -93,10 +101,14 @@ def create_kubernetes_job(file_url, file_name): "envFrom": [{"configMapRef": {"name": config_map_name}}], "env": [ {"name": "IMPOSM_EXPIRED_FILE", "value": file_url}, + {"name": "EXECUTE_PURGE", "value": str(EXECUTE_PURGE)}, + {"name": "EXECUTE_SEED", "value": str(EXECUTE_SEED)}, {"name": "PURGE_MIN_ZOOM", "value": str(PURGE_MIN_ZOOM)}, {"name": "PURGE_MAX_ZOOM", "value": str(PURGE_MAX_ZOOM)}, {"name": "SEED_MIN_ZOOM", "value": str(SEED_MIN_ZOOM)}, {"name": "SEED_MAX_ZOOM", "value": str(SEED_MAX_ZOOM)}, + {"name": "SEED_CONCURRENCY", "value": str(SEED_CONCURRENCY)}, + {"name": "PURGE_CONCURRENCY", "value": str(PURGE_CONCURRENCY)}, ], } ], diff --git a/ohm/templates/tiler-cache-purge/deployment.yaml b/ohm/templates/tiler-cache-purge/deployment.yaml index 2f114440..24fb2289 100644 --- a/ohm/templates/tiler-cache-purge/deployment.yaml +++ b/ohm/templates/tiler-cache-purge/deployment.yaml @@ -46,6 +46,12 @@ spec: value: {{ .Values.ohm.tilerCachePurge.env.MAX_ACTIVE_JOBS | quote }} - name: DELETE_OLD_JOBS_AGE value: {{ .Values.ohm.tilerCachePurge.env.DELETE_OLD_JOBS_AGE | quote }} + # execution parameters + - name: EXECUTE_PURGE + value: {{ .Values.ohm.tilerCachePurge.env.EXECUTE_PURGE | quote }} + - name: EXECUTE_SEED + value: {{ .Values.ohm.tilerCachePurge.env.EXECUTE_SEED | quote }} + # zoom - name: PURGE_MIN_ZOOM value: {{ .Values.ohm.tilerCachePurge.env.PURGE_MIN_ZOOM | quote }} - name: PURGE_MAX_ZOOM @@ -54,6 +60,11 @@ spec: value: {{ .Values.ohm.tilerCachePurge.env.SEED_MIN_ZOOM | quote }} - name: SEED_MAX_ZOOM value: {{ .Values.ohm.tilerCachePurge.env.SEED_MAX_ZOOM | quote }} + # concurrency + - name: SEED_CONCURRENCY + value: {{ .Values.ohm.tilerCachePurge.env.SEED_CONCURRENCY | quote }} + - name: PURGE_CONCURRENCY + value: {{ .Values.ohm.tilerCachePurge.env.PURGE_CONCURRENCY | quote }} envFrom: - configMapRef: name: {{ .Release.Name }}-tiler-server-cm diff --git a/values.production.template.yaml b/values.production.template.yaml index 190d3e0c..4b758e8e 100644 --- a/values.production.template.yaml +++ b/values.production.template.yaml @@ -925,7 +925,7 @@ ohm: env: REGION_NAME: us-east-1 NAMESPACE: default # Namespace to run the job - DOCKER_IMAGE: ghcr.io/openhistoricalmap/tiler-server:0.0.1-0.dev.git.1780.h62561a8 # TODO, this should be automatically updated from tiler server image + DOCKER_IMAGE: ghcr.io/openhistoricalmap/tiler-server:0.0.1-0.dev.git.1791.h27fbedb # TODO, this should be automatically updated from tiler server image SQS_QUEUE_URL: {{PRODUCTION_SQS_QUEUE_URL}} NODEGROUP_TYPE: web_large # Nodegroup type to run the job MAX_ACTIVE_JOBS: 10 # Maximum number of active jobs in high concurrency queue diff --git a/values.staging.template.yaml b/values.staging.template.yaml index 0cd70f1c..fd672593 100644 --- a/values.staging.template.yaml +++ b/values.staging.template.yaml @@ -961,7 +961,7 @@ ohm: env: REGION_NAME: us-east-1 NAMESPACE: default # Namespace to run the job - DOCKER_IMAGE: ghcr.io/openhistoricalmap/tiler-server:0.0.1-0.dev.git.1780.h62561a8 # TODO, this should be automatically updated from tiler server image + DOCKER_IMAGE: ghcr.io/openhistoricalmap/tiler-server:0.0.1-0.dev.git.1791.h27fbedb # TODO, this should be automatically updated from tiler server image SQS_QUEUE_URL: {{STAGING_SQS_QUEUE_URL}} NODEGROUP_TYPE: job_large # Nodegroup type to run the job MAX_ACTIVE_JOBS: 2 # Maximum number of active jobs in high concurrency queue