Config to enable and disable seed tiler option

OpenHistoricalMap · Nov 26, 2024 · 4d358c9 · 4d358c9
1 parent 27fbedb
commit 4d358c9
Show file tree

Hide file tree

Showing 4 changed files with 25 additions and 2 deletions.
diff --git a/images/tiler-cache/purge.py b/images/tiler-cache/purge.py
@@ -24,10 +24,18 @@
 NODEGROUP_TYPE = os.getenv("NODEGROUP_TYPE", "job_large")
 MAX_ACTIVE_JOBS = int(os.getenv("MAX_ACTIVE_JOBS", 2))
 DELETE_OLD_JOBS_AGE = int(os.getenv("DELETE_OLD_JOBS_AGE", 86400))  # default 1 day
+
+# Tiler cache purge and seed settings
+EXECUTE_PURGE = os.getenv("EXECUTE_PURGE", "true")
+EXECUTE_SEED = os.getenv("EXECUTE_SEED", "true")
+# zoom
 PURGE_MIN_ZOOM = os.getenv("PURGE_MIN_ZOOM", 8)
 PURGE_MAX_ZOOM = os.getenv("PURGE_MAX_ZOOM", 20)
 SEED_MIN_ZOOM = os.getenv("SEED_MIN_ZOOM", 8)
 SEED_MAX_ZOOM = os.getenv("SEED_MAX_ZOOM", 14)
+## concurrency
+SEED_CONCURRENCY = os.getenv("SEED_CONCURRENCY", 16)
+PURGE_CONCURRENCY = os.getenv("PURGE_CONCURRENCY", 16)
 
 JOB_NAME_PREFIX = f"{ENVIRONMENT}-tiler-cache-purge-seed"
 POSTGRES_HOST = os.getenv("POSTGRES_HOST", "localhost")
@@ -93,10 +101,14 @@ def create_kubernetes_job(file_url, file_name):
                             "envFrom": [{"configMapRef": {"name": config_map_name}}],
                             "env": [
                                 {"name": "IMPOSM_EXPIRED_FILE", "value": file_url},
+                                {"name": "EXECUTE_PURGE", "value": str(EXECUTE_PURGE)},
+                                {"name": "EXECUTE_SEED", "value": str(EXECUTE_SEED)},
                                 {"name": "PURGE_MIN_ZOOM", "value": str(PURGE_MIN_ZOOM)},
                                 {"name": "PURGE_MAX_ZOOM", "value": str(PURGE_MAX_ZOOM)},
                                 {"name": "SEED_MIN_ZOOM", "value": str(SEED_MIN_ZOOM)},
                                 {"name": "SEED_MAX_ZOOM", "value": str(SEED_MAX_ZOOM)},
+                                {"name": "SEED_CONCURRENCY", "value": str(SEED_CONCURRENCY)},
+                                {"name": "PURGE_CONCURRENCY", "value": str(PURGE_CONCURRENCY)},
                             ],
                         }
                     ],

diff --git a/ohm/templates/tiler-cache-purge/deployment.yaml b/ohm/templates/tiler-cache-purge/deployment.yaml
@@ -46,6 +46,12 @@ spec:
               value: {{ .Values.ohm.tilerCachePurge.env.MAX_ACTIVE_JOBS | quote }}
             - name: DELETE_OLD_JOBS_AGE
               value: {{ .Values.ohm.tilerCachePurge.env.DELETE_OLD_JOBS_AGE | quote }}
+            # execution parameters
+            - name: EXECUTE_PURGE
+              value: {{ .Values.ohm.tilerCachePurge.env.EXECUTE_PURGE | quote }}
+            - name: EXECUTE_SEED
+              value: {{ .Values.ohm.tilerCachePurge.env.EXECUTE_SEED | quote }}
+            # zoom
             - name: PURGE_MIN_ZOOM
               value: {{ .Values.ohm.tilerCachePurge.env.PURGE_MIN_ZOOM | quote }}
             - name: PURGE_MAX_ZOOM
@@ -54,6 +60,11 @@ spec:
               value: {{ .Values.ohm.tilerCachePurge.env.SEED_MIN_ZOOM | quote }}
             - name: SEED_MAX_ZOOM
               value: {{ .Values.ohm.tilerCachePurge.env.SEED_MAX_ZOOM | quote }}
+            # concurrency
+            - name: SEED_CONCURRENCY
+              value: {{ .Values.ohm.tilerCachePurge.env.SEED_CONCURRENCY | quote }}
+            - name: PURGE_CONCURRENCY
+              value: {{ .Values.ohm.tilerCachePurge.env.PURGE_CONCURRENCY | quote }}
           envFrom:
             - configMapRef:
                 name: {{ .Release.Name }}-tiler-server-cm

diff --git a/values.production.template.yaml b/values.production.template.yaml
@@ -925,7 +925,7 @@ ohm:
     env:
       REGION_NAME: us-east-1
       NAMESPACE: default # Namespace to run the job
-      DOCKER_IMAGE: ghcr.io/openhistoricalmap/tiler-server:0.0.1-0.dev.git.1780.h62561a8 # TODO, this should be automatically updated from tiler server image
+      DOCKER_IMAGE: ghcr.io/openhistoricalmap/tiler-server:0.0.1-0.dev.git.1791.h27fbedb # TODO, this should be automatically updated from tiler server image
       SQS_QUEUE_URL: {{PRODUCTION_SQS_QUEUE_URL}}
       NODEGROUP_TYPE: web_large # Nodegroup type to run the job
       MAX_ACTIVE_JOBS: 10 # Maximum number of active jobs in high concurrency queue

diff --git a/values.staging.template.yaml b/values.staging.template.yaml
@@ -961,7 +961,7 @@ ohm:
     env:
       REGION_NAME: us-east-1
       NAMESPACE: default # Namespace to run the job
-      DOCKER_IMAGE: ghcr.io/openhistoricalmap/tiler-server:0.0.1-0.dev.git.1780.h62561a8 # TODO, this should be automatically updated from tiler server image
+      DOCKER_IMAGE: ghcr.io/openhistoricalmap/tiler-server:0.0.1-0.dev.git.1791.h27fbedb # TODO, this should be automatically updated from tiler server image
       SQS_QUEUE_URL: {{STAGING_SQS_QUEUE_URL}}
       NODEGROUP_TYPE: job_large # Nodegroup type to run the job
       MAX_ACTIVE_JOBS: 2 # Maximum number of active jobs in high concurrency queue