Merge pull request #324 from OpenHistoricalMap/staging

Adding TM DB and Update Map styles
OpenHistoricalMap · May 31, 2024 · df4deaf · df4deaf
2 parents ee6a8ba + fb3c5d5
commit df4deaf
Show file tree

Hide file tree

Showing 5 changed files with 103 additions and 17 deletions.
diff --git a/.github/workflows/chartpress.yaml b/.github/workflows/chartpress.yaml
@@ -58,7 +58,7 @@ jobs:
           STAGING_TILER_DB_PASSWORD: ${{ secrets.STAGING_TILER_DB_PASSWORD }}
           # STAGING_TILER_DB_USER: ${{ secrets.STAGING_TILER_DB_USER }}
           # STAGING_TILER_IMPOSM_EBS: ${{ secrets.STAGING_TILER_IMPOSM_EBS }}
-          STAGING_TILER_IMPOSM_REPLICATION_URL: ${{ secrets.STAGING_TILER_IMPOSM_REPLICATION_URL }}
+          # STAGING_TILER_IMPOSM_REPLICATION_URL: ${{ secrets.STAGING_TILER_IMPOSM_REPLICATION_URL }}
           # STAGING_TILER_SERVER_EBS: ${{ secrets.STAGING_TILER_SERVER_EBS }}
           STAGING_TILER_CACHE_AWS_ACCESS_KEY_ID: ${{ secrets.STAGING_TILER_CACHE_AWS_ACCESS_KEY_ID }}
           STAGING_TILER_CACHE_AWS_SECRET_ACCESS_KEY: ${{ secrets.STAGING_TILER_CACHE_AWS_SECRET_ACCESS_KEY }}
@@ -84,6 +84,7 @@ jobs:
           STAGING_OSMCHA_API_CONSUMER_SECRET: ${{ secrets.STAGING_OSMCHA_API_CONSUMER_SECRET }}
           STAGING_OSMCHA_DJANGO_SECRET_KEY: ${{ secrets.STAGING_OSMCHA_DJANGO_SECRET_KEY }}
           STAGING_OSMCHA_REACT_APP_MAPBOX_ACCESS_TOKEN: ${{ secrets.STAGING_OSMCHA_REACT_APP_MAPBOX_ACCESS_TOKEN }}
+          STAGING_TM_DB_PASSWORD: ${{ secrets.STAGING_TM_DB_PASSWORD }}
       - name: Production - substitute secrets
         if: github.ref == 'refs/heads/main'
         uses: bluwy/substitute-string-action@v1
@@ -107,14 +108,14 @@ jobs:
           PRODUCTION_OAUTH_KEY: ${{ secrets.PRODUCTION_OAUTH_KEY }}
           PRODUCTION_S3_BUCKET: ${{ secrets.PRODUCTION_S3_BUCKET }}
           PRODUCTION_DB_BACKUP_S3_BUCKET: ${{ secrets.PRODUCTION_DB_BACKUP_S3_BUCKET }}
-          PRODUCTION_TILER_DB: ${{ secrets.PRODUCTION_TILER_DB }}
-          PRODUCTION_TILER_DB_EBS: ${{ secrets.PRODUCTION_TILER_DB_EBS }}
-          PRODUCTION_TILER_DB_HOST: ${{ secrets.PRODUCTION_TILER_DB_HOST }}
+          # PRODUCTION_TILER_DB: ${{ secrets.PRODUCTION_TILER_DB }}
+          # PRODUCTION_TILER_DB_EBS: ${{ secrets.PRODUCTION_TILER_DB_EBS }}
+          # PRODUCTION_TILER_DB_HOST: ${{ secrets.PRODUCTION_TILER_DB_HOST }}
           PRODUCTION_TILER_DB_PASSWORD: ${{ secrets.PRODUCTION_TILER_DB_PASSWORD }}
-          PRODUCTION_TILER_DB_USER: ${{ secrets.PRODUCTION_TILER_DB_USER }}
-          PRODUCTION_TILER_IMPOSM_EBS: ${{ secrets.PRODUCTION_TILER_IMPOSM_EBS }}
-          PRODUCTION_TILER_IMPOSM_REPLICATION_URL: ${{ secrets.PRODUCTION_TILER_IMPOSM_REPLICATION_URL }}
-          PRODUCTION_TILER_SERVER_EBS: ${{ secrets.PRODUCTION_TILER_SERVER_EBS }}
+          # PRODUCTION_TILER_DB_USER: ${{ secrets.PRODUCTION_TILER_DB_USER }}
+          # PRODUCTION_TILER_IMPOSM_EBS: ${{ secrets.PRODUCTION_TILER_IMPOSM_EBS }}
+          # PRODUCTION_TILER_IMPOSM_REPLICATION_URL: ${{ secrets.PRODUCTION_TILER_IMPOSM_REPLICATION_URL }}
+          # PRODUCTION_TILER_SERVER_EBS: ${{ secrets.PRODUCTION_TILER_SERVER_EBS }}
           PRODUCTION_TILER_CACHE_AWS_ACCESS_KEY_ID: ${{ secrets.PRODUCTION_TILER_CACHE_AWS_ACCESS_KEY_ID }}
           PRODUCTION_TILER_CACHE_AWS_SECRET_ACCESS_KEY: ${{ secrets.PRODUCTION_TILER_CACHE_AWS_SECRET_ACCESS_KEY }}
           PRODUCTION_TM_API_CONSUMER_KEY: ${{ secrets.PRODUCTION_TM_API_CONSUMER_KEY }}
@@ -139,6 +140,7 @@ jobs:
           PRODUCTION_OSMCHA_API_CONSUMER_SECRET: ${{ secrets.PRODUCTION_OSMCHA_API_CONSUMER_SECRET }}
           PRODUCTION_OSMCHA_DJANGO_SECRET_KEY: ${{ secrets.PRODUCTION_OSMCHA_DJANGO_SECRET_KEY }}
           PRODUCTION_OSMCHA_REACT_APP_MAPBOX_ACCESS_TOKEN: ${{ secrets.PRODUCTION_OSMCHA_REACT_APP_MAPBOX_ACCESS_TOKEN }}
+          PRODUCTION_TM_DB_PASSWORD: ${{ secrets.PRODUCTION_TM_DB_PASSWORD }}
       - name: AWS Credentials
         if: github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/main'
         uses: aws-actions/configure-aws-credentials@v1

diff --git a/images/web/Dockerfile b/images/web/Dockerfile
@@ -115,7 +115,7 @@ RUN apache2ctl configtest
 RUN chown -R www-data: $workdir
 
 # Clone Map-styles
-ENV OPENHISTORICALMAP_MAP_STYLES_GITSHA=ad2ea8c7115e42fc73c6788f90ef82c27b23b203
+ENV OPENHISTORICALMAP_MAP_STYLES_GITSHA=ccb32341985a3c318d8476b112579231c4cfccf2
 RUN git clone https://github.com/OpenHistoricalMap/map-styles.git $workdir/map-styles
 RUN cd $workdir/map-styles && git checkout $OPENHISTORICALMAP_MAP_STYLES_GITSHA
 COPY update_map_styles.py $workdir/

diff --git a/ohm/requirements.yaml b/ohm/requirements.yaml
@@ -1,4 +1,4 @@
 dependencies:
   - name: osm-seed
-    version: '1.0.0-dev.he1070f2'
+    version: '0.1.0-n797.h987beee'
     repository: https://devseed.com/osm-seed-chart/
diff --git a/values.production.template.yaml b/values.production.template.yaml
@@ -31,7 +31,7 @@ osm-seed:
   # as the SSL certificate for your services. Else, you will need to configure
   # SSL separately. 
   serviceType: ClusterIP
-
+  createClusterIssuer: true
   # Domain that is pointed to the clusterIP
   # You will need to create an A record like *.osmseed.example.com pointed to the ClusterIP
   # Then, the cluster configuration will setup services at their respective subdomains:
@@ -480,11 +480,42 @@ osm-seed:
         cpu: "2"
 
   # ====================================================================================================
+  # Variables for Tasking Manager DB
+  # ====================================================================================================
+  tmDb:
+    enabled: true
+    image:
+      name: "postgis/postgis"
+      tag: "11-2.5"
+    nodeSelector:
+      enabled: true
+      label_key: nodegroup_type
+      label_value: web
+    env:
+      POSTGRES_DB: tm
+      POSTGRES_PASSWORD: {{PRODUCTION_TM_DB_PASSWORD}}
+      POSTGRES_USER: postgres
+    persistenceDisk:
+      enabled: true
+      accessMode: ReadWriteOnce
+      mountPath: /var/lib/postgresql/data
+      subPath: postgresql-d
+      AWS_ElasticBlockStore_volumeID: vol-03a2f95687a51a531
+      AWS_ElasticBlockStore_size: 20Gi
+    resources:
+      enabled: false
+      requests:
+        memory: "1Gi"
+        cpu: "2"
+      limits:
+        memory: "2Gi"
+        cpu: "2"
+  # ====================================================================================================
   # Variables for Tasking Manager API
   # ====================================================================================================
 
   tmApi:
-    enabled: true
+    enabled: false
     nodeSelector:
       enabled: true
       label_key: nodegroup_type

diff --git a/values.staging.template.yaml b/values.staging.template.yaml
@@ -38,7 +38,7 @@ osm-seed:
     # as the SSL certificate for your services. Else, you will need to configure
     # SSL separately. 
     serviceType: ClusterIP
-
+    createClusterIssuer: true
     # Domain that is pointed to the clusterIP
     # You will need to create an A record like *.osmseed.example.com pointed to the ClusterIP
     # Then, the cluster configuration will setup services at their respective subdomains:
@@ -159,7 +159,7 @@ osm-seed:
     # Variables for osm-seed for osmosis, this configuration os to get the planet dump files from apidb
     # ====================================================================================================
     planetDump:
-      enabled: true
+      enabled: false
       nodeSelector:
         enabled: true
         label_key: nodegroup_type
@@ -290,7 +290,7 @@ osm-seed:
       cronjobs:
       - name: web-db
         enabled: false
-        schedule: '* * * * *'
+        schedule: '0 1 * * *'
         env:
           # backup/restore
           DB_ACTION: backup
@@ -315,9 +315,32 @@ osm-seed:
           enabled: true
           label_key: nodegroup_type
           label_value: job
+      - name: tm-db
+        enabled: true
+        schedule: '0 1 * * *'
+        env:
+          # backup/restore
+          DB_ACTION: backup
+          # Naming backup files
+          SET_DATE_AT_NAME: true
+          BACKUP_CLOUD_FOLDER: database/tm-db
+          BACKUP_CLOUD_FILE: ohm-tm-db
+          AWS_S3_BUCKET: osmseed-staging
+        resources:
+          enabled: false
+          requests:
+            memory: '300Mi'
+            cpu: '0.5'
+          limits:
+            memory: '400Mi'
+            cpu: '0.6'
+        nodeSelector:
+          enabled: true
+          label_key: nodegroup_type
+          label_value: job
       - name: osmcha-db
         enabled: false
-        schedule: '0 0 * * *'
+        schedule: '0 1 * * *'
         env:
           # backup/restore
           DB_ACTION: backup
@@ -520,7 +543,37 @@ osm-seed:
           memory: "2Gi"
           cpu: "2"
 
-
+    # ====================================================================================================
+    # Variables for Tasking Manager DB
+    # ====================================================================================================
+    tmDb:
+      enabled: true
+      image:
+        name: "postgis/postgis"
+        tag: "11-2.5"
+      nodeSelector:
+        enabled: true
+        label_key: nodegroup_type
+        label_value: web
+      env:
+        POSTGRES_DB: tm
+        POSTGRES_PASSWORD: {{STAGING_TM_DB_PASSWORD}}
+        POSTGRES_USER: postgres
+      persistenceDisk:
+        enabled: true
+        accessMode: ReadWriteOnce
+        mountPath: /var/lib/postgresql/data
+        subPath: postgresql-d
+        AWS_ElasticBlockStore_volumeID: vol-05a364a565aea360c
+        AWS_ElasticBlockStore_size: 50Gi
+      resources:
+        enabled: false
+        requests:
+          memory: "1Gi"
+          cpu: "2"
+        limits:
+          memory: "2Gi"
+          cpu: "2"
     # ====================================================================================================
     # Variables for Tasking Manager API
     # ====================================================================================================