Rapid deployment

.github/workflows/chartpress.yaml

@@ -1,5 +1,9 @@
 name: Chartpress Publish and Deploy
-on: push
+on:
+  push:
+    branches:
+      - 'main'
+      - 'staging'
 jobs:
   build:
     runs-on: ubuntu-20.04
@@ -50,14 +54,14 @@ jobs:
         STAGING_OAUTH_CLIENT_ID: ${{ secrets.STAGING_OAUTH_CLIENT_ID }}
         STAGING_OAUTH_KEY: ${{ secrets.STAGING_OAUTH_KEY }}
         STAGING_S3_BUCKET: ${{ secrets.STAGING_S3_BUCKET }}
-        STAGING_TILER_DB: ${{ secrets.STAGING_TILER_DB }}
-        STAGING_TILER_DB_EBS: ${{ secrets.STAGING_TILER_DB_EBS }}
-        STAGING_TILER_DB_HOST: ${{ secrets.STAGING_TILER_DB_HOST }}
+        # STAGING_TILER_DB: ${{ secrets.STAGING_TILER_DB }}
+        # STAGING_TILER_DB_EBS: ${{ secrets.STAGING_TILER_DB_EBS }}
+        # STAGING_TILER_DB_HOST: ${{ secrets.STAGING_TILER_DB_HOST }}
         STAGING_TILER_DB_PASSWORD: ${{ secrets.STAGING_TILER_DB_PASSWORD }}
-        STAGING_TILER_DB_USER: ${{ secrets.STAGING_TILER_DB_USER }}
-        STAGING_TILER_IMPOSM_EBS: ${{ secrets.STAGING_TILER_IMPOSM_EBS }}
+        # STAGING_TILER_DB_USER: ${{ secrets.STAGING_TILER_DB_USER }}
+        # STAGING_TILER_IMPOSM_EBS: ${{ secrets.STAGING_TILER_IMPOSM_EBS }}
         STAGING_TILER_IMPOSM_REPLICATION_URL: ${{ secrets.STAGING_TILER_IMPOSM_REPLICATION_URL }}
-        STAGING_TILER_SERVER_EBS: ${{ secrets.STAGING_TILER_SERVER_EBS }}
+        # STAGING_TILER_SERVER_EBS: ${{ secrets.STAGING_TILER_SERVER_EBS }}
         STAGING_TILER_CACHE_AWS_ACCESS_KEY_ID: ${{ secrets.STAGING_TILER_CACHE_AWS_ACCESS_KEY_ID }}
         STAGING_TILER_CACHE_AWS_SECRET_ACCESS_KEY: ${{ secrets.STAGING_TILER_CACHE_AWS_SECRET_ACCESS_KEY }}
         STAGING_TM_API_CONSUMER_KEY: ${{ secrets.STAGING_TM_API_CONSUMER_KEY }}
@@ -67,12 +71,12 @@ jobs:
         STAGING_TM_API_DB_PASSWORD: ${{ secrets.STAGING_TM_API_DB_PASSWORD }}
         STAGING_TM_API_DB_USER: ${{ secrets.STAGING_TM_API_DB_USER }}
         STAGING_TM_API_SECRET: ${{ secrets.STAGING_TM_API_SECRET }}
-        STAGING_NOMINATIM_DB_EBS: ${{ secrets.STAGING_NOMINATIM_DB_EBS }}
+        # STAGING_NOMINATIM_DB_EBS: ${{ secrets.STAGING_NOMINATIM_DB_EBS }}
         # STAGING_NOMINATIM_PG_PORT: ${{ secrets.STAGING_NOMINATIM_PG_PORT }}
         # STAGING_NOMINATIM_PG_USER: ${{ secrets.STAGING_NOMINATIM_PG_USER }}
         STAGING_NOMINATIM_PG_PASSWORD: ${{ secrets.STAGING_NOMINATIM_PG_PASSWORD }} 
         # STAGING_NOMINATIM_PG_DATABASE: ${{ secrets.STAGING_NOMINATIM_PG_DATABASE }}
-        STAGING_OVERPASS_API_DB_EBS: ${{ secrets.STAGING_OVERPASS_API_DB_EBS }}
+        # STAGING_OVERPASS_API_DB_EBS: ${{ secrets.STAGING_OVERPASS_API_DB_EBS }}
         STAGING_NEW_RELIC_LICENSE_KEY: ${{ secrets.STAGING_NEW_RELIC_LICENSE_KEY }}
         STAGING_NEW_RELIC_APP_NAME: ${{ secrets.STAGING_NEW_RELIC_APP_NAME }}
         STAGING_OSMCHA_PG_USER: ${{ secrets.STAGING_OSMCHA_PG_USER }}

.github/workflows/frontend-rapid.yaml

@@ -0,0 +1,90 @@
+name: Build and Publish Rapid site
+on:
+  push:
+    branches:
+      - 'main'
+      - 'staging'
+jobs:
+  rapid:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Cancel Previous Runs
+        uses: styfle/[email protected]
+        with:
+          access_token: ${{ github.token }}
+      - name: Set environment variables - Staging
+        if: github.ref == 'refs/heads/staging'
+        uses: allenevans/[email protected]
+        with:
+          RAPID_BUCKET: rapid-staging.openhistoricalmap.org
+          CLOUDFRONT_DISTRIBUTION_ID: E12LU6N0NDXVAX
+          OHM_URL: https://staging.openhistoricalmap.org
+          OHM_API_URL: https://staging.openhistoricalmap.org
+          OHM_CLIENT_ID: vwLXYrZ29YiwxWR5LvAcsiG9paLwE7hBod27O-tl16Q
+          OHM_CLIENT_SECRET: ${{ secrets.STAGING_RAPID_CLIENT_SECRET }}
+      - name: Set environment variables - Production
+        if: github.ref == 'refs/heads/main'
+        uses: allenevans/[email protected]
+        with:
+          RAPID_BUCKET: rapid.openhistoricalmap.org
+          CLOUDFRONT_DISTRIBUTION_ID: E3VOVJ6DH8HALM
+          OHM_URL: https://www.openhistoricalmap.org
+          OHM_API_URL: https://www.openhistoricalmap.org
+          OHM_CLIENT_ID: RIIz9U-oZmYzi_LXCKMHwHPgbJxALWAZRcgk9F8cppk
+          OHM_CLIENT_SECRET: ${{ secrets.PRODUCTION_RAPID_CLIENT_SECRET }}
+      - name: Checkout rapid repo
+        uses: actions/checkout@v2
+        with:
+          repository: OpenHistoricalMap/Rapid
+          ref: 042f2e27ab0b9a5192549fc17a1ee0f91acab67f
+      - name: Use Node.js 18
+        uses: actions/setup-node@v1
+        with:
+          node-version: 18
+      - name: Cache node modules
+        id: cache-node-modules
+        uses: actions/cache@v2
+        env:
+          cache-name: cache-node-modules
+        with:
+          path: node_modules
+          key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/yarn.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-build-${{ env.cache-name }}-
+            ${{ runner.os }}-build-
+            ${{ runner.os }}-
+      - name: Install modules
+        if: steps.cache-node-modules.outputs.cache-hit != 'true'
+        run: npm install
+      - name: Build rapid frontend
+        run: |
+          export OSM_URL=https://www.openstreetmap.org
+          export OSM_API_URL=https://api.openstreetmap.org
+          export OSM_CLIENT_ID=O3g0mOUuA2WY5Fs826j5tP260qR3DDX7cIIE2R2WWSc
+          export OSM_CLIENT_SECRET=b4aeHD1cNeapPPQTrvpPoExqQRjybit6JBlNnxh62uE
+          files=("modules/services/OsmService.js" "dist/index.html" "dist/latest.html" "index.html")
+          for file in "${files[@]}"; do
+              sed -e "s|${OSM_URL}|${OHM_URL}|g" "$file" > tmpfile && mv tmpfile "$file"
+              sed -e "s|${OSM_API_URL}|${OHM_API_URL}|g" "$file" > tmpfile && mv tmpfile "$file"
+              sed -e "s|${OSM_CLIENT_ID}|${OHM_CLIENT_ID}|g" "$file" > tmpfile && mv tmpfile "$file"
+              sed -e "s|${OSM_CLIENT_SECRET}|${OHM_CLIENT_SECRET}|g" "$file" > tmpfile && mv tmpfile "$file"
+          done
+          npm run clean
+          npm run build
+          npm run dist
+          sed -e "s|i.client_id|i.client_id,client_secret:\"${OHM_CLIENT_SECRET}\"|g" dist/rapid.min.js > tmpfile && mv tmpfile dist/rapid.min.js
+      - uses: actions/setup-python@v2
+        with:
+          python-version: '3.8'
+      - name: Install aws cli
+        run: |
+          python -m pip install --upgrade pip
+          pip install awscli
+      - name: Push data to s3 and clean cloudfront cache
+        if: ${{ success() }}
+        run: |
+          aws s3 sync dist/ s3://${RAPID_BUCKET}/  --acl public-read
+          aws cloudfront create-invalidation --distribution-id=${CLOUDFRONT_DISTRIBUTION_ID} --paths=/
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}

.gitignore

@@ -9,4 +9,6 @@ images/.env
 images/tiler.yml
 values.dev.yaml
 secrets
-tegola
+tegola
+envs/.env.tiler
+envs/.env.web

compose/tiler.yml

@@ -11,5 +11,5 @@ services:
     ports:
       - "9090:9090"
     env_file:
-      - ../envs/.env.tiler.example
+      - ../envs/.env.tiler
     restart: always

compose/web.yml

@@ -11,7 +11,7 @@ services:
     volumes:
       - ./data/db-data:/var/lib/postgresql/data
     env_file:
-      - ./.env.web.example
+      - ./.env.web
   web:
     image: osmseed-web:v1
     build:
@@ -20,7 +20,7 @@ services:
     ports:
       - '80:80'
     env_file:
-      - ./.env.web.example
+      - ./.env.web
     ####### Enable for development mode
     # volumes:
     #   - ./../../ohm-website:/var/www

images/tiler-server/config/config.toml

@@ -36,35 +36,35 @@ max_connections = 40
 	geometry_fieldname = "geometry"
 	geometry_type = "multipolygon"
 	id_fieldname = "ogc_fid"
-	sql = "SELECT ST_AsMVTGeom(ST_Simplify(wkb_geometry, 750),!BBOX!) AS geometry, ogc_fid FROM land_polygons WHERE wkb_geometry && !BBOX!"
+	sql = "SELECT ST_AsMVTGeom(ST_Simplify(wkb_geometry, 500),!BBOX!) AS geometry, ogc_fid FROM land_polygons WHERE wkb_geometry && !BBOX!"
 
 	[[providers.layers]]
 	name = "land_3-5"
 	geometry_fieldname = "geometry"
 	geometry_type = "multipolygon"
 	id_fieldname = "ogc_fid"
-	sql = "SELECT ST_AsMVTGeom(ST_Simplify(wkb_geometry, 300),!BBOX!) AS geometry, ogc_fid FROM land_polygons WHERE wkb_geometry && !BBOX!"
+	sql = "SELECT ST_AsMVTGeom(ST_Simplify(wkb_geometry, 200),!BBOX!) AS geometry, ogc_fid FROM land_polygons WHERE wkb_geometry && !BBOX!"
 
 	[[providers.layers]]
 	name = "land_6-7"
 	geometry_fieldname = "geometry"
 	geometry_type = "multipolygon"
 	id_fieldname = "ogc_fid"
-	sql = "SELECT ST_AsMVTGeom(ST_Simplify(wkb_geometry, 150),!BBOX!) AS geometry, ogc_fid FROM land_polygons WHERE wkb_geometry && !BBOX!"
+	sql = "SELECT ST_AsMVTGeom(ST_Simplify(wkb_geometry, 70),!BBOX!) AS geometry, ogc_fid FROM land_polygons WHERE wkb_geometry && !BBOX!"
 
 	[[providers.layers]]
 	name = "land_8-9"
 	geometry_fieldname = "geometry"
 	geometry_type = "multipolygon"
 	id_fieldname = "ogc_fid"
-	sql = "SELECT ST_AsMVTGeom(ST_Simplify(wkb_geometry, 75),!BBOX!) AS geometry, ogc_fid FROM land_polygons WHERE wkb_geometry && !BBOX!"
+	sql = "SELECT ST_AsMVTGeom(ST_Simplify(wkb_geometry, 30),!BBOX!) AS geometry, ogc_fid FROM land_polygons WHERE wkb_geometry && !BBOX!"
 
 	[[providers.layers]]
 	name = "land_10-12"
 	geometry_fieldname = "geometry"
 	geometry_type = "multipolygon"
 	id_fieldname = "ogc_fid"
-	sql = "SELECT ST_AsMVTGeom(ST_Simplify(wkb_geometry, 30),!BBOX!) AS geometry, ogc_fid FROM land_polygons WHERE wkb_geometry && !BBOX!"
+	sql = "SELECT ST_AsMVTGeom(ST_Simplify(wkb_geometry, 10),!BBOX!) AS geometry, ogc_fid FROM land_polygons WHERE wkb_geometry && !BBOX!"
 
 	[[providers.layers]]
 	name = "land_13-15"
@@ -245,14 +245,14 @@ max_connections = 40
 	geometry_fieldname = "geometry"
 	geometry_type = "linestring"
 	id_fieldname = "osm_id"
-	sql = "SELECT ST_AsMVTGeom(geometry,!BBOX!) AS geometry, ABS(osm_id) as osm_id, type, tunnel, bridge, ref, tags->'start_date' as start_date, tags->'end_date' as end_date, isodatetodecimaldate(pad_date(tags->'start_date', 'start'), FALSE) AS start_decdate, isodatetodecimaldate(pad_date(tags->'end_date', 'end'), FALSE) AS end_decdate FROM osm_transport_lines_gen0 WHERE type IN ('motorway','trunk','motorway_link','trunk_link','primary') AND tunnel = 0 AND bridge = 0  AND geometry && !BBOX!"
+	sql = "SELECT ST_AsMVTGeom(geometry,!BBOX!) AS geometry, ABS(osm_id) as osm_id, type, tunnel, bridge, ref, tags->'usage' as usage, tags->'start_date' as start_date, tags->'end_date' as end_date, isodatetodecimaldate(pad_date(tags->'start_date', 'start'), FALSE) AS start_decdate, isodatetodecimaldate(pad_date(tags->'end_date', 'end'), FALSE) AS end_decdate FROM osm_transport_lines_gen0 WHERE type IN ('motorway','trunk','motorway_link','trunk_link','primary') AND tunnel = 0 AND bridge = 0  AND geometry && !BBOX!"
 
 	[[providers.layers]]
 	name = "transport_lines_gen1"
 	geometry_fieldname = "geometry"
 	geometry_type = "linestring"
 	id_fieldname = "osm_id"
-	sql = """SELECT ST_AsMVTGeom(geometry,!BBOX!) AS geometry, osm_id, ref, class, type, tags->'start_date' as start_date, tags->'end_date' as end_date, isodatetodecimaldate(pad_date(tags->'start_date', 'start'), FALSE) AS start_decdate, isodatetodecimaldate(pad_date(tags->'end_date', 'end'), FALSE) AS end_decdate 
+	sql = """SELECT ST_AsMVTGeom(geometry,!BBOX!) AS geometry, osm_id, ref, class, type, tags->'usage' as usage, tags->'start_date' as start_date, tags->'end_date' as end_date, isodatetodecimaldate(pad_date(tags->'start_date', 'start'), FALSE) AS start_decdate, isodatetodecimaldate(pad_date(tags->'end_date', 'end'), FALSE) AS end_decdate 
 			FROM osm_transport_lines_gen1 
 			WHERE type IN ('motorway', 'trunk', 'primary', 'primary_link', 'secondary', 'motorway_link', 'trunk_link', 'rail', 'subway', 'light_rail', 'tram', 'narrow_gauge', 'monorail', 'preserved', 'miniature', 'funicular', 'taxiway', 'runway', 'proposed', 'construction', 'disused', 'abandoned', 'razed', 'ferry') 
 			AND geometry && !BBOX!"""
@@ -262,7 +262,7 @@ max_connections = 40
 	geometry_fieldname = "geometry"
 	geometry_type = "linestring"
 	id_fieldname = "osm_id"
-	sql = """SELECT ST_AsMVTGeom(geometry,!BBOX!) AS geometry, osm_id, name, ref, class, type, tunnel, bridge, access, service, ford, tags->'construction' as construction, tags->'start_date' as start_date, tags->'end_date' as end_date, isodatetodecimaldate(pad_date(tags->'start_date', 'start'), FALSE) AS start_decdate, isodatetodecimaldate(pad_date(tags->'end_date', 'end'), FALSE) AS end_decdate 
+	sql = """SELECT ST_AsMVTGeom(geometry,!BBOX!) AS geometry, osm_id, name, ref, class, type, tunnel, bridge, access, service, ford, tags->'construction' as construction, tags->'usage' as usage, tags->'start_date' as start_date, tags->'end_date' as end_date, isodatetodecimaldate(pad_date(tags->'start_date', 'start'), FALSE) AS start_decdate, isodatetodecimaldate(pad_date(tags->'end_date', 'end'), FALSE) AS end_decdate 
 			FROM osm_transport_lines 
 			WHERE type IN ('motorway', 'motorway_link', 'trunk', 'trunk_link', 'primary', 'primary_link', 'secondary', 'secondary_link', 'tertiary', 'tertiary_link', 'rail', 'subway', 'light_rail', 'tram', 'narrow_gauge', 'monorail', 'preserved', 'miniature', 'funicular', 'taxiway', 'runway', 'proposed', 'construction', 'disused', 'abandoned', 'razed', 'apron', 'ferry')  
 			AND geometry && !BBOX!"""
@@ -272,7 +272,7 @@ max_connections = 40
 	geometry_fieldname = "geometry"
 	geometry_type = "linestring"
 	id_fieldname = "id"
-	sql = """SELECT ST_AsMVTGeom(geometry,!BBOX!) AS geometry, id, osm_id, name, ref, class, type, tunnel, bridge, access, service, ford, tags->'construction' as construction, tags->'start_date' as start_date, tags->'end_date' as end_date, isodatetodecimaldate(pad_date(tags->'start_date', 'start'), FALSE) AS start_decdate, isodatetodecimaldate(pad_date(tags->'end_date', 'end'), FALSE) AS end_decdate 
+	sql = """SELECT ST_AsMVTGeom(geometry,!BBOX!) AS geometry, id, osm_id, name, ref, class, type, tunnel, bridge, access, service, ford, tags->'construction' as construction, tags->'usage' as usage, tags->'start_date' as start_date, tags->'end_date' as end_date, isodatetodecimaldate(pad_date(tags->'start_date', 'start'), FALSE) AS start_decdate, isodatetodecimaldate(pad_date(tags->'end_date', 'end'), FALSE) AS end_decdate 
 			FROM osm_transport_lines 
 			WHERE type IN ('motorway', 'motorway_link', 'trunk', 'trunk_link', 'primary', 'primary_link', 'secondary', 'secondary_link', 'tertiary', 'tertiary_link', 'rail', 'subway', 'light_rail', 'tram', 'narrow_gauge', 'monorail', 'preserved', 'miniature', 'funicular', 'taxiway', 'runway', 'residential', 'taxiway', 'runway', 'proposed', 'construction', 'disused', 'abandoned', 'razed', 'apron', 'ferry')  
 			AND geometry && !BBOX!"""
@@ -282,7 +282,7 @@ max_connections = 40
 	geometry_fieldname = "geometry"
 	geometry_type = "linestring"
 	id_fieldname = "id"
-	sql = """SELECT ST_AsMVTGeom(geometry,!BBOX!) AS geometry, id, osm_id, name, ref, class, type, tunnel, bridge, access, service, ford, tags->'construction' as construction, tags->'start_date' as start_date, tags->'end_date' as end_date, isodatetodecimaldate(pad_date(tags->'start_date', 'start'), FALSE) AS start_decdate, isodatetodecimaldate(pad_date(tags->'end_date', 'end'), FALSE) AS end_decdate 
+	sql = """SELECT ST_AsMVTGeom(geometry,!BBOX!) AS geometry, id, osm_id, name, ref, class, type, tunnel, bridge, access, service, ford, tags->'construction' as construction, tags->'usage' as usage, tags->'start_date' as start_date, tags->'end_date' as end_date, isodatetodecimaldate(pad_date(tags->'start_date', 'start'), FALSE) AS start_decdate, isodatetodecimaldate(pad_date(tags->'end_date', 'end'), FALSE) AS end_decdate 
 			FROM osm_transport_lines 
 			WHERE geometry && !BBOX!"""
 

values.staging.template.yaml

@@ -344,9 +344,9 @@ osm-seed:
     tilerDb:
       enabled: true
       env:
-        POSTGRES_HOST: {{STAGING_TILER_DB_HOST}}
-        POSTGRES_DB: {{STAGING_TILER_DB}}
-        POSTGRES_USER: {{STAGING_TILER_DB_USER}}
+        POSTGRES_HOST: staging-tiler-db
+        POSTGRES_DB: tiler-osm
+        POSTGRES_USER: postgres
         POSTGRES_PASSWORD: {{STAGING_TILER_DB_PASSWORD}}
         POSTGRES_PORT: 5432
         # for 20Gi