Split tiles deployment

.github/workflows/deploy.yml

@@ -46,7 +46,6 @@ jobs:
 
     outputs:
       matrix: ${{ steps.bboxes.outputs.matrix }}
-      bboxes: ${{ steps.bboxes.outputs.bboxes }}
 
     steps:
       - name: Checkout
@@ -56,13 +55,51 @@ jobs:
         run: |
           {
             echo -n 'matrix='
-            jq -c '{ bbox: [.[].bboxes] | flatten }' tiles/deploy-bboxes.json
+            jq -c '{ include: [
+              (.["north-america"][].bboxes | map({bbox: ., region: "north-america"}) | .[]),
+              (.["europe"][].bboxes | map({bbox: ., region: "europe"}) | .[])
+            ]}' tiles/deploy-bboxes.json
           } >> $GITHUB_OUTPUT
 
-          {
-            echo -n 'bboxes='
-            jq -r '[.[].bboxes] | flatten | .[]' tiles/deploy-bboxes.json | tr '\n' ' '
-          } >> $GITHUB_OUTPUT
+  generate-tiles-low-med:
+    name: Generate low zoom tiles
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Pull database
+        run: |
+          docker compose pull db
+          docker compose up --no-build --pull never --wait db
+
+      - name: Generate tiles
+        env:
+          TILES: low-med
+          BBOX: -180,-80,180,80
+        run: |
+          docker compose run --no-deps martin-cp
+
+      - name: List generated tiles
+        run: |
+          ls -lah tiles/*.mbtiles
+
+      - name: Deploy
+        env:
+          FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }}
+        run: |
+          flyctl deploy --config martin-static.fly.toml --local-only
 
   generate-tiles:
     name: Generate tiles
@@ -95,6 +132,7 @@ jobs:
 
       - name: Generate tiles
         env:
+          TILES: high
           BBOX: ${{ matrix.bbox }}
         run: |
           docker compose run --no-deps martin-cp
@@ -108,7 +146,7 @@ jobs:
 
       - uses: actions/upload-artifact@v4
         with:
-          name: tiles-${{ matrix.bbox }}
+          name: tiles-${{ matrix.region }}-${{ matrix.bbox }}
           path: |
             tiles/split
           if-no-files-found: error
@@ -120,9 +158,16 @@ jobs:
 
     needs:
       - tiles-bboxes
+      # TODO deploy per region
       - generate-tiles
 
-    environment: tiles-production
+    strategy:
+      matrix:
+        region:
+          - europe
+          - north-america
+
+    environment: tiles-production-${{ matrix.region }}
 
     env:
       BBOXES: ${{ needs.tiles-bboxes.outputs.bboxes }}
@@ -146,15 +191,27 @@ jobs:
 
       - uses: actions/download-artifact@v4
         with:
-          pattern: tiles-*
+          pattern: tiles-${{ matrix.region }}-*
           merge-multiple: true
           path: tiles/split
 
+      - name: Generate bboxes
+        id: bboxes
+        run: |
+          {
+            echo -n 'bboxes='
+            jq -r '[.["${{ matrix.region }}"].[].bboxes] | flatten | .[]' tiles/deploy-bboxes.json | tr '\n' ' '
+          } >> $GITHUB_OUTPUT
+
       - name: Merge tiles
+        env:
+          BBOXES: ${{ steps.bboxes.outputs.bboxes }}
         run: |
           docker compose run --entrypoint /tiles/merge.sh -e BBOXES --no-deps martin-cp
 
       - name: Delete split tiles
+        env:
+          BBOXES: ${{ steps.bboxes.outputs.bboxes }}
         run: |
           for bbox in $BBOXES; do
             rm -rf "tiles/split/$bbox"
@@ -168,7 +225,7 @@ jobs:
         env:
           FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }}
         run: |
-          flyctl deploy --config martin-static.fly.toml --local-only
+          flyctl deploy --app openrailwaymap-tiles-${{ matrix.region }} --config martin-static.fly.toml --local-only
 
   deploy-api:
     name: Deploy API

.github/workflows/nightly-update.yml

@@ -3,8 +3,8 @@ name: Nightly update
 on:
   workflow_dispatch: ~
   schedule:
-    # Run daily at 17:47 UTC
-    - cron: '47 17 * * *'
+    # Run daily at 23:47 or 00:47 Europe/Amsterdam
+    - cron: '47 22 * * *'
 
 jobs:
   update-data:

docker-compose.yml

@@ -70,6 +70,7 @@ services:
       # NL: 2.99,50.74753,7.230455,54.01786
       # PL-west: 15.0590,51.4957,19.5150,53.4155
       - BBOX
+      - TILES
       - MIN_ZOOM=0
       - MAX_ZOOM=14
       - DATABASE_URL=postgresql://postgres@db:5432/gis
@@ -110,6 +111,8 @@ services:
       - '8000:8000'
     environment:
       TILES_UPSTREAM: martin:3000
+      TILES_UPSTREAM_NORTH_AMERICA: martin:3000
+      TILES_UPSTREAM_EUROPE: martin:3000
       API_UPSTREAM: api:5000
       PUBLIC_PROTOCOL: http
       PUBLIC_HOST: localhost:8000

proxy.fly.toml

@@ -18,6 +18,8 @@ primary_region = 'ams'
 
 [env]
   TILES_UPSTREAM = "ams.openrailwaymap-tiles.internal:3000"
+  TILES_UPSTREAM_NORTH_AMERICA = "ams.openrailwaymap-tiles-north-america.internal:3000"
+  TILES_UPSTREAM_EUROPE = "ams.openrailwaymap-tiles-europe.internal:3000"
   API_UPSTREAM = "ams.openrailwaymap-api.internal:5000"
   PUBLIC_PROTOCOL = "https"
   PUBLIC_HOST = "openrailwaymap.fly.dev"

proxy/proxy.conf.template

@@ -16,6 +16,72 @@ map $http_referer $cache_assets {
   default 1d;
 }
 
+map $request_uri $tiles_upstream {
+  # See https://chrishewett.com/blog/slippy-tile-explorer/
+  # Match tiles on the form /name/z/x/y
+  # Tile coordinate recurrence relation: (x_min_{z+1}, y_min_{z+1}, x_max_{z+1}, y_max_{z+1}) = (2 * x_min_{z}, 2 * y_min_{z}, 2 * x_max_{z} + 1, 2 * y_max_{z} + 1)
+  # See https://www.regular-expressions.info/numericranges.html
+  # See https://www.regex-range.com/
+
+  # This is not beautiful, but it works
+
+  # Zoom 0-7 is handled by the default tile server
+
+  # North America (Zoom 4: x=0..5 y=0..6 or lon=-180..45 lat=-45..21.94)
+  # Zoom 8: x=0..95 y=0..111
+  "~^/[^/]+/8/([0-9]|[1-8][0-9]|9[0-5])/([0-9]|[1-9][0-9]|10[0-9]|11[01]{1})$" http://${TILES_UPSTREAM_NORTH_AMERICA};
+  # Zoom 9: x=0..191 y=0..223
+  "~^/[^/]+/9/([0-9]|[1-9][0-9]|1[0-8][0-9]|19[01])/([0-9]|[1-9][0-9]|1[0-9]{2}|2[01][0-9]|22[0-3])$" http://${TILES_UPSTREAM_NORTH_AMERICA};
+  # Zoom 10: x=0..383 y=0..447
+  "~^/[^/]+/10/([0-9]|[1-9][0-9]|[12][0-9]{2}|3[0-7][0-9]|38[0-3])/([0-9]|[1-9][0-9]|[1-3][0-9]{2}|4[0-3][0-9]|44[0-7])$" http://${TILES_UPSTREAM_NORTH_AMERICA};
+  # Zoom 11: x=0..767 y=0..895
+  "~^/[^/]+/11/([0-9]|[1-9][0-9]|[1-6][0-9]{2}|7[0-5][0-9]|76[0-7])/([0-9]|[1-9][0-9]|[1-7][0-9]{2}|8[0-8][0-9]|89[0-5])$" http://${TILES_UPSTREAM_NORTH_AMERICA};
+  # Zoom 12: x=0..1535 y=0..1791
+  "~^/[^/]+/12/([0-9]|[1-9][0-9]{1,2}|1[0-4][0-9]{2}|15[0-2][0-9]|153[0-5])/([0-9]|[1-9][0-9]{1,2}|1[0-6][0-9]{2}|17[0-8][0-9]|179[01])$" http://${TILES_UPSTREAM_NORTH_AMERICA};
+  # Zoom 13: x=0..3071 y=0..3583
+  "~^/[^/]+/13/([0-9]|[1-9][0-9]{1,2}|[12][0-9]{3}|30[0-6][0-9]|307[01])/([0-9]|[1-9][0-9]{1,2}|[12][0-9]{3}|3[0-4][0-9]{2}|35[0-7][0-9]|358[0-3])$" http://${TILES_UPSTREAM_NORTH_AMERICA};
+  # Zoom 14: x=0..6143 y=0..7167
+  "~^/[^/]+/14/([0-9]|[1-9][0-9]{1,2}|[1-5][0-9]{3}|60[0-9]{2}|61[0-3][0-9]|614[0-3])/([0-9]|[1-9][0-9]{1,2}|[1-6][0-9]{3}|70[0-9]{2}|71[0-5][0-9]|716[0-7])$" http://${TILES_UPSTREAM_NORTH_AMERICA};
+
+  # Europe (Zoom 4: x=7..9 y=3..6 or lon=-22.5..45 lat=74.02..21.94)
+  # Zoom 8: x=112..144 y=63..111
+  "~^/[^/]+/8/(11[2-9]|1[23][0-9]|14[0-4])/(6[3-9]|[7-9][0-9]|10[0-9]|11[01])$" http://${TILES_UPSTREAM_EUROPE};
+  # Zoom 9: x=224..288 y=127..223
+  "~^/[^/]+/9/(22[4-9]|2[3-7][0-9]|28[0-8])/(12[7-9]|1[3-9][0-9]|2[01][0-9]|22[0-3])$" http://${TILES_UPSTREAM_EUROPE};
+  # Zoom 10: x=448..576 y=255..447
+  "~^/[^/]+/10/(44[89]|4[5-9][0-9]|5[0-6][0-9]|57[0-6])/(25[5-9]|2[6-9][0-9]|3[0-9]{2}|4[0-3][0-9]|44[0-7])$" http://${TILES_UPSTREAM_EUROPE};
+  # Zoom 11: x=896..1152 y=511..895
+  "~^/[^/]+/11/(89[6-9]|9[0-9]{2}|10[0-9]{2}|11[0-4][0-9]|115[0-2])/(51[1-9]|5[2-9][0-9]|[67][0-9]{2}|8[0-8][0-9]|89[0-5])$" http://${TILES_UPSTREAM_EUROPE};
+  # Zoom 12: x=1792..2304 y=1023..1791
+  "~^/[^/]+/12/(179[2-9]|1[89][0-9]{2}|2[0-2][0-9]{2}|230[0-4])/(102[3-9]|10[3-9][0-9]|1[1-6][0-9]{2}|17[0-8][0-9]|179[01])$" http://${TILES_UPSTREAM_EUROPE};
+  # Zoom 13: x=3584..4608 y=2047..3583
+  "~^/[^/]+/13/(358[4-9]|359[0-9]|3[6-9][0-9]{2}|4[0-5][0-9]{2}|460[0-8])/(204[7-9]|20[5-9][0-9]|2[1-9][0-9]{2}|3[0-4][0-9]{2}|35[0-7][0-9]|358[0-3])$" http://${TILES_UPSTREAM_EUROPE};
+  # Zoom 14: x=7168..9216 y=4095..7167
+  "~^/[^/]+/14/(716[89]|71[7-9][0-9]|7[2-9][0-9]{2}|8[0-9]{3}|9[01][0-9]{2}|920[0-9]|921[0-6])/(409[5-9]|4[1-9][0-9]{2}|[56][0-9]{3}|70[0-9]{2}|71[0-5][0-9]|716[0-7])$" http://${TILES_UPSTREAM_EUROPE};
+
+  # Africa (Zoom 4: x=7..10 y=7..9 or lon=-22.5..67.5 lat=21.94..-40.979)
+  # Zoom 8: x=112..144 y=63..111
+  "~^/[^/]+/8/$" http://${TILES_UPSTREAM_EUROPE};
+  # Zoom 9: x=224..288 y=127..223
+  "~^/[^/]+/9/$" http://${TILES_UPSTREAM_EUROPE};
+  # Zoom 10: x=448..576 y=255..447
+  "~^/[^/]+/10/$" http://${TILES_UPSTREAM_EUROPE};
+  # Zoom 11: x=896..1152 y=511..895
+  "~^/[^/]+/11/$" http://${TILES_UPSTREAM_EUROPE};
+  # Zoom 12: x=1792..2304 y=1023..1791
+  "~^/[^/]+/12/$" http://${TILES_UPSTREAM_EUROPE};
+  # Zoom 13: x=3584..4608 y=2047..3583
+  "~^/[^/]+/13/$" http://${TILES_UPSTREAM_EUROPE};
+  # Zoom 14: x=7168..9216 y=4095..7167
+  "~^/[^/]+/14/$" http://${TILES_UPSTREAM_EUROPE};
+
+  # Tile specifications
+  "~^/(high|standard|speed|signals|electrification)$" http://${TILES_UPSTREAM_EUROPE};
+
+  # Fallback: default tile server
+  default http://${TILES_UPSTREAM};
+}
+
 server {
   listen 8000 default_server;
   server_name localhost;
@@ -100,8 +166,7 @@ server {
       rewrite ^/electrification$ /electrification_signals last;
     }
 
-    set $upstream http://${TILES_UPSTREAM};
-    proxy_pass $upstream;
+    proxy_pass $tiles_upstream;
     proxy_http_version 1.1;
     proxy_cache cache;
     proxy_cache_lock on;