Merge branch 'staging' into date-functions-url

.github/workflows/frontend-nominatim.yaml

@@ -17,7 +17,7 @@ jobs:
         if: github.ref == 'refs/heads/staging'
         uses: allenevans/[email protected]
         with:
-          NOMINATIM_API: 'https://nominatim-api-staging.openhistoricalmap.org/'
+          NOMINATIM_API: 'https://nominatim-api.staging.openhistoricalmap.org/'
           NOMINATIM_BUCKET: 'nominatim-staging.openhistoricalmap.org'
           CLOUDFRONT_DISTRIBUTION_ID: ${{ secrets.STAGING_NOMINATIM_CLOUDFRONT_ID }}
 

.github/workflows/frontend-overpass.yaml

@@ -17,7 +17,7 @@ jobs:
         if: github.ref == 'refs/heads/staging'
         uses: allenevans/[email protected]
         with:
-          OVERPASS_API: overpass-api-staging.openhistoricalmap.org
+          OVERPASS_API: overpass-api.staging.openhistoricalmap.org
           OVERPASS_BUCKET: overpass-turbo-staging.openhistoricalmap.org
           CLOUDFRONT_DISTRIBUTION_ID: ${{ secrets.STAGING_OVERPASS_CLOUDFRONT_ID }}
 

.github/workflows/frontend-rapid.yaml

@@ -0,0 +1,90 @@
+name: Build and Publish Rapid site
+on:
+  push:
+    branches:
+      - 'main'
+      - 'staging'
+jobs:
+  rapid:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Cancel Previous Runs
+        uses: styfle/[email protected]
+        with:
+          access_token: ${{ github.token }}
+      - name: Set environment variables - Staging
+        if: github.ref == 'refs/heads/staging'
+        uses: allenevans/[email protected]
+        with:
+          RAPID_BUCKET: rapid-staging.openhistoricalmap.org
+          CLOUDFRONT_DISTRIBUTION_ID: E12LU6N0NDXVAX
+          OHM_URL: https://staging.openhistoricalmap.org
+          OHM_API_URL: https://staging.openhistoricalmap.org
+          OHM_CLIENT_ID: vwLXYrZ29YiwxWR5LvAcsiG9paLwE7hBod27O-tl16Q
+          OHM_CLIENT_SECRET: ${{ secrets.STAGING_RAPID_CLIENT_SECRET }}
+      - name: Set environment variables - Production
+        if: github.ref == 'refs/heads/main'
+        uses: allenevans/[email protected]
+        with:
+          RAPID_BUCKET: rapid.openhistoricalmap.org
+          CLOUDFRONT_DISTRIBUTION_ID: E3VOVJ6DH8HALM
+          OHM_URL: https://www.openhistoricalmap.org
+          OHM_API_URL: https://www.openhistoricalmap.org
+          OHM_CLIENT_ID: RIIz9U-oZmYzi_LXCKMHwHPgbJxALWAZRcgk9F8cppk
+          OHM_CLIENT_SECRET: ${{ secrets.PRODUCTION_RAPID_CLIENT_SECRET }}
+      - name: Checkout rapid repo
+        uses: actions/checkout@v2
+        with:
+          repository: OpenHistoricalMap/Rapid
+          ref: 042f2e27ab0b9a5192549fc17a1ee0f91acab67f
+      - name: Use Node.js 18
+        uses: actions/setup-node@v1
+        with:
+          node-version: 18
+      - name: Cache node modules
+        id: cache-node-modules
+        uses: actions/cache@v2
+        env:
+          cache-name: cache-node-modules
+        with:
+          path: node_modules
+          key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/yarn.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-build-${{ env.cache-name }}-
+            ${{ runner.os }}-build-
+            ${{ runner.os }}-
+      - name: Install modules
+        if: steps.cache-node-modules.outputs.cache-hit != 'true'
+        run: npm install
+      - name: Build rapid frontend
+        run: |
+          export OSM_URL=https://www.openstreetmap.org
+          export OSM_API_URL=https://api.openstreetmap.org
+          export OSM_CLIENT_ID=O3g0mOUuA2WY5Fs826j5tP260qR3DDX7cIIE2R2WWSc
+          export OSM_CLIENT_SECRET=b4aeHD1cNeapPPQTrvpPoExqQRjybit6JBlNnxh62uE
+          files=("modules/services/OsmService.js" "dist/index.html" "dist/latest.html" "index.html")
+          for file in "${files[@]}"; do
+              sed -e "s|${OSM_URL}|${OHM_URL}|g" "$file" > tmpfile && mv tmpfile "$file"
+              sed -e "s|${OSM_API_URL}|${OHM_API_URL}|g" "$file" > tmpfile && mv tmpfile "$file"
+              sed -e "s|${OSM_CLIENT_ID}|${OHM_CLIENT_ID}|g" "$file" > tmpfile && mv tmpfile "$file"
+              sed -e "s|${OSM_CLIENT_SECRET}|${OHM_CLIENT_SECRET}|g" "$file" > tmpfile && mv tmpfile "$file"
+          done
+          npm run clean
+          npm run build
+          npm run dist
+          sed -e "s|i.client_id|i.client_id,client_secret:\"${OHM_CLIENT_SECRET}\"|g" dist/rapid.min.js > tmpfile && mv tmpfile dist/rapid.min.js
+      - uses: actions/setup-python@v2
+        with:
+          python-version: '3.8'
+      - name: Install aws cli
+        run: |
+          python -m pip install --upgrade pip
+          pip install awscli
+      - name: Push data to s3 and clean cloudfront cache
+        if: ${{ success() }}
+        run: |
+          aws s3 sync dist/ s3://${RAPID_BUCKET}/  --acl public-read
+          aws cloudfront create-invalidation --distribution-id=${CLOUDFRONT_DISTRIBUTION_ID} --paths=/
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}

.github/workflows/frontend-tasking-manager.yaml

@@ -18,7 +18,7 @@ jobs:
         uses: allenevans/[email protected]
         with:
           TM_APP_BASE_URL: https://tasks-staging.openhistoricalmap.org
-          TM_APP_API_URL: https://staging-tm-api.openhistoricalmap.org
+          TM_APP_API_URL: https://tm-api.staging.openhistoricalmap.org
           TM_APP_API_VERSION: v2
           TM_ORG_NAME: OpenHistoricalMap
           TM_ORG_CODE: OHM

.gitignore

@@ -1,10 +1,13 @@
 .env-tiler
-tiler-db-data
-tiler-imposm-data-2
-tiler-imposm-data
-tiler-imposm
 images/data/
 data/
 images/.env
 images/tiler.yml
-values.dev.yaml
+values.dev.yaml
+secrets
+tegola
+envs/.env.tiler
+envs/.env.web
+config.toml
+.DS_Store
+*/.DS_Store

README.md

@@ -15,20 +15,19 @@ You can't really test this code locally, which can make it tempting to commit ch
 
 For updating the OHM website, which is a common reason for a deploy, see these lines:
 
-https://github.com/OpenHistoricalMap/ohm-deploy/blob/main/images/web/Dockerfile#L118-L121
+https://github.com/OpenHistoricalMap/ohm-deploy/blob/main/images/web/Dockerfile#L52
 
+```
+#change commit hash here to pick up latest changes
+ENV OPENHISTORICALMAP_WEBSITE_GITSHA=xyz
+```
+and line 58:
 ```
 # change the echo here with a reason for changing the commithash
-RUN echo 'Set the right commit: iD and staging tiles for staging website'
-RUN git fetch
-RUN git checkout 293d27abe0ed16abba7dd5849a29a5d3c7de4588
+RUN echo 'message about change'
 ```
 Change the message as appropriate and the commit hash to whatever commit in the `ohm-website` repo that you want to deploy.
 
-By practice, commit hashes from the `staging` branch on `ohm-website` should go to `staging` here and commit hashes from `production` on `ohm-website` should go to `main` here. 
-
-That said, there are times when we have published feature branches to `staging` to allow them to be viewed and tested by people who are not running the stack locally. That is fine, but should not be done for `main`.
-
 ## More details on process for changing, testing, and deploying
 
 This is what the process of making changes to openhistoricalmap.org looks like:
@@ -38,4 +37,4 @@ This is what the process of making changes to openhistoricalmap.org looks like:
 3. When your changes are working as desired, submit a PR from your feature branches into staging. Assign Dan, Sanjay, or Sajjad to review that PR. We can merge into `staging` and then update the commit hash on the staging branch of this repo, here https://github.com/OpenHistoricalMap/ohm-deploy/blob/staging/images/web/Dockerfile#L119-L121
 4. When we do that and push here, that kicks off a Github Actions automated deploy that will make your changes live on https://staging.openhistoricalmap.org.
 5. Test on Staging. This is when we can review with folks who are not running locally, share with the community, etc.
-6. When we're all happy with the code on Staging, we go back to `ohm-website` repo and make a PR of `staging` into `production` and then update the commit hash on the `main` branch in this OHM-deploy repo, which then kicks off deploy to production to make changes live on https://openhistoricalmap.org.
+6. When we're all happy with the code on Staging, make a PR of `staging` into `main` branch in this OHM-deploy repo, which then kicks off deploy to production to make changes live on https://openhistoricalmap.org.

chartpress.yaml

@@ -1,14 +1,15 @@
 charts:
   - name: ohm
-    imagePrefix: openhistoricalmap/
-
+    imagePrefix: ghcr.io/openhistoricalmap/
     images:
       web:
         valuesPath: osm-seed.web.image
       db:
         valuesPath: osm-seed.db.image
       tiler-db:
         valuesPath: osm-seed.tilerDb.image
+      tiler-imposm:
+        valuesPath: osm-seed.tilerImposm.image
       tiler-server:
         valuesPath: osm-seed.tilerServer.image
       tasking-manager-api:

compose/tiler.yml

@@ -0,0 +1,41 @@
+version: '3.8'
+services:
+  db:
+    image: ohm-tiler-db:v1
+    build:
+      context: ../images/tiler-db
+      dockerfile: Dockerfile
+    ports:
+      - "5433:5432"
+    volumes:
+      - ../data/tiler-db-data:/var/lib/postgresql/data
+    env_file:
+      - ../envs/.env.tiler
+    restart: always
+  imposm:
+    image: ohm-tiler-imposm:v1
+    build:
+      context: ../images/tiler-imposm
+      dockerfile: Dockerfile
+    volumes:
+      - ../data/tiler-imposm-data:/mnt/data
+      - ../images/tiler-imposm:/app
+    command:
+      - sh
+      - -c
+      - "sleep 60 && ./start.sh"
+    env_file:
+      - ../envs/.env.tiler
+  tiler:
+    image: ohm-tiler-server:v1
+    build:
+      context: ../images/tiler-server
+      dockerfile: Dockerfile
+    volumes:
+      - ../data/tiler-server-data:/mnt/data
+      - ../images/tiler-server:/app
+    ports:
+      - "9090:9090"
+    env_file:
+      - ../envs/.env.tiler
+    restart: always

images/docker-compose.yml → compose/web.yml

@@ -1,26 +1,26 @@
-version: '3.4'
+version: '3.8'
 services:
   db:
     platform: linux/amd64
-    image: osmseed-db:v1
+    image: ohm-db:v2
     build:
-      context: ../images/db
+      context: ./../images/db
       dockerfile: Dockerfile
     ports:
       - '5432:5432'
     volumes:
       - ./data/db-data:/var/lib/postgresql/data
     env_file:
-      - ./.env.example
+      - ./../envs/.env.web
   web:
-    image: osmseed-web:v1
+    image: ohm-web:v1
     build:
-      context: ./web
+      context: ./../images/web
       dockerfile: Dockerfile
     ports:
       - '80:80'
     env_file:
-      - ./.env.example
-    ####### Enable for development mode
-    # volumes:
-    #   - ./../../ohm-website:/var/www
+      - ./../envs/.env.web
+####### Enable for development mode
+# volumes:
+#   - ./../../ohm-website:/var/www

envs/.env.tiler.example

@@ -0,0 +1,23 @@
+#######################################
+# Environment variables for tiler database
+#######################################
+POSTGRES_HOST=host.docker.internal
+POSTGRES_DB=tiler-osm
+POSTGRES_USER=postgres
+POSTGRES_PASSWORD=1234
+POSTGRES_PORT=5432
+POSTGRES_DB_MAX_CONNECTIONS=200
+
+# #######################################
+# tiler-serve env variables,
+# #######################################
+TILER_SERVER_PORT=9090
+TILER_CACHE_BASEPATH=/mnt/data
+TILER_CACHE_MAX_ZOOM=22
+TILER_CACHE_TYPE=file
+TILER_CACHE_TYPE=file
+TILER_CACHE_BUCKET=osmseed-staging
+TILER_CACHE_BASEPATH=tegola
+TILER_CACHE_REGION=us-east-1
+TILER_CACHE_AWS_ACCESS_KEY_ID=...
+TILER_CACHE_AWS_SECRET_ACCESS_KEY=...

envs/.env.web.example

@@ -0,0 +1,7 @@
+POSTGRES_DB=openstreetmap
+POSTGRES_PASSWORD=abcd
+POSTGRES_USER=postgres
+POSTGRES_PORT=5432
+POSTGRES_HOST=host.docker.internal
+SERVER_URL=localhost
+SERVER_PROTOCOL=http

images/db/Dockerfile

@@ -1,24 +1,22 @@
-FROM postgres:11
-RUN rm /etc/apt/sources.list.d/pgdg.list
-RUN sed -i s/deb.debian.org/archive.debian.org/g /etc/apt/sources.list
-RUN sed -i 's|security.debian.org|archive.debian.org|g' /etc/apt/sources.list 
-RUN sed -i '/stretch-updates/d' /etc/apt/sources.list 
-RUN apt-get update && apt-get -y install apt-transport-https
-RUN echo "deb [ signed-by=/usr/local/share/keyrings/postgres.gpg.asc ] https://apt-archive.postgresql.org/pub/repos/apt/ stretch-pgdg main 11" >/etc/apt/sources.list.d/pgdg.list
-RUN apt-get update && apt-get install -y \
-    postgresql-server-dev-11 \
+FROM postgres:14
+RUN apt-get update \
+    && apt-get install -y \
+    postgresql-server-dev-14 \
     make \
     build-essential \
-    postgresql-11-postgis-2.5 && \
-    apt-get clean && rm -rf /var/lib/apt/lists/*
+    postgresql-14-postgis-3 \
+    && apt-get clean && rm -rf /var/lib/apt/lists/*
 
 ADD functions/functions.sql /usr/local/share/osm-db-functions.sql
 ADD docker_postgres.sh /docker-entrypoint-initdb.d/
 RUN mkdir -p db
 RUN mkdir -p lib
 ADD functions/ db/functions/
 ADD lib/quad_tile/ lib/quad_tile/
-
 RUN make -C db/functions/
 RUN chown -R postgres lib/
 RUN chown -R postgres db/
+COPY start.sh /usr/local/bin/
+COPY config/postgresql.*.conf /etc/postgresql/
+
+CMD ["/usr/local/bin/start.sh"]