From 68ee2eced0feb467e7d86bcaf1901fcffbebec5f Mon Sep 17 00:00:00 2001 From: Rub21 Date: Mon, 4 Sep 2023 14:24:27 -0500 Subject: [PATCH 01/16] Optimizing the layers in docker container --- images/web/Dockerfile | 22 +++++++--------------- 1 file changed, 7 insertions(+), 15 deletions(-) diff --git a/images/web/Dockerfile b/images/web/Dockerfile index 9f4d9c8d..a7f44cfa 100644 --- a/images/web/Dockerfile +++ b/images/web/Dockerfile @@ -18,14 +18,17 @@ RUN apt-get update \ RUN curl -sL https://deb.nodesource.com/setup_16.x | bash - RUN curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - RUN echo "deb https://dl.yarnpkg.com/debian/ stable main" | tee /etc/apt/sources.list.d/yarn.list - -RUN apt-get update && apt-get install -y nodejs yarn +RUN apt-get update && apt-get install -y nodejs yarn && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* # Install openstreetmap-cgimap requiriments -RUN apt-get update && apt-get -y install libxml2-dev libpqxx-dev libfcgi-dev zlib1g-dev \ +RUN apt-get update && apt-get -y install libpqxx-dev libfcgi-dev zlib1g-dev \ libboost-dev libboost-program-options-dev libfmt-dev \ libmemcached-dev libcrypto++-dev libargon2-dev libyajl-dev \ - build-essential automake autoconf libtool + automake autoconf libtool && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* # Install cgimap ENV cgimap /tmp/openstreetmap-cgimap @@ -43,7 +46,6 @@ RUN npm install -g svgo # Install openstreetmap-website RUN rm -rf $workdir/html - ENV OPENHISTORICALMAP_WEBSITE_GITSHA=284909db296078187f0338149bc9e7667e5a01c9 RUN git clone -b staging https://github.com/OpenHistoricalMap/ohm-website.git $workdir WORKDIR $workdir @@ -56,10 +58,6 @@ RUN git fetch # Install Ruby packages RUN gem install bundler && bundle install -# update vendored iD -# RUN rm -rf vendor/assets/iD -# RUN vendorer - # Configure database.yml and secrets.yml RUN cp $workdir/config/example.database.yml $workdir/config/database.yml RUN touch $workdir/config/settings.local.yml @@ -99,12 +97,6 @@ RUN /usr/sbin/passenger-memory-stats RUN a2enmod proxy proxy_http rewrite lbmethod_byrequests # Config the virtual host apache2 -# ADD config/cgimap.conf /tmp/ -# RUN sed -e 's/RewriteRule ^(.*)/#RewriteRule ^(.*)/' \ -# -e 's/\/var\/www/\/var\/www\/public/g' \ -# /tmp/cgimap.conf > /etc/apache2/sites-available/cgimap.conf -# RUN chmod 644 /etc/apache2/sites-available/cgimap.conf -# RUN a2ensite cgimap RUN apache2ctl configtest # Set Permissions for www-data From d7f638cdda5c98e9445fec022b9f16af2ddc2bc0 Mon Sep 17 00:00:00 2001 From: Rub21 Date: Mon, 11 Sep 2023 09:26:15 -0500 Subject: [PATCH 02/16] Add osmcha containers --- .github/workflows/chartpress.yaml | 7 ++- ohm/requirements.yaml | 2 +- values.staging.template.yaml | 80 +++++++++++++++++++++++++++++++ 3 files changed, 87 insertions(+), 2 deletions(-) diff --git a/.github/workflows/chartpress.yaml b/.github/workflows/chartpress.yaml index 48ea0465..594daf78 100644 --- a/.github/workflows/chartpress.yaml +++ b/.github/workflows/chartpress.yaml @@ -75,7 +75,9 @@ jobs: STAGING_OVERPASS_API_DB_EBS: ${{ secrets.STAGING_OVERPASS_API_DB_EBS }} STAGING_NEW_RELIC_LICENSE_KEY: ${{ secrets.STAGING_NEW_RELIC_LICENSE_KEY }} STAGING_NEW_RELIC_APP_NAME: ${{ secrets.STAGING_NEW_RELIC_APP_NAME }} - + STAGING_OSMCHA_PG_USER: ${{ secrets.STAGING_OSMCHA_PG_USER }} + STAGING_OSMCHA_PG_PASSWORD: ${{ secrets.STAGING_OSMCHA_PG_PASSWORD }} + STAGING_OSMCHA_PG_DATABASE: ${{ secrets.STAGING_OSMCHA_PG_DATABASE }} - name: Production - substitute secrets if: github.ref == 'refs/heads/main' uses: bluwy/substitute-string-action@v1 @@ -124,6 +126,9 @@ jobs: PRODUCTION_OVERPASS_API_DB_EBS: ${{ secrets.PRODUCTION_OVERPASS_API_DB_EBS }} PRODUCTION_NEW_RELIC_LICENSE_KEY: ${{ secrets.PRODUCTION_NEW_RELIC_LICENSE_KEY }} PRODUCTION_NEW_RELIC_APP_NAME: ${{ secrets.PRODUCTION_NEW_RELIC_APP_NAME }} + PRODUCTION_OSMCHA_PG_USER: ${{ secrets.PRODUCTION_OSMCHA_PG_USER }} + PRODUCTION_OSMCHA_PG_PASSWORD: ${{ secrets.PRODUCTION_OSMCHA_PG_PASSWORD }} + PRODUCTION_OSMCHA_PG_DATABASE: ${{ secrets.PRODUCTION_OSMCHA_PG_DATABASE }} - name: AWS Credentials if: github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/main' diff --git a/ohm/requirements.yaml b/ohm/requirements.yaml index 5e6657d5..af0ba086 100644 --- a/ohm/requirements.yaml +++ b/ohm/requirements.yaml @@ -1,4 +1,4 @@ dependencies: - name: osm-seed - version: '0.1.0-n770.h3ffd454' + version: '0.1.0-n772.hb720b2f' repository: https://devseed.com/osm-seed-chart/ diff --git a/values.staging.template.yaml b/values.staging.template.yaml index 47ab1acb..26e6959a 100644 --- a/values.staging.template.yaml +++ b/values.staging.template.yaml @@ -696,3 +696,83 @@ osm-seed: cpu: '10' nodeSelector: enabled: false + + + # ==================================================================================================== + # Variables for changeset-replication-job, Configuration to create the replication files by, minute, hour, or day + # ==================================================================================================== + osmchaWeb: + enabled: false + image: + name: '' + tag: '' + env: + DJANGO_CACHES: "{'default': {'BACKEND': 'django.core.cache.backends.memcached.MemcachedCache', 'LOCATION': '127.0.0.1:11211'}}" + DJANGO_DEBUG: "False" + DJANGO_SECRET_KEY: "123456789" + DJANGO_SECURE_BROWSER_XSS_FILTER: "True" + DJANGO_SECURE_SSL_REDIRECT: "True" + DJANGO_SECURE_CONTENT_TYPE_NOSNIFF: "True" + DJANGO_SECURE_FRAME_DENY: "True" + DJANGO_SECURE_HSTS_INCLUDE_SUBDOMAINS: "True" + DJANGO_SESSION_COOKIE_HTTPONLY: "True" + DJANGO_SESSION_COOKIE_SECURE: "True" + DJANGO_DEFAULT_FROM_EMAIL: "webmaster@openhistoricalmap.org" + DJANGO_SERVER_EMAIL: "server@openhistoricalmap.org" + DJANGO_EMAIL_SUBJECT_PREFIX: "osmcha" + DJANGO_CHANGESETS_FILTER: "{'some_filter_key': 'some_filter_value'}" + # OAUTH_OSM_KEY: "your_osm_oauth_key" + # OAUTH_OSM_SECRET: "your_osm_oauth_secret" + DJANGO_ANON_USER_THROTTLE_RATE: "10/h" + DJANGO_COMMON_USER_THROTTLE_RATE: "100/h" + DJANGO_NON_STAFF_USER_THROTTLE_RATE: "50/h" + OAUTH_REDIRECT_URI: "https://staging.openhistoricalmap.org/oauth/callback" + OSMCHA_FRONTEND_VERSION: "v1.0.0" + DJANGO_ENABLE_CHANGESET_COMMENTS: "True" + DJANGO_OSM_COMMENTS_API_KEY: "test" + resources: + enabled: false + requests: + memory: '20Gi' + cpu: '8' + limits: + memory: '24Gi' + cpu: '10' + nodeSelector: + enabled: false + + # ==================================================================================================== + # Variables for changeset-replication-job, Configuration to create the replication files by, minute, hour, or day + # ==================================================================================================== + osmchaDb: + enabled: true + image: + name: '' + tag: '' + env: + POSTGRES_DB: {{STAGING_OSMCHA_PG_DATABASE}} + POSTGRES_USER: {{STAGING_OSMCHA_PG_USER}} + POSTGRES_PASSWORD: {{STAGING_OSMCHA_PG_PASSWORD}} + resources: + enabled: false + requests: + memory: '20Gi' + cpu: '8' + limits: + memory: '24Gi' + cpu: '10' + persistenceDisk: + enabled: true + accessMode: ReadWriteOnce + mountPath: /var/lib/postgresql/data + # Minikube + localVolumeHostPath: /mnt/db-data/osmcha-data + localVolumeSize: 10Gi + # AWS + AWS_ElasticBlockStore_volumeID: vol-0462ddf32e5b456df + AWS_ElasticBlockStore_size: 100Gi + # GCP + GCP_gcePersistentDisk_pdName: osmseed-osmcha-disk--v1 + GCP_gcePersistentDisk_size: 50Gi + nodeSelector: + enabled: false From a8ce7f6a962f1c89839595cd387316a51771aa9d Mon Sep 17 00:00:00 2001 From: Rub21 Date: Mon, 11 Sep 2023 10:14:16 -0500 Subject: [PATCH 03/16] Update image --- values.staging.template.yaml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/values.staging.template.yaml b/values.staging.template.yaml index 26e6959a..4b2cddeb 100644 --- a/values.staging.template.yaml +++ b/values.staging.template.yaml @@ -703,9 +703,6 @@ osm-seed: # ==================================================================================================== osmchaWeb: enabled: false - image: - name: '' - tag: '' env: DJANGO_CACHES: "{'default': {'BACKEND': 'django.core.cache.backends.memcached.MemcachedCache', 'LOCATION': '127.0.0.1:11211'}}" DJANGO_DEBUG: "False" @@ -746,9 +743,6 @@ osm-seed: # ==================================================================================================== osmchaDb: enabled: true - image: - name: '' - tag: '' env: POSTGRES_DB: {{STAGING_OSMCHA_PG_DATABASE}} POSTGRES_USER: {{STAGING_OSMCHA_PG_USER}} From 70ef187c93f9681cb15440ac4b772ca47ebaabad Mon Sep 17 00:00:00 2001 From: Rub21 Date: Mon, 11 Sep 2023 10:34:06 -0500 Subject: [PATCH 04/16] Enable osmcha Update apache rules for CORS Set CORS headers Set CORS option for cgimap Set CORS headers --- images/web/Dockerfile | 2 ++ images/web/config/lighttpd.conf | 7 ++++++- images/web/config/production.conf | 28 +++++++++++++++++++--------- values.staging.template.yaml | 2 +- 4 files changed, 28 insertions(+), 11 deletions(-) diff --git a/images/web/Dockerfile b/images/web/Dockerfile index a7f44cfa..0fe062b4 100644 --- a/images/web/Dockerfile +++ b/images/web/Dockerfile @@ -88,6 +88,8 @@ RUN apt-get update && apt-get install -y libapache2-mod-passenger lighttpd # Enable the Passenger Apache module and restart Apache RUN echo "ServerName $(cat /etc/hostname)" >> /etc/apache2/apache2.conf RUN a2enmod passenger +RUN a2enmod headers +RUN a2enmod setenvif # Check installation RUN /usr/bin/passenger-config validate-install diff --git a/images/web/config/lighttpd.conf b/images/web/config/lighttpd.conf index 85520b80..3e11ed9a 100644 --- a/images/web/config/lighttpd.conf +++ b/images/web/config/lighttpd.conf @@ -7,7 +7,12 @@ server.port = 81 server.reject-expect-100-with-417 = "disable" # selecting modules -server.modules = ( "mod_access", "mod_rewrite", "mod_fastcgi", "mod_proxy", "mod_alias" ) +server.modules = ( "mod_access", "mod_rewrite", "mod_fastcgi", "mod_proxy", "mod_alias", "mod_setenv" ) # Added mod_setenv + +# Add CORS headers to all responses +setenv.add-response-header = ( "Access-Control-Allow-Origin" => "*", + "Access-Control-Allow-Methods" => "GET, POST, PUT, OPTIONS", + "Access-Control-Allow-Headers" => "origin, x-requested-with, content-type" ) # handling unknown routes server.error-handler-404 = "/dispatch.map" diff --git a/images/web/config/production.conf b/images/web/config/production.conf index 9920645f..79db2d19 100644 --- a/images/web/config/production.conf +++ b/images/web/config/production.conf @@ -1,40 +1,50 @@ # ServerName localhost + # Tell Apache and Passenger where your app's 'public' directory is DocumentRoot /var/www/public PassengerRuby /usr/bin/ruby + RewriteEngine On - RewriteCond %{HTTP:X-Forwarded-Proto} =http - # Development mode in case domain is localhost + + # Development mode in case domain is localhost # Rewrite to HTTPS + RewriteCond %{HTTP:X-Forwarded-Proto} =http RewriteCond %{HTTP_HOST} !=localhost RewriteCond %{HTTPS} off RewriteRule .* https://%{HTTP_HOST}%{REQUEST_URI} [L,R=301] + # Rewrite to www. [NC] is a case-insensitive match RewriteCond %{HTTP_HOST} !=localhost - # Avoid redirect staging sites RewriteCond %{HTTP_HOST} !=staging.openhistoricalmap.org RewriteCond %{HTTP_HOST} !^www\. [NC] RewriteRule .* https://www.%{HTTP_HOST}%{REQUEST_URI} [L,R=301] - # ======Redirect trafict to CGIMAP==== + # ======Redirect traffic to CGIMAP==== # Rules to redirect to port 81 RewriteCond %{REQUEST_URI} ^/api/0\.6/map RewriteRule ^.*$ http://127.0.0.1:81%{REQUEST_URI} [P,L] + # Existing rewrites RewriteRule ^0\.6/map$ - [L,H=fcgi:127.0.0.1:81] RewriteRule ^0\.6/(nodes|ways|relations)$ - [L,H=fcgi:127.0.0.1:81] RewriteRule ^0\.6/(way|relation)/([^/]+)/full$ - [L,H=fcgi:127.0.0.1:81] - + # If no match found, fall through to port 80 RewriteCond %{REQUEST_METHOD} ^(GET|HEAD)$ RewriteRule ^0\.6/(node|way|relation)/([^/]+)$ - [L,H=fcgi:127.0.0.1:80] + # Set CORS headers for any request within openhistoricalmap.org + Header set Access-Control-Allow-Origin "*" + Header set Access-Control-Allow-Methods "GET, POST, OPTIONS" + Header set Access-Control-Allow-Headers "DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range" + Header set Access-Control-Expose-Headers "Content-Length,Content-Range" + # Relax Apache security settings - AllowOverride None - Allow from all - Options -MultiViews + AllowOverride None + Require all granted + # Allow from all + Options -MultiViews - diff --git a/values.staging.template.yaml b/values.staging.template.yaml index 4b2cddeb..3bf59c4f 100644 --- a/values.staging.template.yaml +++ b/values.staging.template.yaml @@ -742,7 +742,7 @@ osm-seed: # Variables for changeset-replication-job, Configuration to create the replication files by, minute, hour, or day # ==================================================================================================== osmchaDb: - enabled: true + enabled: false env: POSTGRES_DB: {{STAGING_OSMCHA_PG_DATABASE}} POSTGRES_USER: {{STAGING_OSMCHA_PG_USER}} From 1e6a378d6bf1e098a1a3340987f41f542b9784d9 Mon Sep 17 00:00:00 2001 From: Rub21 Date: Tue, 12 Sep 2023 09:42:58 -0500 Subject: [PATCH 05/16] Add osmch containers Update osm-seed version Update osmcha version Update osm-seed version --- .github/workflows/chartpress.yaml | 19 +++++++------------ ohm/requirements.yaml | 2 +- values.staging.template.yaml | 20 ++++++++++++-------- 3 files changed, 20 insertions(+), 21 deletions(-) diff --git a/.github/workflows/chartpress.yaml b/.github/workflows/chartpress.yaml index 594daf78..0bc85af1 100644 --- a/.github/workflows/chartpress.yaml +++ b/.github/workflows/chartpress.yaml @@ -78,6 +78,9 @@ jobs: STAGING_OSMCHA_PG_USER: ${{ secrets.STAGING_OSMCHA_PG_USER }} STAGING_OSMCHA_PG_PASSWORD: ${{ secrets.STAGING_OSMCHA_PG_PASSWORD }} STAGING_OSMCHA_PG_DATABASE: ${{ secrets.STAGING_OSMCHA_PG_DATABASE }} + STAGING_OSMCHA_CONSUMER_KEY: ${{ secrets.STAGING_OSMCHA_CONSUMER_KEY }} + STAGING_OSMCHA_CONSUMER_SECRET: ${{ secrets.STAGING_OSMCHA_CONSUMER_SECRET }} + STAGING_OSMCHA_DJANGO_SECRET_KEY: ${{ secrets.STAGING_OSMCHA_DJANGO_SECRET_KEY }} - name: Production - substitute secrets if: github.ref == 'refs/heads/main' uses: bluwy/substitute-string-action@v1 @@ -129,7 +132,9 @@ jobs: PRODUCTION_OSMCHA_PG_USER: ${{ secrets.PRODUCTION_OSMCHA_PG_USER }} PRODUCTION_OSMCHA_PG_PASSWORD: ${{ secrets.PRODUCTION_OSMCHA_PG_PASSWORD }} PRODUCTION_OSMCHA_PG_DATABASE: ${{ secrets.PRODUCTION_OSMCHA_PG_DATABASE }} - + PRODUCTION_OSMCHA_CONSUMER_KEY: ${{ secrets.PRODUCTION_OSMCHA_CONSUMER_KEY }} + PRODUCTION_OSMCHA_CONSUMER_SECRET: ${{ secrets.PRODUCTION_OSMCHA_CONSUMER_SECRET }} + PRODUCTION_OSMCHA_DJANGO_SECRET_KEY: ${{ secrets.PRODUCTION_OSMCHA_DJANGO_SECRET_KEY }} - name: AWS Credentials if: github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/main' uses: aws-actions/configure-aws-credentials@v1 @@ -140,17 +145,7 @@ jobs: - name: Setup Kubectl and Helm Dependencies if: github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/main' - run: | - sudo pip install awscli --ignore-installed six - sudo curl -L -o /usr/bin/kubectl https://amazon-eks.s3.us-west-2.amazonaws.com/1.17.7/2020-07-08/bin/linux/amd64/kubectl - sudo chmod +x /usr/bin/kubectl - sudo curl -o /usr/bin/aws-iam-authenticator https://amazon-eks.s3.us-west-2.amazonaws.com/1.17.7/2020-07-08/bin/linux/amd64/aws-iam-authenticator - sudo chmod +x /usr/bin/aws-iam-authenticator - wget https://get.helm.sh/helm-v3.5.0-linux-amd64.tar.gz -O helm.tar.gz - tar -xvzf helm.tar.gz - sudo mv linux-amd64/helm /usr/local/bin/ - sudo chmod +x /usr/local/bin/helm - + run: "sudo pip install awscli --ignore-installed six\nsudo curl -L -o /usr/bin/kubectl https://amazon-eks.s3.us-west-2.amazonaws.com/1.17.7/2020-07-08/bin/linux/amd64/kubectl\nsudo chmod +x /usr/bin/kubectl\nsudo curl -o /usr/bin/aws-iam-authenticator https://amazon-eks.s3.us-west-2.amazonaws.com/1.17.7/2020-07-08/bin/linux/amd64/aws-iam-authenticator\nsudo chmod +x /usr/bin/aws-iam-authenticator\nwget https://get.helm.sh/helm-v3.5.0-linux-amd64.tar.gz -O helm.tar.gz\ntar -xvzf helm.tar.gz\nsudo mv linux-amd64/helm /usr/local/bin/ \nsudo chmod +x /usr/local/bin/helm\n #magic___^_^___line\n" - name: Update kube-config staging if: github.ref == 'refs/heads/staging' run: aws eks --region us-east-1 update-kubeconfig --name osmseed-staging diff --git a/ohm/requirements.yaml b/ohm/requirements.yaml index af0ba086..f1a26610 100644 --- a/ohm/requirements.yaml +++ b/ohm/requirements.yaml @@ -1,4 +1,4 @@ dependencies: - name: osm-seed - version: '0.1.0-n772.hb720b2f' + version: '0.1.0-n779.h4437d4d' repository: https://devseed.com/osm-seed-chart/ diff --git a/values.staging.template.yaml b/values.staging.template.yaml index 3bf59c4f..f5b534aa 100644 --- a/values.staging.template.yaml +++ b/values.staging.template.yaml @@ -702,11 +702,12 @@ osm-seed: # Variables for changeset-replication-job, Configuration to create the replication files by, minute, hour, or day # ==================================================================================================== osmchaWeb: - enabled: false + enabled: true env: + OSM_SERVER_URL: 'https://staging.openhistoricalmap.org' DJANGO_CACHES: "{'default': {'BACKEND': 'django.core.cache.backends.memcached.MemcachedCache', 'LOCATION': '127.0.0.1:11211'}}" DJANGO_DEBUG: "False" - DJANGO_SECRET_KEY: "123456789" + DJANGO_SECRET_KEY: {{STAGING_OSMCHA_DJANGO_SECRET_KEY}} DJANGO_SECURE_BROWSER_XSS_FILTER: "True" DJANGO_SECURE_SSL_REDIRECT: "True" DJANGO_SECURE_CONTENT_TYPE_NOSNIFF: "True" @@ -718,15 +719,18 @@ osm-seed: DJANGO_SERVER_EMAIL: "server@openhistoricalmap.org" DJANGO_EMAIL_SUBJECT_PREFIX: "osmcha" DJANGO_CHANGESETS_FILTER: "{'some_filter_key': 'some_filter_value'}" - # OAUTH_OSM_KEY: "your_osm_oauth_key" - # OAUTH_OSM_SECRET: "your_osm_oauth_secret" + OAUTH_OSM_KEY: {{STAGING_OSMCHA_CONSUMER_KEY}} + OAUTH_OSM_SECRET: {{STAGING_OSMCHA_CONSUMER_SECRET}} DJANGO_ANON_USER_THROTTLE_RATE: "10/h" DJANGO_COMMON_USER_THROTTLE_RATE: "100/h" DJANGO_NON_STAFF_USER_THROTTLE_RATE: "50/h" - OAUTH_REDIRECT_URI: "https://staging.openhistoricalmap.org/oauth/callback" - OSMCHA_FRONTEND_VERSION: "v1.0.0" + OAUTH_REDIRECT_URI: https://osmcha.openhistoricalmap.org/oauth-landing.html DJANGO_ENABLE_CHANGESET_COMMENTS: "True" DJANGO_OSM_COMMENTS_API_KEY: "test" + OSMCHA_FRONTEND_VERSION: "v0.86.0-production" + DJANGO_ROOT: /app + DJANGO_SETTINGS_MODULE: config.settings.local + C_FORCE_ROOT: "False" resources: enabled: false requests: @@ -742,7 +746,7 @@ osm-seed: # Variables for changeset-replication-job, Configuration to create the replication files by, minute, hour, or day # ==================================================================================================== osmchaDb: - enabled: false + enabled: true env: POSTGRES_DB: {{STAGING_OSMCHA_PG_DATABASE}} POSTGRES_USER: {{STAGING_OSMCHA_PG_USER}} @@ -756,7 +760,7 @@ osm-seed: memory: '24Gi' cpu: '10' persistenceDisk: - enabled: true + enabled: false accessMode: ReadWriteOnce mountPath: /var/lib/postgresql/data # Minikube From 85094e067f593adfdcb4500083e22682f9993215 Mon Sep 17 00:00:00 2001 From: Rub21 Date: Tue, 12 Sep 2023 15:15:59 -0500 Subject: [PATCH 06/16] Update osm-seed version Update osm-seed version Update osm-seed version Update RewriteRules for web container Fix RewriteRule --- images/web/config/production.conf | 29 +++++++++-------------------- ohm/requirements.yaml | 2 +- values.staging.template.yaml | 2 ++ 3 files changed, 12 insertions(+), 21 deletions(-) diff --git a/images/web/config/production.conf b/images/web/config/production.conf index 79db2d19..0f2b4e33 100644 --- a/images/web/config/production.conf +++ b/images/web/config/production.conf @@ -1,50 +1,39 @@ # ServerName localhost - # Tell Apache and Passenger where your app's 'public' directory is DocumentRoot /var/www/public PassengerRuby /usr/bin/ruby - RewriteEngine On - - # Development mode in case domain is localhost - # Rewrite to HTTPS RewriteCond %{HTTP:X-Forwarded-Proto} =http + # Development mode in case domain is localhost + # Rewrite to HTTPS RewriteCond %{HTTP_HOST} !=localhost RewriteCond %{HTTPS} off RewriteRule .* https://%{HTTP_HOST}%{REQUEST_URI} [L,R=301] - # Rewrite to www. [NC] is a case-insensitive match RewriteCond %{HTTP_HOST} !=localhost + # Avoid redirect staging sites RewriteCond %{HTTP_HOST} !=staging.openhistoricalmap.org RewriteCond %{HTTP_HOST} !^www\. [NC] RewriteRule .* https://www.%{HTTP_HOST}%{REQUEST_URI} [L,R=301] - # ======Redirect traffic to CGIMAP==== + # ======Redirect trafict to CGIMAP==== # Rules to redirect to port 81 RewriteCond %{REQUEST_URI} ^/api/0\.6/map - RewriteRule ^.*$ http://127.0.0.1:81%{REQUEST_URI} [P,L] - + RewriteRule ^.*$ http://127.0.0.1:80%{REQUEST_URI} [P,L] # Existing rewrites RewriteRule ^0\.6/map$ - [L,H=fcgi:127.0.0.1:81] RewriteRule ^0\.6/(nodes|ways|relations)$ - [L,H=fcgi:127.0.0.1:81] RewriteRule ^0\.6/(way|relation)/([^/]+)/full$ - [L,H=fcgi:127.0.0.1:81] - + # If no match found, fall through to port 80 RewriteCond %{REQUEST_METHOD} ^(GET|HEAD)$ RewriteRule ^0\.6/(node|way|relation)/([^/]+)$ - [L,H=fcgi:127.0.0.1:80] - # Set CORS headers for any request within openhistoricalmap.org - Header set Access-Control-Allow-Origin "*" - Header set Access-Control-Allow-Methods "GET, POST, OPTIONS" - Header set Access-Control-Allow-Headers "DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range" - Header set Access-Control-Expose-Headers "Content-Length,Content-Range" - # Relax Apache security settings - AllowOverride None - Require all granted - # Allow from all - Options -MultiViews + AllowOverride None + Allow from all + Options -MultiViews diff --git a/ohm/requirements.yaml b/ohm/requirements.yaml index f1a26610..6d5c47bf 100644 --- a/ohm/requirements.yaml +++ b/ohm/requirements.yaml @@ -1,4 +1,4 @@ dependencies: - name: osm-seed - version: '0.1.0-n779.h4437d4d' + version: '0.1.0-n783.ha5dabc8' repository: https://devseed.com/osm-seed-chart/ diff --git a/values.staging.template.yaml b/values.staging.template.yaml index f5b534aa..aa9649d5 100644 --- a/values.staging.template.yaml +++ b/values.staging.template.yaml @@ -131,6 +131,8 @@ osm-seed: minReplicas: 2 maxReplicas: 10 cpuUtilization: 80 + cgimap: + enabled: true # ==================================================================================================== # Variables for memcached. Memcached is used to store session cookies # ==================================================================================================== From 85fd5b950200c57af67e2cd44292704116c59b88 Mon Sep 17 00:00:00 2001 From: Rub21 Date: Tue, 12 Sep 2023 22:11:22 -0500 Subject: [PATCH 07/16] Set port 81 --- images/web/config/production.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/images/web/config/production.conf b/images/web/config/production.conf index 0f2b4e33..5ef5619b 100644 --- a/images/web/config/production.conf +++ b/images/web/config/production.conf @@ -20,7 +20,7 @@ # ======Redirect trafict to CGIMAP==== # Rules to redirect to port 81 RewriteCond %{REQUEST_URI} ^/api/0\.6/map - RewriteRule ^.*$ http://127.0.0.1:80%{REQUEST_URI} [P,L] + RewriteRule ^.*$ http://127.0.0.1:81%{REQUEST_URI} [P,L] # Existing rewrites RewriteRule ^0\.6/map$ - [L,H=fcgi:127.0.0.1:81] RewriteRule ^0\.6/(nodes|ways|relations)$ - [L,H=fcgi:127.0.0.1:81] From ae0a14caa52ffc4a015d74ae88d45eb3faaf35aa Mon Sep 17 00:00:00 2001 From: Rub21 Date: Wed, 13 Sep 2023 09:59:09 -0500 Subject: [PATCH 08/16] Handle preflight requests --- images/web/config/production.conf | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/images/web/config/production.conf b/images/web/config/production.conf index 5ef5619b..b2ed3717 100644 --- a/images/web/config/production.conf +++ b/images/web/config/production.conf @@ -3,30 +3,38 @@ # Tell Apache and Passenger where your app's 'public' directory is DocumentRoot /var/www/public PassengerRuby /usr/bin/ruby + + # CORS headers + Header set Access-Control-Allow-Origin "*" + Header set Access-Control-Allow-Methods "GET, POST, OPTIONS" + Header set Access-Control-Allow-Headers "Origin, X-Requested-With, Content-Type, Accept" + RewriteEngine On + + # Handle preflight requests + RewriteCond %{REQUEST_METHOD} OPTIONS + RewriteRule ^(.*)$ $1 [R=200,L,E=HTTP_ACCESS_CONTROL_ALLOW_HEADERS: Origin, X-Requested-With, Content-Type, Accept] + + # Avoid redirects for preflight + RewriteCond %{REQUEST_METHOD} !OPTIONS + + # Existing rewrites for HTTPS and www RewriteCond %{HTTP:X-Forwarded-Proto} =http - # Development mode in case domain is localhost - # Rewrite to HTTPS RewriteCond %{HTTP_HOST} !=localhost RewriteCond %{HTTPS} off RewriteRule .* https://%{HTTP_HOST}%{REQUEST_URI} [L,R=301] - # Rewrite to www. [NC] is a case-insensitive match + RewriteCond %{HTTP_HOST} !=localhost - # Avoid redirect staging sites RewriteCond %{HTTP_HOST} !=staging.openhistoricalmap.org RewriteCond %{HTTP_HOST} !^www\. [NC] RewriteRule .* https://www.%{HTTP_HOST}%{REQUEST_URI} [L,R=301] - # ======Redirect trafict to CGIMAP==== - # Rules to redirect to port 81 + # Existing rewrites for CGIMAP RewriteCond %{REQUEST_URI} ^/api/0\.6/map RewriteRule ^.*$ http://127.0.0.1:81%{REQUEST_URI} [P,L] - # Existing rewrites RewriteRule ^0\.6/map$ - [L,H=fcgi:127.0.0.1:81] RewriteRule ^0\.6/(nodes|ways|relations)$ - [L,H=fcgi:127.0.0.1:81] RewriteRule ^0\.6/(way|relation)/([^/]+)/full$ - [L,H=fcgi:127.0.0.1:81] - - # If no match found, fall through to port 80 RewriteCond %{REQUEST_METHOD} ^(GET|HEAD)$ RewriteRule ^0\.6/(node|way|relation)/([^/]+)$ - [L,H=fcgi:127.0.0.1:80] From c16fadb0143af2dbd6619e6485ad568d449a7c53 Mon Sep 17 00:00:00 2001 From: Rub21 Date: Wed, 13 Sep 2023 11:43:23 -0500 Subject: [PATCH 09/16] Handle preflight requests Update headers config Reset apache config --- images/web/Dockerfile | 22 +++++++++++----------- images/web/config/production.conf | 26 +++++++++----------------- 2 files changed, 20 insertions(+), 28 deletions(-) diff --git a/images/web/Dockerfile b/images/web/Dockerfile index 0fe062b4..f69a787c 100644 --- a/images/web/Dockerfile +++ b/images/web/Dockerfile @@ -31,15 +31,15 @@ RUN apt-get update && apt-get -y install libpqxx-dev libfcgi-dev zlib1g-dev \ rm -rf /var/lib/apt/lists/* # Install cgimap -ENV cgimap /tmp/openstreetmap-cgimap -ENV CGIMAP_GITSHA=5cd3d21bebe9d205828608be4c65bbda8b464308 -RUN git clone -b master https://github.com/zerebubuth/openstreetmap-cgimap.git $cgimap -RUN cd $cgimap && git checkout $CGIMAP_GITSHA -RUN cd $cgimap && \ - ./autogen.sh && \ - ./configure && \ - make && \ - make install +# ENV cgimap /tmp/openstreetmap-cgimap +# ENV CGIMAP_GITSHA=5cd3d21bebe9d205828608be4c65bbda8b464308 +# RUN git clone -b master https://github.com/zerebubuth/openstreetmap-cgimap.git $cgimap +# RUN cd $cgimap && git checkout $CGIMAP_GITSHA +# RUN cd $cgimap && \ +# ./autogen.sh && \ +# ./configure && \ +# make && \ +# make install # Install svgo required RUN npm install -g svgo @@ -78,6 +78,8 @@ RUN ln -s /tmp /var/www/tmp # Add Apache configuration file ADD config/production.conf /etc/apache2/sites-available/production.conf +RUN a2enmod headers +RUN a2enmod setenvif RUN a2dissite 000-default RUN a2ensite production @@ -88,8 +90,6 @@ RUN apt-get update && apt-get install -y libapache2-mod-passenger lighttpd # Enable the Passenger Apache module and restart Apache RUN echo "ServerName $(cat /etc/hostname)" >> /etc/apache2/apache2.conf RUN a2enmod passenger -RUN a2enmod headers -RUN a2enmod setenvif # Check installation RUN /usr/bin/passenger-config validate-install diff --git a/images/web/config/production.conf b/images/web/config/production.conf index b2ed3717..5ef5619b 100644 --- a/images/web/config/production.conf +++ b/images/web/config/production.conf @@ -3,38 +3,30 @@ # Tell Apache and Passenger where your app's 'public' directory is DocumentRoot /var/www/public PassengerRuby /usr/bin/ruby - - # CORS headers - Header set Access-Control-Allow-Origin "*" - Header set Access-Control-Allow-Methods "GET, POST, OPTIONS" - Header set Access-Control-Allow-Headers "Origin, X-Requested-With, Content-Type, Accept" - RewriteEngine On - - # Handle preflight requests - RewriteCond %{REQUEST_METHOD} OPTIONS - RewriteRule ^(.*)$ $1 [R=200,L,E=HTTP_ACCESS_CONTROL_ALLOW_HEADERS: Origin, X-Requested-With, Content-Type, Accept] - - # Avoid redirects for preflight - RewriteCond %{REQUEST_METHOD} !OPTIONS - - # Existing rewrites for HTTPS and www RewriteCond %{HTTP:X-Forwarded-Proto} =http + # Development mode in case domain is localhost + # Rewrite to HTTPS RewriteCond %{HTTP_HOST} !=localhost RewriteCond %{HTTPS} off RewriteRule .* https://%{HTTP_HOST}%{REQUEST_URI} [L,R=301] - + # Rewrite to www. [NC] is a case-insensitive match RewriteCond %{HTTP_HOST} !=localhost + # Avoid redirect staging sites RewriteCond %{HTTP_HOST} !=staging.openhistoricalmap.org RewriteCond %{HTTP_HOST} !^www\. [NC] RewriteRule .* https://www.%{HTTP_HOST}%{REQUEST_URI} [L,R=301] - # Existing rewrites for CGIMAP + # ======Redirect trafict to CGIMAP==== + # Rules to redirect to port 81 RewriteCond %{REQUEST_URI} ^/api/0\.6/map RewriteRule ^.*$ http://127.0.0.1:81%{REQUEST_URI} [P,L] + # Existing rewrites RewriteRule ^0\.6/map$ - [L,H=fcgi:127.0.0.1:81] RewriteRule ^0\.6/(nodes|ways|relations)$ - [L,H=fcgi:127.0.0.1:81] RewriteRule ^0\.6/(way|relation)/([^/]+)/full$ - [L,H=fcgi:127.0.0.1:81] + + # If no match found, fall through to port 80 RewriteCond %{REQUEST_METHOD} ^(GET|HEAD)$ RewriteRule ^0\.6/(node|way|relation)/([^/]+)$ - [L,H=fcgi:127.0.0.1:80] From 83112fc00743a3502a708570de5c6b606414cd94 Mon Sep 17 00:00:00 2001 From: Rub21 Date: Wed, 13 Sep 2023 14:45:44 -0500 Subject: [PATCH 10/16] Update overpass page Update overpass deployment config Update gitsha for overpass --- .github/workflows/frontend-overpass.yaml | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/.github/workflows/frontend-overpass.yaml b/.github/workflows/frontend-overpass.yaml index f9434ba7..c4c00939 100644 --- a/.github/workflows/frontend-overpass.yaml +++ b/.github/workflows/frontend-overpass.yaml @@ -33,30 +33,28 @@ jobs: uses: actions/checkout@v2 with: repository: OpenHistoricalMap/overpass-turbo - ref: 43d1d9aa7e8bbb7152c89354900fdd327f9f7ffe + ref: c9defd0a9b32baf3af6599c3cb4a0dbad96b85e8 # token: ${{env.DEV_GITHUB_TOKEN}} - - name: Use Node.js 12 + - name: Use Node.js 16 uses: actions/setup-node@v1 with: - node-version: 12 + node-version: 16 - name: Install modules run: | rm -rf node_modules - rm package-lock.json npm cache clean --force - npm cache verify - npm install + yarn install --no-progress --frozen-lockfile - name: Build frontend run: | sed -i -e 's/overpass-api.openhistoricalmap.org/'${OVERPASS_API}'/g' js/configs.js - npm run build + yarn build - uses: actions/setup-python@v2 with: - python-version: '3.8' + python-version: '3.10' - name: Install aws cli run: | From 7cd8974e30e8c0136ba31dd9feef53699c862a1c Mon Sep 17 00:00:00 2001 From: Rub21 Date: Wed, 13 Sep 2023 18:04:41 -0500 Subject: [PATCH 11/16] Avoid redirects for preflight Reset apache config Handle CORS Preflight Requests in Lighttpd display logs for lighttpd Add CORS headers for preflight requests --- images/web/config/lighttpd.conf | 19 ++++++++++++++----- images/web/config/production.conf | 23 ++++++++++++++++------- 2 files changed, 30 insertions(+), 12 deletions(-) diff --git a/images/web/config/lighttpd.conf b/images/web/config/lighttpd.conf index 3e11ed9a..c21db904 100644 --- a/images/web/config/lighttpd.conf +++ b/images/web/config/lighttpd.conf @@ -7,12 +7,10 @@ server.port = 81 server.reject-expect-100-with-417 = "disable" # selecting modules -server.modules = ( "mod_access", "mod_rewrite", "mod_fastcgi", "mod_proxy", "mod_alias", "mod_setenv" ) # Added mod_setenv +server.modules = ( "mod_access", "mod_rewrite", "mod_fastcgi", "mod_proxy", "mod_alias" ) -# Add CORS headers to all responses -setenv.add-response-header = ( "Access-Control-Allow-Origin" => "*", - "Access-Control-Allow-Methods" => "GET, POST, PUT, OPTIONS", - "Access-Control-Allow-Headers" => "origin, x-requested-with, content-type" ) +accesslog.filename = "/dev/stdout" +server.errorlog = "/dev/stderr" # handling unknown routes server.error-handler-404 = "/dispatch.map" @@ -29,6 +27,17 @@ mimetype.assign = ( ".png" => "image/png" ) +# Handle CORS Preflight Requests +$HTTP["request-method"] == "OPTIONS" { + setenv.add-response-header = ( + "Access-Control-Allow-Origin" => "*", + "Access-Control-Allow-Methods" => "GET, POST, PUT, DELETE, OPTIONS", + "Access-Control-Allow-Headers" => "origin, x-requested-with, content-type", + "Access-Control-Allow-Credentials" => "true", + "Access-Control-Max-Age" => "86400" + ) +} + #debug.log-request-handling = "enable" $HTTP["request-method"] == "GET" { diff --git a/images/web/config/production.conf b/images/web/config/production.conf index 5ef5619b..df9d03f9 100644 --- a/images/web/config/production.conf +++ b/images/web/config/production.conf @@ -3,25 +3,34 @@ # Tell Apache and Passenger where your app's 'public' directory is DocumentRoot /var/www/public PassengerRuby /usr/bin/ruby + + # Add CORS headers for preflight requests + + Header always set Access-Control-Allow-Origin "*" + Header always set Access-Control-Allow-Methods "POST, GET, OPTIONS, DELETE, PUT" + Header always set Access-Control-Max-Age "1000" + Header always set Access-Control-Allow-Headers "x-requested-with, Content-Type, origin, authorization, accept, client-security-token" + + # Respond to preflight requests + RewriteEngine On + RewriteCond %{REQUEST_METHOD} OPTIONS + RewriteRule ^(.*)$ $1 [R=200,L,E=HTTP_ACCESS_CONTROL_ALLOW_HEADERS:Authorization] + + + # Existing rewrite rules RewriteEngine On RewriteCond %{HTTP:X-Forwarded-Proto} =http - # Development mode in case domain is localhost - # Rewrite to HTTPS RewriteCond %{HTTP_HOST} !=localhost RewriteCond %{HTTPS} off RewriteRule .* https://%{HTTP_HOST}%{REQUEST_URI} [L,R=301] - # Rewrite to www. [NC] is a case-insensitive match RewriteCond %{HTTP_HOST} !=localhost - # Avoid redirect staging sites RewriteCond %{HTTP_HOST} !=staging.openhistoricalmap.org RewriteCond %{HTTP_HOST} !^www\. [NC] RewriteRule .* https://www.%{HTTP_HOST}%{REQUEST_URI} [L,R=301] - # ======Redirect trafict to CGIMAP==== - # Rules to redirect to port 81 + # Redirect traffic to CGIMAP RewriteCond %{REQUEST_URI} ^/api/0\.6/map RewriteRule ^.*$ http://127.0.0.1:81%{REQUEST_URI} [P,L] - # Existing rewrites RewriteRule ^0\.6/map$ - [L,H=fcgi:127.0.0.1:81] RewriteRule ^0\.6/(nodes|ways|relations)$ - [L,H=fcgi:127.0.0.1:81] RewriteRule ^0\.6/(way|relation)/([^/]+)/full$ - [L,H=fcgi:127.0.0.1:81] From 58d63ccc76ed044ee934162f42062e3b30bf426a Mon Sep 17 00:00:00 2001 From: Rub21 Date: Thu, 14 Sep 2023 14:38:44 -0500 Subject: [PATCH 12/16] Fix CORS headers for preflight requests Enable cgimap Redirect to cgimap service Update config Reset apache config Avoid redirect request to cgimap --- images/web/Dockerfile | 18 ++++++++--------- images/web/config/production.conf | 32 +++++-------------------------- 2 files changed, 14 insertions(+), 36 deletions(-) diff --git a/images/web/Dockerfile b/images/web/Dockerfile index f69a787c..96e5c0a5 100644 --- a/images/web/Dockerfile +++ b/images/web/Dockerfile @@ -31,15 +31,15 @@ RUN apt-get update && apt-get -y install libpqxx-dev libfcgi-dev zlib1g-dev \ rm -rf /var/lib/apt/lists/* # Install cgimap -# ENV cgimap /tmp/openstreetmap-cgimap -# ENV CGIMAP_GITSHA=5cd3d21bebe9d205828608be4c65bbda8b464308 -# RUN git clone -b master https://github.com/zerebubuth/openstreetmap-cgimap.git $cgimap -# RUN cd $cgimap && git checkout $CGIMAP_GITSHA -# RUN cd $cgimap && \ -# ./autogen.sh && \ -# ./configure && \ -# make && \ -# make install +ENV cgimap /tmp/openstreetmap-cgimap +ENV CGIMAP_GITSHA=5cd3d21bebe9d205828608be4c65bbda8b464308 +RUN git clone -b master https://github.com/zerebubuth/openstreetmap-cgimap.git $cgimap +RUN cd $cgimap && git checkout $CGIMAP_GITSHA +RUN cd $cgimap && \ + ./autogen.sh && \ + ./configure && \ + make && \ + make install # Install svgo required RUN npm install -g svgo diff --git a/images/web/config/production.conf b/images/web/config/production.conf index df9d03f9..5f6a4619 100644 --- a/images/web/config/production.conf +++ b/images/web/config/production.conf @@ -3,46 +3,24 @@ # Tell Apache and Passenger where your app's 'public' directory is DocumentRoot /var/www/public PassengerRuby /usr/bin/ruby - - # Add CORS headers for preflight requests - - Header always set Access-Control-Allow-Origin "*" - Header always set Access-Control-Allow-Methods "POST, GET, OPTIONS, DELETE, PUT" - Header always set Access-Control-Max-Age "1000" - Header always set Access-Control-Allow-Headers "x-requested-with, Content-Type, origin, authorization, accept, client-security-token" - - # Respond to preflight requests - RewriteEngine On - RewriteCond %{REQUEST_METHOD} OPTIONS - RewriteRule ^(.*)$ $1 [R=200,L,E=HTTP_ACCESS_CONTROL_ALLOW_HEADERS:Authorization] - - - # Existing rewrite rules RewriteEngine On RewriteCond %{HTTP:X-Forwarded-Proto} =http + # Development mode in case domain is localhost + # Rewrite to HTTPS RewriteCond %{HTTP_HOST} !=localhost RewriteCond %{HTTPS} off RewriteRule .* https://%{HTTP_HOST}%{REQUEST_URI} [L,R=301] + # Rewrite to www. [NC] is a case-insensitive match RewriteCond %{HTTP_HOST} !=localhost + # Avoid redirect staging sites RewriteCond %{HTTP_HOST} !=staging.openhistoricalmap.org RewriteCond %{HTTP_HOST} !^www\. [NC] RewriteRule .* https://www.%{HTTP_HOST}%{REQUEST_URI} [L,R=301] - # Redirect traffic to CGIMAP - RewriteCond %{REQUEST_URI} ^/api/0\.6/map - RewriteRule ^.*$ http://127.0.0.1:81%{REQUEST_URI} [P,L] - RewriteRule ^0\.6/map$ - [L,H=fcgi:127.0.0.1:81] - RewriteRule ^0\.6/(nodes|ways|relations)$ - [L,H=fcgi:127.0.0.1:81] - RewriteRule ^0\.6/(way|relation)/([^/]+)/full$ - [L,H=fcgi:127.0.0.1:81] - - # If no match found, fall through to port 80 - RewriteCond %{REQUEST_METHOD} ^(GET|HEAD)$ - RewriteRule ^0\.6/(node|way|relation)/([^/]+)$ - [L,H=fcgi:127.0.0.1:80] - # Relax Apache security settings AllowOverride None Allow from all Options -MultiViews - + \ No newline at end of file From 1052110052e135ca14b02c128722d4f6a476587c Mon Sep 17 00:00:00 2001 From: Rub21 Date: Thu, 14 Sep 2023 20:31:23 -0500 Subject: [PATCH 13/16] Add osmcha(disable) for production Update local dev info Update OHM's subdomain in tasking manager --- README.md | 2 + images/DEVELOPMENT.md | 146 -------------------------------- images/README.md | 2 - values.production.template.yaml | 93 ++++++++++++++++++-- 4 files changed, 88 insertions(+), 155 deletions(-) delete mode 100644 images/DEVELOPMENT.md diff --git a/README.md b/README.md index d1c7436b..40c32d6d 100644 --- a/README.md +++ b/README.md @@ -5,6 +5,8 @@ OpenHistoricalMap deploy based on the osm-seed chart This repo is used for deploying to Staging and Production for the main parts of the stack that runs OpenHistoricalMap.org. +This repo is **not** used for local development. Each part of the stack has its own local dev methods. For the OHM Website, following the documentation at https://github.com/OpenHistoricalMap/ohm-website#docker-for-local-development. + Commits to `main` and `staging` will kick off a Github Actions build process that takes somewhere between 30 and 45 minutes. You can't really test this code locally, which can make it tempting to commit changes directly to `staging` or `main`. Don't do it! Since every commit kicks off a build, it is still best practice to make all changes in a branch so you can review them, or ask someone else to do so, before merging in and kicking off a build. diff --git a/images/DEVELOPMENT.md b/images/DEVELOPMENT.md deleted file mode 100644 index e0160e14..00000000 --- a/images/DEVELOPMENT.md +++ /dev/null @@ -1,146 +0,0 @@ -# Setting up [ohm-website](https://github.com/OpenHistoricalMap/ohm-website) for development mode - -### Step 1: Clone [ohm-website](https://github.com/OpenHistoricalMap/ohm-website) and [ohm-deploy](https://github.com/OpenHistoricalMap/ohm-deploy/) repositories in the same level - -```sh - -git clone https://github.com/OpenHistoricalMap/ohm-website.git && cd ohm-website && git checkout merge-osm-website -cd ../ -git clone https://github.com/OpenHistoricalMap/ohm-deploy.git && cd ohm-deploy && git checkout new_web_version -cd ../ -``` - -### Step 2: - -Replace the `web` section in the file `web.yml` with the following configuration: - -```yaml -web: - image: osmseed-web:v1 - build: - context: ./web - dockerfile: Dockerfile - ports: - - '80:80' - env_file: - - ./.env.example - volumes: - - ./../../ohm-website:/var/www -``` - -### Step 3: - -Edit `ohm-deploy/images/web/Dockerfile` and replace the CMD line as following: - -``` -# CMD sh $workdir/start.sh -CMD ["tail", "-f", "/dev/null"] -``` - -### Step 4: Open a new terminal tab and build and start the containers - -```sh -cd ohm-deploy/images/ -docker compose -f web.yml up --build -``` - -This will show the PostgreSQL server setting up its initial database then becoming ready for connections. - -### Step 5: Creating config files in the container environment - -Run the following CLI under `ohm-deploy/images/` where the `docker-compose.yml` is found. - -The following code will connect to the "web" container which attaches the `../../ohm-website` folder to `/var/www`, so any change in `ohm-website` will be reflected in the container's `/var/www` and vice versa. - -```sh -docker-compose exec web bash -``` - -Once in the container run the following CLI to fill in settings: - -```sh -#### MOST ENV VARIABLES ARE SET IN DOCKER CONFIG e.g. DB, MAILER, ETC. - -export workdir="/var/www" -export RAILS_ENV=production - -#### SETTING UP THE PRODUCTION DATABASE -echo " # Production DB -production: - adapter: postgresql - host: ${POSTGRES_HOST} - database: ${POSTGRES_DB} - username: ${POSTGRES_USER} - password: ${POSTGRES_PASSWORD} - encoding: utf8" >$workdir/config/database.yml - -#### SETTING UP SERVER_URL AND SERVER_PROTOCOL -sed -i -e 's/server_url: "openstreetmap.example.com"/server_url: "'$SERVER_URL'"/g' $workdir/config/settings.yml -sed -i -e 's/server_protocol: "http"/server_protocol: "'$SERVER_PROTOCOL'"/g' $workdir/config/settings.yml - -#### SETTING UP MAIL SENDER -sed -i -e 's/smtp_address: "localhost"/smtp_address: "'$MAILER_ADDRESS'"/g' $workdir/config/settings.yml -sed -i -e 's/smtp_domain: "localhost"/smtp_domain: "'$MAILER_DOMAIN'"/g' $workdir/config/settings.yml -sed -i -e 's/smtp_enable_starttls_auto: false/smtp_enable_starttls_auto: true/g' $workdir/config/settings.yml -sed -i -e 's/smtp_authentication: null/smtp_authentication: "login"/g' $workdir/config/settings.yml -sed -i -e 's/smtp_user_name: null/smtp_user_name: "'$MAILER_USERNAME'"/g' $workdir/config/settings.yml -sed -i -e 's/smtp_password: null/smtp_password: "'$MAILER_PASSWORD'"/g' $workdir/config/settings.yml -sed -i -e 's/openstreetmap@example.com/'$MAILER_FROM'/g' $workdir/config/settings.yml -sed -i -e 's/smtp_port: 25/smtp_port: '$MAILER_PORT'/g' $workdir/config/settings.yml - -#### SET UP ID KEY -sed -i -e 's/#id_key: ""/id_key: "'$OSM_id_key'"/g' $workdir/config/settings.yml - -#### SET NOMINATIM URL -sed -i -e "s@https://nominatim.openstreetmap.org/@$NOMINATIM_URL@g" $workdir/config/settings.yml - -#### CREATE A BLANK LOCAL SETTINGS FILE -touch $workdir/config/settings.local.yml - -### STORAGE CONFIG -cp $workdir/config/example.storage.yml $workdir/config/storage.yml -``` - -### Step 6: Running Rails CLI - -Still within the container, run these commands to install Rake packages and to test everything. - -You will see warnings about pngcrush, jpegtran, and other image format tools; ignore them. - -```sh -bundle exec rails db:migrate - -bundle exec rake yarn:install -yarnpkg --ignore-engines install - -bundle exec rake i18n:js:export - -## asset compilation fails the first time; run yarnpkg after it fails, then try again -bundle exec rake assets:precompile --trace -yarnpkg --ignore-engines install -bundle exec rake assets:precompile --trace - -# bundle exec rake jobs:work -# bundle exec rails test:all -``` - -### Step 6b: Proxy to staging - -If you want to proxy to staging database to more easily test login, editing, etc., then do the following: - -1. Inside the container, set this ENV variable: -`POSTGRES_HOST=host.docker.internal` - -2. In another terminal window, run this proxy command: -`kubectl port-forward staging-db-0 5432:5432` - -Note this assumes you have permissions to access that Kubernetes context. - -### Step 7: Start the web server - -Still within the container: - -``` -## Start server in port 80 -bundle exec rails server -p 80 -``` diff --git a/images/README.md b/images/README.md index 25f0aef2..deca9e56 100644 --- a/images/README.md +++ b/images/README.md @@ -19,5 +19,3 @@ docker-compose exec web bash ``` - Wait couple of minutes and open http://localhost, Follow the [documentation](CONFIGURE.md) for interact with the page in local mode. - -- For development follow the [documentation](DEVELOPMENT.md). diff --git a/values.production.template.yaml b/values.production.template.yaml index 847f3419..bfb59ffe 100644 --- a/values.production.template.yaml +++ b/values.production.template.yaml @@ -120,6 +120,8 @@ osm-seed: minReplicas: 2 maxReplicas: 10 cpuUtilization: 80 + cgimap: + enabled: false # ==================================================================================================== # Variables for memcached. Memcached is used to store session cookies # ==================================================================================================== @@ -470,14 +472,14 @@ osm-seed: POSTGRES_PORT: 5432 TM_ORG_NAME: 'OpenHistoricalMap' TM_ORG_CODE: 'OHM' - TM_ORG_URL: 'openhistoricalmap.org' - TM_ORG_PRIVACY_POLICY_URL: 'openhistoricalmap.org/copyright' + TM_ORG_URL: 'www.openhistoricalmap.org' + TM_ORG_PRIVACY_POLICY_URL: 'www.openhistoricalmap.org/copyright' TM_ORG_GITHUB: 'github.com/openhistoricalmap' - OSM_SERVER_URL: 'https://openhistoricalmap.org' + OSM_SERVER_URL: 'https://www.openhistoricalmap.org' OSM_NOMINATIM_SERVER_URL: 'https://nominatim-api.openhistoricalmap.org' - OSM_REGISTER_URL: 'https://openhistoricalmap.org/user/new' - ID_EDITOR_URL: 'https://openhistoricalmap.org/edit?editor=id' - POTLATCH2_EDITOR_URL: 'https://openhistoricalmap.org/edit?editor=potlatch2' + OSM_REGISTER_URL: 'https://www.openhistoricalmap.org/user/new' + ID_EDITOR_URL: 'https://www.openhistoricalmap.org/edit?editor=id' + POTLATCH2_EDITOR_URL: 'https://www.openhistoricalmap.org/edit?editor=potlatch2' TM_SECRET: {{PRODUCTION_TM_API_SECRET}} TM_CONSUMER_KEY: {{PRODUCTION_TM_API_CONSUMER_KEY}} TM_CONSUMER_SECRET: {{PRODUCTION_TM_API_CONSUMER_SECRET}} @@ -667,4 +669,81 @@ osm-seed: memory: '24Gi' cpu: '10' nodeSelector: - enabled: false \ No newline at end of file + enabled: false + + # ==================================================================================================== + # Variables for changeset-replication-job, Configuration to create the replication files by, minute, hour, or day + # ==================================================================================================== + osmchaWeb: + enabled: false + env: + OSM_SERVER_URL: 'https://staging.openhistoricalmap.org' + DJANGO_CACHES: "{'default': {'BACKEND': 'django.core.cache.backends.memcached.MemcachedCache', 'LOCATION': '127.0.0.1:11211'}}" + DJANGO_DEBUG: "False" + DJANGO_SECRET_KEY: {{PRODUCTION_OSMCHA_DJANGO_SECRET_KEY}} + DJANGO_SECURE_BROWSER_XSS_FILTER: "True" + DJANGO_SECURE_SSL_REDIRECT: "True" + DJANGO_SECURE_CONTENT_TYPE_NOSNIFF: "True" + DJANGO_SECURE_FRAME_DENY: "True" + DJANGO_SECURE_HSTS_INCLUDE_SUBDOMAINS: "True" + DJANGO_SESSION_COOKIE_HTTPONLY: "True" + DJANGO_SESSION_COOKIE_SECURE: "True" + DJANGO_DEFAULT_FROM_EMAIL: "webmaster@openhistoricalmap.org" + DJANGO_SERVER_EMAIL: "server@openhistoricalmap.org" + DJANGO_EMAIL_SUBJECT_PREFIX: "osmcha" + DJANGO_CHANGESETS_FILTER: "{'some_filter_key': 'some_filter_value'}" + OAUTH_OSM_KEY: {{PRODUCTION_OSMCHA_CONSUMER_KEY}} + OAUTH_OSM_SECRET: {{PRODUCTION_OSMCHA_CONSUMER_SECRET}} + DJANGO_ANON_USER_THROTTLE_RATE: "10/h" + DJANGO_COMMON_USER_THROTTLE_RATE: "100/h" + DJANGO_NON_STAFF_USER_THROTTLE_RATE: "50/h" + OAUTH_REDIRECT_URI: https://osmcha.openhistoricalmap.org/oauth-landing.html + DJANGO_ENABLE_CHANGESET_COMMENTS: "True" + DJANGO_OSM_COMMENTS_API_KEY: "test" + OSMCHA_FRONTEND_VERSION: "v0.86.0-production" + DJANGO_ROOT: /app + DJANGO_SETTINGS_MODULE: config.settings.local + C_FORCE_ROOT: "False" + resources: + enabled: false + requests: + memory: '20Gi' + cpu: '8' + limits: + memory: '24Gi' + cpu: '10' + nodeSelector: + enabled: false + + # ==================================================================================================== + # Variables for changeset-replication-job, Configuration to create the replication files by, minute, hour, or day + # ==================================================================================================== + osmchaDb: + enabled: false + env: + POSTGRES_DB: {{PRODUCTION_OSMCHA_PG_DATABASE}} + POSTGRES_USER: {{PRODUCTION_OSMCHA_PG_USER}} + POSTGRES_PASSWORD: {{PRODUCTION_OSMCHA_PG_PASSWORD}} + resources: + enabled: false + requests: + memory: '20Gi' + cpu: '8' + limits: + memory: '24Gi' + cpu: '10' + persistenceDisk: + enabled: false + accessMode: ReadWriteOnce + mountPath: /var/lib/postgresql/data + # Minikube + localVolumeHostPath: /mnt/db-data/osmcha-data + localVolumeSize: 10Gi + # AWS + AWS_ElasticBlockStore_volumeID: vol-0462ddf32e5b456df + AWS_ElasticBlockStore_size: 100Gi + # GCP + GCP_gcePersistentDisk_pdName: osmseed-osmcha-disk--v1 + GCP_gcePersistentDisk_size: 50Gi + nodeSelector: + enabled: false From f969c25dc324777d8d88676fd38dbec24e596069 Mon Sep 17 00:00:00 2001 From: Rub21 Date: Thu, 14 Sep 2023 11:48:57 -0500 Subject: [PATCH 14/16] Update OSM_SERVER_URL for production --- .github/workflows/frontend-tasking-manager.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/frontend-tasking-manager.yaml b/.github/workflows/frontend-tasking-manager.yaml index f5dbdc5a..98797bf5 100644 --- a/.github/workflows/frontend-tasking-manager.yaml +++ b/.github/workflows/frontend-tasking-manager.yaml @@ -52,7 +52,7 @@ jobs: TM_ORG_GITHUB: github.com/openhistoricalmap TM_CONSUMER_KEY: ${{ secrets.PRODUCTION_TM_API_CONSUMER_KEY }} TM_CONSUMER_SECRET: ${{ secrets.PRODUCTION_TM_API_CONSUMER_SECRET }} - OSM_SERVER_URL: https://www.openhistoricalmap.org + OSM_SERVER_URL: https://openhistoricalmap.org OSM_REGISTER_URL: https://www.openhistoricalmap.org/user/new ID_EDITOR_URL: https://www.openhistoricalmap.org/edit?editor=id POTLATCH2_EDITOR_URL: https://www.openhistoricalmap.org/edit?editor=potlatch2 From d157ce78eef38880b150eeaed905a3eaf10f779b Mon Sep 17 00:00:00 2001 From: Rub21 Date: Thu, 14 Sep 2023 20:34:54 -0500 Subject: [PATCH 15/16] Set www for OSM_SERVER_URL in tasking manager --- .github/workflows/frontend-tasking-manager.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/frontend-tasking-manager.yaml b/.github/workflows/frontend-tasking-manager.yaml index 98797bf5..f5dbdc5a 100644 --- a/.github/workflows/frontend-tasking-manager.yaml +++ b/.github/workflows/frontend-tasking-manager.yaml @@ -52,7 +52,7 @@ jobs: TM_ORG_GITHUB: github.com/openhistoricalmap TM_CONSUMER_KEY: ${{ secrets.PRODUCTION_TM_API_CONSUMER_KEY }} TM_CONSUMER_SECRET: ${{ secrets.PRODUCTION_TM_API_CONSUMER_SECRET }} - OSM_SERVER_URL: https://openhistoricalmap.org + OSM_SERVER_URL: https://www.openhistoricalmap.org OSM_REGISTER_URL: https://www.openhistoricalmap.org/user/new ID_EDITOR_URL: https://www.openhistoricalmap.org/edit?editor=id POTLATCH2_EDITOR_URL: https://www.openhistoricalmap.org/edit?editor=potlatch2 From e12ddf30105ee7c8c3902d942b5c434450ffcb57 Mon Sep 17 00:00:00 2001 From: Rub21 Date: Thu, 14 Sep 2023 20:56:11 -0500 Subject: [PATCH 16/16] Disable cgimap that creates a ELB service --- values.staging.template.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/values.staging.template.yaml b/values.staging.template.yaml index aa9649d5..ca85d866 100644 --- a/values.staging.template.yaml +++ b/values.staging.template.yaml @@ -132,7 +132,7 @@ osm-seed: maxReplicas: 10 cpuUtilization: 80 cgimap: - enabled: true + enabled: false # ==================================================================================================== # Variables for memcached. Memcached is used to store session cookies # ====================================================================================================