Merge pull request #231 from OpenHistoricalMap/staging

Disable redirect request to CGIMAP

.github/workflows/chartpress.yaml

@@ -75,7 +75,12 @@ jobs:
         STAGING_OVERPASS_API_DB_EBS: ${{ secrets.STAGING_OVERPASS_API_DB_EBS }}
         STAGING_NEW_RELIC_LICENSE_KEY: ${{ secrets.STAGING_NEW_RELIC_LICENSE_KEY }}
         STAGING_NEW_RELIC_APP_NAME: ${{ secrets.STAGING_NEW_RELIC_APP_NAME }}
-
+        STAGING_OSMCHA_PG_USER: ${{ secrets.STAGING_OSMCHA_PG_USER }}
+        STAGING_OSMCHA_PG_PASSWORD: ${{ secrets.STAGING_OSMCHA_PG_PASSWORD }}
+        STAGING_OSMCHA_PG_DATABASE: ${{ secrets.STAGING_OSMCHA_PG_DATABASE }}
+        STAGING_OSMCHA_CONSUMER_KEY: ${{ secrets.STAGING_OSMCHA_CONSUMER_KEY }}
+        STAGING_OSMCHA_CONSUMER_SECRET: ${{ secrets.STAGING_OSMCHA_CONSUMER_SECRET }}
+        STAGING_OSMCHA_DJANGO_SECRET_KEY: ${{ secrets.STAGING_OSMCHA_DJANGO_SECRET_KEY }}
     - name: Production - substitute secrets
       if: github.ref == 'refs/heads/main'
       uses: bluwy/substitute-string-action@v1
@@ -124,7 +129,12 @@ jobs:
         PRODUCTION_OVERPASS_API_DB_EBS: ${{ secrets.PRODUCTION_OVERPASS_API_DB_EBS }}
         PRODUCTION_NEW_RELIC_LICENSE_KEY: ${{ secrets.PRODUCTION_NEW_RELIC_LICENSE_KEY }}
         PRODUCTION_NEW_RELIC_APP_NAME: ${{ secrets.PRODUCTION_NEW_RELIC_APP_NAME }}
-
+        PRODUCTION_OSMCHA_PG_USER: ${{ secrets.PRODUCTION_OSMCHA_PG_USER }}
+        PRODUCTION_OSMCHA_PG_PASSWORD: ${{ secrets.PRODUCTION_OSMCHA_PG_PASSWORD }}
+        PRODUCTION_OSMCHA_PG_DATABASE: ${{ secrets.PRODUCTION_OSMCHA_PG_DATABASE }}
+        PRODUCTION_OSMCHA_CONSUMER_KEY: ${{ secrets.PRODUCTION_OSMCHA_CONSUMER_KEY }}
+        PRODUCTION_OSMCHA_CONSUMER_SECRET: ${{ secrets.PRODUCTION_OSMCHA_CONSUMER_SECRET }}
+        PRODUCTION_OSMCHA_DJANGO_SECRET_KEY: ${{ secrets.PRODUCTION_OSMCHA_DJANGO_SECRET_KEY }}
     - name: AWS Credentials
       if: github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/main'
       uses: aws-actions/configure-aws-credentials@v1
@@ -135,17 +145,7 @@ jobs:
 
     - name: Setup Kubectl and Helm Dependencies
       if: github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/main'
-      run: |
-       sudo pip install awscli --ignore-installed six
-       sudo curl -L -o /usr/bin/kubectl https://amazon-eks.s3.us-west-2.amazonaws.com/1.17.7/2020-07-08/bin/linux/amd64/kubectl
-       sudo chmod +x /usr/bin/kubectl
-       sudo curl -o /usr/bin/aws-iam-authenticator https://amazon-eks.s3.us-west-2.amazonaws.com/1.17.7/2020-07-08/bin/linux/amd64/aws-iam-authenticator
-       sudo chmod +x /usr/bin/aws-iam-authenticator
-       wget https://get.helm.sh/helm-v3.5.0-linux-amd64.tar.gz -O helm.tar.gz
-       tar -xvzf helm.tar.gz
-       sudo mv linux-amd64/helm /usr/local/bin/ 
-       sudo chmod +x /usr/local/bin/helm
-
+      run: "sudo pip install awscli --ignore-installed six\nsudo curl -L -o /usr/bin/kubectl https://amazon-eks.s3.us-west-2.amazonaws.com/1.17.7/2020-07-08/bin/linux/amd64/kubectl\nsudo chmod +x /usr/bin/kubectl\nsudo curl -o /usr/bin/aws-iam-authenticator https://amazon-eks.s3.us-west-2.amazonaws.com/1.17.7/2020-07-08/bin/linux/amd64/aws-iam-authenticator\nsudo chmod +x /usr/bin/aws-iam-authenticator\nwget https://get.helm.sh/helm-v3.5.0-linux-amd64.tar.gz -O helm.tar.gz\ntar -xvzf helm.tar.gz\nsudo mv linux-amd64/helm /usr/local/bin/ \nsudo chmod +x /usr/local/bin/helm\n #magic___^_^___line\n"
     - name: Update kube-config staging
       if: github.ref == 'refs/heads/staging'
       run: aws eks --region us-east-1 update-kubeconfig --name osmseed-staging

.github/workflows/frontend-overpass.yaml

@@ -33,30 +33,28 @@ jobs:
         uses: actions/checkout@v2
         with:
           repository: OpenHistoricalMap/overpass-turbo
-          ref: 43d1d9aa7e8bbb7152c89354900fdd327f9f7ffe
+          ref: c9defd0a9b32baf3af6599c3cb4a0dbad96b85e8
           # token: ${{env.DEV_GITHUB_TOKEN}}
 
-      - name: Use Node.js 12
+      - name: Use Node.js 16
         uses: actions/setup-node@v1
         with:
-          node-version: 12
+          node-version: 16
 
       - name: Install modules
         run: |
           rm -rf node_modules
-          rm package-lock.json
           npm cache clean --force
-          npm cache verify
-          npm install
+          yarn install --no-progress --frozen-lockfile
 
       - name: Build frontend
         run: |
           sed -i -e 's/overpass-api.openhistoricalmap.org/'${OVERPASS_API}'/g' js/configs.js
-          npm run build
+          yarn build
       
       - uses: actions/setup-python@v2
         with:
-          python-version: '3.8'
+          python-version: '3.10'
 
       - name: Install aws cli
         run: |

images/web/Dockerfile

@@ -18,14 +18,17 @@ RUN apt-get update \
 RUN curl -sL https://deb.nodesource.com/setup_16.x | bash -
 RUN curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add -
 RUN echo "deb https://dl.yarnpkg.com/debian/ stable main" | tee /etc/apt/sources.list.d/yarn.list
-
-RUN apt-get update && apt-get install -y nodejs yarn
+RUN apt-get update && apt-get install -y nodejs yarn && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
 
 # Install openstreetmap-cgimap requiriments
-RUN apt-get update &&  apt-get -y install libxml2-dev libpqxx-dev libfcgi-dev zlib1g-dev \
+RUN apt-get update &&  apt-get -y install libpqxx-dev libfcgi-dev zlib1g-dev \
   libboost-dev libboost-program-options-dev libfmt-dev \
   libmemcached-dev libcrypto++-dev libargon2-dev libyajl-dev \
-  build-essential automake autoconf libtool
+  automake autoconf libtool && \
+  apt-get clean && \
+  rm -rf /var/lib/apt/lists/*
 
 # Install cgimap
 ENV cgimap /tmp/openstreetmap-cgimap
@@ -43,7 +46,6 @@ RUN npm install -g svgo
 
 # Install openstreetmap-website
 RUN rm -rf $workdir/html
-
 ENV OPENHISTORICALMAP_WEBSITE_GITSHA=284909db296078187f0338149bc9e7667e5a01c9
 RUN git clone -b staging https://github.com/OpenHistoricalMap/ohm-website.git $workdir
 WORKDIR $workdir
@@ -56,10 +58,6 @@ RUN git fetch
 # Install Ruby packages
 RUN gem install bundler && bundle install
 
-# update vendored iD
-# RUN rm -rf vendor/assets/iD
-# RUN vendorer
-
 # Configure database.yml and secrets.yml
 RUN cp $workdir/config/example.database.yml $workdir/config/database.yml
 RUN touch $workdir/config/settings.local.yml
@@ -80,6 +78,8 @@ RUN ln -s /tmp /var/www/tmp
 
 # Add Apache configuration file
 ADD config/production.conf /etc/apache2/sites-available/production.conf
+RUN a2enmod headers
+RUN a2enmod setenvif
 RUN a2dissite 000-default
 RUN a2ensite production
 
@@ -99,12 +99,6 @@ RUN /usr/sbin/passenger-memory-stats
 RUN a2enmod proxy proxy_http rewrite lbmethod_byrequests
 
 # Config the virtual host apache2
-# ADD config/cgimap.conf /tmp/
-# RUN sed -e 's/RewriteRule ^(.*)/#RewriteRule ^(.*)/' \
-#         -e 's/\/var\/www/\/var\/www\/public/g' \
-#         /tmp/cgimap.conf > /etc/apache2/sites-available/cgimap.conf
-# RUN chmod 644 /etc/apache2/sites-available/cgimap.conf
-# RUN a2ensite cgimap
 RUN apache2ctl configtest
 
 # Set Permissions for www-data

images/web/config/lighttpd.conf

@@ -9,6 +9,9 @@ server.reject-expect-100-with-417 = "disable"
 # selecting modules
 server.modules = ( "mod_access", "mod_rewrite", "mod_fastcgi",  "mod_proxy", "mod_alias" )
 
+accesslog.filename          = "/dev/stdout"
+server.errorlog             = "/dev/stderr"
+
 # handling unknown routes
 server.error-handler-404   = "/dispatch.map"
 
@@ -24,6 +27,17 @@ mimetype.assign = (
   ".png" => "image/png"
 )
 
+# Handle CORS Preflight Requests
+$HTTP["request-method"] == "OPTIONS" {
+    setenv.add-response-header = (
+        "Access-Control-Allow-Origin" => "*",
+        "Access-Control-Allow-Methods" => "GET, POST, PUT, DELETE, OPTIONS",
+        "Access-Control-Allow-Headers" => "origin, x-requested-with, content-type",
+        "Access-Control-Allow-Credentials" => "true",
+        "Access-Control-Max-Age" => "86400"
+    )
+}
+
 #debug.log-request-handling = "enable"
 
 $HTTP["request-method"] == "GET" {

images/web/config/production.conf

@@ -17,24 +17,10 @@
     RewriteCond %{HTTP_HOST} !^www\. [NC]
     RewriteRule .* https://www.%{HTTP_HOST}%{REQUEST_URI} [L,R=301]
 
-    # ======Redirect trafict to CGIMAP====
-    # Rules to redirect to port 81
-    RewriteCond %{REQUEST_URI} ^/api/0\.6/map
-    RewriteRule ^.*$ http://127.0.0.1:81%{REQUEST_URI} [P,L]
-    # Existing rewrites
-    RewriteRule ^0\.6/map$ - [L,H=fcgi:127.0.0.1:81]
-    RewriteRule ^0\.6/(nodes|ways|relations)$ - [L,H=fcgi:127.0.0.1:81]
-    RewriteRule ^0\.6/(way|relation)/([^/]+)/full$ - [L,H=fcgi:127.0.0.1:81]
-
-    # If no match found, fall through to port 80
-    RewriteCond %{REQUEST_METHOD} ^(GET|HEAD)$
-    RewriteRule ^0\.6/(node|way|relation)/([^/]+)$ - [L,H=fcgi:127.0.0.1:80]
-
     # Relax Apache security settings
     <Directory /var/www/public>
       AllowOverride None
       Allow from all
       Options -MultiViews
     </Directory>
-</VirtualHost>
-
+</VirtualHost>

ohm/requirements.yaml

@@ -1,4 +1,4 @@
 dependencies:
 - name: osm-seed
-  version: '0.1.0-n770.h3ffd454'
+  version: '0.1.0-n783.ha5dabc8'
   repository: https://devseed.com/osm-seed-chart/

values.production.template.yaml

@@ -120,6 +120,8 @@ osm-seed:
       minReplicas: 2
       maxReplicas: 10
       cpuUtilization: 80
+    cgimap:
+      enabled: false
   # ====================================================================================================
   # Variables for memcached. Memcached is used to store session cookies
   # ====================================================================================================
@@ -667,4 +669,81 @@ osm-seed:
         memory: '24Gi'
         cpu: '10'
     nodeSelector:
-      enabled: false
+      enabled: false
+
+  # ====================================================================================================
+  # Variables for changeset-replication-job, Configuration to create the replication files by, minute, hour, or day
+  # ====================================================================================================
+  osmchaWeb:
+    enabled: false
+    env:
+      OSM_SERVER_URL: 'https://staging.openhistoricalmap.org'
+      DJANGO_CACHES: "{'default': {'BACKEND': 'django.core.cache.backends.memcached.MemcachedCache', 'LOCATION': '127.0.0.1:11211'}}"
+      DJANGO_DEBUG: "False"
+      DJANGO_SECRET_KEY: {{PRODUCTION_OSMCHA_DJANGO_SECRET_KEY}}
+      DJANGO_SECURE_BROWSER_XSS_FILTER: "True"
+      DJANGO_SECURE_SSL_REDIRECT: "True"
+      DJANGO_SECURE_CONTENT_TYPE_NOSNIFF: "True"
+      DJANGO_SECURE_FRAME_DENY: "True"
+      DJANGO_SECURE_HSTS_INCLUDE_SUBDOMAINS: "True"
+      DJANGO_SESSION_COOKIE_HTTPONLY: "True"
+      DJANGO_SESSION_COOKIE_SECURE: "True"
+      DJANGO_DEFAULT_FROM_EMAIL: "[email protected]"
+      DJANGO_SERVER_EMAIL: "[email protected]"
+      DJANGO_EMAIL_SUBJECT_PREFIX: "osmcha"
+      DJANGO_CHANGESETS_FILTER: "{'some_filter_key': 'some_filter_value'}"
+      OAUTH_OSM_KEY: {{PRODUCTION_OSMCHA_CONSUMER_KEY}}
+      OAUTH_OSM_SECRET: {{PRODUCTION_OSMCHA_CONSUMER_SECRET}}
+      DJANGO_ANON_USER_THROTTLE_RATE: "10/h"
+      DJANGO_COMMON_USER_THROTTLE_RATE: "100/h"
+      DJANGO_NON_STAFF_USER_THROTTLE_RATE: "50/h"
+      OAUTH_REDIRECT_URI: https://osmcha.openhistoricalmap.org/oauth-landing.html
+      DJANGO_ENABLE_CHANGESET_COMMENTS: "True"
+      DJANGO_OSM_COMMENTS_API_KEY: "test"
+      OSMCHA_FRONTEND_VERSION: "v0.86.0-production"
+      DJANGO_ROOT: /app
+      DJANGO_SETTINGS_MODULE: config.settings.local
+      C_FORCE_ROOT: "False"
+    resources:
+      enabled: false
+      requests:
+        memory: '20Gi'
+        cpu: '8'
+      limits:
+        memory: '24Gi'
+        cpu: '10'
+    nodeSelector:
+      enabled: false
+
+  # ====================================================================================================
+  # Variables for changeset-replication-job, Configuration to create the replication files by, minute, hour, or day
+  # ====================================================================================================
+  osmchaDb:
+    enabled: false
+    env:
+      POSTGRES_DB: {{PRODUCTION_OSMCHA_PG_DATABASE}}
+      POSTGRES_USER: {{PRODUCTION_OSMCHA_PG_USER}}
+      POSTGRES_PASSWORD: {{PRODUCTION_OSMCHA_PG_PASSWORD}}
+    resources:
+      enabled: false
+      requests:
+        memory: '20Gi'
+        cpu: '8'
+      limits:
+        memory: '24Gi'
+        cpu: '10'
+    persistenceDisk:
+      enabled: false
+      accessMode: ReadWriteOnce
+      mountPath: /var/lib/postgresql/data
+      # Minikube
+      localVolumeHostPath: /mnt/db-data/osmcha-data
+      localVolumeSize: 10Gi
+      # AWS
+      AWS_ElasticBlockStore_volumeID: vol-0462ddf32e5b456df
+      AWS_ElasticBlockStore_size: 100Gi
+      # GCP
+      GCP_gcePersistentDisk_pdName: osmseed-osmcha-disk--v1
+      GCP_gcePersistentDisk_size: 50Gi
+    nodeSelector:
+      enabled: false

values.staging.template.yaml

@@ -131,6 +131,8 @@ osm-seed:
         minReplicas: 2
         maxReplicas: 10
         cpuUtilization: 80
+      cgimap:
+        enabled: false
     # ====================================================================================================
     # Variables for memcached. Memcached is used to store session cookies
     # ====================================================================================================
@@ -696,3 +698,81 @@ osm-seed:
           cpu: '10'
       nodeSelector:
         enabled: false
+
+
+    # ====================================================================================================
+    # Variables for changeset-replication-job, Configuration to create the replication files by, minute, hour, or day
+    # ====================================================================================================
+    osmchaWeb:
+      enabled: true
+      env:
+        OSM_SERVER_URL: 'https://staging.openhistoricalmap.org'
+        DJANGO_CACHES: "{'default': {'BACKEND': 'django.core.cache.backends.memcached.MemcachedCache', 'LOCATION': '127.0.0.1:11211'}}"
+        DJANGO_DEBUG: "False"
+        DJANGO_SECRET_KEY: {{STAGING_OSMCHA_DJANGO_SECRET_KEY}}
+        DJANGO_SECURE_BROWSER_XSS_FILTER: "True"
+        DJANGO_SECURE_SSL_REDIRECT: "True"
+        DJANGO_SECURE_CONTENT_TYPE_NOSNIFF: "True"
+        DJANGO_SECURE_FRAME_DENY: "True"
+        DJANGO_SECURE_HSTS_INCLUDE_SUBDOMAINS: "True"
+        DJANGO_SESSION_COOKIE_HTTPONLY: "True"
+        DJANGO_SESSION_COOKIE_SECURE: "True"
+        DJANGO_DEFAULT_FROM_EMAIL: "[email protected]"
+        DJANGO_SERVER_EMAIL: "[email protected]"
+        DJANGO_EMAIL_SUBJECT_PREFIX: "osmcha"
+        DJANGO_CHANGESETS_FILTER: "{'some_filter_key': 'some_filter_value'}"
+        OAUTH_OSM_KEY: {{STAGING_OSMCHA_CONSUMER_KEY}}
+        OAUTH_OSM_SECRET: {{STAGING_OSMCHA_CONSUMER_SECRET}}
+        DJANGO_ANON_USER_THROTTLE_RATE: "10/h"
+        DJANGO_COMMON_USER_THROTTLE_RATE: "100/h"
+        DJANGO_NON_STAFF_USER_THROTTLE_RATE: "50/h"
+        OAUTH_REDIRECT_URI: https://osmcha.openhistoricalmap.org/oauth-landing.html
+        DJANGO_ENABLE_CHANGESET_COMMENTS: "True"
+        DJANGO_OSM_COMMENTS_API_KEY: "test"
+        OSMCHA_FRONTEND_VERSION: "v0.86.0-production"
+        DJANGO_ROOT: /app
+        DJANGO_SETTINGS_MODULE: config.settings.local
+        C_FORCE_ROOT: "False"
+      resources:
+        enabled: false
+        requests:
+          memory: '20Gi'
+          cpu: '8'
+        limits:
+          memory: '24Gi'
+          cpu: '10'
+      nodeSelector:
+        enabled: false
+
+    # ====================================================================================================
+    # Variables for changeset-replication-job, Configuration to create the replication files by, minute, hour, or day
+    # ====================================================================================================
+    osmchaDb:
+      enabled: true
+      env:
+        POSTGRES_DB: {{STAGING_OSMCHA_PG_DATABASE}}
+        POSTGRES_USER: {{STAGING_OSMCHA_PG_USER}}
+        POSTGRES_PASSWORD: {{STAGING_OSMCHA_PG_PASSWORD}}
+      resources:
+        enabled: false
+        requests:
+          memory: '20Gi'
+          cpu: '8'
+        limits:
+          memory: '24Gi'
+          cpu: '10'
+      persistenceDisk:
+        enabled: false
+        accessMode: ReadWriteOnce
+        mountPath: /var/lib/postgresql/data
+        # Minikube
+        localVolumeHostPath: /mnt/db-data/osmcha-data
+        localVolumeSize: 10Gi
+        # AWS
+        AWS_ElasticBlockStore_volumeID: vol-0462ddf32e5b456df
+        AWS_ElasticBlockStore_size: 100Gi
+        # GCP
+        GCP_gcePersistentDisk_pdName: osmseed-osmcha-disk--v1
+        GCP_gcePersistentDisk_size: 50Gi
+      nodeSelector:
+        enabled: false