diff --git a/.github/workflows/chartpress.yaml b/.github/workflows/chartpress.yaml index 48ea0465..0bc85af1 100644 --- a/.github/workflows/chartpress.yaml +++ b/.github/workflows/chartpress.yaml @@ -75,7 +75,12 @@ jobs: STAGING_OVERPASS_API_DB_EBS: ${{ secrets.STAGING_OVERPASS_API_DB_EBS }} STAGING_NEW_RELIC_LICENSE_KEY: ${{ secrets.STAGING_NEW_RELIC_LICENSE_KEY }} STAGING_NEW_RELIC_APP_NAME: ${{ secrets.STAGING_NEW_RELIC_APP_NAME }} - + STAGING_OSMCHA_PG_USER: ${{ secrets.STAGING_OSMCHA_PG_USER }} + STAGING_OSMCHA_PG_PASSWORD: ${{ secrets.STAGING_OSMCHA_PG_PASSWORD }} + STAGING_OSMCHA_PG_DATABASE: ${{ secrets.STAGING_OSMCHA_PG_DATABASE }} + STAGING_OSMCHA_CONSUMER_KEY: ${{ secrets.STAGING_OSMCHA_CONSUMER_KEY }} + STAGING_OSMCHA_CONSUMER_SECRET: ${{ secrets.STAGING_OSMCHA_CONSUMER_SECRET }} + STAGING_OSMCHA_DJANGO_SECRET_KEY: ${{ secrets.STAGING_OSMCHA_DJANGO_SECRET_KEY }} - name: Production - substitute secrets if: github.ref == 'refs/heads/main' uses: bluwy/substitute-string-action@v1 @@ -124,7 +129,12 @@ jobs: PRODUCTION_OVERPASS_API_DB_EBS: ${{ secrets.PRODUCTION_OVERPASS_API_DB_EBS }} PRODUCTION_NEW_RELIC_LICENSE_KEY: ${{ secrets.PRODUCTION_NEW_RELIC_LICENSE_KEY }} PRODUCTION_NEW_RELIC_APP_NAME: ${{ secrets.PRODUCTION_NEW_RELIC_APP_NAME }} - + PRODUCTION_OSMCHA_PG_USER: ${{ secrets.PRODUCTION_OSMCHA_PG_USER }} + PRODUCTION_OSMCHA_PG_PASSWORD: ${{ secrets.PRODUCTION_OSMCHA_PG_PASSWORD }} + PRODUCTION_OSMCHA_PG_DATABASE: ${{ secrets.PRODUCTION_OSMCHA_PG_DATABASE }} + PRODUCTION_OSMCHA_CONSUMER_KEY: ${{ secrets.PRODUCTION_OSMCHA_CONSUMER_KEY }} + PRODUCTION_OSMCHA_CONSUMER_SECRET: ${{ secrets.PRODUCTION_OSMCHA_CONSUMER_SECRET }} + PRODUCTION_OSMCHA_DJANGO_SECRET_KEY: ${{ secrets.PRODUCTION_OSMCHA_DJANGO_SECRET_KEY }} - name: AWS Credentials if: github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/main' uses: aws-actions/configure-aws-credentials@v1 @@ -135,17 +145,7 @@ jobs: - name: Setup Kubectl and Helm Dependencies if: github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/main' - run: | - sudo pip install awscli --ignore-installed six - sudo curl -L -o /usr/bin/kubectl https://amazon-eks.s3.us-west-2.amazonaws.com/1.17.7/2020-07-08/bin/linux/amd64/kubectl - sudo chmod +x /usr/bin/kubectl - sudo curl -o /usr/bin/aws-iam-authenticator https://amazon-eks.s3.us-west-2.amazonaws.com/1.17.7/2020-07-08/bin/linux/amd64/aws-iam-authenticator - sudo chmod +x /usr/bin/aws-iam-authenticator - wget https://get.helm.sh/helm-v3.5.0-linux-amd64.tar.gz -O helm.tar.gz - tar -xvzf helm.tar.gz - sudo mv linux-amd64/helm /usr/local/bin/ - sudo chmod +x /usr/local/bin/helm - + run: "sudo pip install awscli --ignore-installed six\nsudo curl -L -o /usr/bin/kubectl https://amazon-eks.s3.us-west-2.amazonaws.com/1.17.7/2020-07-08/bin/linux/amd64/kubectl\nsudo chmod +x /usr/bin/kubectl\nsudo curl -o /usr/bin/aws-iam-authenticator https://amazon-eks.s3.us-west-2.amazonaws.com/1.17.7/2020-07-08/bin/linux/amd64/aws-iam-authenticator\nsudo chmod +x /usr/bin/aws-iam-authenticator\nwget https://get.helm.sh/helm-v3.5.0-linux-amd64.tar.gz -O helm.tar.gz\ntar -xvzf helm.tar.gz\nsudo mv linux-amd64/helm /usr/local/bin/ \nsudo chmod +x /usr/local/bin/helm\n #magic___^_^___line\n" - name: Update kube-config staging if: github.ref == 'refs/heads/staging' run: aws eks --region us-east-1 update-kubeconfig --name osmseed-staging diff --git a/.github/workflows/frontend-overpass.yaml b/.github/workflows/frontend-overpass.yaml index f9434ba7..c4c00939 100644 --- a/.github/workflows/frontend-overpass.yaml +++ b/.github/workflows/frontend-overpass.yaml @@ -33,30 +33,28 @@ jobs: uses: actions/checkout@v2 with: repository: OpenHistoricalMap/overpass-turbo - ref: 43d1d9aa7e8bbb7152c89354900fdd327f9f7ffe + ref: c9defd0a9b32baf3af6599c3cb4a0dbad96b85e8 # token: ${{env.DEV_GITHUB_TOKEN}} - - name: Use Node.js 12 + - name: Use Node.js 16 uses: actions/setup-node@v1 with: - node-version: 12 + node-version: 16 - name: Install modules run: | rm -rf node_modules - rm package-lock.json npm cache clean --force - npm cache verify - npm install + yarn install --no-progress --frozen-lockfile - name: Build frontend run: | sed -i -e 's/overpass-api.openhistoricalmap.org/'${OVERPASS_API}'/g' js/configs.js - npm run build + yarn build - uses: actions/setup-python@v2 with: - python-version: '3.8' + python-version: '3.10' - name: Install aws cli run: | diff --git a/images/web/Dockerfile b/images/web/Dockerfile index 9f4d9c8d..96e5c0a5 100644 --- a/images/web/Dockerfile +++ b/images/web/Dockerfile @@ -18,14 +18,17 @@ RUN apt-get update \ RUN curl -sL https://deb.nodesource.com/setup_16.x | bash - RUN curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - RUN echo "deb https://dl.yarnpkg.com/debian/ stable main" | tee /etc/apt/sources.list.d/yarn.list - -RUN apt-get update && apt-get install -y nodejs yarn +RUN apt-get update && apt-get install -y nodejs yarn && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* # Install openstreetmap-cgimap requiriments -RUN apt-get update && apt-get -y install libxml2-dev libpqxx-dev libfcgi-dev zlib1g-dev \ +RUN apt-get update && apt-get -y install libpqxx-dev libfcgi-dev zlib1g-dev \ libboost-dev libboost-program-options-dev libfmt-dev \ libmemcached-dev libcrypto++-dev libargon2-dev libyajl-dev \ - build-essential automake autoconf libtool + automake autoconf libtool && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* # Install cgimap ENV cgimap /tmp/openstreetmap-cgimap @@ -43,7 +46,6 @@ RUN npm install -g svgo # Install openstreetmap-website RUN rm -rf $workdir/html - ENV OPENHISTORICALMAP_WEBSITE_GITSHA=284909db296078187f0338149bc9e7667e5a01c9 RUN git clone -b staging https://github.com/OpenHistoricalMap/ohm-website.git $workdir WORKDIR $workdir @@ -56,10 +58,6 @@ RUN git fetch # Install Ruby packages RUN gem install bundler && bundle install -# update vendored iD -# RUN rm -rf vendor/assets/iD -# RUN vendorer - # Configure database.yml and secrets.yml RUN cp $workdir/config/example.database.yml $workdir/config/database.yml RUN touch $workdir/config/settings.local.yml @@ -80,6 +78,8 @@ RUN ln -s /tmp /var/www/tmp # Add Apache configuration file ADD config/production.conf /etc/apache2/sites-available/production.conf +RUN a2enmod headers +RUN a2enmod setenvif RUN a2dissite 000-default RUN a2ensite production @@ -99,12 +99,6 @@ RUN /usr/sbin/passenger-memory-stats RUN a2enmod proxy proxy_http rewrite lbmethod_byrequests # Config the virtual host apache2 -# ADD config/cgimap.conf /tmp/ -# RUN sed -e 's/RewriteRule ^(.*)/#RewriteRule ^(.*)/' \ -# -e 's/\/var\/www/\/var\/www\/public/g' \ -# /tmp/cgimap.conf > /etc/apache2/sites-available/cgimap.conf -# RUN chmod 644 /etc/apache2/sites-available/cgimap.conf -# RUN a2ensite cgimap RUN apache2ctl configtest # Set Permissions for www-data diff --git a/images/web/config/lighttpd.conf b/images/web/config/lighttpd.conf index 85520b80..c21db904 100644 --- a/images/web/config/lighttpd.conf +++ b/images/web/config/lighttpd.conf @@ -9,6 +9,9 @@ server.reject-expect-100-with-417 = "disable" # selecting modules server.modules = ( "mod_access", "mod_rewrite", "mod_fastcgi", "mod_proxy", "mod_alias" ) +accesslog.filename = "/dev/stdout" +server.errorlog = "/dev/stderr" + # handling unknown routes server.error-handler-404 = "/dispatch.map" @@ -24,6 +27,17 @@ mimetype.assign = ( ".png" => "image/png" ) +# Handle CORS Preflight Requests +$HTTP["request-method"] == "OPTIONS" { + setenv.add-response-header = ( + "Access-Control-Allow-Origin" => "*", + "Access-Control-Allow-Methods" => "GET, POST, PUT, DELETE, OPTIONS", + "Access-Control-Allow-Headers" => "origin, x-requested-with, content-type", + "Access-Control-Allow-Credentials" => "true", + "Access-Control-Max-Age" => "86400" + ) +} + #debug.log-request-handling = "enable" $HTTP["request-method"] == "GET" { diff --git a/images/web/config/production.conf b/images/web/config/production.conf index 9920645f..5f6a4619 100644 --- a/images/web/config/production.conf +++ b/images/web/config/production.conf @@ -17,24 +17,10 @@ RewriteCond %{HTTP_HOST} !^www\. [NC] RewriteRule .* https://www.%{HTTP_HOST}%{REQUEST_URI} [L,R=301] - # ======Redirect trafict to CGIMAP==== - # Rules to redirect to port 81 - RewriteCond %{REQUEST_URI} ^/api/0\.6/map - RewriteRule ^.*$ http://127.0.0.1:81%{REQUEST_URI} [P,L] - # Existing rewrites - RewriteRule ^0\.6/map$ - [L,H=fcgi:127.0.0.1:81] - RewriteRule ^0\.6/(nodes|ways|relations)$ - [L,H=fcgi:127.0.0.1:81] - RewriteRule ^0\.6/(way|relation)/([^/]+)/full$ - [L,H=fcgi:127.0.0.1:81] - - # If no match found, fall through to port 80 - RewriteCond %{REQUEST_METHOD} ^(GET|HEAD)$ - RewriteRule ^0\.6/(node|way|relation)/([^/]+)$ - [L,H=fcgi:127.0.0.1:80] - # Relax Apache security settings AllowOverride None Allow from all Options -MultiViews - - + \ No newline at end of file diff --git a/ohm/requirements.yaml b/ohm/requirements.yaml index 5e6657d5..6d5c47bf 100644 --- a/ohm/requirements.yaml +++ b/ohm/requirements.yaml @@ -1,4 +1,4 @@ dependencies: - name: osm-seed - version: '0.1.0-n770.h3ffd454' + version: '0.1.0-n783.ha5dabc8' repository: https://devseed.com/osm-seed-chart/ diff --git a/values.production.template.yaml b/values.production.template.yaml index fbe81f25..bfb59ffe 100644 --- a/values.production.template.yaml +++ b/values.production.template.yaml @@ -120,6 +120,8 @@ osm-seed: minReplicas: 2 maxReplicas: 10 cpuUtilization: 80 + cgimap: + enabled: false # ==================================================================================================== # Variables for memcached. Memcached is used to store session cookies # ==================================================================================================== @@ -667,4 +669,81 @@ osm-seed: memory: '24Gi' cpu: '10' nodeSelector: - enabled: false \ No newline at end of file + enabled: false + + # ==================================================================================================== + # Variables for changeset-replication-job, Configuration to create the replication files by, minute, hour, or day + # ==================================================================================================== + osmchaWeb: + enabled: false + env: + OSM_SERVER_URL: 'https://staging.openhistoricalmap.org' + DJANGO_CACHES: "{'default': {'BACKEND': 'django.core.cache.backends.memcached.MemcachedCache', 'LOCATION': '127.0.0.1:11211'}}" + DJANGO_DEBUG: "False" + DJANGO_SECRET_KEY: {{PRODUCTION_OSMCHA_DJANGO_SECRET_KEY}} + DJANGO_SECURE_BROWSER_XSS_FILTER: "True" + DJANGO_SECURE_SSL_REDIRECT: "True" + DJANGO_SECURE_CONTENT_TYPE_NOSNIFF: "True" + DJANGO_SECURE_FRAME_DENY: "True" + DJANGO_SECURE_HSTS_INCLUDE_SUBDOMAINS: "True" + DJANGO_SESSION_COOKIE_HTTPONLY: "True" + DJANGO_SESSION_COOKIE_SECURE: "True" + DJANGO_DEFAULT_FROM_EMAIL: "webmaster@openhistoricalmap.org" + DJANGO_SERVER_EMAIL: "server@openhistoricalmap.org" + DJANGO_EMAIL_SUBJECT_PREFIX: "osmcha" + DJANGO_CHANGESETS_FILTER: "{'some_filter_key': 'some_filter_value'}" + OAUTH_OSM_KEY: {{PRODUCTION_OSMCHA_CONSUMER_KEY}} + OAUTH_OSM_SECRET: {{PRODUCTION_OSMCHA_CONSUMER_SECRET}} + DJANGO_ANON_USER_THROTTLE_RATE: "10/h" + DJANGO_COMMON_USER_THROTTLE_RATE: "100/h" + DJANGO_NON_STAFF_USER_THROTTLE_RATE: "50/h" + OAUTH_REDIRECT_URI: https://osmcha.openhistoricalmap.org/oauth-landing.html + DJANGO_ENABLE_CHANGESET_COMMENTS: "True" + DJANGO_OSM_COMMENTS_API_KEY: "test" + OSMCHA_FRONTEND_VERSION: "v0.86.0-production" + DJANGO_ROOT: /app + DJANGO_SETTINGS_MODULE: config.settings.local + C_FORCE_ROOT: "False" + resources: + enabled: false + requests: + memory: '20Gi' + cpu: '8' + limits: + memory: '24Gi' + cpu: '10' + nodeSelector: + enabled: false + + # ==================================================================================================== + # Variables for changeset-replication-job, Configuration to create the replication files by, minute, hour, or day + # ==================================================================================================== + osmchaDb: + enabled: false + env: + POSTGRES_DB: {{PRODUCTION_OSMCHA_PG_DATABASE}} + POSTGRES_USER: {{PRODUCTION_OSMCHA_PG_USER}} + POSTGRES_PASSWORD: {{PRODUCTION_OSMCHA_PG_PASSWORD}} + resources: + enabled: false + requests: + memory: '20Gi' + cpu: '8' + limits: + memory: '24Gi' + cpu: '10' + persistenceDisk: + enabled: false + accessMode: ReadWriteOnce + mountPath: /var/lib/postgresql/data + # Minikube + localVolumeHostPath: /mnt/db-data/osmcha-data + localVolumeSize: 10Gi + # AWS + AWS_ElasticBlockStore_volumeID: vol-0462ddf32e5b456df + AWS_ElasticBlockStore_size: 100Gi + # GCP + GCP_gcePersistentDisk_pdName: osmseed-osmcha-disk--v1 + GCP_gcePersistentDisk_size: 50Gi + nodeSelector: + enabled: false diff --git a/values.staging.template.yaml b/values.staging.template.yaml index 47ab1acb..ca85d866 100644 --- a/values.staging.template.yaml +++ b/values.staging.template.yaml @@ -131,6 +131,8 @@ osm-seed: minReplicas: 2 maxReplicas: 10 cpuUtilization: 80 + cgimap: + enabled: false # ==================================================================================================== # Variables for memcached. Memcached is used to store session cookies # ==================================================================================================== @@ -696,3 +698,81 @@ osm-seed: cpu: '10' nodeSelector: enabled: false + + + # ==================================================================================================== + # Variables for changeset-replication-job, Configuration to create the replication files by, minute, hour, or day + # ==================================================================================================== + osmchaWeb: + enabled: true + env: + OSM_SERVER_URL: 'https://staging.openhistoricalmap.org' + DJANGO_CACHES: "{'default': {'BACKEND': 'django.core.cache.backends.memcached.MemcachedCache', 'LOCATION': '127.0.0.1:11211'}}" + DJANGO_DEBUG: "False" + DJANGO_SECRET_KEY: {{STAGING_OSMCHA_DJANGO_SECRET_KEY}} + DJANGO_SECURE_BROWSER_XSS_FILTER: "True" + DJANGO_SECURE_SSL_REDIRECT: "True" + DJANGO_SECURE_CONTENT_TYPE_NOSNIFF: "True" + DJANGO_SECURE_FRAME_DENY: "True" + DJANGO_SECURE_HSTS_INCLUDE_SUBDOMAINS: "True" + DJANGO_SESSION_COOKIE_HTTPONLY: "True" + DJANGO_SESSION_COOKIE_SECURE: "True" + DJANGO_DEFAULT_FROM_EMAIL: "webmaster@openhistoricalmap.org" + DJANGO_SERVER_EMAIL: "server@openhistoricalmap.org" + DJANGO_EMAIL_SUBJECT_PREFIX: "osmcha" + DJANGO_CHANGESETS_FILTER: "{'some_filter_key': 'some_filter_value'}" + OAUTH_OSM_KEY: {{STAGING_OSMCHA_CONSUMER_KEY}} + OAUTH_OSM_SECRET: {{STAGING_OSMCHA_CONSUMER_SECRET}} + DJANGO_ANON_USER_THROTTLE_RATE: "10/h" + DJANGO_COMMON_USER_THROTTLE_RATE: "100/h" + DJANGO_NON_STAFF_USER_THROTTLE_RATE: "50/h" + OAUTH_REDIRECT_URI: https://osmcha.openhistoricalmap.org/oauth-landing.html + DJANGO_ENABLE_CHANGESET_COMMENTS: "True" + DJANGO_OSM_COMMENTS_API_KEY: "test" + OSMCHA_FRONTEND_VERSION: "v0.86.0-production" + DJANGO_ROOT: /app + DJANGO_SETTINGS_MODULE: config.settings.local + C_FORCE_ROOT: "False" + resources: + enabled: false + requests: + memory: '20Gi' + cpu: '8' + limits: + memory: '24Gi' + cpu: '10' + nodeSelector: + enabled: false + + # ==================================================================================================== + # Variables for changeset-replication-job, Configuration to create the replication files by, minute, hour, or day + # ==================================================================================================== + osmchaDb: + enabled: true + env: + POSTGRES_DB: {{STAGING_OSMCHA_PG_DATABASE}} + POSTGRES_USER: {{STAGING_OSMCHA_PG_USER}} + POSTGRES_PASSWORD: {{STAGING_OSMCHA_PG_PASSWORD}} + resources: + enabled: false + requests: + memory: '20Gi' + cpu: '8' + limits: + memory: '24Gi' + cpu: '10' + persistenceDisk: + enabled: false + accessMode: ReadWriteOnce + mountPath: /var/lib/postgresql/data + # Minikube + localVolumeHostPath: /mnt/db-data/osmcha-data + localVolumeSize: 10Gi + # AWS + AWS_ElasticBlockStore_volumeID: vol-0462ddf32e5b456df + AWS_ElasticBlockStore_size: 100Gi + # GCP + GCP_gcePersistentDisk_pdName: osmseed-osmcha-disk--v1 + GCP_gcePersistentDisk_size: 50Gi + nodeSelector: + enabled: false