From e422ad273532f9d149bf938aa87f407a4d528ec2 Mon Sep 17 00:00:00 2001 From: Rub21 Date: Wed, 11 Dec 2024 10:20:23 -0500 Subject: [PATCH 1/5] Add leaflet-ohm-timeslider-v2 in the web container --- images/web/Dockerfile | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/images/web/Dockerfile b/images/web/Dockerfile index 08361ee3..99d00ffc 100644 --- a/images/web/Dockerfile +++ b/images/web/Dockerfile @@ -1,6 +1,6 @@ FROM ruby:3.3.0 ENV DEBIAN_FRONTEND=noninteractive -ENV workdir /var/www +ENV workdir=/var/www # Production OSM setup ENV RAILS_ENV=production @@ -32,7 +32,7 @@ RUN apt-get update && apt-get -y install libxml2-dev libpqxx-dev libfcgi-dev zl rm -rf /var/lib/apt/lists/* # Install cgimap -ENV cgimap /tmp/openstreetmap-cgimap +ENV cgimap=/tmp/openstreetmap-cgimap ENV CGIMAP_GITSHA=26cd7fa10affe5dbd13dbe16de34421059f53f18 RUN git clone -b master https://github.com/zerebubuth/openstreetmap-cgimap.git $cgimap \ && cd $cgimap \ @@ -120,6 +120,20 @@ RUN cd $workdir/public/map-styles && git fetch --depth 1 origin $OPENHISTORICALM RUN rm -rf $workdir/public/map-styles/.git COPY update_map_styles.py $workdir/ + +# Clone leaflet-ohm-timeslider-v2 +ENV LEAFLET_OHM_TIMESLIDER_V2=dd0acbdc9432fae6a4d09a17a4848c391e5064f0 +RUN git clone https://github.com/OpenHistoricalMap/leaflet-ohm-timeslider-v2.git $workdir/public/leaflet-ohm-timeslider-v2 \ + && cd $workdir/public/leaflet-ohm-timeslider-v2 \ + && git checkout $LEAFLET_OHM_TIMESLIDER_V2 \ + && rm -rf .git \ + && cp decimaldate.* $workdir/app/assets/javascripts/ \ + && cp leaflet-ohm-timeslider.* $workdir/app/assets/javascripts/ \ + && cp leaflet-ohm-timeslider.* $workdir/app/assets/stylesheets/ \ + && assets/* $workdir/app/assets/images/ \ + && cd ..\ + && rm -rf $workdir/public/leaflet-ohm-timeslider-v2 + # Add settings ADD config/settings.local.yml $workdir/config/ From 3640a2c125e398e3df28511b4027cc5e7250ab02 Mon Sep 17 00:00:00 2001 From: Rub21 Date: Wed, 11 Dec 2024 11:02:37 -0500 Subject: [PATCH 2/5] Fix ref for timeslider scripts --- images/web/Dockerfile | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/images/web/Dockerfile b/images/web/Dockerfile index 99d00ffc..a126f581 100644 --- a/images/web/Dockerfile +++ b/images/web/Dockerfile @@ -120,18 +120,18 @@ RUN cd $workdir/public/map-styles && git fetch --depth 1 origin $OPENHISTORICALM RUN rm -rf $workdir/public/map-styles/.git COPY update_map_styles.py $workdir/ - # Clone leaflet-ohm-timeslider-v2 ENV LEAFLET_OHM_TIMESLIDER_V2=dd0acbdc9432fae6a4d09a17a4848c391e5064f0 -RUN git clone https://github.com/OpenHistoricalMap/leaflet-ohm-timeslider-v2.git $workdir/public/leaflet-ohm-timeslider-v2 \ +RUN git clone https://github.com/OpenHistoricalMap/leaflet-ohm-timeslider-v2.git $workdir/public/leaflet-ohm-timeslider-v2 \ && cd $workdir/public/leaflet-ohm-timeslider-v2 \ - && git checkout $LEAFLET_OHM_TIMESLIDER_V2 \ + && git checkout dd0acbdc9432fae6a4d09a17a4848c391e5064f0 \ && rm -rf .git \ + && chmod 777 -R assets/ \ && cp decimaldate.* $workdir/app/assets/javascripts/ \ && cp leaflet-ohm-timeslider.* $workdir/app/assets/javascripts/ \ && cp leaflet-ohm-timeslider.* $workdir/app/assets/stylesheets/ \ - && assets/* $workdir/app/assets/images/ \ - && cd ..\ + && cp assets/* $workdir/app/assets/images/ \ + && cd .. \ && rm -rf $workdir/public/leaflet-ohm-timeslider-v2 # Add settings From f01843f50a6d207f089aa5c8a911e39b3666de5e Mon Sep 17 00:00:00 2001 From: Rub21 Date: Wed, 11 Dec 2024 11:48:53 -0500 Subject: [PATCH 3/5] Copy svg images for timeslider --- images/web/Dockerfile | 4 +--- images/web/start.sh | 26 +++++++++++++++----------- 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/images/web/Dockerfile b/images/web/Dockerfile index a126f581..216c639a 100644 --- a/images/web/Dockerfile +++ b/images/web/Dockerfile @@ -130,9 +130,7 @@ RUN git clone https://github.com/OpenHistoricalMap/leaflet-ohm-timeslider-v2.git && cp decimaldate.* $workdir/app/assets/javascripts/ \ && cp leaflet-ohm-timeslider.* $workdir/app/assets/javascripts/ \ && cp leaflet-ohm-timeslider.* $workdir/app/assets/stylesheets/ \ - && cp assets/* $workdir/app/assets/images/ \ - && cd .. \ - && rm -rf $workdir/public/leaflet-ohm-timeslider-v2 + && cp assets/* $workdir/app/assets/images/ # Add settings ADD config/settings.local.yml $workdir/config/ diff --git a/images/web/start.sh b/images/web/start.sh index 19c34431..eb60ddbb 100755 --- a/images/web/start.sh +++ b/images/web/start.sh @@ -3,7 +3,7 @@ workdir="/var/www" export RAILS_ENV=production #### Because we can not set up many env variable in build process, we are going to process here! -#### SETTING UP THE PRODUCTION DATABASE +#### Setting up the production database echo " # Production DB production: adapter: postgresql @@ -13,14 +13,14 @@ production: password: ${POSTGRES_PASSWORD} encoding: utf8" >$workdir/config/database.yml -#### SETTING UP SERVER_URL AND SERVER_PROTOCOL +#### Setting up server_url and server_protocol sed -i -e 's/server_url: "openhistoricalmap.example.com"/server_url: "'$SERVER_URL'"/g' $workdir/config/settings.local.yml sed -i -e 's/server_protocol: "http"/server_protocol: "'$SERVER_PROTOCOL'"/g' $workdir/config/settings.local.yml -### WEBSITE STATUS +### Setting up website status sed -i "s/online/$WEBSITE_STATUS/g" $workdir/config/settings.yml -#### SETTING UP MAIL SENDER +#### Setting up mail sender sed -i -e 's/smtp_address: "localhost"/smtp_address: "'$MAILER_ADDRESS'"/g' $workdir/config/settings.local.yml sed -i -e 's/smtp_domain: "localhost"/smtp_domain: "'$MAILER_DOMAIN'"/g' $workdir/config/settings.local.yml sed -i -e 's/smtp_enable_starttls_auto: false/smtp_enable_starttls_auto: true/g' $workdir/config/settings.local.yml @@ -30,11 +30,11 @@ sed -i -e 's/smtp_password: null/smtp_password: "'$MAILER_PASSWORD'"/g' $workdir sed -i -e 's/openstreetmap@example.com/'$MAILER_FROM'/g' $workdir/config/settings.local.yml sed -i -e 's/smtp_port: 25/smtp_port: '$MAILER_PORT'/g' $workdir/config/settings.local.yml -#### SET UP ID KEY +#### Setting up id key fro the website sed -i -e 's/id_application: ""/id_application: "'$OPENSTREETMAP_id_key'"/g' $workdir/config/settings.local.yml sed -i -e 's/#id_application: ""/id_application: "'$OPENSTREETMAP_id_key'"/g' $workdir/config/settings.yml -### SET UP OAUTH ID AND KEY +#### Setting up oauth id and key for iD editor sed -i -e 's/OAUTH_CLIENT_ID/'$OAUTH_CLIENT_ID'/g' $workdir/config/settings.local.yml sed -i -e 's/OAUTH_KEY/'$OAUTH_KEY'/g' $workdir/config/settings.local.yml sed -i -e 's/# oauth_application: "OAUTH_CLIENT_ID"/oauth_application: "'$OAUTH_CLIENT_ID'"/g' $workdir/config/settings.yml @@ -43,24 +43,25 @@ sed -i -e 's/# oauth_key: "OAUTH_CLIENT_ID"/oauth_key: "'$OAUTH_KEY'"/g' $workdi #### Setup env vars for memcached server sed -i -e 's/memcache_servers: \[\]/memcache_servers: "'$OPENSTREETMAP_memcache_servers'"/g' $workdir/config/settings.local.yml -## SET NOMINATIM URL +#### Setting up nominatim url sed -i -e 's/nominatim.openhistoricalmap.org/'$NOMINATIM_URL'/g' $workdir/config/settings.local.yml -## SET OVERPASS URL +#### Setting up overpass url sed -i -e 's/overpass-api.de/'$OVERPASS_URL'/g' $workdir/config/settings.local.yml sed -i -e 's/overpass-api.de/'$OVERPASS_URL'/g' $workdir/app/views/site/export.html.erb sed -i -e 's/overpass-api.de/'$OVERPASS_URL'/g' $workdir/app/assets/javascripts/index/export.js -# ADD DOORKEEPER_SIGNING_KEY +#### Adding doorkeeper_signing_key openssl genpkey -algorithm RSA -out private.pem chmod 400 /var/www/private.pem export DOORKEEPER_SIGNING_KEY=$(cat /var/www/private.pem | sed -e '1d;$d' | tr -d '\n') sed -i "s#PRIVATE_KEY#${DOORKEEPER_SIGNING_KEY}#" $workdir/config/settings.local.yml -# UPDATE MAP-STYLES +#### Updating map-styles python3 update_map_styles.py -#### CHECK IF DB IS ALREADY UP AND START THE APP + +#### Checking if db is already up and start the app flag=true while "$flag" = true; do pg_isready -h $POSTGRES_HOST -p 5432 >/dev/null 2>&2 || continue @@ -74,6 +75,9 @@ while "$flag" = true; do # Enable assets:precompile, to take lates changes for assets in $workdir/config/settings.local.yml. time bundle exec rake i18n:js:export assets:precompile + # Since leaflet-ohm-timeslider.css points directly to the svg files, they need to be copied to the public/assets directory. + cp $workdir/public/leaflet-ohm-timeslider-v2/* $$workdir/public/assets/ + bundle exec rails db:migrate # Start cgimap From 73220fb16cacea3306889646a699b600d41c5214 Mon Sep 17 00:00:00 2001 From: Rub21 Date: Thu, 12 Dec 2024 06:43:35 -0500 Subject: [PATCH 4/5] Copy timeslider assets into public assets folder --- images/web/start.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/images/web/start.sh b/images/web/start.sh index eb60ddbb..69a1f705 100755 --- a/images/web/start.sh +++ b/images/web/start.sh @@ -76,7 +76,7 @@ while "$flag" = true; do time bundle exec rake i18n:js:export assets:precompile # Since leaflet-ohm-timeslider.css points directly to the svg files, they need to be copied to the public/assets directory. - cp $workdir/public/leaflet-ohm-timeslider-v2/* $$workdir/public/assets/ + cp $workdir/public/leaflet-ohm-timeslider-v2/assets/* $$workdir/public/assets/ bundle exec rails db:migrate From 2cae2b41c7dc150b80d5c0f6342e5ef1d35ccb75 Mon Sep 17 00:00:00 2001 From: Rub21 Date: Thu, 12 Dec 2024 08:20:26 -0500 Subject: [PATCH 5/5] Update concurrency for tiler seed --- images/web/start.sh | 2 +- values.production.template.yaml | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/images/web/start.sh b/images/web/start.sh index 69a1f705..a47d8731 100755 --- a/images/web/start.sh +++ b/images/web/start.sh @@ -76,7 +76,7 @@ while "$flag" = true; do time bundle exec rake i18n:js:export assets:precompile # Since leaflet-ohm-timeslider.css points directly to the svg files, they need to be copied to the public/assets directory. - cp $workdir/public/leaflet-ohm-timeslider-v2/assets/* $$workdir/public/assets/ + cp $workdir/public/leaflet-ohm-timeslider-v2/assets/* $workdir/public/assets/ bundle exec rails db:migrate diff --git a/values.production.template.yaml b/values.production.template.yaml index ef436f08..fb01e719 100644 --- a/values.production.template.yaml +++ b/values.production.template.yaml @@ -934,7 +934,7 @@ ohm: NODEGROUP_TYPE: web_large # Nodegroup type to run the job # Maximum number of active jobs in high concurrency queue MAX_ACTIVE_JOBS: 10 - DELETE_OLD_JOBS_AGE: 60 # 1 hours + DELETE_OLD_JOBS_AGE: 7200 # 2 hours ## Execute purging EXECUTE_PURGE: true PURGE_CONCURRENCY: 64 @@ -967,9 +967,9 @@ ohm: enabled: true schedule: '0 */6 * * *' env: - GEOJSON_URL: https://osmseed-dev.s3.us-east-1.amazonaws.com/tiler/wold-usa-eu.geojson + GEOJSON_URL: https://osmseed-dev.s3.us-east-1.amazonaws.com/tiler/usa-eu-lat-afr.geojson ZOOM_LEVELS: '7,8,9,10' - CONCURRENCY: 32 + CONCURRENCY: 128 S3_BUCKET: osmseed-dev OUTPUT_FILE: /logs/tiler_benchmark.log resources: