Update script for tiler - imposm to inser minute replication files

OpenHistoricalMap · Dec 18, 2024 · 1de8ee8 · 1de8ee8
1 parent d6ad0bb
commit 1de8ee8
Show file tree

Hide file tree

Showing 2 changed files with 51 additions and 15 deletions.
diff --git a/images/tiler-imposm/start.sh b/images/tiler-imposm/start.sh
@@ -105,25 +105,59 @@ function uploadExpiredFiles() {
     done
 }
 
+function uploadLastState() {
+    # Path to the last.state.txt file
+    local state_file="$DIFF_DIR/last.state.txt"
+    local s3_path="${AWS_S3_BUCKET}/${BUCKET_IMPOSM_FOLDER}/last.state.txt"
+    local checksum_file="$DIFF_DIR/last.state.md5"
+
+    # Check if the last.state.txt file exists
+    if [ ! -f "$state_file" ]; then
+        echo "No last.state.txt file found at $state_file. Skipping upload."
+        return
+    fi
+
+    # Calculate the current checksum of the file
+    local current_checksum=$(md5sum "$state_file" | awk '{ print $1 }')
+
+    # Compare with the previous checksum
+    if [ -f "$checksum_file" ]; then
+        local previous_checksum=$(cat "$checksum_file")
+        if [ "$current_checksum" == "$previous_checksum" ]; then
+            echo "No changes in last.state.txt. Skipping upload."
+            return
+        fi
+    fi
+
+    # Attempt to upload the file to S3
+    echo "Uploading $state_file to S3 at $s3_path..."
+    if aws s3 cp "$state_file" "s3://${s3_path}" --acl private; then
+        # Update the checksum file after a successful upload
+        echo "$current_checksum" > "$checksum_file"
+        echo "Successfully uploaded $state_file to S3."
+    fi
+}
+
 function updateData() {
-    ### Update the DB with the new data form minute replication
-    if [ "$OVERWRITE_STATE" = "true" ]; then
-        rm -f $DIFF_DIR/last.state.txt
+
+    local s3_last_state_path="s3://${AWS_S3_BUCKET}/${BUCKET_IMPOSM_FOLDER}${DIFF_DIR}/last.state.txt"
+    local local_last_state_path="$DIFF_DIR/last.state.txt"
+    echo "Checking if $s3_last_state_path exists in S3..."
+    if aws s3 ls "$s3_last_state_path" > /dev/null 2>&1; then
+        echo "Found $s3_last_state_path. Downloading..."
+        aws s3 cp "$s3_last_state_path" "$local_last_state_path"
     fi
 
-    # Check if last.state.txt exists
-    if [ -f "$DIFF_DIR/last.state.txt" ]; then
-        echo "Exist... $DIFF_DIR/last.state.txt"
-    else
-        # Create last.state.txt file with REPLICATION_URL and SEQUENCE_NUMBER from env vars
+    ### Update the DB with the new data from minute replication
+    if [ "$OVERWRITE_STATE" = "true" ]; then
+        echo "Overwriting last.state.txt..."
         echo "timestamp=0001-01-01T00\:00\:00Z 
-        sequenceNumber=$SEQUENCE_NUMBER
-        replicationUrl=$REPLICATION_URL" >$DIFF_DIR/last.state.txt
+        sequenceNumber=${SEQUENCE_NUMBER:-0}
+        replicationUrl=${REPLICATION_URL}" > "$local_last_state_path"
     fi
 
-    # Check if the limit file exists
+    # Run the Imposm update process
     if [ -z "$TILER_IMPORT_LIMIT" ]; then
-        # imposm run -config "$WORKDIR/config.json" -expiretiles-dir "$IMPOSM3_EXPIRE_DIR" -httpprofile ":6060" &
         imposm run \
         -config "${WORKDIR}/config.json" \
         -cachedir "${CACHE_DIR}" \
@@ -144,10 +178,12 @@ function updateData() {
         -quiet &
     fi
 
+    # Continuously upload expired files and last state
     while true; do
         echo "Upload expired files... $(date +%F_%H-%M-%S)"
         uploadExpiredFiles
-        sleep 1m
+        uploadLastState
+        sleep 30s
     done
 }
 

diff --git a/values.staging.template.yaml b/values.staging.template.yaml
@@ -485,8 +485,8 @@ osm-seed:
         TILER_IMPORT_FROM: osm
         TILER_IMPORT_PBF_URL: https://s3.amazonaws.com/planet.openhistoricalmap.org/planet/planet-241214_1202.osm.pbf
         REPLICATION_URL: http://s3.amazonaws.com/planet.openhistoricalmap.org/replication/minute/
-        SEQUENCE_NUMBER: '1675330'
-        OVERWRITE_STATE: false
+        SEQUENCE_NUMBER: '1677612'
+        OVERWRITE_STATE: true
         UPLOAD_EXPIRED_FILES: true
         IMPORT_NATURAL_EARTH: true
         IMPORT_OSM_LAND: true