migrated the docker based download into a download script as the comp…

…lexity is not managable otherwise
TUM-Dev · Oct 21, 2024 · d686d32 · d686d32
1 parent 2b11cbe
commit d686d32
Show file tree

Hide file tree

Showing 3 changed files with 50 additions and 32 deletions.
diff --git a/docker-compose.local.yml b/docker-compose.local.yml
@@ -89,27 +89,21 @@ services:
       # requires docker v25 to be available for ubuntu
       # start_interval: 20s
       start_period: 20s
-  osm-download-data:
+  download-data:
     image: alpine:latest
-    command: sh -c "apk --update add --quiet wget && wget https://download.geofabrik.de/europe/germany/bayern/oberbayern-latest.osm.pbf"
+    command: sh /map/download.sh europe/germany/bayern/oberbayern
     environment:
       TZ: Europe/Berlin
-    working_dir: /map/data
+    working_dir: /map
     volumes:
       - ./map/data/:/map/data/:rw
-  gtfs-download-data:
-    image: alpine:latest
-    ## aah, MVV, warum machst du so sachen???
-    command: sh -c "apk --update add --quiet wget && wget https://www.opendata-oepnv.de/dataset/17065229-c3fd-46d7-84a9-aae55aadbf40/resource/927d0830-2a40-4702-acc6-f5716352b666/download/gtfs_mvv_mitshape_240814.zip && unzip -d mvv gtfs_mvv*.zip && rm *.zip"
-    environment:
-      TZ: Europe/Berlin
-    working_dir: /gtfs_feeds
-    volumes:
-      - ./map/gtfs_feeds/:/gtfs_feeds/:rw
+      - ./map/gtfs_feeds/:/map/gtfs_feeds/:rw
+      - ./map/download.sh:/map/download.sh:ro
   valhalla-build:
     image: ghcr.io/gis-ops/docker-valhalla/valhalla:latest
     volumes:
       - ./map/data/:/custom_files:rw
+      - ./map/download.sh:/local/bin:ro
       - ./map/gtfs_feeds:/gtfs_feeds:rw
     environment:
       TZ: Europe/Berlin
@@ -120,9 +114,7 @@ services:
       use_default_speeds_config: True
       serve_tiles: False
     depends_on:
-      osm-download-data:
-        condition: service_completed_successfully
-      gtfs-download-data:
+      download-data:
         condition: service_completed_successfully
   valhalla:
     image: ghcr.io/gis-ops/docker-valhalla/valhalla:latest
@@ -170,7 +162,7 @@ services:
       - ./map/data/:/map/data/:ro
       - ./map/osm2pgsql/:/map/osm2pgsql/:ro
     depends_on:
-      osm-download-data:
+      download-data:
         condition: service_completed_successfully
       db:
         condition: service_healthy

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -193,23 +193,16 @@ services:
       # requires docker v25 to be available for ubuntu
       # start_interval: 20s
       start_period: 20s
-  osm-download-data:
+  download-data:
     image: alpine:latest
-    command: sh -c "apk --update add --quiet wget && wget https://download.geofabrik.de/europe/germany/bayern/oberbayern-latest.osm.pbf -o nul"
+    command: sh /map/download.sh europe/germany/bayern/oberbayern
     environment:
       TZ: Europe/Berlin
-    working_dir: /map/data
+    working_dir: /map
     volumes:
       - ./map/data/:/map/data/:rw
-  gtfs-download-data:
-    image: alpine:latest
-    ## aah, MVV, warum machst du so sachen???
-    command: sh -c "apk --update add --quiet wget && wget https://www.opendata-oepnv.de/dataset/17065229-c3fd-46d7-84a9-aae55aadbf40/resource/927d0830-2a40-4702-acc6-f5716352b666/download/gtfs_mvv_mitshape_240814.zip && unzip -d mvv gtfs_mvv*.zip && rm *.zip"
-    environment:
-      TZ: Europe/Berlin
-    working_dir: /gtfs_feeds
-    volumes:
-      - ./map/gtfs_feeds/:/gtfs_feeds/:rw
+      - ./map/gtfs_feeds/:/map/gtfs_feeds/:rw
+      - ./map/download.sh:/map/download.sh:ro
   valhalla-build:
     image: ghcr.io/gis-ops/docker-valhalla/valhalla:latest
     volumes:
@@ -224,9 +217,7 @@ services:
       use_default_speeds_config: True
       serve_tiles: False
     depends_on:
-      osm-download-data:
-        condition: service_completed_successfully
-      gtfs-download-data:
+      download-data:
         condition: service_completed_successfully
   valhalla:
     image: ghcr.io/gis-ops/docker-valhalla/valhalla:latest
@@ -284,7 +275,7 @@ services:
       - ./map/data/:/map/data/:ro
       - ./map/osm2pgsql/:/map/osm2pgsql/:ro
     depends_on:
-      osm-download-data:
+      download-data:
         condition: service_completed_successfully
       db:
         condition: service_healthy

diff --git a/map/download.sh b/map/download.sh
@@ -0,0 +1,35 @@
+#!/bin/bash
+
+set -o errexit
+
+cd "$(dirname "$0")"
+
+echo "-- cleanup $(dirname "$0") --"
+rm -rf ./data/* ./gtfs_feeds/*
+mkdir --parents gtfs_feeds
+mkdir --parents data/transit_tiles
+
+echo -- download dependencys --
+if command -v apk > /dev/null 2>&1
+then
+  apk --update add --quiet wget
+else
+  echo "[WARNING] skipping apk installation of wget. Please make sure it is installed"
+fi
+
+
+echo -- download geodata --
+cd data || exit 1
+wget "https://download.geofabrik.de/$1-latest.osm.pbf" --tries=5 --random-wait --wait=5
+cd .. || exit 1
+
+echo -- download gtfs feeds --
+cd gtfs_feeds || exit 1
+# aah, MVV, warum machst du so Sachen???
+# TODO: look into https://www.delfi.de/de/leistungen-produkte/daten-dienste/
+# TODO: look into https://www.transit.land/feeds?search=germany
+# TODO: look into https://github.com/transitland/transitland-atlas/pull/1268
+wget --tries=5 https://www.opendata-oepnv.de/dataset/17065229-c3fd-46d7-84a9-aae55aadbf40/resource/927d0830-2a40-4702-acc6-f5716352b666/download/gtfs_mvv_mitshape_240814.zip --output-document mvv.zip --tries=5 --random-wait --wait=5
+unzip -q -d mvv mvv.zip
+rm mvv.zip
+cd .. || exit 1