reworked osm2pgsql to work in docker-compose

TUM-Dev · Aug 8, 2024 · 9262582 · 9262582
1 parent b75c402
commit 9262582
Show file tree

Hide file tree

Showing 4 changed files with 174 additions and 22 deletions.
diff --git a/docker-compose.local.yml b/docker-compose.local.yml
@@ -78,26 +78,94 @@ services:
       POSTGRES_DB: ${POSTGRES_DB}
     ports:
       - "5432:5432"
+    volumes:
+      - ./resources/init-db.sh:/docker-entrypoint-initdb.d/init-db.sh:ro
     healthcheck:
       test: [ "CMD", "pg_isready", "-U", "${POSTGRES_USER}" ]
       retries: 5
       interval: 10s
       start_period: 10s
-  osm2pgsql:
+  osm-download-data:
+    image: alpine:latest
+    command:
+      - wget
+      - -c
+      - -O
+      - data.pbf
+      - https://download.geofabrik.de/europe/germany/bayern/oberbayern-latest.osm.pbf
+    working_dir: /data
+    volumes:
+      - ./map/data/:/data/:rw
+  osm2pgsql-init:
+    image: iboates/osm2pgsql:latest
+    environment:
+      TZ: Europe/Berlin
+      PGPASSWORD: ${POSTGRES_PASSWORD}
+    command:
+      - osm2pgsql
+      - --create
+      - --slim
+      - --database
+      - ${POSTGRES_DB}
+      - --user
+      - ${POSTGRES_USER}
+      - --host
+      - db
+      - --port
+      - "5432"
+      - data.pbf
+      - --hstore
+      - --hstore-add-index
+      - --hstore-column
+      - raw
+    working_dir: /data
+    volumes:
+      - ./map/data/:/data/:ro
+    depends_on:
+      osm-download-data:
+        condition: service_completed_successfully
+      db:
+        condition: service_healthy
+  osm2pgsql-replication-init:
     image: iboates/osm2pgsql:latest
     environment:
       TZ: Europe/Berlin
       PGPASSWORD: ${POSTGRES_PASSWORD}
     command:
       - replication
-      - update
-      - --database ${POSTGRES_DB}
-      - --username ${POSTGRES_USER}
-      - --host db
-      - --port 5432
+      - init
+      - --database
+      - ${POSTGRES_DB}
+      - --username
+      - ${POSTGRES_USER}
+      - --host
+      - db
+      - --port
+      - "5432"
     depends_on:
+      osm2pgsql-init:
+        condition: service_completed_successfully
       db:
         condition: service_healthy
+  osm2pgsql-replication:
+    image: iboates/osm2pgsql:latest
+    environment:
+      TZ: Europe/Berlin
+      PGPASSWORD: ${POSTGRES_PASSWORD}
+    command:
+      - replication
+      - update
+      - --database
+      - ${POSTGRES_DB}
+      - --username
+      - ${POSTGRES_USER}
+      - --host
+      - db
+      - --port
+      - "5432"
+    depends_on:
+      osm2pgsql-replication-init:
+        condition: service_completed_successfully
   martin-init-config:
     image: alpine:latest
     command:
@@ -125,6 +193,8 @@ services:
         condition: service_healthy
       martin-init-config:
         condition: service_completed_successfully
+      osm2pgsql-init:
+        condition: service_completed_successfully
     volumes:
       - ./map/martin/:/map/:ro
     healthcheck:

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -183,26 +183,93 @@ services:
       - "5432"
     volumes:
       - postgis-data:/var/lib/postgresql/data
+      - ./resources/init-db.sh:/docker-entrypoint-initdb.d/init-db.sh:ro
     healthcheck:
       test: [ "CMD", "pg_isready", "-U", "${POSTGRES_USER}" ]
       retries: 5
       interval: 10s
       start_period: 10s
-  osm2pgsql:
+  osm-download-data:
+    image: alpine:latest
+    command:
+      - wget
+      - -c
+      - -O
+      - data.pbf
+      - https://download.geofabrik.de/europe/germany-latest.osm.pbf
+    working_dir: /data
+    volumes:
+      - ./map/data/:/data/:rw
+  osm2pgsql-init:
+    image: iboates/osm2pgsql:latest
+    environment:
+      TZ: Europe/Berlin
+      PGPASSWORD: ${POSTGRES_PASSWORD}
+    command:
+      - osm2pgsql
+      - --create
+      - --slim
+      - --database
+      - ${POSTGRES_DB}
+      - --user
+      - ${POSTGRES_USER}
+      - --host
+      - db
+      - --port
+      - "5432"
+      - data.pbf
+      - --hstore
+      - --hstore-add-index
+      - --hstore-column
+      - raw
+    working_dir: /data
+    volumes:
+      - ./map/data/:/data/:ro
+    depends_on:
+      osm-download-data:
+        condition: service_completed_successfully
+      db:
+        condition: service_healthy
+  osm2pgsql-replication-init:
     image: iboates/osm2pgsql:latest
     environment:
       TZ: Europe/Berlin
       PGPASSWORD: ${POSTGRES_PASSWORD}
     command:
       - replication
-      - update
-      - --database ${POSTGRES_DB}
-      - --username ${POSTGRES_USER}
-      - --host db
-      - --port 5432
+      - init
+      - --database
+      - ${POSTGRES_DB}
+      - --username
+      - ${POSTGRES_USER}
+      - --host
+      - db
+      - --port
+      - "5432"
     depends_on:
+      osm2pgsql-init:
+        condition: service_completed_successfully
       db:
         condition: service_healthy
+  osm2pgsql-replication:
+    image: iboates/osm2pgsql:latest
+    environment:
+      TZ: Europe/Berlin
+      PGPASSWORD: ${POSTGRES_PASSWORD}
+    command:
+      - replication
+      - update
+      - --database
+      - ${POSTGRES_DB}
+      - --username
+      - ${POSTGRES_USER}
+      - --host
+      - db
+      - --port
+      - "5432"
+    depends_on:
+      osm2pgsql-replication-init:
+        condition: service_completed_successfully
   martin-init-config:
     image: alpine:latest
     command:
@@ -240,6 +307,8 @@ services:
         condition: service_healthy
       martin-init-config:
         condition: service_completed_successfully
+      osm2pgsql-init:
+        condition: service_completed_successfully
     volumes:
       - ./map/martin/:/map/:ro
     healthcheck:

diff --git a/resources/init-db.sh b/resources/init-db.sh
@@ -0,0 +1,6 @@
+#!/bin/bash
+set -e
+
+psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-EOSQL
+    create extension IF NOT EXISTS hstore
+EOSQL
diff --git a/webclient/README.md b/webclient/README.md
@@ -6,20 +6,22 @@ This folder contains the JavaScript based webclient for NavigaTUM.
 
 ### Prerequisites
 
-For getting started, there are some system dependencys which you will need.
-Please follow the [system dependencys docs](/resources/documentation/Dependencys.md) before trying to run this part of our project.
+For getting started, there are some system dependencies which you will need.
+Please follow the [system dependencies docs](/resources/documentation/Dependencys.md) before trying to run this part of
+our project.
 
 ### Recommended IDE Setup
 
-[VSCode](https://code.visualstudio.com/) + [Volar](https://marketplace.visualstudio.com/items?itemName=Vue.volar) (and disable Vetur) + [TypeScript Vue Plugin (Volar)](https://marketplace.visualstudio.com/items?itemName=Vue.vscode-typescript-vue-plugin).
-Most modern IDEs (like PyCharm) should work as well and have a Plugin.
+[Zed](https://zed.dev/) comes with a profile for all our languages out of the box.
+Most modern IDEs (like the PyCharm+RustRover+WebStorm combination) should work as well.
 
 ## Dependencies
 
 ### Prerequisites
 
 For getting started, there are some system dependencys which you will need.
-Please follow the [system dependencys docs](/resources/documentation/Dependencys.md) before trying to run this part of our project.
+Please follow the [system dependencys docs](/resources/documentation/Dependencys.md) before trying to run this part of
+our project.
 
 ### Installing Dependency's
 
@@ -31,7 +33,8 @@ pnpm install
 
 Ensure that _NavigaTUM-server_ is running in the background.
 By default, the webclient will connect to the server on `https://nav.tum.de`.  
-If you want to connect to a local version instead, change the environemnt variable `NUXT_PUBLIC_{API,CDN,FEEDBACK,MAPS}_URL` to the appropriate value.
+If you want to connect to a local version instead, change the environemnt
+variable `NUXT_PUBLIC_{API,CDN,FEEDBACK,MAPS}_URL` to the appropriate value.
 
 To get a local server running, please:
 
@@ -68,7 +71,8 @@ pnpm run type-refresh
 
 ## Architecture
 
-The NavigaTUM webclient is made as a nuxt3 server side rendered application based on [Vue.js](https://vuejs.org/) and [Vue Router](https://router.vuejs.org/).
+The NavigaTUM webclient is made as a nuxt3 server side rendered application based on [Vue.js](https://vuejs.org/)
+and [Vue Router](https://router.vuejs.org/).
 Our CSS framework is [Tailwind](https://tailwindcss.com/).
 
 ### Directory structure (only the important parts)
@@ -97,14 +101,17 @@ To add a new view, you need to add a new route in `router.ts`.
 
 For this part of the project, the tests consist mainly of hot-path e2e tests and tests of critical components.
 PRs improving the coverage are very likely to be accepted.
-The reason behind these tests is that they fundamentally increase the future productivity by allowing faster review cycles.
+The reason behind these tests is that they fundamentally increase the future productivity by allowing faster review
+cycles.
 
 ### Continuous Integration
 
-Every push and pull request triggers a build that runs linting issues (cypress is currently temporarily disabled to help in the nuxt transition).
+Every push and pull request triggers a build that runs linting issues (cypress is currently temporarily disabled to help
+in the nuxt transition).
 This helps us catch any regressions or issues early in the development process.
 
 ### Reporting Issues
 
-If you encounter any problems while running the Cypress tests or have suggestions for improving the testing framework, please open an issue/pull request on this repository.
+If you encounter any problems while running the Cypress tests or have suggestions for improving the testing framework,
+please open an issue/pull request on this repository.
 We appreciate your feedback and contributions.