From bad2001594e096964af6ffbdcb1ee247c08f113e Mon Sep 17 00:00:00 2001 From: Frank Elsinga Date: Tue, 13 Aug 2024 01:23:36 +0200 Subject: [PATCH] Flex indoor style (#1435) * migrated osm2pgsql to the flex export format * first draft at a simple indoor import * made martin expose the relevant tables in the catalog * apk usage * removed odd `sh -c` --- docker-compose.local.yml | 30 +++++----- docker-compose.yml | 32 +++++----- map/martin/config.yaml | 30 +++++++++- map/osm2pgsql/style.lua | 125 +++++++++++++++++++++++++++++++++++++++ server/Dockerfile | 2 +- webclient/Dockerfile | 3 +- 6 files changed, 188 insertions(+), 34 deletions(-) create mode 100644 map/osm2pgsql/style.lua diff --git a/docker-compose.local.yml b/docker-compose.local.yml index 943b07f57..b186f1953 100644 --- a/docker-compose.local.yml +++ b/docker-compose.local.yml @@ -84,17 +84,14 @@ services: test: [ "CMD", "pg_isready", "-U", "${POSTGRES_USER}" ] retries: 5 interval: 10s - start_period: 10s + start_interval: 20s + start_period: 20s osm-download-data: image: alpine:latest - command: - - wget - - -O - - data.pbf - - https://download.geofabrik.de/europe/germany/bayern/oberbayern-latest.osm.pbf - working_dir: /data + command: apk --update --quiet add wget && wget --continue --timestamping https://download.geofabrik.de/europe/germany/bayern/oberbayern-latest.osm.pbf + working_dir: /map/data volumes: - - ./map/data/:/data/:rw + - ./map/data/:/map/data/:rw osm2pgsql-init: image: iboates/osm2pgsql:latest environment: @@ -114,14 +111,13 @@ services: - db - --port - "5432" - - data.pbf - - --hstore - - --hstore-add-index - - --hstore-column - - raw - working_dir: /data + - /map/data/oberbayern-latest.osm.pbf + - --output=flex + - --style + - /map/osm2pgsql/style.lua volumes: - - ./map/data/:/data/:ro + - ./map/data/:/map/data/:ro + - ./map/osm2pgsql/:/map/osm2pgsql/:ro depends_on: osm-download-data: condition: service_completed_successfully @@ -148,6 +144,8 @@ services: condition: service_completed_successfully db: condition: service_healthy + volumes: + - ./map/osm2pgsql/:/map/osm2pgsql/:ro osm2pgsql-replication: image: iboates/osm2pgsql:latest environment: @@ -167,6 +165,8 @@ services: depends_on: osm2pgsql-replication-init: condition: service_completed_successfully + volumes: + - ./map/osm2pgsql/:/map/osm2pgsql/:ro martin-init-config: image: alpine:latest command: diff --git a/docker-compose.yml b/docker-compose.yml index 0b9e7a0a0..2f5201743 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -188,17 +188,14 @@ services: test: [ "CMD", "pg_isready", "-U", "${POSTGRES_USER}" ] retries: 5 interval: 10s - start_period: 10s + start_interval: 20s + start_period: 20s osm-download-data: image: alpine:latest - command: - - wget - - -O - - data.pbf - - https://download.geofabrik.de/europe/germany-latest.osm.pbf - working_dir: /data + command: apk --update --quiet add wget && wget --continue --timestamping https://download.geofabrik.de/europe/germany-latest.osm.pbf + working_dir: /map/data volumes: - - ./map/data/:/data/:rw + - ./map/data/:/map/data/:rw osm2pgsql-init: image: iboates/osm2pgsql:latest networks: @@ -211,7 +208,7 @@ services: - --create - --slim - --cache - - "4096" # cache size in MB, maybe underspeced + - "4096" # cache size in MB, maybe too low for optimum - --database - ${POSTGRES_DB} - --user @@ -220,14 +217,13 @@ services: - db - --port - "5432" - - data.pbf - - --hstore - - --hstore-add-index - - --hstore-column - - raw - working_dir: /data + - /map/data/germany-latest.osm.pbf + - --output=flex + - --style + - /map/osm2pgsql/style.lua volumes: - - ./map/data/:/data/:ro + - ./map/data/:/map/data/:ro + - ./map/osm2pgsql/:/map/osm2pgsql/:ro depends_on: osm-download-data: condition: service_completed_successfully @@ -256,6 +252,8 @@ services: condition: service_completed_successfully db: condition: service_healthy + volumes: + - ./map/osm2pgsql/:/map/osm2pgsql/:ro osm2pgsql-replication: image: iboates/osm2pgsql:latest networks: @@ -277,6 +275,8 @@ services: depends_on: osm2pgsql-replication-init: condition: service_completed_successfully + volumes: + - ./map/osm2pgsql/:/map/osm2pgsql/:ro martin-init-config: image: alpine:latest command: diff --git a/map/martin/config.yaml b/map/martin/config.yaml index b31226df7..17533f140 100644 --- a/map/martin/config.yaml +++ b/map/martin/config.yaml @@ -8,7 +8,7 @@ fonts: postgres: connection_string: $DATABASE_URL auto_publish: false - tables: {} + tables: # addrfeat: # schema: tiger # table: addrfeat @@ -231,6 +231,34 @@ postgres: # vtdst00: varchar # zcta5ce: varchar # zcta5ce00: varchar + indoor_nodes: + schema: public + table: indoor_nodes + srid: 3857 + geometry_column: geom + geometry_type: POINT + properties: + node_id: int8 + tags: jsonb + indoor_polygons: + schema: public + table: indoor_polygons + srid: 3857 + geometry_column: geom + geometry_type: GEOMETRY + properties: + area_id: int8 + tags: jsonb + type: text + indoor_ways: + schema: public + table: indoor_ways + srid: 3857 + geometry_column: geom + geometry_type: LINESTRING + properties: + tags: jsonb + way_id: int8 # place: # schema: tiger # table: place diff --git a/map/osm2pgsql/style.lua b/map/osm2pgsql/style.lua new file mode 100644 index 000000000..5b36e3795 --- /dev/null +++ b/map/osm2pgsql/style.lua @@ -0,0 +1,125 @@ +-- See https://github.com/osm2pgsql-dev/osm2pgsql/tree/master/flex-config +-- for configuration examples + +-- For debugging +-- inspect = require('inspect') +-- print(inspect(object)) + +print('osm2pgsql version: ' .. osm2pgsql.version) + +local tables = {} + + +tables.indoor_nodes = osm2pgsql.define_node_table('indoor_nodes', { + { column = 'tags', type = 'jsonb' }, + { column = 'geom', type = 'point', not_null = true }, +}) + +tables.indoor_ways = osm2pgsql.define_way_table('indoor_ways', { + { column = 'tags', type = 'jsonb' }, + { column = 'geom', type = 'linestring', not_null = true }, +}) + +tables.indoor_polygons = osm2pgsql.define_area_table('indoor_polygons', { + { column = 'type', type = 'text' }, + { column = 'tags', type = 'jsonb' }, + -- The type of the `geom` column is `geometry`, because we need to store + -- polygons AND multipolygons + { column = 'geom', type = 'geometry', not_null = true }, +}) + +-- Debug output: Show definition of tables +for name, dtable in pairs(tables) do + print("\ntable '" .. name .. "':") + print(" name='" .. dtable:name() .. "'") +-- print(" columns=" .. inspect(dtable:columns())) +end + +-- Helper function to remove some of the tags we usually are not interested in. +-- Returns true if there are no tags left. +local function clean_tags(tags) + tags.odbl = nil + tags.created_by = nil + tags.source = nil + tags['source:ref'] = nil + + return next(tags) == nil +end + +-- Called for every node in the input. The `object` argument contains all the +-- attributes of the node like `id`, `version`, etc. as well as all tags as a +-- Lua table (`object.tags`). +function osm2pgsql.process_node(object) + -- Uncomment next line to look at the object data: + -- print(inspect(object)) + + if clean_tags(object.tags) then + return + end + + if object.tags.indoor == nil and object.tags.level == nil then + return + end + + tables.indoor_nodes:insert({ + tags = object.tags, + geom = object:as_point() + }) +end + +-- Called for every way in the input. The `object` argument contains the same +-- information as with nodes and additionally a boolean `is_closed` flag and +-- the list of node IDs referenced by the way (`object.nodes`). +function osm2pgsql.process_way(object) + -- Uncomment next line to look at the object data: + -- print(inspect(object)) + + if clean_tags(object.tags) then + return + end + + if object.tags.indoor == nil and object.tags.level == nil then + return + end + + -- Very simple check to decide whether a way is a polygon or not, in a + -- real stylesheet we'd have to also look at the tags... + if object.is_closed then + tables.indoor_polygons:insert({ + type = object.type, + tags = object.tags, + geom = object:as_polygon() + }) + else + tables.indoor_ways:insert({ + tags = object.tags, + geom = object:as_linestring() + }) + end +end + +-- Called for every relation in the input. The `object` argument contains the +-- same information as with nodes and additionally an array of members +-- (`object.members`). +function osm2pgsql.process_relation(object) + -- Uncomment next line to look at the object data: + -- print(inspect(object)) + + if clean_tags(object.tags) then + return + end + + if object.tags.indoor == nil and object.tags.level == nil then + return + end + + -- Store multipolygons and boundaries as polygons + if object.tags.type == 'multipolygon' or + object.tags.type == 'boundary' then + tables.indoor_polygons:insert({ + type = object.type, + tags = object.tags, + geom = object:as_multipolygon() + }) + end +end diff --git a/server/Dockerfile b/server/Dockerfile index 453213a38..1d6553b3a 100644 --- a/server/Dockerfile +++ b/server/Dockerfile @@ -9,7 +9,7 @@ FROM rust:1.80-alpine AS compiler # - musl-dev is needed for musl to compile the binary # - mold is used to link faster # - I somehow could not get openssl to cooperate => we are contibuing with libpq-dev -RUN apk add -q --update-cache --no-cache musl-dev libpq-dev mold +RUN apk --update add -q --update-cache --no-cache musl-dev libpq-dev mold WORKDIR /compiler ENV USER=root diff --git a/webclient/Dockerfile b/webclient/Dockerfile index 9ef5107ff..8cd946e83 100644 --- a/webclient/Dockerfile +++ b/webclient/Dockerfile @@ -27,7 +27,8 @@ RUN pnpm run build FROM node:21-alpine3.19 AS production-stage -RUN apk update --no-progress --quiet && apk add curl --no-progress --quiet +RUN apk update --no-progress --quiet && \ + apk add --no-cache --no-progress --quiet curl COPY --from=build-stage /app/.output /app/.output COPY --from=build-stage /app/node_modules /app/node_modules