From 7009b883fc5e303ea7e10ef91bf7a70017f04a9e Mon Sep 17 00:00:00 2001 From: Frank Elsinga Date: Sun, 11 Aug 2024 01:06:16 +0200 Subject: [PATCH 1/5] migrated osm2pgsql to the flex export format --- docker-compose.local.yml | 17 +++-- docker-compose.yml | 19 +++-- map/osm2pgsql/style.lua | 160 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 181 insertions(+), 15 deletions(-) create mode 100644 map/osm2pgsql/style.lua diff --git a/docker-compose.local.yml b/docker-compose.local.yml index 943b07f57..c70cf9e60 100644 --- a/docker-compose.local.yml +++ b/docker-compose.local.yml @@ -114,14 +114,13 @@ services: - db - --port - "5432" - - data.pbf - - --hstore - - --hstore-add-index - - --hstore-column - - raw - working_dir: /data + - /map/data/data.pbf + - --output=flex + - --style + - /map/osm2pgsql/style.lua volumes: - - ./map/data/:/data/:ro + - ./map/data/:/map/data/:ro + - ./map/osm2pgsql/:/map/osm2pgsql/:ro depends_on: osm-download-data: condition: service_completed_successfully @@ -148,6 +147,8 @@ services: condition: service_completed_successfully db: condition: service_healthy + volumes: + - ./map/osm2pgsql/:/map/osm2pgsql/:ro osm2pgsql-replication: image: iboates/osm2pgsql:latest environment: @@ -167,6 +168,8 @@ services: depends_on: osm2pgsql-replication-init: condition: service_completed_successfully + volumes: + - ./map/osm2pgsql/:/map/osm2pgsql/:ro martin-init-config: image: alpine:latest command: diff --git a/docker-compose.yml b/docker-compose.yml index 0b9e7a0a0..dc22f53d1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -211,7 +211,7 @@ services: - --create - --slim - --cache - - "4096" # cache size in MB, maybe underspeced + - "4096" # cache size in MB, maybe too low for optimum - --database - ${POSTGRES_DB} - --user @@ -220,14 +220,13 @@ services: - db - --port - "5432" - - data.pbf - - --hstore - - --hstore-add-index - - --hstore-column - - raw - working_dir: /data + - /map/data/data.pbf + - --output=flex + - --style + - /map/osm2pgsql/style.lua volumes: - - ./map/data/:/data/:ro + - ./map/data/:/map/data/:ro + - ./map/osm2pgsql/:/map/osm2pgsql/:ro depends_on: osm-download-data: condition: service_completed_successfully @@ -256,6 +255,8 @@ services: condition: service_completed_successfully db: condition: service_healthy + volumes: + - ./map/osm2pgsql/:/map/osm2pgsql/:ro osm2pgsql-replication: image: iboates/osm2pgsql:latest networks: @@ -277,6 +278,8 @@ services: depends_on: osm2pgsql-replication-init: condition: service_completed_successfully + volumes: + - ./map/osm2pgsql/:/map/osm2pgsql/:ro martin-init-config: image: alpine:latest command: diff --git a/map/osm2pgsql/style.lua b/map/osm2pgsql/style.lua new file mode 100644 index 000000000..4654b6399 --- /dev/null +++ b/map/osm2pgsql/style.lua @@ -0,0 +1,160 @@ +-- This config exmple file is released into the Public Domain. + +-- This is a very simple Lua config for the Flex output not intended for +-- real-world use. Use it do understand the basic principles of the +-- configuration. After reading and understanding this, have a look at +-- "geometries.lua". + +-- For debugging +-- inspect = require('inspect') + +-- The global variable "osm2pgsql" is used to talk to the main osm2pgsql code. +-- You can, for instance, get the version of osm2pgsql: +print('osm2pgsql version: ' .. osm2pgsql.version) + +-- A place to store the SQL tables we will define shortly. +local tables = {} + +-- Create a new table called "pois" with the given columns. When running in +-- "create" mode, this will do the `CREATE TABLE`, when running in "append" +-- mode, this will only declare the table for use. +-- +-- This is a "node table", it can only contain data derived from nodes and will +-- contain a "node_id" column (SQL type INT8) as first column. When running in +-- "append" mode, osm2pgsql will automatically update this table using the node +-- ids. +tables.pois = osm2pgsql.define_node_table('pois', { + { column = 'tags', type = 'jsonb' }, + { column = 'geom', type = 'point', not_null = true }, -- will be something like `GEOMETRY(Point, 4326)` in SQL +}) + +-- A special table for restaurants to demonstrate that we can have any tables +-- with any columns we want. +tables.restaurants = osm2pgsql.define_node_table('restaurants', { + { column = 'name', type = 'text' }, + { column = 'cuisine', type = 'text' }, + -- We declare all geometry columns as "NOT NULL". If osm2pgsql encounters + -- an invalid geometry (for whatever reason) it will generate a null + -- geometry which will not be written to the database if "not_null" is + -- set. The result is that broken geometries will just be silently + -- ignored. + { column = 'geom', type = 'point', not_null = true }, +}) + +-- This is a "way table", it can only contain data derived from ways and will +-- contain a "way_id" column. When running in "append" mode, osm2pgsql will +-- automatically update this table using the way ids. +tables.ways = osm2pgsql.define_way_table('ways', { + { column = 'tags', type = 'jsonb' }, + { column = 'geom', type = 'linestring', not_null = true }, +}) + +-- This is an "area table", it can contain data derived from ways or relations +-- and will contain an "area_id" column. Way ids will be stored "as is" in the +-- "area_id" column, for relations the negative id will be stored. When +-- running in "append" mode, osm2pgsql will automatically update this table +-- using the way/relation ids. +tables.polygons = osm2pgsql.define_area_table('polygons', { + { column = 'type', type = 'text' }, + { column = 'tags', type = 'jsonb' }, + -- The type of the `geom` column is `geometry`, because we need to store + -- polygons AND multipolygons + { column = 'geom', type = 'geometry', not_null = true }, +}) + +-- Debug output: Show definition of tables +for name, dtable in pairs(tables) do + print("\ntable '" .. name .. "':") + print(" name='" .. dtable:name() .. "'") +-- print(" columns=" .. inspect(dtable:columns())) +end + +-- Helper function to remove some of the tags we usually are not interested in. +-- Returns true if there are no tags left. +local function clean_tags(tags) + tags.odbl = nil + tags.created_by = nil + tags.source = nil + tags['source:ref'] = nil + + return next(tags) == nil +end + +-- Called for every node in the input. The `object` argument contains all the +-- attributes of the node like `id`, `version`, etc. as well as all tags as a +-- Lua table (`object.tags`). +function osm2pgsql.process_node(object) + -- Uncomment next line to look at the object data: + -- print(inspect(object)) + + if clean_tags(object.tags) then + return + end + + if object.tags.amenity == 'restaurant' then + -- Add a row to the SQL table. The keys in the parameter table + -- correspond to the table columns, if one is missing the column will + -- be NULL. Id and geometry columns will be filled automatically. + tables.restaurants:insert({ + name = object.tags.name, + cuisine = object.tags.cuisine, + geom = object:as_point() + }) + else + tables.pois:insert({ + -- We know `tags` is of type `jsonb` so this will do the + -- right thing. + tags = object.tags, + geom = object:as_point() + }) + end +end + +-- Called for every way in the input. The `object` argument contains the same +-- information as with nodes and additionally a boolean `is_closed` flag and +-- the list of node IDs referenced by the way (`object.nodes`). +function osm2pgsql.process_way(object) + -- Uncomment next line to look at the object data: + -- print(inspect(object)) + + if clean_tags(object.tags) then + return + end + + -- Very simple check to decide whether a way is a polygon or not, in a + -- real stylesheet we'd have to also look at the tags... + if object.is_closed then + tables.polygons:insert({ + type = object.type, + tags = object.tags, + geom = object:as_polygon() + }) + else + tables.ways:insert({ + tags = object.tags, + geom = object:as_linestring() + }) + end +end + +-- Called for every relation in the input. The `object` argument contains the +-- same information as with nodes and additionally an array of members +-- (`object.members`). +function osm2pgsql.process_relation(object) + -- Uncomment next line to look at the object data: + -- print(inspect(object)) + + if clean_tags(object.tags) then + return + end + + -- Store multipolygons and boundaries as polygons + if object.tags.type == 'multipolygon' or + object.tags.type == 'boundary' then + tables.polygons:insert({ + type = object.type, + tags = object.tags, + geom = object:as_multipolygon() + }) + end +end From 48911f5b22d36a105b9ae92a3ddb889c6949a8e2 Mon Sep 17 00:00:00 2001 From: Frank Elsinga Date: Sun, 11 Aug 2024 01:32:59 +0200 Subject: [PATCH 2/5] first draft at a simple indoor import --- docker-compose.local.yml | 15 +++---- docker-compose.yml | 12 ++---- map/osm2pgsql/style.lua | 85 ++++++++++++---------------------------- 3 files changed, 35 insertions(+), 77 deletions(-) diff --git a/docker-compose.local.yml b/docker-compose.local.yml index c70cf9e60..e5af7627c 100644 --- a/docker-compose.local.yml +++ b/docker-compose.local.yml @@ -84,17 +84,14 @@ services: test: [ "CMD", "pg_isready", "-U", "${POSTGRES_USER}" ] retries: 5 interval: 10s - start_period: 10s + start_interval: 20s + start_period: 20s osm-download-data: image: alpine:latest - command: - - wget - - -O - - data.pbf - - https://download.geofabrik.de/europe/germany/bayern/oberbayern-latest.osm.pbf - working_dir: /data + command: sh -c "apk add wget && wget --continue --timestamping https://download.geofabrik.de/europe/germany/bayern/oberbayern-latest.osm.pbf" + working_dir: /map/data volumes: - - ./map/data/:/data/:rw + - ./map/data/:/map/data/:rw osm2pgsql-init: image: iboates/osm2pgsql:latest environment: @@ -114,7 +111,7 @@ services: - db - --port - "5432" - - /map/data/data.pbf + - /map/data/oberbayern-latest.osm.pbf - --output=flex - --style - /map/osm2pgsql/style.lua diff --git a/docker-compose.yml b/docker-compose.yml index dc22f53d1..737dcc0bd 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -191,14 +191,10 @@ services: start_period: 10s osm-download-data: image: alpine:latest - command: - - wget - - -O - - data.pbf - - https://download.geofabrik.de/europe/germany-latest.osm.pbf - working_dir: /data + command: apk --update add wget && wget --continue --timestamping https://download.geofabrik.de/europe/germany-latest.osm.pbf + working_dir: /map/data volumes: - - ./map/data/:/data/:rw + - ./map/data/:/map/data/:rw osm2pgsql-init: image: iboates/osm2pgsql:latest networks: @@ -220,7 +216,7 @@ services: - db - --port - "5432" - - /map/data/data.pbf + - /map/data/germany-latest.osm.pbf - --output=flex - --style - /map/osm2pgsql/style.lua diff --git a/map/osm2pgsql/style.lua b/map/osm2pgsql/style.lua index 4654b6399..5b36e3795 100644 --- a/map/osm2pgsql/style.lua +++ b/map/osm2pgsql/style.lua @@ -1,60 +1,26 @@ --- This config exmple file is released into the Public Domain. - --- This is a very simple Lua config for the Flex output not intended for --- real-world use. Use it do understand the basic principles of the --- configuration. After reading and understanding this, have a look at --- "geometries.lua". +-- See https://github.com/osm2pgsql-dev/osm2pgsql/tree/master/flex-config +-- for configuration examples -- For debugging -- inspect = require('inspect') +-- print(inspect(object)) --- The global variable "osm2pgsql" is used to talk to the main osm2pgsql code. --- You can, for instance, get the version of osm2pgsql: print('osm2pgsql version: ' .. osm2pgsql.version) --- A place to store the SQL tables we will define shortly. local tables = {} --- Create a new table called "pois" with the given columns. When running in --- "create" mode, this will do the `CREATE TABLE`, when running in "append" --- mode, this will only declare the table for use. --- --- This is a "node table", it can only contain data derived from nodes and will --- contain a "node_id" column (SQL type INT8) as first column. When running in --- "append" mode, osm2pgsql will automatically update this table using the node --- ids. -tables.pois = osm2pgsql.define_node_table('pois', { - { column = 'tags', type = 'jsonb' }, - { column = 'geom', type = 'point', not_null = true }, -- will be something like `GEOMETRY(Point, 4326)` in SQL -}) --- A special table for restaurants to demonstrate that we can have any tables --- with any columns we want. -tables.restaurants = osm2pgsql.define_node_table('restaurants', { - { column = 'name', type = 'text' }, - { column = 'cuisine', type = 'text' }, - -- We declare all geometry columns as "NOT NULL". If osm2pgsql encounters - -- an invalid geometry (for whatever reason) it will generate a null - -- geometry which will not be written to the database if "not_null" is - -- set. The result is that broken geometries will just be silently - -- ignored. +tables.indoor_nodes = osm2pgsql.define_node_table('indoor_nodes', { + { column = 'tags', type = 'jsonb' }, { column = 'geom', type = 'point', not_null = true }, }) --- This is a "way table", it can only contain data derived from ways and will --- contain a "way_id" column. When running in "append" mode, osm2pgsql will --- automatically update this table using the way ids. -tables.ways = osm2pgsql.define_way_table('ways', { +tables.indoor_ways = osm2pgsql.define_way_table('indoor_ways', { { column = 'tags', type = 'jsonb' }, { column = 'geom', type = 'linestring', not_null = true }, }) --- This is an "area table", it can contain data derived from ways or relations --- and will contain an "area_id" column. Way ids will be stored "as is" in the --- "area_id" column, for relations the negative id will be stored. When --- running in "append" mode, osm2pgsql will automatically update this table --- using the way/relation ids. -tables.polygons = osm2pgsql.define_area_table('polygons', { +tables.indoor_polygons = osm2pgsql.define_area_table('indoor_polygons', { { column = 'type', type = 'text' }, { column = 'tags', type = 'jsonb' }, -- The type of the `geom` column is `geometry`, because we need to store @@ -91,23 +57,14 @@ function osm2pgsql.process_node(object) return end - if object.tags.amenity == 'restaurant' then - -- Add a row to the SQL table. The keys in the parameter table - -- correspond to the table columns, if one is missing the column will - -- be NULL. Id and geometry columns will be filled automatically. - tables.restaurants:insert({ - name = object.tags.name, - cuisine = object.tags.cuisine, - geom = object:as_point() - }) - else - tables.pois:insert({ - -- We know `tags` is of type `jsonb` so this will do the - -- right thing. - tags = object.tags, - geom = object:as_point() - }) + if object.tags.indoor == nil and object.tags.level == nil then + return end + + tables.indoor_nodes:insert({ + tags = object.tags, + geom = object:as_point() + }) end -- Called for every way in the input. The `object` argument contains the same @@ -121,16 +78,20 @@ function osm2pgsql.process_way(object) return end + if object.tags.indoor == nil and object.tags.level == nil then + return + end + -- Very simple check to decide whether a way is a polygon or not, in a -- real stylesheet we'd have to also look at the tags... if object.is_closed then - tables.polygons:insert({ + tables.indoor_polygons:insert({ type = object.type, tags = object.tags, geom = object:as_polygon() }) else - tables.ways:insert({ + tables.indoor_ways:insert({ tags = object.tags, geom = object:as_linestring() }) @@ -148,10 +109,14 @@ function osm2pgsql.process_relation(object) return end + if object.tags.indoor == nil and object.tags.level == nil then + return + end + -- Store multipolygons and boundaries as polygons if object.tags.type == 'multipolygon' or object.tags.type == 'boundary' then - tables.polygons:insert({ + tables.indoor_polygons:insert({ type = object.type, tags = object.tags, geom = object:as_multipolygon() From 92c54d5aeef90ae77a2ff10126504f03a88a3043 Mon Sep 17 00:00:00 2001 From: Frank Elsinga Date: Mon, 12 Aug 2024 23:56:58 +0200 Subject: [PATCH 3/5] made martin expose the relevant tables in the catalog --- map/martin/config.yaml | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/map/martin/config.yaml b/map/martin/config.yaml index b31226df7..17533f140 100644 --- a/map/martin/config.yaml +++ b/map/martin/config.yaml @@ -8,7 +8,7 @@ fonts: postgres: connection_string: $DATABASE_URL auto_publish: false - tables: {} + tables: # addrfeat: # schema: tiger # table: addrfeat @@ -231,6 +231,34 @@ postgres: # vtdst00: varchar # zcta5ce: varchar # zcta5ce00: varchar + indoor_nodes: + schema: public + table: indoor_nodes + srid: 3857 + geometry_column: geom + geometry_type: POINT + properties: + node_id: int8 + tags: jsonb + indoor_polygons: + schema: public + table: indoor_polygons + srid: 3857 + geometry_column: geom + geometry_type: GEOMETRY + properties: + area_id: int8 + tags: jsonb + type: text + indoor_ways: + schema: public + table: indoor_ways + srid: 3857 + geometry_column: geom + geometry_type: LINESTRING + properties: + tags: jsonb + way_id: int8 # place: # schema: tiger # table: place From b913c3fe0aa25a2ace89cb1d4b994e0167c9e9c1 Mon Sep 17 00:00:00 2001 From: Frank Elsinga Date: Tue, 13 Aug 2024 00:48:00 +0200 Subject: [PATCH 4/5] apk usage --- docker-compose.local.yml | 2 +- docker-compose.yml | 5 +++-- server/Dockerfile | 2 +- webclient/Dockerfile | 3 ++- 4 files changed, 7 insertions(+), 5 deletions(-) diff --git a/docker-compose.local.yml b/docker-compose.local.yml index e5af7627c..bb5a946bd 100644 --- a/docker-compose.local.yml +++ b/docker-compose.local.yml @@ -88,7 +88,7 @@ services: start_period: 20s osm-download-data: image: alpine:latest - command: sh -c "apk add wget && wget --continue --timestamping https://download.geofabrik.de/europe/germany/bayern/oberbayern-latest.osm.pbf" + command: sh -c "apk --update --quiet add wget && wget --continue --timestamping https://download.geofabrik.de/europe/germany/bayern/oberbayern-latest.osm.pbf" working_dir: /map/data volumes: - ./map/data/:/map/data/:rw diff --git a/docker-compose.yml b/docker-compose.yml index 737dcc0bd..2f5201743 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -188,10 +188,11 @@ services: test: [ "CMD", "pg_isready", "-U", "${POSTGRES_USER}" ] retries: 5 interval: 10s - start_period: 10s + start_interval: 20s + start_period: 20s osm-download-data: image: alpine:latest - command: apk --update add wget && wget --continue --timestamping https://download.geofabrik.de/europe/germany-latest.osm.pbf + command: apk --update --quiet add wget && wget --continue --timestamping https://download.geofabrik.de/europe/germany-latest.osm.pbf working_dir: /map/data volumes: - ./map/data/:/map/data/:rw diff --git a/server/Dockerfile b/server/Dockerfile index 453213a38..1d6553b3a 100644 --- a/server/Dockerfile +++ b/server/Dockerfile @@ -9,7 +9,7 @@ FROM rust:1.80-alpine AS compiler # - musl-dev is needed for musl to compile the binary # - mold is used to link faster # - I somehow could not get openssl to cooperate => we are contibuing with libpq-dev -RUN apk add -q --update-cache --no-cache musl-dev libpq-dev mold +RUN apk --update add -q --update-cache --no-cache musl-dev libpq-dev mold WORKDIR /compiler ENV USER=root diff --git a/webclient/Dockerfile b/webclient/Dockerfile index 9ef5107ff..8cd946e83 100644 --- a/webclient/Dockerfile +++ b/webclient/Dockerfile @@ -27,7 +27,8 @@ RUN pnpm run build FROM node:21-alpine3.19 AS production-stage -RUN apk update --no-progress --quiet && apk add curl --no-progress --quiet +RUN apk update --no-progress --quiet && \ + apk add --no-cache --no-progress --quiet curl COPY --from=build-stage /app/.output /app/.output COPY --from=build-stage /app/node_modules /app/node_modules From 257e207b3da7bda7b4efca3005cdf582bf67af0c Mon Sep 17 00:00:00 2001 From: Frank Elsinga Date: Tue, 13 Aug 2024 00:49:28 +0200 Subject: [PATCH 5/5] removed odd `sh -c` --- docker-compose.local.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.local.yml b/docker-compose.local.yml index bb5a946bd..b186f1953 100644 --- a/docker-compose.local.yml +++ b/docker-compose.local.yml @@ -88,7 +88,7 @@ services: start_period: 20s osm-download-data: image: alpine:latest - command: sh -c "apk --update --quiet add wget && wget --continue --timestamping https://download.geofabrik.de/europe/germany/bayern/oberbayern-latest.osm.pbf" + command: apk --update --quiet add wget && wget --continue --timestamping https://download.geofabrik.de/europe/germany/bayern/oberbayern-latest.osm.pbf working_dir: /map/data volumes: - ./map/data/:/map/data/:rw