From f440eced4d5eb6b9c0e2e031e9e02e5c2acb61ad Mon Sep 17 00:00:00 2001 From: Thomas Petillon Date: Sun, 17 Nov 2024 12:29:28 +0100 Subject: [PATCH 1/5] Adapt Docker setup to latest changes And use newer versions of base images. --- .dockerignore | 1 - Dockerfile | 11 ++++++----- Dockerfile.db | 2 +- Dockerfile.import | 16 ++-------------- docker-compose.yml | 6 +++--- scripts/docker-startup.sh | 11 ++++++++--- 6 files changed, 20 insertions(+), 27 deletions(-) diff --git a/.dockerignore b/.dockerignore index ad148a9e84..afd540194c 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,3 +1,2 @@ * !scripts/tune-postgis.sh -!openstreetmap-carto.style diff --git a/Dockerfile b/Dockerfile index 45d346f8f4..0e2ba3f139 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,18 +1,19 @@ -FROM ubuntu:focal +FROM ubuntu:noble # https://serverfault.com/questions/949991/how-to-install-tzdata-on-a-ubuntu-docker-image ARG DEBIAN_FRONTEND=noninteractive # Style dependencies RUN apt-get update && apt-get install --no-install-recommends -y \ - ca-certificates curl gnupg postgresql-client python3 python3-distutils \ - fonts-hanazono fonts-noto-cjk fonts-noto-hinted fonts-noto-unhinted \ - mapnik-utils nodejs npm ttf-unifont unzip git && rm -rf /var/lib/apt/lists/* + ca-certificates curl gnupg postgresql-client python3 \ + fonts-hanazono fonts-noto-cjk fonts-noto-hinted fonts-noto-unhinted fonts-unifont \ + mapnik-utils nodejs npm unzip git && rm -rf /var/lib/apt/lists/* # Kosmtik with plugins, forcing prefix to /usr because Ubuntu sets # npm prefix to /usr/local, which breaks the install # We install kosmtik not from release channel, but directly from a specific commit on github. -RUN npm set prefix /usr && npm install -g --unsafe-perm "git+https://git@github.com/kosmtik/kosmtik.git" +# 5dbde8db6b5e22073951066b0646a91c10bb81a5 is master's tip as of 2024-11-17. +RUN npm set prefix /usr && npm install -g --unsafe-perm "git+https://git@github.com/kosmtik/kosmtik.git#5dbde8db6b5e22073951066b0646a91c10bb81a5" WORKDIR /usr/lib/node_modules/kosmtik/ RUN kosmtik plugins --install kosmtik-overpass-layer \ diff --git a/Dockerfile.db b/Dockerfile.db index 489c073455..17a0e01e51 100644 --- a/Dockerfile.db +++ b/Dockerfile.db @@ -1,3 +1,3 @@ -FROM postgis/postgis:10-2.5-alpine +FROM postgis/postgis:15-3.5-alpine COPY ./scripts/tune-postgis.sh /docker-entrypoint-initdb.d/tune-postgis.sh diff --git a/Dockerfile.import b/Dockerfile.import index 685471657e..f10ec45ad3 100644 --- a/Dockerfile.import +++ b/Dockerfile.import @@ -1,21 +1,9 @@ -FROM ubuntu:bionic +FROM ubuntu:noble RUN apt-get update && apt-get install --no-install-recommends -y \ - ca-certificates curl gnupg && rm -rf /var/lib/apt/lists/* - -RUN echo 'deb http://ppa.launchpad.net/osmadmins/ppa/ubuntu bionic main\n\ -deb-src http://ppa.launchpad.net/osmadmins/ppa/ubuntu bionic main' > \ - /etc/apt/sources.list.d/osmadmins-ppa.list - -RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 \ - --recv A438A16C88C6BE41CB1616B8D57F48750AC4F2CB - -RUN apt-get update && apt-get install --no-install-recommends -y \ - osm2pgsql gdal-bin python3-psycopg2 python3-yaml unzip \ + osm2pgsql gdal-bin python3-psycopg2 python3-yaml curl unzip \ python3-requests postgresql-client && rm -rf /var/lib/apt/lists/* -ADD openstreetmap-carto-flex.lua / - RUN mkdir -p /openstreetmap-carto WORKDIR /openstreetmap-carto diff --git a/docker-compose.yml b/docker-compose.yml index 8f1aa62990..4e91674c2c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,7 +1,7 @@ version: '2' services: kosmtik: - image: kosmtik:v1 + image: openstreetmap-carto-kosmtik:v1 platform: linux/amd64 build: context: . @@ -16,7 +16,7 @@ services: - PGHOST=db - PGUSER=postgres db: - image: db:v1 + image: openstreetmap-carto-db:v1 build: context: . dockerfile: Dockerfile.db @@ -25,7 +25,7 @@ services: - PG_WORK_MEM - PG_MAINTENANCE_WORK_MEM import: - image: import:v1 + image: openstreetmap-carto-import:v1 build: context: . dockerfile: Dockerfile.import diff --git a/scripts/docker-startup.sh b/scripts/docker-startup.sh index 6b60fe7b58..0ecc25041b 100644 --- a/scripts/docker-startup.sh +++ b/scripts/docker-startup.sh @@ -18,9 +18,10 @@ test $i -gt $MAXCOUNT && echo "Timeout while waiting for PostgreSQL to be runnin case "$1" in import) # Creating default database - psql -c "SELECT 1 FROM pg_database WHERE datname = 'gis';" | grep -q 1 || createdb gis && \ - psql -d gis -c 'CREATE EXTENSION IF NOT EXISTS postgis;' && \ - psql -d gis -c 'CREATE EXTENSION IF NOT EXISTS hstore;' && \ + psql -c "SELECT 1 FROM pg_database WHERE datname = 'gis';" | grep -q 1 || createdb gis + psql -d gis -c 'CREATE EXTENSION IF NOT EXISTS postgis;' + psql -d gis -c 'CREATE EXTENSION IF NOT EXISTS hstore;' + psql -d gis -c 'ALTER SYSTEM SET jit=off;' -c 'SELECT pg_reload_conf();' # Creating default import settings file editable by user and passing values for osm2pgsql if [ ! -e ".env" ]; then @@ -50,6 +51,10 @@ EOF --style openstreetmap-carto-flex.lua \ $OSM2PGSQL_DATAFILE + # Setting up indexes and functions + psql -d gis -f indexes.sql + psql -d gis -f functions.sql + # Downloading and importing needed shapefiles scripts/get-external-data.py $EXTERNAL_DATA_SCRIPT_FLAGS From a431b51e8373a9a8f6efbe7655d9e8c6831849b7 Mon Sep 17 00:00:00 2001 From: Thomas Petillon Date: Sat, 14 Dec 2024 00:05:25 +0100 Subject: [PATCH 2/5] Remove fonts from Dockerfile, as they are downloaded separately --- Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 0e2ba3f139..f526dd1ff1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,9 +5,9 @@ ARG DEBIAN_FRONTEND=noninteractive # Style dependencies RUN apt-get update && apt-get install --no-install-recommends -y \ - ca-certificates curl gnupg postgresql-client python3 \ - fonts-hanazono fonts-noto-cjk fonts-noto-hinted fonts-noto-unhinted fonts-unifont \ - mapnik-utils nodejs npm unzip git && rm -rf /var/lib/apt/lists/* + ca-certificates gnupg postgresql-client curl unzip python3 \ + nodejs npm git fonts-unifont mapnik-utils \ + && rm -rf /var/lib/apt/lists/* # Kosmtik with plugins, forcing prefix to /usr because Ubuntu sets # npm prefix to /usr/local, which breaks the install From 13dabdf37354dd108a9932e41216b788f2a90d62 Mon Sep 17 00:00:00 2001 From: Thomas Petillon Date: Fri, 13 Dec 2024 23:38:35 +0100 Subject: [PATCH 3/5] Mention the external data download in Docker documentation --- DOCKER.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/DOCKER.md b/DOCKER.md index 8d6964d832..3973cca760 100644 --- a/DOCKER.md +++ b/DOCKER.md @@ -11,9 +11,9 @@ PostgreSQL database to store the OpenStreetMap data and [Kosmtik](https://github to be able to run Docker containers. You also need Docker Compose, which should be available once you install Docker itself. Otherwise, you need to [install Docker Compose manually](https://docs.docker.com/compose/install/). -* You need sufficient disk space of _several Gigabytes_. - * Docker creates a disk image for its virtual machine that holds the virtualised operating system and the containers. - * The format (Docker.raw, Docker.qcow2, \*.vhdx, etc.) depends on the host system. It can be a sparse file allocating large amounts of disk space, but still the physical size starts with 2-3 GB for the virtual OS and grows to 6-7 GB when filled with the containers needed for the database, Kosmtik, and a chosen small region of OSM data. +* You need sufficient disk space of _several Gigabytes_. + * Docker creates a disk image for its virtual machine that holds the virtualised operating system and the containers. + * The format (Docker.raw, Docker.qcow2, \*.vhdx, etc.) depends on the host system. It can be a sparse file allocating large amounts of disk space, but still the physical size starts with 2-3 GB for the virtual OS and grows to 6-7 GB when filled with the containers needed for the database, Kosmtik, and a chosen small region of OSM data. * An additional 1-2 GB are needed for shapefiles to be downloaded and stored in the openstreetmap-carto/data repository. ## Quick start @@ -23,7 +23,7 @@ If you are eager to get started, here is an overview over the necessary steps: * `git clone https://github.com/gravitystorm/openstreetmap-carto.git` to clone openstreetmap-carto repository into a directory on your host system * Download OpenStreetMap data in osm.pbf format to a file `data.osm.pbf` and place it within the openstreetmap-carto directory (for example some small area from [Geofabrik](https://download.geofabrik.de/)) * If necessary, `sudo service postgresql stop` to make sure you don't have a currently-running native PostgreSQL server which would conflict with Docker's PostgreSQL server. -* `docker-compose up import` to import the data (only necessary the first time or when you change the data file). Additionally, you can set import options through [environment variables](#Importing-data). More on that [later](#Hands-on-approach) +* `docker-compose up import` to import the osm.pbf data file and download additional external data (only necessary the first time or when you change the data file). Additionally, you can set import options through [environment variables](#Importing-data). More on that [later](#Hands-on-approach) * `docker-compose up kosmtik` to run the style preview application * Browse to [http://127.0.0.1:6789](http://127.0.0.1:6789) to view the output of Kosmtik * Ctrl+C to stop the style preview application @@ -45,7 +45,7 @@ At startup of the container the script `scripts/docker-startup.sh` is invoked wh ### Supplying command line options as environment variables -**osm2pgsql** has a few [command line options](https://manpages.debian.org/testing/osm2pgsql/osm2pgsql.1.en.html) and the import by default uses a RAM cache of 512 MB, 1 worker, **and expects the import file to be named `data.osm.pbf`**. +**osm2pgsql** has a few [command line options](https://manpages.debian.org/testing/osm2pgsql/osm2pgsql.1.en.html) and the import by default uses a RAM cache of 512 MB, 1 worker, **and expects the import file to be named `data.osm.pbf`**. If you want to customize any of these parameters, you have to set any number of the following environment variables: * `OSM2PGSQL_CACHE` (e.g. `export OSM2PGSQL_CACHE=1024` on Linux to set the cache to 1 GB) for the RAM cache (the value depends on the amount of RAM you have available - the more you can use here the faster the import may be) * `OSM2PGSQL_NUMPROC` for the number of workers (this depends on the number of processors you have and whether your hard disk is fast enough e.g. is a SSD) @@ -74,7 +74,7 @@ EXTERNAL_DATA_SCRIPT_FLAGS="--cache --no-update" \ docker-compose up import ``` -Note that on Linux you need to export those environment variables before calling docker-compose. If you are using sudo to call docker (because your user is not in the docker group (which we don't recommend)), you need to also use sudo -E option +Note that on Linux you need to export those environment variables before calling `docker-compose`. If you are using sudo to call docker (because your user is not in the docker group (which we don't recommend)), you need to also use sudo -E option Variables will be remembered in `.env` if you don't have that file, and values in the file will be applied unless you manually assign them. Keep in mind this means if you change your `.env` file, but keep your environment variables untouched (you haven't unset them or you haven't rebooted your host), they will be used instead of anything that you changed in `.env`. From ffd739a9ec9ba51ee59637cd835d560ffbdb849f Mon Sep 17 00:00:00 2001 From: Thomas Petillon Date: Fri, 13 Dec 2024 23:41:02 +0100 Subject: [PATCH 4/5] Add more log messages to track the import process --- scripts/docker-startup.sh | 3 +++ scripts/get-external-data.py | 3 ++- scripts/get-fonts.sh | 9 ++++++--- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/scripts/docker-startup.sh b/scripts/docker-startup.sh index 0ecc25041b..5d4df2ed61 100644 --- a/scripts/docker-startup.sh +++ b/scripts/docker-startup.sh @@ -18,6 +18,7 @@ test $i -gt $MAXCOUNT && echo "Timeout while waiting for PostgreSQL to be runnin case "$1" in import) # Creating default database + echo "Creating database" psql -c "SELECT 1 FROM pg_database WHERE datname = 'gis';" | grep -q 1 || createdb gis psql -d gis -c 'CREATE EXTENSION IF NOT EXISTS postgis;' psql -d gis -c 'CREATE EXTENSION IF NOT EXISTS hstore;' @@ -41,6 +42,7 @@ EOF fi # Importing data to a database + echo "Importing data from $OSM2PGSQL_DATAFILE" osm2pgsql \ --cache $OSM2PGSQL_CACHE \ --number-processes $OSM2PGSQL_NUMPROC \ @@ -52,6 +54,7 @@ EOF $OSM2PGSQL_DATAFILE # Setting up indexes and functions + echo "Setting up indexes and functions" psql -d gis -f indexes.sql psql -d gis -f functions.sql diff --git a/scripts/get-external-data.py b/scripts/get-external-data.py index 8f3822da5f..10694add53 100755 --- a/scripts/get-external-data.py +++ b/scripts/get-external-data.py @@ -313,7 +313,7 @@ def main(): config["settings"]["metadata_table"]) for name, source in config["sources"].items(): - logging.info("Checking table {}".format(name)) + logging.info("Table {}".format(name)) # Don't attempt to handle strange names # Even if there was code to escape them properly here, you don't want # in a style with all the quoting headaches @@ -327,6 +327,7 @@ def main(): config["settings"]["metadata_table"]) this_table.clean_temp() + logging.info(" Downloading file {}".format(source["url"])) # This will fetch data needed for import download = d.download(source["url"], name, opts, data_dir, this_table.last_modified()) diff --git a/scripts/get-fonts.sh b/scripts/get-fonts.sh index b25d3ad2fb..c204570535 100755 --- a/scripts/get-fonts.sh +++ b/scripts/get-fonts.sh @@ -7,8 +7,9 @@ mkdir -p "${FONTDIR}" # download filename url download() { + echo "Downloading file $2" ## Download if newer, and if curl fails, clean up and exit - curl --fail --compressed -A "get-fonts.sh/osm-carto" -o "$1" -z "$1" -L "$2" || { echo "Failed to download $1 $2"; rm -f "$1"; exit 1; } + curl --fail -s --compressed -A "get-fonts.sh/osm-carto" -o "$1" -z "$1" -L "$2" || { echo "Failed to download $1 $2"; rm -f "$1"; exit 1; } } # TTF Hinted Noto Fonts @@ -84,6 +85,8 @@ NotoSansYi" # Download the fonts in the lists above +echo "Downloading fonts" + for font in $REGULAR_BOLD_ITALIC; do regular="$font-Regular.ttf" bold="$font-Bold.ttf" @@ -121,12 +124,12 @@ TMPDIR=$(mktemp -d -t get-fonts.XXXXXXXXX) trap "rm -rf ${TMPDIR} ${FONTDIR}/static" EXIT # Noto Emoji B&W isn't available as a separate download, so we need to download the package and unzip it -curl --fail -A "get-fonts.sh/osm-carto" -o "${TMPDIR}/Noto_Emoji.zip" -L 'https://fonts.google.com/download?family=Noto%20Emoji' +download "${TMPDIR}/Noto_Emoji.zip" 'https://fonts.google.com/download?family=Noto%20Emoji' unzip -oqq "${TMPDIR}/Noto_Emoji.zip" static/NotoEmoji-Regular.ttf static/NotoEmoji-Bold.ttf -d "${FONTDIR}" mv "${FONTDIR}/static/NotoEmoji-Regular.ttf" "${FONTDIR}" mv "${FONTDIR}/static/NotoEmoji-Bold.ttf" "${FONTDIR}" -curl --fail -A "get-fonts.sh/osm-carto" -o "${TMPDIR}/hanazono.zip" -L 'https://mirrors.dotsrc.org/osdn/hanazono-font/68253/hanazono-20170904.zip' +download "${TMPDIR}/hanazono.zip" 'https://mirrors.dotsrc.org/osdn/hanazono-font/68253/hanazono-20170904.zip' unzip -oqq "${TMPDIR}/hanazono.zip" HanaMinA.ttf HanaMinB.ttf -d "${FONTDIR}" From 0d04886378282a5f5a64b95997fac8862545c34d Mon Sep 17 00:00:00 2001 From: Thomas Petillon Date: Fri, 13 Dec 2024 23:44:40 +0100 Subject: [PATCH 5/5] Change URL for Noto Emoji font --- scripts/get-fonts.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/get-fonts.sh b/scripts/get-fonts.sh index c204570535..76ca03f187 100755 --- a/scripts/get-fonts.sh +++ b/scripts/get-fonts.sh @@ -124,7 +124,7 @@ TMPDIR=$(mktemp -d -t get-fonts.XXXXXXXXX) trap "rm -rf ${TMPDIR} ${FONTDIR}/static" EXIT # Noto Emoji B&W isn't available as a separate download, so we need to download the package and unzip it -download "${TMPDIR}/Noto_Emoji.zip" 'https://fonts.google.com/download?family=Noto%20Emoji' +download "${TMPDIR}/Noto_Emoji.zip" 'https://archive.org/download/noto-emoji/Noto_Emoji.zip' unzip -oqq "${TMPDIR}/Noto_Emoji.zip" static/NotoEmoji-Regular.ttf static/NotoEmoji-Bold.ttf -d "${FONTDIR}" mv "${FONTDIR}/static/NotoEmoji-Regular.ttf" "${FONTDIR}"