From 2c639fe102a26bbbe144f9ab20c4e40dee11a487 Mon Sep 17 00:00:00 2001 From: Robert Juzak Date: Thu, 29 Aug 2024 10:27:21 +0200 Subject: [PATCH] - [nominatim][fix] Wrong indentation resolve #88 - [nominatim][fix] Remove resource preset from Postgres - [nominatim][feature] Support import continue in init job --- charts/nominatim/Chart.yaml | 2 +- charts/nominatim/README.md | 81 +++++++++------ charts/nominatim/templates/deployment.yaml | 114 ++++++++++----------- charts/nominatim/templates/initJob.yaml | 11 +- charts/nominatim/values.yaml | 27 ++++- 5 files changed, 145 insertions(+), 90 deletions(-) diff --git a/charts/nominatim/Chart.yaml b/charts/nominatim/Chart.yaml index 6aca4e8..9dcb3b9 100644 --- a/charts/nominatim/Chart.yaml +++ b/charts/nominatim/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 4.0.0 +version: 4.1.0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/charts/nominatim/README.md b/charts/nominatim/README.md index 4ff26d4..497d046 100644 --- a/charts/nominatim/README.md +++ b/charts/nominatim/README.md @@ -47,18 +47,18 @@ initJob: postgresql: primary: - extendedConfiguration: | - shared_buffers = 2GB - maintenance_work_mem = 10GB - autovacuum_work_mem = 2GB - work_mem = 50MB - effective_cache_size = 24GB - synchronous_commit = off - max_wal_size = 1GB - checkpoint_timeout = 10min - checkpoint_completion_target = 0.9 - fsync = off - full_page_writes = off + extendedConfiguration: | + shared_buffers = 2GB + maintenance_work_mem = 10GB + autovacuum_work_mem = 2GB + work_mem = 50MB + effective_cache_size = 24GB + synchronous_commit = off + max_wal_size = 1GB + checkpoint_timeout = 10min + checkpoint_completion_target = 0.9 + fsync = off + full_page_writes = off ``` To install the chart with the release name `nominatim`: @@ -422,22 +422,24 @@ Note: The command above may differ a little depending the k8s cluster version yo ### Database Parameters -| Name | Description | Value | -|-----------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------| -| `postgresql.enabled` | Deploy a PostgreSQL server to satisfy the applications database requirements | `true` | -| `postgresql.image.repository` | PostgreSQL image repository | `robjuz/postgresql-nominatim` | -| `postgresql.image.tag` | PostgreSQL image tag | `14.4.0-4.0.1` | -| `postgresql.auth.postgresPassword` | PostgreSQL root password | `nominatim` | -| `postgresql.primary.persistence.enabled` | Enable persistence on PostgreSQL using PVC(s) | `true` | -| `postgresql.primary.persistence.storageClass` | Persistent Volume storage class | `nil` | -| `postgresql.primary.persistence.accessModes` | Persistent Volume access modes | `[ReadWriteOnce]` | -| `postgresql.primary.persistence.size` | Persistent Volume size | `500Gi` | -| `externalDatabase.host` | External PostgreSQL host (ignored if `postgresql.enabled = true`) | localhost | -| `externalDatabase.port` | External PostgreSQL post (ignored if `postgresql.enabled = true`) | 5432 | -| `externalDatabase.user` | External PostgreSQL user (ignored if `postgresql.enabled = true`) | nominatim | -| `externalDatabase.password` | External PostgreSQL password (ignored if `postgresql.enabled = true`) | "" | -| `externalDatabase.existingSecretDsn` | Name of existing secret to use to set full PostgreSQL DataSourceName (overrides `externalDatabase.*`) | `nil` | -| `externalDatabase.existingSecretDsnKey` | Name of key in existing secret to use to set full PostgreSQL DataSourceName. Only used when `externalDatabase.existingSecretDsn` is set. | POSTGRESQL_DSN | +| Name | Description | Value | +|-----------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------| +| `postgresql.enabled` | Deploy a PostgreSQL server to satisfy the applications database requirements | `true` | +| `postgresql.image.repository` | PostgreSQL image repository | `robjuz/postgresql-nominatim` | +| `postgresql.image.tag` | PostgreSQL image tag | `14.4.0-4.0.1` | +| `postgresql.auth.postgresPassword` | PostgreSQL root password | `nominatim` | +| `postgresql.primary.persistence.enabled` | Enable persistence on PostgreSQL using PVC(s) | `true` | +| `postgresql.primary.persistence.storageClass` | Persistent Volume storage class | `nil` | +| `postgresql.primary.persistence.accessModes` | Persistent Volume access modes | `[ReadWriteOnce]` | +| `postgresql.primary.persistence.size` | Persistent Volume size | `500Gi` | +| `postgresql.primary.resourcesPreset` | Set container resources according to one common preset (allowed values: none, nano, micro, small, medium, large, xlarge, 2xlarge). This is ignored if postgresql.primary.resources is set (postgresql.primary.resources is recommended for production). | `none` | +| `postgresql.primary.resources` | Set container requests and limits for different resources like CPU or memory (essential for production workloads) | `{}` | +| `externalDatabase.host` | External PostgreSQL host (ignored if `postgresql.enabled = true`) | localhost | +| `externalDatabase.port` | External PostgreSQL post (ignored if `postgresql.enabled = true`) | 5432 | +| `externalDatabase.user` | External PostgreSQL user (ignored if `postgresql.enabled = true`) | nominatim | +| `externalDatabase.password` | External PostgreSQL password (ignored if `postgresql.enabled = true`) | "" | +| `externalDatabase.existingSecretDsn` | Name of existing secret to use to set full PostgreSQL DataSourceName (overrides `externalDatabase.*`) | `nil` | +| `externalDatabase.existingSecretDsnKey` | Name of key in existing secret to use to set full PostgreSQL DataSourceName. Only used when `externalDatabase.existingSecretDsn` is set. | POSTGRESQL_DSN | ### Nominatim Appserver Parameters @@ -463,13 +465,32 @@ Using flatnode with replication enabled requires the usage of a ReadWriteMany vo be shared within the pods. This also applies when scaling the nominatim deployment. - - ### PVC For data When importing large extracts (Europe/Planet) the data needed to be downloaded are quite big. If your server has not enough disk space to store the data, you can use a dedicated PV for this. +### Dealing with import errors and continuing the import + +When there is an error during importing, you can check for logs: +```console +kubectl logs jobs/nominatim-init +``` +To continue, you first need to delete the job +```console +kubectl delete jobs nominatim-init +``` + +then add the `initJob.continue` +```yaml +initJob: + continue: load-data +``` + +and reinstall the chart +```console +helm upgrade --install nominatim robjuz/nominatim -f values.yaml +``` ### External database support You may want to have Nominatim connect to an external database rather than installing one inside your cluster. Typical diff --git a/charts/nominatim/templates/deployment.yaml b/charts/nominatim/templates/deployment.yaml index ea781a0..94b7f3e 100644 --- a/charts/nominatim/templates/deployment.yaml +++ b/charts/nominatim/templates/deployment.yaml @@ -75,66 +75,66 @@ spec: {{- if .Values.topologySpreadConstraints }} topologySpreadConstraints: {{- include "common.tplvalues.render" (dict "value" .Values.topologySpreadConstraints "context" .) | nindent 8 }} {{- end }} - {{- if or (and .Values.podSecurityContext.enabled .Values.volumePermissions.enabled .Values.flatnode.enabled) (.Values.initContainers) }} - initContainers: - {{- if .Values.nominatimUi.enabled }} - - name: nominatim-ui-download - image: curlimages/curl - command: - - /bin/sh - args: - - -ec - - | - mkdir -p /nominatim/nominatim-ui - cd /nominatim/nominatim-ui - curl -L {{ include "nominatim.uiUrl" . }} | tar -xz --strip-components 1 - {{- if eq ( toString ( .Values.volumePermissions.containerSecurityContext.runAsUser )) "auto" }} - find /nominatim/flatnode -mindepth 0 -maxdepth 1 -not -name ".snapshot" -not -name "lost+found" | xargs -r chown -R $(id -u):$(id -G | cut -d " " -f2) - {{- else }} - find /nominatim/flatnode -mindepth 0 -maxdepth 1 -not -name ".snapshot" -not -name "lost+found" | xargs -r chown -R {{ .Values.containerSecurityContext.runAsUser }}:{{ .Values.podSecurityContext.fsGroup }} - {{- end }} - {{- if eq ( toString ( .Values.volumePermissions.containerSecurityContext.runAsUser )) "auto " }} - securityContext: {{- omit .Values.volumePermissions.containerSecurityContext "runAsUser" | toYaml | nindent 12 }} - {{- else }} - securityContext: {{- .Values.volumePermissions.containerSecurityContext | toYaml | nindent 12 }} - {{- end }} - {{- if .Values.volumePermissions.resources }} - resources: {{- toYaml .Values.volumePermissions.resources | nindent 12 }} - {{- end }} - volumeMounts: - - mountPath: /nominatim/nominatim-ui - name: nominatim-ui + {{- if or (and .Values.podSecurityContext.enabled .Values.volumePermissions.enabled .Values.flatnode.enabled) (.Values.nominatimUi.enabled) (.Values.initContainers) }} + initContainers: + {{- if .Values.nominatimUi.enabled }} + - name: nominatim-ui-download + image: curlimages/curl + command: + - /bin/sh + args: + - -ec + - | + mkdir -p /nominatim/nominatim-ui + cd /nominatim/nominatim-ui + curl -L {{ include "nominatim.uiUrl" . }} | tar -xz --strip-components 1 + {{- if eq ( toString ( .Values.volumePermissions.containerSecurityContext.runAsUser )) "auto" }} + find /nominatim/nominatim-ui -mindepth 0 -maxdepth 1 -not -name ".snapshot" -not -name "lost+found" | xargs -r chown -R $(id -u):$(id -G | cut -d " " -f2) + {{- else }} + find /nominatim/nominatim-ui -mindepth 0 -maxdepth 1 -not -name ".snapshot" -not -name "lost+found" | xargs -r chown -R {{ .Values.containerSecurityContext.runAsUser }}:{{ .Values.podSecurityContext.fsGroup }} + {{- end }} + {{- if eq ( toString ( .Values.volumePermissions.containerSecurityContext.runAsUser )) "auto " }} + securityContext: {{- omit .Values.volumePermissions.containerSecurityContext "runAsUser" | toYaml | nindent 12 }} + {{- else }} + securityContext: {{- .Values.volumePermissions.containerSecurityContext | toYaml | nindent 12 }} {{- end }} - {{- if and .Values.podSecurityContext.enabled .Values.volumePermissions.enabled .Values.flatnode.enabled }} - - name: volume-permissions - image: "{{ include "nominatim.volumePermissions.image" . }}" - imagePullPolicy: {{ .Values.volumePermissions.image.pullPolicy | quote }} - command: - - /bin/bash - args: - - -ec - - | - mkdir -p /nominatim/flatnode - {{- if eq ( toString ( .Values.volumePermissions.containerSecurityContext.runAsUser )) "auto" }} - find /nominatim/flatnode -mindepth 0 -maxdepth 1 -not -name ".snapshot" -not -name "lost+found" | xargs -r chown -R $(id -u):$(id -G | cut -d " " -f2) - {{- else }} - find /nominatim/flatnode -mindepth 0 -maxdepth 1 -not -name ".snapshot" -not -name "lost+found" | xargs -r chown -R {{ .Values.containerSecurityContext.runAsUser }}:{{ .Values.podSecurityContext.fsGroup }} - {{- end }} - {{- if eq ( toString ( .Values.volumePermissions.containerSecurityContext.runAsUser )) "auto " }} - securityContext: {{- omit .Values.volumePermissions.containerSecurityContext "runAsUser" | toYaml | nindent 12 }} - {{- else }} - securityContext: {{- .Values.volumePermissions.containerSecurityContext | toYaml | nindent 12 }} - {{- end }} - {{- if .Values.volumePermissions.resources }} - resources: {{- toYaml .Values.volumePermissions.resources | nindent 12 }} - {{- end }} - volumeMounts: - - mountPath: /nominatim/flatnode - name: flatnode + {{- if .Values.volumePermissions.resources }} + resources: {{- toYaml .Values.volumePermissions.resources | nindent 12 }} + {{- end }} + volumeMounts: + - mountPath: /nominatim/nominatim-ui + name: nominatim-ui + {{- end }} + {{- if and .Values.podSecurityContext.enabled .Values.volumePermissions.enabled .Values.flatnode.enabled }} + - name: volume-permissions + image: "{{ include "nominatim.volumePermissions.image" . }}" + imagePullPolicy: {{ .Values.volumePermissions.image.pullPolicy | quote }} + command: + - /bin/bash + args: + - -ec + - | + mkdir -p /nominatim/flatnode + {{- if eq ( toString ( .Values.volumePermissions.containerSecurityContext.runAsUser )) "auto" }} + find /nominatim/flatnode -mindepth 0 -maxdepth 1 -not -name ".snapshot" -not -name "lost+found" | xargs -r chown -R $(id -u):$(id -G | cut -d " " -f2) + {{- else }} + find /nominatim/flatnode -mindepth 0 -maxdepth 1 -not -name ".snapshot" -not -name "lost+found" | xargs -r chown -R {{ .Values.containerSecurityContext.runAsUser }}:{{ .Values.podSecurityContext.fsGroup }} + {{- end }} + {{- if eq ( toString ( .Values.volumePermissions.containerSecurityContext.runAsUser )) "auto " }} + securityContext: {{- omit .Values.volumePermissions.containerSecurityContext "runAsUser" | toYaml | nindent 12 }} + {{- else }} + securityContext: {{- .Values.volumePermissions.containerSecurityContext | toYaml | nindent 12 }} {{- end }} - {{- if .Values.initContainers }} - {{- include "common.tplvalues.render" (dict "value" .Values.initContainers "context" $) | nindent 8 }} + {{- if .Values.volumePermissions.resources }} + resources: {{- toYaml .Values.volumePermissions.resources | nindent 12 }} {{- end }} + volumeMounts: + - mountPath: /nominatim/flatnode + name: flatnode + {{- end }} + {{- if .Values.initContainers }} + {{- include "common.tplvalues.render" (dict "value" .Values.initContainers "context" $) | nindent 8 }} + {{- end }} {{- end }} containers: - name: nominatim diff --git a/charts/nominatim/templates/initJob.yaml b/charts/nominatim/templates/initJob.yaml index afd04e2..867a7a5 100644 --- a/charts/nominatim/templates/initJob.yaml +++ b/charts/nominatim/templates/initJob.yaml @@ -127,11 +127,20 @@ spec: value: {{ .Values.initJob.threads | quote }} - name: NOMINATIM_DATABASE_WEBUSER value: {{ include "nominatim.databaseUser" . }} + - name: CONTINUE + value: {{ .Values.initJob.continue | quote }} command: - /bin/bash - -ec - | - nominatim import --osm-file data.osm.pbf --threads $THREADS + if [ -z "$CONTINUE" ]; then + echo "Starting data import." + nominatim import --osm-file data.osm.pbf --threads $THREADS + else + echo "Continuing initialization from step: '$CONTINUE'." + nominatim import --continue $CONTINUE --osm-file data.osm.pbf --threads $THREADS + fi + nominatim index --threads $THREADS nominatim admin --check-database diff --git a/charts/nominatim/values.yaml b/charts/nominatim/values.yaml index b970a38..a0f4b15 100644 --- a/charts/nominatim/values.yaml +++ b/charts/nominatim/values.yaml @@ -568,9 +568,16 @@ initJob: freeze: false wikipediaUrl: https://nominatim.org/data/wikimedia-importance.sql.gz + ## @param initJob.continue Nominatim import CLI continue arg + ## If undefined (default) `--continue` flag is not set on nominatim import + ## If defined, `nominatim import --continue ` is set + ## Supported steps: `[ import-from-file load-data | indexing | db-postprocess ]` + ## More info on: https://nominatim.org/release-docs/latest/admin/Faq/#can-a-stoppedkilled-import-process-be-resumed + ## + continue: ## @param initJob.backoffLimit set backoff limit of the job ## - backoffLimit: 1 + backoffLimit: 0 ## @param initJob.annotations [object] Add annotations to the job ## annotations: {} @@ -1095,6 +1102,24 @@ postgresql: persistence: size: 500Gi + ## PostgreSQL Primary resource requests and limits + ## ref: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/ + ## @param postgresql.primary.resourcesPreset Set container resources according to one common preset (allowed values: none, nano, micro, small, medium, large, xlarge, 2xlarge). This is ignored if primary.resources is set (primary.resources is recommended for production). + ## More information: https://github.com/bitnami/charts/blob/main/bitnami/common/templates/_resources.tpl#L15 + ## + resourcesPreset: "none" + ## @param postgresql.primary.resources Set container requests and limits for different resources like CPU or memory (essential for production workloads) + ## Example: + ## resources: + ## requests: + ## cpu: 2 + ## memory: 512Mi + ## limits: + ## cpu: 3 + ## memory: 1024Mi + ## + resources: { } + extendedConfiguration: | shared_buffers = 2GB maintenance_work_mem = 10GB