Tiler server/database and cache cleaning refactoring

.github/workflows/chartpress.yaml

@@ -5,7 +5,7 @@ on:
       - 'main'
       - 'staging'
       - 'development'
-      - 'tiler-cache'
+      - 'transport_lines'
 jobs:
   build:
     runs-on: ubuntu-20.04
@@ -56,6 +56,8 @@ jobs:
           DEVELOPMENT_OAUTH_KEY: ${{ secrets.STAGING_OAUTH_KEY }}
           DEVELOPMENT_S3_BUCKET: osmseed-dev
           ## tiler
+
+          DEVELOPMENT_TILER_DB_HOST: ${{ secrets.STAGING_TILER_DB_HOST }}
           DEVELOPMENT_TILER_DB_PASSWORD: ${{ secrets.STAGING_TILER_DB_PASSWORD }}
           DEVELOPMENT_TILER_CACHE_AWS_ACCESS_KEY_ID: ${{ secrets.STAGING_TILER_CACHE_AWS_ACCESS_KEY_ID }}
           DEVELOPMENT_TILER_CACHE_AWS_SECRET_ACCESS_KEY: ${{ secrets.STAGING_TILER_CACHE_AWS_SECRET_ACCESS_KEY }}
@@ -99,6 +101,7 @@ jobs:
           STAGING_OAUTH_KEY: ${{ secrets.STAGING_OAUTH_KEY }}
           STAGING_S3_BUCKET: ${{ secrets.STAGING_S3_BUCKET }}
           ## tiler
+          STAGING_TILER_DB_HOST: ${{ secrets.STAGING_TILER_DB_HOST }}
           STAGING_TILER_DB_PASSWORD: ${{ secrets.STAGING_TILER_DB_PASSWORD }}
           STAGING_TILER_CACHE_AWS_ACCESS_KEY_ID: ${{ secrets.STAGING_TILER_CACHE_AWS_ACCESS_KEY_ID }}
           STAGING_TILER_CACHE_AWS_SECRET_ACCESS_KEY: ${{ secrets.STAGING_TILER_CACHE_AWS_SECRET_ACCESS_KEY }}
@@ -146,6 +149,7 @@ jobs:
           PRODUCTION_S3_BUCKET: ${{ secrets.PRODUCTION_S3_BUCKET }}
           PRODUCTION_DB_BACKUP_S3_BUCKET: ${{ secrets.PRODUCTION_DB_BACKUP_S3_BUCKET }}
           ## tiler
+          PRODUCTION_TILER_DB_HOST: ${{ secrets.PRODUCTION_TILER_DB_HOST }}
           PRODUCTION_TILER_DB_PASSWORD: ${{ secrets.PRODUCTION_TILER_DB_PASSWORD }}
           PRODUCTION_TILER_CACHE_AWS_ACCESS_KEY_ID: ${{ secrets.PRODUCTION_TILER_CACHE_AWS_ACCESS_KEY_ID }}
           PRODUCTION_SQS_QUEUE_URL: ${{ secrets.PRODUCTION_SQS_QUEUE_URL }}

.gitignore

@@ -11,4 +11,5 @@ envs/.env.web
 config.toml
 .DS_Store
 */.DS_Store
-.env
+.env
+values.dev.staging.yaml

compose/README.md

@@ -0,0 +1,9 @@
+## Development Mode
+
+Compose files are for development mode, e.g.:
+
+- Tiler DB
+
+```sh
+docker compose -f compose/tiler.yml run --service-ports tiler bash
+```

compose/tiler.yml

@@ -6,23 +6,23 @@ services:
       dockerfile: Dockerfile
     ports:
       - "5432:5432"
-    # volumes:
-    #   - ../data/tiler-db-data:/var/lib/postgresql/data
+    volumes:
+      - ../data/tiler-db-data2:/var/lib/postgresql/data
     env_file:
       - ../envs/.env.tiler
     restart: always
   imposm:
-    image: ohm-tiler-imposm:v1
+    image: rub21/tiler-imposm:v15
     build:
       context: ../images/tiler-imposm
       dockerfile: Dockerfile
-    # volumes:
-      # - ../data/tiler-imposm-data:/mnt/data
-      # - ../images/tiler-imposm:/app
-    # command:
-    #   - sh
-    #   - -c
-    #   - "sleep 60 && ./start.sh"
+    volumes:
+      - ../data/tiler-imposm-data2:/mnt/data
+      - ../images/tiler-imposm:/app
+    command:
+      - sh
+      - -c
+      - "sleep 20 && ./start.sh"
     env_file:
       - ../envs/.env.tiler
   tiler:
@@ -31,19 +31,11 @@ services:
       context: ../images/tiler-server
       dockerfile: Dockerfile
     volumes:
-      # - ../data/tiler-server-data:/mnt/data
+      - ../data/tiler-server-data:/mnt/data
       - ../images/tiler-server:/app
     ports:
       - "9090:9090"
     env_file:
       - ../envs/.env.tiler
     restart: always
-  tilercache:
-    image: ohm-tiler-cache:v1
-    build:
-      context: ../images/tiler-cache
-      dockerfile: Dockerfile
-    volumes:
-      # - ../data/tiler-cache-data:/mnt/data
-      - ../images/tiler-cache:/app
-
+

images/tiler-cache/Dockerfile

@@ -1,4 +1,4 @@
-FROM ghcr.io/openhistoricalmap/tiler-server:0.0.1-0.dev.git.1734.h5b4d15d
+FROM ghcr.io/openhistoricalmap/tiler-server:0.0.1-0.dev.git.1880.h0f4c894
 
 RUN apk update && apk add --no-cache \
     proj \

images/tiler-cache/purge.py

@@ -87,7 +87,9 @@ def get_active_jobs_count():
 
 def create_kubernetes_job(file_url, file_name):
     """Create a Kubernetes Job to process a file."""
-    config_map_name = f"{ENVIRONMENT}-tiler-server-cm"
+    configmap_tiler_server = f"{ENVIRONMENT}-tiler-server-cm"
+    configmap_tiler_db = f"{ENVIRONMENT}-tiler-db-cm"
+
     job_name = f"{JOB_NAME_PREFIX}-{file_name}"
     job_manifest = {
         "apiVersion": "batch/v1",
@@ -103,7 +105,7 @@ def create_kubernetes_job(file_url, file_name):
                             "name": "tiler-purge-seed",
                             "image": DOCKER_IMAGE,
                             "command": ["sh", "./purge_and_seed.sh"],
-                            "envFrom": [{"configMapRef": {"name": config_map_name}}],
+                            "envFrom": [{"configMapRef": {"name": configmap_tiler_server}},{"configMapRef": {"name": configmap_tiler_db}}],
                             "env": [
                                 {"name": "IMPOSM_EXPIRED_FILE", "value": file_url},
                                 {"name": "EXECUTE_PURGE", "value": str(EXECUTE_PURGE)},

images/tiler-imposm/Dockerfile

@@ -1,7 +1,6 @@
 FROM osgeo/gdal:ubuntu-small-3.2.3
 
-RUN apt-get -y update
-RUN apt-get install -y \
+RUN apt-get -y update && apt-get install -y \
     g++ \
     libboost-dev \
     libboost-system-dev \
@@ -22,46 +21,37 @@ RUN apt-get install -y \
     software-properties-common && \
     rm -rf /var/lib/apt/lists/*
 
-# Install python
 RUN add-apt-repository ppa:deadsnakes/ppa && \
     apt-get update && \
-    apt-get install -y build-essential python3.6 python3.6-dev python3-pip && \
+    apt-get install -y python3.9 python3.9-dev python3-pip && \
     rm -rf /var/lib/apt/lists/* && \
     python3 -m pip install pip --upgrade && \
     python3 -m pip install wheel
 
-# Install postgresql-client
 RUN apt-get update && apt-get install -y postgresql-client && \
     rm -rf /var/lib/apt/lists/*
 
-# Install Go
 RUN wget -c https://dl.google.com/go/go1.21.0.linux-amd64.tar.gz -O - | tar -xz -C /usr/local
 ENV PATH $PATH:/usr/local/go/bin
 
+RUN git clone https://github.com/OpenHistoricalMap/DateFunctions-plpgsql.git /usr/local/datefunctions
+
 WORKDIR /go
 ENV GOPATH /go
-
-# Clone the repository directly
 RUN git clone https://github.com/omniscale/imposm3.git $GOPATH/src/github.com/omniscale/imposm3
 
 WORKDIR $GOPATH/src/github.com/omniscale/imposm3
 RUN git checkout v0.14.0
 
-# Fix cache package conflict
-RUN sed -i '/setMaxFileSize/d' cache/ldb_pre_121.go
 
+# Apply fixes or patches (if necessary)
+RUN sed -i '/setMaxFileSize/d' cache/ldb_pre_121.go
 RUN go install github.com/omniscale/imposm3/cmd/imposm
 ENV PATH $PATH:$GOPATH/bin
 
-# Install awscli and gsutil to get the pbf file
-RUN pip3 install -U setuptools
-RUN pip3 install awscli
-
-ENV IMPOSMDATA /mnt/data
-RUN mkdir -p $IMPOSMDATA && chmod 777 -R $IMPOSMDATA
-WORKDIR $IMPOSMDATA
-VOLUME $IMPOSMDATA
-
+# Install AWS CLI for downloading files
+RUN pip3 install -U setuptools awscli psycopg2
 WORKDIR /osm
 COPY . .
-CMD ./start.sh
+RUN chmod +x ./start.sh
+CMD ["./start.sh"]

images/tiler-imposm/build_imposm3_config.py

@@ -0,0 +1,88 @@
+import json
+import os
+import logging
+
+# Configure logging
+logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s")
+logger = logging.getLogger(__name__)
+
+def load_json(file_path):
+    """Load a JSON file."""
+    try:
+        with open(file_path, 'r', encoding='utf-8') as file:
+            return json.load(file)
+    except FileNotFoundError:
+        logger.error(f"File not found: {file_path}")
+        raise
+    except json.JSONDecodeError as e:
+        logger.error(f"Error parsing JSON in file {file_path}: {e}")
+        raise
+
+def merge_configs(template, configs):
+    """Merge multiple JSON configs into the template."""
+    merged = template.copy()
+    for config in configs:
+        if 'generalized_tables' in config:
+            merged['generalized_tables'].update(config['generalized_tables'])
+        if 'tables' in config:
+            merged['tables'].update(config['tables'])
+    return merged
+
+def main(folder_path, template_path, output_path):
+    """Main function to merge JSON configs."""
+    logger.info("Loading template configuration...")
+    template = load_json(template_path)
+
+    import_layers = os.getenv("IMPOSM3_IMPORT_LAYERS", "all").strip()
+    logger.info(f"IMPOSM3_IMPORT_LAYERS: {import_layers}")
+
+    configs = []
+    if "all" in import_layers:
+        logger.info("Importing all layer files.")
+        # Import all JSON files in the folder
+        json_files = [f for f in os.listdir(folder_path) if f.endswith('.json')]
+        for json_file in json_files:
+            file_path = os.path.join(folder_path, json_file)
+            try:
+                logger.info(f"Importing {file_path}")
+                configs.append(load_json(file_path))
+            except Exception as e:
+                logger.error(f"Error reading {file_path}: {e}")
+    else:
+        # Import only specified layers
+        layer_names = [layer.strip() for layer in import_layers.split(",") if layer.strip()]
+        if not layer_names:
+            logger.error("No layers specified in IMPOSM3_IMPORT_LAYERS. Exiting.")
+            return
+
+        for layer_name in layer_names:
+            file_path = os.path.join(folder_path, f"{layer_name}.json")
+            if os.path.exists(file_path):
+                try:
+                    logger.info(f"Importing {file_path}")
+                    configs.append(load_json(file_path))
+                except Exception as e:
+                    logger.error(f"Error reading {file_path}: {e}")
+            else:
+                logger.warning(f"Layer config file {file_path} not found. Skipping.")
+
+    if not configs:
+        logger.error("No valid layer configurations found. Exiting.")
+        return
+
+    logger.info("Merging configurations...")
+    merged_config = merge_configs(template, configs)
+    try:
+        with open(output_path, 'w', encoding='utf-8') as output_file:
+            json.dump(merged_config, output_file, indent=2)
+        logger.info(f"Merged configuration saved to {output_path}")
+    except Exception as e:
+        logger.error(f"Error writing merged configuration to {output_path}: {e}")
+        raise
+
+if __name__ == "__main__":
+    folder_path = "./config/layers"
+    template_path = "./config/imposm3.template.json"
+    output_path = "./config/imposm3.json"
+    main(folder_path, template_path, output_path)
+