Merge pull request #440 from OpenHistoricalMap/staging

Update tiler imposm script to insert data in hetzner db - tiler production
OpenHistoricalMap · Dec 19, 2024 · 5e1250a · 5e1250a
2 parents 1a3260b + 6a82e5e
commit 5e1250a
Show file tree

Hide file tree

Showing 14 changed files with 591 additions and 387 deletions.
diff --git a/.github/workflows/chartpress.yaml b/.github/workflows/chartpress.yaml
@@ -5,7 +5,7 @@ on:
       - 'main'
       - 'staging'
       - 'development'
-      - 'imposm_transport'
+      - 'refactor/imposm'
 jobs:
   build:
     runs-on: ubuntu-20.04

diff --git a/compose/tiler.yml b/compose/tiler.yml
@@ -1,30 +1,36 @@
 services:
-  db:
-    image: ohm-tiler-db:v1
-    build:
-      context: ../images/tiler-db
-      dockerfile: Dockerfile
-    ports:
-      - "5432:5432"
-    volumes:
-      - ../data/tiler-db-data2:/var/lib/postgresql/data
-    env_file:
-      - ../envs/.env.tiler
-    restart: always
-  imposm:
-    image: rub21/tiler-imposm:v15
-    build:
-      context: ../images/tiler-imposm
-      dockerfile: Dockerfile
-    volumes:
-      - ../data/tiler-imposm-data2:/mnt/data
-      - ../images/tiler-imposm:/app
-    command:
-      - sh
-      - -c
-      - "sleep 20 && ./start.sh"
-    env_file:
-      - ../envs/.env.tiler
+  # db:
+  #   image: ohm-tiler-db:v1
+  #   build:
+  #     context: ../images/tiler-db
+  #     dockerfile: Dockerfile
+  #   ports:
+  #     - "5432:5432"
+  #   # volumes:
+  #   #   - ../data/tiler-db-data2:/var/lib/postgresql/data
+  #   env_file:
+  #     - ../envs/.env.tiler
+  #   restart: always
+  #   networks:
+  #     - tiler_network
+
+  # imposm:
+  #   image: rub21/tiler-imposm:v16
+  #   build:
+  #     context: ../images/tiler-imposm
+  #     dockerfile: Dockerfile
+  #   volumes:
+  #   #   - ../data/tiler-imposm-data2:/mnt/data
+  #     - ../images/tiler-imposm:/osm
+  #   command:
+  #     - sh
+  #     - -c
+  #     - "./start.sh"
+  #   env_file:
+  #     - ../envs/.env.tiler
+  #   networks:
+  #     - tiler_network
+
   tiler:
     image: ohm-tiler-server:v1
     build:
@@ -38,4 +44,9 @@ services:
     env_file:
       - ../envs/.env.tiler
     restart: always
-
+    networks:
+      - tiler_network
+
+networks:
+  tiler_network:
+    driver: bridge
diff --git a/images/tiler-cache/Dockerfile b/images/tiler-cache/Dockerfile
@@ -1,4 +1,4 @@
-FROM ghcr.io/openhistoricalmap/tiler-server:0.0.1-0.dev.git.1920.h4abe268
+FROM ghcr.io/openhistoricalmap/tiler-server:0.0.1-0.dev.git.1951.hc772955
 
 RUN apk update && apk add --no-cache \
     proj \

diff --git a/images/tiler-imposm/config/layers/admin_areas.json b/images/tiler-imposm/config/layers/admin_areas.json
@@ -8,95 +8,123 @@
     ]
   },
   "generalized_tables": {
-    "admin_boundaries_lines_z0_2": {
+    "admin_areas_z0_2": {
       "source": "admin_areas",
       "tolerance": 5000,
-      "geometry_transform": "ST_Boundary(geometry)",
-      "geometry_transform_types": "GeometryType(geometry) IN ('POLYGON', 'MULTIPOLYGON')",
-      "sql_filter": "admin_level IN (1,2)"
-    },
-    "admin_boundaries_centroid_z0_2": {
-      "source": "admin_areas",
-      "geometry_transform": "(ST_MaximumInscribedCircle(geometry)).center",
-      "geometry_transform_types": "GeometryType(geometry) IN ('POLYGON', 'MULTIPOLYGON')",
-      "sql_filter": "admin_level IN (1,2) AND name IS NOT NULL AND name <> ''"
+      "sql_filter": "admin_level IN (1,2)",
+      "sub_tables": [
+        {
+          "table": "admin_boundaries_lines_z0_2",
+          "geometry_transform": "ST_Boundary(geometry)"
+        },
+        {
+          "table": "admin_boundaries_centroid_z0_2",
+          "geometry_transform": "(ST_MaximumInscribedCircle(geometry)).center",
+          "sql_filter": "name IS NOT NULL AND name <> ''"
+        }
+      ]
     },
-    "admin_boundaries_lines_z3_5": {
+
+    "admin_areas_z3_5": {
       "source": "admin_areas",
       "tolerance": 1000,
-      "geometry_transform": "ST_Boundary(geometry)",
-      "geometry_transform_types": "GeometryType(geometry) IN ('POLYGON', 'MULTIPOLYGON')",
-      "sql_filter": "admin_level IN (1,2,3,4)"
-    },
-    "admin_boundaries_centroid_z3_5": {
-      "source": "admin_areas",
-      "geometry_transform": "(ST_MaximumInscribedCircle(geometry)).center",
-      "geometry_transform_types": "GeometryType(geometry) IN ('POLYGON', 'MULTIPOLYGON')",
-      "sql_filter": "admin_level IN (1,2,3,4) AND name IS NOT NULL AND name <> ''"
+      "sql_filter": "admin_level IN (1,2,3,4)",
+      "sub_tables": [
+        {
+          "table": "admin_boundaries_lines_z3_5",
+          "geometry_transform": "ST_Boundary(geometry)"
+        },
+        {
+          "table": "admin_boundaries_centroid_z3_5",
+          "geometry_transform": "(ST_MaximumInscribedCircle(geometry)).center",
+          "sql_filter": "name IS NOT NULL AND name <> ''"
+        }
+      ]
     },
-    "admin_boundaries_lines_z6_7": {
+
+    "admin_areas_z6_7": {
       "source": "admin_areas",
       "tolerance": 200,
-      "geometry_transform": "ST_Boundary(geometry)",
-      "geometry_transform_types": "GeometryType(geometry) IN ('POLYGON', 'MULTIPOLYGON')",
-      "sql_filter": "admin_level IN (1,2,3,4,5,6)"
-    },
-    "admin_boundaries_centroid_z6_7": {
-      "source": "admin_areas",
-      "geometry_transform": "(ST_MaximumInscribedCircle(geometry)).center",
-      "geometry_transform_types": "GeometryType(geometry) IN ('POLYGON', 'MULTIPOLYGON')",
-      "sql_filter": "admin_level IN (1,2,3,4,5,6) AND name IS NOT NULL AND name <> ''"
+      "sql_filter": "admin_level IN (1,2,3,4,5,6)",
+      "sub_tables": [
+        {
+          "table": "admin_boundaries_lines_z6_7",
+          "geometry_transform": "ST_Boundary(geometry)"
+        },
+        {
+          "table": "admin_boundaries_centroid_z6_7",
+          "geometry_transform": "(ST_MaximumInscribedCircle(geometry)).center",
+          "sql_filter": "name IS NOT NULL AND name <> ''"
+        }
+      ]
     },
-    "admin_boundaries_lines_z8_9": {
+
+    "admin_areas_z8_9": {
       "source": "admin_areas",
       "tolerance": 100,
-      "geometry_transform": "ST_Boundary(geometry)",
-      "geometry_transform_types": "GeometryType(geometry) IN ('POLYGON', 'MULTIPOLYGON')",
-      "sql_filter": "admin_level IN (1,2,3,4,5,6,7,8,9)"
-    },
-    "admin_boundaries_centroid_z8_9": {
-      "source": "admin_areas",
-      "geometry_transform": "(ST_MaximumInscribedCircle(geometry)).center",
-      "geometry_transform_types": "GeometryType(geometry) IN ('POLYGON', 'MULTIPOLYGON')",
-      "sql_filter": "admin_level IN (1,2,3,4,5,6,7,8,9) AND name IS NOT NULL AND name <> ''"
+      "sql_filter": "admin_level IN (1,2,3,4,5,6,7,8,9)",
+      "sub_tables": [
+        {
+          "table": "admin_boundaries_lines_z8_9",
+          "geometry_transform": "ST_Boundary(geometry)"
+        },
+        {
+          "table": "admin_boundaries_centroid_z8_9",
+          "geometry_transform": "(ST_MaximumInscribedCircle(geometry)).center",
+          "sql_filter": "name IS NOT NULL AND name <> ''"
+        }
+      ]
     },
-    "admin_boundaries_lines_z10_12": {
+
+    "admin_areas_z10_12": {
       "source": "admin_areas",
       "tolerance": 20,
-      "geometry_transform": "ST_Boundary(geometry)",
-      "geometry_transform_types": "GeometryType(geometry) IN ('POLYGON', 'MULTIPOLYGON')",
-      "sql_filter": "admin_level IN (1,2,3,4,5,6,7,8,9,10)"
-    },
-    "admin_boundaries_centroid_z10_12": {
-      "source": "admin_areas",
-      "geometry_transform": "(ST_MaximumInscribedCircle(geometry)).center",
-      "geometry_transform_types": "GeometryType(geometry) IN ('POLYGON', 'MULTIPOLYGON')",
-      "sql_filter": "admin_level IN (1,2,3,4,5,6,7,8,9,10) AND name IS NOT NULL AND name <> ''"
+      "sql_filter": "admin_level IN (1,2,3,4,5,6,7,8,9,10)",
+      "sub_tables": [
+        {
+          "table": "admin_boundaries_lines_z10_12",
+          "geometry_transform": "ST_Boundary(geometry)"
+        },
+        {
+          "table": "admin_boundaries_centroid_z10_12",
+          "geometry_transform": "(ST_MaximumInscribedCircle(geometry)).center",
+          "sql_filter": "name IS NOT NULL AND name <> ''"
+        }
+      ]
     },
-    "admin_boundaries_lines_z13_15": {
+
+    "admin_areas_z13_15": {
       "source": "admin_areas",
       "tolerance": 5,
-      "geometry_transform": "ST_Boundary(geometry)",
-      "geometry_transform_types": "GeometryType(geometry) IN ('POLYGON', 'MULTIPOLYGON')",
-      "sql_filter": "admin_level IN (1,2,3,4,5,6,7,8,9,10)"
-    },
-    "admin_boundaries_centroid_z13_15": {
-      "source": "admin_areas",
-      "geometry_transform": "(ST_MaximumInscribedCircle(geometry)).center",
-      "geometry_transform_types": "GeometryType(geometry) IN ('POLYGON', 'MULTIPOLYGON')",
-      "sql_filter": "admin_level IN (1,2,3,4,5,6,7,8,9,10) AND name IS NOT NULL AND name <> ''"
+      "sql_filter": "admin_level IN (1,2,3,4,5,6,7,8,9,10)",
+      "sub_tables": [
+        {
+          "table": "admin_boundaries_lines_z13_15",
+          "geometry_transform": "ST_Boundary(geometry)"
+        },
+        {
+          "table": "admin_boundaries_centroid_z13_15",
+          "geometry_transform": "(ST_MaximumInscribedCircle(geometry)).center",
+          "sql_filter": "name IS NOT NULL AND name <> ''"
+        }
+      ]
     },
-    "admin_boundaries_lines_z16_20": {
+    "admin_areas_z16_20": {
       "source": "admin_areas",
       "geometry_transform": "ST_Boundary(geometry)",
       "geometry_transform_types": "GeometryType(geometry) IN ('POLYGON', 'MULTIPOLYGON')",
-      "sql_filter": "admin_level IN (1,2,3,4,5,6,7,8,9,10)"
-    },
-    "admin_boundaries_centroid_z16_20": {
-      "source": "admin_areas",
-      "geometry_transform": "(ST_MaximumInscribedCircle(geometry)).center",
-      "geometry_transform_types": "GeometryType(geometry) IN ('POLYGON', 'MULTIPOLYGON')",
-      "sql_filter": "admin_level IN (1,2,3,4,5,6,7,8,9,10) AND name IS NOT NULL AND name <> ''"
+      "sql_filter": "admin_level IN (1,2,3,4,5,6,7,8,9,10)",
+      "sub_tables": [
+        {
+          "table": "admin_boundaries_lines_z16_20",
+          "geometry_transform": "ST_Boundary(geometry)"
+        },
+        {
+          "table": "admin_boundaries_centroid_z16_20",
+          "geometry_transform": "(ST_MaximumInscribedCircle(geometry)).center",
+          "sql_filter": "name IS NOT NULL AND name <> ''"
+        }
+      ]
     }
   },
   "tables": {

diff --git a/images/tiler-imposm/config/layers/admin_lines.json b/images/tiler-imposm/config/layers/admin_lines.json
@@ -8,33 +8,18 @@
     ]
   },
   "generalized_tables": {
-    "admin_lines_z0_2": {
-      "source": "admin_lines",
-      "sql_filter": "maritime='yes'",
-      "tolerance": 5000
-    },
-    "admin_lines_z3_5": {
-      "source": "admin_lines",
+    "admin_lines_z0_5": {
+      "source": "admin_lines_z6_9",
       "sql_filter": "maritime='yes'",
       "tolerance": 2000
     },
-    "admin_lines_z6_7": {
-      "source": "admin_lines",
-      "sql_filter": "maritime='yes'",
-      "tolerance": 1000
-    },
-    "admin_lines_z8_9": {
-      "source": "admin_lines",
+    "admin_lines_z6_9": {
+      "source": "admin_lines_z10_15",
       "sql_filter": "maritime='yes'",
       "tolerance": 500
     },
-    "admin_lines_z10_12": {
-      "source": "admin_lines",
-      "sql_filter": "maritime='yes'",
-      "tolerance": 100
-    },
-    "admin_lines_z13_15": {
-      "source": "admin_lines",
+    "admin_lines_z10_15": {
+      "source": "admin_lines_z16_20",
       "sql_filter": "maritime='yes'",
       "tolerance": 10
     },