From 5a91a98ad3c81709892574d4bf0037ab05c62188 Mon Sep 17 00:00:00 2001 From: Rub21 Date: Wed, 18 Dec 2024 10:48:39 -0500 Subject: [PATCH 1/4] Create separed table for simplification --- .github/workflows/chartpress.yaml | 2 +- compose/tiler.yml | 51 ++- .../config/layers/admin_areas.json | 164 ++++---- .../config/layers/water_areas.json | 92 ++--- .../queries/admin_boundaries_centroids.sql | 81 +++- images/tiler-imposm/update_tables.py | 359 ++++++++++++------ 6 files changed, 477 insertions(+), 272 deletions(-) diff --git a/.github/workflows/chartpress.yaml b/.github/workflows/chartpress.yaml index 3df7c758..f92fc6f5 100644 --- a/.github/workflows/chartpress.yaml +++ b/.github/workflows/chartpress.yaml @@ -5,7 +5,7 @@ on: - 'main' - 'staging' - 'development' - - 'imposm_transport' + - 'refactor/imposm' jobs: build: runs-on: ubuntu-20.04 diff --git a/compose/tiler.yml b/compose/tiler.yml index cd84ea41..800a93ea 100644 --- a/compose/tiler.yml +++ b/compose/tiler.yml @@ -6,36 +6,47 @@ services: dockerfile: Dockerfile ports: - "5432:5432" - volumes: - - ../data/tiler-db-data2:/var/lib/postgresql/data + # volumes: + # - ../data/tiler-db-data2:/var/lib/postgresql/data env_file: - ../envs/.env.tiler restart: always + networks: + - tiler_network + imposm: - image: rub21/tiler-imposm:v15 + image: rub21/tiler-imposm:v16 build: context: ../images/tiler-imposm dockerfile: Dockerfile volumes: - - ../data/tiler-imposm-data2:/mnt/data - - ../images/tiler-imposm:/app + # - ../data/tiler-imposm-data2:/mnt/data + - ../images/tiler-imposm:/osm command: - sh - -c - - "sleep 20 && ./start.sh" - env_file: - - ../envs/.env.tiler - tiler: - image: ohm-tiler-server:v1 - build: - context: ../images/tiler-server - dockerfile: Dockerfile - volumes: - - ../data/tiler-server-data:/mnt/data - - ../images/tiler-server:/app - ports: - - "9090:9090" + - "./start.sh" env_file: - ../envs/.env.tiler - restart: always - \ No newline at end of file + networks: + - tiler_network + + # tiler: + # image: ohm-tiler-server:v1 + # build: + # context: ../images/tiler-server + # dockerfile: Dockerfile + # volumes: + # - ../data/tiler-server-data:/mnt/data + # - ../images/tiler-server:/app + # ports: + # - "9090:9090" + # env_file: + # - ../envs/.env.tiler + # restart: always + # networks: + # - tiler_network + +networks: + tiler_network: + driver: bridge \ No newline at end of file diff --git a/images/tiler-imposm/config/layers/admin_areas.json b/images/tiler-imposm/config/layers/admin_areas.json index 53665575..f79a7e16 100644 --- a/images/tiler-imposm/config/layers/admin_areas.json +++ b/images/tiler-imposm/config/layers/admin_areas.json @@ -8,95 +8,123 @@ ] }, "generalized_tables": { - "admin_boundaries_lines_z0_2": { + "admin_areas_z0_2": { "source": "admin_areas", "tolerance": 5000, - "geometry_transform": "ST_Boundary(geometry)", - "geometry_transform_types": "GeometryType(geometry) IN ('POLYGON', 'MULTIPOLYGON')", - "sql_filter": "admin_level IN (1,2)" - }, - "admin_boundaries_centroid_z0_2": { - "source": "admin_areas", - "geometry_transform": "(ST_MaximumInscribedCircle(geometry)).center", - "geometry_transform_types": "GeometryType(geometry) IN ('POLYGON', 'MULTIPOLYGON')", - "sql_filter": "admin_level IN (1,2) AND name IS NOT NULL AND name <> ''" + "sql_filter": "admin_level IN (1,2)", + "sub_tables": [ + { + "table": "admin_boundaries_lines_z0_2", + "geometry_transform": "ST_Boundary(geometry)" + }, + { + "table": "admin_boundaries_centroid_z0_2", + "geometry_transform": "(ST_MaximumInscribedCircle(geometry)).center", + "sql_filter": "name IS NOT NULL AND name <> ''" + } + ] }, - "admin_boundaries_lines_z3_5": { + + "admin_areas_z3_5": { "source": "admin_areas", "tolerance": 1000, - "geometry_transform": "ST_Boundary(geometry)", - "geometry_transform_types": "GeometryType(geometry) IN ('POLYGON', 'MULTIPOLYGON')", - "sql_filter": "admin_level IN (1,2,3,4)" - }, - "admin_boundaries_centroid_z3_5": { - "source": "admin_areas", - "geometry_transform": "(ST_MaximumInscribedCircle(geometry)).center", - "geometry_transform_types": "GeometryType(geometry) IN ('POLYGON', 'MULTIPOLYGON')", - "sql_filter": "admin_level IN (1,2,3,4) AND name IS NOT NULL AND name <> ''" + "sql_filter": "admin_level IN (1,2,3,4)", + "sub_tables": [ + { + "table": "admin_boundaries_lines_z3_5", + "geometry_transform": "ST_Boundary(geometry)" + }, + { + "table": "admin_boundaries_centroid_z3_5", + "geometry_transform": "(ST_MaximumInscribedCircle(geometry)).center", + "sql_filter": "name IS NOT NULL AND name <> ''" + } + ] }, - "admin_boundaries_lines_z6_7": { + + "admin_areas_z6_7": { "source": "admin_areas", "tolerance": 200, - "geometry_transform": "ST_Boundary(geometry)", - "geometry_transform_types": "GeometryType(geometry) IN ('POLYGON', 'MULTIPOLYGON')", - "sql_filter": "admin_level IN (1,2,3,4,5,6)" - }, - "admin_boundaries_centroid_z6_7": { - "source": "admin_areas", - "geometry_transform": "(ST_MaximumInscribedCircle(geometry)).center", - "geometry_transform_types": "GeometryType(geometry) IN ('POLYGON', 'MULTIPOLYGON')", - "sql_filter": "admin_level IN (1,2,3,4,5,6) AND name IS NOT NULL AND name <> ''" + "sql_filter": "admin_level IN (1,2,3,4,5,6)", + "sub_tables": [ + { + "table": "admin_boundaries_lines_z6_7", + "geometry_transform": "ST_Boundary(geometry)" + }, + { + "table": "admin_boundaries_centroid_z6_7", + "geometry_transform": "(ST_MaximumInscribedCircle(geometry)).center", + "sql_filter": "name IS NOT NULL AND name <> ''" + } + ] }, - "admin_boundaries_lines_z8_9": { + + "admin_areas_z8_9": { "source": "admin_areas", "tolerance": 100, - "geometry_transform": "ST_Boundary(geometry)", - "geometry_transform_types": "GeometryType(geometry) IN ('POLYGON', 'MULTIPOLYGON')", - "sql_filter": "admin_level IN (1,2,3,4,5,6,7,8,9)" - }, - "admin_boundaries_centroid_z8_9": { - "source": "admin_areas", - "geometry_transform": "(ST_MaximumInscribedCircle(geometry)).center", - "geometry_transform_types": "GeometryType(geometry) IN ('POLYGON', 'MULTIPOLYGON')", - "sql_filter": "admin_level IN (1,2,3,4,5,6,7,8,9) AND name IS NOT NULL AND name <> ''" + "sql_filter": "admin_level IN (1,2,3,4,5,6,7,8,9)", + "sub_tables": [ + { + "table": "admin_boundaries_lines_z8_9", + "geometry_transform": "ST_Boundary(geometry)" + }, + { + "table": "admin_boundaries_centroid_z8_9", + "geometry_transform": "(ST_MaximumInscribedCircle(geometry)).center", + "sql_filter": "name IS NOT NULL AND name <> ''" + } + ] }, - "admin_boundaries_lines_z10_12": { + + "admin_areas_z10_12": { "source": "admin_areas", "tolerance": 20, - "geometry_transform": "ST_Boundary(geometry)", - "geometry_transform_types": "GeometryType(geometry) IN ('POLYGON', 'MULTIPOLYGON')", - "sql_filter": "admin_level IN (1,2,3,4,5,6,7,8,9,10)" - }, - "admin_boundaries_centroid_z10_12": { - "source": "admin_areas", - "geometry_transform": "(ST_MaximumInscribedCircle(geometry)).center", - "geometry_transform_types": "GeometryType(geometry) IN ('POLYGON', 'MULTIPOLYGON')", - "sql_filter": "admin_level IN (1,2,3,4,5,6,7,8,9,10) AND name IS NOT NULL AND name <> ''" + "sql_filter": "admin_level IN (1,2,3,4,5,6,7,8,9,10)", + "sub_tables": [ + { + "table": "admin_boundaries_lines_z10_12", + "geometry_transform": "ST_Boundary(geometry)" + }, + { + "table": "admin_boundaries_centroid_z10_12", + "geometry_transform": "(ST_MaximumInscribedCircle(geometry)).center", + "sql_filter": "name IS NOT NULL AND name <> ''" + } + ] }, - "admin_boundaries_lines_z13_15": { + + "admin_areas_z13_15": { "source": "admin_areas", "tolerance": 5, - "geometry_transform": "ST_Boundary(geometry)", - "geometry_transform_types": "GeometryType(geometry) IN ('POLYGON', 'MULTIPOLYGON')", - "sql_filter": "admin_level IN (1,2,3,4,5,6,7,8,9,10)" - }, - "admin_boundaries_centroid_z13_15": { - "source": "admin_areas", - "geometry_transform": "(ST_MaximumInscribedCircle(geometry)).center", - "geometry_transform_types": "GeometryType(geometry) IN ('POLYGON', 'MULTIPOLYGON')", - "sql_filter": "admin_level IN (1,2,3,4,5,6,7,8,9,10) AND name IS NOT NULL AND name <> ''" + "sql_filter": "admin_level IN (1,2,3,4,5,6,7,8,9,10)", + "sub_tables": [ + { + "table": "admin_boundaries_lines_z13_15", + "geometry_transform": "ST_Boundary(geometry)" + }, + { + "table": "admin_boundaries_centroid_z13_15", + "geometry_transform": "(ST_MaximumInscribedCircle(geometry)).center", + "sql_filter": "name IS NOT NULL AND name <> ''" + } + ] }, - "admin_boundaries_lines_z16_20": { + "admin_areas_z16_20": { "source": "admin_areas", "geometry_transform": "ST_Boundary(geometry)", "geometry_transform_types": "GeometryType(geometry) IN ('POLYGON', 'MULTIPOLYGON')", - "sql_filter": "admin_level IN (1,2,3,4,5,6,7,8,9,10)" - }, - "admin_boundaries_centroid_z16_20": { - "source": "admin_areas", - "geometry_transform": "(ST_MaximumInscribedCircle(geometry)).center", - "geometry_transform_types": "GeometryType(geometry) IN ('POLYGON', 'MULTIPOLYGON')", - "sql_filter": "admin_level IN (1,2,3,4,5,6,7,8,9,10) AND name IS NOT NULL AND name <> ''" + "sql_filter": "admin_level IN (1,2,3,4,5,6,7,8,9,10)", + "sub_tables": [ + { + "table": "admin_boundaries_lines_z16_20", + "geometry_transform": "ST_Boundary(geometry)" + }, + { + "table": "admin_boundaries_centroid_z16_20", + "geometry_transform": "(ST_MaximumInscribedCircle(geometry)).center", + "sql_filter": "name IS NOT NULL AND name <> ''" + } + ] } }, "tables": { diff --git a/images/tiler-imposm/config/layers/water_areas.json b/images/tiler-imposm/config/layers/water_areas.json index ec519585..4aa79da3 100644 --- a/images/tiler-imposm/config/layers/water_areas.json +++ b/images/tiler-imposm/config/layers/water_areas.json @@ -3,68 +3,74 @@ "water_areas_z0_2": { "source": "water_areas", "sql_filter": "type IN ('water', 'riverbank') AND area > 100000000", - "tolerance": 5000 - }, - "water_areas_centroid_z0_2": { - "source": "water_areas", - "sql_filter": "type IN ('water', 'riverbank') AND area > 100000000 AND name IS NOT NULL AND name <> ''", - "geometry_transform": "(ST_MaximumInscribedCircle(geometry)).center", - "geometry_transform_types": "GeometryType(geometry) IN ('POLYGON', 'MULTIPOLYGON')" + "tolerance": 5000, + "sub_tables": [ + { + "table": "water_areas_centroid_z0_2", + "geometry_transform": "(ST_MaximumInscribedCircle(geometry)).center", + "sql_filter": "name IS NOT NULL AND name <> ''" + } + ] }, "water_areas_z3_5": { "source": "water_areas", "sql_filter": "type IN ('water', 'pond', 'basin', 'canal', 'mill_pond', 'riverbank') AND area > 50000000", - "tolerance": 1000 - }, - "water_areas_centroid_z3_5": { - "source": "water_areas", - "sql_filter": "type IN ('water', 'pond', 'basin', 'canal', 'mill_pond', 'riverbank') AND area > 50000000 AND name IS NOT NULL AND name <> ''", - "geometry_transform": "(ST_MaximumInscribedCircle(geometry)).center", - "geometry_transform_types": "GeometryType(geometry) IN ('POLYGON', 'MULTIPOLYGON')" + "tolerance": 1000, + "sub_tables": [ + { + "table": "water_areas_centroid_z3_5", + "geometry_transform": "(ST_MaximumInscribedCircle(geometry)).center", + "sql_filter": "name IS NOT NULL AND name <> ''" + } + ] }, "water_areas_z6_7": { "source": "water_areas", "sql_filter": "type IN ('water', 'pond', 'basin', 'canal', 'mill_pond', 'riverbank') AND area > 1000000", - "tolerance": 200 - }, - "water_areas_centroid_z6_7": { - "source": "water_areas", - "sql_filter": "type IN ('water', 'pond', 'basin', 'canal', 'mill_pond', 'riverbank') AND area > 1000000 AND name IS NOT NULL AND name <> ''", - "geometry_transform": "(ST_MaximumInscribedCircle(geometry)).center", - "geometry_transform_types": "GeometryType(geometry) IN ('POLYGON', 'MULTIPOLYGON')" + "tolerance": 200, + "sub_tables": [ + { + "table": "water_areas_centroid_z6_7", + "geometry_transform": "(ST_MaximumInscribedCircle(geometry)).center", + "sql_filter": "name IS NOT NULL AND name <> ''" + } + ] }, "water_areas_z8_9": { "source": "water_areas", "sql_filter": "type IN ('water', 'pond', 'basin', 'canal', 'mill_pond', 'riverbank') AND area > 10000", - "tolerance": 100 - }, - "water_areas_centroid_z8_9": { - "source": "water_areas", - "sql_filter": "type IN ('water', 'pond', 'basin', 'canal', 'mill_pond', 'riverbank') AND area > 10000 AND name IS NOT NULL AND name <> ''", - "geometry_transform": "(ST_MaximumInscribedCircle(geometry)).center", - "geometry_transform_types": "GeometryType(geometry) IN ('POLYGON', 'MULTIPOLYGON')" + "tolerance": 100, + "sub_tables": [ + { + "table": "water_areas_centroid_z8_9", + "geometry_transform": "(ST_MaximumInscribedCircle(geometry)).center", + "sql_filter": "name IS NOT NULL AND name <> ''" + } + ] }, "water_areas_z10_12": { "source": "water_areas", "sql_filter": "type IN ('water', 'pond', 'basin', 'canal', 'mill_pond', 'riverbank')", - "tolerance": 20 - }, - "water_areas_centroid_z10_12": { - "source": "water_areas", - "sql_filter": "type IN ('water', 'pond', 'basin', 'canal', 'mill_pond', 'riverbank') AND name IS NOT NULL AND name <> ''", - "geometry_transform": "(ST_MaximumInscribedCircle(geometry)).center", - "geometry_transform_types": "GeometryType(geometry) IN ('POLYGON', 'MULTIPOLYGON')" + "tolerance": 20, + "sub_tables": [ + { + "table": "water_areas_centroid_z10_12", + "geometry_transform": "(ST_MaximumInscribedCircle(geometry)).center", + "sql_filter": "name IS NOT NULL AND name <> ''" + } + ] }, "water_areas_z13_15": { "source": "water_areas", "sql_filter": "type IN ('water', 'pond', 'basin', 'canal', 'mill_pond', 'riverbank', 'dock')", - "tolerance": 5 - }, - "water_areas_centroid_z13_15": { - "source": "water_areas", - "sql_filter": "type IN ('water', 'pond', 'basin', 'canal', 'mill_pond', 'riverbank', 'dock') AND name IS NOT NULL AND name <> ''", - "geometry_transform": "(ST_MaximumInscribedCircle(geometry)).center", - "geometry_transform_types": "GeometryType(geometry) IN ('POLYGON', 'MULTIPOLYGON')" + "tolerance": 5, + "sub_tables": [ + { + "table": "water_areas_centroid_z13_15", + "geometry_transform": "(ST_MaximumInscribedCircle(geometry)).center", + "sql_filter": "name IS NOT NULL AND name <> ''" + } + ] } }, "tables": { @@ -126,4 +132,4 @@ } } } -} +} \ No newline at end of file diff --git a/images/tiler-imposm/queries/admin_boundaries_centroids.sql b/images/tiler-imposm/queries/admin_boundaries_centroids.sql index b955df66..e9b7a6a0 100644 --- a/images/tiler-imposm/queries/admin_boundaries_centroids.sql +++ b/images/tiler-imposm/queries/admin_boundaries_centroids.sql @@ -1,9 +1,35 @@ +DO $$ +DECLARE + trigger_name TEXT; + table_name TEXT; +BEGIN + -- Loop through all triggers for tables matching the pattern + FOR trigger_name, table_name IN + SELECT tgname, relname + FROM pg_trigger + JOIN pg_class ON pg_trigger.tgrelid = pg_class.oid + JOIN pg_namespace ON pg_class.relnamespace = pg_namespace.oid + WHERE nspname = 'public' -- Adjust schema if needed + AND relname LIKE 'osm_admin_boundaries_centroid_%' + AND NOT tgisinternal -- Exclude internal triggers + LOOP + -- Dynamically drop the trigger if it exists + EXECUTE format('DROP TRIGGER IF EXISTS %I ON %I;', trigger_name, table_name); + RAISE NOTICE 'Dropped trigger % on table %', trigger_name, table_name; + END LOOP; +END; +$$ LANGUAGE plpgsql; + -- Create a function that updates the osm_admin_boundaries_centroid_* if relations have labels or not CREATE OR REPLACE FUNCTION update_admin_boundaries_centroids() RETURNS void AS $$ DECLARE table_name text; + updated_rows integer; BEGIN + -- Log the start of the update process + RAISE NOTICE 'Starting the update of admin boundaries centroids at %', clock_timestamp(); + -- Loop through all table names that match the pattern FOR table_name IN SELECT t.table_name @@ -13,30 +39,46 @@ BEGIN -- Dynamically execute the update query for each table BEGIN EXECUTE format(' - UPDATE %I - SET has_label = 1 - WHERE osm_id IN ( - SELECT osm_id - FROM osm_relation_members - WHERE role = ''label'' - );', table_name); + WITH updated AS ( + UPDATE %I + SET has_label = 1 + WHERE osm_id IN ( + SELECT osm_id + FROM osm_relation_members + WHERE role = ''label'' + ) + RETURNING * + ) + SELECT COUNT(*) FROM updated;', table_name) + INTO updated_rows; + + -- Log the number of updated rows + RAISE NOTICE 'Table % updated. Rows affected: %', table_name, updated_rows; + EXCEPTION WHEN OTHERS THEN + -- Log any errors encountered RAISE NOTICE 'Error updating table %: %', table_name, SQLERRM; END; END LOOP; -- Log completion - RAISE NOTICE 'Update completed for all matching tables.'; + RAISE NOTICE 'Update process completed for all matching tables at %.', clock_timestamp(); END; $$ LANGUAGE plpgsql; --- Create a function that updates the osm_admin_boundaries_centroid_* if relations have labels or not -CREATE OR REPLACE FUNCTION create_update_has_label_triggers(pattern text) +-- Execute the update function when the import is done +SELECT update_admin_boundaries_centroids(); + +CREATE OR REPLACE FUNCTION create_update_has_label_triggers() RETURNS void AS $$ DECLARE table_name text; + pattern text := 'osm_admin_boundaries_centroid_%'; -- Fixed pattern BEGIN - -- Loop through all tables matching the provided pattern + -- Log the start of the trigger creation process + RAISE NOTICE 'Starting the creation of triggers for tables matching pattern: %', pattern; + + -- Loop through all tables matching the fixed pattern FOR table_name IN SELECT t.table_name FROM information_schema.tables AS t @@ -47,15 +89,20 @@ BEGIN 'CREATE OR REPLACE FUNCTION %I_update_has_label_row() RETURNS TRIGGER AS $trigger_body$ BEGIN + RAISE NOTICE ''Trigger activated for table %% with osm_id: %%'', TG_TABLE_NAME, NEW.osm_id; + IF EXISTS ( SELECT 1 FROM osm_relation_members WHERE osm_id = NEW.osm_id AND role = ''label'' ) THEN NEW.has_label := 1; + RAISE NOTICE ''Set has_label to 1 for osm_id: %%'', NEW.osm_id; ELSE NEW.has_label := 0; + RAISE NOTICE ''Set has_label to 0 for osm_id: %%'', NEW.osm_id; END IF; + RETURN NEW; END; $trigger_body$ LANGUAGE plpgsql;', @@ -72,15 +119,15 @@ BEGIN table_name, -- Table name table_name -- Function name ); + + -- Log the trigger creation for the current table + RAISE NOTICE 'Trigger and function created for table: %', table_name; END LOOP; - RAISE NOTICE 'Triggers and functions created for tables matching %.', pattern; + -- Log completion of the trigger creation process + RAISE NOTICE 'All triggers created for tables matching pattern: %.', pattern; END; $$ LANGUAGE plpgsql; - --- Execute the update function when the import is done -SELECT update_admin_boundaries_centroids(); - -- Execute the function to create triggers for tables matching the pattern -SELECT create_update_has_label_triggers('osm_admin_boundaries_centroid%'); +SELECT create_update_has_label_triggers(); diff --git a/images/tiler-imposm/update_tables.py b/images/tiler-imposm/update_tables.py index d116e697..936fe287 100644 --- a/images/tiler-imposm/update_tables.py +++ b/images/tiler-imposm/update_tables.py @@ -4,9 +4,14 @@ import subprocess import time -logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s") +# Configuración del logger +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s - %(levelname)s - %(message)s" +) logger = logging.getLogger(__name__) +# Cargar configuración DB desde variables de entorno DB_CONFIG = { "dbname": os.getenv("POSTGRES_DB"), "user": os.getenv("POSTGRES_USER"), @@ -15,166 +20,274 @@ "port": int(os.getenv("POSTGRES_PORT", 5432)) } -PSQL_CONN = f"postgresql://{DB_CONFIG['user']}:{DB_CONFIG['password']}@{DB_CONFIG['host']}:{DB_CONFIG['port']}/{DB_CONFIG['dbname']}" - REQUIRED_ENV_VARS = ["POSTGRES_DB", "POSTGRES_USER", "POSTGRES_PASSWORD", "POSTGRES_HOST"] for var in REQUIRED_ENV_VARS: if not os.getenv(var): - logger.error(f"Environment variable {var} is not defined. Exiting.") - raise EnvironmentError(f"Environment variable {var} is not defined.") + logger.error(f"La variable de entorno {var} no está definida. Saliendo.") + raise EnvironmentError(f"La variable de entorno {var} no está definida.") -def load_imposm_config(filepath): - """Load the imposm3.json configuration file.""" - logger.info(f"Loading configuration from {filepath}") - try: - with open(filepath, "r") as f: - return json.load(f) - except FileNotFoundError: - logger.error(f"Configuration file {filepath} not found.") - raise - except json.JSONDecodeError: - logger.error(f"Error parsing JSON from {filepath}.") - raise +PSQL_CONN = f"postgresql://{DB_CONFIG['user']}:{DB_CONFIG['password']}@{DB_CONFIG['host']}:{DB_CONFIG['port']}/{DB_CONFIG['dbname']}" -def execute_psql_query(query): - """Execute a query using psql and print the output.""" +def load_imposm_config(filepath: str) -> dict: + """ + Carga la configuración desde un archivo JSON. + """ + logger.info(f"Cargando configuración desde {filepath}") + with open(filepath, "r") as f: + return json.load(f) + +def execute_psql_query(query: str): + """ + Ejecuta una consulta SQL usando psql. + """ try: - # logger.info(f"Executing query:\t{query}") result = subprocess.run( ["psql", PSQL_CONN, "-c", query], text=True, capture_output=True ) if result.returncode != 0: - logger.error(f"Error executing query: {result.stderr}") + logger.error(f"Error al ejecutar la consulta: {result.stderr.strip()}") else: - logger.info(f"Query executed successfully:\n{result.stdout}") + logger.info(f"Consulta ejecutada con éxito:\n{result.stdout.strip()}") except Exception as e: - logger.error(f"Error while executing query with psql: {e}") - -def delete_existing_triggers(generalized_tables): - """Delete existing triggers before applying transformations.""" - logger.info("Deleting existing triggers...") - for table_name in generalized_tables.keys(): - fixed_table_name = f"osm_{table_name}" - trigger_name = f"{fixed_table_name}_before_insert_update" + logger.error(f"Error ejecutando la consulta con psql: {e}") - drop_trigger_query = f""" - DROP TRIGGER IF EXISTS {trigger_name} ON {fixed_table_name}; - DROP FUNCTION IF EXISTS {fixed_table_name}_transform_trigger(); - """ - execute_psql_query(drop_trigger_query) - -def apply_geometry_transformations(generalized_tables): - """Apply geometry transformations using psql.""" - logger.info("Starting geometry transformations...") +def delete_sub_tables(generalized_tables: dict): + """ + Deletes all sub-tables defined in the configuration. + """ + logger.info("Starting deletion of sub-tables...") for table_name, table_info in generalized_tables.items(): fixed_table_name = f"osm_{table_name}" - geometry_transform = table_info.get("geometry_transform") - geometry_transform_types = table_info.get("geometry_transform_types") - - # Skip if transform or types are not defined - if not geometry_transform or not geometry_transform_types: - logger.warning( - f"Skipping transformations for {fixed_table_name}: " - "'geometry_transform' or 'geometry_transform_types' not defined." - ) + sub_tables = table_info.get("sub_tables") + + # If no sub-tables are defined, skip + if not sub_tables: + logger.info(f"No sub-tables defined for {fixed_table_name}. Skipping.") continue - # Build the SQL query - sql_query = f"""UPDATE {fixed_table_name} SET geometry = {geometry_transform} WHERE {geometry_transform_types};""" - start_time = time.time() - execute_psql_query(sql_query) - elapsed_time = time.time() - start_time + for sub_table in sub_tables: + sub_table_name = sub_table.get("table") + if not sub_table_name: + logger.warning( + f"Sub-table for {fixed_table_name} does not have a defined name. Skipping." + ) + continue + + sub_fixed_table_name = f"osm_{sub_table_name}" + + # Check if the table exists before attempting to delete it + if table_exists(sub_fixed_table_name): + logger.info(f"Deleting sub-table {sub_fixed_table_name}...") + drop_table_query = f"DROP TABLE IF EXISTS {sub_fixed_table_name} CASCADE;" + execute_psql_query(drop_table_query) + else: + logger.info(f"Sub-table {sub_fixed_table_name} does not exist. Skipping.") + + logger.info("Sub-table deletion process completed.") + + +def table_exists(table_name: str) -> bool: + """ + Verifica si una tabla existe en la base de datos. + """ + query = f"SELECT to_regclass('public.{table_name}');" + result = subprocess.run( + ["psql", PSQL_CONN, "-t", "-c", query], + text=True, + capture_output=True + ) + output = result.stdout.strip() + return output != "" and output != "-" + + +def create_trigger_for_sub_table(sub_table_name: str, geometry_transform: str, sql_filter: str = None): + """ + Creates triggers for a sub-table, applying the geometric transformation + and optionally a SQL filter for future INSERT/UPDATE operations. + Handles row deletions. + If triggers already exist, they are dropped and recreated. + """ + fixed_table_name = f"osm_{sub_table_name}" + insert_update_trigger_name = f"{fixed_table_name}_before_insert_update" + delete_trigger_name = f"{fixed_table_name}_before_delete" + + # Drop existing triggers if they exist + drop_trigger_query = f""" + DO $$ + BEGIN + IF EXISTS ( + SELECT 1 + FROM pg_trigger + WHERE tgname = '{insert_update_trigger_name}' + ) THEN + EXECUTE format('DROP TRIGGER IF EXISTS %I ON %I;', '{insert_update_trigger_name}', '{fixed_table_name}'); + END IF; + + IF EXISTS ( + SELECT 1 + FROM pg_trigger + WHERE tgname = '{delete_trigger_name}' + ) THEN + EXECUTE format('DROP TRIGGER IF EXISTS %I ON %I;', '{delete_trigger_name}', '{fixed_table_name}'); + END IF; + END $$; + """ + execute_psql_query(drop_trigger_query) + logger.info(f"Existing triggers removed on table {fixed_table_name} (if they existed).") + + # Prepare the SQL filter clause if provided + sql_filter_clause = f"({sql_filter})" if sql_filter else "TRUE" - logger.info(f"Transformation for table {fixed_table_name} completed in {elapsed_time:.2f} seconds.") + # Create the trigger for INSERT and UPDATE + transform_for_trigger = geometry_transform.replace('geometry', 'NEW.geometry') -def create_triggers(generalized_tables): - """Create triggers for future updates using psql.""" - logger.info("Creating triggers for future geometry transformations...") + insert_update_trigger_function = f""" + CREATE OR REPLACE FUNCTION {fixed_table_name}_transform_trigger() + RETURNS TRIGGER AS $$ + BEGIN + -- Apply geometric transformation + NEW.geometry = {transform_for_trigger}; + + -- Apply optional SQL filter + IF {sql_filter_clause} THEN + RETURN NEW; + ELSE + RETURN NULL; -- Ignore the row if it doesn't match the filter + END IF; + END; + $$ LANGUAGE plpgsql; + + CREATE TRIGGER {insert_update_trigger_name} + BEFORE INSERT OR UPDATE ON {fixed_table_name} + FOR EACH ROW + EXECUTE FUNCTION {fixed_table_name}_transform_trigger(); + """ + execute_psql_query(insert_update_trigger_function) + logger.info(f"Trigger {insert_update_trigger_name} created for INSERT/UPDATE on table {fixed_table_name}.") + + # Create the trigger for DELETE + delete_trigger_function = f""" + CREATE OR REPLACE FUNCTION {fixed_table_name}_delete_trigger() + RETURNS TRIGGER AS $$ + BEGIN + -- Log information about the deleted row + RAISE NOTICE 'Row deleted from table % with osm_id: %', TG_TABLE_NAME, OLD.osm_id; + RETURN OLD; + END; + $$ LANGUAGE plpgsql; + + CREATE TRIGGER {delete_trigger_name} + BEFORE DELETE ON {fixed_table_name} + FOR EACH ROW + EXECUTE FUNCTION {fixed_table_name}_delete_trigger(); + """ + execute_psql_query(delete_trigger_function) + logger.info(f"Trigger {delete_trigger_name} created for DELETE on table {fixed_table_name}.") + + +def apply_geometry_transformations(generalized_tables: dict): + """ + Applies initial geometric transformations to generalized tables + and creates sub-tables with their transformations and triggers. + """ + logger.info("Starting initial geometric transformations...") for table_name, table_info in generalized_tables.items(): fixed_table_name = f"osm_{table_name}" - geometry_transform = table_info.get("geometry_transform") - geometry_transform_types = table_info.get("geometry_transform_types") + sub_tables = table_info.get("sub_tables") - if not geometry_transform or not geometry_transform_types: - logger.warning(f"Skipping trigger creation for {fixed_table_name}.") + # Skip if no sub-tables are defined + if not sub_tables: + logger.info(f"No sub_tables defined for {fixed_table_name}. Skipping.") continue - # Check if the trigger already exists - check_trigger_query = f""" - SELECT 1 - FROM pg_trigger - WHERE tgname = '{fixed_table_name}_before_insert_update'; - """ - result = subprocess.run( - ["psql", PSQL_CONN, "-c", check_trigger_query], - text=True, - capture_output=True - ) - if "1" in result.stdout: - logger.info(f"Trigger {fixed_table_name}_before_insert_update already exists. Skipping.") - continue + for sub_table in sub_tables: + sub_table_name = sub_table.get("table") + if not sub_table_name: + logger.warning(f"Sub-table for {fixed_table_name} has no defined name. Skipping.") + continue - # Create the trigger if it does not exist - trigger_function = f""" - CREATE OR REPLACE FUNCTION {fixed_table_name}_transform_trigger() - RETURNS TRIGGER AS $$ - DECLARE - start_time TIMESTAMP; - end_time TIMESTAMP; - elapsed_time INTERVAL; - BEGIN - -- Record the start time - start_time := clock_timestamp(); - - -- Perform the transformation - NEW.geometry = {geometry_transform.replace('geometry', 'NEW.geometry')}; - - -- Record the end time - end_time := clock_timestamp(); - elapsed_time := end_time - start_time; - - -- Log the time taken and the object being updated - RAISE NOTICE 'Table: %, ID: %, Time Taken: % ms', - TG_TABLE_NAME, NEW.id, EXTRACT(MILLISECOND FROM elapsed_time); + sub_table_fixed_name = f"osm_{sub_table_name}" + sub_geometry_transform = sub_table["geometry_transform"] + sub_sql_filter = sub_table.get("sql_filter", None) - RETURN NEW; - END; - $$ LANGUAGE plpgsql; - - CREATE TRIGGER {fixed_table_name}_before_insert_update - BEFORE INSERT OR UPDATE ON {fixed_table_name} - FOR EACH ROW - EXECUTE FUNCTION {fixed_table_name}_transform_trigger(); - """ - execute_psql_query(trigger_function) - -def main(imposm3_config_path): - """Main execution flow.""" + # Retrieve the list of columns from the main table + get_columns_query = f""" + SELECT column_name + FROM information_schema.columns + WHERE table_schema = 'public' + AND table_name = '{fixed_table_name}' + ORDER BY ordinal_position; + """ + result = subprocess.run( + ["psql", PSQL_CONN, "-t", "-c", get_columns_query], + text=True, + capture_output=True + ) + if result.returncode != 0: + logger.error(f"Error retrieving columns for {fixed_table_name}: {result.stderr.strip()}") + continue + + # Clean and retrieve the list of columns + retrieved_columns = [col.strip() for col in result.stdout.strip().split('\n') if col.strip()] + + if not retrieved_columns: + logger.warning(f"No columns found for {fixed_table_name}. Skipping.") + continue + + # Create the sub-table if it doesn't exist + if not table_exists(sub_table_fixed_name): + logger.info(f"Creating sub-table {sub_table_fixed_name} based on {fixed_table_name}...") + create_table_query = f""" + CREATE TABLE {sub_table_fixed_name} (LIKE {fixed_table_name} INCLUDING ALL); + """ + execute_psql_query(create_table_query) + + # Build the column list for SELECT, replacing geometry with geometry_transform + selected_columns = [] + for col in retrieved_columns: + if col == 'geometry': + selected_columns.append(f"{sub_geometry_transform} AS geometry") + else: + selected_columns.append(col) + + # Generate the INSERT query with optional SQL filter + where_clause = f"WHERE {sub_sql_filter}" if sub_sql_filter else "" + insert_query = f""" + INSERT INTO {sub_table_fixed_name} ({", ".join(retrieved_columns)}) + SELECT {", ".join(selected_columns)} + FROM {fixed_table_name} + {where_clause}; + """ + execute_psql_query(insert_query) + else: + logger.info(f"Sub-table {sub_table_fixed_name} already exists. Skipping creation.") + + # Create triggers for the sub-table + create_trigger_for_sub_table(sub_table_name, sub_geometry_transform, sub_sql_filter) + +def main(imposm3_config_path: str): + """ + Flujo principal: + 1. Carga la configuración. + 2. Aplica transformaciones geométricas iniciales y crea tablas derivadas con sus triggers. + """ try: - # Load the imposm3.json configuration config = load_imposm_config(imposm3_config_path) generalized_tables = config.get("generalized_tables", {}) - # Delete existing triggers - logger.info("Deleting existing triggers...") - delete_existing_triggers(generalized_tables) + ## Delete tables + delete_sub_tables(generalized_tables) - # Apply initial geometry transformations - logger.info("Starting initial geometry transformations...") + # Aplicar transformaciones iniciales y crear tablas derivadas apply_geometry_transformations(generalized_tables) - # Recreate triggers for future transformations - logger.info("Recreating triggers for future updates...") - create_triggers(generalized_tables) - - logger.info("All transformations and triggers completed successfully.") + logger.info("Proceso completado con éxito.") except Exception as e: - logger.error(f"An error occurred during execution: {e}") + logger.error(f"Ocurrió un error durante la ejecución: {e}") raise if __name__ == "__main__": imposm3_config_path = "config/imposm3.json" main(imposm3_config_path) - \ No newline at end of file From c772955b640ef45d7a361bff06448ad06d5658e8 Mon Sep 17 00:00:00 2001 From: Rub21 Date: Wed, 18 Dec 2024 13:26:25 -0500 Subject: [PATCH 2/4] Update tegola config acoording to the imposm tables --- compose/tiler.yml | 78 +++++++++---------- .../config/layers/admin_lines.json | 27 ++----- .../tiler-server/config/config.template.toml | 2 +- .../providers/admin_boundaries_maritime.toml | 62 ++------------- .../admin_boundaries_maritime.zoom.toml | 18 +---- 5 files changed, 56 insertions(+), 131 deletions(-) diff --git a/compose/tiler.yml b/compose/tiler.yml index 800a93ea..7507905a 100644 --- a/compose/tiler.yml +++ b/compose/tiler.yml @@ -1,52 +1,52 @@ services: - db: - image: ohm-tiler-db:v1 - build: - context: ../images/tiler-db - dockerfile: Dockerfile - ports: - - "5432:5432" - # volumes: - # - ../data/tiler-db-data2:/var/lib/postgresql/data - env_file: - - ../envs/.env.tiler - restart: always - networks: - - tiler_network - - imposm: - image: rub21/tiler-imposm:v16 - build: - context: ../images/tiler-imposm - dockerfile: Dockerfile - volumes: - # - ../data/tiler-imposm-data2:/mnt/data - - ../images/tiler-imposm:/osm - command: - - sh - - -c - - "./start.sh" - env_file: - - ../envs/.env.tiler - networks: - - tiler_network - - # tiler: - # image: ohm-tiler-server:v1 + # db: + # image: ohm-tiler-db:v1 # build: - # context: ../images/tiler-server + # context: ../images/tiler-db # dockerfile: Dockerfile - # volumes: - # - ../data/tiler-server-data:/mnt/data - # - ../images/tiler-server:/app # ports: - # - "9090:9090" + # - "5432:5432" + # # volumes: + # # - ../data/tiler-db-data2:/var/lib/postgresql/data # env_file: # - ../envs/.env.tiler # restart: always # networks: # - tiler_network + # imposm: + # image: rub21/tiler-imposm:v16 + # build: + # context: ../images/tiler-imposm + # dockerfile: Dockerfile + # volumes: + # # - ../data/tiler-imposm-data2:/mnt/data + # - ../images/tiler-imposm:/osm + # command: + # - sh + # - -c + # - "./start.sh" + # env_file: + # - ../envs/.env.tiler + # networks: + # - tiler_network + + tiler: + image: ohm-tiler-server:v1 + build: + context: ../images/tiler-server + dockerfile: Dockerfile + volumes: + - ../data/tiler-server-data:/mnt/data + - ../images/tiler-server:/app + ports: + - "9090:9090" + env_file: + - ../envs/.env.tiler + restart: always + networks: + - tiler_network + networks: tiler_network: driver: bridge \ No newline at end of file diff --git a/images/tiler-imposm/config/layers/admin_lines.json b/images/tiler-imposm/config/layers/admin_lines.json index 294f3457..73f33bee 100644 --- a/images/tiler-imposm/config/layers/admin_lines.json +++ b/images/tiler-imposm/config/layers/admin_lines.json @@ -8,33 +8,18 @@ ] }, "generalized_tables": { - "admin_lines_z0_2": { - "source": "admin_lines", - "sql_filter": "maritime='yes'", - "tolerance": 5000 - }, - "admin_lines_z3_5": { - "source": "admin_lines", + "admin_lines_z0_5": { + "source": "admin_lines_z6_9", "sql_filter": "maritime='yes'", "tolerance": 2000 }, - "admin_lines_z6_7": { - "source": "admin_lines", - "sql_filter": "maritime='yes'", - "tolerance": 1000 - }, - "admin_lines_z8_9": { - "source": "admin_lines", + "admin_lines_z6_9": { + "source": "admin_lines_z10_15", "sql_filter": "maritime='yes'", "tolerance": 500 }, - "admin_lines_z10_12": { - "source": "admin_lines", - "sql_filter": "maritime='yes'", - "tolerance": 100 - }, - "admin_lines_z13_15": { - "source": "admin_lines", + "admin_lines_z10_15": { + "source": "admin_lines_z16_20", "sql_filter": "maritime='yes'", "tolerance": 10 }, diff --git a/images/tiler-server/config/config.template.toml b/images/tiler-server/config/config.template.toml index 3a6d39d0..6e8812ec 100644 --- a/images/tiler-server/config/config.template.toml +++ b/images/tiler-server/config/config.template.toml @@ -98,7 +98,7 @@ max_connections = 50 [[maps]] name = "osm" attribution = "OpenHistoricalMap" -center = [-74.275329586789, -12.153492567373, 8.0] +center = [-74.275329586789, -12.153492567373, 1.0] ###### Land [['providers/land.zoom.toml']] diff --git a/images/tiler-server/config/providers/admin_boundaries_maritime.toml b/images/tiler-server/config/providers/admin_boundaries_maritime.toml index 02e3bc86..5de4577a 100644 --- a/images/tiler-server/config/providers/admin_boundaries_maritime.toml +++ b/images/tiler-server/config/providers/admin_boundaries_maritime.toml @@ -1,5 +1,5 @@ [[providers.layers]] -name = "admin_lines_z0_2" +name = "admin_lines_z0_5" geometry_fieldname = "geometry" geometry_type = "linestring" id_fieldname = "osm_id" @@ -17,13 +17,13 @@ SELECT isodatetodecimaldate(pad_date(end_date, 'end'), FALSE) AS end_decdate, {{LENGUAGES}} FROM - osm_admin_lines_z0_2 + osm_admin_lines_z0_5 WHERE geometry && !BBOX! """ [[providers.layers]] -name = "admin_lines_z3_5" +name = "admin_lines_z6_9" geometry_fieldname = "geometry" geometry_type = "linestring" id_fieldname = "osm_id" @@ -41,13 +41,13 @@ SELECT isodatetodecimaldate(pad_date(end_date, 'end'), FALSE) AS end_decdate, {{LENGUAGES}} FROM - osm_admin_lines_z3_5 + osm_admin_lines_z6_9 WHERE geometry && !BBOX! """ [[providers.layers]] -name = "admin_lines_z6_7" +name = "admin_lines_z10_15" geometry_fieldname = "geometry" geometry_type = "linestring" id_fieldname = "osm_id" @@ -65,55 +65,7 @@ SELECT isodatetodecimaldate(pad_date(end_date, 'end'), FALSE) AS end_decdate, {{LENGUAGES}} FROM - osm_admin_lines_z6_7 + osm_admin_lines_z10_15 WHERE geometry && !BBOX! -""" - -[[providers.layers]] -name = "admin_lines_z8_9" -geometry_fieldname = "geometry" -geometry_type = "linestring" -id_fieldname = "osm_id" -sql = """ -SELECT - ST_AsMVTGeom(geometry, !BBOX!) AS geometry, - osm_id, - admin_level, - name, - type, - maritime, - start_date, - end_date, - isodatetodecimaldate(pad_date(start_date, 'start'), FALSE) AS start_decdate, - isodatetodecimaldate(pad_date(end_date, 'end'), FALSE) AS end_decdate, - {{LENGUAGES}} -FROM - osm_admin_lines_z8_9 -WHERE - geometry && !BBOX! -""" - -[[providers.layers]] -name = "admin_lines_z10_12" -geometry_fieldname = "geometry" -geometry_type = "linestring" -id_fieldname = "osm_id" -sql = """ -SELECT - ST_AsMVTGeom(geometry, !BBOX!) AS geometry, - osm_id, - admin_level, - name, - type, - maritime, - start_date, - end_date, - isodatetodecimaldate(pad_date(start_date, 'start'), FALSE) AS start_decdate, - isodatetodecimaldate(pad_date(end_date, 'end'), FALSE) AS end_decdate, - {{LENGUAGES}} -FROM - osm_admin_lines_z10_12 -WHERE - geometry && !BBOX! -""" +""" \ No newline at end of file diff --git a/images/tiler-server/config/providers/admin_boundaries_maritime.zoom.toml b/images/tiler-server/config/providers/admin_boundaries_maritime.zoom.toml index a6de8555..fbc69373 100644 --- a/images/tiler-server/config/providers/admin_boundaries_maritime.zoom.toml +++ b/images/tiler-server/config/providers/admin_boundaries_maritime.zoom.toml @@ -1,29 +1,17 @@ [[maps.layers]] name = "land_ohm_maritime" -provider_layer = "osm.admin_lines_z0_2" +provider_layer = "osm.admin_lines_z0_5" min_zoom = 0 -max_zoom = 2 - -[[maps.layers]] -name = "land_ohm_maritime" -provider_layer = "osm.admin_lines_z3_5" -min_zoom = 3 max_zoom = 5 [[maps.layers]] name = "land_ohm_maritime" -provider_layer = "osm.admin_lines_z6_7" +provider_layer = "osm.admin_lines_z6_9" min_zoom = 6 -max_zoom = 7 - -[[maps.layers]] -name = "land_ohm_maritime" -provider_layer = "osm.admin_lines_z8_9" -min_zoom = 8 max_zoom = 9 [[maps.layers]] name = "land_ohm_maritime" -provider_layer = "osm.admin_lines_z10_12" +provider_layer = "osm.admin_lines_z10_15" min_zoom = 10 max_zoom = 12 From 628dd44f9de9d3d8d7ced6ebc00ba101b256dc37 Mon Sep 17 00:00:00 2001 From: Rub21 Date: Thu, 19 Dec 2024 10:12:53 -0500 Subject: [PATCH 3/4] Update db fot tiler staging --- values.staging.template.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/values.staging.template.yaml b/values.staging.template.yaml index 42e67a0a..96b5676e 100644 --- a/values.staging.template.yaml +++ b/values.staging.template.yaml @@ -388,7 +388,7 @@ osm-seed: enabled: true env: POSTGRES_HOST: {{STAGING_TILER_DB_HOST}} - POSTGRES_DB: tiler_osm_prod_v2 # Kuberntes existing db called, tiler-osm + POSTGRES_DB: tiler_osm_prod_v4 # Kuberntes existing db called, tiler-osm POSTGRES_USER: postgres POSTGRES_PASSWORD: {{STAGING_TILER_DB_PASSWORD}} POSTGRES_PORT: 5432 @@ -486,7 +486,7 @@ osm-seed: TILER_IMPORT_PBF_URL: https://s3.amazonaws.com/planet.openhistoricalmap.org/planet/planet-241214_1202.osm.pbf REPLICATION_URL: http://s3.amazonaws.com/planet.openhistoricalmap.org/replication/minute/ SEQUENCE_NUMBER: '1677612' - OVERWRITE_STATE: true + OVERWRITE_STATE: false UPLOAD_EXPIRED_FILES: true IMPORT_NATURAL_EARTH: true IMPORT_OSM_LAND: true From 7fa9bfd5b9f976f5ccc34ed4a5f24d7169e49a24 Mon Sep 17 00:00:00 2001 From: Rub21 Date: Thu, 19 Dec 2024 10:32:06 -0500 Subject: [PATCH 4/4] Update comments for tables processor script --- images/tiler-imposm/update_tables.py | 36 ++++++++++++++-------------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/images/tiler-imposm/update_tables.py b/images/tiler-imposm/update_tables.py index 936fe287..47a4d867 100644 --- a/images/tiler-imposm/update_tables.py +++ b/images/tiler-imposm/update_tables.py @@ -4,14 +4,14 @@ import subprocess import time -# Configuración del logger +# Logger configuration logging.basicConfig( level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s" ) logger = logging.getLogger(__name__) -# Cargar configuración DB desde variables de entorno +# Load DB configuration from environment variables DB_CONFIG = { "dbname": os.getenv("POSTGRES_DB"), "user": os.getenv("POSTGRES_USER"), @@ -23,22 +23,22 @@ REQUIRED_ENV_VARS = ["POSTGRES_DB", "POSTGRES_USER", "POSTGRES_PASSWORD", "POSTGRES_HOST"] for var in REQUIRED_ENV_VARS: if not os.getenv(var): - logger.error(f"La variable de entorno {var} no está definida. Saliendo.") - raise EnvironmentError(f"La variable de entorno {var} no está definida.") + logger.error(f"The environment variable {var} is not defined. Exiting.") + raise EnvironmentError(f"The environment variable {var} is not defined.") PSQL_CONN = f"postgresql://{DB_CONFIG['user']}:{DB_CONFIG['password']}@{DB_CONFIG['host']}:{DB_CONFIG['port']}/{DB_CONFIG['dbname']}" def load_imposm_config(filepath: str) -> dict: """ - Carga la configuración desde un archivo JSON. + Load the configuration from a JSON file. """ - logger.info(f"Cargando configuración desde {filepath}") + logger.info(f"Loading configuration from {filepath}") with open(filepath, "r") as f: return json.load(f) def execute_psql_query(query: str): """ - Ejecuta una consulta SQL usando psql. + Execute an SQL query using psql. """ try: result = subprocess.run( @@ -47,11 +47,11 @@ def execute_psql_query(query: str): capture_output=True ) if result.returncode != 0: - logger.error(f"Error al ejecutar la consulta: {result.stderr.strip()}") + logger.error(f"Error executing the query: {result.stderr.strip()}") else: - logger.info(f"Consulta ejecutada con éxito:\n{result.stdout.strip()}") + logger.info(f"Query executed successfully:\n{result.stdout.strip()}") except Exception as e: - logger.error(f"Error ejecutando la consulta con psql: {e}") + logger.error(f"Error executing the query with psql: {e}") def delete_sub_tables(generalized_tables: dict): """ @@ -90,7 +90,7 @@ def delete_sub_tables(generalized_tables: dict): def table_exists(table_name: str) -> bool: """ - Verifica si una tabla existe en la base de datos. + Checks if a table exists in the database. """ query = f"SELECT to_regclass('public.{table_name}');" result = subprocess.run( @@ -269,9 +269,9 @@ def apply_geometry_transformations(generalized_tables: dict): def main(imposm3_config_path: str): """ - Flujo principal: - 1. Carga la configuración. - 2. Aplica transformaciones geométricas iniciales y crea tablas derivadas con sus triggers. + Main flow: + 1. Load the configuration. + 2. Apply initial geometric transformations and create derived tables with their triggers. """ try: config = load_imposm_config(imposm3_config_path) @@ -280,14 +280,14 @@ def main(imposm3_config_path: str): ## Delete tables delete_sub_tables(generalized_tables) - # Aplicar transformaciones iniciales y crear tablas derivadas + # Apply initial transformations and create derived tables apply_geometry_transformations(generalized_tables) - logger.info("Proceso completado con éxito.") + logger.info("Process completed successfully.") except Exception as e: - logger.error(f"Ocurrió un error durante la ejecución: {e}") + logger.error(f"An error occurred during execution: {e}") raise if __name__ == "__main__": imposm3_config_path = "config/imposm3.json" - main(imposm3_config_path) + main(imposm3_config_path) \ No newline at end of file