diff --git a/.github/workflows/chartpress.yaml b/.github/workflows/chartpress.yaml index 81db24fd..edce8666 100644 --- a/.github/workflows/chartpress.yaml +++ b/.github/workflows/chartpress.yaml @@ -5,7 +5,7 @@ on: - 'main' - 'staging' - 'development' - - 'imposm3' + - 'imposm/routes' jobs: build: runs-on: ubuntu-20.04 diff --git a/compose/tiler.yml b/compose/tiler.yml index b130c4a0..b96d1e08 100644 --- a/compose/tiler.yml +++ b/compose/tiler.yml @@ -6,23 +6,23 @@ services: dockerfile: Dockerfile ports: - "5432:5432" - # volumes: - # - ../data/tiler-db-data:/var/lib/postgresql/data + volumes: + - ../data/tiler-db-data:/var/lib/postgresql/data env_file: - ../envs/.env.tiler restart: always imposm: - image: ohm-tiler-imposm:v1 + image: rub21/tiler-imposm:v15 build: context: ../images/tiler-imposm dockerfile: Dockerfile - # volumes: - # - ../data/tiler-imposm-data:/mnt/data - # - ../images/tiler-imposm:/app - # command: - # - sh - # - -c - # - "sleep 60 && ./start.sh" + volumes: + - ../data/tiler-imposm-data:/mnt/data + - ../images/tiler-imposm:/app + command: + - sh + - -c + - "sleep 60 && ./start.sh" env_file: - ../envs/.env.tiler tiler: @@ -31,7 +31,7 @@ services: context: ../images/tiler-server dockerfile: Dockerfile volumes: - # - ../data/tiler-server-data:/mnt/data + - ../data/tiler-server-data:/mnt/data - ../images/tiler-server:/app ports: - "9090:9090" @@ -44,6 +44,6 @@ services: context: ../images/tiler-cache dockerfile: Dockerfile volumes: - # - ../data/tiler-cache-data:/mnt/data + - ../data/tiler-cache-data:/mnt/data - ../images/tiler-cache:/app diff --git a/images/tiler-imposm/build_imposm3_config.py b/images/tiler-imposm/build_imposm3_config.py index f68c34aa..11029037 100644 --- a/images/tiler-imposm/build_imposm3_config.py +++ b/images/tiler-imposm/build_imposm3_config.py @@ -1,73 +1,88 @@ import json import os +import logging + +# Configure logging +logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s") +logger = logging.getLogger(__name__) def load_json(file_path): """Load a JSON file.""" - with open(file_path, 'r', encoding='utf-8') as file: - return json.load(file) + try: + with open(file_path, 'r', encoding='utf-8') as file: + return json.load(file) + except FileNotFoundError: + logger.error(f"File not found: {file_path}") + raise + except json.JSONDecodeError as e: + logger.error(f"Error parsing JSON in file {file_path}: {e}") + raise def merge_configs(template, configs): """Merge multiple JSON configs into the template.""" merged = template.copy() - for config in configs: if 'generalized_tables' in config: merged['generalized_tables'].update(config['generalized_tables']) - - for config in configs: if 'tables' in config: merged['tables'].update(config['tables']) - return merged def main(folder_path, template_path, output_path): """Main function to merge JSON configs.""" + logger.info("Loading template configuration...") template = load_json(template_path) - + import_layers = os.getenv("IMPOSM3_IMPORT_LAYERS", "all").strip() - + logger.info(f"IMPOSM3_IMPORT_LAYERS: {import_layers}") + configs = [] - if "all" in import_layers: - print("Importing all layer files.") + if "all" in import_layers: + logger.info("Importing all layer files.") # Import all JSON files in the folder json_files = [f for f in os.listdir(folder_path) if f.endswith('.json')] for json_file in json_files: file_path = os.path.join(folder_path, json_file) try: - print(f"Importing {file_path}") + logger.info(f"Importing {file_path}") configs.append(load_json(file_path)) - except json.JSONDecodeError as e: - print(f"Error reading {file_path}: {e}") + except Exception as e: + logger.error(f"Error reading {file_path}: {e}") else: # Import only specified layers layer_names = [layer.strip() for layer in import_layers.split(",") if layer.strip()] if not layer_names: - print("No layers specified in IMPOSM3_IMPORT_LAYERS. Exiting.") + logger.error("No layers specified in IMPOSM3_IMPORT_LAYERS. Exiting.") return - + for layer_name in layer_names: file_path = os.path.join(folder_path, f"{layer_name}.json") if os.path.exists(file_path): try: - print(f"Importing {file_path}") + logger.info(f"Importing {file_path}") configs.append(load_json(file_path)) - except json.JSONDecodeError as e: - print(f"Error reading {file_path}: {e}") + except Exception as e: + logger.error(f"Error reading {file_path}: {e}") else: - print(f"Layer config file {file_path} not found. Skipping.") + logger.warning(f"Layer config file {file_path} not found. Skipping.") if not configs: - print("No valid layer configurations found. Exiting.") + logger.error("No valid layer configurations found. Exiting.") return - + + logger.info("Merging configurations...") merged_config = merge_configs(template, configs) - with open(output_path, 'w', encoding='utf-8') as output_file: - json.dump(merged_config, output_file, indent=2) - - print(f"Merged configuration saved to {output_path}") + try: + with open(output_path, 'w', encoding='utf-8') as output_file: + json.dump(merged_config, output_file, indent=2) + logger.info(f"Merged configuration saved to {output_path}") + except Exception as e: + logger.error(f"Error writing merged configuration to {output_path}: {e}") + raise if __name__ == "__main__": folder_path = "./config/layers" template_path = "./config/imposm3.template.json" output_path = "./config/imposm3.json" main(folder_path, template_path, output_path) + \ No newline at end of file diff --git a/images/tiler-imposm/config/layers/admin_areas.json b/images/tiler-imposm/config/layers/admin_areas.json index 398655f0..50cf8bb9 100644 --- a/images/tiler-imposm/config/layers/admin_areas.json +++ b/images/tiler-imposm/config/layers/admin_areas.json @@ -17,7 +17,6 @@ }, "admin_boundaries_centroid_z0_2": { "source": "admin_areas", - "tolerance": 5000, "geometry_transform": "(ST_MaximumInscribedCircle(geometry)).center", "geometry_transform_types": "GeometryType(geometry) IN ('POLYGON', 'MULTIPOLYGON')", "sql_filter": "admin_level IN (1,2) AND name IS NOT NULL AND name <> ''" @@ -31,7 +30,6 @@ }, "admin_boundaries_centroid_z3_5": { "source": "admin_areas", - "tolerance": 1000, "geometry_transform": "(ST_MaximumInscribedCircle(geometry)).center", "geometry_transform_types": "GeometryType(geometry) IN ('POLYGON', 'MULTIPOLYGON')", "sql_filter": "admin_level IN (1,2,3,4) AND name IS NOT NULL AND name <> ''" @@ -45,7 +43,6 @@ }, "admin_boundaries_centroid_z6_7": { "source": "admin_areas", - "tolerance": 500, "geometry_transform": "(ST_MaximumInscribedCircle(geometry)).center", "geometry_transform_types": "GeometryType(geometry) IN ('POLYGON', 'MULTIPOLYGON')", "sql_filter": "admin_level IN (1,2,3,4,5,6) AND name IS NOT NULL AND name <> ''" diff --git a/images/tiler-imposm/config/layers/relation_members_routes.json b/images/tiler-imposm/config/layers/relation_members_routes.json index 0608baba..9cd93243 100644 --- a/images/tiler-imposm/config/layers/relation_members_routes.json +++ b/images/tiler-imposm/config/layers/relation_members_routes.json @@ -7,27 +7,7 @@ "source:datetime" ] }, - "generalized_tables": { - "route_lines_z5_7": { - "source": "relation_members_routes", - "sql_filter": "type IS NOT NULL AND route IS NOT NULL", - "tolerance": 500 - }, - "route_lines_z8_9": { - "source": "relation_members_routes", - "sql_filter": "type IS NOT NULL AND route IS NOT NULL", - "tolerance": 200 - }, - "route_lines_z10_12": { - "source": "relation_members_routes", - "sql_filter": "type IS NOT NULL AND route IS NOT NULL", - "tolerance": 100 - }, - "route_lines_z12_20": { - "source": "relation_members_routes", - "sql_filter": "type IS NOT NULL AND route IS NOT NULL" - } - }, + "generalized_tables": {}, "tables": { "relation_members_routes": { "type": "relation_member", @@ -47,11 +27,6 @@ "name": "type", "key": null }, - { - "type": "integer", - "name": "admin_level", - "key": "admin_level" - }, { "key": "route", "name": "route", @@ -136,12 +111,6 @@ "name": "member", "type": "member_id" }, - { - "name": "me_maritime", - "type": "string", - "key": "maritime", - "from_member": true - }, { "type": "hstore_tags", "name": "me_tags", diff --git a/images/tiler-imposm/config/postgis_post_import.sql b/images/tiler-imposm/config/postgis_post_import.sql deleted file mode 100644 index c11d7d84..00000000 --- a/images/tiler-imposm/config/postgis_post_import.sql +++ /dev/null @@ -1,71 +0,0 @@ --- Create a function that updates the osm_admin_areas if the polygon has a label in the relations -CREATE OR REPLACE FUNCTION update_admin_areas() -RETURNS void AS $$ -BEGIN - UPDATE osm_admin_areas - SET has_label = 1 - WHERE osm_id IN ( - SELECT osm_id - FROM osm_relation_members - WHERE role = 'label' - ); -END; -$$ LANGUAGE plpgsql; - --- Execute the function -SELECT update_admin_areas(); - - --- Create a function and trigger that will update every time a new admin area is inserted or updated in osm_admin_areas table - -CREATE OR REPLACE FUNCTION update_has_label_row() -RETURNS TRIGGER AS $$ -BEGIN - IF EXISTS ( - SELECT 1 - FROM osm_relation_members - WHERE osm_id = NEW.osm_id AND role = 'label' - ) THEN - NEW.has_label := 1; - ELSE - NEW.has_label := 0; - END IF; - RETURN NEW; -END; -$$ LANGUAGE plpgsql; - - -CREATE TRIGGER trigger_update_has_label -BEFORE INSERT OR UPDATE ON osm_admin_areas -FOR EACH ROW -EXECUTE FUNCTION update_has_label_row(); - --- Create a function to validate dates - -CREATE OR REPLACE FUNCTION is_date_valid(date_text text) RETURNS boolean AS $$ -DECLARE - tmp_date date; -BEGIN - -- Try the format YYYY-MM-DD - BEGIN - tmp_date := to_date(date_text, 'YYYY-MM-DD'); - RETURN TRUE; - EXCEPTION WHEN others THEN - END; - - -- Try the format YYYY-MM - BEGIN - tmp_date := to_date(date_text || '-01', 'YYYY-MM-DD'); - RETURN TRUE; - EXCEPTION WHEN others THEN - END; - - -- Try the format YYYY - BEGIN - tmp_date := to_date(date_text || '-01-01', 'YYYY-MM-DD'); - RETURN TRUE; - EXCEPTION WHEN others THEN - END; - RETURN FALSE; -END; -$$ LANGUAGE plpgsql; \ No newline at end of file diff --git a/images/tiler-imposm/queries/admin_boundaries_centroids.sql b/images/tiler-imposm/queries/admin_boundaries_centroids.sql new file mode 100644 index 00000000..c0e98cd4 --- /dev/null +++ b/images/tiler-imposm/queries/admin_boundaries_centroids.sql @@ -0,0 +1,86 @@ +-- Create a function that updates the osm_admin_boundaries_centroid_* if relations have labels or not +CREATE OR REPLACE FUNCTION update_admin_boundaries_centroids() +RETURNS void AS $$ +DECLARE + table_name text; +BEGIN + -- Loop through all table names that match the pattern + FOR table_name IN + SELECT t.table_name + FROM information_schema.tables AS t + WHERE t.table_name LIKE 'osm_admin_boundaries_centroid%' + LOOP + -- Dynamically execute the update query for each table + BEGIN + EXECUTE format(' + UPDATE %I + SET has_label = 1 + WHERE osm_id IN ( + SELECT osm_id + FROM osm_relation_members + WHERE role = ''label'' + );', table_name); + EXCEPTION WHEN OTHERS THEN + RAISE NOTICE ''Error updating table %: %'', table_name, SQLERRM; + END; + END LOOP; + + -- Log completion + RAISE NOTICE ''Update completed for all matching tables.''; +END; +$$ LANGUAGE plpgsql; + +-- Create a function that updates the osm_admin_boundaries_centroid_* if relations have labels or not +CREATE OR REPLACE FUNCTION create_update_has_label_triggers(pattern text) +RETURNS void AS $$ +DECLARE + table_name text; +BEGIN + -- Loop through all tables matching the provided pattern + FOR table_name IN + SELECT t.table_name + FROM information_schema.tables AS t + WHERE t.table_name LIKE pattern + LOOP + -- Create a dynamic function for each table + EXECUTE format( + 'CREATE OR REPLACE FUNCTION %I_update_has_label_row() + RETURNS TRIGGER AS $trigger_body$ + BEGIN + IF EXISTS ( + SELECT 1 + FROM osm_relation_members + WHERE osm_id = NEW.osm_id AND role = ''label'' + ) THEN + NEW.has_label := 1; + ELSE + NEW.has_label := 0; + END IF; + RETURN NEW; + END; + $trigger_body$ LANGUAGE plpgsql;', + table_name + ); + + -- Attach a trigger to each table + EXECUTE format( + 'CREATE TRIGGER %I_trigger_update_has_label + BEFORE INSERT OR UPDATE ON %I + FOR EACH ROW + EXECUTE FUNCTION %I_update_has_label_row();', + table_name, -- Trigger name + table_name, -- Table name + table_name -- Function name + ); + END LOOP; + + RAISE NOTICE 'Triggers and functions created for tables matching %.', pattern; +END; +$$ LANGUAGE plpgsql; + + +-- Execute the update function when the import is done +SELECT update_admin_boundaries_centroids(); + +-- Execute the function to create triggers for tables matching the pattern +SELECT create_update_has_label_triggers('osm_admin_boundaries_centroid%'); diff --git a/images/tiler-imposm/queries/osm_relation_menbers_routes_table.sql b/images/tiler-imposm/queries/osm_relation_menbers_routes_table.sql new file mode 100644 index 00000000..f8669690 --- /dev/null +++ b/images/tiler-imposm/queries/osm_relation_menbers_routes_table.sql @@ -0,0 +1,54 @@ +DO $$ +BEGIN + -- Check if the table osm_relation_members_routes exists + IF EXISTS ( + SELECT 1 + FROM information_schema.tables + WHERE table_schema = 'public' + AND table_name = 'osm_relation_members_routes' + ) THEN + -- Drop the merged table if it exists + DROP TABLE IF EXISTS osm_relation_members_routes_merged; + + -- Create the merged table + CREATE TABLE osm_relation_members_routes_merged AS + SELECT + osm_id, + name, + type, + route, + ref, + network, + direction, + operator, + state, + symbol, + distance, + roundtrip, + interval, + duration, + tourism, + start_date, + end_date, + tags, + ST_Union(geometry) AS geometry + FROM + osm_relation_members_routes + GROUP BY + osm_id, name, type, route, ref, network, direction, operator, + state, symbol, distance, roundtrip, interval, duration, tourism, + start_date, end_date, tags; + + -- Add primary key + ALTER TABLE osm_relation_members_routes_merged ADD PRIMARY KEY (osm_id); + + -- Create geometry index + CREATE INDEX idx_osm_relation_members_routes_merged_geometry + ON osm_relation_members_routes_merged + USING GIST (geometry); + + RAISE NOTICE 'Merged table created successfully.'; + ELSE + RAISE NOTICE 'Table osm_relation_members_routes does not exist. Skipping execution.'; + END IF; +END $$; \ No newline at end of file diff --git a/images/tiler-imposm/queries/osm_relation_menbers_routes_trigger.sql b/images/tiler-imposm/queries/osm_relation_menbers_routes_trigger.sql new file mode 100644 index 00000000..d8b26c1e --- /dev/null +++ b/images/tiler-imposm/queries/osm_relation_menbers_routes_trigger.sql @@ -0,0 +1,107 @@ +DO $$ +BEGIN + -- Check if the table exists + IF EXISTS ( + SELECT 1 + FROM information_schema.tables + WHERE table_schema = 'public' + AND table_name = 'osm_relation_members_routes' + ) THEN + -- Create the trigger function + CREATE OR REPLACE FUNCTION update_osm_relation_members_routes_merged() + RETURNS TRIGGER AS $$ + BEGIN + -- Handle DELETE operation + IF (TG_OP = 'DELETE') THEN + DELETE FROM osm_relation_members_routes_merged + WHERE osm_id = OLD.osm_id; + RETURN OLD; + END IF; + + -- Handle INSERT and UPDATE operations + IF (TG_OP = 'INSERT' OR TG_OP = 'UPDATE') THEN + -- Merge geometry and update the aggregated row + INSERT INTO osm_relation_members_routes_merged ( + osm_id, + name, + type, + route, + ref, + network, + direction, + operator, + state, + symbol, + distance, + roundtrip, + interval, + duration, + tourism, + start_date, + end_date, + tags, + geometry + ) + SELECT + osm_id, + name, + type, + route, + ref, + network, + direction, + operator, + state, + symbol, + distance, + roundtrip, + interval, + duration, + tourism, + start_date, + end_date, + tags, + ST_Union(geometry) AS geometry + FROM osm_relation_members_routes + WHERE osm_id = NEW.osm_id + GROUP BY + osm_id, name, type, route, ref, network, direction, operator, + state, symbol, distance, roundtrip, interval, duration, tourism, + start_date, end_date, tags + ON CONFLICT (osm_id) + DO UPDATE SET + name = EXCLUDED.name, + type = EXCLUDED.type, + route = EXCLUDED.route, + ref = EXCLUDED.ref, + network = EXCLUDED.network, + direction = EXCLUDED.direction, + operator = EXCLUDED.operator, + state = EXCLUDED.state, + symbol = EXCLUDED.symbol, + distance = EXCLUDED.distance, + roundtrip = EXCLUDED.roundtrip, + interval = EXCLUDED.interval, + duration = EXCLUDED.duration, + tourism = EXCLUDED.tourism, + start_date = EXCLUDED.start_date, + end_date = EXCLUDED.end_date, + tags = EXCLUDED.tags, + geometry = EXCLUDED.geometry; + RETURN NEW; + END IF; + END; + $$ LANGUAGE plpgsql; + + -- Attach the trigger to the table + CREATE TRIGGER osm_relation_members_routes_trigger + AFTER INSERT OR UPDATE OR DELETE ON osm_relation_members_routes + FOR EACH ROW + EXECUTE FUNCTION update_osm_relation_members_routes_merged(); + + ELSE + -- Log or handle the case where the table doesn't exist + RAISE NOTICE 'Table osm_relation_members_routes does not exist. Skipping trigger creation.'; + END IF; +END; +$$; \ No newline at end of file diff --git a/images/tiler-imposm/config/postgis_helpers.sql b/images/tiler-imposm/queries/postgis_helpers.sql similarity index 100% rename from images/tiler-imposm/config/postgis_helpers.sql rename to images/tiler-imposm/queries/postgis_helpers.sql diff --git a/images/tiler-imposm/config/postgis_index.sql b/images/tiler-imposm/queries/postgis_index.sql similarity index 100% rename from images/tiler-imposm/config/postgis_index.sql rename to images/tiler-imposm/queries/postgis_index.sql diff --git a/images/tiler-imposm/queries/postgis_post_import.sql b/images/tiler-imposm/queries/postgis_post_import.sql new file mode 100644 index 00000000..02198f51 --- /dev/null +++ b/images/tiler-imposm/queries/postgis_post_import.sql @@ -0,0 +1,30 @@ + +-- Create a function to validate dates + +CREATE OR REPLACE FUNCTION is_date_valid(date_text text) RETURNS boolean AS $$ +DECLARE + tmp_date date; +BEGIN + -- Try the format YYYY-MM-DD + BEGIN + tmp_date := to_date(date_text, 'YYYY-MM-DD'); + RETURN TRUE; + EXCEPTION WHEN others THEN + END; + + -- Try the format YYYY-MM + BEGIN + tmp_date := to_date(date_text || '-01', 'YYYY-MM-DD'); + RETURN TRUE; + EXCEPTION WHEN others THEN + END; + + -- Try the format YYYY + BEGIN + tmp_date := to_date(date_text || '-01-01', 'YYYY-MM-DD'); + RETURN TRUE; + EXCEPTION WHEN others THEN + END; + RETURN FALSE; +END; +$$ LANGUAGE plpgsql; \ No newline at end of file diff --git a/images/tiler-imposm/start.sh b/images/tiler-imposm/start.sh index c900e43a..08e24445 100755 --- a/images/tiler-imposm/start.sh +++ b/images/tiler-imposm/start.sh @@ -136,7 +136,7 @@ function updateData() { function importData() { ### Import the PBF and Natural Earth files to the DB echo "Execute the missing functions" - psql "postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@$POSTGRES_HOST/$POSTGRES_DB" -a -f config/postgis_helpers.sql + psql "postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@$POSTGRES_HOST/$POSTGRES_DB" -f queries/postgis_helpers.sql if [ "$IMPORT_NATURAL_EARTH" = "true" ]; then echo "Importing Natural Earth..." @@ -170,14 +170,19 @@ function importData() { -deployproduction # These index will help speed up tegola tile generation - # psql "postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@$POSTGRES_HOST/$POSTGRES_DB" -a -f config/postgis_index.sql - psql "postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@$POSTGRES_HOST/$POSTGRES_DB" -a -f config/postgis_post_import.sql + # psql "postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@$POSTGRES_HOST/$POSTGRES_DB" -f queries/postgis_index.sql + psql "postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@$POSTGRES_HOST/$POSTGRES_DB" -f queries/postgis_post_import.sql touch $INIT_FILE # Update tables python update_tables.py + echo "Create Table/Tigger for osm_relation_menbers_routes_merged" + # psql "postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@$POSTGRES_HOST/$POSTGRES_DB" -f queries/osm_relation_menbers_routes_table.sql + # psql "postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@$POSTGRES_HOST/$POSTGRES_DB" -f queries/osm_relation_menbers_routes_trigger.sql + psql "postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@$POSTGRES_HOST/$POSTGRES_DB" -f queries/admin_boundaries_centroids.sql + # Updata data with minute replication updateData } diff --git a/images/tiler-imposm/transform_tables.py b/images/tiler-imposm/transform_tables.py new file mode 100644 index 00000000..76527e00 --- /dev/null +++ b/images/tiler-imposm/transform_tables.py @@ -0,0 +1,195 @@ +import os +import json +import logging +import psycopg2 + +logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s") +logger = logging.getLogger(__name__) + +# Database configuration +DB_CONFIG = { + "dbname": os.getenv("POSTGRES_DB"), + "user": os.getenv("POSTGRES_USER"), + "password": os.getenv("POSTGRES_PASSWORD"), + "host": os.getenv("POSTGRES_HOST"), + "port": int(os.getenv("POSTGRES_PORT", 5432)) +} + +REQUIRED_ENV_VARS = ["POSTGRES_DB", "POSTGRES_USER", "POSTGRES_PASSWORD", "POSTGRES_HOST"] +for var in REQUIRED_ENV_VARS: + if not os.getenv(var): + logger.error(f"Environment variable {var} is not defined. Exiting.") + raise EnvironmentError(f"Environment variable {var} is not defined.") + + +def load_imposm_config(filepath): + """Load the imposm3.json configuration file.""" + logger.info(f"Loading configuration from {filepath}") + try: + with open(filepath, "r") as f: + return json.load(f) + except FileNotFoundError: + logger.error(f"Configuration file {filepath} not found.") + raise + except json.JSONDecodeError: + logger.error(f"Error parsing JSON from {filepath}.") + raise + +def get_column_data_type(cur, table_name, column_name): + """Obtener el tipo de datos de una columna.""" + cur.execute(f""" + SELECT data_type + FROM information_schema.columns + WHERE table_name = %s AND column_name = %s; + """, (table_name, column_name)) + result = cur.fetchone() + return result[0] if result else "TEXT" + +def create_transform_table_and_trigger(conn, table_name, table_info): + """Crear tabla de transformación y disparador dinámicamente.""" + transform_table_name = f"{table_name}_transform" + print("================") + print(transform_table_name) + + geometry_transform = table_info.get("geometry_transform") + print(geometry_transform) + + if not geometry_transform: + logger.warning(f"No se definió la transformación de geometría para {table_name}. Saltando.") + return + + logger.info(f"Creando tabla de transformación y disparador para {table_name}...") + + with conn.cursor() as cur: + # Obtener los nombres de las columnas dinámicamente + cur.execute(f"SELECT column_name FROM information_schema.columns WHERE table_name = %s", (table_name,)) + columns = [row[0] for row in cur.fetchall()] + + if not columns: + logger.error(f"La tabla {table_name} no tiene columnas. Saltando.") + return + + if "geometry" not in columns: + logger.error(f"La tabla {table_name} debe tener columnas 'geometry' y 'osm_id'. Saltando.") + return + + # Excluir la columna de geometría para la transformación + columns_without_geometry = [col for col in columns if col != "geometry"] + columns_select = ", ".join(columns_without_geometry) + + # Crear la tabla de transformación vacía + create_table_query = f""" + CREATE TABLE IF NOT EXISTS {transform_table_name} AS + SELECT * FROM {table_name} WHERE FALSE; + CREATE UNIQUE INDEX {transform_table_name}_pkey ON {transform_table_name}(osm_id int8_ops); + CREATE INDEX {transform_table_name}_geom ON {transform_table_name} USING GIST (geometry gist_geometry_ops_2d); + """ + + cur.execute(create_table_query) + conn.commit() + logger.info(f"Tabla de transformación {transform_table_name} creada.") + + # Copiar los datos aplicando la transformación + insert_data_query = f""" + INSERT INTO {transform_table_name} ( + {columns_select}, geometry + ) + SELECT + {columns_select}, {geometry_transform} AS geometry + FROM {table_name}; + """ + cur.execute(insert_data_query) + conn.commit() + logger.info(f"Datos copiados a {transform_table_name} aplicando la transformación.") + + # Crear la función del disparador + trigger_function_name = f"{table_name}_transform_trigger" + trigger_function_query = f""" + CREATE OR REPLACE FUNCTION {trigger_function_name}() + RETURNS TRIGGER AS $$ + BEGIN + -- Handle DELETE operation + IF TG_OP = 'DELETE' THEN + DELETE FROM {transform_table_name} + WHERE osm_id = OLD.osm_id; + RETURN OLD; + END IF; + + -- Handle INSERT and UPDATE operations + IF TG_OP = 'INSERT' OR TG_OP = 'UPDATE' THEN + -- Check if the row exists in the transform table + IF EXISTS ( + SELECT 1 + FROM {transform_table_name} + WHERE osm_id = NEW.osm_id + ) THEN + -- If it exists, update the row + UPDATE {transform_table_name} + SET + {", ".join([f"{col} = NEW.{col}" for col in columns_without_geometry])}, + geometry = {geometry_transform.replace('geometry', 'NEW.geometry')} + WHERE osm_id = NEW.osm_id; + ELSE + -- If it does not exist, insert a new row + INSERT INTO {transform_table_name} ( + {columns_select}, geometry + ) + VALUES ( + {", ".join([f"NEW.{col}" for col in columns_without_geometry])}, + {geometry_transform.replace('geometry', 'NEW.geometry')} + ); + END IF; + + RETURN NEW; + END IF; + + -- If no matching operation, raise an exception (optional for debugging) + RAISE EXCEPTION 'Unexpected trigger operation: %', TG_OP; + END; + $$ LANGUAGE plpgsql; + """ + print("================="*20) + print(trigger_function_query) + cur.execute(trigger_function_query) + conn.commit() + logger.info(f"Función del disparador {trigger_function_name} creada.") + + # Crear el disparador + trigger_name = f"{table_name}_after_insert_update_delete" + create_trigger_query = f""" + CREATE TRIGGER {trigger_name} + AFTER INSERT OR UPDATE OR DELETE ON {table_name} + FOR EACH ROW + EXECUTE FUNCTION {trigger_function_name}(); + """ + cur.execute(create_trigger_query) + conn.commit() + logger.info(f"Disparador {trigger_name} creado para la tabla {table_name}.") + + + +def main(imposm3_config_path): + """Main execution flow.""" + try: + conn = psycopg2.connect(**DB_CONFIG) + # Load the imposm3.json configuration + config = load_imposm_config(imposm3_config_path) + generalized_tables = config.get("generalized_tables", {}) + + # Create transform tables and triggers + for table_name, table_info in generalized_tables.items(): + table_name = f"osm_{table_name}" + create_transform_table_and_trigger(conn, table_name, table_info) + + logger.info("All transform tables and triggers created successfully.") + except Exception as e: + logger.error(f"An error occurred during execution: {e}") + raise + finally: + if conn: + conn.close() + + +if __name__ == "__main__": + imposm3_config_path = "./config/imposm3.json" + main(imposm3_config_path) \ No newline at end of file diff --git a/images/tiler-imposm/update_tables.py b/images/tiler-imposm/update_tables.py index ff5b43bf..b442c9eb 100644 --- a/images/tiler-imposm/update_tables.py +++ b/images/tiler-imposm/update_tables.py @@ -39,7 +39,7 @@ def load_imposm_config(filepath): def execute_psql_query(query): """Execute a query using psql and print the output.""" try: - logger.info(f"Executing query:\t{query}") + # logger.info(f"Executing query:\t{query}") result = subprocess.run( ["psql", PSQL_CONN, "-c", query], text=True, @@ -97,37 +97,42 @@ def create_triggers(generalized_tables): geometry_transform = table_info.get("geometry_transform") geometry_transform_types = table_info.get("geometry_transform_types") - # Skip if transform or types are not defined if not geometry_transform or not geometry_transform_types: - logger.warning( - f"Skipping trigger creation for {fixed_table_name}: " - "'geometry_transform' or 'geometry_transform_types' not defined." - ) + logger.warning(f"Skipping trigger creation for {fixed_table_name}.") continue - # Create the trigger function SQL + # Check if the trigger already exists + check_trigger_query = f""" + SELECT 1 + FROM pg_trigger + WHERE tgname = '{fixed_table_name}_before_insert_update'; + """ + result = subprocess.run( + ["psql", PSQL_CONN, "-c", check_trigger_query], + text=True, + capture_output=True + ) + if "1" in result.stdout: + logger.info(f"Trigger {fixed_table_name}_before_insert_update already exists. Skipping.") + continue + + # Create the trigger if it does not exist trigger_function = f""" CREATE OR REPLACE FUNCTION {fixed_table_name}_transform_trigger() RETURNS TRIGGER AS $$ BEGIN - IF {geometry_transform_types} THEN - NEW.geometry = {geometry_transform}; - END IF; + NEW.geometry = {geometry_transform.replace('geometry', 'NEW.geometry')}; RETURN NEW; END; $$ LANGUAGE plpgsql; - """ - execute_psql_query(trigger_function) - # Create the trigger SQL - trigger = f""" CREATE TRIGGER {fixed_table_name}_before_insert_update BEFORE INSERT OR UPDATE ON {fixed_table_name} FOR EACH ROW EXECUTE FUNCTION {fixed_table_name}_transform_trigger(); """ - execute_psql_query(trigger) - + execute_psql_query(trigger_function) + def main(imposm3_config_path): """Main execution flow.""" try: diff --git a/images/tiler-server/config/config.template.toml b/images/tiler-server/config/config.template.toml index 88bafd93..3a6d39d0 100644 --- a/images/tiler-server/config/config.template.toml +++ b/images/tiler-server/config/config.template.toml @@ -78,7 +78,7 @@ max_connections = 50 [['providers/buildings.toml']] [['providers/buildings.centroids.toml']] - ###### transport_associated_streets + ##### transport_associated_streets [['providers/transport_associated_streets.toml']] ###### admin_boundaries_maritime diff --git a/images/tiler-server/config/providers/admin_boundaries_centroids.toml b/images/tiler-server/config/providers/admin_boundaries_centroids.toml index a309f074..56101be2 100644 --- a/images/tiler-server/config/providers/admin_boundaries_centroids.toml +++ b/images/tiler-server/config/providers/admin_boundaries_centroids.toml @@ -18,7 +18,8 @@ SELECT FROM osm_admin_boundaries_centroid_z0_2 WHERE - geometry && !BBOX! + has_label = 0 + AND geometry && !BBOX! """ [[providers.layers]] @@ -41,7 +42,8 @@ SELECT FROM osm_admin_boundaries_centroid_z3_5 WHERE - geometry && !BBOX! + has_label = 0 + AND geometry && !BBOX! """ [[providers.layers]] @@ -64,10 +66,7 @@ SELECT FROM osm_admin_boundaries_centroid_z6_7 WHERE - admin_level IN (1,2,3,4,5,6) - AND has_label = 0 - AND name IS NOT NULL - AND name <> '' + has_label = 0 AND geometry && !BBOX! """ @@ -91,7 +90,8 @@ SELECT FROM osm_admin_boundaries_centroid_z8_9 WHERE - geometry && !BBOX! + has_label = 0 + AND geometry && !BBOX! """ [[providers.layers]] @@ -114,7 +114,8 @@ SELECT FROM osm_admin_boundaries_centroid_z10_12 WHERE - geometry && !BBOX! + has_label = 0 + AND geometry && !BBOX! """ @@ -138,7 +139,8 @@ SELECT FROM osm_admin_boundaries_centroid_z13_15 WHERE - geometry && !BBOX! + has_label = 0 + AND geometry && !BBOX! """ [[providers.layers]] @@ -161,5 +163,6 @@ SELECT FROM osm_admin_boundaries_centroid_z16_20 WHERE - geometry && !BBOX! + has_label = 0 + AND geometry && !BBOX! """ \ No newline at end of file diff --git a/images/tiler-server/config/providers/route_lines.toml b/images/tiler-server/config/providers/route_lines.toml index 2a715fcf..bfab752a 100644 --- a/images/tiler-server/config/providers/route_lines.toml +++ b/images/tiler-server/config/providers/route_lines.toml @@ -1,39 +1,31 @@ [[providers.layers]] -name = "route_lines_5_20" +name = "routes_lines" geometry_fieldname = "geometry" geometry_type = "LineString" id_fieldname = "osm_id" sql = """ SELECT - ST_AsMVTGeom(geometry,!BBOX!) AS geometry, - ABS(r.osm_id) AS osm_id, - r.type, - r.tags->'route' AS route, - r.name, - r.tags->'ref' AS ref, - r.tags->'network' AS network, - r.tags->'direction' AS direction, - r.tags->'operator' AS operator, - r.tags->'state' AS state, - r.tags->'symbol' AS symbol, - r.tags->'distance' AS distance, - r.tags->'roundtrip' AS roundtrip, - r.tags->'interval' AS interval, - r.tags->'duration' AS duration, - r.tags->'tourism' AS tourism, - r.start_date, - r.end_date, - CASE - WHEN is_date_valid(r.start_date) THEN isodatetodecimaldate(pad_date(r.start_date, 'start')) - ELSE NULL - END AS start_decdate, - CASE - WHEN is_date_valid(r.end_date) THEN isodatetodecimaldate(pad_date(r.end_date, 'end')) - ELSE NULL - END AS end_decdate, - {{LENGUAGES_RELATION}} -FROM osm_relations r -JOIN osm_relation_members m ON r.osm_id = m.osm_id -WHERE r.type = 'route' -AND geometry && !BBOX! -""" + ST_AsMVTGeom(geometry, !BBOX!) AS geometry, + ABS(osm_id) AS osm_id, + name, + type, + route, + ref, + network, + direction, + operator, + state, + symbol, + distance, + roundtrip, + interval, + duration, + tourism, + start_date, + end_date, + isodatetodecimaldate(pad_date(start_date, 'start'), FALSE) AS start_decdate, + isodatetodecimaldate(pad_date(end_date, 'end'), FALSE) AS end_decdate, + {{LENGUAGES}} +FROM osm_relation_members_routes +WHERE geometry && !BBOX! +""" \ No newline at end of file diff --git a/images/tiler-server/config/providers/route_lines.zoom.toml b/images/tiler-server/config/providers/route_lines.zoom.toml index 6fea0412..61e269bd 100644 --- a/images/tiler-server/config/providers/route_lines.zoom.toml +++ b/images/tiler-server/config/providers/route_lines.zoom.toml @@ -1,5 +1,5 @@ [[maps.layers]] name = "route_lines" -provider_layer = "osm.route_lines_5_20" +provider_layer = "osm.routes_lines" min_zoom = 5 max_zoom = 20 diff --git a/values.staging.template.yaml b/values.staging.template.yaml index 04183997..bc689148 100644 --- a/values.staging.template.yaml +++ b/values.staging.template.yaml @@ -397,7 +397,7 @@ osm-seed: mountPath: /var/lib/postgresql/data subPath: postgresql-d # In case cloudProvider: aws - AWS_ElasticBlockStore_volumeID : vol-0492574788bacbabb + AWS_ElasticBlockStore_volumeID : vol-070a59c4b4d7c0b32 AWS_ElasticBlockStore_size: 200Gi resources: enabled: false @@ -498,7 +498,7 @@ osm-seed: accessMode: ReadWriteOnce mountPath: /mnt/data # In case cloudProvider: aws - AWS_ElasticBlockStore_volumeID: vol-02af939ea87b8c5db + AWS_ElasticBlockStore_volumeID: vol-088adc6b334905d19 AWS_ElasticBlockStore_size: 50Gi resources: enabled: false