From 94af5259efb69fa392a66ff7ef1bffe44224768d Mon Sep 17 00:00:00 2001 From: Biel Stela Date: Mon, 20 Nov 2023 11:57:37 +0100 Subject: [PATCH] Drop contextuals that don't have a url (are raster tiles) and are not linked in h3data --- data/h3_data_importer/delete_h3_tables.py | 43 +++++++++++++++-------- 1 file changed, 29 insertions(+), 14 deletions(-) diff --git a/data/h3_data_importer/delete_h3_tables.py b/data/h3_data_importer/delete_h3_tables.py index df71d0c5e..c72f9adfb 100644 --- a/data/h3_data_importer/delete_h3_tables.py +++ b/data/h3_data_importer/delete_h3_tables.py @@ -10,8 +10,9 @@ @click.command() +@click.option("--drop-contextuals", is_flag=True) @click.option("--dry-run", is_flag=True) -def main(dry_run: bool): +def main(drop_contextuals: bool, dry_run: bool): with psycopg.connect(get_connection_info()) as conn: with conn.cursor() as cursor: # find all the tables that start with h3_grid* @@ -25,22 +26,36 @@ def main(dry_run: bool): ) tables_to_drop = cursor.fetchall() if tables_to_drop: - for table in tables_to_drop: - if not dry_run: + if not dry_run: + for table in tables_to_drop: cursor.execute(f"DROP TABLE {table[0]}") - log.info(f"Tables {', '.join(table[0] for table in tables_to_drop)} were deleted") + log.info(f"Deleted tables {', '.join(table[0] for table in tables_to_drop)}") + else: + log.info("All tables are linked properly to h3_data. Nothing to delete.") + if drop_contextuals: + # Contextual layer entries with a tileUrl defined are rasters that don't have a h3 dataset related. + # We can't know if they are used or not, so we don't delete them. cursor.execute( - """ - SELECT "contextualLayerId" FROM h3_data - WHERE "h3tableName" in (%s) - """, - (tables_to_drop,), + """SELECT contextual_layer.id + FROM contextual_layer + LEFT JOIN h3_data ON contextual_layer.id = h3_data."contextualLayerId" + WHERE h3_data."contextualLayerId" IS NULL AND contextual_layer."tilerUrl" IS NULL; + """ ) - tables_deleted_with_contextuals = cursor.fetchall() - print(tables_deleted_with_contextuals) - # TODO: Find some logic to delete the contextual layers safely if they are not used anymore. - else: - log.info("No tables to delete") + contextuals_to_drop = cursor.fetchall() + if contextuals_to_drop: + if dry_run: + print(contextuals_to_drop) + return + cursor.execute( + """DELETE FROM contextual_layer + WHERE id = ANY(%s); + """, + (list(ctx[0] for ctx in contextuals_to_drop),), + ) + log.info(f"Deleted contextual layers {', '.join(str(ctx[0]) for ctx in contextuals_to_drop)}") + else: + log.info("All contextual layers are linked properly to h3_data. Nothing to delete.") if __name__ == "__main__":