From c6f14d05d9897e1189f2d63f98d4161d367ac609 Mon Sep 17 00:00:00 2001 From: ebocher Date: Sat, 12 Oct 2024 09:44:16 +0200 Subject: [PATCH] Fix worldpop service request plus url database syntax --- .../bdtopo/AbstractBDTopoWorkflow.groovy | 50 +++++++++++-------- .../geoclimate/utils/AbstractScript.groovy | 8 +-- docs/CHANGELOG.md | 2 + .../WorkflowGeoIndicators.groovy | 9 ++-- .../geoindicators/WorkflowUtilities.groovy | 20 +++++++- .../geoclimate/osm/WorkflowOSM.groovy | 45 ++++++++++------- .../geoclimate/osm/WorflowOSMTest.groovy | 17 +++++-- .../worldpoptools/WorldPopExtractTest.groovy | 29 +++++------ 8 files changed, 113 insertions(+), 67 deletions(-) diff --git a/bdtopo/src/main/groovy/org/orbisgis/geoclimate/bdtopo/AbstractBDTopoWorkflow.groovy b/bdtopo/src/main/groovy/org/orbisgis/geoclimate/bdtopo/AbstractBDTopoWorkflow.groovy index 5992fd5a58..e0acdfea48 100644 --- a/bdtopo/src/main/groovy/org/orbisgis/geoclimate/bdtopo/AbstractBDTopoWorkflow.groovy +++ b/bdtopo/src/main/groovy/org/orbisgis/geoclimate/bdtopo/AbstractBDTopoWorkflow.groovy @@ -977,38 +977,48 @@ abstract class AbstractBDTopoWorkflow extends BDTopoUtils { //Extract and compute population indicators for the specified year //This data can be used by the grid_indicators process if (worldpop_indicators) { - def envelope = h2gis_datasource.firstRow("select st_transform(the_geom, 4326) as geom from $zone".toString()).geom.getEnvelopeInternal() - def bbox = [envelope.getMinY() as Float, envelope.getMinX() as Float, - envelope.getMaxY() as Float, envelope.getMaxX() as Float] String coverageId = "wpGlobal:ppp_2018" - String worldPopFile = WorldPopTools.Extract.extractWorldPopLayer(coverageId, bbox) - if (worldPopFile) { - String worldPopTableName = WorldPopTools.Extract.importAscGrid(h2gis_datasource, worldPopFile, srid, coverageId.replaceAll(":", "_")) - if (worldPopTableName) { - results.put("population", worldPopTableName) - building = Geoindicators.BuildingIndicators.buildingPopulation(h2gis_datasource, results.building, - worldPopTableName, ["pop"]) - if (!building) { - info "Cannot compute any population data at building level" + //Test if the coverage is available + if(WorldPopTools.Extract.isCoverageAvailable(coverageId)) { + def envelope = h2gis_datasource.firstRow("select st_transform(the_geom, 4326) as geom from $zone".toString()).geom.getEnvelopeInternal() + def bbox = [envelope.getMinY() as Float, envelope.getMinX() as Float, + envelope.getMaxY() as Float, envelope.getMaxX() as Float] + + String worldPopFile = WorldPopTools.Extract.extractWorldPopLayer(coverageId, bbox) + if (worldPopFile) { + String worldPopTableName = WorldPopTools.Extract.importAscGrid(h2gis_datasource, worldPopFile, srid, coverageId.replaceAll(":", "_")) + if (worldPopTableName) { + results.put("population", worldPopTableName) + building = Geoindicators.BuildingIndicators.buildingPopulation(h2gis_datasource, results.building, + worldPopTableName, ["pop"]) + if (!building) { + info "Cannot compute any population data at building level" + } + tablesToDrop << results.building + //Update the building table with the population data + results.put("building", building) + + } else { + info "Cannot import the worldpop asc file $worldPopFile" + info "Create a default empty worldpop table" + def outputTableWorldPopName = postfix "world_pop" + h2gis_datasource.execute("""drop table if exists $outputTableWorldPopName; + create table $outputTableWorldPopName (the_geom GEOMETRY(POLYGON, $srid), ID_POP INTEGER, POP FLOAT);""".toString()) + results.put("population", outputTableWorldPopName) } - tablesToDrop << results.building - //Update the building table with the population data - results.put("building", building) } else { - info "Cannot import the worldpop asc file $worldPopFile" - info "Create a default empty worldpop table" + info "Cannot find the population grid $coverageId \n Create a default empty worldpop table" def outputTableWorldPopName = postfix "world_pop" h2gis_datasource.execute("""drop table if exists $outputTableWorldPopName; - create table $outputTableWorldPopName (the_geom GEOMETRY(POLYGON, $srid), ID_POP INTEGER, POP FLOAT);""".toString()) + create table $outputTableWorldPopName (the_geom GEOMETRY(POLYGON, $srid), ID_POP INTEGER, POP FLOAT);""".toString()) results.put("population", outputTableWorldPopName) } - } else { info "Cannot find the population grid $coverageId \n Create a default empty worldpop table" def outputTableWorldPopName = postfix "world_pop" h2gis_datasource.execute("""drop table if exists $outputTableWorldPopName; - create table $outputTableWorldPopName (the_geom GEOMETRY(POLYGON, $srid), ID_POP INTEGER, POP FLOAT);""".toString()) + create table $outputTableWorldPopName (the_geom GEOMETRY(POLYGON, $srid), ID_POP INTEGER, POP FLOAT);""".toString()) results.put("population", outputTableWorldPopName) } } diff --git a/common-utils/src/main/groovy/org/orbisgis/geoclimate/utils/AbstractScript.groovy b/common-utils/src/main/groovy/org/orbisgis/geoclimate/utils/AbstractScript.groovy index 56bcb97a38..cf789ca0e0 100644 --- a/common-utils/src/main/groovy/org/orbisgis/geoclimate/utils/AbstractScript.groovy +++ b/common-utils/src/main/groovy/org/orbisgis/geoclimate/utils/AbstractScript.groovy @@ -26,19 +26,19 @@ abstract class AbstractScript extends Script { static String uuid() { UUID.randomUUID().toString().replaceAll("-", "_") } - void info(def message) { + static void info(def message) { LoggerUtils.info(message.toString()) } - void warn(def message) { + static void warn(def message) { LoggerUtils.warn(message.toString()) } - void error(def message) { + static void error(def message) { LoggerUtils.error(message.toString()) } - void debug(def message) { + static void debug(def message) { LoggerUtils.debug(message.toString()) } diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index ace789e8c2..e99acfaa70 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -4,3 +4,5 @@ - Fix some tests due to some rounding precision in JTS 1.20 - Fix bad OSM geometry representation #994 - Fix height value parsing with OSM #995 +- Simplify database url in config file. Use only the name of the database. e.g : h2://, postgis:// +- Add a test to check if the worldpop service is available diff --git a/geoindicators/src/main/groovy/org/orbisgis/geoclimate/geoindicators/WorkflowGeoIndicators.groovy b/geoindicators/src/main/groovy/org/orbisgis/geoclimate/geoindicators/WorkflowGeoIndicators.groovy index c276bb6122..5295583b10 100644 --- a/geoindicators/src/main/groovy/org/orbisgis/geoclimate/geoindicators/WorkflowGeoIndicators.groovy +++ b/geoindicators/src/main/groovy/org/orbisgis/geoclimate/geoindicators/WorkflowGeoIndicators.groovy @@ -1384,13 +1384,13 @@ Map estimateBuildingHeight(JdbcDataSource datasource, String zone, String buildi def estimated_building_with_indicators = "ESTIMATED_BUILDING_INDICATORS_${UUID.randomUUID().toString().replaceAll("-", "_")}" - datasource.execute """DROP TABLE IF EXISTS $estimated_building_with_indicators; + datasource.execute("""DROP TABLE IF EXISTS $estimated_building_with_indicators; CREATE TABLE $estimated_building_with_indicators AS SELECT a.* FROM $buildingIndicatorsForHeightEst a RIGHT JOIN $building_estimate b ON a.id_build=b.id_build - WHERE a.ID_RSU IS NOT NULL;""".toString() + WHERE a.ID_RSU IS NOT NULL;""") info "Collect building indicators to estimate the height" @@ -1405,11 +1405,10 @@ Map estimateBuildingHeight(JdbcDataSource datasource, String zone, String buildi def buildEstimatedHeight if (datasource.getTable(gatheredScales).isEmpty()) { info "No building height to estimate" - nbBuildingEstimated = 0 - datasource.execute """DROP TABLE IF EXISTS $buildingTableName; + datasource.execute("""DROP TABLE IF EXISTS $buildingTableName; CREATE TABLE $buildingTableName AS SELECT THE_GEOM, ID_BUILD, ID_SOURCE, HEIGHT_WALL , - HEIGHT_ROOF, NB_LEV, TYPE, MAIN_USE, ZINDEX, ID_BLOCK, ID_RSU from $estimated_building_with_indicators""".toString() + HEIGHT_ROOF, NB_LEV, TYPE, MAIN_USE, ZINDEX, ID_BLOCK, ID_RSU from $estimated_building_with_indicators""") } else { info "Start estimating the building height" diff --git a/geoindicators/src/main/groovy/org/orbisgis/geoclimate/geoindicators/WorkflowUtilities.groovy b/geoindicators/src/main/groovy/org/orbisgis/geoclimate/geoindicators/WorkflowUtilities.groovy index 0c0cb619f1..aec3722979 100644 --- a/geoindicators/src/main/groovy/org/orbisgis/geoclimate/geoindicators/WorkflowUtilities.groovy +++ b/geoindicators/src/main/groovy/org/orbisgis/geoclimate/geoindicators/WorkflowUtilities.groovy @@ -78,6 +78,9 @@ def createDatasource(def database_properties) { String url = db_output_url.substring("jdbc:".length()); if (url.startsWith("h2")) { return H2GIS.open(database_properties) + } else if (url.startsWith("postgis")) { + database_properties.url = "jdbc:postgresql"+url.substring("postgis".length()) + return POSTGIS.open(database_properties) } else if (url.startsWith("postgresql")) { return POSTGIS.open(database_properties) } else { @@ -85,8 +88,21 @@ def createDatasource(def database_properties) { return } } else { - error "Invalid output database url" - return + //Try to create the URL without JDBC prefix + if (db_output_url.startsWith("h2")) { + database_properties.url = "jdbc:"+db_output_url + return H2GIS.open(database_properties) + } else if (db_output_url.startsWith("postgresql")) { + database_properties.url = "jdbc:"+db_output_url + return POSTGIS.open(database_properties) + }else if (db_output_url.startsWith("postgis")) { + database_properties.url = "jdbc:postgresql"+db_output_url.substring("postgis".length()) + return POSTGIS.open(database_properties) + } + else { + error "Unsupported database" + return + } } } else { diff --git a/osm/src/main/groovy/org/orbisgis/geoclimate/osm/WorkflowOSM.groovy b/osm/src/main/groovy/org/orbisgis/geoclimate/osm/WorkflowOSM.groovy index b062505195..6e03e86846 100644 --- a/osm/src/main/groovy/org/orbisgis/geoclimate/osm/WorkflowOSM.groovy +++ b/osm/src/main/groovy/org/orbisgis/geoclimate/osm/WorkflowOSM.groovy @@ -539,33 +539,42 @@ Map osm_processing(JdbcDataSource h2gis_datasource, def processing_parameters, d //Extract and compute population indicators for the specified year //This data can be used by the grid_indicators process if (worldpop_indicators) { - def bbox = [zones.osm_envelope_extented.getMinY() as Float, zones.osm_envelope_extented.getMinX() as Float, - zones.osm_envelope_extented.getMaxY() as Float, zones.osm_envelope_extented.getMaxX() as Float] String coverageId = "wpGlobal:ppp_2020" - String worldPopFile = WorldPopTools.Extract.extractWorldPopLayer(coverageId, bbox) - if (worldPopFile) { - String worldPopTableName = WorldPopTools.Extract.importAscGrid(h2gis_datasource, worldPopFile, srid, coverageId.replaceAll(":", "_")) - if (worldPopTableName) { - results.put("population", worldPopTableName) - String buildingWithPop = Geoindicators.BuildingIndicators.buildingPopulation(h2gis_datasource, results.building, worldPopTableName, ["pop"]) - h2gis_datasource.dropTable(worldPopTableName) - if (!buildingWithPop) { - info "Cannot compute any population data at building level" + if(WorldPopTools.Extract.isCoverageAvailable(coverageId)) { + def bbox = [zones.osm_envelope_extented.getMinY() as Float, zones.osm_envelope_extented.getMinX() as Float, + zones.osm_envelope_extented.getMaxY() as Float, zones.osm_envelope_extented.getMaxX() as Float] + + String worldPopFile = WorldPopTools.Extract.extractWorldPopLayer(coverageId, bbox) + if (worldPopFile) { + String worldPopTableName = WorldPopTools.Extract.importAscGrid(h2gis_datasource, worldPopFile, srid, coverageId.replaceAll(":", "_")) + if (worldPopTableName) { + results.put("population", worldPopTableName) + String buildingWithPop = Geoindicators.BuildingIndicators.buildingPopulation(h2gis_datasource, results.building, worldPopTableName, ["pop"]) + h2gis_datasource.dropTable(worldPopTableName) + if (!buildingWithPop) { + info "Cannot compute any population data at building level" + } else { + h2gis_datasource.dropTable(results.building) + //Update the building table with the population data + results.put("building", buildingWithPop) + } + } else { - h2gis_datasource.dropTable(results.building) - //Update the building table with the population data - results.put("building", buildingWithPop) + info "Cannot import the worldpop asc file $worldPopFile" + info "Create a default empty worldpop table" + def outputTableWorldPopName = postfix "world_pop" + h2gis_datasource.execute("""drop table if exists $outputTableWorldPopName; + create table $outputTableWorldPopName (the_geom GEOMETRY(POLYGON, $srid), ID_POP INTEGER, POP FLOAT);""".toString()) + results.put("population", outputTableWorldPopName) } } else { - info "Cannot import the worldpop asc file $worldPopFile" - info "Create a default empty worldpop table" + info "Cannot find the population grid $coverageId \n Create a default empty worldpop table" def outputTableWorldPopName = postfix "world_pop" h2gis_datasource.execute("""drop table if exists $outputTableWorldPopName; - create table $outputTableWorldPopName (the_geom GEOMETRY(POLYGON, $srid), ID_POP INTEGER, POP FLOAT);""".toString()) + create table $outputTableWorldPopName (the_geom GEOMETRY(POLYGON, $srid), ID_POP INTEGER, POP FLOAT);""".toString()) results.put("population", outputTableWorldPopName) } - } else { info "Cannot find the population grid $coverageId \n Create a default empty worldpop table" def outputTableWorldPopName = postfix "world_pop" diff --git a/osm/src/test/groovy/org/orbisgis/geoclimate/osm/WorflowOSMTest.groovy b/osm/src/test/groovy/org/orbisgis/geoclimate/osm/WorflowOSMTest.groovy index f6334ff4ad..fbea159870 100644 --- a/osm/src/test/groovy/org/orbisgis/geoclimate/osm/WorflowOSMTest.groovy +++ b/osm/src/test/groovy/org/orbisgis/geoclimate/osm/WorflowOSMTest.groovy @@ -649,11 +649,11 @@ class WorflowOSMTest extends WorkflowAbstractTest { File dirFile = new File(directory) dirFile.delete() dirFile.mkdir() - def location = "Nantes" + def location = "Hesdin" def nominatim = org.orbisgis.geoclimate.osmtools.OSMTools.Utilities.getNominatimData(location) - def grid_size = 250 - location = nominatim.bbox - location=[51.2, 1.0, 51.4, 1.2] + def grid_size = 100 + //location = nominatim.bbox + //location=[51.2, 1.0, 51.4, 1.2] /* location =[ 48.84017284026897, 2.3061887733275785, 48.878115442982086, @@ -674,6 +674,15 @@ class WorflowOSMTest extends WorkflowAbstractTest { "endpoint":"https://lz4.overpass-api.de/api"*/], "output" : [ "folder": directory] + /*[ + "database": + ["user" : "ebocher", + "password": "k@ndinsky5622", + "url" : "postgis://localhost:5432/orbisgis_db", + "tables" : [ + "rsu_indicators" : "rsu_indicators", + "rsu_lcz" : "rsu_lcz", + "zone" : "zone"]]]*/ , "parameters" : ["distance" : 0, diff --git a/worldpoptools/src/test/groovy/org/orbisgis/geoclimate/worldpoptools/WorldPopExtractTest.groovy b/worldpoptools/src/test/groovy/org/orbisgis/geoclimate/worldpoptools/WorldPopExtractTest.groovy index 60baed26ee..b7980ad67d 100644 --- a/worldpoptools/src/test/groovy/org/orbisgis/geoclimate/worldpoptools/WorldPopExtractTest.groovy +++ b/worldpoptools/src/test/groovy/org/orbisgis/geoclimate/worldpoptools/WorldPopExtractTest.groovy @@ -57,19 +57,21 @@ class WorldPopExtractTest { */ @Test void extractGrid() { - def outputGridFile = new File(folder, "extractGrid.asc") - if (outputGridFile.exists()) { - outputGridFile.delete() - } def coverageId = "wpGlobal:ppp_2018" - def bbox = [47.63324, -2.78087, 47.65749, -2.75979] - if (WorldPopTools.Extract.grid(coverageId, bbox, outputGridFile)) { - AscReaderDriver ascReaderDriver = new AscReaderDriver() - ascReaderDriver.setAs3DPoint(false) - ascReaderDriver.setEncoding("UTF-8") - ascReaderDriver.setDeleteTable(true) - ascReaderDriver.read(h2GIS.getConnection(), outputGridFile, new EmptyProgressVisitor(), "grid", 4326) - assertEquals(720, h2GIS.getSpatialTable("grid").rowCount) + if (WorldPopExtract.Extract.isCoverageAvailable(coverageId)) { + def outputGridFile = new File(folder, "extractGrid.asc") + if (outputGridFile.exists()) { + outputGridFile.delete() + } + def bbox = [47.63324, -2.78087, 47.65749, -2.75979] + if (WorldPopTools.Extract.grid(coverageId, bbox, outputGridFile)) { + AscReaderDriver ascReaderDriver = new AscReaderDriver() + ascReaderDriver.setAs3DPoint(false) + ascReaderDriver.setEncoding("UTF-8") + ascReaderDriver.setDeleteTable(true) + ascReaderDriver.read(h2GIS.getConnection(), outputGridFile, new EmptyProgressVisitor(), "grid", 4326) + assertEquals(720, h2GIS.getSpatialTable("grid").rowCount) + } } } @@ -107,7 +109,6 @@ class WorldPopExtractTest { */ @Test void testCoverageAvailable() { - assertFalse(WorldPopExtract.Extract.isCoverageAvailable("wpGlobal:ppp_2050")) - assertTrue(WorldPopExtract.Extract.isCoverageAvailable("wpGlobal:ppp_2018")) + assertFalse(WorldPopExtract.Extract.isCoverageAvailable("wpGlobal:ppp_2150")) } }