Skip to content

Commit

Permalink
Fix worldpop service request plus url database syntax
Browse files Browse the repository at this point in the history
  • Loading branch information
@ebocher
ebocher committed Oct 12, 2024
1 parent 5df3e3c commit c6f14d0
Show file tree
Hide file tree
Showing 8 changed files with 113 additions and 67 deletions.
50 changes: 30 additions & 20 deletions bdtopo/src/main/groovy/org/orbisgis/geoclimate/bdtopo/AbstractBDTopoWorkflow.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -977,38 +977,48 @@ abstract class AbstractBDTopoWorkflow extends BDTopoUtils {
//Extract and compute population indicators for the specified year
//This data can be used by the grid_indicators process
if (worldpop_indicators) {
def envelope = h2gis_datasource.firstRow("select st_transform(the_geom, 4326) as geom from $zone".toString()).geom.getEnvelopeInternal()
def bbox = [envelope.getMinY() as Float, envelope.getMinX() as Float,
envelope.getMaxY() as Float, envelope.getMaxX() as Float]
String coverageId = "wpGlobal:ppp_2018"
String worldPopFile = WorldPopTools.Extract.extractWorldPopLayer(coverageId, bbox)
if (worldPopFile) {
String worldPopTableName = WorldPopTools.Extract.importAscGrid(h2gis_datasource, worldPopFile, srid, coverageId.replaceAll(":", "_"))
if (worldPopTableName) {
results.put("population", worldPopTableName)
building = Geoindicators.BuildingIndicators.buildingPopulation(h2gis_datasource, results.building,
worldPopTableName, ["pop"])
if (!building) {
info "Cannot compute any population data at building level"
//Test if the coverage is available
if(WorldPopTools.Extract.isCoverageAvailable(coverageId)) {
def envelope = h2gis_datasource.firstRow("select st_transform(the_geom, 4326) as geom from $zone".toString()).geom.getEnvelopeInternal()
def bbox = [envelope.getMinY() as Float, envelope.getMinX() as Float,
envelope.getMaxY() as Float, envelope.getMaxX() as Float]

String worldPopFile = WorldPopTools.Extract.extractWorldPopLayer(coverageId, bbox)
if (worldPopFile) {
String worldPopTableName = WorldPopTools.Extract.importAscGrid(h2gis_datasource, worldPopFile, srid, coverageId.replaceAll(":", "_"))
if (worldPopTableName) {
results.put("population", worldPopTableName)
building = Geoindicators.BuildingIndicators.buildingPopulation(h2gis_datasource, results.building,
worldPopTableName, ["pop"])
if (!building) {
info "Cannot compute any population data at building level"
}
tablesToDrop << results.building
//Update the building table with the population data
results.put("building", building)

} else {
info "Cannot import the worldpop asc file $worldPopFile"
info "Create a default empty worldpop table"
def outputTableWorldPopName = postfix "world_pop"
h2gis_datasource.execute("""drop table if exists $outputTableWorldPopName;
create table $outputTableWorldPopName (the_geom GEOMETRY(POLYGON, $srid), ID_POP INTEGER, POP FLOAT);""".toString())
results.put("population", outputTableWorldPopName)
}
tablesToDrop << results.building
//Update the building table with the population data
results.put("building", building)

} else {
info "Cannot import the worldpop asc file $worldPopFile"
info "Create a default empty worldpop table"
info "Cannot find the population grid $coverageId \n Create a default empty worldpop table"
def outputTableWorldPopName = postfix "world_pop"
h2gis_datasource.execute("""drop table if exists $outputTableWorldPopName;
create table $outputTableWorldPopName (the_geom GEOMETRY(POLYGON, $srid), ID_POP INTEGER, POP FLOAT);""".toString())
create table $outputTableWorldPopName (the_geom GEOMETRY(POLYGON, $srid), ID_POP INTEGER, POP FLOAT);""".toString())
results.put("population", outputTableWorldPopName)
}

} else {
info "Cannot find the population grid $coverageId \n Create a default empty worldpop table"
def outputTableWorldPopName = postfix "world_pop"
h2gis_datasource.execute("""drop table if exists $outputTableWorldPopName;
create table $outputTableWorldPopName (the_geom GEOMETRY(POLYGON, $srid), ID_POP INTEGER, POP FLOAT);""".toString())
create table $outputTableWorldPopName (the_geom GEOMETRY(POLYGON, $srid), ID_POP INTEGER, POP FLOAT);""".toString())
results.put("population", outputTableWorldPopName)
}
}
Expand Down
8 changes: 4 additions & 4 deletions common-utils/src/main/groovy/org/orbisgis/geoclimate/utils/AbstractScript.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,19 +26,19 @@ abstract class AbstractScript extends Script {

static String uuid() { UUID.randomUUID().toString().replaceAll("-", "_") }

void info(def message) {
static void info(def message) {
LoggerUtils.info(message.toString())
}

void warn(def message) {
static void warn(def message) {
LoggerUtils.warn(message.toString())
}

void error(def message) {
static void error(def message) {
LoggerUtils.error(message.toString())
}

void debug(def message) {
static void debug(def message) {
LoggerUtils.debug(message.toString())
}

Expand Down
2 changes: 2 additions & 0 deletions docs/CHANGELOG.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,3 +4,5 @@
- Fix some tests due to some rounding precision in JTS 1.20
- Fix bad OSM geometry representation #994
- Fix height value parsing with OSM #995
- Simplify database url in config file. Use only the name of the database. e.g : h2://, postgis://
- Add a test to check if the worldpop service is available
9 changes: 4 additions & 5 deletions ...cators/src/main/groovy/org/orbisgis/geoclimate/geoindicators/WorkflowGeoIndicators.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1384,13 +1384,13 @@ Map estimateBuildingHeight(JdbcDataSource datasource, String zone, String buildi

def estimated_building_with_indicators = "ESTIMATED_BUILDING_INDICATORS_${UUID.randomUUID().toString().replaceAll("-", "_")}"

datasource.execute """DROP TABLE IF EXISTS $estimated_building_with_indicators;
datasource.execute("""DROP TABLE IF EXISTS $estimated_building_with_indicators;
CREATE TABLE $estimated_building_with_indicators
AS SELECT a.*
FROM $buildingIndicatorsForHeightEst a
RIGHT JOIN $building_estimate b
ON a.id_build=b.id_build
WHERE a.ID_RSU IS NOT NULL;""".toString()
WHERE a.ID_RSU IS NOT NULL;""")

info "Collect building indicators to estimate the height"

Expand All @@ -1405,11 +1405,10 @@ Map estimateBuildingHeight(JdbcDataSource datasource, String zone, String buildi
def buildEstimatedHeight
if (datasource.getTable(gatheredScales).isEmpty()) {
info "No building height to estimate"
nbBuildingEstimated = 0
datasource.execute """DROP TABLE IF EXISTS $buildingTableName;
datasource.execute("""DROP TABLE IF EXISTS $buildingTableName;
CREATE TABLE $buildingTableName
AS SELECT THE_GEOM, ID_BUILD, ID_SOURCE, HEIGHT_WALL ,
HEIGHT_ROOF, NB_LEV, TYPE, MAIN_USE, ZINDEX, ID_BLOCK, ID_RSU from $estimated_building_with_indicators""".toString()
HEIGHT_ROOF, NB_LEV, TYPE, MAIN_USE, ZINDEX, ID_BLOCK, ID_RSU from $estimated_building_with_indicators""")

} else {
info "Start estimating the building height"
Expand Down
20 changes: 18 additions & 2 deletions geoindicators/src/main/groovy/org/orbisgis/geoclimate/geoindicators/WorkflowUtilities.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -78,15 +78,31 @@ def createDatasource(def database_properties) {
String url = db_output_url.substring("jdbc:".length());
if (url.startsWith("h2")) {
return H2GIS.open(database_properties)
} else if (url.startsWith("postgis")) {
database_properties.url = "jdbc:postgresql"+url.substring("postgis".length())
return POSTGIS.open(database_properties)
} else if (url.startsWith("postgresql")) {
return POSTGIS.open(database_properties)
} else {
error "Unsupported database"
return
}
} else {
error "Invalid output database url"
return
//Try to create the URL without JDBC prefix
if (db_output_url.startsWith("h2")) {
database_properties.url = "jdbc:"+db_output_url
return H2GIS.open(database_properties)
} else if (db_output_url.startsWith("postgresql")) {
database_properties.url = "jdbc:"+db_output_url
return POSTGIS.open(database_properties)
}else if (db_output_url.startsWith("postgis")) {
database_properties.url = "jdbc:postgresql"+db_output_url.substring("postgis".length())
return POSTGIS.open(database_properties)
}
else {
error "Unsupported database"
return
}
}

} else {
Expand Down
45 changes: 27 additions & 18 deletions osm/src/main/groovy/org/orbisgis/geoclimate/osm/WorkflowOSM.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -539,33 +539,42 @@ Map osm_processing(JdbcDataSource h2gis_datasource, def processing_parameters, d
//Extract and compute population indicators for the specified year
//This data can be used by the grid_indicators process
if (worldpop_indicators) {
def bbox = [zones.osm_envelope_extented.getMinY() as Float, zones.osm_envelope_extented.getMinX() as Float,
zones.osm_envelope_extented.getMaxY() as Float, zones.osm_envelope_extented.getMaxX() as Float]
String coverageId = "wpGlobal:ppp_2020"
String worldPopFile = WorldPopTools.Extract.extractWorldPopLayer(coverageId, bbox)
if (worldPopFile) {
String worldPopTableName = WorldPopTools.Extract.importAscGrid(h2gis_datasource, worldPopFile, srid, coverageId.replaceAll(":", "_"))
if (worldPopTableName) {
results.put("population", worldPopTableName)
String buildingWithPop = Geoindicators.BuildingIndicators.buildingPopulation(h2gis_datasource, results.building, worldPopTableName, ["pop"])
h2gis_datasource.dropTable(worldPopTableName)
if (!buildingWithPop) {
info "Cannot compute any population data at building level"
if(WorldPopTools.Extract.isCoverageAvailable(coverageId)) {
def bbox = [zones.osm_envelope_extented.getMinY() as Float, zones.osm_envelope_extented.getMinX() as Float,
zones.osm_envelope_extented.getMaxY() as Float, zones.osm_envelope_extented.getMaxX() as Float]

String worldPopFile = WorldPopTools.Extract.extractWorldPopLayer(coverageId, bbox)
if (worldPopFile) {
String worldPopTableName = WorldPopTools.Extract.importAscGrid(h2gis_datasource, worldPopFile, srid, coverageId.replaceAll(":", "_"))
if (worldPopTableName) {
results.put("population", worldPopTableName)
String buildingWithPop = Geoindicators.BuildingIndicators.buildingPopulation(h2gis_datasource, results.building, worldPopTableName, ["pop"])
h2gis_datasource.dropTable(worldPopTableName)
if (!buildingWithPop) {
info "Cannot compute any population data at building level"
} else {
h2gis_datasource.dropTable(results.building)
//Update the building table with the population data
results.put("building", buildingWithPop)
}

} else {
h2gis_datasource.dropTable(results.building)
//Update the building table with the population data
results.put("building", buildingWithPop)
info "Cannot import the worldpop asc file $worldPopFile"
info "Create a default empty worldpop table"
def outputTableWorldPopName = postfix "world_pop"
h2gis_datasource.execute("""drop table if exists $outputTableWorldPopName;
create table $outputTableWorldPopName (the_geom GEOMETRY(POLYGON, $srid), ID_POP INTEGER, POP FLOAT);""".toString())
results.put("population", outputTableWorldPopName)
}

} else {
info "Cannot import the worldpop asc file $worldPopFile"
info "Create a default empty worldpop table"
info "Cannot find the population grid $coverageId \n Create a default empty worldpop table"
def outputTableWorldPopName = postfix "world_pop"
h2gis_datasource.execute("""drop table if exists $outputTableWorldPopName;
create table $outputTableWorldPopName (the_geom GEOMETRY(POLYGON, $srid), ID_POP INTEGER, POP FLOAT);""".toString())
create table $outputTableWorldPopName (the_geom GEOMETRY(POLYGON, $srid), ID_POP INTEGER, POP FLOAT);""".toString())
results.put("population", outputTableWorldPopName)
}

} else {
info "Cannot find the population grid $coverageId \n Create a default empty worldpop table"
def outputTableWorldPopName = postfix "world_pop"
Expand Down
17 changes: 13 additions & 4 deletions osm/src/test/groovy/org/orbisgis/geoclimate/osm/WorflowOSMTest.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -649,11 +649,11 @@ class WorflowOSMTest extends WorkflowAbstractTest {
File dirFile = new File(directory)
dirFile.delete()
dirFile.mkdir()
def location = "Nantes"
def location = "Hesdin"
def nominatim = org.orbisgis.geoclimate.osmtools.OSMTools.Utilities.getNominatimData(location)
def grid_size = 250
location = nominatim.bbox
location=[51.2, 1.0, 51.4, 1.2]
def grid_size = 100
//location = nominatim.bbox
//location=[51.2, 1.0, 51.4, 1.2]
/* location =[ 48.84017284026897,
2.3061887733275785,
48.878115442982086,
Expand All @@ -674,6 +674,15 @@ class WorflowOSMTest extends WorkflowAbstractTest {
"endpoint":"https://lz4.overpass-api.de/api"*/],
"output" : [
"folder": directory]
/*[
"database":
["user" : "ebocher",
"password": "k@ndinsky5622",
"url" : "postgis://localhost:5432/orbisgis_db",
"tables" : [
"rsu_indicators" : "rsu_indicators",
"rsu_lcz" : "rsu_lcz",
"zone" : "zone"]]]*/
,
"parameters" :
["distance" : 0,
Expand Down
29 changes: 15 additions & 14 deletions ...poptools/src/test/groovy/org/orbisgis/geoclimate/worldpoptools/WorldPopExtractTest.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,19 +57,21 @@ class WorldPopExtractTest {
*/
@Test
void extractGrid() {
def outputGridFile = new File(folder, "extractGrid.asc")
if (outputGridFile.exists()) {
outputGridFile.delete()
}
def coverageId = "wpGlobal:ppp_2018"
def bbox = [47.63324, -2.78087, 47.65749, -2.75979]
if (WorldPopTools.Extract.grid(coverageId, bbox, outputGridFile)) {
AscReaderDriver ascReaderDriver = new AscReaderDriver()
ascReaderDriver.setAs3DPoint(false)
ascReaderDriver.setEncoding("UTF-8")
ascReaderDriver.setDeleteTable(true)
ascReaderDriver.read(h2GIS.getConnection(), outputGridFile, new EmptyProgressVisitor(), "grid", 4326)
assertEquals(720, h2GIS.getSpatialTable("grid").rowCount)
if (WorldPopExtract.Extract.isCoverageAvailable(coverageId)) {
def outputGridFile = new File(folder, "extractGrid.asc")
if (outputGridFile.exists()) {
outputGridFile.delete()
}
def bbox = [47.63324, -2.78087, 47.65749, -2.75979]
if (WorldPopTools.Extract.grid(coverageId, bbox, outputGridFile)) {
AscReaderDriver ascReaderDriver = new AscReaderDriver()
ascReaderDriver.setAs3DPoint(false)
ascReaderDriver.setEncoding("UTF-8")
ascReaderDriver.setDeleteTable(true)
ascReaderDriver.read(h2GIS.getConnection(), outputGridFile, new EmptyProgressVisitor(), "grid", 4326)
assertEquals(720, h2GIS.getSpatialTable("grid").rowCount)
}
}
}

Expand Down Expand Up @@ -107,7 +109,6 @@ class WorldPopExtractTest {
*/
@Test
void testCoverageAvailable() {
assertFalse(WorldPopExtract.Extract.isCoverageAvailable("wpGlobal:ppp_2050"))
assertTrue(WorldPopExtract.Extract.isCoverageAvailable("wpGlobal:ppp_2018"))
assertFalse(WorldPopExtract.Extract.isCoverageAvailable("wpGlobal:ppp_2150"))
}
}

0 comments on commit c6f14d0

Please sign in to comment.