Merge pull request #1000 from ebocher/incorrect_osm_geom

Fix worldpop service request plus url database syntax

bdtopo/src/main/groovy/org/orbisgis/geoclimate/bdtopo/AbstractBDTopoWorkflow.groovy

@@ -977,38 +977,48 @@ abstract class AbstractBDTopoWorkflow extends BDTopoUtils {
             //Extract and compute population indicators for the specified year
             //This data can be used by the grid_indicators process
             if (worldpop_indicators) {
-                def envelope = h2gis_datasource.firstRow("select st_transform(the_geom, 4326) as geom from $zone".toString()).geom.getEnvelopeInternal()
-                def bbox = [envelope.getMinY() as Float, envelope.getMinX() as Float,
-                            envelope.getMaxY() as Float, envelope.getMaxX() as Float]
                 String coverageId = "wpGlobal:ppp_2018"
-                String worldPopFile = WorldPopTools.Extract.extractWorldPopLayer(coverageId, bbox)
-                if (worldPopFile) {
-                    String worldPopTableName = WorldPopTools.Extract.importAscGrid(h2gis_datasource, worldPopFile, srid, coverageId.replaceAll(":", "_"))
-                    if (worldPopTableName) {
-                        results.put("population", worldPopTableName)
-                        building = Geoindicators.BuildingIndicators.buildingPopulation(h2gis_datasource, results.building,
-                                worldPopTableName, ["pop"])
-                        if (!building) {
-                            info "Cannot compute any population data at building level"
+                //Test if the coverage is available
+                if(WorldPopTools.Extract.isCoverageAvailable(coverageId)) {
+                    def envelope = h2gis_datasource.firstRow("select st_transform(the_geom, 4326) as geom from $zone".toString()).geom.getEnvelopeInternal()
+                    def bbox = [envelope.getMinY() as Float, envelope.getMinX() as Float,
+                                envelope.getMaxY() as Float, envelope.getMaxX() as Float]
+
+                    String worldPopFile = WorldPopTools.Extract.extractWorldPopLayer(coverageId, bbox)
+                    if (worldPopFile) {
+                        String worldPopTableName = WorldPopTools.Extract.importAscGrid(h2gis_datasource, worldPopFile, srid, coverageId.replaceAll(":", "_"))
+                        if (worldPopTableName) {
+                            results.put("population", worldPopTableName)
+                            building = Geoindicators.BuildingIndicators.buildingPopulation(h2gis_datasource, results.building,
+                                    worldPopTableName, ["pop"])
+                            if (!building) {
+                                info "Cannot compute any population data at building level"
+                            }
+                            tablesToDrop << results.building
+                            //Update the building table with the population data
+                            results.put("building", building)
+
+                        } else {
+                            info "Cannot import the worldpop asc file $worldPopFile"
+                            info "Create a default empty worldpop table"
+                            def outputTableWorldPopName = postfix "world_pop"
+                            h2gis_datasource.execute("""drop table if exists $outputTableWorldPopName;
+                    create table $outputTableWorldPopName (the_geom GEOMETRY(POLYGON, $srid), ID_POP INTEGER, POP FLOAT);""".toString())
+                            results.put("population", outputTableWorldPopName)
                         }
-                        tablesToDrop << results.building
-                        //Update the building table with the population data
-                        results.put("building", building)
 
                     } else {
-                        info "Cannot import the worldpop asc file $worldPopFile"
-                        info "Create a default empty worldpop table"
+                        info "Cannot find the population grid $coverageId \n Create a default empty worldpop table"
                         def outputTableWorldPopName = postfix "world_pop"
                         h2gis_datasource.execute("""drop table if exists $outputTableWorldPopName;
-                    create table $outputTableWorldPopName (the_geom GEOMETRY(POLYGON, $srid), ID_POP INTEGER, POP FLOAT);""".toString())
+                        create table $outputTableWorldPopName (the_geom GEOMETRY(POLYGON, $srid), ID_POP INTEGER, POP FLOAT);""".toString())
                         results.put("population", outputTableWorldPopName)
                     }
-
                 } else {
                     info "Cannot find the population grid $coverageId \n Create a default empty worldpop table"
                     def outputTableWorldPopName = postfix "world_pop"
                     h2gis_datasource.execute("""drop table if exists $outputTableWorldPopName;
-                    create table $outputTableWorldPopName (the_geom GEOMETRY(POLYGON, $srid), ID_POP INTEGER, POP FLOAT);""".toString())
+                        create table $outputTableWorldPopName (the_geom GEOMETRY(POLYGON, $srid), ID_POP INTEGER, POP FLOAT);""".toString())
                     results.put("population", outputTableWorldPopName)
                 }
             }

common-utils/src/main/groovy/org/orbisgis/geoclimate/utils/AbstractScript.groovy

@@ -26,19 +26,19 @@ abstract class AbstractScript extends Script {
 
     static String uuid() { UUID.randomUUID().toString().replaceAll("-", "_") }
 
-    void info(def message) {
+    static void info(def message) {
         LoggerUtils.info(message.toString())
     }
 
-    void warn(def message) {
+    static void warn(def message) {
         LoggerUtils.warn(message.toString())
     }
 
-    void error(def message) {
+    static void error(def message) {
         LoggerUtils.error(message.toString())
     }
 
-    void debug(def message) {
+    static void debug(def message) {
         LoggerUtils.debug(message.toString())
     }
 

docs/CHANGELOG.md

@@ -4,3 +4,5 @@
 - Fix some tests due to some rounding precision in JTS 1.20
 - Fix bad OSM geometry representation #994
 - Fix height value parsing with OSM #995
+- Simplify database url in config file. Use only the name of the database. e.g : h2://, postgis://
+- Add a test to check if the worldpop service is available

geoindicators/src/main/groovy/org/orbisgis/geoclimate/geoindicators/WorkflowGeoIndicators.groovy

@@ -1384,13 +1384,13 @@ Map estimateBuildingHeight(JdbcDataSource datasource, String zone, String buildi
 
     def estimated_building_with_indicators = "ESTIMATED_BUILDING_INDICATORS_${UUID.randomUUID().toString().replaceAll("-", "_")}"
 
-    datasource.execute """DROP TABLE IF EXISTS $estimated_building_with_indicators;
+    datasource.execute("""DROP TABLE IF EXISTS $estimated_building_with_indicators;
                                                CREATE TABLE $estimated_building_with_indicators 
                                                         AS SELECT a.*
                                                         FROM $buildingIndicatorsForHeightEst a 
                                                             RIGHT JOIN $building_estimate b 
                                                             ON a.id_build=b.id_build
-                                                        WHERE a.ID_RSU IS NOT NULL;""".toString()
+                                                        WHERE a.ID_RSU IS NOT NULL;""")
 
     info "Collect building indicators to estimate the height"
 
@@ -1405,11 +1405,10 @@ Map estimateBuildingHeight(JdbcDataSource datasource, String zone, String buildi
     def buildEstimatedHeight
     if (datasource.getTable(gatheredScales).isEmpty()) {
         info "No building height to estimate"
-        nbBuildingEstimated = 0
-        datasource.execute """DROP TABLE IF EXISTS $buildingTableName;
+        datasource.execute("""DROP TABLE IF EXISTS $buildingTableName;
                                            CREATE TABLE $buildingTableName 
                                                 AS SELECT  THE_GEOM, ID_BUILD, ID_SOURCE, HEIGHT_WALL ,
-                                                    HEIGHT_ROOF, NB_LEV, TYPE, MAIN_USE, ZINDEX, ID_BLOCK, ID_RSU from $estimated_building_with_indicators""".toString()
+                                                    HEIGHT_ROOF, NB_LEV, TYPE, MAIN_USE, ZINDEX, ID_BLOCK, ID_RSU from $estimated_building_with_indicators""")
 
     } else {
         info "Start estimating the building height"

geoindicators/src/main/groovy/org/orbisgis/geoclimate/geoindicators/WorkflowUtilities.groovy

@@ -78,15 +78,31 @@ def createDatasource(def database_properties) {
             String url = db_output_url.substring("jdbc:".length());
             if (url.startsWith("h2")) {
                 return H2GIS.open(database_properties)
+            } else if (url.startsWith("postgis")) {
+                database_properties.url = "jdbc:postgresql"+url.substring("postgis".length())
+                return POSTGIS.open(database_properties)
             } else if (url.startsWith("postgresql")) {
                 return POSTGIS.open(database_properties)
             } else {
                 error "Unsupported database"
                 return
             }
         } else {
-            error "Invalid output database url"
-            return
+            //Try to create the URL without JDBC prefix
+            if (db_output_url.startsWith("h2")) {
+                database_properties.url = "jdbc:"+db_output_url
+                return H2GIS.open(database_properties)
+            } else if (db_output_url.startsWith("postgresql")) {
+                database_properties.url = "jdbc:"+db_output_url
+                return POSTGIS.open(database_properties)
+            }else if (db_output_url.startsWith("postgis")) {
+                database_properties.url = "jdbc:postgresql"+db_output_url.substring("postgis".length())
+                return POSTGIS.open(database_properties)
+            }
+            else {
+                error "Unsupported database"
+                return
+            }
         }
 
     } else {

osm/src/main/groovy/org/orbisgis/geoclimate/osm/WorkflowOSM.groovy

@@ -539,33 +539,42 @@ Map osm_processing(JdbcDataSource h2gis_datasource, def processing_parameters, d
                 //Extract and compute population indicators for the specified year
                 //This data can be used by the grid_indicators process
                 if (worldpop_indicators) {
-                    def bbox = [zones.osm_envelope_extented.getMinY() as Float, zones.osm_envelope_extented.getMinX() as Float,
-                                zones.osm_envelope_extented.getMaxY() as Float, zones.osm_envelope_extented.getMaxX() as Float]
                     String coverageId = "wpGlobal:ppp_2020"
-                    String worldPopFile = WorldPopTools.Extract.extractWorldPopLayer(coverageId, bbox)
-                    if (worldPopFile) {
-                        String worldPopTableName = WorldPopTools.Extract.importAscGrid(h2gis_datasource, worldPopFile, srid, coverageId.replaceAll(":", "_"))
-                        if (worldPopTableName) {
-                            results.put("population", worldPopTableName)
-                            String buildingWithPop = Geoindicators.BuildingIndicators.buildingPopulation(h2gis_datasource, results.building, worldPopTableName, ["pop"])
-                            h2gis_datasource.dropTable(worldPopTableName)
-                            if (!buildingWithPop) {
-                                info "Cannot compute any population data at building level"
+                    if(WorldPopTools.Extract.isCoverageAvailable(coverageId)) {
+                        def bbox = [zones.osm_envelope_extented.getMinY() as Float, zones.osm_envelope_extented.getMinX() as Float,
+                                    zones.osm_envelope_extented.getMaxY() as Float, zones.osm_envelope_extented.getMaxX() as Float]
+
+                        String worldPopFile = WorldPopTools.Extract.extractWorldPopLayer(coverageId, bbox)
+                        if (worldPopFile) {
+                            String worldPopTableName = WorldPopTools.Extract.importAscGrid(h2gis_datasource, worldPopFile, srid, coverageId.replaceAll(":", "_"))
+                            if (worldPopTableName) {
+                                results.put("population", worldPopTableName)
+                                String buildingWithPop = Geoindicators.BuildingIndicators.buildingPopulation(h2gis_datasource, results.building, worldPopTableName, ["pop"])
+                                h2gis_datasource.dropTable(worldPopTableName)
+                                if (!buildingWithPop) {
+                                    info "Cannot compute any population data at building level"
+                                } else {
+                                    h2gis_datasource.dropTable(results.building)
+                                    //Update the building table with the population data
+                                    results.put("building", buildingWithPop)
+                                }
+
                             } else {
-                                h2gis_datasource.dropTable(results.building)
-                                //Update the building table with the population data
-                                results.put("building", buildingWithPop)
+                                info "Cannot import the worldpop asc file $worldPopFile"
+                                info "Create a default empty worldpop table"
+                                def outputTableWorldPopName = postfix "world_pop"
+                                h2gis_datasource.execute("""drop table if exists $outputTableWorldPopName;
+                                        create table $outputTableWorldPopName (the_geom GEOMETRY(POLYGON, $srid), ID_POP INTEGER, POP FLOAT);""".toString())
+                                results.put("population", outputTableWorldPopName)
                             }
 
                         } else {
-                            info "Cannot import the worldpop asc file $worldPopFile"
-                            info "Create a default empty worldpop table"
+                            info "Cannot find the population grid $coverageId \n Create a default empty worldpop table"
                             def outputTableWorldPopName = postfix "world_pop"
                             h2gis_datasource.execute("""drop table if exists $outputTableWorldPopName;
-                                        create table $outputTableWorldPopName (the_geom GEOMETRY(POLYGON, $srid), ID_POP INTEGER, POP FLOAT);""".toString())
+                                    create table $outputTableWorldPopName (the_geom GEOMETRY(POLYGON, $srid), ID_POP INTEGER, POP FLOAT);""".toString())
                             results.put("population", outputTableWorldPopName)
                         }
-
                     } else {
                         info "Cannot find the population grid $coverageId \n Create a default empty worldpop table"
                         def outputTableWorldPopName = postfix "world_pop"

osm/src/test/groovy/org/orbisgis/geoclimate/osm/WorflowOSMTest.groovy

@@ -649,11 +649,11 @@ class WorflowOSMTest extends WorkflowAbstractTest {
         File dirFile = new File(directory)
         dirFile.delete()
         dirFile.mkdir()
-        def location = "Nantes"
+        def location = "Hesdin"
         def nominatim = org.orbisgis.geoclimate.osmtools.OSMTools.Utilities.getNominatimData(location)
-        def grid_size = 250
-        location = nominatim.bbox
-        location=[51.2, 1.0, 51.4, 1.2]
+        def grid_size = 100
+        //location = nominatim.bbox
+        //location=[51.2, 1.0, 51.4, 1.2]
         /* location =[ 48.84017284026897,
                     2.3061887733275785,
                     48.878115442982086,
@@ -674,6 +674,15 @@ class WorflowOSMTest extends WorkflowAbstractTest {
                         "endpoint":"https://lz4.overpass-api.de/api"*/],
                 "output"      : [
                         "folder": directory]
+                        /*[
+                        "database":
+                                ["user"    : "ebocher",
+                                 "password": "k@ndinsky5622",
+                                 "url"     : "postgis://localhost:5432/orbisgis_db",
+                                 "tables"  : [
+                                         "rsu_indicators"         : "rsu_indicators",
+                                         "rsu_lcz"                : "rsu_lcz",
+                                         "zone"                   : "zone"]]]*/
                 ,
                 "parameters"  :
                         ["distance"             : 0,

worldpoptools/src/test/groovy/org/orbisgis/geoclimate/worldpoptools/WorldPopExtractTest.groovy

@@ -57,19 +57,21 @@ class WorldPopExtractTest {
      */
     @Test
     void extractGrid() {
-        def outputGridFile = new File(folder, "extractGrid.asc")
-        if (outputGridFile.exists()) {
-            outputGridFile.delete()
-        }
         def coverageId = "wpGlobal:ppp_2018"
-        def bbox = [47.63324, -2.78087, 47.65749, -2.75979]
-        if (WorldPopTools.Extract.grid(coverageId, bbox, outputGridFile)) {
-            AscReaderDriver ascReaderDriver = new AscReaderDriver()
-            ascReaderDriver.setAs3DPoint(false)
-            ascReaderDriver.setEncoding("UTF-8")
-            ascReaderDriver.setDeleteTable(true)
-            ascReaderDriver.read(h2GIS.getConnection(), outputGridFile, new EmptyProgressVisitor(), "grid", 4326)
-            assertEquals(720, h2GIS.getSpatialTable("grid").rowCount)
+        if (WorldPopExtract.Extract.isCoverageAvailable(coverageId)) {
+            def outputGridFile = new File(folder, "extractGrid.asc")
+            if (outputGridFile.exists()) {
+                outputGridFile.delete()
+            }
+            def bbox = [47.63324, -2.78087, 47.65749, -2.75979]
+            if (WorldPopTools.Extract.grid(coverageId, bbox, outputGridFile)) {
+                AscReaderDriver ascReaderDriver = new AscReaderDriver()
+                ascReaderDriver.setAs3DPoint(false)
+                ascReaderDriver.setEncoding("UTF-8")
+                ascReaderDriver.setDeleteTable(true)
+                ascReaderDriver.read(h2GIS.getConnection(), outputGridFile, new EmptyProgressVisitor(), "grid", 4326)
+                assertEquals(720, h2GIS.getSpatialTable("grid").rowCount)
+            }
         }
     }
 
@@ -107,7 +109,6 @@ class WorldPopExtractTest {
      */
     @Test
     void testCoverageAvailable() {
-        assertFalse(WorldPopExtract.Extract.isCoverageAvailable("wpGlobal:ppp_2050"))
-        assertTrue(WorldPopExtract.Extract.isCoverageAvailable("wpGlobal:ppp_2018"))
+        assertFalse(WorldPopExtract.Extract.isCoverageAvailable("wpGlobal:ppp_2150"))
     }
 }