From b3f6e9eb5430dff5a94bedb18b495e81a6f02bd4 Mon Sep 17 00:00:00 2001 From: ebocher Date: Thu, 12 Oct 2023 07:22:49 +0200 Subject: [PATCH 1/2] Fix a zero DIV when a building doesn't have any pop. Improve OSMTools --- .../geoindicators/BuildingIndicators.groovy | 4 +- .../osm/InputDataFormattingTest.groovy | 2 +- .../osm/InputDataLoadingTest.groovy | 2 +- .../geoclimate/osm/WorflowOSMTest.groovy | 11 ++-- .../geoclimate/osmtools/Loader.groovy | 8 +-- .../geoclimate/osmtools/OSMTools.groovy | 14 +++-- .../osmtools/utils/TransformUtils.groovy | 4 +- .../geoclimate/osmtools/LoaderTest.groovy | 51 +++++++++---------- .../geoclimate/osmtools/TransformTest.groovy | 17 ++++++- 9 files changed, 63 insertions(+), 50 deletions(-) diff --git a/geoindicators/src/main/groovy/org/orbisgis/geoclimate/geoindicators/BuildingIndicators.groovy b/geoindicators/src/main/groovy/org/orbisgis/geoclimate/geoindicators/BuildingIndicators.groovy index 9c092123cc..d94c77981c 100644 --- a/geoindicators/src/main/groovy/org/orbisgis/geoclimate/geoindicators/BuildingIndicators.groovy +++ b/geoindicators/src/main/groovy/org/orbisgis/geoclimate/geoindicators/BuildingIndicators.groovy @@ -520,7 +520,9 @@ String buildingPopulation(JdbcDataSource datasource, String inputBuilding, Strin create index on $inputBuildingTableName_area_sum($ID_POP); create table $inputBuildingTableName_pop_sum as select a.$ID_BUILDING, ${sum_popColumns.join(",")} - from $inputBuildingTableName_pop as a, $inputBuildingTableName_area_sum as b where a.$ID_POP=b.$ID_POP group by $ID_BUILDING; + from $inputBuildingTableName_pop as a, $inputBuildingTableName_area_sum as b where a.$ID_POP=b.$ID_POP and + b.sum_area_building!=0 + group by $ID_BUILDING; CREATE INDEX ON $inputBuildingTableName_pop_sum ($ID_BUILDING); DROP TABLE IF EXISTS $outputTableName; CREATE TABLE $outputTableName AS SELECT a.*, ${popColumns.join(",")} from $inputBuilding a diff --git a/osm/src/test/groovy/org/orbisgis/geoclimate/osm/InputDataFormattingTest.groovy b/osm/src/test/groovy/org/orbisgis/geoclimate/osm/InputDataFormattingTest.groovy index 7c8ef62e05..60feb893c8 100644 --- a/osm/src/test/groovy/org/orbisgis/geoclimate/osm/InputDataFormattingTest.groovy +++ b/osm/src/test/groovy/org/orbisgis/geoclimate/osm/InputDataFormattingTest.groovy @@ -311,7 +311,7 @@ class InputDataFormattingTest { zoneToExtract = "Göteborgs Stad" - zoneToExtract = "Ploemeur" + zoneToExtract = "Barcelona Eixample" //zoneToExtract = [69.24666, 28.3359, 69.32999, 28.41923] diff --git a/osm/src/test/groovy/org/orbisgis/geoclimate/osm/InputDataLoadingTest.groovy b/osm/src/test/groovy/org/orbisgis/geoclimate/osm/InputDataLoadingTest.groovy index 0fa82566c7..d940cb6fc7 100644 --- a/osm/src/test/groovy/org/orbisgis/geoclimate/osm/InputDataLoadingTest.groovy +++ b/osm/src/test/groovy/org/orbisgis/geoclimate/osm/InputDataLoadingTest.groovy @@ -73,7 +73,7 @@ class InputDataLoadingTest { @Test @Disabled void createGISLayersFromFileTestIntegration() { - Map extract = OSM.InputDataLoading.createGISLayers(h2GIS, "/tmp/map.osm", 2154) + Map extract = OSM.InputDataLoading.createGISLayers(h2GIS, "/tmp/3be46f5e1060bd53fe7c5b97a04102b3f36fa6f6257c87664dbaec3e954879ab.osm", 2154) //h2GIS.getTable(extract.vegetation).save("./target/osm_vegetation.shp") diff --git a/osm/src/test/groovy/org/orbisgis/geoclimate/osm/WorflowOSMTest.groovy b/osm/src/test/groovy/org/orbisgis/geoclimate/osm/WorflowOSMTest.groovy index d68a1e9305..eb9d0c5b8f 100644 --- a/osm/src/test/groovy/org/orbisgis/geoclimate/osm/WorflowOSMTest.groovy +++ b/osm/src/test/groovy/org/orbisgis/geoclimate/osm/WorflowOSMTest.groovy @@ -652,6 +652,7 @@ class WorflowOSMTest extends WorkflowAbstractTest { dirFile.delete() dirFile.mkdir() + def location = "Redon" //def nominatim = OSMTools.Utilities.getNominatimData("Lorient") @@ -663,7 +664,7 @@ class WorflowOSMTest extends WorkflowAbstractTest { "delete": false ], "input" : [ - "locations": [[47.081403,-1.481352,47.110619,-1.423845]],//["Pont-de-Veyle"],//[nominatim["bbox"]],//["Lorient"], + "locations": [location],//["Pont-de-Veyle"],//[nominatim["bbox"]],//["Lorient"], "area": 2800, /*"timeout":182, "maxsize": 536870918, @@ -683,12 +684,12 @@ class WorflowOSMTest extends WorkflowAbstractTest { "BUILDING_TYPE_FRACTION","WATER_FRACTION","VEGETATION_FRACTION", "ROAD_FRACTION", "IMPERVIOUS_FRACTION", "BUILDING_HEIGHT_WEIGHTED", "BUILDING_SURFACE_DENSITY", "SEA_LAND_FRACTION", - "ASPECT_RATIO","SVF", + "ASPECT_RATIO",//"SVF", "HEIGHT_OF_ROUGHNESS_ELEMENTS", "TERRAIN_ROUGHNESS_CLASS"] - ], "worldpop_indicators": false, - "road_traffic" : false, + ], "worldpop_indicators": true, + "road_traffic" : true, "noise_indicators" : [ - "ground_acoustic": false + "ground_acoustic": true ] ] ] diff --git a/osmtools/src/main/groovy/org/orbisgis/geoclimate/osmtools/Loader.groovy b/osmtools/src/main/groovy/org/orbisgis/geoclimate/osmtools/Loader.groovy index 7d651c9cb4..07e47a899c 100644 --- a/osmtools/src/main/groovy/org/orbisgis/geoclimate/osmtools/Loader.groovy +++ b/osmtools/src/main/groovy/org/orbisgis/geoclimate/osmtools/Loader.groovy @@ -98,8 +98,8 @@ Map fromArea(JdbcDataSource datasource, Object filterArea, float distance = 0) { if (load(datasource, osmTablesPrefix, extract)) { info "Loading OSM data from the area $filterArea" return [zone : outputZoneTable, - zoneEnvelopeTableName: outputZoneEnvelopeTable, - osmTablesPrefix : osmTablesPrefix, + envelope : outputZoneEnvelopeTable, + prefix : osmTablesPrefix, epsg : epsg] } else { error "Cannot load the OSM data from the area $filterArea" @@ -175,8 +175,8 @@ Map fromPlace(JdbcDataSource datasource, String placeName, float distance = 0) { if (load(datasource, osmTablesPrefix, extract)) { info "Loading OSM data from the place $placeName" return [zone : outputZoneTable, - zoneEnvelopeTableName: outputZoneEnvelopeTable, - osmTablesPrefix : osmTablesPrefix] + envelope: outputZoneEnvelopeTable, + prefix : osmTablesPrefix] } else { error "Cannot load the OSM data from the place $placeName" } diff --git a/osmtools/src/main/groovy/org/orbisgis/geoclimate/osmtools/OSMTools.groovy b/osmtools/src/main/groovy/org/orbisgis/geoclimate/osmtools/OSMTools.groovy index 7c6d77472b..5bcd0c55ce 100644 --- a/osmtools/src/main/groovy/org/orbisgis/geoclimate/osmtools/OSMTools.groovy +++ b/osmtools/src/main/groovy/org/orbisgis/geoclimate/osmtools/OSMTools.groovy @@ -19,10 +19,8 @@ */ package org.orbisgis.geoclimate.osmtools -import org.orbisgis.geoclimate.osmtools.Loader as LOADER -import org.orbisgis.geoclimate.osmtools.Transform as TRANSFORM -import org.orbisgis.geoclimate.osmtools.utils.TransformUtils as TRANSFORM_UTILS -import org.orbisgis.geoclimate.osmtools.utils.Utilities as UTILITIES +import org.orbisgis.geoclimate.osmtools.utils.TransformUtils +import org.orbisgis.geoclimate.osmtools.utils.Utilities import org.orbisgis.geoclimate.utils.AbstractScript /** @@ -34,10 +32,10 @@ import org.orbisgis.geoclimate.utils.AbstractScript */ abstract class OSMTools extends AbstractScript { - def static Loader = new LOADER() - def static Transform = new TRANSFORM() - def static Utilities = new UTILITIES() - def static TransformUtils = new TRANSFORM_UTILS() + static Loader Loader = new Loader() + static Transform Transform = new Transform() + static Utilities Utilities = new Utilities() + static TransformUtils TransformUtils = new TransformUtils() OSMTools() { super(OSMTools.class) diff --git a/osmtools/src/main/groovy/org/orbisgis/geoclimate/osmtools/utils/TransformUtils.groovy b/osmtools/src/main/groovy/org/orbisgis/geoclimate/osmtools/utils/TransformUtils.groovy index 60c0bbe66a..8d24ed87b4 100644 --- a/osmtools/src/main/groovy/org/orbisgis/geoclimate/osmtools/utils/TransformUtils.groovy +++ b/osmtools/src/main/groovy/org/orbisgis/geoclimate/osmtools/utils/TransformUtils.groovy @@ -94,10 +94,10 @@ String toPolygonOrLine(GeometryTypes type, JdbcDataSource datasource, String osm error "Invalid EPSG code : $epsgCode" return } - if (!tags && !columnsToKeep) { + /*if (!tags && !columnsToKeep) { error "No tags nor columns to keep" return - } + }*/ //Start the transformation def outputTableName = postfix("OSM_${type.name()}_") diff --git a/osmtools/src/test/groovy/org/orbisgis/geoclimate/osmtools/LoaderTest.groovy b/osmtools/src/test/groovy/org/orbisgis/geoclimate/osmtools/LoaderTest.groovy index 2bdd405e61..ff7e80b198 100644 --- a/osmtools/src/test/groovy/org/orbisgis/geoclimate/osmtools/LoaderTest.groovy +++ b/osmtools/src/test/groovy/org/orbisgis/geoclimate/osmtools/LoaderTest.groovy @@ -98,10 +98,10 @@ class LoaderTest extends AbstractOSMToolsTest { assertFalse r.isEmpty() assertTrue r.containsKey("zone") assertTrue Pattern.compile("ZONE_$uuidRegex").matcher(r.zone as String).matches() - assertTrue r.containsKey("zoneEnvelopeTableName") - assertTrue Pattern.compile("ZONE_ENVELOPE_$uuidRegex").matcher(r.zoneEnvelopeTableName as String).matches() - assertTrue r.containsKey("osmTablesPrefix") - assertTrue Pattern.compile("OSM_DATA_$uuidRegex").matcher(r.osmTablesPrefix as String).matches() + assertTrue r.containsKey("envelope") + assertTrue Pattern.compile("ZONE_ENVELOPE_$uuidRegex").matcher(r.envelope as String).matches() + assertTrue r.containsKey("prefix") + assertTrue Pattern.compile("OSM_DATA_$uuidRegex").matcher(r.prefix as String).matches() assertTrue r.containsKey("epsg") assertEquals 4326, r.epsg @@ -112,7 +112,7 @@ class LoaderTest extends AbstractOSMToolsTest { zone.next() assertEquals "POLYGON ((-3.076869 48.733493, -3.076869 48.733995, -3.075829 48.733995, -3.075829 48.733493, -3.076869 48.733493))", zone.getGeometry(1).toText() - def zoneEnv = ds.getSpatialTable(r.zoneEnvelopeTableName) + def zoneEnv = ds.getSpatialTable(r.envelope) assertEquals 1, zoneEnv.rowCount assertEquals 1, zoneEnv.getColumnCount() assertTrue zoneEnv.columns.contains("THE_GEOM") @@ -126,10 +126,10 @@ class LoaderTest extends AbstractOSMToolsTest { assertFalse r.isEmpty() assertTrue r.containsKey("zone") assertTrue Pattern.compile("ZONE_$uuidRegex").matcher(r.zone as String).matches() - assertTrue r.containsKey("zoneEnvelopeTableName") - assertTrue Pattern.compile("ZONE_ENVELOPE_$uuidRegex").matcher(r.zoneEnvelopeTableName as String).matches() - assertTrue r.containsKey("osmTablesPrefix") - assertTrue Pattern.compile("OSM_DATA_$uuidRegex").matcher(r.osmTablesPrefix as String).matches() + assertTrue r.containsKey("envelope") + assertTrue Pattern.compile("ZONE_ENVELOPE_$uuidRegex").matcher(r.envelope as String).matches() + assertTrue r.containsKey("prefix") + assertTrue Pattern.compile("OSM_DATA_$uuidRegex").matcher(r.prefix as String).matches() assertTrue r.containsKey("epsg") assertEquals 4326, r.epsg @@ -140,7 +140,7 @@ class LoaderTest extends AbstractOSMToolsTest { zone.next() assertEquals "POLYGON ((-3.076869 48.733493, -3.076869 48.733995, -3.075829 48.733995, -3.075829 48.733493, -3.076869 48.733493))", zone.getGeometry(1).toText() - zoneEnv = ds.getSpatialTable(r.zoneEnvelopeTableName) + zoneEnv = ds.getSpatialTable(r.envelope) assertEquals 1, zoneEnv.rowCount assertEquals 1, zoneEnv.getColumnCount() assertTrue zoneEnv.columns.contains("THE_GEOM") @@ -170,10 +170,10 @@ class LoaderTest extends AbstractOSMToolsTest { assertFalse r.isEmpty() assertTrue r.containsKey("zone") assertTrue Pattern.compile("ZONE_$uuidRegex").matcher(r.zone as String).matches() - assertTrue r.containsKey("zoneEnvelopeTableName") - assertTrue Pattern.compile("ZONE_ENVELOPE_$uuidRegex").matcher(r.zoneEnvelopeTableName as String).matches() - assertTrue r.containsKey("osmTablesPrefix") - assertTrue Pattern.compile("OSM_DATA_$uuidRegex").matcher(r.osmTablesPrefix as String).matches() + assertTrue r.containsKey("envelope") + assertTrue Pattern.compile("ZONE_ENVELOPE_$uuidRegex").matcher(r.envelope as String).matches() + assertTrue r.containsKey("prefix") + assertTrue Pattern.compile("OSM_DATA_$uuidRegex").matcher(r.prefix as String).matches() def zone = ds.getSpatialTable(r.zone) assertEquals 1, zone.rowCount @@ -182,7 +182,7 @@ class LoaderTest extends AbstractOSMToolsTest { zone.next() assertEquals wktReader.read("POLYGON ((0 0, 2 0, 2 2, 0 2, 0 0))"), zone.getGeometry(1) - def zoneEnv = ds.getSpatialTable(r.zoneEnvelopeTableName) + def zoneEnv = ds.getSpatialTable(r.envelope) assertEquals 1, zoneEnv.rowCount assertEquals 1, zoneEnv.getColumnCount() assertTrue zoneEnv.columns.contains("THE_GEOM") @@ -196,10 +196,10 @@ class LoaderTest extends AbstractOSMToolsTest { assertFalse r.isEmpty() assertTrue r.containsKey("zone") assertTrue Pattern.compile("ZONE_$uuidRegex").matcher(r.zone as String).matches() - assertTrue r.containsKey("zoneEnvelopeTableName") - assertTrue Pattern.compile("ZONE_ENVELOPE_$uuidRegex").matcher(r.zoneEnvelopeTableName as String).matches() - assertTrue r.containsKey("osmTablesPrefix") - assertTrue Pattern.compile("OSM_DATA_$uuidRegex").matcher(r.osmTablesPrefix as String).matches() + assertTrue r.containsKey("envelope") + assertTrue Pattern.compile("ZONE_ENVELOPE_$uuidRegex").matcher(r.envelope as String).matches() + assertTrue r.containsKey("prefix") + assertTrue Pattern.compile("OSM_DATA_$uuidRegex").matcher(r.prefix as String).matches() zone = ds.getSpatialTable(r.zone) assertEquals 1, zone.rowCount @@ -208,7 +208,7 @@ class LoaderTest extends AbstractOSMToolsTest { zone.next() assertEquals wktReader.read("POLYGON ((0 0, 0 2, 2 2, 2 0, 0 0))"), zone.getGeometry(1) - zoneEnv = ds.getSpatialTable(r.zoneEnvelopeTableName) + zoneEnv = ds.getSpatialTable(r.envelope) assertEquals 1, zoneEnv.rowCount assertEquals 1, zoneEnv.getColumnCount() assertTrue zoneEnv.columns.contains("THE_GEOM") @@ -227,9 +227,9 @@ class LoaderTest extends AbstractOSMToolsTest { Map r = OSMTools.Loader.fromPlace(ds, placeName) assertFalse r.isEmpty() assertTrue r.containsKey("zone") - assertTrue r.containsKey("zoneEnvelopeTableName") - assertTrue r.containsKey("osmTablesPrefix") - assertTrue Pattern.compile("OSM_DATA_$formattedPlaceName$uuidRegex").matcher(r.osmTablesPrefix as String).matches() + assertTrue r.containsKey("envelope") + assertTrue r.containsKey("prefix") + assertTrue Pattern.compile("OSM_DATA_$formattedPlaceName$uuidRegex").matcher(r.prefix as String).matches() def zone = ds.getSpatialTable(r.zone) assertEquals 1, zone.rowCount @@ -239,7 +239,7 @@ class LoaderTest extends AbstractOSMToolsTest { zone.next() assertNotNull zone.getGeometry(1) - def zoneEnv = ds.getSpatialTable(r.zoneEnvelopeTableName) + def zoneEnv = ds.getSpatialTable(r.envelope) assertEquals 1, zoneEnv.rowCount assertEquals 2, zoneEnv.getColumnCount() assertTrue zoneEnv.columns.contains("THE_GEOM") @@ -269,8 +269,7 @@ class LoaderTest extends AbstractOSMToolsTest { assertTrue zone.columns.contains("ID_ZONE") zone.next() assertNotNull zone.getGeometry(1) - - def zoneEnv = ds.getSpatialTable(r.zoneEnvelopeTableName) + def zoneEnv = ds.getSpatialTable(r.envelope) assertEquals 1, zoneEnv.rowCount assertEquals 2, zoneEnv.getColumnCount() assertTrue zoneEnv.columns.contains("THE_GEOM") diff --git a/osmtools/src/test/groovy/org/orbisgis/geoclimate/osmtools/TransformTest.groovy b/osmtools/src/test/groovy/org/orbisgis/geoclimate/osmtools/TransformTest.groovy index 6b36ce5463..8c98fc7058 100644 --- a/osmtools/src/test/groovy/org/orbisgis/geoclimate/osmtools/TransformTest.groovy +++ b/osmtools/src/test/groovy/org/orbisgis/geoclimate/osmtools/TransformTest.groovy @@ -138,7 +138,7 @@ class TransformTest extends AbstractOSMToolsTest { def columnsToKeep = [] LOGGER.warn("An error will be thrown next") - assertNull OSMTools.Transform.toLines(null, prefix, epsgCode, tags, columnsToKeep) + assertNull OSMTools.Transform.toLines(null, prefix, epsgCode, tags, columnsToKeep) LOGGER.warn("An error will be thrown next") assertNull OSMTools.Transform.toLines(ds, prefix, -1, tags, columnsToKeep) @@ -875,7 +875,20 @@ class TransformTest extends AbstractOSMToolsTest { } } } + } - + /** + * It uses for test purpose + */ + @Disabled + @Test + void testTransformForDebug() { + H2GIS h2GIS = H2GIS.open("/tmp/geoclimate;AUTO_SERVER=TRUE") + Map r = OSMTools.Loader.fromArea(h2GIS, [48.733493,-3.076869,48.733995,-3.075829]) + println(r) + def lines = OSMTools.Transform.toPolygons(h2GIS, r.prefix,4326, [], []) + h2GIS.save(lines, "/tmp/building.geojson") } + + } From 5d42510189f97100ee5922fded6802a544bcc57d Mon Sep 17 00:00:00 2001 From: ebocher Date: Thu, 12 Oct 2023 10:35:20 +0200 Subject: [PATCH 2/2] Enable tags and columns test --- .../orbisgis/geoclimate/osmtools/utils/TransformUtils.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/osmtools/src/main/groovy/org/orbisgis/geoclimate/osmtools/utils/TransformUtils.groovy b/osmtools/src/main/groovy/org/orbisgis/geoclimate/osmtools/utils/TransformUtils.groovy index 8d24ed87b4..60c0bbe66a 100644 --- a/osmtools/src/main/groovy/org/orbisgis/geoclimate/osmtools/utils/TransformUtils.groovy +++ b/osmtools/src/main/groovy/org/orbisgis/geoclimate/osmtools/utils/TransformUtils.groovy @@ -94,10 +94,10 @@ String toPolygonOrLine(GeometryTypes type, JdbcDataSource datasource, String osm error "Invalid EPSG code : $epsgCode" return } - /*if (!tags && !columnsToKeep) { + if (!tags && !columnsToKeep) { error "No tags nor columns to keep" return - }*/ + } //Start the transformation def outputTableName = postfix("OSM_${type.name()}_")