From 42e5159d82f82e307c334769bfb990867ea75e35 Mon Sep 17 00:00:00 2001 From: ebocher Date: Thu, 13 Jun 2024 14:18:02 +0200 Subject: [PATCH 1/3] Fix OSM water infinite process on large water geomtries --- .../geoclimate/osm/InputDataLoading.groovy | 2 +- .../osm/InputDataFormattingTest.groovy | 10 +++++----- .../geoclimate/osm/WorflowOSMTest.groovy | 14 ++++++++------ .../geoclimate/osmtools/Transform.groovy | 17 +++++------------ 4 files changed, 19 insertions(+), 24 deletions(-) diff --git a/osm/src/main/groovy/org/orbisgis/geoclimate/osm/InputDataLoading.groovy b/osm/src/main/groovy/org/orbisgis/geoclimate/osm/InputDataLoading.groovy index 5f4cf5ad59..374130fb3e 100644 --- a/osm/src/main/groovy/org/orbisgis/geoclimate/osm/InputDataLoading.groovy +++ b/osm/src/main/groovy/org/orbisgis/geoclimate/osm/InputDataLoading.groovy @@ -221,7 +221,7 @@ Map createGISLayers(JdbcDataSource datasource, String osmFilePath, parametersMap = readJSONParameters(paramsDefaultFile) tags = parametersMap.get("tags") columnsToKeep = parametersMap.get("columns") - String water = OSMTools.Transform.toPolygons(datasource, prefix, epsg, tags, columnsToKeep, geometry, true) + String water = OSMTools.Transform.toPolygons(datasource, prefix, epsg, tags, columnsToKeep, geometry, false) debug "Create the water layer" if (water) { outputHydroTableName = postfix("OSM_WATER") diff --git a/osm/src/test/groovy/org/orbisgis/geoclimate/osm/InputDataFormattingTest.groovy b/osm/src/test/groovy/org/orbisgis/geoclimate/osm/InputDataFormattingTest.groovy index a77854ca4a..1ffd7c73eb 100644 --- a/osm/src/test/groovy/org/orbisgis/geoclimate/osm/InputDataFormattingTest.groovy +++ b/osm/src/test/groovy/org/orbisgis/geoclimate/osm/InputDataFormattingTest.groovy @@ -293,7 +293,7 @@ class InputDataFormattingTest { //def nominatim = org.orbisgis.geoclimate.osmtools.OSMTools.Utilities.getNominatimData(zoneToExtract) // zoneToExtract = nominatim.bbox - zoneToExtract = [43.824643, 4.383599, 43.827271, 4.388207] + zoneToExtract = [62.2, 28.2, 62.4, 28.4] //zoneToExtract =[44.795480,12.323227,45.004622,12.627411] Map extractData = OSM.InputDataLoading.extractAndCreateGISLayers(h2GIS, zoneToExtract) @@ -311,7 +311,7 @@ class InputDataFormattingTest { //Zone envelope h2GIS.getTable(extractData.zone_envelope).save("${file.absolutePath + File.separator}zone_envelope.fgb", true) -/* + //Urban Areas def inputUrbanAreas = OSM.InputDataFormatting.formatUrbanAreas(h2GIS, extractData.urban_areas,extractData.zone) @@ -343,17 +343,17 @@ class InputDataFormattingTest { h2GIS,extractData.vegetation,extractData.zone_envelope) h2GIS.save(inputVegetationTableName,"${file.absolutePath + File.separator}vegetation.fgb", true) - println("Vegetation formatted")*/ + println("Vegetation formatted") //Hydrography def inputWaterTableName = OSM.InputDataFormatting.formatWaterLayer(h2GIS, extractData.water, extractData.zone_envelope) //Impervious - /* String imperviousTable = OSM.InputDataFormatting.formatImperviousLayer(h2GIS, extractData.impervious, + String imperviousTable = OSM.InputDataFormatting.formatImperviousLayer(h2GIS, extractData.impervious, extractData.zone_envelope) h2GIS.save(imperviousTable,"${file.absolutePath + File.separator}impervious.fgb", true) - println("Impervious formatted")*/ + println("Impervious formatted") //Save coastlines to debug h2GIS.save(extractData.coastline, "${file.absolutePath + File.separator}coastlines.fgb", true) diff --git a/osm/src/test/groovy/org/orbisgis/geoclimate/osm/WorflowOSMTest.groovy b/osm/src/test/groovy/org/orbisgis/geoclimate/osm/WorflowOSMTest.groovy index fcb92a7dc9..a390cc0ad7 100644 --- a/osm/src/test/groovy/org/orbisgis/geoclimate/osm/WorflowOSMTest.groovy +++ b/osm/src/test/groovy/org/orbisgis/geoclimate/osm/WorflowOSMTest.groovy @@ -649,11 +649,12 @@ class WorflowOSMTest extends WorkflowAbstractTest { File dirFile = new File(directory) dirFile.delete() dirFile.mkdir() - def location = "Mâcon" + def location = "Genève, Suisse" //def nominatim = org.orbisgis.geoclimate.osmtools.OSMTools.Utilities.getNominatimData(location) def grid_size = 100 //location = nominatim.bbox - // location=[44.795480,12.323227,45.004622,12.627411] + location=[46.178404,6.095524,46.222959,6.190109] + //location =[62.2, 28.2, 62.4, 28.4] def osm_parmeters = [ "description" : "Example of configuration file to run the OSM workflow and store the result in a folder", "geoclimatedb": [ @@ -674,7 +675,7 @@ class WorflowOSMTest extends WorkflowAbstractTest { "parameters" : ["distance" : 0, "rsu_indicators" : [ - "indicatorUse": ["LCZ"] //, "UTRF", "TEB"] + "indicatorUse": ["LCZ", "TEB"] //, "UTRF", "TEB"] ], "grid_indicators" : [ "x_size" : grid_size, @@ -692,9 +693,10 @@ class WorflowOSMTest extends WorkflowAbstractTest { // "HEIGHT_OF_ROUGHNESS_ELEMENTS", "TERRAIN_ROUGHNESS_CLASS", "URBAN_SPRAWL_AREAS", "URBAN_SPRAWL_DISTANCES", - "URBAN_SPRAWL_COOL_DISTANCE"], - "lcz_lod":1 - ], "worldpop_indicators": true/* + "URBAN_SPRAWL_COOL_DISTANCE"]/*, + "lcz_lod":1*/ + ], "worldpop_indicators": true + /* "road_traffic" : true, "noise_indicators" : [ diff --git a/osmtools/src/main/groovy/org/orbisgis/geoclimate/osmtools/Transform.groovy b/osmtools/src/main/groovy/org/orbisgis/geoclimate/osmtools/Transform.groovy index a3663d7402..3e964cd09e 100644 --- a/osmtools/src/main/groovy/org/orbisgis/geoclimate/osmtools/Transform.groovy +++ b/osmtools/src/main/groovy/org/orbisgis/geoclimate/osmtools/Transform.groovy @@ -544,27 +544,20 @@ def extractRelationsAsPolygons(JdbcDataSource datasource, String osmTablesPrefix debug "Build all polygon relations" def relationsMpHoles = postfix "RELATIONS_MP_HOLES" + datasource.save(relationsPolygonsOuterExploded, "/tmp/outer.geojson", true) + datasource.save(relationsPolygonsInnerExploded, "/tmp/inner.geojson", true) datasource """ CREATE SPATIAL INDEX ON $relationsPolygonsOuterExploded (the_geom); CREATE SPATIAL INDEX ON $relationsPolygonsInnerExploded (the_geom); CREATE INDEX ON $relationsPolygonsOuterExploded(id_relation); CREATE INDEX ON $relationsPolygonsInnerExploded(id_relation); DROP TABLE IF EXISTS $relationsMpHoles; - CREATE TABLE $relationsMpHoles AS ( + CREATE TABLE $relationsMpHoles AS SELECT ST_MAKEPOLYGON(ST_EXTERIORRING(a.the_geom), ST_ACCUM(b.the_geom)) AS the_geom, a.ID_RELATION FROM $relationsPolygonsOuterExploded AS a LEFT JOIN $relationsPolygonsInnerExploded AS b - ON( - a.the_geom && b.the_geom - AND st_contains(a.the_geom, b.THE_GEOM) - AND a.ID_RELATION=b.ID_RELATION) - GROUP BY a.the_geom, a.id_relation) - UNION( - SELECT a.the_geom as the_geom , a.ID_RELATION - FROM $relationsPolygonsOuterExploded AS a - LEFT JOIN $relationsPolygonsInnerExploded AS b - ON a.id_relation=b.id_relation - WHERE b.id_relation IS NULL); + ON( a.ID_RELATION=b.ID_RELATION) + GROUP BY a.the_geom, a.id_relation; CREATE INDEX ON $relationsMpHoles(id_relation); """.toString() From ad3264219b0bcd7134a2109aa2c17de754cb77b6 Mon Sep 17 00:00:00 2001 From: ebocher Date: Thu, 13 Jun 2024 14:21:42 +0200 Subject: [PATCH 2/3] Remove methods used for debug purpose --- .../groovy/org/orbisgis/geoclimate/osmtools/Transform.groovy | 2 -- 1 file changed, 2 deletions(-) diff --git a/osmtools/src/main/groovy/org/orbisgis/geoclimate/osmtools/Transform.groovy b/osmtools/src/main/groovy/org/orbisgis/geoclimate/osmtools/Transform.groovy index 3e964cd09e..28bc44a46f 100644 --- a/osmtools/src/main/groovy/org/orbisgis/geoclimate/osmtools/Transform.groovy +++ b/osmtools/src/main/groovy/org/orbisgis/geoclimate/osmtools/Transform.groovy @@ -544,8 +544,6 @@ def extractRelationsAsPolygons(JdbcDataSource datasource, String osmTablesPrefix debug "Build all polygon relations" def relationsMpHoles = postfix "RELATIONS_MP_HOLES" - datasource.save(relationsPolygonsOuterExploded, "/tmp/outer.geojson", true) - datasource.save(relationsPolygonsInnerExploded, "/tmp/inner.geojson", true) datasource """ CREATE SPATIAL INDEX ON $relationsPolygonsOuterExploded (the_geom); CREATE SPATIAL INDEX ON $relationsPolygonsInnerExploded (the_geom); From 40a2733e3101a7aa8afb78c2d018804810c0661c Mon Sep 17 00:00:00 2001 From: ebocher Date: Thu, 13 Jun 2024 15:07:26 +0200 Subject: [PATCH 3/3] update cool_distance to cool_dsitances --- .../geoclimate/bdtopo/AbstractBDTopoWorkflow.groovy | 2 +- .../geoclimate/geoindicators/WorkflowGeoIndicators.groovy | 8 ++++---- .../groovy/org/orbisgis/geoclimate/osm/WorkflowOSM.groovy | 2 +- .../org/orbisgis/geoclimate/osm/WorflowOSMTest.groovy | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/bdtopo/src/main/groovy/org/orbisgis/geoclimate/bdtopo/AbstractBDTopoWorkflow.groovy b/bdtopo/src/main/groovy/org/orbisgis/geoclimate/bdtopo/AbstractBDTopoWorkflow.groovy index 62320a27a7..979f5e7bd3 100644 --- a/bdtopo/src/main/groovy/org/orbisgis/geoclimate/bdtopo/AbstractBDTopoWorkflow.groovy +++ b/bdtopo/src/main/groovy/org/orbisgis/geoclimate/bdtopo/AbstractBDTopoWorkflow.groovy @@ -558,7 +558,7 @@ abstract class AbstractBDTopoWorkflow extends BDTopoUtils { "BUILDING_HEIGHT_WEIGHTED", "BUILDING_SURFACE_DENSITY", "BUILDING_HEIGHT_DIST", "FRONTAL_AREA_INDEX", "SEA_LAND_FRACTION", "ASPECT_RATIO", "SVF", "HEIGHT_OF_ROUGHNESS_ELEMENTS", "TERRAIN_ROUGHNESS_CLASS", "URBAN_SPRAWL_AREAS", - "URBAN_SPRAWL_DISTANCES", "URBAN_SPRAWL_COOL_DISTANCE"] + "URBAN_SPRAWL_DISTANCES", "URBAN_SPRAWL_COOL_DISTANCES"] def allowedOutputIndicators = allowed_grid_indicators.intersect(list_indicators*.toUpperCase()) if (allowedOutputIndicators) { //Update the RSU indicators list according the grid indicators diff --git a/geoindicators/src/main/groovy/org/orbisgis/geoclimate/geoindicators/WorkflowGeoIndicators.groovy b/geoindicators/src/main/groovy/org/orbisgis/geoclimate/geoindicators/WorkflowGeoIndicators.groovy index e6bd064b62..ee33d62f91 100644 --- a/geoindicators/src/main/groovy/org/orbisgis/geoclimate/geoindicators/WorkflowGeoIndicators.groovy +++ b/geoindicators/src/main/groovy/org/orbisgis/geoclimate/geoindicators/WorkflowGeoIndicators.groovy @@ -1597,7 +1597,7 @@ String rasterizeIndicators(JdbcDataSource datasource, /* * Make aggregation process with previous grid and current rsu lcz */ - if (list_indicators_upper.intersect(["LCZ_FRACTION", "LCZ_PRIMARY", "URBAN_SPRAWL_AREAS", "URBAN_SPRAWL_DISTANCES", "URBAN_SPRAWL_COOL_DISTANCE"]) && rsu_lcz) { + if (list_indicators_upper.intersect(["LCZ_FRACTION", "LCZ_PRIMARY", "URBAN_SPRAWL_AREAS", "URBAN_SPRAWL_DISTANCES", "URBAN_SPRAWL_COOL_DISTANCES"]) && rsu_lcz) { def indicatorName = "LCZ_PRIMARY" String upperScaleAreaStatistics = Geoindicators.GenericIndicators.upperScaleAreaStatistics( datasource, grid, grid_column_identifier, @@ -2176,7 +2176,7 @@ Map sprawlIndicators(JdbcDataSource datasource, String grid_indicators, String i } //Concert the list of indicators to upper case - allowed_indicators = ["URBAN_SPRAWL_AREAS", "URBAN_SPRAWL_DISTANCES", "URBAN_SPRAWL_COOL_DISTANCE"] + allowed_indicators = ["URBAN_SPRAWL_AREAS", "URBAN_SPRAWL_DISTANCES", "URBAN_SPRAWL_COOL_DISTANCES"] def list_indicators_upper = list_indicators.collect { it.toUpperCase() } def tablesToDrop = [] @@ -2184,7 +2184,7 @@ Map sprawlIndicators(JdbcDataSource datasource, String grid_indicators, String i tablesToJoin.put(grid_indicators, id_grid) String sprawl_areas String cool_areas - if (list_indicators_upper.intersect(["URBAN_SPRAWL_AREAS", "URBAN_SPRAWL_DISTANCES", "URBAN_SPRAWL_COOL_DISTANCE"]) && grid_indicators) { + if (list_indicators_upper.intersect(["URBAN_SPRAWL_AREAS", "URBAN_SPRAWL_DISTANCES", "URBAN_SPRAWL_COOL_DISTANCES"]) && grid_indicators) { sprawl_areas = Geoindicators.SpatialUnits.computeSprawlAreas(datasource, grid_indicators, distance ) } if (sprawl_areas) { @@ -2207,7 +2207,7 @@ Map sprawlIndicators(JdbcDataSource datasource, String grid_indicators, String i } } } - if (list_indicators_upper.contains("URBAN_SPRAWL_COOL_DISTANCE")) { + if (list_indicators_upper.contains("URBAN_SPRAWL_COOL_DISTANCES")) { cool_areas = Geoindicators.SpatialUnits.extractCoolAreas(datasource, grid_indicators, sprawl_areas, (distance / 2) as float) if (cool_areas) { String inverse_cool_areas = Geoindicators.SpatialUnits.inversePolygonsLayer(datasource, sprawl_areas,cool_areas) diff --git a/osm/src/main/groovy/org/orbisgis/geoclimate/osm/WorkflowOSM.groovy b/osm/src/main/groovy/org/orbisgis/geoclimate/osm/WorkflowOSM.groovy index 0563ea9683..0848625790 100644 --- a/osm/src/main/groovy/org/orbisgis/geoclimate/osm/WorkflowOSM.groovy +++ b/osm/src/main/groovy/org/orbisgis/geoclimate/osm/WorkflowOSM.groovy @@ -865,7 +865,7 @@ def extractProcessingParameters(def processing_parameters) throws Exception { "BUILDING_HEIGHT_WEIGHTED", "BUILDING_SURFACE_DENSITY", "BUILDING_HEIGHT_DIST", "FRONTAL_AREA_INDEX", "SEA_LAND_FRACTION", "ASPECT_RATIO", "SVF", "HEIGHT_OF_ROUGHNESS_ELEMENTS", "TERRAIN_ROUGHNESS_CLASS", "URBAN_SPRAWL_AREAS", - "URBAN_SPRAWL_DISTANCES", "URBAN_SPRAWL_COOL_DISTANCE"] + "URBAN_SPRAWL_DISTANCES", "URBAN_SPRAWL_COOL_DISTANCES"] def allowedOutputIndicators = allowed_grid_indicators.intersect(list_indicators*.toUpperCase()) if (allowedOutputIndicators) { //Update the RSU indicators list according the grid indicators diff --git a/osm/src/test/groovy/org/orbisgis/geoclimate/osm/WorflowOSMTest.groovy b/osm/src/test/groovy/org/orbisgis/geoclimate/osm/WorflowOSMTest.groovy index a390cc0ad7..c8992e317e 100644 --- a/osm/src/test/groovy/org/orbisgis/geoclimate/osm/WorflowOSMTest.groovy +++ b/osm/src/test/groovy/org/orbisgis/geoclimate/osm/WorflowOSMTest.groovy @@ -693,7 +693,7 @@ class WorflowOSMTest extends WorkflowAbstractTest { // "HEIGHT_OF_ROUGHNESS_ELEMENTS", "TERRAIN_ROUGHNESS_CLASS", "URBAN_SPRAWL_AREAS", "URBAN_SPRAWL_DISTANCES", - "URBAN_SPRAWL_COOL_DISTANCE"]/*, + "URBAN_SPRAWL_COOL_DISTANCES"]/*, "lcz_lod":1*/ ], "worldpop_indicators": true /*