From 7e6e631b0440db5f6ee58a1a69373274cdd9f421 Mon Sep 17 00:00:00 2001 From: jeremy Date: Mon, 20 Nov 2023 15:13:31 +0100 Subject: [PATCH 1/2] rebase --- .../bdtopo/AbstractBDTopoWorkflow.groovy | 6 + .../geoclimate/bdtopo/InputDataLoading.groovy | 7 +- .../geoclimate/utils/AbstractScript.groovy | 1 + .../geoindicators/SpatialUnits.groovy | 26 ++-- .../WorkflowGeoIndicators.groovy | 18 +-- .../geoindicators/RsuIndicatorsTests.groovy | 2 +- .../geoindicators/SpatialUnitsTests.groovy | 6 +- .../WorkflowGeoIndicatorsTest.groovy | 10 +- .../geoclimate/osm/InputDataFormatting.groovy | 121 +++++++++++----- .../geoclimate/osm/InputDataLoading.groovy | 3 +- .../geoclimate/osm/WorkflowOSM.groovy | 6 + .../geoclimate/osm/buildingParams.json | 19 ++- .../geoclimate/osm/urbanAreasParams.json | 56 ++++++-- .../osm/InputDataFormattingTest.groovy | 6 +- .../osm/InputDataLoadingTest.groovy | 2 +- .../geoclimate/osm/WorflowOSMTest.groovy | 43 +----- .../osm/WorkflowAbstractTest.groovy | 2 +- .../osmtools/utils/Utilities.groovy | 135 ++++++++++++++---- .../geoclimate/osmtools/LoaderTest.groovy | 62 ++++---- .../geoclimate/osmtools/TransformTest.groovy | 21 ++- .../osmtools/utils/UtilitiesTest.groovy | 20 ++- 21 files changed, 385 insertions(+), 187 deletions(-) diff --git a/bdtopo/src/main/groovy/org/orbisgis/geoclimate/bdtopo/AbstractBDTopoWorkflow.groovy b/bdtopo/src/main/groovy/org/orbisgis/geoclimate/bdtopo/AbstractBDTopoWorkflow.groovy index 06ae84953a..816ca83561 100644 --- a/bdtopo/src/main/groovy/org/orbisgis/geoclimate/bdtopo/AbstractBDTopoWorkflow.groovy +++ b/bdtopo/src/main/groovy/org/orbisgis/geoclimate/bdtopo/AbstractBDTopoWorkflow.groovy @@ -451,6 +451,7 @@ abstract class AbstractBDTopoWorkflow extends BDTopoUtils { svfSimplified : true, surface_vegetation: 10000f, surface_hydro : 2500f, + surface_urban_areas: 10000f, snappingTolerance : 0.01f, mapOfWeights : ["sky_view_factor" : 4, "aspect_ratio" : 3, @@ -505,6 +506,11 @@ abstract class AbstractBDTopoWorkflow extends BDTopoUtils { if (surface_hydroP && surface_hydroP in Number) { rsu_indicators_default.surface_hydro = surface_hydroP } + def surface_UrbanAreasP = rsu_indicators.surface_urban_areas + if (surface_UrbanAreasP && surface_UrbanAreasP in Number) { + rsu_indicators_default.surface_urban_areas = surface_UrbanAreasP + } + def svfSimplifiedP = rsu_indicators.svfSimplified if (svfSimplifiedP && svfSimplifiedP in Boolean) { rsu_indicators_default.svfSimplified = svfSimplifiedP diff --git a/bdtopo/src/main/groovy/org/orbisgis/geoclimate/bdtopo/InputDataLoading.groovy b/bdtopo/src/main/groovy/org/orbisgis/geoclimate/bdtopo/InputDataLoading.groovy index 3a608a0eee..0dfb087be4 100644 --- a/bdtopo/src/main/groovy/org/orbisgis/geoclimate/bdtopo/InputDataLoading.groovy +++ b/bdtopo/src/main/groovy/org/orbisgis/geoclimate/bdtopo/InputDataLoading.groovy @@ -142,7 +142,7 @@ def loadV2( String troncon_voie_ferree = tablesExist.get("troncon_voie_ferree") if (!troncon_voie_ferree) { troncon_voie_ferree = "troncon_voie_ferree" - datasource.execute("DROP TABLE IF EXISTS $troncon_voie_ferree; CREATE TABLE $troncon_voie_ferree (THE_GEOM geometry(linestring, $srid), ID varchar, NATURE varchar, POS_SOL integer, FRANCHISST varchar,LARGEUR DOUBLE PRECISION);".toString()) + datasource.execute("DROP TABLE IF EXISTS $troncon_voie_ferree; CREATE TABLE $troncon_voie_ferree (THE_GEOM geometry(linestring, $srid), ID varchar, NATURE varchar, POS_SOL integer, FRANCHISST varchar,LARGEUR DOUBLE PRECISION, NB_VOIES INTEGER);".toString()) } String surface_eau = tablesExist.get("surface_eau") if (!surface_eau) { @@ -252,7 +252,8 @@ def loadV2( datasource.execute(""" DROP TABLE IF EXISTS INPUT_RAIL; CREATE TABLE INPUT_RAIL (THE_GEOM geometry, ID_SOURCE varchar(24), TYPE varchar, ZINDEX integer, CROSSING varchar, WIDTH DOUBLE PRECISION) - AS SELECT ST_FORCE2D(ST_MAKEVALID(a.THE_GEOM)) as the_geom, a.ID, a.NATURE, a.POS_SOL, a.FRANCHISST, CASE WHEN a.NB_VOIES = 0 THEN 1.435 ELSE 1.435 * a.NB_VOIES END FROM $troncon_voie_ferree a, $zoneTable b WHERE a.the_geom && b.the_geom AND ST_INTERSECTS(a.the_geom, b.the_geom) and a.POS_SOL>=0; + AS SELECT ST_FORCE2D(ST_MAKEVALID(a.THE_GEOM)) as the_geom, a.ID, a.NATURE, a.POS_SOL, a.FRANCHISST, + CASE WHEN a.NB_VOIES = 0 THEN 1.435 ELSE 1.435 * a.NB_VOIES END FROM $troncon_voie_ferree a, $zoneTable b WHERE a.the_geom && b.the_geom AND ST_INTERSECTS(a.the_geom, b.the_geom) and a.POS_SOL>=0; """.toString()) @@ -445,7 +446,7 @@ Map loadV3(JdbcDataSource datasource, troncon_de_voie_ferree = "troncon_de_voie_ferree" datasource.execute("""DROP TABLE IF EXISTS $troncon_de_voie_ferree; CREATE TABLE $troncon_de_voie_ferree (THE_GEOM geometry(linestring, $srid), ID varchar, NATURE varchar, - POS_SOL integer, FRANCHISST varchar, LARGEUR DOUBLE PRECISION);""".toString()) + POS_SOL integer, FRANCHISST varchar, LARGEUR DOUBLE PRECISION, NB_VOIES INTEGER);""".toString()) } String surface_hydrographique = tablesExist.get("surface_hydrographique") diff --git a/common-utils/src/main/groovy/org/orbisgis/geoclimate/utils/AbstractScript.groovy b/common-utils/src/main/groovy/org/orbisgis/geoclimate/utils/AbstractScript.groovy index 28c8ce1e2f..32ca86df18 100644 --- a/common-utils/src/main/groovy/org/orbisgis/geoclimate/utils/AbstractScript.groovy +++ b/common-utils/src/main/groovy/org/orbisgis/geoclimate/utils/AbstractScript.groovy @@ -4,6 +4,7 @@ import org.slf4j.Logger abstract class AbstractScript extends Script { + public Logger logger diff --git a/geoindicators/src/main/groovy/org/orbisgis/geoclimate/geoindicators/SpatialUnits.groovy b/geoindicators/src/main/groovy/org/orbisgis/geoclimate/geoindicators/SpatialUnits.groovy index c8844ba123..f73a388d89 100644 --- a/geoindicators/src/main/groovy/org/orbisgis/geoclimate/geoindicators/SpatialUnits.groovy +++ b/geoindicators/src/main/groovy/org/orbisgis/geoclimate/geoindicators/SpatialUnits.groovy @@ -55,13 +55,15 @@ import static org.h2gis.network.functions.ST_ConnectedComponents.getConnectedCom * Expressed in geometry unit of the vegetationTable, default 10000 * @param surface_hydro A double value to select the hydrographic geometry areas. * Expressed in geometry unit of the vegetationTable, default 2500 + * @param surface_urban_areas A double value to select the urban areas. + * Expressed in geometry unit of the urban_areas table, default 10000 * * @return A database table name and the name of the column ID */ String createTSU(JdbcDataSource datasource, String zone, double area = 1f, String road, String rail, String vegetation, String water, String sea_land_mask,String urban_areas, - double surface_vegetation, double surface_hydro, String prefixName) { + double surface_vegetation, double surface_hydro, double surface_urban_areas, String prefixName) { def BASE_NAME = "rsu" debug "Creating the reference spatial units" @@ -71,7 +73,7 @@ String createTSU(JdbcDataSource datasource, String zone, def tsuDataPrepared = prepareTSUData(datasource, zone, road, rail, - vegetation, water, sea_land_mask, urban_areas, surface_vegetation, surface_hydro, prefixName) + vegetation, water, sea_land_mask, urban_areas, surface_vegetation, surface_hydro, surface_urban_areas, prefixName) if (!tsuDataPrepared) { info "Cannot prepare the data for RSU calculation." return @@ -114,7 +116,7 @@ String createTSU(JdbcDataSource datasource, String inputTableName, String inputz error "The input data to compute the TSU cannot be null or empty" return null } - def epsg = datasource.getSpatialTable(inputTableName).srid + def epsg = datasource.getSrid(inputTableName) if (area <= 0) { error "The area value to filter the TSU must be greater to 0" @@ -125,9 +127,9 @@ String createTSU(JdbcDataSource datasource, String inputTableName, String inputz datasource """ DROP TABLE IF EXISTS $outputTableName; CREATE TABLE $outputTableName AS - SELECT EXPLOD_ID AS $COLUMN_ID_NAME, ST_SETSRID(a.the_geom, $epsg) AS the_geom + SELECT EXPLOD_ID AS $COLUMN_ID_NAME, ST_SETSRID(ST_BUFFER(a.the_geom,0.01), $epsg) AS the_geom FROM ST_EXPLODE('( - SELECT ST_BUFFER(ST_POLYGONIZE(ST_UNION(ST_NODE(ST_ACCUM(the_geom)))), -0.01) AS the_geom + SELECT ST_POLYGONIZE(ST_UNION(ST_NODE(ST_ACCUM(the_geom)))) AS the_geom FROM $inputTableName)') AS a, $inputzone AS b WHERE a.the_geom && b.the_geom @@ -139,7 +141,7 @@ String createTSU(JdbcDataSource datasource, String inputTableName, String inputz CREATE TABLE $outputTableName AS SELECT EXPLOD_ID AS $COLUMN_ID_NAME, ST_SETSRID(st_buffer(the_geom, -0.01), $epsg) AS the_geom FROM ST_EXPLODE('( - SELECT ST_BUFFER(ST_POLYGONIZE(ST_UNION(ST_NODE(ST_ACCUM(the_geom)))),-0.01) AS the_geom + SELECT ST_POLYGONIZE(ST_UNION(ST_NODE(ST_ACCUM(the_geom)))) AS the_geom FROM $inputTableName)') where st_area(the_geom) > $area""".toString() } @@ -171,7 +173,8 @@ String createTSU(JdbcDataSource datasource, String inputTableName, String inputz */ String prepareTSUData(JdbcDataSource datasource, String zone, String road, String rail, String vegetation, String water, String sea_land_mask, String urban_areas, - double surface_vegetation, double surface_hydro, String prefixName = "unified_abstract_model") { + double surface_vegetation, double surface_hydro, double surface_urban_areas, String prefixName = "unified_abstract_model") { + if (surface_vegetation <= 100) { error("The surface of vegetation must be greater or equal than 100 m²") return @@ -180,6 +183,12 @@ String prepareTSUData(JdbcDataSource datasource, String zone, String road, Strin error("The surface of water must be greater or equal than 100 m²") return } + + if (surface_urban_areas <= 100) { + error("The surface of urban areas must be greater or equal than 100 m²") + return + } + def BASE_NAME = "prepared_tsu_data" debug "Preparing the abstract model to build the TSU" @@ -326,8 +335,7 @@ String prepareTSUData(JdbcDataSource datasource, String zone, String road, Strin if (water && datasource.hasTable(urban_areas)) { if (datasource.getColumnNames(urban_areas).size() > 0) { debug "Preparing urban areas..." - queryCreateOutputTable += [urban_areas_tmp: "(SELECT ST_ToMultiLine(THE_GEOM) FROM $urban_areas)"] - + queryCreateOutputTable += [urban_areas_tmp: "(SELECT ST_ToMultiLine(THE_GEOM) FROM $urban_areas WHERE st_area(the_geom)>=$surface_urban_areas and type not in ('social_building'))"] } } diff --git a/geoindicators/src/main/groovy/org/orbisgis/geoclimate/geoindicators/WorkflowGeoIndicators.groovy b/geoindicators/src/main/groovy/org/orbisgis/geoclimate/geoindicators/WorkflowGeoIndicators.groovy index f3421baf6d..bb9c4795f6 100644 --- a/geoindicators/src/main/groovy/org/orbisgis/geoclimate/geoindicators/WorkflowGeoIndicators.groovy +++ b/geoindicators/src/main/groovy/org/orbisgis/geoclimate/geoindicators/WorkflowGeoIndicators.groovy @@ -1029,7 +1029,8 @@ Map createUnitsOfAnalysis(JdbcDataSource datasource, String zone, String buildin String road, String rail, String vegetation, String water, String sea_land_mask, String urban_areas, String rsu, double surface_vegetation, - double surface_hydro, double snappingTolerance, List indicatorUse = ["LCZ", "UTRF", "TEB"], String prefixName = "") { + double surface_hydro, double surface_urban_areas, + double snappingTolerance, List indicatorUse = ["LCZ", "UTRF", "TEB"], String prefixName = "") { info "Create the units of analysis..." def idRsu = "id_rsu" def tablesToDrop = [] @@ -1038,7 +1039,7 @@ Map createUnitsOfAnalysis(JdbcDataSource datasource, String zone, String buildin rsu = Geoindicators.SpatialUnits.createTSU(datasource, zone, road, rail, vegetation, water, sea_land_mask, urban_areas, surface_vegetation, - surface_hydro, prefixName) + surface_hydro,surface_urban_areas, prefixName) if (!rsu) { info "Cannot compute the RSU." return @@ -1111,7 +1112,7 @@ Map createUnitsOfAnalysis(JdbcDataSource datasource, String zone, String buildin */ Map getParameters() { return [ - "surface_vegetation" : 10000d, "surface_hydro": 2500d, + "surface_vegetation" : 10000d, "surface_hydro": 2500d, "surface_urban_areas":10000d, "snappingTolerance" : 0.01d, "indicatorUse": ["LCZ", "UTRF", "TEB"], "mapOfWeights" : ["sky_view_factor" : 1, "aspect_ratio": 1, "building_surface_fraction": 1, "impervious_surface_fraction" : 1, "pervious_surface_fraction": 1, @@ -1218,6 +1219,7 @@ Map computeAllGeoIndicators(JdbcDataSource datasource, String zone, String build def surface_vegetation = inputParameters.surface_vegetation def surface_hydro = inputParameters.surface_hydro + def surface_urban_areas = inputParameters.surface_urban_areas def snappingTolerance = inputParameters.snappingTolerance def buildingHeightModelName = inputParameters.buildingHeightModelName def indicatorUse = inputParameters.indicatorUse @@ -1239,7 +1241,7 @@ Map computeAllGeoIndicators(JdbcDataSource datasource, String zone, String build water, impervious, buildingEstimateTableName, sea_land_mask, urban_areas, rsuTable, - surface_vegetation, surface_hydro, + surface_vegetation, surface_hydro,surface_urban_areas, snappingTolerance, buildingHeightModelName, prefixName) if (!estimHeight) { @@ -1280,7 +1282,7 @@ Map computeAllGeoIndicators(JdbcDataSource datasource, String zone, String build vegetation, water, sea_land_mask, "", rsuTable, surface_vegetation, - surface_hydro, snappingTolerance, indicatorUse, + surface_hydro, surface_urban_areas, snappingTolerance, indicatorUse, prefixName) if (!spatialUnitsForCalc) { error "Cannot create the spatial units" @@ -1330,7 +1332,7 @@ Map computeAllGeoIndicators(JdbcDataSource datasource, String zone, String build rail, vegetation, water, sea_land_mask, "","", surface_vegetation, - surface_hydro, snappingTolerance, indicatorUse, + surface_hydro,surface_urban_areas, snappingTolerance, indicatorUse, prefixName) if (!spatialUnits) { error "Cannot create the spatial units" @@ -1366,7 +1368,7 @@ Map estimateBuildingHeight(JdbcDataSource datasource, String zone, String buildi String road, String rail, String vegetation, String water, String impervious, String building_estimate, String sea_land_mask, String urban_areas, String rsu, - double surface_vegetation, double surface_hydro, + double surface_vegetation, double surface_hydro,double surface_urban_areas, double snappingTolerance, String buildingHeightModelName, String prefixName = "") { if (!building_estimate) { error "To estimate the building height a table that contains the list of building to estimate must be provided" @@ -1382,7 +1384,7 @@ Map estimateBuildingHeight(JdbcDataSource datasource, String zone, String buildi Map spatialUnits = createUnitsOfAnalysis(datasource, zone, building, road, rail, vegetation, water, sea_land_mask, urban_areas, rsu, - surface_vegetation, surface_hydro, snappingTolerance, ["UTRF"], + surface_vegetation, surface_hydro,surface_urban_areas, snappingTolerance, ["UTRF"], prefixName) if (!spatialUnits) { error "Cannot create the spatial units" diff --git a/geoindicators/src/test/groovy/org/orbisgis/geoclimate/geoindicators/RsuIndicatorsTests.groovy b/geoindicators/src/test/groovy/org/orbisgis/geoclimate/geoindicators/RsuIndicatorsTests.groovy index db6476f092..6f0e824a05 100644 --- a/geoindicators/src/test/groovy/org/orbisgis/geoclimate/geoindicators/RsuIndicatorsTests.groovy +++ b/geoindicators/src/test/groovy/org/orbisgis/geoclimate/geoindicators/RsuIndicatorsTests.groovy @@ -414,7 +414,7 @@ class RsuIndicatorsTests { def outputTableGeoms = Geoindicators.SpatialUnits.prepareTSUData(h2GIS, 'zone_test', 'road_test', '', 'veget_test', 'hydro_test', "","", - 10000, 2500, "prepare_rsu") + 10000, 2500,10000, "prepare_rsu") assertNotNull h2GIS.getTable(outputTableGeoms) diff --git a/geoindicators/src/test/groovy/org/orbisgis/geoclimate/geoindicators/SpatialUnitsTests.groovy b/geoindicators/src/test/groovy/org/orbisgis/geoclimate/geoindicators/SpatialUnitsTests.groovy index 812b8bf789..29f987da8d 100644 --- a/geoindicators/src/test/groovy/org/orbisgis/geoclimate/geoindicators/SpatialUnitsTests.groovy +++ b/geoindicators/src/test/groovy/org/orbisgis/geoclimate/geoindicators/SpatialUnitsTests.groovy @@ -80,7 +80,7 @@ class SpatialUnitsTests { def outputTableGeoms = Geoindicators.SpatialUnits.prepareTSUData(h2GIS, 'zone_test', 'road_test', 'rail_test', 'veget_test', 'hydro_test', "","", - 10000, 2500, "block") + 10000, 2500,10000, "block") assertNotNull(outputTableGeoms) @@ -103,7 +103,7 @@ class SpatialUnitsTests { def createRSU = Geoindicators.SpatialUnits.createTSU(h2GIS, "zone_test", 'road_test', 'rail_test', 'veget_test', 'hydro_test', - "","", 10000, 2500, "block") + "","", 10000, 2500,10000, "block") assert createRSU assert h2GIS.getSpatialTable(createRSU).save(new File(folder, "rsu.shp").getAbsolutePath(), true) @@ -188,7 +188,7 @@ class SpatialUnitsTests { def outputTableGeoms = Geoindicators.SpatialUnits.prepareTSUData(h2GIS, 'zone_test', 'road_test', 'rail_test', 'veget_test', - 'hydro_test', "", "",10000, 2500, "block") + 'hydro_test', "", "",10000, 2500, 10000, "block") assertNotNull(outputTableGeoms) diff --git a/geoindicators/src/test/groovy/org/orbisgis/geoclimate/geoindicators/WorkflowGeoIndicatorsTest.groovy b/geoindicators/src/test/groovy/org/orbisgis/geoclimate/geoindicators/WorkflowGeoIndicatorsTest.groovy index 9300a07382..c966f7e9e9 100644 --- a/geoindicators/src/test/groovy/org/orbisgis/geoclimate/geoindicators/WorkflowGeoIndicatorsTest.groovy +++ b/geoindicators/src/test/groovy/org/orbisgis/geoclimate/geoindicators/WorkflowGeoIndicatorsTest.groovy @@ -489,15 +489,15 @@ class WorkflowGeoIndicatorsTest { def checkRSUIndicators(def datasource, def rsuIndicatorsTableName) { //Check road_fraction > 0 def countResult = datasource.firstRow("select count(*) as count from ${rsuIndicatorsTableName} WHERE ROAD_FRACTION>0".toString()) - assertEquals(184, countResult.count) + assertEquals(195, countResult.count) //Check building_fraction > 0 countResult = datasource.firstRow("select count(*) as count from ${rsuIndicatorsTableName} WHERE BUILDING_FRACTION>0".toString()) - assertEquals(70, countResult.count) + assertEquals(73, countResult.count) //Check high_vegetation_fraction > 0 countResult = datasource.firstRow("select count(*) as count from ${rsuIndicatorsTableName} WHERE high_vegetation_fraction>0".toString()) - assertEquals(18, countResult.count) + assertEquals(43, countResult.count) //Check high_vegetation_water_fraction > 0 countResult = datasource.firstRow("select count(*) as count from ${rsuIndicatorsTableName} WHERE high_vegetation_water_fraction>0".toString()) @@ -513,7 +513,7 @@ class WorkflowGeoIndicatorsTest { //Check high_vegetation_road_fraction > 0 countResult = datasource.firstRow("select count(*) as count from ${rsuIndicatorsTableName} WHERE high_vegetation_road_fraction>0".toString()) - assertEquals(20, countResult.count) + assertEquals(43, countResult.count) //Check high_vegetation_impervious_fraction > 0 countResult = datasource.firstRow("select count(*) as count from ${rsuIndicatorsTableName} WHERE high_vegetation_impervious_fraction>0".toString()) @@ -525,7 +525,7 @@ class WorkflowGeoIndicatorsTest { //Check low_vegetation_fraction > 0 countResult = datasource.firstRow("select count(*) as count from ${rsuIndicatorsTableName} WHERE low_vegetation_fraction>0".toString()) - assertEquals(50, countResult.count) + assertEquals(76, countResult.count) //Check low_vegetation_fraction > 0 countResult = datasource.firstRow("select count(*) as count from ${rsuIndicatorsTableName} WHERE impervious_fraction>0".toString()) diff --git a/osm/src/main/groovy/org/orbisgis/geoclimate/osm/InputDataFormatting.groovy b/osm/src/main/groovy/org/orbisgis/geoclimate/osm/InputDataFormatting.groovy index 3a0722b2f3..8992bb21a3 100644 --- a/osm/src/main/groovy/org/orbisgis/geoclimate/osm/InputDataFormatting.groovy +++ b/osm/src/main/groovy/org/orbisgis/geoclimate/osm/InputDataFormatting.groovy @@ -145,28 +145,64 @@ Map formatBuildingLayer(JdbcDataSource datasource, String building, String zone datasource.createIndex(outputTableName, "id_build") datasource.createIndex(outputTableName, "type") datasource.createSpatialIndex(urban_areas, "the_geom") + def urbanAreasPart = "urbanAreasPart${UUID.randomUUID().toString().replaceAll("-", "_")}" def buildinType = "BUILDING_TYPE_${UUID.randomUUID().toString().replaceAll("-", "_")}" - datasource.execute """create table $buildinType as SELECT max(b.type) as type, max(b.main_use) as main_use, a.id_build FROM $outputTableName a, $urban_areas b - WHERE a.the_geom && b.the_geom and st_intersects(ST_POINTONSURFACE(a.the_geom), b.the_geom) AND a.TYPE ='building' group by a.id_build""".toString() - - datasource.execute("CREATE INDEX ON $buildinType(id_build)".toString()) - - def newBuildingWithType = "NEW_BUILDING_TYPE_${UUID.randomUUID().toString().replaceAll("-", "_")}" + datasource.execute """DROP TABLE IF EXISTS $urbanAreasPart, $buildinType ; + CREATE TABLE $urbanAreasPart as SELECT b.type, a.id_build, st_area(st_intersection(a.the_geom,b.the_geom))/st_area(a.the_geom) as part FROM $outputTableName a, $urban_areas b + WHERE a.the_geom && b.the_geom and st_intersects(a.the_geom, b.the_geom) AND a.TYPE ='building'; + CREATE INDEX ON $urbanAreasPart(id_build); + create table $buildinType as select + MAX(part) FILTER (WHERE type = 'residential') as "residential", + MAX(part) FILTER (WHERE type = 'education') as "education", + MAX(part) FILTER (WHERE type = 'commercial') as "commercial", + MAX(part) FILTER (WHERE type = 'heavy_industry') as "heavy_industry", + MAX(part) FILTER (WHERE type = 'light_industry') as "light_industry", + MAX(part) FILTER (WHERE type = 'government') as "government", + MAX(part) FILTER (WHERE type = 'military') as "military", + MAX(part) FILTER (WHERE type = 'transport') as "transport", + MAX(part) FILTER (WHERE type = 'construction') as "construction", + MAX(part) FILTER (WHERE type = 'agricultural') as "agricultural", + MAX(part) FILTER (WHERE type = 'social_building') as "social_building", + id_build FROM $urbanAreasPart where part > 0.9 group by id_build """.toString() + + datasource.withBatch(100) { stmt -> + datasource.eachRow("SELECT * FROM $buildinType".toString()) { row -> + def new_type = "building" + def id_build_ = row.id_build + def mapTypes = row.toRowResult() + mapTypes.remove("ID_BUILD") + mapTypes.removeAll{ it.value == null } + //Only one value type, no overlaps + if(mapTypes.size()==1){ + new_type= mapTypes.keySet().stream().findFirst().get() + }else { + String maxKeyType = mapTypes.max { it.value }.key + def sortValues = mapTypes.sort { -it.value }.keySet() + sortValues.remove(maxKeyType) + switch (maxKeyType) { + case "residential": + if (sortValues.contains("education")) { + new_type = "education" + } + else { + new_type = sortValues.take(1)[0] + } + break + case "education": + new_type="education" + break + default: + new_type = sortValues.take(1)[0] + break + } - datasource.execute """DROP TABLE IF EXISTS $newBuildingWithType; - CREATE TABLE $newBuildingWithType as - SELECT a.THE_GEOM, a.ID_BUILD,a.ID_SOURCE, - a.HEIGHT_WALL, - a.HEIGHT_ROOF, - a.NB_LEV, - COALESCE(b.TYPE, a.TYPE) AS TYPE , - COALESCE(b.MAIN_USE, a.MAIN_USE) AS MAIN_USE - , a.ZINDEX, a.ROOF_SHAPE from $outputTableName - a LEFT JOIN $buildinType b on a.id_build=b.id_build""".toString() + } + stmt.addBatch("UPDATE $outputTableName set type = '${new_type}' , main_use = '${new_type}'where id_build=${id_build_}") + } + } + datasource.execute("""DROP TABLE IF EXISTS $urbanAreasPart, $buildinType""".toString()) - datasource.execute """DROP TABLE IF EXISTS $buildinType, $outputTableName; - ALTER TABLE $newBuildingWithType RENAME TO $outputTableName;""".toString() } } } @@ -610,7 +646,7 @@ String formatImperviousLayer(JdbcDataSource datasource, String impervious, Strin g1.EXPLOD_ID = g2.EXPLOD_ID AND g1.AREA_IMP > g2.AREA_IMP WHERE g2.EXPLOD_ID IS NULL ORDER BY g1.EXPLOD_ID ASC""".toString()) { row -> - //println(row) + def typeAndUseValues = getTypeAndUse(row, columnNames, mappingTypeAndUse) def use = typeAndUseValues[1] def type = typeAndUseValues[0] @@ -978,8 +1014,7 @@ String formatUrbanAreas(JdbcDataSource datasource, String urban_areas, String zo debug('Urban areas transformation starts') def outputTableName = "INPUT_URBAN_AREAS_${UUID.randomUUID().toString().replaceAll("-", "_")}" datasource.execute """Drop table if exists $outputTableName; - CREATE TABLE $outputTableName (THE_GEOM GEOMETRY, id_urban serial, ID_SOURCE VARCHAR, TYPE VARCHAR, MAIN_USE VARCHAR);""".toString() - + CREATE TABLE $outputTableName (THE_GEOM GEOMETRY, id_urban serial, ID_SOURCE VARCHAR, TYPE VARCHAR);""".toString() if (urban_areas != null) { def paramsDefaultFile = this.class.getResourceAsStream("urbanAreasParams.json") def parametersMap = parametersMapping(jsonFilename, paramsDefaultFile) @@ -1001,32 +1036,42 @@ String formatUrbanAreas(JdbcDataSource datasource, String urban_areas, String zo queryMapper += ", a.the_geom as the_geom FROM $urban_areas as a" } - def constructions = ["industrial", "commercial", "residential"] int rowcount = 1 datasource.withBatch(100) { stmt -> datasource.eachRow(queryMapper) { row -> - def typeAndUseValues = getTypeAndUse(row, columnNames, mappingType) - def use = typeAndUseValues[1] - def type = typeAndUseValues[0] - //Check if the urban areas is under construction - if(type == "construction"){ - def construction = row."construction" - if(construction && construction in constructions){ - type = construction - use = construction + if (!row.building) { + def typeAndUseValues = getTypeAndUse(row, columnNames, mappingType) + def type = typeAndUseValues[0] + //Check if the urban areas is under construction + if (type == "construction") { + def construction = row."construction" + if (construction && construction in constructions) { + type = construction + } } - } - Geometry geom = row.the_geom - int epsg = geom.getSRID() - for (int i = 0; i < geom.getNumGeometries(); i++) { - Geometry subGeom = geom.getGeometryN(i) - if (subGeom instanceof Polygon && subGeom.getArea()>1) { - stmt.addBatch "insert into $outputTableName values(ST_GEOMFROMTEXT('${subGeom}',$epsg), ${rowcount++}, '${row.id}', ${singleQuote(type)},${singleQuote(use)})".toString() + if (type) { + Geometry geom = row.the_geom + int epsg = geom.getSRID() + for (int i = 0; i < geom.getNumGeometries(); i++) { + Geometry subGeom = geom.getGeometryN(i) + if (subGeom instanceof Polygon && subGeom.getArea() > 1) { + stmt.addBatch "insert into $outputTableName values(ST_GEOMFROMTEXT('${subGeom}',$epsg), ${rowcount++}, '${row.id}', ${singleQuote(type)})".toString() + } + } } } } } + //Merging landuse + def mergingUrbanAreas = postfix("merging_urban_areas") + datasource.execute(""" + CREATE TABLE $mergingUrbanAreas as select CAST((row_number() over()) as Integer) as id_urban,the_geom, type from + ST_EXPLODE('(SELECT ST_UNION(ST_ACCUM(the_geom)) as the_geom, type from $outputTableName group by type)'); + DROP TABLE IF EXISTS $outputTableName; + ALTER TABLE $mergingUrbanAreas RENAME to $outputTableName; + """.toString()) + } } debug('Urban areas transformation finishes') diff --git a/osm/src/main/groovy/org/orbisgis/geoclimate/osm/InputDataLoading.groovy b/osm/src/main/groovy/org/orbisgis/geoclimate/osm/InputDataLoading.groovy index 113d0d45b3..044b842907 100644 --- a/osm/src/main/groovy/org/orbisgis/geoclimate/osm/InputDataLoading.groovy +++ b/osm/src/main/groovy/org/orbisgis/geoclimate/osm/InputDataLoading.groovy @@ -244,13 +244,12 @@ Map createGISLayers(JdbcDataSource datasource, String osmFilePath, org.locationt datasource.execute("ALTER TABLE ${impervious} RENAME TO $outputImperviousTableName".toString()) info "Impervious layer created" } - //Create urban areas layer paramsDefaultFile = this.class.getResourceAsStream("urbanAreasParams.json") parametersMap = readJSONParameters(paramsDefaultFile) tags = parametersMap.get("tags") columnsToKeep = parametersMap.get("columns") - String urban_areas = OSMTools.Transform.toPolygons(datasource, prefix, epsg, tags, [],geometry, true) + String urban_areas = OSMTools.Transform.toPolygons(datasource, prefix, epsg, tags,columnsToKeep,geometry, true) debug "Create the urban areas layer" if (urban_areas) { outputUrbanAreasTableName = postfix("OSM_URBAN_AREAS") diff --git a/osm/src/main/groovy/org/orbisgis/geoclimate/osm/WorkflowOSM.groovy b/osm/src/main/groovy/org/orbisgis/geoclimate/osm/WorkflowOSM.groovy index b8e097ef9f..0ed21dbfb9 100644 --- a/osm/src/main/groovy/org/orbisgis/geoclimate/osm/WorkflowOSM.groovy +++ b/osm/src/main/groovy/org/orbisgis/geoclimate/osm/WorkflowOSM.groovy @@ -753,6 +753,7 @@ def extractProcessingParameters(def processing_parameters) { svfSimplified : true, surface_vegetation: 10000f, surface_hydro : 2500f, + surface_urban_areas : 10000f, snappingTolerance : 0.01f, mapOfWeights : ["sky_view_factor" : 4, "aspect_ratio" : 3, @@ -808,6 +809,11 @@ def extractProcessingParameters(def processing_parameters) { if (surface_hydroP && surface_hydroP in Number) { rsu_indicators_default.surface_hydro = surface_hydroP } + def surface_urbanAreasP = rsu_indicators.surface_urban_areas + if (surface_urbanAreasP && surface_urbanAreasP in Number) { + rsu_indicators_default.surface_urban_areas = surface_urbanAreasP + } + def svfSimplifiedP = rsu_indicators.svfSimplified if (svfSimplifiedP && svfSimplifiedP in Boolean) { rsu_indicators_default.svfSimplified = svfSimplifiedP diff --git a/osm/src/main/resources/org/orbisgis/geoclimate/osm/buildingParams.json b/osm/src/main/resources/org/orbisgis/geoclimate/osm/buildingParams.json index 5d32d551b0..314ac157a5 100644 --- a/osm/src/main/resources/org/orbisgis/geoclimate/osm/buildingParams.json +++ b/osm/src/main/resources/org/orbisgis/geoclimate/osm/buildingParams.json @@ -37,7 +37,8 @@ "sustenance", "office", "tourism", - "roof:shape" + "roof:shape", + "wall" ], "level": { "building": 1, @@ -82,7 +83,9 @@ "emergency": 0, "hotel": 2, "hospital": 2, - "parking": 1 + "parking": 1, + "slight_construction": 0, + "water_tower" : 0 }, "type": { "terminal:transport": { @@ -597,7 +600,7 @@ "community_centre", "conference_centre", "events_venue", "exhibition_centre", "gambling", "music_venue", "nightclub", "planetarium", - "social_centre","studio", "theatre" + "social_centre","studio", "theatre", "library" ], "building": [ "cinema","arts_centre", "brothel", "casino", @@ -605,6 +608,8 @@ "events_venue", "exhibition_centre", "gambling", "music_venue", "nightclub", "planetarium", "social_centre","studio", "theatre" + ], "tourism": [ + "museum", "aquarium", "gallery", "information" ] }, "sustenance:commercial": { @@ -650,7 +655,7 @@ "! no" ] }, - "building:public": { + "government:public": { "building": [ "public" ] @@ -673,6 +678,12 @@ "attraction" ] }, + "slight_construction": { + "wall": ["no"] + }, + "water:service": { + "man_made": ["water_tower"] + }, "building": { "building": [ "yes" diff --git a/osm/src/main/resources/org/orbisgis/geoclimate/osm/urbanAreasParams.json b/osm/src/main/resources/org/orbisgis/geoclimate/osm/urbanAreasParams.json index b9d18e6f0b..bb1fae88a6 100644 --- a/osm/src/main/resources/org/orbisgis/geoclimate/osm/urbanAreasParams.json +++ b/osm/src/main/resources/org/orbisgis/geoclimate/osm/urbanAreasParams.json @@ -1,20 +1,34 @@ { "tags": { - "landuse": [ - "commercial", + "landuse": [ "commercial", "residential", "retail", "industrial", - "construction" - ], - "construction":[] + "construction", "military", + "railway", "farmyard"], + "construction":[], + "amenity": ["school", + "university", + "research_institute", "community_centre"] }, "columns": [ "landuse", "industrial", - "construction" + "construction", + "amenity", + "building" ], "type": { + "education": { + "landuse": [ + "education" + ], + "amenity": [ + "school", + "university", + "research_institute" + ] + }, "construction": { "landuse": [ "construction" @@ -31,14 +45,38 @@ "residential" ] }, - "heavy_industry:industrial": { + "heavy_industry": { "industrial": [ "refinery" ] }, - "light_industry:industrial": { + "light_industry": { + "landuse": [ + "industrial", "port" + ] + }, + "government": { + "landuse": [ + "institutional" + ] + }, + "social_building": { + "amenity": [ + "community_centre" + ]}, + "military": { + "landuse": [ + "military" + ] + }, + "transport": { + "landuse": [ + "railway" + ] + }, + "agricultural": { "landuse": [ - "industrial" + "farmyard" ] } } diff --git a/osm/src/test/groovy/org/orbisgis/geoclimate/osm/InputDataFormattingTest.groovy b/osm/src/test/groovy/org/orbisgis/geoclimate/osm/InputDataFormattingTest.groovy index a15ef82313..ebc446360d 100644 --- a/osm/src/test/groovy/org/orbisgis/geoclimate/osm/InputDataFormattingTest.groovy +++ b/osm/src/test/groovy/org/orbisgis/geoclimate/osm/InputDataFormattingTest.groovy @@ -67,7 +67,7 @@ class InputDataFormattingTest { assertEquals 136, h2GIS.getTable(extractData.vegetation).rowCount assertEquals 10, h2GIS.getTable(extractData.water).rowCount assertEquals 47, h2GIS.getTable(extractData.impervious).rowCount - assertEquals 7, h2GIS.getTable(extractData.urban_areas).rowCount + assertEquals 11, h2GIS.getTable(extractData.urban_areas).rowCount assertEquals 0, h2GIS.getTable(extractData.coastline).rowCount //Buildings @@ -312,7 +312,9 @@ class InputDataFormattingTest { zoneToExtract = "Göteborgs Stad" zoneToExtract = "Riantec" - zoneToExtract =[45.575525,5.913734,45.578859,5.919549] + zoneToExtract =[45.185546,5.751944,45.204296,5.784216] + + zoneToExtract="Sassenage" Map extractData = OSM.InputDataLoading.extractAndCreateGISLayers(h2GIS, zoneToExtract) diff --git a/osm/src/test/groovy/org/orbisgis/geoclimate/osm/InputDataLoadingTest.groovy b/osm/src/test/groovy/org/orbisgis/geoclimate/osm/InputDataLoadingTest.groovy index d940cb6fc7..c7a0a9f9ac 100644 --- a/osm/src/test/groovy/org/orbisgis/geoclimate/osm/InputDataLoadingTest.groovy +++ b/osm/src/test/groovy/org/orbisgis/geoclimate/osm/InputDataLoadingTest.groovy @@ -66,7 +66,7 @@ class InputDataLoadingTest { assertEquals 47, h2GIS.getTable(extract.impervious).rowCount - assertEquals 7, h2GIS.getTable(extract.urban_areas).rowCount + assertEquals 11, h2GIS.getTable(extract.urban_areas).rowCount } //This test is used for debug purpose diff --git a/osm/src/test/groovy/org/orbisgis/geoclimate/osm/WorflowOSMTest.groovy b/osm/src/test/groovy/org/orbisgis/geoclimate/osm/WorflowOSMTest.groovy index fb48a37f2f..f50b71e03e 100644 --- a/osm/src/test/groovy/org/orbisgis/geoclimate/osm/WorflowOSMTest.groovy +++ b/osm/src/test/groovy/org/orbisgis/geoclimate/osm/WorflowOSMTest.groovy @@ -645,19 +645,15 @@ class WorflowOSMTest extends WorkflowAbstractTest { //Use it for debug @Test void testIntegration() { - String directory = "/tmp/geoclimate" File dirFile = new File(directory) dirFile.delete() dirFile.mkdir() def location = "Nice" - //def nominatim = OSMTools.Utilities.getNominatimData("Nantes") - - location=[47.2, -1.6, 47.4, -1.4] + location="Sassenage" //location = nominatim.bbox - def osm_parmeters = [ "description" : "Example of configuration file to run the OSM workflow and store the result in a folder", "geoclimatedb": [ @@ -678,7 +674,7 @@ class WorflowOSMTest extends WorkflowAbstractTest { ["distance" : 0, "rsu_indicators" : [ - "indicatorUse": ["LCZ", "TEB"] //, "UTRF", "TEB"] + "indicatorUse": ["LCZ"] //, "UTRF", "TEB"] ]/*,"grid_indicators": [ "x_size": 200, @@ -777,41 +773,6 @@ class WorflowOSMTest extends WorkflowAbstractTest { Geoindicators.WorkflowUtilities.saveToAscGrid("grid_indicators", subFolder, "grid_indicators", datasource, 3007, reproject, deleteOutputData) } - @Disabled - @Test - void testPopulation_Indicators() { - String directory = "/tmp/geoclimate" - File dirFile = new File(directory) - dirFile.delete() - dirFile.mkdir() - //H2GIS h2gis = H2GIS.open("${directory+File.separator}geoclimate_chain_db;AUTO_SERVER=TRUE") - //h2gis.load("../cities/Barcelona/lden_barcelona_ref.geojson", "area_zone", true) - //def env = h2gis.firstRow("(select st_transform(st_extent(the_geom), 4326) as the_geom from area_zone)").the_geom.getEnvelopeInternal() - //def location = [[env.getMinY()as float, env.getMinX() as float, env.getMaxY() as float, env.getMaxX() as float]]; - def location = [[57.651096, 11.821976, 57.708916, 11.89235]] - def tables = ["building", "population", "road_traffic"] - tables = ["road_traffic"] - def osm_parmeters = [ - "description" : "Example of configuration file to run only the road traffic estimation", - "geoclimatedb": [ - "folder": dirFile.absolutePath, - "name" : "geoclimate_chain_db;AUTO_SERVER=TRUE", - "delete": false - ], - "input" : [ - "locations": location], - "output" : [ - "folder": ["path" : directory, - "tables": tables]], - "parameters" : - ["worldpop_indicators": false, "road_traffic": true] - ] - OSM.workflow(osm_parmeters) - //assertEquals(10, process.getResults().output["Pont-de-Veyle"].size()) - //assertTrue h2gis.firstRow("select count(*) as count from ${process.results.output["Pont-de-Veyle"].population} where pop is not null").count>0 - } - - @Test //Integration tests @Disabled diff --git a/osm/src/test/groovy/org/orbisgis/geoclimate/osm/WorkflowAbstractTest.groovy b/osm/src/test/groovy/org/orbisgis/geoclimate/osm/WorkflowAbstractTest.groovy index 7e2c6dee7c..fca48c3aeb 100644 --- a/osm/src/test/groovy/org/orbisgis/geoclimate/osm/WorkflowAbstractTest.groovy +++ b/osm/src/test/groovy/org/orbisgis/geoclimate/osm/WorkflowAbstractTest.groovy @@ -52,7 +52,7 @@ class WorkflowAbstractTest { Map spatialUnits = Geoindicators.WorkflowGeoIndicators.createUnitsOfAnalysis(datasource, zone, buildingTableName, roadTableName, railTableName, vegetationTableName, hydrographicTableName, sealandmaskTableName, urban_areas, "", 10000, - 2500, 0.01, indicatorUse, prefixName) + 2500, 10000, 0.01, indicatorUse, prefixName) String relationBuildings = spatialUnits.building String relationBlocks = spatialUnits.block diff --git a/osmtools/src/main/groovy/org/orbisgis/geoclimate/osmtools/utils/Utilities.groovy b/osmtools/src/main/groovy/org/orbisgis/geoclimate/osmtools/utils/Utilities.groovy index 8f57ad23c2..6b11f1d42c 100644 --- a/osmtools/src/main/groovy/org/orbisgis/geoclimate/osmtools/utils/Utilities.groovy +++ b/osmtools/src/main/groovy/org/orbisgis/geoclimate/osmtools/utils/Utilities.groovy @@ -77,9 +77,10 @@ Map getNominatimData(def placeName) { GeometryFactory geometryFactory = new GeometryFactory() + def geometry_admin= 0 def data = [:] jsonRoot.features.find() { feature -> - if (feature.geometry != null) { + if (feature.geometry != null ) { if (feature.geometry.type.equalsIgnoreCase("polygon")) { def area = parsePolygon(feature.geometry.coordinates, geometryFactory) area.setSRID(4326) @@ -88,6 +89,9 @@ Map getNominatimData(def placeName) { data.putAll(feature.properties) def bbox = feature.bbox data.put("bbox", [bbox[1], bbox[0], bbox[3], bbox[2]]) + if(feature.properties.type=="administrative" && feature.properties.category=='boundary'){ + return true + } } else if (feature.geometry.type.equalsIgnoreCase("multipolygon")) { def mp = feature.geometry.coordinates.collect { it -> parsePolygon(it, geometryFactory) @@ -99,10 +103,12 @@ Map getNominatimData(def placeName) { data.putAll(feature.properties) def bbox = feature.bbox data.put("bbox", [bbox[1], bbox[0], bbox[3], bbox[2]]) + if(feature.properties.type=="administrative"&& feature.properties.category=='boundary'){ + return true + } } else { return false } - return true } return false } @@ -242,7 +248,7 @@ boolean executeNominatimQuery(def query, def outputOSMFile) { error "The OSM file should be an instance of File" return false } - def endPoint = System.getProperty("NOMINATIM_ENPOINT"); + def endPoint = System.getProperty("NOMINATIM_ENDPOINT"); if (!endPoint) { /** nominatim server endpoint as defined by WSDL2 definition */ endPoint = "https://nominatim.openstreetmap.org/"; @@ -263,11 +269,7 @@ boolean executeNominatimQuery(def query, def outputOSMFile) { connection = url.openConnection() } connection.requestMethod = "GET" - def user_agent = System.getProperty("OVERPASS_USER_AGENT") - if (!user_agent) { - user_agent = OSM_USER_AGENT - } - connection.setRequestProperty("User-Agent", user_agent) + connection.setRequestProperty("User-Agent", "GEOCLIMATE_${System.currentTimeMillis()}") connection.connect() @@ -339,6 +341,21 @@ String toPoly(Geometry geometry) { polyStr += "${coord.getY()} ${coord.getX()}" return polyStr + "\")" } +/** + * Method to build a valid OSM query with a bbox. + * + * @author Erwan Bocher (CNRS LAB-STICC) + * @author Elisabeth Le Saux (UBS LAB-STICC) + * + * @param envelope The envelope to filter. + * @param keys A list of OSM keys. Must be null + * + * @return A string representation of the OSM query. + */ +String buildOSMQuery(Envelope envelope, def keys=null) { + return buildOSMQuery(envelope, keys, OSMElement.NODE, OSMElement.WAY, OSMElement.RELATION) +} + /** * Method to build a valid OSM query with a bbox. @@ -352,7 +369,7 @@ String toPoly(Geometry geometry) { * * @return A string representation of the OSM query. */ -String buildOSMQuery(Envelope envelope, def keys, OSMElement... osmElement) { + String buildOSMQuery(Envelope envelope, def keys, OSMElement... osmElement) { if (!envelope) { error "Cannot create the overpass query from the bbox $envelope." return null @@ -371,6 +388,44 @@ String buildOSMQuery(Envelope envelope, def keys, OSMElement... osmElement) { return query } +/** + * Method to build a valid OSM query with a list of coordinates that define a Bbox. + * + * @author Erwan Bocher (CNRS LAB-STICC) + * + * @param latLonCoordinates an array of 4 coordinates in lat/lon + * @param keys A list of OSM keys. + * + * @return A string representation of the OSM query. + */ +String buildOSMQuery(List latLonCoordinates, def keys=null) { + return buildOSMQuery(latLonCoordinates, keys, OSMElement.NODE, OSMElement.WAY, OSMElement.RELATION) +} + +/** + * Method to build a valid OSM query with a list of coordinates that define a Bbox. + * + * @author Erwan Bocher (CNRS LAB-STICC) + * + * @param latLonCoordinates an array of 4 coordinates in lat/lon + * @param keys A list of OSM keys. + * @param osmElement A list of OSM elements to build the query (node, way, relation). + * + * @return A string representation of the OSM query. + */ + String buildOSMQuery(List latLonCoordinates, def keys, OSMElement... osmElement) { + if (!latLonCoordinates) { + error "Cannot create the overpass query from the bbox $latLonCoordinates." + return null + } + Geometry geom = OSMTools.Utilities.geometryFromValues(latLonCoordinates) + if(geom==null) { + error "Invalid BBOX" + return null + } + return buildOSMQuery(geom.getEnvelopeInternal(), keys, osmElement) +} + /** * Method to build a valid OSM query with a bbox to * download all the osm data concerning @@ -384,7 +439,7 @@ String buildOSMQuery(Envelope envelope, def keys, OSMElement... osmElement) { * * @return A string representation of the OSM query. */ -static String buildOSMQueryWithAllData(Envelope envelope, def keys, OSMElement... osmElement) { + String buildOSMQueryWithAllData(Envelope envelope, def keys, OSMElement... osmElement) { if (!envelope) { error "Cannot create the overpass query from the bbox $envelope." return null @@ -403,6 +458,21 @@ static String buildOSMQueryWithAllData(Envelope envelope, def keys, OSMElement.. return query } +/** + * Method to build a valid and optimized OSM query + * + * @author Erwan Bocher (CNRS LAB-STICC) + * @author Elisabeth Le Saux (UBS LAB-STICC) + * + * @param polygon The polygon to filter. + * @param keys A list of OSM keys. + * + * @return A string representation of the OSM query. + */ +String buildOSMQuery(Polygon polygon, def keys=null) { + return buildOSMQuery(polygon, keys, OSMElement.NODE, OSMElement.WAY, OSMElement.RELATION) +} + /** * Method to build a valid and optimized OSM query * @@ -617,9 +687,6 @@ static @Field OVERPASS_BASE_URL = "${OVERPASS_ENDPOINT}/interpreter?data=" /** Url of the status of the Overpass server */ static @Field OVERPASS_STATUS_URL = "${OVERPASS_ENDPOINT}/status" -/** Default user agent*/ -static @Field OSM_USER_AGENT = "geoclimate" - /** OVERPASS TIMEOUT */ static @Field int OVERPASS_TIMEOUT = 180 /** @@ -630,7 +697,7 @@ def getServerStatus() { final String proxyHost = System.getProperty("http.proxyHost"); final int proxyPort = Integer.parseInt(System.getProperty("http.proxyPort", "80")); def connection - def endPoint = System.getProperty("OVERPASS_ENPOINT"); + def endPoint = System.getProperty("OVERPASS_ENDPOINT"); if (endPoint) { OVERPASS_STATUS_URL = "${endPoint}/status" } @@ -686,11 +753,7 @@ boolean executeOverPassQuery(URL queryUrl, def outputOSMFile) { timeout = (int) TimeUnit.MINUTES.toMillis(3); } - def user_agent = System.getProperty("OVERPASS_USER_AGENT") - if (!user_agent) { - user_agent = OSM_USER_AGENT - } - connection.setRequestProperty("User-Agent", user_agent) + connection.setRequestProperty("User-Agent", "GEOCLIMATE_${System.currentTimeMillis()}") connection.setConnectTimeout(timeout); connection.setReadTimeout(timeout); @@ -729,7 +792,7 @@ boolean executeOverPassQuery(def query, def outputOSMFile) { error "The output file should not be null or empty." return false } - def endPoint = System.getProperty("OVERPASS_ENPOINT"); + def endPoint = System.getProperty("OVERPASS_ENDPOINT"); if (endPoint) { OVERPASS_BASE_URL = "${endPoint}/interpreter?data=" } @@ -752,11 +815,8 @@ boolean executeOverPassQuery(def query, def outputOSMFile) { } else { timeout = (int) TimeUnit.MINUTES.toMillis(3); } - def user_agent = System.getProperty("OVERPASS_USER_AGENT") - if (!user_agent) { - user_agent = OSM_USER_AGENT - } - connection.setRequestProperty("User-Agent", user_agent) + connection.setRequestProperty("User-Agent", "GEOCLIMATE_${System.currentTimeMillis()}") + connection.setConnectTimeout(timeout) connection.setReadTimeout(timeout) @@ -773,3 +833,28 @@ boolean executeOverPassQuery(def query, def outputOSMFile) { return false } } + +/** + * Return the status of the Nominatim server. + * @return true is the server is OK + */ +def isNominatimReady() { + final String proxyHost = System.getProperty("http.proxyHost"); + final int proxyPort = Integer.parseInt(System.getProperty("http.proxyPort", "80")); + def connection + def endPoint = "https://nominatim.openstreetmap.org/status" + + if (proxyHost != null) { + def proxy = new Proxy(Proxy.Type.HTTP, new InetSocketAddress(proxyHost, proxyPort)); + connection = new URL(endPoint).openConnection(proxy) as HttpURLConnection + } else { + connection = new URL(endPoint).openConnection() as HttpURLConnection + } + connection.requestMethod = GET + connection.connect() + if (connection.responseCode == 200) { + return true + } else { + return false + } +} diff --git a/osmtools/src/test/groovy/org/orbisgis/geoclimate/osmtools/LoaderTest.groovy b/osmtools/src/test/groovy/org/orbisgis/geoclimate/osmtools/LoaderTest.groovy index ff7e80b198..652537265d 100644 --- a/osmtools/src/test/groovy/org/orbisgis/geoclimate/osmtools/LoaderTest.groovy +++ b/osmtools/src/test/groovy/org/orbisgis/geoclimate/osmtools/LoaderTest.groovy @@ -221,32 +221,33 @@ class LoaderTest extends AbstractOSMToolsTest { */ @Test void fromPlaceNoDistTest() { - def placeName = "Lezoen, Plourivo" - def formattedPlaceName = "Lezoen_Plourivo_" - - Map r = OSMTools.Loader.fromPlace(ds, placeName) - assertFalse r.isEmpty() - assertTrue r.containsKey("zone") - assertTrue r.containsKey("envelope") - assertTrue r.containsKey("prefix") - assertTrue Pattern.compile("OSM_DATA_$formattedPlaceName$uuidRegex").matcher(r.prefix as String).matches() - - def zone = ds.getSpatialTable(r.zone) - assertEquals 1, zone.rowCount - assertEquals 2, zone.getColumnCount() - assertTrue zone.columns.contains("THE_GEOM") - assertTrue zone.columns.contains("ID_ZONE") - zone.next() - assertNotNull zone.getGeometry(1) - - def zoneEnv = ds.getSpatialTable(r.envelope) - assertEquals 1, zoneEnv.rowCount - assertEquals 2, zoneEnv.getColumnCount() - assertTrue zoneEnv.columns.contains("THE_GEOM") - assertTrue zoneEnv.columns.contains("ID_ZONE") - zoneEnv.next() - assertEquals "POLYGON ((-3.0790622 48.7298266, -3.0790622 48.7367393, -3.0739517 48.7367393, -3.0739517 48.7298266, -3.0790622 48.7298266))", zoneEnv.getGeometry(1).toText() - assertEquals "Lezoen, Plourivo", zoneEnv.getString(2) + if(OSMTools.Utilities.isNominatimReady()) { + def placeName = "Lezoen, Plourivo" + def formattedPlaceName = "Lezoen_Plourivo_" + Map r = OSMTools.Loader.fromPlace(ds, placeName) + assertFalse r.isEmpty() + assertTrue r.containsKey("zone") + assertTrue r.containsKey("envelope") + assertTrue r.containsKey("prefix") + assertTrue Pattern.compile("OSM_DATA_$formattedPlaceName$uuidRegex").matcher(r.prefix as String).matches() + + def zone = ds.getSpatialTable(r.zone) + assertEquals 1, zone.rowCount + assertEquals 2, zone.getColumnCount() + assertTrue zone.columns.contains("THE_GEOM") + assertTrue zone.columns.contains("ID_ZONE") + zone.next() + assertNotNull zone.getGeometry(1) + + def zoneEnv = ds.getSpatialTable(r.envelope) + assertEquals 1, zoneEnv.rowCount + assertEquals 2, zoneEnv.getColumnCount() + assertTrue zoneEnv.columns.contains("THE_GEOM") + assertTrue zoneEnv.columns.contains("ID_ZONE") + zoneEnv.next() + assertEquals "POLYGON ((-3.0790622 48.7298266, -3.0790622 48.7367393, -3.0739517 48.7367393, -3.0739517 48.7298266, -3.0790622 48.7298266))", zoneEnv.getGeometry(1).toText() + assertEquals "Lezoen, Plourivo", zoneEnv.getString(2) + } } /** @@ -635,4 +636,13 @@ class LoaderTest extends AbstractOSMToolsTest { } } } + + /** + * For debug purpose + */ + @Disabled + @Test + void forDebug() { + println(OSMTools.Utilities.getNominatimData("Sassenage")) + } } diff --git a/osmtools/src/test/groovy/org/orbisgis/geoclimate/osmtools/TransformTest.groovy b/osmtools/src/test/groovy/org/orbisgis/geoclimate/osmtools/TransformTest.groovy index 8c98fc7058..822449b31a 100644 --- a/osmtools/src/test/groovy/org/orbisgis/geoclimate/osmtools/TransformTest.groovy +++ b/osmtools/src/test/groovy/org/orbisgis/geoclimate/osmtools/TransformTest.groovy @@ -608,11 +608,11 @@ class TransformTest extends AbstractOSMToolsTest { def extract = OSMTools.Loader.extract(query) if (extract) { def prefix = "OSM_FILE_${OSMTools.Utilities.uuid()}" - if (OSMTools.Loader.load(h2GIS, prefix, extract)) { + if (OSMTools.Loader.load(ds, prefix, extract)) { def tags = ['building'] def transform = OSMTools.Transform.toPolygons(h2GIS, prefix, tags) assertNotNull(transform) - assertTrue(h2GIS.getTable(transform).getRowCount() > 0) + assertTrue(ds.getTable(transform).getRowCount() > 0) } } } @@ -877,6 +877,23 @@ class TransformTest extends AbstractOSMToolsTest { } } + @Test + void buildAllPolygons() { + def bbox =[47.647353,-2.090192,47.649413,-2.087274] + def query = OSMTools.Utilities.buildOSMQuery(bbox) + if (!query.isEmpty()) { + def extract = OSMTools.Loader.extract(query) + if (extract) { + def prefix = "OSM" + assertTrue OSMTools.Loader.load(ds, prefix,extract) + //Create building layer + def tags = ["amenity", "landuse", "railway", "water"] + String outputTableName = OSMTools.Transform.toPolygons(ds, prefix, 4326, tags) + ds.save(outputTableName, "/tmp/polygons.geojson", true) + } + } + } + /** * It uses for test purpose */ diff --git a/osmtools/src/test/groovy/org/orbisgis/geoclimate/osmtools/utils/UtilitiesTest.groovy b/osmtools/src/test/groovy/org/orbisgis/geoclimate/osmtools/utils/UtilitiesTest.groovy index 74c557521a..15b6e06e92 100644 --- a/osmtools/src/test/groovy/org/orbisgis/geoclimate/osmtools/utils/UtilitiesTest.groovy +++ b/osmtools/src/test/groovy/org/orbisgis/geoclimate/osmtools/utils/UtilitiesTest.groovy @@ -278,9 +278,12 @@ class UtilitiesTest extends AbstractOSMToolsTest { "out;", OSMTools.Utilities.buildOSMQuery(enveloppe, ["building", "water"], OSMElement.NODE, OSMElement.WAY) assertEquals "[bbox:7.6,0.0,8.9,2.3];\n" + "(\n" + + "\tnode[\"building\"];\n" + + "\tway[\"building\"];\n" + + "\trelation[\"building\"];\n" + ");\n" + "(._;>;);\n" + - "out;", OSMTools.Utilities.buildOSMQuery(enveloppe, ["building", "water"]) + "out;", OSMTools.Utilities.buildOSMQuery(enveloppe, ["building"]) assertEquals "[bbox:7.6,0.0,8.9,2.3];\n" + "(\n" + ");\n" + @@ -315,19 +318,19 @@ class UtilitiesTest extends AbstractOSMToolsTest { "\tway[\"water\"];\n" + ");\n" + ">;);\n" + - "out;", Utilities.buildOSMQueryWithAllData(enveloppe, ["building", "water"], OSMElement.NODE, OSMElement.WAY) + "out;", OSMTools.Utilities.buildOSMQueryWithAllData(enveloppe, ["building", "water"], OSMElement.NODE, OSMElement.WAY) assertEquals "[bbox:7.6,0.0,8.9,2.3];\n" + "((\n" + ");\n" + ">;);\n" + - "out;", Utilities.buildOSMQueryWithAllData(enveloppe, ["building", "water"]) + "out;", OSMTools.Utilities.buildOSMQueryWithAllData(enveloppe, ["building", "water"]) assertEquals "[bbox:7.6,0.0,8.9,2.3];\n" + "((\n" + "\tnode;\n" + "\tway;\n" + ");\n" + ">;);\n" + - "out;", Utilities.buildOSMQueryWithAllData(enveloppe, [], OSMElement.NODE, OSMElement.WAY) + "out;", OSMTools.Utilities.buildOSMQueryWithAllData(enveloppe, [], OSMElement.NODE, OSMElement.WAY) } /** @@ -366,9 +369,12 @@ class UtilitiesTest extends AbstractOSMToolsTest { "out;", OSMTools.Utilities.buildOSMQuery(polygon, ["building", "water"], OSMElement.NODE, OSMElement.WAY) assertEquals "[bbox:2.3,0.0,8.9,7.6];\n" + "(\n" + + "\tnode[\"building\"](poly:\"2.3 0.0 2.3 7.6 8.9 7.6 8.9 0.0\");\n" + + "\tway[\"building\"](poly:\"2.3 0.0 2.3 7.6 8.9 7.6 8.9 0.0\");\n" + + "\trelation[\"building\"](poly:\"2.3 0.0 2.3 7.6 8.9 7.6 8.9 0.0\");\n" + ");\n" + "(._;>;);\n" + - "out;", OSMTools.Utilities.buildOSMQuery(polygon, ["building", "water"]) + "out;", OSMTools.Utilities.buildOSMQuery(polygon, ["building"]) assertEquals "[bbox:2.3,0.0,8.9,7.6];\n" + "(\n" + ");\n" + @@ -542,12 +548,12 @@ class UtilitiesTest extends AbstractOSMToolsTest { @Disabled void getNominatimDataTest() { def pattern = Pattern.compile("^POLYGON \\(\\((?>-?\\d+(?>\\.\\d+)? -?\\d+(?>\\.\\d+)?(?>, )?)*\\)\\)\$") - def data = Utilities.getNominatimData("Paimpol") + def data = OSMTools.Utilities.getNominatimData("Paimpol") assertTrue pattern.matcher(data["geom"].toString()).matches() Envelope env = data["geom"].getEnvelopeInternal() assertEquals([env.getMinY(), env.getMinX(), env.getMaxY(), env.getMaxX()].toString(), data["bbox"].toString()) assertEquals(data["extratags"]["ref:INSEE"], "22162") - data = Utilities.getNominatimData("Boston") + data = OSMTools.Utilities.getNominatimData("Boston") assertTrue pattern.matcher(data["geom"].toString()).matches() assertEquals(data["extratags"]["population"], "689326") } From 06365b73d27334ce5393e5b81ac69790e92a077d Mon Sep 17 00:00:00 2001 From: jeremy Date: Tue, 5 Dec 2023 21:02:50 +0100 Subject: [PATCH 2/2] Modify english sld version of lcz style --- .../src/main/resources/styles/en/sld/rsu_lcz_primary.sld | 4 ++-- .../orbisgis/geoclimate/osm/InputDataFormattingTest.groovy | 2 +- .../groovy/org/orbisgis/geoclimate/osm/WorflowOSMTest.groovy | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/geoindicators/src/main/resources/styles/en/sld/rsu_lcz_primary.sld b/geoindicators/src/main/resources/styles/en/sld/rsu_lcz_primary.sld index b35ef49960..50e6a879fe 100644 --- a/geoindicators/src/main/resources/styles/en/sld/rsu_lcz_primary.sld +++ b/geoindicators/src/main/resources/styles/en/sld/rsu_lcz_primary.sld @@ -2,7 +2,7 @@ - + @@ -330,5 +330,5 @@ - + diff --git a/osm/src/test/groovy/org/orbisgis/geoclimate/osm/InputDataFormattingTest.groovy b/osm/src/test/groovy/org/orbisgis/geoclimate/osm/InputDataFormattingTest.groovy index ebc446360d..e21c64d8d5 100644 --- a/osm/src/test/groovy/org/orbisgis/geoclimate/osm/InputDataFormattingTest.groovy +++ b/osm/src/test/groovy/org/orbisgis/geoclimate/osm/InputDataFormattingTest.groovy @@ -314,7 +314,7 @@ class InputDataFormattingTest { zoneToExtract = "Riantec" zoneToExtract =[45.185546,5.751944,45.204296,5.784216] - zoneToExtract="Sassenage" + zoneToExtract=[50, 8.6, 50.2, 8.8] Map extractData = OSM.InputDataLoading.extractAndCreateGISLayers(h2GIS, zoneToExtract) diff --git a/osm/src/test/groovy/org/orbisgis/geoclimate/osm/WorflowOSMTest.groovy b/osm/src/test/groovy/org/orbisgis/geoclimate/osm/WorflowOSMTest.groovy index f50b71e03e..c7b0472f13 100644 --- a/osm/src/test/groovy/org/orbisgis/geoclimate/osm/WorflowOSMTest.groovy +++ b/osm/src/test/groovy/org/orbisgis/geoclimate/osm/WorflowOSMTest.groovy @@ -651,7 +651,7 @@ class WorflowOSMTest extends WorkflowAbstractTest { dirFile.mkdir() def location = "Nice" //def nominatim = OSMTools.Utilities.getNominatimData("Nantes") - location="Sassenage" + //location=[50, 8.6, 50.2, 8.8] //location = nominatim.bbox def osm_parmeters = [ @@ -674,7 +674,7 @@ class WorflowOSMTest extends WorkflowAbstractTest { ["distance" : 0, "rsu_indicators" : [ - "indicatorUse": ["LCZ"] //, "UTRF", "TEB"] + "indicatorUse": ["LCZ", "UTRF"] //, "UTRF", "TEB"] ]/*,"grid_indicators": [ "x_size": 200,