From 1394c0f7f0e37edb603a252550d381972dd9a50c Mon Sep 17 00:00:00 2001 From: ebocher Date: Thu, 23 May 2024 08:50:43 +0200 Subject: [PATCH 1/4] Allow workflow to only format the GIS layers --- .../bdtopo/AbstractBDTopoWorkflow.groovy | 14 ++---- .../bdtopo/WorkflowAbstractTest.groovy | 47 +++++++++++++++++++ .../bdtopo/v2/WorkflowBDTopoV2Test.groovy | 2 - 3 files changed, 51 insertions(+), 12 deletions(-) diff --git a/bdtopo/src/main/groovy/org/orbisgis/geoclimate/bdtopo/AbstractBDTopoWorkflow.groovy b/bdtopo/src/main/groovy/org/orbisgis/geoclimate/bdtopo/AbstractBDTopoWorkflow.groovy index f92bc8207c..773aaade43 100644 --- a/bdtopo/src/main/groovy/org/orbisgis/geoclimate/bdtopo/AbstractBDTopoWorkflow.groovy +++ b/bdtopo/src/main/groovy/org/orbisgis/geoclimate/bdtopo/AbstractBDTopoWorkflow.groovy @@ -160,9 +160,6 @@ abstract class AbstractBDTopoWorkflow extends BDTopoUtils { //Get processing parameters def processing_parameters = extractProcessingParameters(parameters.parameters) - if (!processing_parameters) { - return - } //Get the out put parameters def outputDatasource @@ -444,8 +441,7 @@ abstract class AbstractBDTopoWorkflow extends BDTopoUtils { * @return a filled map of parameters */ def extractProcessingParameters(def processing_parameters) { - def defaultParameters = [distance : 1000f, - distance_buffer: 500f, prefixName: "", + def defaultParameters = [distance : 1000f, prefixName: "", hLevMin : 3] def rsu_indicators_default = [indicatorUse : [], svfSimplified : true, @@ -461,7 +457,6 @@ abstract class AbstractBDTopoWorkflow extends BDTopoUtils { "height_of_roughness_elements": 6, "terrain_roughness_length" : 0.5], utrfModelName : "UTRF_BDTOPO_V2_RF_2_2.model"] - defaultParameters.put("rsu_indicators", rsu_indicators_default) if (processing_parameters) { def distanceP = processing_parameters.distance @@ -526,10 +521,9 @@ abstract class AbstractBDTopoWorkflow extends BDTopoUtils { rsu_indicators_default.mapOfWeights = mapOfWeightsP } } - } else { - rsu_indicators = rsu_indicators_default - } + defaultParameters.put("rsu_indicators", rsu_indicators_default) + } //Check for grid indicators def grid_indicators = processing_parameters.grid_indicators @@ -949,7 +943,7 @@ abstract class AbstractBDTopoWorkflow extends BDTopoUtils { results.put("building", building) //Compute the RSU indicators - if (rsu_indicators_params.indicatorUse) { + if (rsu_indicators_params && rsu_indicators_params.indicatorUse) { //Build the indicators rsu_indicators_params.put("utrfModelName", "UTRF_BDTOPO_V2_RF_2_2.model") Map geoIndicators = Geoindicators.WorkflowGeoIndicators.computeAllGeoIndicators(h2gis_datasource, zone, diff --git a/bdtopo/src/test/groovy/org/orbisgis/geoclimate/bdtopo/WorkflowAbstractTest.groovy b/bdtopo/src/test/groovy/org/orbisgis/geoclimate/bdtopo/WorkflowAbstractTest.groovy index 43dd7322e0..1121ce4c6b 100644 --- a/bdtopo/src/test/groovy/org/orbisgis/geoclimate/bdtopo/WorkflowAbstractTest.groovy +++ b/bdtopo/src/test/groovy/org/orbisgis/geoclimate/bdtopo/WorkflowAbstractTest.groovy @@ -389,4 +389,51 @@ abstract class WorkflowAbstractTest { checkFormatData() } + @Test + void testOnlyFormatData() { + String dataFolder = getDataFolderPath() + def bdTopoParameters = [ + "description" : "Workflow format data", + "geoclimatedb": [ + "folder": folder.absolutePath, + "name" : "testFormat;AUTO_SERVER=TRUE", + "delete": false + ], + "input" : [ + "folder" : dataFolder, + "locations": [getInseeCode()]], + "output" : [ + "folder": ["path": folder.absolutePath]] + ] + + Map process = BDTopo.workflow(bdTopoParameters, getVersion()) + assertNotNull(process) + + def tableNames = process[getInseeCode()] + assertTrue(tableNames.size() > 0) + H2GIS h2gis = H2GIS.open(folder.absolutePath + File.separator + "testFormat;AUTO_SERVER=TRUE") + + //Test zone + assertTrue h2gis.firstRow("select count(*) as count from ${tableNames.zone} where the_geom is not null").count > 0 + + //Test building + String building_table = tableNames.building + assertTrue(h2gis.firstRow("""SELECT count(*) as count from $building_table where TYPE is not null;""".toString()).count > 0) + assertTrue(h2gis.firstRow("""SELECT count(*) as count from $building_table where MAIN_USE is not null;""".toString()).count > 0) + assertTrue(h2gis.firstRow("""SELECT count(*) as count from $building_table where NB_LEV is not null or NB_LEV>0 ;""".toString()).count > 0) + assertTrue(h2gis.firstRow("""SELECT count(*) as count from $building_table where HEIGHT_WALL is not null or HEIGHT_WALL>0 ;""".toString()).count > 0) + assertTrue(h2gis.firstRow("""SELECT count(*) as count from $building_table where HEIGHT_ROOF is not null or HEIGHT_ROOF>0 ;""".toString()).count > 0) + + //Test water + assertTrue(h2gis.firstRow("""SELECT count(*) as count from ${tableNames.water} where TYPE is not null;""".toString()).count > 0) + + //Test vegetation + assertTrue(h2gis.firstRow("""SELECT count(*) as count from ${tableNames.vegetation} where TYPE is not null;""".toString()).count > 0) + assertTrue(h2gis.firstRow("""SELECT count(*) as count from ${tableNames.vegetation} where HEIGHT_CLASS is not null;""".toString()).count > 0) + + //Test road + assertTrue(h2gis.firstRow("""SELECT count(*) as count from ${tableNames.road} where TYPE is not null;""".toString()).count > 0) + assertTrue(h2gis.firstRow("""SELECT count(*) as count from ${tableNames.road} where WIDTH is not null or WIDTH>0 ;""".toString()).count > 0) + } + } \ No newline at end of file diff --git a/bdtopo/src/test/groovy/org/orbisgis/geoclimate/bdtopo/v2/WorkflowBDTopoV2Test.groovy b/bdtopo/src/test/groovy/org/orbisgis/geoclimate/bdtopo/v2/WorkflowBDTopoV2Test.groovy index 3aec0944b1..b674e51a90 100644 --- a/bdtopo/src/test/groovy/org/orbisgis/geoclimate/bdtopo/v2/WorkflowBDTopoV2Test.groovy +++ b/bdtopo/src/test/groovy/org/orbisgis/geoclimate/bdtopo/v2/WorkflowBDTopoV2Test.groovy @@ -132,7 +132,5 @@ class WorkflowBDTopoV2Test extends WorkflowAbstractTest { assertEquals(count, h2GIS.firstRow("SELECT COUNT(*) as count FROM urban_areas where type is not null").count) assertEquals(count, h2GIS.firstRow("SELECT COUNT(*) as count FROM urban_areas where ST_ISEMPTY(THE_GEOM)=false OR THE_GEOM IS NOT NULL").count) - - } } \ No newline at end of file From eb0bc8951d18d7c45df4074c19631399e31456c1 Mon Sep 17 00:00:00 2001 From: jeremy Date: Mon, 27 May 2024 21:48:43 +0200 Subject: [PATCH 2/4] Fix issue #968 --- .../bdtopo/WorkflowDebugTest.groovy | 26 +++++-------------- .../WorkflowGeoIndicators.groovy | 2 +- .../GenericIndicatorsTests.groovy | 21 +++++++++++++++ 3 files changed, 28 insertions(+), 21 deletions(-) diff --git a/bdtopo/src/test/groovy/org/orbisgis/geoclimate/bdtopo/WorkflowDebugTest.groovy b/bdtopo/src/test/groovy/org/orbisgis/geoclimate/bdtopo/WorkflowDebugTest.groovy index bafbf0834b..3189494ad4 100644 --- a/bdtopo/src/test/groovy/org/orbisgis/geoclimate/bdtopo/WorkflowDebugTest.groovy +++ b/bdtopo/src/test/groovy/org/orbisgis/geoclimate/bdtopo/WorkflowDebugTest.groovy @@ -136,9 +136,9 @@ class WorkflowDebugTest { @Disabled @Test void testIntegrationFolderInput() { - def input_data = "/media/ebocher/Extreme SSD/bdtopo/bdtopo2/BDTOPO_2-2_TOUSTHEMES_SHP_LAMB93_D035_2018-09-25/BDTOPO/1_DONNEES_LIVRAISON_2018-11-00144/BDT_2-2_SHP_LAMB93_D035-ED182" - def locations = [ "Bordeaux"] - String directory = "/tmp/bdtopo2" + def input_data = "/home/decide/Data/URBIO/Donnees_brutes/BD_TOPO/BDTOPO_3-3_TOUSTHEMES_SHP_LAMB93_D076_2024-03-15" + def locations = ["76005","76020"] + String directory = "/tmp/bdtopo3" File dirFile = new File(directory) dirFile.delete() dirFile.mkdir() @@ -157,30 +157,16 @@ class WorkflowDebugTest { "folder": ["path": directory]], "parameters" : ["distance" : 0, - rsu_indicators : [ - "indicatorUse": ["LCZ", "UTRF", "TEB"] - - ], "grid_indicators": [ "x_size" : 100, "y_size" : 100, - "indicators" :["BUILDING_FRACTION", "BUILDING_HEIGHT", "BUILDING_POP", - "BUILDING_TYPE_FRACTION", "WATER_FRACTION", "VEGETATION_FRACTION", - "ROAD_FRACTION", "IMPERVIOUS_FRACTION", - //"FREE_EXTERNAL_FACADE_DENSITY", - "BUILDING_HEIGHT_WEIGHTED", "BUILDING_SURFACE_DENSITY", - "SEA_LAND_FRACTION", "ASPECT_RATIO", - //"SVF", - "HEIGHT_OF_ROUGHNESS_ELEMENTS", "TERRAIN_ROUGHNESS_CLASS", - "UTRF_AREA_FRACTION", "UTRF_FLOOR_AREA_FRACTION", - "LCZ_PRIMARY", "SPRAWL_AREAS", - "SPRAWL_DISTANCES", "SPRAWL_COOL_DISTANCE"] + "indicators" :["FREE_EXTERNAL_FACADE_DENSITY", "BUILDING_FRACTION"] //, "lcz_lod":2 ] ] ] //BDTopo.v2(bdTopoParameters) - + /* input_data = "/home/ebocher/Téléchargements/BDTOPO_3-3_TOUSTHEMES_SHP_LAMB93_D033_2023-12-15/BDTOPO/1_DONNEES_LIVRAISON_2023-12-00099" directory = "/tmp/bdtopo3" @@ -189,7 +175,7 @@ class WorkflowDebugTest { dirFile.mkdir() bdTopoParameters.input.folder=input_data - bdTopoParameters.output.folder.path=directory + bdTopoParameters.output.folder.path=directory*/ BDTopo.v3(bdTopoParameters) } diff --git a/geoindicators/src/main/groovy/org/orbisgis/geoclimate/geoindicators/WorkflowGeoIndicators.groovy b/geoindicators/src/main/groovy/org/orbisgis/geoclimate/geoindicators/WorkflowGeoIndicators.groovy index d7d5bfb4ff..0a9b581c65 100644 --- a/geoindicators/src/main/groovy/org/orbisgis/geoclimate/geoindicators/WorkflowGeoIndicators.groovy +++ b/geoindicators/src/main/groovy/org/orbisgis/geoclimate/geoindicators/WorkflowGeoIndicators.groovy @@ -1898,7 +1898,7 @@ String rasterizeIndicators(JdbcDataSource datasource, list_indicators_upper.each { if (it == "BUILDING_FRACTION" || it == "BUILDING_SURFACE_DENSITY" || - it == "ASPECT_RATIO") { + it == "ASPECT_RATIO" || it == "FREE_EXTERNAL_FACADE_DENSITY") { columnFractionsList.put(priorities.indexOf("building"), "building") } else if (it == "WATER_FRACTION") { columnFractionsList.put(priorities.indexOf("water"), "water") diff --git a/geoindicators/src/test/groovy/org/orbisgis/geoclimate/geoindicators/GenericIndicatorsTests.groovy b/geoindicators/src/test/groovy/org/orbisgis/geoclimate/geoindicators/GenericIndicatorsTests.groovy index 4a70d7d4ea..ded502f0df 100644 --- a/geoindicators/src/test/groovy/org/orbisgis/geoclimate/geoindicators/GenericIndicatorsTests.groovy +++ b/geoindicators/src/test/groovy/org/orbisgis/geoclimate/geoindicators/GenericIndicatorsTests.groovy @@ -371,6 +371,27 @@ class GenericIndicatorsTests { assertNull(p) } + @Test + void typeProportionTest3() { + // + h2GIS.load('/tmp/inputTableName.fgb', 'inputTableName') + h2GIS.load('/tmp/inputUpperTableName.fgb', 'inputUpperTableName') + + // Test 1 + def p = Geoindicators.GenericIndicators.typeProportion(h2GIS, + "inputTableName", "id_rsu", "type", + "inputUpperTableName", + ["undefined_lcz": ["building"], + "light_industry_lcz": ["industrial", "factory", "warehouse", "port"], + "commercial_lcz": ["commercial", "shop", "retail", "port", + "exhibition_centre", "cinema"], + "heavy_industry_lcz": ["refinery"], + "residential_lcz" : ["house", "detached", "bungalow", "farm", "apartments", "barracks", + "abbey", "condominium", "villa", "dormitory", "sheltered_housing", + "workers_dormitory", "terrace", "residential", "cabin"]], [:], "rsu_indicator_LCZ") + + } + @Test void gatherScalesTest() { h2GIS """ From 85d967740407871142c3ac504a15426a7d85c5f6 Mon Sep 17 00:00:00 2001 From: jeremy Date: Tue, 28 May 2024 10:30:47 +0200 Subject: [PATCH 3/4] Remove old test... --- .../GenericIndicatorsTests.groovy | 21 ------------------- 1 file changed, 21 deletions(-) diff --git a/geoindicators/src/test/groovy/org/orbisgis/geoclimate/geoindicators/GenericIndicatorsTests.groovy b/geoindicators/src/test/groovy/org/orbisgis/geoclimate/geoindicators/GenericIndicatorsTests.groovy index ded502f0df..4a70d7d4ea 100644 --- a/geoindicators/src/test/groovy/org/orbisgis/geoclimate/geoindicators/GenericIndicatorsTests.groovy +++ b/geoindicators/src/test/groovy/org/orbisgis/geoclimate/geoindicators/GenericIndicatorsTests.groovy @@ -371,27 +371,6 @@ class GenericIndicatorsTests { assertNull(p) } - @Test - void typeProportionTest3() { - // - h2GIS.load('/tmp/inputTableName.fgb', 'inputTableName') - h2GIS.load('/tmp/inputUpperTableName.fgb', 'inputUpperTableName') - - // Test 1 - def p = Geoindicators.GenericIndicators.typeProportion(h2GIS, - "inputTableName", "id_rsu", "type", - "inputUpperTableName", - ["undefined_lcz": ["building"], - "light_industry_lcz": ["industrial", "factory", "warehouse", "port"], - "commercial_lcz": ["commercial", "shop", "retail", "port", - "exhibition_centre", "cinema"], - "heavy_industry_lcz": ["refinery"], - "residential_lcz" : ["house", "detached", "bungalow", "farm", "apartments", "barracks", - "abbey", "condominium", "villa", "dormitory", "sheltered_housing", - "workers_dormitory", "terrace", "residential", "cabin"]], [:], "rsu_indicator_LCZ") - - } - @Test void gatherScalesTest() { h2GIS """ From 40dcedcbf8865000f9181218ef46dabf92f0327f Mon Sep 17 00:00:00 2001 From: ebocher Date: Wed, 29 May 2024 09:32:38 +0200 Subject: [PATCH 4/4] Allow workflow to only format the GIS layers --- .../bdtopo/AbstractBDTopoWorkflow.groovy | 2 +- .../geoclimate/osm/WorflowOSMTest.groovy | 54 +++++++++++++++++++ 2 files changed, 55 insertions(+), 1 deletion(-) diff --git a/bdtopo/src/main/groovy/org/orbisgis/geoclimate/bdtopo/AbstractBDTopoWorkflow.groovy b/bdtopo/src/main/groovy/org/orbisgis/geoclimate/bdtopo/AbstractBDTopoWorkflow.groovy index 773aaade43..aec6d9e74b 100644 --- a/bdtopo/src/main/groovy/org/orbisgis/geoclimate/bdtopo/AbstractBDTopoWorkflow.groovy +++ b/bdtopo/src/main/groovy/org/orbisgis/geoclimate/bdtopo/AbstractBDTopoWorkflow.groovy @@ -441,7 +441,7 @@ abstract class AbstractBDTopoWorkflow extends BDTopoUtils { * @return a filled map of parameters */ def extractProcessingParameters(def processing_parameters) { - def defaultParameters = [distance : 1000f, prefixName: "", + def defaultParameters = [distance : 500f, prefixName: "", hLevMin : 3] def rsu_indicators_default = [indicatorUse : [], svfSimplified : true, diff --git a/osm/src/test/groovy/org/orbisgis/geoclimate/osm/WorflowOSMTest.groovy b/osm/src/test/groovy/org/orbisgis/geoclimate/osm/WorflowOSMTest.groovy index 06c6d8ffe8..707679e3cf 100644 --- a/osm/src/test/groovy/org/orbisgis/geoclimate/osm/WorflowOSMTest.groovy +++ b/osm/src/test/groovy/org/orbisgis/geoclimate/osm/WorflowOSMTest.groovy @@ -858,6 +858,60 @@ class WorflowOSMTest extends WorkflowAbstractTest { assertTrue h2gis.firstRow("select count(*) as count from $building where HEIGHT_WALL>0 and HEIGHT_ROOF>0").count > 0 } + + @Test + void testCreateGISLayers() { + String directory = folder.absolutePath + File.separator + "test_creategislayers" + File dirFile = new File(directory) + dirFile.delete() + dirFile.mkdir() + def osm_parmeters = [ + "geoclimatedb": [ + "folder": dirFile.absolutePath, + "name" : "geoclimate_chain_db", + "delete": false + ], + "input" : [ + "locations": [[43.726898,7.298452,100]]], + "output" : [ + "folder": ["path" : directory, + "tables": ["building", "zone"]]] + ] + Map process = OSM.WorkflowOSM.workflow(osm_parmeters) + def tableNames = process.values() + def building = tableNames.building[0] + def zone = tableNames.zone[0] + H2GIS h2gis = H2GIS.open("${directory + File.separator}geoclimate_chain_db;AUTO_SERVER=TRUE") + assertTrue h2gis.firstRow("select count(*) as count from $building where HEIGHT_WALL>0 and HEIGHT_ROOF>0").count > 0 + assertEquals(1, h2gis.firstRow("select count(*) as count from $zone").count) + } + + @Test + void testCreateGISLayersNoOutput() { + String directory = folder.absolutePath + File.separator + "test_no_output" + directory = "/tmp/db" + File dirFile = new File(directory) + dirFile.delete() + dirFile.mkdir() + def osm_parmeters = [ + "geoclimatedb": [ + "folder": dirFile.absolutePath, + "name" : "geoclimate_chain_db", + "delete": false + ], + "input" : [ + "locations": [[43.726898,7.298452,100]]] + ] + Map process = OSM.WorkflowOSM.workflow(osm_parmeters) + def tableNames = process.values()[0] + H2GIS h2gis = H2GIS.open("${directory + File.separator}geoclimate_chain_db") + //All tables must exist in the database + tableNames.each {it-> + assertTrue(h2gis.hasTable(it.value)) + } + } + + @Disabled //Because it takes some time to build the OSM query @Test void testEstimateBuildingWithAllInputHeightDate() {