Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Improve OSM data extraction and OSM data formating #946

Merged
merged 3 commits into from
Mar 20, 2024
Merged

Improve OSM data extraction and OSM data formating #946

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
0f12ceb
Improve OSM data extraction and OSM data formating
ebocher Mar 19, 2024
7d1b7a6
Improve tsu, remove small gaps
ebocher Mar 19, 2024
8f8651b
Fix test
ebocher Mar 20, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
48 changes: 24 additions & 24 deletions bdtopo/src/main/groovy/org/orbisgis/geoclimate/bdtopo/AbstractBDTopoWorkflow.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -447,20 +447,20 @@ abstract class AbstractBDTopoWorkflow extends BDTopoUtils {
def defaultParameters = [distance : 1000f,
distance_buffer: 500f, prefixName: "",
hLevMin : 3]
def rsu_indicators_default = [indicatorUse : [],
svfSimplified : true,
surface_vegetation: 10000f,
surface_hydro : 2500f,
def rsu_indicators_default = [indicatorUse : [],
svfSimplified : true,
surface_vegetation : 10000f,
surface_hydro : 2500f,
surface_urban_areas: 10000f,
snappingTolerance : 0.01f,
mapOfWeights : ["sky_view_factor" : 4,
"aspect_ratio" : 3,
"building_surface_fraction" : 8,
"impervious_surface_fraction" : 0,
"pervious_surface_fraction" : 0,
"height_of_roughness_elements": 6,
"terrain_roughness_length" : 0.5],
utrfModelName : "UTRF_BDTOPO_V2_RF_2_2.model"]
snappingTolerance : 0.01f,
mapOfWeights : ["sky_view_factor" : 4,
"aspect_ratio" : 3,
"building_surface_fraction" : 8,
"impervious_surface_fraction" : 0,
"pervious_surface_fraction" : 0,
"height_of_roughness_elements": 6,
"terrain_roughness_length" : 0.5],
utrfModelName : "UTRF_BDTOPO_V2_RF_2_2.model"]
defaultParameters.put("rsu_indicators", rsu_indicators_default)

if (processing_parameters) {
Expand Down Expand Up @@ -564,13 +564,13 @@ abstract class AbstractBDTopoWorkflow extends BDTopoUtils {
def grid_indicators_tmp = [
"x_size" : x_size,
"y_size" : y_size,
"output" : "geojson",
"output" : "fgb",
"rowCol" : false,
"indicators": allowedOutputIndicators
]
def grid_output = grid_indicators.output
if (grid_output) {
if (grid_output.toLowerCase() in ["asc", "geojson"]) {
if (grid_output.toLowerCase() in ["asc", "fgb"]) {
grid_indicators_tmp.output = grid_output.toLowerCase()
}
}
Expand Down Expand Up @@ -675,7 +675,7 @@ abstract class AbstractBDTopoWorkflow extends BDTopoUtils {
//Add the GIS layers to the list of results
def outputTableNamesResult = [:]
def grid_indicators_params = processing_parameters.grid_indicators
def outputGrid = "geojson"
def outputGrid = "fgb"
if (grid_indicators_params) {
outputGrid = grid_indicators_params.output
}
Expand Down Expand Up @@ -936,7 +936,7 @@ abstract class AbstractBDTopoWorkflow extends BDTopoUtils {
Map geoIndicators = Geoindicators.WorkflowGeoIndicators.computeAllGeoIndicators(h2gis_datasource, zone,
building, road,
rail, vegetation,
water, impervious, "", "", urban_areas,"",
water, impervious, "", "", urban_areas, "",
rsu_indicators_params, processing_parameters.prefixName)
if (!geoIndicators) {
error "Cannot build the geoindicators for the zone $id_zone"
Expand All @@ -961,7 +961,7 @@ abstract class AbstractBDTopoWorkflow extends BDTopoUtils {
if (!building) {
info "Cannot compute any population data at building level"
}
tablesToDrop<<results.building
tablesToDrop << results.building
//Update the building table with the population data
results.put("building", building)

Expand Down Expand Up @@ -1003,12 +1003,12 @@ abstract class AbstractBDTopoWorkflow extends BDTopoUtils {
if (ground_acoustic) {
results.put("ground_acoustic", ground_acoustic)
}
tablesToDrop<<gridP
tablesToDrop << gridP
}
}
}
//Clean the database
if(tablesToDrop) {
if (tablesToDrop) {
h2gis_datasource.execute("DROP TABLE IF EXISTS ${tablesToDrop.join(",")}".toString())
}
info "${id_zone} has been processed"
Expand All @@ -1018,7 +1018,7 @@ abstract class AbstractBDTopoWorkflow extends BDTopoUtils {
}

/**
* Save the geoclimate tables into geojson files
* Save the geoclimate tables into files
* @param id_zone the id of the zone
* @param results a list of tables computed by geoclimate
* @param outputFolder the output folder
Expand All @@ -1029,13 +1029,13 @@ abstract class AbstractBDTopoWorkflow extends BDTopoUtils {
def saveOutputFiles(def h2gis_datasource, def results, def outputFiles, def outputFolder, def outputSRID, def reproject, def deleteOutputData, def outputGrid) {
outputFiles.each {
if (it == "grid_indicators") {
if (outputGrid == "geojson") {
Geoindicators.WorkflowUtilities.saveToGeojson(results."$it", "${outputFolder + File.separator + it}.geojson", h2gis_datasource, outputSRID, reproject, deleteOutputData)
if (outputGrid == "fgb") {
Geoindicators.WorkflowUtilities.saveInFile(results."$it", "${outputFolder + File.separator + it}.fgb", h2gis_datasource, outputSRID, reproject, deleteOutputData)
} else if (outputGrid == "asc") {
Geoindicators.WorkflowUtilities.saveToAscGrid(results."$it", outputFolder, it, h2gis_datasource, outputSRID, reproject, deleteOutputData)
}
} else {
Geoindicators.WorkflowUtilities.saveToGeojson(results."$it", "${outputFolder + File.separator + it}.geojson", h2gis_datasource, outputSRID, reproject, deleteOutputData)
Geoindicators.WorkflowUtilities.saveInFile(results."$it", "${outputFolder + File.separator + it}.fgb", h2gis_datasource, outputSRID, reproject, deleteOutputData)
}
}
}
Expand Down
7 changes: 3 additions & 4 deletions bdtopo/src/test/groovy/org/orbisgis/geoclimate/bdtopo/WorkflowAbstractTest.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,8 +30,7 @@ import static org.junit.jupiter.api.Assertions.*

abstract class WorkflowAbstractTest {

@TempDir(cleanup = CleanupMode.ON_SUCCESS)
static File folder


/**
* Get the version of the workflow
Expand Down Expand Up @@ -66,7 +65,7 @@ abstract class WorkflowAbstractTest {
if(dataFolder){
def files = [:]
new File(dataFolder).eachFileRecurse groovy.io.FileType.FILES, { file ->
if (file.name.toLowerCase().endsWith(".geojson")) {
if (file.name.toLowerCase().endsWith(".fgb")) {
files.put((file.name.take(file.name.lastIndexOf('.'))), file.getAbsolutePath())
}
}
Expand Down Expand Up @@ -210,7 +209,7 @@ abstract class WorkflowAbstractTest {
H2GIS h2gis = H2GIS.open("${folder.absolutePath + File.separator}testFullWorflowSRID;AUTO_SERVER=TRUE")
assertTrue h2gis.firstRow("select count(*) as count from $grid_table".toString()).count == 100
assertTrue h2gis.firstRow("select count(*) as count from $grid_table where BUILDING_FRACTION>0".toString()).count > 0
File grid_file = new File(folder.absolutePath + File.separator + "bdtopo_" + getVersion() + "_" + location.join("_") + File.separator + "grid_indicators.geojson")
File grid_file = new File(folder.absolutePath + File.separator + "bdtopo_" + getVersion() + "_" + location.join("_") + File.separator + "grid_indicators.fgb")
assertTrue(grid_file.exists())
h2gis.load(grid_file.absolutePath, "grid_indicators_file", true)
assertEquals(4326, h2gis.getSpatialTable("grid_indicators_file").srid)
Expand Down
27 changes: 4 additions & 23 deletions bdtopo/src/test/groovy/org/orbisgis/geoclimate/bdtopo/v2/WorkflowBDTopoV2Test.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,36 +19,17 @@
*/
package org.orbisgis.geoclimate.bdtopo.v2

import org.junit.jupiter.api.io.CleanupMode
import org.junit.jupiter.api.io.TempDir
import org.orbisgis.data.H2GIS
import org.orbisgis.geoclimate.bdtopo.WorkflowAbstractTest

import static org.junit.jupiter.api.Assertions.assertEquals
import static org.junit.jupiter.api.Assertions.assertEquals
import static org.junit.jupiter.api.Assertions.assertEquals
import static org.junit.jupiter.api.Assertions.assertEquals
import static org.junit.jupiter.api.Assertions.assertEquals
import static org.junit.jupiter.api.Assertions.assertEquals
import static org.junit.jupiter.api.Assertions.assertEquals
import static org.junit.jupiter.api.Assertions.assertEquals
import static org.junit.jupiter.api.Assertions.assertEquals
import static org.junit.jupiter.api.Assertions.assertEquals
import static org.junit.jupiter.api.Assertions.assertEquals
import static org.junit.jupiter.api.Assertions.assertEquals
import static org.junit.jupiter.api.Assertions.assertEquals
import static org.junit.jupiter.api.Assertions.assertEquals
import static org.junit.jupiter.api.Assertions.assertEquals
import static org.junit.jupiter.api.Assertions.assertEquals
import static org.junit.jupiter.api.Assertions.assertEquals
import static org.junit.jupiter.api.Assertions.assertEquals
import static org.junit.jupiter.api.Assertions.assertEquals
import static org.junit.jupiter.api.Assertions.assertTrue
import static org.junit.jupiter.api.Assertions.assertTrue
import static org.junit.jupiter.api.Assertions.assertTrue
import static org.junit.jupiter.api.Assertions.assertTrue
import static org.junit.jupiter.api.Assertions.assertTrue

class WorkflowBDTopoV2Test extends WorkflowAbstractTest {

@TempDir(cleanup = CleanupMode.ON_SUCCESS)
static File folder

@Override
ArrayList getFileNames() {
Expand Down
10 changes: 5 additions & 5 deletions bdtopo/src/test/groovy/org/orbisgis/geoclimate/bdtopo/v3/WorkflowBDTopoV3Test.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,8 @@
package org.orbisgis.geoclimate.bdtopo.v3


import org.junit.jupiter.api.*
import org.junit.jupiter.api.io.CleanupMode
import org.junit.jupiter.api.io.TempDir
import org.orbisgis.data.H2GIS
import org.orbisgis.geoclimate.bdtopo.WorkflowAbstractTest

Expand All @@ -29,11 +30,10 @@ import static org.junit.jupiter.api.Assertions.assertTrue

class WorkflowBDTopoV3Test extends WorkflowAbstractTest {

@TempDir(cleanup = CleanupMode.ON_SUCCESS)
static File folder


public WorkflowBDTopoV3Test(){
new File("/tmp/test_bd").mkdir()
folder = new File("/tmp/test_bd")
}
@Override
int getVersion() {
return 3
Expand Down
20 changes: 11 additions & 9 deletions geoindicators/src/main/groovy/org/orbisgis/geoclimate/geoindicators/DataUtils.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -89,14 +89,16 @@ String joinTables(JdbcDataSource datasource, Map inputTableNamesWithId, String o
/**
* An utility process to save several tables in a folder
*
*
* @param datasource connection to the database
* @param inputTableNames to be stored in the directory.
* Note : A spatial table is saved in a geojson file and the other in csv
* Note : A spatial table is saved in a flatgeobuffer file and the other in csv
* @param delete true to delete the file is exist
* @param directory folder to save the tables
* @param datasource connection to the database
*
* @return the directory where the tables are saved
*/
String saveTablesAsFiles(JdbcDataSource datasource, List inputTableNames, boolean delete = false, String directory) {
String saveTablesAsFiles(JdbcDataSource datasource, List inputTableNames, boolean delete = true, String directory) {
if (directory == null) {
error "The directory to save the data cannot be null"
return
Expand All @@ -113,7 +115,7 @@ String saveTablesAsFiles(JdbcDataSource datasource, List inputTableNames, boolea
inputTableNames.each { tableName ->
if (tableName) {
def fileToSave = dirFile.absolutePath + File.separator + tableName +
(datasource."$tableName".spatial ? ".geojson" : ".csv")
(datasource."$tableName".spatial ? ".fgb" : ".csv")
def table = datasource.getTable(tableName)
if (table) {
table.save(fileToSave, delete)
Expand Down Expand Up @@ -164,10 +166,10 @@ static String aliasColumns(JdbcDataSource datasource, String tableName, String a

/**
* Create the select projection and alias all columns
* @param datasource
* @param tableName
* @param alias
* @param exceptColumns
* @param datasource connection to the database
* @param tableName table name
* @param alias for the columns
* @param exceptColumns columns to remove
* @return
*/
static String aliasColumns(JdbcDataSource datasource, def tableName, def alias, def exceptColumns){
Expand All @@ -176,4 +178,4 @@ static String aliasColumns(JdbcDataSource datasource, def tableName, def alias,
return columnNames.inject([]) { result, iter ->
result += "$alias.$iter"
}.join(",")
}
}
2 changes: 1 addition & 1 deletion geoindicators/src/main/groovy/org/orbisgis/geoclimate/geoindicators/GenericIndicators.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -108,7 +108,7 @@ String unweightedOperationFromLowerScale(JdbcDataSource datasource, String input
}
}
} else {
warn """ The column $var doesn't exist or should be numeric"""
debug("""The column $var doesn't exist or should be numeric""")
}
}
query += "b.$inputIdUp FROM $inputLowerScaleTableName a RIGHT JOIN $inputUpperScaleTableName b " +
Expand Down
20 changes: 13 additions & 7 deletions geoindicators/src/main/groovy/org/orbisgis/geoclimate/geoindicators/SpatialUnits.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -123,23 +123,29 @@ String createTSU(JdbcDataSource datasource, String inputTableName, String inputz
return null
}
if (inputzone) {
datasource.createSpatialIndex(inputTableName, "the_geom")
String polygons = postfix("polygons")
datasource.execute("""DROP TABLE IF EXISTS $polygons;
CREATE TABLE $polygons as SELECT EXPLOD_ID AS $COLUMN_ID_NAME, ST_SETSRID(ST_BUFFER(ST_BUFFER(the_geom,-0.01, 'join=mitre'), 0.01, 'join=mitre'), $epsg) AS the_geom FROM
ST_EXPLODE('(SELECT ST_POLYGONIZE(ST_UNION(ST_NODE(ST_ACCUM(the_geom)))) AS the_geom
FROM $inputTableName)')
""".toString())
datasource.createSpatialIndex(polygons)
datasource.createSpatialIndex(inputTableName)
datasource """
DROP TABLE IF EXISTS $outputTableName;
CREATE TABLE $outputTableName AS
SELECT EXPLOD_ID AS $COLUMN_ID_NAME, ST_SETSRID(ST_BUFFER(a.the_geom,0.01), $epsg) AS the_geom
FROM ST_EXPLODE('(
SELECT ST_POLYGONIZE(ST_UNION(ST_NODE(ST_ACCUM(the_geom)))) AS the_geom
FROM $inputTableName)') AS a,
SELECT a.*
FROM $polygons AS a,
$inputzone AS b
WHERE a.the_geom && b.the_geom
AND ST_INTERSECTS(ST_POINTONSURFACE(a.THE_GEOM), b.the_geom) and st_area(a.the_geom) > $area
AND ST_INTERSECTS(ST_POINTONSURFACE(a.THE_GEOM), b.the_geom) and st_area(a.the_geom) > $area;
DROP TABLE IF EXISTS $polygons;
""".toString()
} else {
datasource """
DROP TABLE IF EXISTS $outputTableName;
CREATE TABLE $outputTableName AS
SELECT EXPLOD_ID AS $COLUMN_ID_NAME, ST_SETSRID(st_buffer(the_geom, -0.01), $epsg) AS the_geom
SELECT EXPLOD_ID AS $COLUMN_ID_NAME, ST_SETSRID(ST_BUFFER(ST_BUFFER(the_geom,-0.01, 'join=mitre'), 0.01, 'join=mitre'), $epsg) AS the_geom
FROM ST_EXPLODE('(
SELECT ST_POLYGONIZE(ST_UNION(ST_NODE(ST_ACCUM(the_geom)))) AS the_geom
FROM $inputTableName)') where st_area(the_geom) > $area""".toString()
Expand Down
4 changes: 2 additions & 2 deletions geoindicators/src/main/groovy/org/orbisgis/geoclimate/geoindicators/WorkflowUtilities.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -212,15 +212,15 @@ def saveToAscGrid(def outputTable, def subFolder, def filePrefix, JdbcDataSource
}

/**
* Method to save a table into a geojson file
* Method to save a table into a file
* @param outputTable name of the table to export
* @param filePath path to save the table
* @param h2gis_datasource connection to the database
* @param outputSRID srid code to reproject the outputTable.
* @param reproject true if the file must be reprojected
* @param deleteOutputData true to delete the file if exists
*/
def saveToGeojson(def outputTable, def filePath, H2GIS h2gis_datasource, def outputSRID, def reproject, def deleteOutputData) {
def saveInFile(def outputTable, def filePath, H2GIS h2gis_datasource, def outputSRID, def reproject, def deleteOutputData) {
if (outputTable && h2gis_datasource.hasTable(outputTable)) {
if (!reproject) {
h2gis_datasource.save(outputTable, filePath, deleteOutputData)
Expand Down
2 changes: 1 addition & 1 deletion geoindicators/src/test/groovy/org/orbisgis/geoclimate/geoindicators/DataUtilsTests.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,7 @@ class DataUtilsTests {
directory)
assert p

assert 1 == h2GIS.table(h2GIS.load(directory + File.separator + "tablegeom.geojson", true)).rowCount
assert 1 == h2GIS.table(h2GIS.load(directory + File.separator + "tablegeom.fgb", true)).rowCount
assert 1 == h2GIS.table(h2GIS.load(directory + File.separator + "tablea.csv", true)).rowCount
}
}
4 changes: 2 additions & 2 deletions ...est/groovy/org/orbisgis/geoclimate/geoindicators/GeoIndicatorsExtensionModuleTests.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ class GeoIndicatorsExtensionModuleTests {
@Test
void saveGeometryToFile() {
Geometry geom = new GeometryFactory().createPoint(new Coordinate(10, 10))
File outputFile = new File(folder, "geometry.geojson")
File outputFile = new File(folder, "geometry.fgb")
geom.save(h2GIS, outputFile.getAbsolutePath())
h2GIS.load(outputFile.getAbsolutePath(), "mygeom", true)
ISpatialTable table = h2GIS.getSpatialTable("mygeom")
Expand All @@ -58,7 +58,7 @@ class GeoIndicatorsExtensionModuleTests {
void saveGeometrySridToFile() {
Geometry geom = new GeometryFactory().createPoint(new Coordinate(10, 10))
geom.setSRID(4326)
File outputFile = new File(folder, "geometry.geojson")
File outputFile = new File(folder, "geometry.fgb")
geom.save(h2GIS, outputFile.getAbsolutePath())
h2GIS.load(outputFile.getAbsolutePath(), "mygeom", true)
ISpatialTable table = h2GIS.getSpatialTable("mygeom")
Expand Down
2 changes: 1 addition & 1 deletion ...dicators/src/test/groovy/org/orbisgis/geoclimate/geoindicators/GridIndicatorsTests.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,7 @@ class GridIndicatorsTests {
from $grid_scale
group by ID_ROW_LOD_${i}, ID_COL_LOD_${i};
""".toString())
h2GIS.save(grid_lod, "/tmp/grid_lod_${i}.geojson", true)
h2GIS.save(grid_lod, "/tmp/grid_lod_${i}.fgb", true)
h2GIS.dropTable(grid_lod)
}
}
Expand Down
2 changes: 1 addition & 1 deletion ...rs/src/test/groovy/org/orbisgis/geoclimate/geoindicators/PopulationIndicatorsTests.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,7 @@ class PopulationIndicatorsTests {
"building", "rsu", "grid")

results.each { it ->
h2GIS.save(it.value, "./target/${it.value}.geojson", true)
h2GIS.save(it.value, "./target/${it.value}.fgb", true)
}

def rows = h2GIS.rows("SELECT id_build, pop from ${results.buildingTable} order by id_build".toString())
Expand Down
Loading
Loading