Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Many improvements on OSM module #871

Merged
merged 8 commits into from
Nov 8, 2023
Merged

Many improvements on OSM module #871

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
8459bd6
Many improvement on OSM module
ebocher Nov 8, 2023
8e3f3b8
Identify shelter
ebocher Nov 8, 2023
a62fb91
Identify roundhouse
ebocher Nov 8, 2023
5773f15
Merge branch 'master' into couple_of_fixes
ebocher Nov 8, 2023
59fb228
Merge branch 'master' into couple_of_fixes
ebocher Nov 8, 2023
83a5782
Fix a drop
ebocher Nov 8, 2023
cb6e219
Merge remote-tracking branch 'ebocher/couple_of_fixes' into couple_of…
ebocher Nov 8, 2023
b46ddd8
Fix a expected value
ebocher Nov 8, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
41 changes: 18 additions & 23 deletions osm/src/main/groovy/org/orbisgis/geoclimate/osm/InputDataFormatting.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ import java.util.regex.Pattern
* @return outputTableName The name of the final buildings table
* @return outputEstimatedTableName The name of the table containing the state of estimation for each building
*/
Map formatBuildingLayer(JdbcDataSource datasource, String building, String zone = "", String urban_areas = "", int h_lev_min = 3, String jsonFilename = null) {
Map formatBuildingLayer(JdbcDataSource datasource, String building, String zone = "", String urban_areas = "", int h_lev_min = 3, String jsonFilename = "") {
if (!h_lev_min) {
h_lev_min = 3
}
Expand Down Expand Up @@ -1056,7 +1056,6 @@ String formatSeaLandMask(JdbcDataSource datasource, String coastline, String zon
def coastLinesIntersectsPoints = "coatline_intersect_points_zone${UUID.randomUUID().toString().replaceAll("-", "_")}"
def coastLinesPoints = "coatline_points_zone${UUID.randomUUID().toString().replaceAll("-", "_")}"
def sea_land_mask = "sea_land_mask${UUID.randomUUID().toString().replaceAll("-", "_")}"
def sea_land_mask_in_zone = "sea_land_mask_in_zone${UUID.randomUUID().toString().replaceAll("-", "_")}"
def water_to_be_filtered = "water_to_be_filtered${UUID.randomUUID().toString().replaceAll("-", "_")}"
def water_filtered_exploded = "water_filtered_exploded${UUID.randomUUID().toString().replaceAll("-", "_")}"
def sea_land_triangles = "sea_land_triangles${UUID.randomUUID().toString().replaceAll("-", "_")}"
Expand All @@ -1066,10 +1065,10 @@ String formatSeaLandMask(JdbcDataSource datasource, String coastline, String zon
datasource.createSpatialIndex(coastline, "the_geom")
datasource.execute """DROP TABLE IF EXISTS $coastLinesIntersects,
$islands_mark, $mergingDataTable, $coastLinesIntersectsPoints, $coastLinesPoints,$sea_land_mask,
$sea_land_mask_in_zone,$water_filtered_exploded,$water_to_be_filtered, $sea_land_triangles, $sea_id_triangles, $water_id_triangles;
$water_filtered_exploded,$water_to_be_filtered, $sea_land_triangles, $sea_id_triangles, $water_id_triangles;
CREATE TABLE $coastLinesIntersects AS SELECT ST_intersection(a.the_geom, b.the_geom) as the_geom
from $coastline AS a, $zone AS b WHERE
a.the_geom && b.the_geom AND st_intersects(a.the_geom, b.the_geom);
a.the_geom && b.the_geom AND st_intersects(a.the_geom, b.the_geom) and "natural"= 'coastline';
""".toString()

if (water) {
Expand All @@ -1088,17 +1087,12 @@ String formatSeaLandMask(JdbcDataSource datasource, String coastline, String zon
UNION ALL
SELECT st_tomultiline(the_geom)
from $water ;

CREATE TABLE $sea_land_mask (THE_GEOM GEOMETRY,ID serial, TYPE VARCHAR, ZINDEX INTEGER) AS SELECT THE_GEOM, EXPLOD_ID, 'land', 0 AS ZINDEX FROM
st_explode('(SELECT st_polygonize(st_union(ST_NODE(st_accum(the_geom)))) AS the_geom FROM $mergingDataTable)'); """.toString()
st_explode('(SELECT st_polygonize(st_union(ST_NODE(st_accum(the_geom)))) AS the_geom FROM $mergingDataTable)')
as foo where ST_DIMENSION(the_geom) = 2 AND st_area(the_geom) >0; """.toString()

datasource.execute """
CREATE SPATIAL INDEX IF NOT EXISTS ${sea_land_mask}_the_geom_idx ON $sea_land_mask (THE_GEOM);

CREATE TABLE $sea_land_mask_in_zone as select the_geom, id, type, ZINDEX
from st_explode('(SELECT st_intersection(a.THE_GEOM, b.the_geom) as the_geom, a.id, a.type,a.ZINDEX
FROM $sea_land_mask as a, $zone as b WHERE a.the_geom && b.the_geom AND st_intersects(a.the_geom, b.the_geom))')
where ST_DIMENSION(the_geom) = 2 AND st_area(the_geom) >0;

CREATE SPATIAL INDEX IF NOT EXISTS ${islands_mark}_the_geom_idx ON $islands_mark (THE_GEOM);

Expand All @@ -1116,19 +1110,23 @@ String formatSeaLandMask(JdbcDataSource datasource, String coastline, String zon
CREATE TABLE $sea_land_triangles AS
SELECT * FROM
st_explode('(SELECT CASE WHEN ST_AREA(THE_GEOM) > 100000 THEN ST_Tessellate(the_geom) ELSE THE_GEOM END AS THE_GEOM,
ID, TYPE, ZINDEX FROM $sea_land_mask_in_zone)');
ID, TYPE, ZINDEX FROM $sea_land_mask)');

CREATE SPATIAL INDEX IF NOT EXISTS ${sea_land_triangles}_the_geom_idx ON $sea_land_triangles (THE_GEOM);

DROP TABLE IF EXISTS $sea_id_triangles;
CREATE TABLE $sea_id_triangles AS SELECT DISTINCT a.id FROM $sea_land_triangles a,
$coastLinesIntersectsPoints b WHERE a.THE_GEOM && b.THE_GEOM AND
st_contains(a.THE_GEOM, b.THE_GEOM);
CREATE INDEX ON $sea_id_triangles (id);

--Update sea triangles
st_intersects(a.THE_GEOM, b.THE_GEOM);
CREATE INDEX ON $sea_id_triangles (id);""".toString()

//Set the triangles to sea
datasource.execute """
UPDATE ${sea_land_triangles} SET TYPE='sea' WHERE ID IN(SELECT ID FROM $sea_id_triangles);

""".toString()

//Set the triangles to water
datasource.execute """
DROP TABLE IF EXISTS $water_id_triangles;
CREATE TABLE $water_id_triangles AS SELECT a.ID
FROM ${sea_land_triangles} a, $water b WHERE a.THE_GEOM && b.THE_GEOM AND
Expand Down Expand Up @@ -1160,13 +1158,10 @@ String formatSeaLandMask(JdbcDataSource datasource, String coastline, String zon
from $zone ;

CREATE TABLE $sea_land_mask (THE_GEOM GEOMETRY,ID serial, TYPE VARCHAR, ZINDEX INTEGER) AS SELECT THE_GEOM, EXPLOD_ID, 'land', 0 AS ZINDEX FROM
st_explode('(SELECT st_polygonize(st_union(ST_NODE(st_accum(the_geom)))) AS the_geom FROM $mergingDataTable)');
st_explode('(SELECT st_polygonize(st_union(ST_NODE(st_accum(the_geom)))) AS the_geom FROM $mergingDataTable)') as foo where ST_DIMENSION(the_geom) = 2 AND st_area(the_geom) >0;

CREATE SPATIAL INDEX IF NOT EXISTS ${sea_land_mask}_the_geom_idx ON $sea_land_mask (THE_GEOM);

CREATE TABLE $sea_land_mask_in_zone as select the_geom, id, type, ZINDEX from st_explode('(SELECT st_intersection(a.THE_GEOM, b.the_geom) as the_geom, a.id, a.type,a.ZINDEX
FROM $sea_land_mask as a, $zone as b where a.the_geom && b.the_geom AND st_intersects(a.the_geom, b.the_geom))') where ST_DIMENSION(the_geom) = 2 AND st_area(the_geom) >0;

CREATE SPATIAL INDEX IF NOT EXISTS ${islands_mark}_the_geom_idx ON $islands_mark (THE_GEOM);

CREATE TABLE $coastLinesPoints as SELECT ST_LocateAlong(the_geom, 0.5, -0.01) AS the_geom FROM
Expand All @@ -1186,7 +1181,7 @@ String formatSeaLandMask(JdbcDataSource datasource, String coastline, String zon
CREATE TABLE $sea_land_triangles AS
SELECT * FROM
st_explode('(SELECT CASE WHEN ST_AREA(THE_GEOM) > 100000 THEN ST_Tessellate(the_geom) ELSE THE_GEOM END AS THE_GEOM,
ID, TYPE, ZINDEX FROM $sea_land_mask_in_zone)');
ID, TYPE, ZINDEX FROM $sea_land_mask)');

CREATE SPATIAL INDEX IF NOT EXISTS ${sea_land_triangles}_the_geom_idx ON $sea_land_triangles (THE_GEOM);

Expand All @@ -1209,7 +1204,7 @@ String formatSeaLandMask(JdbcDataSource datasource, String coastline, String zon

datasource.execute("""DROP TABLE IF EXISTS $coastLinesIntersects,
$islands_mark, $mergingDataTable, $coastLinesIntersectsPoints, $coastLinesPoints,$sea_land_mask,
$sea_land_mask_in_zone,$water_filtered_exploded,$water_to_be_filtered, $sea_land_triangles, $sea_id_triangles, $water_id_triangles
$water_filtered_exploded,$water_to_be_filtered, $sea_land_triangles, $sea_id_triangles, $water_id_triangles
""".toString())
debug 'The sea/land mask has been computed'
return outputTableName
Expand Down
38 changes: 29 additions & 9 deletions osm/src/main/resources/org/orbisgis/geoclimate/osm/buildingParams.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,10 +1,6 @@
{
"tags": {
"building": [],
"railway": [
"station",
"train_station"
]
"building": []
},
"columns": [
"height",
Expand Down Expand Up @@ -183,7 +179,7 @@
"! church"
]
},
"sport:entertainment_arts_culture": {
"sport:sport": {
"building": [
"swimming_pool",
"fitness_centre",
Expand Down Expand Up @@ -212,7 +208,7 @@
"track"
]
},
"sports_centre:entertainment_arts_culture": {
"sports_centre:sport": {
"building": [
"sports_centre",
"sports_hall"
Expand Down Expand Up @@ -344,7 +340,7 @@
"kiosk",
"money_transfer",
"market",
"market_place",
"marketplace",
"pharmacy",
"post_office",
"retail",
Expand Down Expand Up @@ -519,6 +515,16 @@
"! no"
]
},
"shelter": {
"amenity": [
"shelter"
]
},
"shelter:transport": {
"shelter_type": [
"public_transport"
]
},
"transport": {
"building": [
"train_station",
Expand All @@ -535,7 +541,8 @@
"railway": [
"station",
"train_station",
"building"
"roundhouse"

],
"public_transport": [
"train_station",
Expand Down Expand Up @@ -585,6 +592,19 @@
"entertainment_arts_culture": {
"leisure": [
"! no"
], "amenity": [
"cinema","arts_centre", "brothel", "casino",
"community_centre", "conference_centre",
"events_venue", "exhibition_centre", "gambling",
"music_venue", "nightclub", "planetarium",
"social_centre","studio", "theatre"
],
"building": [
"cinema","arts_centre", "brothel", "casino",
"community_centre", "conference_centre",
"events_venue", "exhibition_centre", "gambling",
"music_venue", "nightclub", "planetarium",
"social_centre","studio", "theatre"
]
},
"sustenance:commercial": {
Expand Down
55 changes: 21 additions & 34 deletions osm/src/test/groovy/org/orbisgis/geoclimate/osm/InputDataFormattingTest.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -216,8 +216,8 @@ class InputDataFormattingTest {

//Sea/Land mask
String inputSeaLandTableName = OSM.InputDataFormatting.formatSeaLandMask(h2GIS, extractData.coastline, zoneEnvelopeTableName)
assertEquals(4, h2GIS.getTable(inputSeaLandTableName).getRowCount())
assertTrue h2GIS.firstRow("select count(*) as count from ${inputSeaLandTableName} where type='land'").count == 3
assertEquals(2, h2GIS.getTable(inputSeaLandTableName).getRowCount())
assertTrue h2GIS.firstRow("select count(*) as count from ${inputSeaLandTableName} where type='land'").count == 1
h2GIS.getTable(inputSeaLandTableName).save(new File(folder, "osm_sea_land.geojson").getAbsolutePath(), true)
}

Expand Down Expand Up @@ -309,11 +309,10 @@ class InputDataFormattingTest {
//aeroway Toulouse https://www.openstreetmap.org/way/739797641#map=14/43.6316/1.3590
zoneToExtract = [43.610539, 1.334152, 43.648808, 1.392689]

zoneToExtract = "Göteborgs Stad"
zoneToExtract = "Göteborgs Stad"

zoneToExtract = "Barcelona Eixample"

//zoneToExtract = [69.24666, 28.3359, 69.32999, 28.41923]
zoneToExtract = "Riantec"
zoneToExtract =[45.575525,5.913734,45.578859,5.919549]

Map extractData = OSM.InputDataLoading.extractAndCreateGISLayers(h2GIS, zoneToExtract)

Expand All @@ -332,37 +331,21 @@ class InputDataFormattingTest {
h2GIS.getTable(extractData.zone_envelope).save("${file.absolutePath + File.separator}osm_zone_envelope_${formatedPlaceName}.geojson", true)

//Urban Areas
/*format = OSM.InputDataFormatting.formatUrbanAreas()
format.execute([
datasource : h2GIS,
inputTableName : extractData.results.urbanAreasTableName,
inputZoneEnvelopeTableName: extractData.results.zoneEnvelopeTableName,
epsg : epsg])
def urbanAreasTableName = format.results.outputTableName;
h2GIS.getTable(format.results.outputTableName).save("./target/osm_urban_areas_${formatedPlaceName}.geojson", true)
def inputUrbanAreas = OSM.InputDataFormatting.formatUrbanAreas(h2GIS,
extractData.urban_areas,extractData.zone)
h2GIS.save(inputUrbanAreas,"${file.absolutePath + File.separator}osm_urban_areas_${formatedPlaceName}.geojson", true)

//Buildings
format = OSM.InputDataFormatting.formatBuildingLayer()
format.execute([
datasource : h2GIS,
inputTableName : extractData.results.buildingTableName,
inputZoneEnvelopeTableName: extractData.results.zoneEnvelopeTableName,
epsg : epsg,
urbanAreasTableName : urbanAreasTableName])
h2GIS.getTable(format.results.outputTableName).save("./target/osm_building_${formatedPlaceName}.geojson", true)
assertTrue h2GIS.firstRow("select count(*) as count from ${format.results.outputTableName} where NB_LEV is null").count == 0
assertTrue h2GIS.firstRow("select count(*) as count from ${format.results.outputTableName} where NB_LEV<0").count == 0
assertTrue h2GIS.firstRow("select count(*) as count from ${format.results.outputTableName} where HEIGHT_WALL is null").count == 0
assertTrue h2GIS.firstRow("select count(*) as count from ${format.results.outputTableName} where HEIGHT_WALL<0").count == 0
assertTrue h2GIS.firstRow("select count(*) as count from ${format.results.outputTableName} where HEIGHT_ROOF is null").count == 0
assertTrue h2GIS.firstRow("select count(*) as count from ${format.results.outputTableName} where HEIGHT_ROOF<0").count == 0
h2GIS.save(extractData.building,"${file.absolutePath + File.separator}building_${formatedPlaceName}.geojson", true)
def inputBuildings = OSM.InputDataFormatting.formatBuildingLayer(h2GIS,
extractData.building,extractData.zone,inputUrbanAreas)
h2GIS.save(inputBuildings.building,"${file.absolutePath + File.separator}osm_building_${formatedPlaceName}.geojson", true)


//Roads

*/
def inputRoadTableName = OSM.InputDataFormatting.formatRoadLayer( h2GIS,extractData.road, extractData.zone_envelope)
h2GIS.getTable(inputRoadTableName).save("${file.absolutePath + File.separator}osm_road_${formatedPlaceName}.geojson", true)
h2GIS.save(inputRoadTableName,"${file.absolutePath + File.separator}osm_road_${formatedPlaceName}.geojson", true)

//Rails
/*format = OSM.InputDataFormatting.formatRailsLayer()
Expand All @@ -377,22 +360,26 @@ class InputDataFormattingTest {
//Vegetation
def inputVegetationTableName = OSM.InputDataFormatting.formatVegetationLayer(
h2GIS,extractData.vegetation,extractData.zone_envelope)
h2GIS.getTable(inputVegetationTableName).save("${file.absolutePath + File.separator}osm_vegetation_${formatedPlaceName}.geojson", true)
h2GIS.save(inputVegetationTableName,"${file.absolutePath + File.separator}osm_vegetation_${formatedPlaceName}.geojson", true)


//Hydrography
def inputWaterTableName = OSM.InputDataFormatting.formatWaterLayer(h2GIS, extractData.water, extractData.zone_envelope)
h2GIS.getTable(inputWaterTableName).save("${file.absolutePath + File.separator}osm_water_${formatedPlaceName}.geojson", true)
h2GIS.save(inputWaterTableName,"${file.absolutePath + File.separator}osm_water_${formatedPlaceName}.geojson", true)

//Impervious
String imperviousTable = OSM.InputDataFormatting.formatImperviousLayer(h2GIS, extractData.impervious,
extractData.zone_envelope)
h2GIS.getTable(imperviousTable).save("${file.absolutePath + File.separator}osm_impervious_${formatedPlaceName}.geojson", true)
h2GIS.save(imperviousTable,"${file.absolutePath + File.separator}osm_impervious_${formatedPlaceName}.geojson", true)

//Save coastlines to debug
h2GIS.save(extractData.coastline,"${file.absolutePath + File.separator}osm_coastlines_${formatedPlaceName}.geojson", true)


//Sea/Land mask
def inputSeaLandTableName = OSM.InputDataFormatting.formatSeaLandMask(h2GIS, extractData.coastline,
extractData.zone_envelope, inputWaterTableName)
h2GIS.getTable(inputSeaLandTableName).save("${file.absolutePath + File.separator}osm_sea_land_${formatedPlaceName}.geojson", true)
h2GIS.save(inputSeaLandTableName,"${file.absolutePath + File.separator}osm_sea_land_${formatedPlaceName}.geojson", true)

} else {
assertTrue(false)
Expand Down
14 changes: 8 additions & 6 deletions osm/src/test/groovy/org/orbisgis/geoclimate/osm/WorflowOSMTest.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,6 @@ import org.junit.jupiter.api.io.TempDir
import org.orbisgis.data.H2GIS
import org.orbisgis.data.POSTGIS
import org.orbisgis.geoclimate.Geoindicators
import org.orbisgis.geoclimate.osmtools.OSMTools

import static org.junit.jupiter.api.Assertions.*

Expand Down Expand Up @@ -651,13 +650,14 @@ class WorflowOSMTest extends WorkflowAbstractTest {
File dirFile = new File(directory)
dirFile.delete()
dirFile.mkdir()
def location = "Nice"

def location = "Redon"
//def nominatim = OSMTools.Utilities.getNominatimData("Nantes")

location=[47.2, -1.6, 47.4, -1.4]
//location = nominatim.bbox

def nominatim = OSMTools.Utilities.getNominatimData("Redon")

location = nominatim.bbox
location = [33.8, 7.6, 34, 7.8]
def osm_parmeters = [
"description" : "Example of configuration file to run the OSM workflow and store the result in a folder",
"geoclimatedb": [
Expand All @@ -677,7 +677,9 @@ class WorflowOSMTest extends WorkflowAbstractTest {
"parameters" :
["distance" : 0,
"rsu_indicators" : [

"indicatorUse": ["LCZ", "TEB"] //, "UTRF", "TEB"]

]/*,"grid_indicators": [
"x_size": 200,
"y_size": 200,
Expand All @@ -688,7 +690,7 @@ class WorflowOSMTest extends WorkflowAbstractTest {
"ROAD_FRACTION", "IMPERVIOUS_FRACTION",
"LCZ_PRIMARY",
//"BUILDING_HEIGHT_WEIGHTED", //"BUILDING_SURFACE_DENSITY", "SEA_LAND_FRACTION",
"ASPECT_RATIO","SVF",
"ASPECT_RATIO",//"SVF",
"HEIGHT_OF_ROUGHNESS_ELEMENTS", "TERRAIN_ROUGHNESS_CLASS"]
], "worldpop_indicators": true,
"road_traffic" : true,
Expand Down