Skip to content

Commit

Permalink
Merge pull request #3402 from matsim-org/noiseAnalysis
Browse files Browse the repository at this point in the history 
Include Damages in NoiseDashboard
  • Loading branch information
@tschlenther
tschlenther authored Aug 9, 2024
2 parents 4c42086 + 2d8aece commit 9948285
Show file tree
Hide file tree
Showing 11 changed files with 366 additions and 122 deletions.
97 changes: 52 additions & 45 deletions ...ibs/application/src/main/java/org/matsim/application/analysis/noise/MergeNoiseOutput.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,11 +13,14 @@
import org.apache.avro.file.DataFileWriter;
import org.apache.avro.io.DatumWriter;
import org.apache.avro.specific.SpecificDatumWriter;
import org.apache.commons.lang.StringUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.matsim.api.core.v01.Coord;
import org.matsim.application.avro.XYTData;
import org.matsim.core.config.Config;
import org.matsim.core.utils.io.IOUtils;
import org.matsim.core.utils.misc.Time;
import tech.tablesaw.api.*;
import tech.tablesaw.io.csv.CsvReadOptions;

Expand All @@ -29,7 +32,8 @@
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.*;
import java.util.stream.Collectors;

import static org.geotools.gml3.v3_2.GML.coordinateSystem;

/**
* Merges noise data from multiple files into one file.
Expand All @@ -44,17 +48,16 @@ final class MergeNoiseOutput {
*/
private static final boolean CREATE_CSV_FILES = false;

private final String[] inputPath;
private final Path outputDirectory;
private final String crs;
private final String[] labels = {"immission", "emission"};
private final int minTime = 3600;
private int maxTime = 24 * 3600;

MergeNoiseOutput(String[] inputPath, Path outputDirectory, String crs) {
this.inputPath = inputPath;
this.outputDirectory = outputDirectory;
this.crs = crs;
private final Map<String,Float> totalReceiverPointValues = new HashMap<>();

MergeNoiseOutput(Path path, String coordinateSystem ) {
this.outputDirectory = path;
this.crs = coordinateSystem;
}

/**
Expand Down Expand Up @@ -90,25 +93,9 @@ public void setMaxTime(int maxTime) {
* Merges noise data from multiple files into one file.
*/
public void run() {

// Loop over all paths
for (int i = 0; i < labels.length; i++) {

// Select the correct method based on the label
switch (labels[i]) {
case "immission" -> {
if (CREATE_CSV_FILES) {
mergeImmissionsCSV(inputPath[i], labels[i]);
} else {
mergeImissions(inputPath[i], labels[i]);
}

}
case "emission" -> mergeEmissions(inputPath[i], labels[i]);
default -> log.warn("Unknown path: " + inputPath[i]);
}

}
mergeReceiverPointData(outputDirectory + "/immissions/", "immission");
mergeReceiverPointData(outputDirectory + "/damages_receiverPoint/", "damages_receiverPoint");
mergeLinkData(outputDirectory.toString() + "/emissions/", "emission");
}

/**
Expand All @@ -118,6 +105,7 @@ public void run() {
* @param output
*/
private void writeAvro(XYTData xytData, File output) {
log.info(String.format("Start writing avro file to %s", output.toString() ));
DatumWriter<XYTData> datumWriter = new SpecificDatumWriter<>(XYTData.class);
try (DataFileWriter<XYTData> dataFileWriter = new DataFileWriter<>(datumWriter)) {
dataFileWriter.setCodec(CodecFactory.deflateCodec(9));
Expand All @@ -128,7 +116,7 @@ private void writeAvro(XYTData xytData, File output) {
}
}

private void mergeEmissions(String pathParameter, String label) {
private void mergeLinkData(String pathParameter, String label) {
log.info("Merging emissions data for label {}", label);
Object2DoubleMap<String> mergedData = new Object2DoubleOpenHashMap<>();
Table csvOutputMerged = Table.create(TextColumn.create("Link Id"), DoubleColumn.create("value"));
Expand All @@ -143,9 +131,8 @@ private void mergeEmissions(String pathParameter, String label) {
.separator(';').build());

for (Row row : table) {
// index for Noise Emission xx:xx:xx -> 7
String linkId = row.getString("Link Id");
double value = row.getDouble(7);
double value = row.getDouble(row.columnCount() - 1);
mergedData.mergeDouble(linkId, value, Double::max);

}
Expand All @@ -165,38 +152,49 @@ private void mergeEmissions(String pathParameter, String label) {
}

/**
* Merges the immissions data
* Merges receiverPoint data (written by {@link org.matsim.contrib.noise.NoiseWriter}
*
* @param pathParameter path to the immissions data
* @param label label for the immissions data
* @param outputDir path to the receiverPoint data
* @param label label for the receiverPoint data (which kind of data)
*/
private void mergeImissions(String pathParameter, String label) {
private void mergeReceiverPointData(String outputDir, String label) {

// data per time step, maps coord to value
Int2ObjectMap<Object2FloatMap<FloatFloatPair>> data = new Int2ObjectOpenHashMap<>();

// Loop over all files
//TODO could be adjusted to time bin size from noise config group
String substrToCapitalize = null;
for (int time = minTime; time <= maxTime; time += 3600) {

String path = pathParameter + label + "_" + round(time, 1) + ".csv";
String timeDataFile = outputDir + label + "_" + round(time, 1) + ".csv";

Object2FloatOpenHashMap<FloatFloatPair> values = new Object2FloatOpenHashMap<>();

if (!Files.exists(Path.of(path))) {
log.warn("File {} does not exist", path);
if (!Files.exists(Path.of(timeDataFile))) {
log.warn("File {} does not exist", timeDataFile);
continue;
}

// Read the file
Table table = Table.read().csv(CsvReadOptions.builder(IOUtils.getBufferedReader(path))
.columnTypesPartial(Map.of("x", ColumnType.FLOAT, "y", ColumnType.FLOAT, "Receiver Point Id", ColumnType.INTEGER, "t", ColumnType.DOUBLE))
//we need "damages_receiverPoint" -> "Damages 01:00:00" and "immission" -> "Immision 01:00:00"
substrToCapitalize = label.contains("_") ? label.substring(0, label.lastIndexOf("_")) : label;
String valueHeader = StringUtils.capitalize(substrToCapitalize) + " " + Time.writeTime(time, Time.TIMEFORMAT_HHMMSS);

// Read the data file
Table dataTable = Table.read().csv(CsvReadOptions.builder(IOUtils.getBufferedReader(timeDataFile))
.columnTypesPartial(Map.of("x", ColumnType.FLOAT,
"y", ColumnType.FLOAT,
"Receiver Point Id", ColumnType.INTEGER,
"t", ColumnType.DOUBLE,
valueHeader, ColumnType.DOUBLE))
.sample(false)
.separator(';').build());

// Loop over all rows in the file
for (Row row : table) {
// Loop over all rows in the data file
for (Row row : dataTable) {
float x = row.getFloat("x");
float y = row.getFloat("y");
float value = (float) row.getDouble(1); // 1
float value = (float) row.getDouble(valueHeader);
FloatFloatPair coord = FloatFloatPair.of(x, y);
values.put(coord, value);
}
Expand Down Expand Up @@ -232,7 +230,7 @@ private void mergeImissions(String pathParameter, String label) {
}
}

xytHourData.setData(Map.of("imissions", raw));
xytHourData.setData(Map.of(label, raw));
xytHourData.setCrs(crs);

File out = outputDirectory.getParent().resolve(label + "_per_hour.avro").toFile();
Expand All @@ -254,15 +252,18 @@ private void mergeImissions(String pathParameter, String label) {
xytDayData.setTimestamps(List.of(0));
xytDayData.setXCoords(xCoords);
xytDayData.setYCoords(yCoords);
xytDayData.setData(Map.of("imissions", raw));
xytDayData.setData(Map.of(label, raw));
xytDayData.setCrs(crs);

File outDay = outputDirectory.getParent().resolve(label + "_per_day.avro").toFile();

writeAvro(xytDayData, outDay);
//cache the overall sum
this.totalReceiverPointValues.put(substrToCapitalize, raw.stream().reduce(0f, Float::sum));
}

// Merges the immissions data

@Deprecated
private void mergeImmissionsCSV(String pathParameter, String label) {
log.info("Merging immissions data for label {}", label);
Expand All @@ -278,7 +279,10 @@ private void mergeImmissionsCSV(String pathParameter, String label) {

// Read the file
Table table = Table.read().csv(CsvReadOptions.builder(IOUtils.getBufferedReader(path))
.columnTypesPartial(Map.of("x", ColumnType.DOUBLE, "y", ColumnType.DOUBLE, "Receiver Point Id", ColumnType.INTEGER, "t", ColumnType.DOUBLE))
.columnTypesPartial(Map.of("x", ColumnType.DOUBLE,
"y", ColumnType.DOUBLE,
"Receiver Point Id", ColumnType.INTEGER,
"t", ColumnType.DOUBLE))
.sample(false)
.separator(';').build());

Expand Down Expand Up @@ -319,4 +323,7 @@ private void mergeImmissionsCSV(String pathParameter, String label) {
log.info("Merged noise data written to {} ", outPerDay);

}
public Map<String, Float> getTotalReceiverPointValues() {
return totalReceiverPointValues;
}
}
92 changes: 69 additions & 23 deletions contribs/application/src/main/java/org/matsim/application/analysis/noise/NoiseAnalysis.java
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
package org.matsim.application.analysis.noise;

import org.apache.commons.csv.CSVFormat;
import org.apache.commons.csv.CSVPrinter;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.locationtech.jts.geom.Envelope;
Expand All @@ -19,9 +21,15 @@
import org.matsim.core.config.ConfigUtils;
import org.matsim.core.scenario.ScenarioUtils;
import org.matsim.core.utils.geometry.CoordinateTransformation;
import org.matsim.core.utils.io.IOUtils;
import picocli.CommandLine;

import java.io.IOException;
import java.nio.file.Path;
import java.text.DecimalFormat;
import java.text.DecimalFormatSymbols;
import java.util.Locale;
import java.util.Map;
import java.util.Objects;
import java.util.Set;

Expand All @@ -36,7 +44,10 @@
produces = {
"emission_per_day.csv",
"immission_per_day.%s",
"immission_per_hour.%s"
"immission_per_hour.%s",
"damages_receiverPoint_per_hour.%s",
"damages_receiverPoint_per_day.%s",
"noise_stats.csv"
}
)
public class NoiseAnalysis implements MATSimAppCommand {
Expand All @@ -55,7 +66,7 @@ public class NoiseAnalysis implements MATSimAppCommand {
private final SampleOptions sampleOptions = new SampleOptions();

@CommandLine.Option(names = "--consider-activities", split = ",", description = "Considered activities for noise calculation." +
" Use asterisk ('*') for acttype prefixes, if all such acts shall be considered.", defaultValue = "h,w,home*,work*")
" Use asterisk ('*') for acttype prefixes, if all such acts shall be considered.", defaultValue = "home*,work*,educ*,leisure*")
private Set<String> considerActivities;

@CommandLine.Option(names = "--noise-barrier", description = "Path to the noise barrier File", defaultValue = "")
Expand All @@ -71,32 +82,55 @@ public Integer call() throws Exception {

config.controller().setOutputDirectory(input.getRunDirectory().toString());

// adjust the default noise parameters
//trying to set noise parameters more explicitly, here...
//if NoiseConfigGroup was added before. do not override (most) parameters
boolean overrideParameters = ! ConfigUtils.hasModule(config, NoiseConfigGroup.class);
NoiseConfigGroup noiseParameters = ConfigUtils.addOrGetModule(config, NoiseConfigGroup.class);
noiseParameters.setConsideredActivitiesForReceiverPointGridArray(considerActivities.toArray(String[]::new));
noiseParameters.setConsideredActivitiesForDamageCalculationArray(considerActivities.toArray(String[]::new));
if (shp.getShapeFile() != null) {
CoordinateTransformation ct = shp.createInverseTransformation(config.global().getCoordinateSystem());

Envelope bbox = shp.getGeometry().getEnvelopeInternal();
if(overrideParameters){
log.warn("no NoiseConfigGroup was configured before. Will set some standards. You should check the next lines in the log file!");
noiseParameters.setConsideredActivitiesForReceiverPointGridArray(considerActivities.toArray(String[]::new));
noiseParameters.setConsideredActivitiesForDamageCalculationArray(considerActivities.toArray(String[]::new));

Coord minCoord = ct.transform(new Coord(bbox.getMinX(), bbox.getMinY()));
Coord maxCoord = ct.transform(new Coord(bbox.getMaxX(), bbox.getMaxY()));
//use actual speed and not freespeed
noiseParameters.setUseActualSpeedLevel(true);
//use the valid speed range (recommended by IK)
noiseParameters.setAllowForSpeedsOutsideTheValidRange(false);

noiseParameters.setReceiverPointsGridMinX(minCoord.getX());
noiseParameters.setReceiverPointsGridMinY(minCoord.getY());
noiseParameters.setReceiverPointsGridMaxX(maxCoord.getX());
noiseParameters.setReceiverPointsGridMaxY(maxCoord.getY());
}
if (shp.getShapeFile() != null) {
CoordinateTransformation ct = shp.createInverseTransformation(config.global().getCoordinateSystem());

Envelope bbox = shp.getGeometry().getEnvelopeInternal();

Coord minCoord = ct.transform(new Coord(bbox.getMinX(), bbox.getMinY()));
Coord maxCoord = ct.transform(new Coord(bbox.getMaxX(), bbox.getMaxY()));

noiseParameters.setReceiverPointsGridMinX(minCoord.getX());
noiseParameters.setReceiverPointsGridMinY(minCoord.getY());
noiseParameters.setReceiverPointsGridMaxX(maxCoord.getX());
noiseParameters.setReceiverPointsGridMaxY(maxCoord.getY());
}

noiseParameters.setNoiseComputationMethod(NoiseConfigGroup.NoiseComputationMethod.RLS19);
noiseParameters.setNoiseComputationMethod(NoiseConfigGroup.NoiseComputationMethod.RLS19);

if (!Objects.equals(noiseBarrierFile, "")) {
noiseParameters.setNoiseBarriersSourceCRS(config.global().getCoordinateSystem());
noiseParameters.setConsiderNoiseBarriers(true);
noiseParameters.setNoiseBarriersFilePath(noiseBarrierFile);
if (!Objects.equals(noiseBarrierFile, "")) {
noiseParameters.setNoiseBarriersSourceCRS(config.global().getCoordinateSystem());
noiseParameters.setConsiderNoiseBarriers(true);
noiseParameters.setNoiseBarriersFilePath(noiseBarrierFile);
}
} else {
log.warn("will override a few settings in NoiseConfigGroup, as we are now doing postprocessing and do not want any internalization etc." +
" You should check the next lines in the log file!");
}

// we only mean to do postprocessing here, thus no internalization etc
noiseParameters.setInternalizeNoiseDamages(false);
noiseParameters.setComputeCausingAgents(false);
//we don't need events (for Dashboard) - spare disk space.
noiseParameters.setThrowNoiseEventsAffected(false);
noiseParameters.setThrowNoiseEventsCaused(false);
noiseParameters.setComputeNoiseDamages(true);

if(! sampleOptions.isSet() && noiseParameters.getScaleFactor() == 1d){
log.warn("You didn't provide the simulation sample size via command line option --sample-size! This means, noise damages are not scaled!!!");
} else if (noiseParameters.getScaleFactor() == 1d){
Expand All @@ -111,23 +145,35 @@ public Integer call() throws Exception {

String outputFilePath = output.getPath().getParent() == null ? "." : output.getPath().getParent().toString();

log.info("starting " + NoiseOfflineCalculation.class + " with the following parameters:\n"
+ noiseParameters);

NoiseOfflineCalculation noiseCalculation = new NoiseOfflineCalculation(scenario, outputFilePath);
outputFilePath += "/noise-analysis";
noiseCalculation.run();

ProcessNoiseImmissions process = new ProcessNoiseImmissions(outputFilePath + "/immissions/", outputFilePath + "/receiverPoints/receiverPoints.csv", noiseParameters.getReceiverPointGap());
process.run();

final String[] paths = {outputFilePath + "/immissions/", outputFilePath + "/emissions/"};
MergeNoiseOutput mergeNoiseOutput = new MergeNoiseOutput(paths, Path.of(outputFilePath), config.global().getCoordinateSystem());
MergeNoiseOutput mergeNoiseOutput = new MergeNoiseOutput(Path.of(outputFilePath), config.global().getCoordinateSystem());
mergeNoiseOutput.run();

// Total stats
DecimalFormat df = new DecimalFormat("#.###", DecimalFormatSymbols.getInstance(Locale.US));
try (CSVPrinter printer = new CSVPrinter(IOUtils.getBufferedWriter(output.getPath("noise_stats.csv").toString()), CSVFormat.DEFAULT)) {
printer.printRecord("Annual cost rate per pop. unit [€]:", df.format(noiseParameters.getAnnualCostRate()));
for (Map.Entry<String, Float> labelValueEntry : mergeNoiseOutput.getTotalReceiverPointValues().entrySet()) {
printer.printRecord("Total " + labelValueEntry.getKey() + " at receiver points", df.format(labelValueEntry.getValue()));
}
} catch (IOException ex) {
log.error(ex);
}

return 0;
}

private Config prepareConfig() {
Config config = ConfigUtils.loadConfig(ApplicationUtils.matchInput("config.xml", input.getRunDirectory()).toAbsolutePath().toString(), new NoiseConfigGroup());
Config config = ConfigUtils.loadConfig(ApplicationUtils.matchInput("config.xml", input.getRunDirectory()).toAbsolutePath().toString());

//it is important to match "output_vehicles.xml.gz" specifically, because otherwise dvrpVehicle files might be matched and the code crashes later
config.vehicles().setVehiclesFile(ApplicationUtils.matchInput("output_vehicles.xml.gz", input.getRunDirectory()).toAbsolutePath().toString());
Expand Down
1 change: 1 addition & 0 deletions contribs/noise/src/main/java/org/matsim/contrib/noise/MergeNoiseCSVFile.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -392,6 +392,7 @@ private void readReceiverPoints() {
}
}

//TODO this should be updated to use CSVReader or something as robust
private void readValues() {
for (int ll = 0; ll < this.labels.length; ll++) {

Expand Down
4 changes: 2 additions & 2 deletions contribs/noise/src/main/java/org/matsim/contrib/noise/NoiseConfigGroup.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -95,8 +95,8 @@ public NoiseConfigGroup() {

private double receiverPointGap = 250.;

private String[] consideredActivitiesForReceiverPointGrid = {"home", "work"};
private String[] consideredActivitiesForDamageCalculation = {"home", "work"};
private String[] consideredActivitiesForReceiverPointGrid = {"home*", "work*"};
private String[] consideredActivitiesForDamageCalculation = {"home*", "work*"};

private double receiverPointsGridMinX = 0.;
private double receiverPointsGridMinY = 0.;
Expand Down
Loading

0 comments on commit 9948285

Please sign in to comment.