Include Damages in NoiseDashboard

contribs/application/src/main/java/org/matsim/application/analysis/noise/MergeNoiseOutput.java

@@ -13,11 +13,14 @@
 import org.apache.avro.file.DataFileWriter;
 import org.apache.avro.io.DatumWriter;
 import org.apache.avro.specific.SpecificDatumWriter;
+import org.apache.commons.lang.StringUtils;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.matsim.api.core.v01.Coord;
 import org.matsim.application.avro.XYTData;
+import org.matsim.core.config.Config;
 import org.matsim.core.utils.io.IOUtils;
+import org.matsim.core.utils.misc.Time;
 import tech.tablesaw.api.*;
 import tech.tablesaw.io.csv.CsvReadOptions;
 
@@ -29,7 +32,8 @@
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.*;
-import java.util.stream.Collectors;
+
+import static org.geotools.gml3.v3_2.GML.coordinateSystem;
 
 /**
  * Merges noise data from multiple files into one file.
@@ -44,17 +48,16 @@ final class MergeNoiseOutput {
 	 */
 	private static final boolean CREATE_CSV_FILES = false;
 
-	private final String[] inputPath;
 	private final Path outputDirectory;
 	private final String crs;
-	private final String[] labels = {"immission", "emission"};
 	private final int minTime = 3600;
 	private int maxTime = 24 * 3600;
 
-	MergeNoiseOutput(String[] inputPath, Path outputDirectory, String crs) {
-		this.inputPath = inputPath;
-		this.outputDirectory = outputDirectory;
-		this.crs = crs;
+	private final Map<String,Float> totalReceiverPointValues = new HashMap<>();
+
+	MergeNoiseOutput(Path path, String coordinateSystem ) {
+		this.outputDirectory = path;
+		this.crs = coordinateSystem;
 	}
 
 	/**
@@ -90,25 +93,9 @@ public void setMaxTime(int maxTime) {
 	 * Merges noise data from multiple files into one file.
 	 */
 	public void run() {
-
-		// Loop over all paths
-		for (int i = 0; i < labels.length; i++) {
-
-			// Select the correct method based on the label
-			switch (labels[i]) {
-				case "immission" -> {
-					if (CREATE_CSV_FILES) {
-						mergeImmissionsCSV(inputPath[i], labels[i]);
-					} else {
-						mergeImissions(inputPath[i], labels[i]);
-					}
-
-				}
-				case "emission" -> mergeEmissions(inputPath[i], labels[i]);
-				default -> log.warn("Unknown path: " + inputPath[i]);
-			}
-
-		}
+		mergeReceiverPointData(outputDirectory + "/immissions/", "immission");
+		mergeReceiverPointData(outputDirectory + "/damages_receiverPoint/", "damages_receiverPoint");
+		mergeLinkData(outputDirectory.toString() + "/emissions/", "emission");
 	}
 
 	/**
@@ -118,6 +105,7 @@ public void run() {
 	 * @param output
 	 */
 	private void writeAvro(XYTData xytData, File output) {
+		log.info(String.format("Start writing avro file to %s", output.toString() ));
 		DatumWriter<XYTData> datumWriter = new SpecificDatumWriter<>(XYTData.class);
 		try (DataFileWriter<XYTData> dataFileWriter = new DataFileWriter<>(datumWriter)) {
 			dataFileWriter.setCodec(CodecFactory.deflateCodec(9));
@@ -128,7 +116,7 @@ private void writeAvro(XYTData xytData, File output) {
 		}
 	}
 
-	private void mergeEmissions(String pathParameter, String label) {
+	private void mergeLinkData(String pathParameter, String label) {
 		log.info("Merging emissions data for label {}", label);
 		Object2DoubleMap<String> mergedData = new Object2DoubleOpenHashMap<>();
 		Table csvOutputMerged = Table.create(TextColumn.create("Link Id"), DoubleColumn.create("value"));
@@ -143,9 +131,8 @@ private void mergeEmissions(String pathParameter, String label) {
 				.separator(';').build());
 
 			for (Row row : table) {
-				// index for Noise Emission xx:xx:xx -> 7
 				String linkId = row.getString("Link Id");
-				double value = row.getDouble(7);
+				double value = row.getDouble(row.columnCount() - 1);
 				mergedData.mergeDouble(linkId, value, Double::max);
 
 			}
@@ -165,38 +152,49 @@ private void mergeEmissions(String pathParameter, String label) {
 	}
 
 	/**
-	 * Merges the immissions data
+	 * Merges receiverPoint data (written by {@link org.matsim.contrib.noise.NoiseWriter}
 	 *
-	 * @param pathParameter path to the immissions data
-	 * @param label         label for the immissions data
+	 * @param outputDir path to the receiverPoint data
+	 * @param label         label for the receiverPoint data (which kind of data)
 	 */
-	private void mergeImissions(String pathParameter, String label) {
+	private void mergeReceiverPointData(String outputDir, String label) {
 
 		// data per time step, maps coord to value
 		Int2ObjectMap<Object2FloatMap<FloatFloatPair>> data = new Int2ObjectOpenHashMap<>();
 
 		// Loop over all files
+		//TODO could be adjusted to time bin size from noise config group
+		String substrToCapitalize = null;
 		for (int time = minTime; time <= maxTime; time += 3600) {
 
-			String path = pathParameter + label + "_" + round(time, 1) + ".csv";
+			String timeDataFile = outputDir + label + "_" + round(time, 1) + ".csv";
+
 			Object2FloatOpenHashMap<FloatFloatPair> values = new Object2FloatOpenHashMap<>();
 
-			if (!Files.exists(Path.of(path))) {
-				log.warn("File {} does not exist", path);
+			if (!Files.exists(Path.of(timeDataFile))) {
+				log.warn("File {} does not exist", timeDataFile);
 				continue;
 			}
 
-			// Read the file
-			Table table = Table.read().csv(CsvReadOptions.builder(IOUtils.getBufferedReader(path))
-				.columnTypesPartial(Map.of("x", ColumnType.FLOAT, "y", ColumnType.FLOAT, "Receiver Point Id", ColumnType.INTEGER, "t", ColumnType.DOUBLE))
+			//we need "damages_receiverPoint" -> "Damages 01:00:00" and "immission" -> "Immision 01:00:00"
+			substrToCapitalize = label.contains("_") ? label.substring(0, label.lastIndexOf("_")) : label;
+			String valueHeader = StringUtils.capitalize(substrToCapitalize) + " " + Time.writeTime(time, Time.TIMEFORMAT_HHMMSS);
+
+			// Read the data file
+			Table dataTable = Table.read().csv(CsvReadOptions.builder(IOUtils.getBufferedReader(timeDataFile))
+				.columnTypesPartial(Map.of("x", ColumnType.FLOAT,
+					"y", ColumnType.FLOAT,
+					"Receiver Point Id", ColumnType.INTEGER,
+					"t", ColumnType.DOUBLE,
+					valueHeader, ColumnType.DOUBLE))
 				.sample(false)
 				.separator(';').build());
 
-			// Loop over all rows in the file
-			for (Row row : table) {
+			// Loop over all rows in the data file
+			for (Row row : dataTable) {
 				float x = row.getFloat("x");
 				float y = row.getFloat("y");
-				float value = (float) row.getDouble(1); // 1
+				float value = (float) row.getDouble(valueHeader);
 				FloatFloatPair coord = FloatFloatPair.of(x, y);
 				values.put(coord, value);
 			}
@@ -232,7 +230,7 @@ private void mergeImissions(String pathParameter, String label) {
 			}
 		}
 
-		xytHourData.setData(Map.of("imissions", raw));
+		xytHourData.setData(Map.of(label, raw));
 		xytHourData.setCrs(crs);
 
 		File out = outputDirectory.getParent().resolve(label + "_per_hour.avro").toFile();
@@ -254,15 +252,18 @@ private void mergeImissions(String pathParameter, String label) {
 		xytDayData.setTimestamps(List.of(0));
 		xytDayData.setXCoords(xCoords);
 		xytDayData.setYCoords(yCoords);
-		xytDayData.setData(Map.of("imissions", raw));
+		xytDayData.setData(Map.of(label, raw));
 		xytDayData.setCrs(crs);
 
 		File outDay = outputDirectory.getParent().resolve(label + "_per_day.avro").toFile();
 
 		writeAvro(xytDayData, outDay);
+		//cache the overall sum
+		this.totalReceiverPointValues.put(substrToCapitalize, raw.stream().reduce(0f, Float::sum));
 	}
 
 	// Merges the immissions data
+
 	@Deprecated
 	private void mergeImmissionsCSV(String pathParameter, String label) {
 		log.info("Merging immissions data for label {}", label);
@@ -278,7 +279,10 @@ private void mergeImmissionsCSV(String pathParameter, String label) {
 
 			// Read the file
 			Table table = Table.read().csv(CsvReadOptions.builder(IOUtils.getBufferedReader(path))
-				.columnTypesPartial(Map.of("x", ColumnType.DOUBLE, "y", ColumnType.DOUBLE, "Receiver Point Id", ColumnType.INTEGER, "t", ColumnType.DOUBLE))
+				.columnTypesPartial(Map.of("x", ColumnType.DOUBLE,
+					"y", ColumnType.DOUBLE,
+					"Receiver Point Id", ColumnType.INTEGER,
+					"t", ColumnType.DOUBLE))
 				.sample(false)
 				.separator(';').build());
 
@@ -319,4 +323,7 @@ private void mergeImmissionsCSV(String pathParameter, String label) {
 		log.info("Merged noise data written to {} ", outPerDay);
 
 	}
+	public Map<String, Float> getTotalReceiverPointValues() {
+		return totalReceiverPointValues;
+	}
 }

contribs/application/src/main/java/org/matsim/application/analysis/noise/NoiseAnalysis.java

@@ -1,5 +1,7 @@
 package org.matsim.application.analysis.noise;
 
+import org.apache.commons.csv.CSVFormat;
+import org.apache.commons.csv.CSVPrinter;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.locationtech.jts.geom.Envelope;
@@ -19,9 +21,15 @@
 import org.matsim.core.config.ConfigUtils;
 import org.matsim.core.scenario.ScenarioUtils;
 import org.matsim.core.utils.geometry.CoordinateTransformation;
+import org.matsim.core.utils.io.IOUtils;
 import picocli.CommandLine;
 
+import java.io.IOException;
 import java.nio.file.Path;
+import java.text.DecimalFormat;
+import java.text.DecimalFormatSymbols;
+import java.util.Locale;
+import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
 
@@ -36,7 +44,10 @@
 	produces = {
 		"emission_per_day.csv",
 		"immission_per_day.%s",
-		"immission_per_hour.%s"
+		"immission_per_hour.%s",
+		"damages_receiverPoint_per_hour.%s",
+		"damages_receiverPoint_per_day.%s",
+		"noise_stats.csv"
 	}
 )
 public class NoiseAnalysis implements MATSimAppCommand {
@@ -55,7 +66,7 @@ public class NoiseAnalysis implements MATSimAppCommand {
 	private final SampleOptions sampleOptions = new SampleOptions();
 
 	@CommandLine.Option(names = "--consider-activities", split = ",", description = "Considered activities for noise calculation." +
-		" Use asterisk ('*') for acttype prefixes, if all such acts shall be considered.", defaultValue = "h,w,home*,work*")
+		" Use asterisk ('*') for acttype prefixes, if all such acts shall be considered.", defaultValue = "home*,work*,educ*,leisure*")
 	private Set<String> considerActivities;
 
 	@CommandLine.Option(names = "--noise-barrier", description = "Path to the noise barrier File", defaultValue = "")
@@ -71,32 +82,55 @@ public Integer call() throws Exception {
 
 		config.controller().setOutputDirectory(input.getRunDirectory().toString());
 
-		// adjust the default noise parameters
+		//trying to set noise parameters more explicitly, here...
+		//if NoiseConfigGroup was added before. do not override (most) parameters
+		boolean overrideParameters = ! ConfigUtils.hasModule(config, NoiseConfigGroup.class);
 		NoiseConfigGroup noiseParameters = ConfigUtils.addOrGetModule(config, NoiseConfigGroup.class);
-		noiseParameters.setConsideredActivitiesForReceiverPointGridArray(considerActivities.toArray(String[]::new));
-		noiseParameters.setConsideredActivitiesForDamageCalculationArray(considerActivities.toArray(String[]::new));
-		if (shp.getShapeFile() != null) {
-			CoordinateTransformation ct = shp.createInverseTransformation(config.global().getCoordinateSystem());
 
-			Envelope bbox = shp.getGeometry().getEnvelopeInternal();
+		if(overrideParameters){
+			log.warn("no NoiseConfigGroup was configured before. Will set some standards. You should check the next lines in the log file!");
+			noiseParameters.setConsideredActivitiesForReceiverPointGridArray(considerActivities.toArray(String[]::new));
+			noiseParameters.setConsideredActivitiesForDamageCalculationArray(considerActivities.toArray(String[]::new));
 
-			Coord minCoord = ct.transform(new Coord(bbox.getMinX(), bbox.getMinY()));
-			Coord maxCoord = ct.transform(new Coord(bbox.getMaxX(), bbox.getMaxY()));
+			//use actual speed and not freespeed
+			noiseParameters.setUseActualSpeedLevel(true);
+			//use the valid speed range (recommended by IK)
+			noiseParameters.setAllowForSpeedsOutsideTheValidRange(false);
 
-			noiseParameters.setReceiverPointsGridMinX(minCoord.getX());
-			noiseParameters.setReceiverPointsGridMinY(minCoord.getY());
-			noiseParameters.setReceiverPointsGridMaxX(maxCoord.getX());
-			noiseParameters.setReceiverPointsGridMaxY(maxCoord.getY());
-		}
+			if (shp.getShapeFile() != null) {
+				CoordinateTransformation ct = shp.createInverseTransformation(config.global().getCoordinateSystem());
+
+				Envelope bbox = shp.getGeometry().getEnvelopeInternal();
+
+				Coord minCoord = ct.transform(new Coord(bbox.getMinX(), bbox.getMinY()));
+				Coord maxCoord = ct.transform(new Coord(bbox.getMaxX(), bbox.getMaxY()));
+
+				noiseParameters.setReceiverPointsGridMinX(minCoord.getX());
+				noiseParameters.setReceiverPointsGridMinY(minCoord.getY());
+				noiseParameters.setReceiverPointsGridMaxX(maxCoord.getX());
+				noiseParameters.setReceiverPointsGridMaxY(maxCoord.getY());
+			}
 
-		noiseParameters.setNoiseComputationMethod(NoiseConfigGroup.NoiseComputationMethod.RLS19);
+			noiseParameters.setNoiseComputationMethod(NoiseConfigGroup.NoiseComputationMethod.RLS19);
 
-		if (!Objects.equals(noiseBarrierFile, "")) {
-			noiseParameters.setNoiseBarriersSourceCRS(config.global().getCoordinateSystem());
-			noiseParameters.setConsiderNoiseBarriers(true);
-			noiseParameters.setNoiseBarriersFilePath(noiseBarrierFile);
+			if (!Objects.equals(noiseBarrierFile, "")) {
+				noiseParameters.setNoiseBarriersSourceCRS(config.global().getCoordinateSystem());
+				noiseParameters.setConsiderNoiseBarriers(true);
+				noiseParameters.setNoiseBarriersFilePath(noiseBarrierFile);
+			}
+		} else {
+			log.warn("will override a few settings in NoiseConfigGroup, as we are now doing postprocessing and do not want any internalization etc." +
+				" You should check the next lines in the log file!");
 		}
 
+		// we only mean to do postprocessing here, thus no internalization etc
+		noiseParameters.setInternalizeNoiseDamages(false);
+		noiseParameters.setComputeCausingAgents(false);
+		//we don't need events (for Dashboard) - spare disk space.
+		noiseParameters.setThrowNoiseEventsAffected(false);
+		noiseParameters.setThrowNoiseEventsCaused(false);
+		noiseParameters.setComputeNoiseDamages(true);
+
 		if(! sampleOptions.isSet() && noiseParameters.getScaleFactor() == 1d){
 			log.warn("You didn't provide the simulation sample size via command line option --sample-size! This means, noise damages are not scaled!!!");
 		} else if (noiseParameters.getScaleFactor() == 1d){
@@ -111,23 +145,35 @@ public Integer call() throws Exception {
 
 		String outputFilePath = output.getPath().getParent() == null ? "." : output.getPath().getParent().toString();
 
+		log.info("starting " + NoiseOfflineCalculation.class + " with the following parameters:\n"
+			+ noiseParameters);
+
 		NoiseOfflineCalculation noiseCalculation = new NoiseOfflineCalculation(scenario, outputFilePath);
 		outputFilePath += "/noise-analysis";
 		noiseCalculation.run();
 
 		ProcessNoiseImmissions process = new ProcessNoiseImmissions(outputFilePath + "/immissions/", outputFilePath + "/receiverPoints/receiverPoints.csv", noiseParameters.getReceiverPointGap());
 		process.run();
 
-		final String[] paths = {outputFilePath + "/immissions/", outputFilePath + "/emissions/"};
-		MergeNoiseOutput mergeNoiseOutput = new MergeNoiseOutput(paths, Path.of(outputFilePath), config.global().getCoordinateSystem());
+		MergeNoiseOutput mergeNoiseOutput = new MergeNoiseOutput(Path.of(outputFilePath), config.global().getCoordinateSystem());
 		mergeNoiseOutput.run();
 
+		// Total stats
+		DecimalFormat df = new DecimalFormat("#.###", DecimalFormatSymbols.getInstance(Locale.US));
+		try (CSVPrinter printer = new CSVPrinter(IOUtils.getBufferedWriter(output.getPath("noise_stats.csv").toString()), CSVFormat.DEFAULT)) {
+			printer.printRecord("Annual cost rate per pop. unit [€]:", df.format(noiseParameters.getAnnualCostRate()));
+			for (Map.Entry<String, Float> labelValueEntry : mergeNoiseOutput.getTotalReceiverPointValues().entrySet()) {
+				printer.printRecord("Total " + labelValueEntry.getKey() + " at receiver points", df.format(labelValueEntry.getValue()));
+			}
+		} catch (IOException ex) {
+			log.error(ex);
+		}
 
 		return 0;
 	}
 
 	private Config prepareConfig() {
-		Config config = ConfigUtils.loadConfig(ApplicationUtils.matchInput("config.xml", input.getRunDirectory()).toAbsolutePath().toString(), new NoiseConfigGroup());
+		Config config = ConfigUtils.loadConfig(ApplicationUtils.matchInput("config.xml", input.getRunDirectory()).toAbsolutePath().toString());
 
 		//it is important to match "output_vehicles.xml.gz" specifically, because otherwise dvrpVehicle files might be matched and the code crashes later
 		config.vehicles().setVehiclesFile(ApplicationUtils.matchInput("output_vehicles.xml.gz", input.getRunDirectory()).toAbsolutePath().toString());

contribs/noise/src/main/java/org/matsim/contrib/noise/MergeNoiseCSVFile.java

@@ -392,6 +392,7 @@ private void readReceiverPoints() {
 		}
 	}
 
+	//TODO this should be updated to use CSVReader or something as robust
 	private void readValues() {
 		for (int ll = 0; ll < this.labels.length; ll++) {
 

contribs/noise/src/main/java/org/matsim/contrib/noise/NoiseConfigGroup.java

@@ -95,8 +95,8 @@ public NoiseConfigGroup() {
 
 	private double receiverPointGap = 250.;
 
-	private String[] consideredActivitiesForReceiverPointGrid = {"home", "work"};
-	private String[] consideredActivitiesForDamageCalculation = {"home", "work"};
+	private String[] consideredActivitiesForReceiverPointGrid = {"home*", "work*"};
+	private String[] consideredActivitiesForDamageCalculation = {"home*", "work*"};
 
 	private double receiverPointsGridMinX = 0.;
 	private double receiverPointsGridMinY = 0.;