From e981bde003b91aa599b51cac12c43d36ea35ed6a Mon Sep 17 00:00:00 2001 From: schlenther Date: Thu, 8 Aug 2024 16:34:20 +0200 Subject: [PATCH 1/6] better defaults for NoiseDashboard --- .../org/matsim/simwrapper/dashboard/NoiseDashboard.java | 8 ++++---- .../main/java/org/matsim/simwrapper/viz/ColorScheme.java | 1 + 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/contribs/simwrapper/src/main/java/org/matsim/simwrapper/dashboard/NoiseDashboard.java b/contribs/simwrapper/src/main/java/org/matsim/simwrapper/dashboard/NoiseDashboard.java index dc934e6c2e2..b502f3fbf57 100644 --- a/contribs/simwrapper/src/main/java/org/matsim/simwrapper/dashboard/NoiseDashboard.java +++ b/contribs/simwrapper/src/main/java/org/matsim/simwrapper/dashboard/NoiseDashboard.java @@ -42,7 +42,7 @@ public void configure(Header header, Layout layout) { viz.maxHeight = 20; viz.center = data.context().getCenter(); viz.zoom = data.context().mapZoomLevel; - viz.setColorRamp(new double[]{40, 50, 60}, new String[]{"#1175b3", "#95c7df", "#f4a986", "#cc0c27"}); + viz.setColorRamp(new double[]{30, 40, 50, 60, 70}, new String[]{"#1175b3", "#95c7df", "#dfdb95", "#dfb095", "#f4a986", "#cc0c27"}); viz.file = data.computeWithPlaceholder(NoiseAnalysis.class, "immission_per_day.%s", "avro"); }) .el(MapPlot.class, (viz, data) -> { @@ -58,8 +58,8 @@ public void configure(Header header, Layout layout) { viz.display.lineColor.dataset = "noise"; viz.display.lineColor.columnName = "value"; viz.display.lineColor.join = "Link Id"; - viz.display.lineColor.fixedColors = new String[]{"#1175b3", "#95c7df", "#f4a986", "#cc0c27"}; - viz.display.lineColor.setColorRamp(ColorScheme.RdYlBu, 4, true, "45, 55, 65"); + //viz.display.lineColor.fixedColors = new String[]{"#1175b3", "#95c7df", "#f4a986", "#cc0c27"}; + viz.display.lineColor.setColorRamp(ColorScheme.Oranges, 8, false, "35, 45, 55, 65, 75, 85, 95"); viz.display.lineWidth.dataset = "noise"; viz.display.lineWidth.columnName = "value"; viz.display.lineWidth.scaleFactor = 8d; @@ -75,7 +75,7 @@ public void configure(Header header, Layout layout) { viz.maxHeight = 20; viz.center = data.context().getCenter(); viz.zoom = data.context().mapZoomLevel; - viz.setColorRamp(new double[]{40, 50, 60}, new String[]{"#1175b3", "#95c7df", "#f4a986", "#cc0c27"}); + viz.setColorRamp(new double[]{30, 40, 50, 60, 70}, new String[]{"#1175b3", "#95c7df", "#dfdb95", "#dfb095", "#f4a986", "#cc0c27"}); viz.file = data.computeWithPlaceholder(NoiseAnalysis.class, "immission_per_hour.%s", "avro"); }); } diff --git a/contribs/simwrapper/src/main/java/org/matsim/simwrapper/viz/ColorScheme.java b/contribs/simwrapper/src/main/java/org/matsim/simwrapper/viz/ColorScheme.java index 37e31cc97ca..df8f6410679 100644 --- a/contribs/simwrapper/src/main/java/org/matsim/simwrapper/viz/ColorScheme.java +++ b/contribs/simwrapper/src/main/java/org/matsim/simwrapper/viz/ColorScheme.java @@ -27,6 +27,7 @@ public final class ColorScheme { public static final String Inferne = "Inferne"; public static final String Cividis = "Cividis"; public static final String Rainbow = "Rainbow"; + public static final String Oranges = "Oranges"; private ColorScheme() { From 624ca13d060c4313c753868fc14f5c947da6d845 Mon Sep 17 00:00:00 2001 From: schlenther Date: Thu, 8 Aug 2024 18:05:56 +0200 Subject: [PATCH 2/6] NoiseAnalysis: more explicit parameter settings + bug fix in MergeNoiseOutput --- .../analysis/noise/MergeNoiseOutput.java | 14 ++-- .../analysis/noise/NoiseAnalysis.java | 67 +++++++++++++------ .../contrib/noise/NoiseConfigGroup.java | 4 +- .../simwrapper/dashboard/NoiseDashboard.java | 4 +- 4 files changed, 60 insertions(+), 29 deletions(-) diff --git a/contribs/application/src/main/java/org/matsim/application/analysis/noise/MergeNoiseOutput.java b/contribs/application/src/main/java/org/matsim/application/analysis/noise/MergeNoiseOutput.java index aef1ec23d05..88f8f9354e5 100644 --- a/contribs/application/src/main/java/org/matsim/application/analysis/noise/MergeNoiseOutput.java +++ b/contribs/application/src/main/java/org/matsim/application/analysis/noise/MergeNoiseOutput.java @@ -118,6 +118,7 @@ public void run() { * @param output */ private void writeAvro(XYTData xytData, File output) { + log.info(String.format("Start writing avro file to %s", output.toString() )); DatumWriter datumWriter = new SpecificDatumWriter<>(XYTData.class); try (DataFileWriter dataFileWriter = new DataFileWriter<>(datumWriter)) { dataFileWriter.setCodec(CodecFactory.deflateCodec(9)); @@ -143,9 +144,8 @@ private void mergeEmissions(String pathParameter, String label) { .separator(';').build()); for (Row row : table) { - // index for Noise Emission xx:xx:xx -> 7 String linkId = row.getString("Link Id"); - double value = row.getDouble(7); + double value = row.getDouble(row.columnCount() - 1); mergedData.mergeDouble(linkId, value, Double::max); } @@ -188,7 +188,10 @@ private void mergeImissions(String pathParameter, String label) { // Read the file Table table = Table.read().csv(CsvReadOptions.builder(IOUtils.getBufferedReader(path)) - .columnTypesPartial(Map.of("x", ColumnType.FLOAT, "y", ColumnType.FLOAT, "Receiver Point Id", ColumnType.INTEGER, "t", ColumnType.DOUBLE)) + .columnTypesPartial(Map.of("x", ColumnType.FLOAT, + "y", ColumnType.FLOAT, + "Receiver Point Id", ColumnType.INTEGER, + "t", ColumnType.DOUBLE)) .sample(false) .separator(';').build()); @@ -278,7 +281,10 @@ private void mergeImmissionsCSV(String pathParameter, String label) { // Read the file Table table = Table.read().csv(CsvReadOptions.builder(IOUtils.getBufferedReader(path)) - .columnTypesPartial(Map.of("x", ColumnType.DOUBLE, "y", ColumnType.DOUBLE, "Receiver Point Id", ColumnType.INTEGER, "t", ColumnType.DOUBLE)) + .columnTypesPartial(Map.of("x", ColumnType.DOUBLE, + "y", ColumnType.DOUBLE, + "Receiver Point Id", ColumnType.INTEGER, + "t", ColumnType.DOUBLE)) .sample(false) .separator(';').build()); diff --git a/contribs/application/src/main/java/org/matsim/application/analysis/noise/NoiseAnalysis.java b/contribs/application/src/main/java/org/matsim/application/analysis/noise/NoiseAnalysis.java index 6730cae8996..02d6337de15 100644 --- a/contribs/application/src/main/java/org/matsim/application/analysis/noise/NoiseAnalysis.java +++ b/contribs/application/src/main/java/org/matsim/application/analysis/noise/NoiseAnalysis.java @@ -55,7 +55,7 @@ public class NoiseAnalysis implements MATSimAppCommand { private final SampleOptions sampleOptions = new SampleOptions(); @CommandLine.Option(names = "--consider-activities", split = ",", description = "Considered activities for noise calculation." + - " Use asterisk ('*') for acttype prefixes, if all such acts shall be considered.", defaultValue = "h,w,home*,work*") + " Use asterisk ('*') for acttype prefixes, if all such acts shall be considered.", defaultValue = "home*,work*,educ*,leisure*") private Set considerActivities; @CommandLine.Option(names = "--noise-barrier", description = "Path to the noise barrier File", defaultValue = "") @@ -71,32 +71,55 @@ public Integer call() throws Exception { config.controller().setOutputDirectory(input.getRunDirectory().toString()); - // adjust the default noise parameters + //trying to set noise parameters more explicitly, here... + //if NoiseConfigGroup was added before. do not override (most) parameters + boolean overrideParameters = ! ConfigUtils.hasModule(config, NoiseConfigGroup.class); NoiseConfigGroup noiseParameters = ConfigUtils.addOrGetModule(config, NoiseConfigGroup.class); - noiseParameters.setConsideredActivitiesForReceiverPointGridArray(considerActivities.toArray(String[]::new)); - noiseParameters.setConsideredActivitiesForDamageCalculationArray(considerActivities.toArray(String[]::new)); - if (shp.getShapeFile() != null) { - CoordinateTransformation ct = shp.createInverseTransformation(config.global().getCoordinateSystem()); - Envelope bbox = shp.getGeometry().getEnvelopeInternal(); + if(overrideParameters){ + log.warn("no NoiseConfigGroup was configured before. Will set som estandards. You should check the next lines in the log file!"); + noiseParameters.setConsideredActivitiesForReceiverPointGridArray(considerActivities.toArray(String[]::new)); + noiseParameters.setConsideredActivitiesForDamageCalculationArray(considerActivities.toArray(String[]::new)); - Coord minCoord = ct.transform(new Coord(bbox.getMinX(), bbox.getMinY())); - Coord maxCoord = ct.transform(new Coord(bbox.getMaxX(), bbox.getMaxY())); + //use actual speed and not freespeed + noiseParameters.setUseActualSpeedLevel(true); + //use the valid speed range (recommended by IK) + noiseParameters.setAllowForSpeedsOutsideTheValidRange(false); - noiseParameters.setReceiverPointsGridMinX(minCoord.getX()); - noiseParameters.setReceiverPointsGridMinY(minCoord.getY()); - noiseParameters.setReceiverPointsGridMaxX(maxCoord.getX()); - noiseParameters.setReceiverPointsGridMaxY(maxCoord.getY()); - } + if (shp.getShapeFile() != null) { + CoordinateTransformation ct = shp.createInverseTransformation(config.global().getCoordinateSystem()); + + Envelope bbox = shp.getGeometry().getEnvelopeInternal(); + + Coord minCoord = ct.transform(new Coord(bbox.getMinX(), bbox.getMinY())); + Coord maxCoord = ct.transform(new Coord(bbox.getMaxX(), bbox.getMaxY())); + + noiseParameters.setReceiverPointsGridMinX(minCoord.getX()); + noiseParameters.setReceiverPointsGridMinY(minCoord.getY()); + noiseParameters.setReceiverPointsGridMaxX(maxCoord.getX()); + noiseParameters.setReceiverPointsGridMaxY(maxCoord.getY()); + } - noiseParameters.setNoiseComputationMethod(NoiseConfigGroup.NoiseComputationMethod.RLS19); + noiseParameters.setNoiseComputationMethod(NoiseConfigGroup.NoiseComputationMethod.RLS19); - if (!Objects.equals(noiseBarrierFile, "")) { - noiseParameters.setNoiseBarriersSourceCRS(config.global().getCoordinateSystem()); - noiseParameters.setConsiderNoiseBarriers(true); - noiseParameters.setNoiseBarriersFilePath(noiseBarrierFile); + if (!Objects.equals(noiseBarrierFile, "")) { + noiseParameters.setNoiseBarriersSourceCRS(config.global().getCoordinateSystem()); + noiseParameters.setConsiderNoiseBarriers(true); + noiseParameters.setNoiseBarriersFilePath(noiseBarrierFile); + } + } else { + log.warn("will override a few settings in NoiseConfigGroup, as we are now doing postprocessing and do not want any internalization etc." + + " You should check the next lines in the log file!"); } + // we only mean to do postprocessing here, thus no internalization etc + noiseParameters.setInternalizeNoiseDamages(false); + noiseParameters.setComputeCausingAgents(false); + //we don't need events (for Dashboard) - spare disk space. + noiseParameters.setThrowNoiseEventsAffected(false); + noiseParameters.setThrowNoiseEventsCaused(false); + noiseParameters.setComputeNoiseDamages(true); + if(! sampleOptions.isSet() && noiseParameters.getScaleFactor() == 1d){ log.warn("You didn't provide the simulation sample size via command line option --sample-size! This means, noise damages are not scaled!!!"); } else if (noiseParameters.getScaleFactor() == 1d){ @@ -111,6 +134,9 @@ public Integer call() throws Exception { String outputFilePath = output.getPath().getParent() == null ? "." : output.getPath().getParent().toString(); + log.info("starting " + NoiseOfflineCalculation.class + " with the following parameters:\n" + + noiseParameters); + NoiseOfflineCalculation noiseCalculation = new NoiseOfflineCalculation(scenario, outputFilePath); outputFilePath += "/noise-analysis"; noiseCalculation.run(); @@ -122,12 +148,11 @@ public Integer call() throws Exception { MergeNoiseOutput mergeNoiseOutput = new MergeNoiseOutput(paths, Path.of(outputFilePath), config.global().getCoordinateSystem()); mergeNoiseOutput.run(); - return 0; } private Config prepareConfig() { - Config config = ConfigUtils.loadConfig(ApplicationUtils.matchInput("config.xml", input.getRunDirectory()).toAbsolutePath().toString(), new NoiseConfigGroup()); + Config config = ConfigUtils.loadConfig(ApplicationUtils.matchInput("config.xml", input.getRunDirectory()).toAbsolutePath().toString()); //it is important to match "output_vehicles.xml.gz" specifically, because otherwise dvrpVehicle files might be matched and the code crashes later config.vehicles().setVehiclesFile(ApplicationUtils.matchInput("output_vehicles.xml.gz", input.getRunDirectory()).toAbsolutePath().toString()); diff --git a/contribs/noise/src/main/java/org/matsim/contrib/noise/NoiseConfigGroup.java b/contribs/noise/src/main/java/org/matsim/contrib/noise/NoiseConfigGroup.java index cd7b2a500ab..06e793c85d9 100644 --- a/contribs/noise/src/main/java/org/matsim/contrib/noise/NoiseConfigGroup.java +++ b/contribs/noise/src/main/java/org/matsim/contrib/noise/NoiseConfigGroup.java @@ -95,8 +95,8 @@ public NoiseConfigGroup() { private double receiverPointGap = 250.; - private String[] consideredActivitiesForReceiverPointGrid = {"home", "work"}; - private String[] consideredActivitiesForDamageCalculation = {"home", "work"}; + private String[] consideredActivitiesForReceiverPointGrid = {"home*", "work*"}; + private String[] consideredActivitiesForDamageCalculation = {"home*", "work*"}; private double receiverPointsGridMinX = 0.; private double receiverPointsGridMinY = 0.; diff --git a/contribs/simwrapper/src/main/java/org/matsim/simwrapper/dashboard/NoiseDashboard.java b/contribs/simwrapper/src/main/java/org/matsim/simwrapper/dashboard/NoiseDashboard.java index b502f3fbf57..036b3c5a718 100644 --- a/contribs/simwrapper/src/main/java/org/matsim/simwrapper/dashboard/NoiseDashboard.java +++ b/contribs/simwrapper/src/main/java/org/matsim/simwrapper/dashboard/NoiseDashboard.java @@ -35,7 +35,7 @@ public void configure(Header header, Layout layout) { layout.row("aggregate noise") .el(GridMap.class, (viz, data) -> { viz.title = "Noise Immissions (Grid)"; - viz.description = "Aggregate Noise Immissions per day"; + viz.description = "Total Noise Immissions per day"; viz.height = 12.0; viz.cellSize = 250; viz.opacity = 0.2; @@ -47,7 +47,7 @@ public void configure(Header header, Layout layout) { }) .el(MapPlot.class, (viz, data) -> { viz.title = "Noise Emissions (Link)"; - viz.description = "Aggregate Noise Emissions per day"; + viz.description = "Maximum Noise Level per day [dB]"; viz.height = 12.0; viz.center = data.context().getCenter(); viz.zoom = data.context().mapZoomLevel; From 1730ac8213fb52066926563386badd19b3e9e71f Mon Sep 17 00:00:00 2001 From: schlenther Date: Fri, 9 Aug 2024 11:01:06 +0200 Subject: [PATCH 3/6] NoiseAnalysis: merge damages files --- .../analysis/noise/MergeNoiseOutput.java | 76 +++++++++---------- .../analysis/noise/NoiseAnalysis.java | 3 +- .../contrib/noise/MergeNoiseCSVFile.java | 1 + .../org/matsim/contrib/noise/NoiseWriter.java | 14 ++-- 4 files changed, 42 insertions(+), 52 deletions(-) diff --git a/contribs/application/src/main/java/org/matsim/application/analysis/noise/MergeNoiseOutput.java b/contribs/application/src/main/java/org/matsim/application/analysis/noise/MergeNoiseOutput.java index 88f8f9354e5..e59c4aee75c 100644 --- a/contribs/application/src/main/java/org/matsim/application/analysis/noise/MergeNoiseOutput.java +++ b/contribs/application/src/main/java/org/matsim/application/analysis/noise/MergeNoiseOutput.java @@ -13,11 +13,14 @@ import org.apache.avro.file.DataFileWriter; import org.apache.avro.io.DatumWriter; import org.apache.avro.specific.SpecificDatumWriter; +import org.apache.commons.lang.StringUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.matsim.api.core.v01.Coord; import org.matsim.application.avro.XYTData; +import org.matsim.core.config.Config; import org.matsim.core.utils.io.IOUtils; +import org.matsim.core.utils.misc.Time; import tech.tablesaw.api.*; import tech.tablesaw.io.csv.CsvReadOptions; @@ -29,7 +32,8 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.*; -import java.util.stream.Collectors; + +import static org.geotools.gml3.v3_2.GML.coordinateSystem; /** * Merges noise data from multiple files into one file. @@ -44,17 +48,14 @@ final class MergeNoiseOutput { */ private static final boolean CREATE_CSV_FILES = false; - private final String[] inputPath; private final Path outputDirectory; private final String crs; - private final String[] labels = {"immission", "emission"}; private final int minTime = 3600; private int maxTime = 24 * 3600; - MergeNoiseOutput(String[] inputPath, Path outputDirectory, String crs) { - this.inputPath = inputPath; - this.outputDirectory = outputDirectory; - this.crs = crs; + MergeNoiseOutput(Path path, String coordinateSystem ) { + this.outputDirectory = path; + this.crs = coordinateSystem; } /** @@ -90,25 +91,9 @@ public void setMaxTime(int maxTime) { * Merges noise data from multiple files into one file. */ public void run() { - - // Loop over all paths - for (int i = 0; i < labels.length; i++) { - - // Select the correct method based on the label - switch (labels[i]) { - case "immission" -> { - if (CREATE_CSV_FILES) { - mergeImmissionsCSV(inputPath[i], labels[i]); - } else { - mergeImissions(inputPath[i], labels[i]); - } - - } - case "emission" -> mergeEmissions(inputPath[i], labels[i]); - default -> log.warn("Unknown path: " + inputPath[i]); - } - - } + mergeReceiverPointData(outputDirectory + "/immissions/", "immission"); + mergeReceiverPointData(outputDirectory + "/damages_receiverPoint/", "damages_receiverPoint"); + mergeLinkData(outputDirectory.toString() + "/emissions/", "emission"); } /** @@ -129,7 +114,7 @@ private void writeAvro(XYTData xytData, File output) { } } - private void mergeEmissions(String pathParameter, String label) { + private void mergeLinkData(String pathParameter, String label) { log.info("Merging emissions data for label {}", label); Object2DoubleMap mergedData = new Object2DoubleOpenHashMap<>(); Table csvOutputMerged = Table.create(TextColumn.create("Link Id"), DoubleColumn.create("value")); @@ -165,41 +150,48 @@ private void mergeEmissions(String pathParameter, String label) { } /** - * Merges the immissions data + * Merges receiverPoint data (written by {@link org.matsim.contrib.noise.NoiseWriter} * - * @param pathParameter path to the immissions data - * @param label label for the immissions data + * @param outputDir path to the receiverPoint data + * @param label label for the receiverPoint data (which kind of data) */ - private void mergeImissions(String pathParameter, String label) { + private void mergeReceiverPointData(String outputDir, String label) { // data per time step, maps coord to value Int2ObjectMap> data = new Int2ObjectOpenHashMap<>(); // Loop over all files + //TODO could be adjusted to time bin size from noise config group for (int time = minTime; time <= maxTime; time += 3600) { - String path = pathParameter + label + "_" + round(time, 1) + ".csv"; + String timeDataFile = outputDir + label + "_" + round(time, 1) + ".csv"; + Object2FloatOpenHashMap values = new Object2FloatOpenHashMap<>(); - if (!Files.exists(Path.of(path))) { - log.warn("File {} does not exist", path); + if (!Files.exists(Path.of(timeDataFile))) { + log.warn("File {} does not exist", timeDataFile); continue; } - // Read the file - Table table = Table.read().csv(CsvReadOptions.builder(IOUtils.getBufferedReader(path)) + //we need "damages_receiverPoint" -> "Damages 01:00:00" and "immission" -> "Immision 01:00:00" + String substrToCapitalize = label.contains("_") ? label.substring(0, label.lastIndexOf("_")) : label; + String valueHeader = StringUtils.capitalize(substrToCapitalize) + " " + Time.writeTime(time, Time.TIMEFORMAT_HHMMSS); + + // Read the data file + Table dataTable = Table.read().csv(CsvReadOptions.builder(IOUtils.getBufferedReader(timeDataFile)) .columnTypesPartial(Map.of("x", ColumnType.FLOAT, "y", ColumnType.FLOAT, "Receiver Point Id", ColumnType.INTEGER, - "t", ColumnType.DOUBLE)) + "t", ColumnType.DOUBLE, + valueHeader, ColumnType.DOUBLE)) .sample(false) .separator(';').build()); - // Loop over all rows in the file - for (Row row : table) { + // Loop over all rows in the data file + for (Row row : dataTable) { float x = row.getFloat("x"); float y = row.getFloat("y"); - float value = (float) row.getDouble(1); // 1 + float value = (float) row.getDouble(valueHeader); FloatFloatPair coord = FloatFloatPair.of(x, y); values.put(coord, value); } @@ -235,7 +227,7 @@ private void mergeImissions(String pathParameter, String label) { } } - xytHourData.setData(Map.of("imissions", raw)); + xytHourData.setData(Map.of(label, raw)); xytHourData.setCrs(crs); File out = outputDirectory.getParent().resolve(label + "_per_hour.avro").toFile(); @@ -257,7 +249,7 @@ private void mergeImissions(String pathParameter, String label) { xytDayData.setTimestamps(List.of(0)); xytDayData.setXCoords(xCoords); xytDayData.setYCoords(yCoords); - xytDayData.setData(Map.of("imissions", raw)); + xytDayData.setData(Map.of(label, raw)); xytDayData.setCrs(crs); File outDay = outputDirectory.getParent().resolve(label + "_per_day.avro").toFile(); diff --git a/contribs/application/src/main/java/org/matsim/application/analysis/noise/NoiseAnalysis.java b/contribs/application/src/main/java/org/matsim/application/analysis/noise/NoiseAnalysis.java index 02d6337de15..deceb84624e 100644 --- a/contribs/application/src/main/java/org/matsim/application/analysis/noise/NoiseAnalysis.java +++ b/contribs/application/src/main/java/org/matsim/application/analysis/noise/NoiseAnalysis.java @@ -144,8 +144,7 @@ public Integer call() throws Exception { ProcessNoiseImmissions process = new ProcessNoiseImmissions(outputFilePath + "/immissions/", outputFilePath + "/receiverPoints/receiverPoints.csv", noiseParameters.getReceiverPointGap()); process.run(); - final String[] paths = {outputFilePath + "/immissions/", outputFilePath + "/emissions/"}; - MergeNoiseOutput mergeNoiseOutput = new MergeNoiseOutput(paths, Path.of(outputFilePath), config.global().getCoordinateSystem()); + MergeNoiseOutput mergeNoiseOutput = new MergeNoiseOutput(Path.of(outputFilePath), config.global().getCoordinateSystem()); mergeNoiseOutput.run(); return 0; diff --git a/contribs/noise/src/main/java/org/matsim/contrib/noise/MergeNoiseCSVFile.java b/contribs/noise/src/main/java/org/matsim/contrib/noise/MergeNoiseCSVFile.java index d052f341cfe..7abae434d94 100644 --- a/contribs/noise/src/main/java/org/matsim/contrib/noise/MergeNoiseCSVFile.java +++ b/contribs/noise/src/main/java/org/matsim/contrib/noise/MergeNoiseCSVFile.java @@ -392,6 +392,7 @@ private void readReceiverPoints() { } } + //TODO this should be updated to use CSVReader or something as robust private void readValues() { for (int ll = 0; ll < this.labels.length; ll++) { diff --git a/contribs/noise/src/main/java/org/matsim/contrib/noise/NoiseWriter.java b/contribs/noise/src/main/java/org/matsim/contrib/noise/NoiseWriter.java index de465940c1a..6187e0b2f06 100644 --- a/contribs/noise/src/main/java/org/matsim/contrib/noise/NoiseWriter.java +++ b/contribs/noise/src/main/java/org/matsim/contrib/noise/NoiseWriter.java @@ -78,17 +78,17 @@ public static void writeReceiverPoints(NoiseContext noiseContext, String outputP // .addAttribute("Id", String.class) // .create(); // Collection features = new ArrayList(); -// +// // for (ReceiverPoint rp : noiseContext.getReceiverPoints().values()) { -// +// // SimpleFeature feature = factory.createPoint(MGC.coord2Coordinate(rp.getCoord()), new Object[] {rp.getId().toString()}, null); // features.add(feature); // } -// +// // String filePath = outputPath; // File file = new File(filePath); // file.mkdirs(); -// +// // log.info("Writing receiver points to shapefile... "); // ShapeFileWriter.writeGeometries(features, filePath + "receiverPoints.shp"); // log.info("Writing receiver points to shapefile... Done. "); @@ -318,13 +318,11 @@ public static void writeDamageInfoPerHour(NoiseContext noiseContext, String outp try { BufferedWriter bw = new BufferedWriter(new FileWriter(file)); - - bw.write("Receiver Point Id;Damages " + Time.writeTime(timeInterval, Time.TIMEFORMAT_HHMMSS)); + bw.write("Receiver Point Id;Damages " + Time.writeTime(timeInterval, Time.TIMEFORMAT_HHMMSS) + ";x;y;t"); bw.newLine(); for (NoiseReceiverPoint rp : noiseContext.getReceiverPoints().values()) { - - bw.write(rp.getId() + ";" + rp.getDamageCosts()); + bw.write(rp.getId() + ";" + rp.getDamageCosts() + ";" + rp.getCoord().getX() + ";" + rp.getCoord().getY() + ";" + timeInterval ); bw.newLine(); } From 6ba2578003eaed40f431e3401b88c3a98035aefc Mon Sep 17 00:00:00 2001 From: schlenther Date: Fri, 9 Aug 2024 13:11:10 +0200 Subject: [PATCH 4/6] NoiseDashboard: include stats and damage plots --- .../analysis/noise/MergeNoiseOutput.java | 11 +++- .../analysis/noise/NoiseAnalysis.java | 26 +++++++- .../simwrapper/dashboard/NoiseDashboard.java | 65 ++++++++++++++----- .../dashboard/NoiseDashboardTests.java | 7 +- 4 files changed, 89 insertions(+), 20 deletions(-) diff --git a/contribs/application/src/main/java/org/matsim/application/analysis/noise/MergeNoiseOutput.java b/contribs/application/src/main/java/org/matsim/application/analysis/noise/MergeNoiseOutput.java index e59c4aee75c..b5b85336c1c 100644 --- a/contribs/application/src/main/java/org/matsim/application/analysis/noise/MergeNoiseOutput.java +++ b/contribs/application/src/main/java/org/matsim/application/analysis/noise/MergeNoiseOutput.java @@ -53,6 +53,8 @@ final class MergeNoiseOutput { private final int minTime = 3600; private int maxTime = 24 * 3600; + private final Map totalReceiverPointValues = new HashMap<>(); + MergeNoiseOutput(Path path, String coordinateSystem ) { this.outputDirectory = path; this.crs = coordinateSystem; @@ -162,6 +164,7 @@ private void mergeReceiverPointData(String outputDir, String label) { // Loop over all files //TODO could be adjusted to time bin size from noise config group + String substrToCapitalize = null; for (int time = minTime; time <= maxTime; time += 3600) { String timeDataFile = outputDir + label + "_" + round(time, 1) + ".csv"; @@ -174,7 +177,7 @@ private void mergeReceiverPointData(String outputDir, String label) { } //we need "damages_receiverPoint" -> "Damages 01:00:00" and "immission" -> "Immision 01:00:00" - String substrToCapitalize = label.contains("_") ? label.substring(0, label.lastIndexOf("_")) : label; + substrToCapitalize = label.contains("_") ? label.substring(0, label.lastIndexOf("_")) : label; String valueHeader = StringUtils.capitalize(substrToCapitalize) + " " + Time.writeTime(time, Time.TIMEFORMAT_HHMMSS); // Read the data file @@ -255,9 +258,12 @@ private void mergeReceiverPointData(String outputDir, String label) { File outDay = outputDirectory.getParent().resolve(label + "_per_day.avro").toFile(); writeAvro(xytDayData, outDay); + //cache the overall sum + this.totalReceiverPointValues.put(substrToCapitalize, raw.stream().reduce(0f, Float::sum)); } // Merges the immissions data + @Deprecated private void mergeImmissionsCSV(String pathParameter, String label) { log.info("Merging immissions data for label {}", label); @@ -317,4 +323,7 @@ private void mergeImmissionsCSV(String pathParameter, String label) { log.info("Merged noise data written to {} ", outPerDay); } + public Map getTotalReceiverPointValues() { + return totalReceiverPointValues; + } } diff --git a/contribs/application/src/main/java/org/matsim/application/analysis/noise/NoiseAnalysis.java b/contribs/application/src/main/java/org/matsim/application/analysis/noise/NoiseAnalysis.java index deceb84624e..1892de7fb74 100644 --- a/contribs/application/src/main/java/org/matsim/application/analysis/noise/NoiseAnalysis.java +++ b/contribs/application/src/main/java/org/matsim/application/analysis/noise/NoiseAnalysis.java @@ -1,5 +1,7 @@ package org.matsim.application.analysis.noise; +import org.apache.commons.csv.CSVFormat; +import org.apache.commons.csv.CSVPrinter; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.locationtech.jts.geom.Envelope; @@ -19,9 +21,15 @@ import org.matsim.core.config.ConfigUtils; import org.matsim.core.scenario.ScenarioUtils; import org.matsim.core.utils.geometry.CoordinateTransformation; +import org.matsim.core.utils.io.IOUtils; import picocli.CommandLine; +import java.io.IOException; import java.nio.file.Path; +import java.text.DecimalFormat; +import java.text.DecimalFormatSymbols; +import java.util.Locale; +import java.util.Map; import java.util.Objects; import java.util.Set; @@ -36,7 +44,10 @@ produces = { "emission_per_day.csv", "immission_per_day.%s", - "immission_per_hour.%s" + "immission_per_hour.%s", + "damages_receiverPoint_per_hour.%s", + "damages_receiverPoint_per_day.%s", + "noise_stats.csv" } ) public class NoiseAnalysis implements MATSimAppCommand { @@ -77,7 +88,7 @@ public Integer call() throws Exception { NoiseConfigGroup noiseParameters = ConfigUtils.addOrGetModule(config, NoiseConfigGroup.class); if(overrideParameters){ - log.warn("no NoiseConfigGroup was configured before. Will set som estandards. You should check the next lines in the log file!"); + log.warn("no NoiseConfigGroup was configured before. Will set some standards. You should check the next lines in the log file!"); noiseParameters.setConsideredActivitiesForReceiverPointGridArray(considerActivities.toArray(String[]::new)); noiseParameters.setConsideredActivitiesForDamageCalculationArray(considerActivities.toArray(String[]::new)); @@ -147,6 +158,17 @@ public Integer call() throws Exception { MergeNoiseOutput mergeNoiseOutput = new MergeNoiseOutput(Path.of(outputFilePath), config.global().getCoordinateSystem()); mergeNoiseOutput.run(); + // Total stats + DecimalFormat df = new DecimalFormat("#.###", DecimalFormatSymbols.getInstance(Locale.US)); + try (CSVPrinter printer = new CSVPrinter(IOUtils.getBufferedWriter(output.getPath("noise_stats.csv").toString()), CSVFormat.DEFAULT)) { + printer.printRecord("Annual cost rate per pop. unit [€]:", df.format(noiseParameters.getAnnualCostRate())); + for (Map.Entry labelValueEntry : mergeNoiseOutput.getTotalReceiverPointValues().entrySet()) { + printer.printRecord("Total " + labelValueEntry.getKey() + " at receiver points", df.format(labelValueEntry.getValue())); + } + } catch (IOException ex) { + log.error(ex); + } + return 0; } diff --git a/contribs/simwrapper/src/main/java/org/matsim/simwrapper/dashboard/NoiseDashboard.java b/contribs/simwrapper/src/main/java/org/matsim/simwrapper/dashboard/NoiseDashboard.java index 036b3c5a718..48f173a8adb 100644 --- a/contribs/simwrapper/src/main/java/org/matsim/simwrapper/dashboard/NoiseDashboard.java +++ b/contribs/simwrapper/src/main/java/org/matsim/simwrapper/dashboard/NoiseDashboard.java @@ -1,6 +1,7 @@ package org.matsim.simwrapper.dashboard; import org.matsim.application.analysis.noise.NoiseAnalysis; +import org.matsim.application.analysis.population.StuckAgentAnalysis; import org.matsim.application.prepare.network.CreateAvroNetwork; import org.matsim.simwrapper.Dashboard; import org.matsim.simwrapper.Header; @@ -8,6 +9,7 @@ import org.matsim.simwrapper.viz.ColorScheme; import org.matsim.simwrapper.viz.GridMap; import org.matsim.simwrapper.viz.MapPlot; +import org.matsim.simwrapper.viz.Tile; /** * Shows emission in the scenario. @@ -32,19 +34,12 @@ public void configure(Header header, Layout layout) { header.title = "Noise"; header.description = "Shows the noise footprint and spatial distribution."; - layout.row("aggregate noise") - .el(GridMap.class, (viz, data) -> { - viz.title = "Noise Immissions (Grid)"; - viz.description = "Total Noise Immissions per day"; - viz.height = 12.0; - viz.cellSize = 250; - viz.opacity = 0.2; - viz.maxHeight = 20; - viz.center = data.context().getCenter(); - viz.zoom = data.context().mapZoomLevel; - viz.setColorRamp(new double[]{30, 40, 50, 60, 70}, new String[]{"#1175b3", "#95c7df", "#dfdb95", "#dfb095", "#f4a986", "#cc0c27"}); - viz.file = data.computeWithPlaceholder(NoiseAnalysis.class, "immission_per_day.%s", "avro"); - }) + layout.row("stats") + .el(Tile.class, (viz, data) -> { + viz.dataset = data.compute(NoiseAnalysis.class, "noise_stats.csv"); + viz.height = 0.1; + }); + layout.row("emissions") .el(MapPlot.class, (viz, data) -> { viz.title = "Noise Emissions (Link)"; viz.description = "Maximum Noise Level per day [dB]"; @@ -65,18 +60,56 @@ public void configure(Header header, Layout layout) { viz.display.lineWidth.scaleFactor = 8d; viz.display.lineWidth.join = "Link Id"; }); - layout.row("hourly noise") + layout.row("imissions") + .el(GridMap.class, (viz, data) -> { + viz.title = "Noise Immissions (Grid)"; + viz.description = "Total Noise Immissions per day"; + viz.height = 12.0; + viz.cellSize = 250; + viz.opacity = 0.1; + viz.maxHeight = 40; + viz.center = data.context().getCenter(); + viz.zoom = data.context().mapZoomLevel; + viz.setColorRamp(new double[]{30, 40, 50, 60, 70}, new String[]{"#1175b3", "#95c7df", "#dfdb95", "#dfb095", "#f4a986", "#cc0c27"}); + viz.file = data.computeWithPlaceholder(NoiseAnalysis.class, "immission_per_day.%s", "avro"); + }) .el(GridMap.class, (viz, data) -> { viz.title = "Hourly Noise Immissions (Grid)"; viz.description = "Noise Immissions per hour"; viz.height = 12.0; viz.cellSize = 250; - viz.opacity = 0.2; - viz.maxHeight = 20; + viz.opacity = 0.1; + viz.maxHeight = 40; viz.center = data.context().getCenter(); viz.zoom = data.context().mapZoomLevel; viz.setColorRamp(new double[]{30, 40, 50, 60, 70}, new String[]{"#1175b3", "#95c7df", "#dfdb95", "#dfb095", "#f4a986", "#cc0c27"}); viz.file = data.computeWithPlaceholder(NoiseAnalysis.class, "immission_per_hour.%s", "avro"); }); + layout.row("damages") + .el(GridMap.class, (viz, data) -> { + viz.title = "Daily Noise Damages (Grid)"; + viz.description = "Total Noise Damages per day [€]"; + viz.height = 12.0; + viz.cellSize = 250; + viz.opacity = 0.1; + viz.maxHeight = 40; + viz.center = data.context().getCenter(); + viz.zoom = data.context().mapZoomLevel; + viz.setColorRamp(ColorScheme.Oranges); + viz.file = data.computeWithPlaceholder(NoiseAnalysis.class, "damages_receiverPoint_per_day.%s", "avro"); + }) + .el(GridMap.class, (viz, data) -> { + viz.title = "Hourly Noise Damages (Grid)"; + viz.description = "Noise Damages per hour [€]"; + viz.height = 12.0; + viz.cellSize = 250; + viz.opacity = 0.2; + viz.maxHeight = 40; + viz.center = data.context().getCenter(); + viz.zoom = data.context().mapZoomLevel; +// viz.setColorRamp(new double[]{30, 40, 50, 60, 70}, new String[]{"#1175b3", "#95c7df", "#dfdb95", "#dfb095", "#f4a986", "#cc0c27"}); + viz.setColorRamp(ColorScheme.Oranges); + viz.file = data.computeWithPlaceholder(NoiseAnalysis.class, "damages_receiverPoint_per_hour.%s", "avro"); + }); } } diff --git a/contribs/simwrapper/src/test/java/org/matsim/simwrapper/dashboard/NoiseDashboardTests.java b/contribs/simwrapper/src/test/java/org/matsim/simwrapper/dashboard/NoiseDashboardTests.java index 6ca2a08cee3..fdc846cdff5 100644 --- a/contribs/simwrapper/src/test/java/org/matsim/simwrapper/dashboard/NoiseDashboardTests.java +++ b/contribs/simwrapper/src/test/java/org/matsim/simwrapper/dashboard/NoiseDashboardTests.java @@ -46,6 +46,11 @@ void generate() { Assertions.assertThat(out) .isDirectoryContaining("glob:**emission_per_day.csv") .isDirectoryContaining("glob:**immission_per_day.avro") - .isDirectoryContaining("glob:**immission_per_hour.avro"); + .isDirectoryContaining("glob:**immission_per_hour.avro") + .isDirectoryContaining("glob:**damages_receiverPoint_per_hour.avro") + .isDirectoryContaining("glob:**damages_receiverPoint_per_day.avro") + .isDirectoryContaining("glob:**noise_stats.csv"); + + //TODO check content / values of the files } } From e2f7c360eda279ed4a0b07ec3298b5bb910413af Mon Sep 17 00:00:00 2001 From: schlenther Date: Fri, 9 Aug 2024 14:18:15 +0200 Subject: [PATCH 5/6] CreateSingleSimWrapperDashboard --- .../CreateSingleSimWrapperDashboard.java | 148 ++++++++++++++++++ 1 file changed, 148 insertions(+) create mode 100644 contribs/simwrapper/src/main/java/org/matsim/simwrapper/CreateSingleSimWrapperDashboard.java diff --git a/contribs/simwrapper/src/main/java/org/matsim/simwrapper/CreateSingleSimWrapperDashboard.java b/contribs/simwrapper/src/main/java/org/matsim/simwrapper/CreateSingleSimWrapperDashboard.java new file mode 100644 index 00000000000..fce848736a6 --- /dev/null +++ b/contribs/simwrapper/src/main/java/org/matsim/simwrapper/CreateSingleSimWrapperDashboard.java @@ -0,0 +1,148 @@ +/* *********************************************************************** * + * project: org.matsim.* + * Controler.java + * * + * *********************************************************************** * + * * + * copyright : (C) 2007 by the members listed in the COPYING, * + * LICENSE and WARRANTY file. * + * email : info at matsim dot org * + * * + * *********************************************************************** * + * * + * This program is free software; you can redistribute it and/or modify * + * it under the terms of the GNU General Public License as published by * + * the Free Software Foundation; either version 2 of the License, or * + * (at your option) any later version. * + * See also COPYING, LICENSE and WARRANTY file * + * * + * *********************************************************************** */ + +package org.matsim.simwrapper; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.matsim.application.ApplicationUtils; +import org.matsim.application.MATSimAppCommand; +import org.matsim.application.options.ShpOptions; +import org.matsim.core.config.Config; +import org.matsim.core.config.ConfigUtils; +import org.matsim.simwrapper.dashboard.*; +import picocli.CommandLine; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.List; + +@CommandLine.Command( + name = "dashboard", + description = "Run analysis and create SimWrapper dashboard for existing run output." +) + +/** + * This class creates single SimWrapper dashboards for multiple output directories. It is meant to be run as a post-process, + * e.g. when a specific dashboard was missing after the initial run (for whatever reason). + * It will create the dashboard with all standard settings. + * Depending on the dashboard type, it might be required to provide ShpOptions for data filtering. + * TODO: test whether this works for the non-noise DashboardTypes. + */ +final class CreateSingleSimWrapperDashboard implements MATSimAppCommand { + + private static final Logger log = LogManager.getLogger(CreateSingleSimWrapperDashboard.class); + + @CommandLine.Option(names = "--type", required = true, description = "Provide the dashboard type to be generated. See DashboardType enum within this class.") + private DashboardType dashboardType; + + @CommandLine.Parameters(arity = "1..*", description = "Path to run output directories for which the dashboards is to be generated.") + private List inputPaths; + + @CommandLine.Mixin + private final ShpOptions shp = new ShpOptions(); + + enum DashboardType{ + noise, + emissions, + traffic, + overview, + stuckAgent, + populationAttribute, + ODTrip, + trip, + publicTransit + } + + private CreateSingleSimWrapperDashboard(){ + } + + @Override + public Integer call() throws Exception { + + for (Path runDirectory : inputPaths) { + log.info("Creating " + dashboardType + " for {}", runDirectory); + + Path configPath = ApplicationUtils.matchInput("config.xml", runDirectory); + Config config = ConfigUtils.loadConfig(configPath.toString()); + SimWrapper sw = SimWrapper.create(config); + + SimWrapperConfigGroup simwrapperCfg = ConfigUtils.addOrGetModule(config, SimWrapperConfigGroup.class); + + if (shp.isDefined()){ + //not sure if this is the best way to go, might be that the shape file would be automatically read by providing the --shp command line option + simwrapperCfg.defaultParams().shp = shp.getShapeFile().toString(); + } + + //skip default dashboards + simwrapperCfg.defaultDashboards = SimWrapperConfigGroup.Mode.disabled; + + //add dashboard + switch (dashboardType) { + case noise -> { + sw.addDashboard(new NoiseDashboard()); + } + case emissions -> { + sw.addDashboard(new EmissionsDashboard()); + } + case traffic -> { + sw.addDashboard(new TrafficDashboard()); + } + case overview -> { + sw.addDashboard(new OverviewDashboard()); + } + case stuckAgent -> { + sw.addDashboard(new StuckAgentDashboard()); + } + case populationAttribute -> { + sw.addDashboard(new PopulationAttributeDashboard()); + } + case ODTrip -> { + throw new RuntimeException("ODTripDashboard needs additional information. Single creation is currently not implemented"); +// sw.addDashboard(new ODTripDashboard()); + } + case trip -> { + sw.addDashboard(new TripDashboard()); + } + case publicTransit -> { + sw.addDashboard(new PublicTransitDashboard()); + } + default -> throw new IllegalArgumentException("unkown dashboard type: " + dashboardType); + } + + try { + //append dashboard to existing ones + boolean append = true; + sw.generate(runDirectory, append); + sw.run(runDirectory); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + return 0; + } + + public static void main(String[] args) { + new CreateSingleSimWrapperDashboard().execute(args); + } + +} + From 9175ec805299815d0d8797c57de367e908aa31eb Mon Sep 17 00:00:00 2001 From: schlenther Date: Fri, 9 Aug 2024 14:40:02 +0200 Subject: [PATCH 6/6] fix noise tests: explicitly set activity types for rp grid, when setting act type for damage calculation --- .../org/matsim/contrib/noise/NoiseIT.java | 4 ++ .../noise/NoiseConfigGroupTest/config1.xml | 47 ++++++++++--------- 2 files changed, 28 insertions(+), 23 deletions(-) diff --git a/contribs/noise/src/test/java/org/matsim/contrib/noise/NoiseIT.java b/contribs/noise/src/test/java/org/matsim/contrib/noise/NoiseIT.java index e8aa97a3ab3..542580dcb44 100644 --- a/contribs/noise/src/test/java/org/matsim/contrib/noise/NoiseIT.java +++ b/contribs/noise/src/test/java/org/matsim/contrib/noise/NoiseIT.java @@ -93,6 +93,7 @@ final void test1(){ String[] consideredActivities = {"home", "work"}; noiseParameters.setConsideredActivitiesForDamageCalculationArray(consideredActivities); + noiseParameters.setConsideredActivitiesForReceiverPointGridArray(consideredActivities); com.google.inject.Injector injector = Injector.createInjector( scenario.getConfig() , new AbstractModule(){ @Override public void install(){ @@ -197,6 +198,7 @@ private static void runTest2a( Config runConfig ) { String[] consideredActivities = {"home", "work"}; noiseParameters.setConsideredActivitiesForDamageCalculationArray(consideredActivities); + noiseParameters.setConsideredActivitiesForReceiverPointGridArray(consideredActivities); noiseParameters.setScaleFactor(1.); noiseParameters.setUseActualSpeedLevel(false); @@ -984,6 +986,7 @@ final void test2b(){ String[] consideredActivities = {"home", "work"}; noiseParameters.setConsideredActivitiesForDamageCalculationArray(consideredActivities); + noiseParameters.setConsideredActivitiesForReceiverPointGridArray(consideredActivities); noiseParameters.setScaleFactor(1.); noiseParameters.setNoiseAllocationApproach(NoiseConfigGroup.NoiseAllocationApproach.MarginalCost); @@ -1073,6 +1076,7 @@ final void test2c(){ String[] consideredActivities = {"home", "work"}; noiseParameters.setConsideredActivitiesForDamageCalculationArray(consideredActivities); + noiseParameters.setConsideredActivitiesForReceiverPointGridArray(consideredActivities); noiseParameters.setScaleFactor(1.); noiseParameters.setUseActualSpeedLevel(true); diff --git a/contribs/noise/test/input/org/matsim/contrib/noise/NoiseConfigGroupTest/config1.xml b/contribs/noise/test/input/org/matsim/contrib/noise/NoiseConfigGroupTest/config1.xml index e2d42df47cf..f4830014284 100644 --- a/contribs/noise/test/input/org/matsim/contrib/noise/NoiseConfigGroupTest/config1.xml +++ b/contribs/noise/test/input/org/matsim/contrib/noise/NoiseConfigGroupTest/config1.xml @@ -1,24 +1,25 @@ - + - + + - + - + - + @@ -34,20 +35,20 @@ - - + + - + - + - + - + @@ -66,26 +67,26 @@ - + - + - + - + - + - + @@ -171,20 +172,20 @@ - + - + - + - + - + @@ -199,5 +200,5 @@ - +