Skip to content

Commit

Permalink
Counts v2 format (#2840)
Browse files Browse the repository at this point in the history 
* created classes for multi mode counts

* started writer class, added element names

* Renamed Volumes.java to Measurable.java to make the class more flexible. Updated MultiModeCount class to new functionality

* finished writer class, adapted count classes for better writing performance.

* added reader, cleaned up writer class. checkstyle fixing

* test class for general handling, writer and reader

* added null value filter to writer

* added new multi mode counts

* extended test

* bug fixing, code improvements

* started cleanup of new multi mode counts

* improved API

* update reader and writer and tests

* start to provide old api in the new format

* use id class in v2 reader

* separated v1 and v2, updated tests

* update tests

* fix type error with new counts

* fix counts v1 tests

* add more backwards compatible API

* change count resolution to seconds instead of minutes

* update warning for backwards compatibility

* counts v2 is now zero based, deprecated old methods

* mode is now networkMode

* fix hour offsets in compatibility class

* remove old uncheckedioexception

* fix backwards compatibility

* make refType optional for now

* update traffic counts dashboard for new counts format

* update count dashboard to new format

---------

Co-authored-by: hzoerner <[email protected]>
  • Loading branch information
@rakow @hzoerner
rakow and hzoerner authored Nov 28, 2023
1 parent 32c0d83 commit de7b2a7
Show file tree
Hide file tree
Showing 24 changed files with 7,247 additions and 3,335 deletions.
111 changes: 73 additions & 38 deletions ...cation/src/main/java/org/matsim/application/analysis/traffic/CountComparisonAnalysis.java
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
package org.matsim.application.analysis.traffic;

import it.unimi.dsi.fastutil.ints.Int2DoubleMap;
import it.unimi.dsi.fastutil.ints.Int2DoubleOpenHashMap;
import org.matsim.analysis.VolumesAnalyzer;
import org.matsim.api.core.v01.Id;
import org.matsim.api.core.v01.TransportMode;
Expand All @@ -13,10 +15,7 @@
import org.matsim.core.api.experimental.events.EventsManager;
import org.matsim.core.events.EventsUtils;
import org.matsim.core.network.NetworkUtils;
import org.matsim.counts.Count;
import org.matsim.counts.Counts;
import org.matsim.counts.MatsimCountsReader;
import org.matsim.counts.Volume;
import org.matsim.counts.*;
import picocli.CommandLine;
import tech.tablesaw.api.*;
import tech.tablesaw.selection.Selection;
Expand All @@ -29,7 +28,7 @@

@CommandLine.Command(name = "count-comparison", description = "Produces comparisons of observed and simulated counts.")
@CommandSpec(requireEvents = true, requireCounts = true, requireNetwork = true,
produces = {"count_comparison_by_hour.csv", "count_comparison_daily.csv", "count_comparison_quality.csv", "count_error_by_hour.csv"})
produces = {"count_comparison_by_hour.csv", "count_comparison_daily.csv", "count_comparison_quality.csv", "count_error_by_hour.csv"})
public class CountComparisonAnalysis implements MATSimAppCommand {

@CommandLine.Mixin
Expand All @@ -47,7 +46,7 @@ public class CountComparisonAnalysis implements MATSimAppCommand {
@CommandLine.Option(names = "--labels", split = ",", description = "Labels for quality categories", defaultValue = "major under,under,ok,over,major over")
private List<String> labels;

@CommandLine.Option(names = "--transport-mode", description = "Mode to analyze", split = ",", defaultValue = TransportMode.car)
@CommandLine.Option(names = {"--network-mode", "--transport-mode"}, description = "Mode(s) to analyze", split = ",", defaultValue = TransportMode.car)
private Set<String> modes;

public static void main(String[] args) {
Expand All @@ -65,6 +64,9 @@ private static String cut(double relError, List<Double> errorGroups, List<String
return labels.get(ins);
}

/**
* Sum two arrays element-wise into new array.
*/
private static int[] sum(int[] a, int[] b) {
int[] counts = new int[a.length];
for (int i = 0; i < counts.length; i++) {
Expand Down Expand Up @@ -107,25 +109,27 @@ private Table writeOutput(Counts<Link> counts, Network network, VolumesAnalyzer
Map<Id<Link>, ? extends Link> links = network.getLinks();

Table byHour = Table.create(
StringColumn.create("link_id"),
StringColumn.create("name"),
StringColumn.create("road_type"),
IntColumn.create("hour"),
DoubleColumn.create("observed_traffic_volume"),
DoubleColumn.create("simulated_traffic_volume")
StringColumn.create("link_id"),
StringColumn.create("name"),
StringColumn.create("road_type"),
IntColumn.create("hour"),
DoubleColumn.create("observed_traffic_volume"),
DoubleColumn.create("simulated_traffic_volume")
);

Table dailyTrafficVolume = Table.create(StringColumn.create("link_id"),
StringColumn.create("name"),
StringColumn.create("road_type"),
DoubleColumn.create("observed_traffic_volume"),
DoubleColumn.create("simulated_traffic_volume")
StringColumn.create("name"),
StringColumn.create("road_type"),
DoubleColumn.create("observed_traffic_volume"),
DoubleColumn.create("simulated_traffic_volume")
);

for (Map.Entry<Id<Link>, Count<Link>> entry : counts.getCounts().entrySet()) {
for (Map.Entry<Id<Link>, MeasurementLocation<Link>> entry : counts.getMeasureLocations().entrySet()) {
Id<Link> key = entry.getKey();
Map<Integer, Volume> countVolume = entry.getValue().getVolumes();
String name = entry.getValue().getCsLabel();

Int2DoubleMap countVolume = aggregateObserved(entry.getValue(), modes);

String name = entry.getValue().getDisplayName();

Link link = links.get(key);
String type = NetworkUtils.getHighwayType(link);
Expand All @@ -134,9 +138,9 @@ private Table writeOutput(Counts<Link> counts, Network network, VolumesAnalyzer
continue;

Optional<int[]> opt = modes.stream()
.map(mode -> volumes.getVolumesForLink(key, mode))
.filter(Objects::nonNull)
.reduce(CountComparisonAnalysis::sum);
.map(mode -> volumes.getVolumesForLink(key, mode))
.filter(Objects::nonNull)
.reduce(CountComparisonAnalysis::sum);

int[] volumesForLink;
if (countVolume.isEmpty() || opt.isEmpty()) {
Expand All @@ -148,12 +152,13 @@ private Table writeOutput(Counts<Link> counts, Network network, VolumesAnalyzer
double simulatedTrafficVolumeByDay = 0;
double observedTrafficVolumeByDay = 0;

if (countVolume.size() == 24) {
// Hourly values are present
if (countVolume.size() > 1 || !countVolume.containsKey(24)) {

for (int hour = 1; hour < 25; hour++) {
for (int hour = 0; hour < 24; hour++) {

double observedTrafficVolumeAtHour = countVolume.get(hour).getValue();
double simulatedTrafficVolumeAtHour = (double) volumesForLink[hour - 1] / this.sample.getSample();
double observedTrafficVolumeAtHour = countVolume.get(hour);
double simulatedTrafficVolumeAtHour = (double) volumesForLink[hour] / this.sample.getSample();

simulatedTrafficVolumeByDay += simulatedTrafficVolumeAtHour;
observedTrafficVolumeByDay += observedTrafficVolumeAtHour;
Expand All @@ -162,12 +167,15 @@ private Table writeOutput(Counts<Link> counts, Network network, VolumesAnalyzer
row.setString("link_id", key.toString());
row.setString("name", name);
row.setString("road_type", type);
row.setInt("hour", hour - 1);
row.setInt("hour", hour);
row.setDouble("observed_traffic_volume", observedTrafficVolumeAtHour);
row.setDouble("simulated_traffic_volume", simulatedTrafficVolumeAtHour);
}
} else
observedTrafficVolumeByDay = countVolume.values().stream().mapToDouble(Volume::getValue).sum();
} else {
// Get the daily values
observedTrafficVolumeByDay = countVolume.get(24);
simulatedTrafficVolumeByDay = Arrays.stream(volumesForLink).sum() / this.sample.getSample();
}

Row row = dailyTrafficVolume.appendRow();
row.setString("link_id", key.toString());
Expand All @@ -178,12 +186,12 @@ private Table writeOutput(Counts<Link> counts, Network network, VolumesAnalyzer
}

DoubleColumn relError = dailyTrafficVolume.doubleColumn("simulated_traffic_volume")
.divide(dailyTrafficVolume.doubleColumn("observed_traffic_volume"))
.setName("rel_error");
.divide(dailyTrafficVolume.doubleColumn("observed_traffic_volume"))
.setName("rel_error");

StringColumn qualityLabel = relError.copy()
.map(err -> cut(err, limits, labels), ColumnType.STRING::create)
.setName("quality");
.map(err -> cut(err, limits, labels), ColumnType.STRING::create)
.setName("quality");

dailyTrafficVolume.addColumns(relError, qualityLabel);

Expand Down Expand Up @@ -220,20 +228,47 @@ private Table writeOutput(Counts<Link> counts, Network network, VolumesAnalyzer
return byHour;
}

/**
* Aggregate observed counts by hour. Starting at 0.
*/
private static Int2DoubleMap aggregateObserved(MeasurementLocation<Link> m, Set<String> modes) {

Int2DoubleOpenHashMap map = new Int2DoubleOpenHashMap();

for (String mode : modes) {
Measurable volumes = m.getVolumesForMode(mode);
if (volumes == null)
continue;

if (volumes.supportsHourlyAggregate()) {
for (int i = 0; i < 24; i++) {
OptionalDouble v = volumes.aggregateAtHour(i);
if (v.isPresent())
map.mergeDouble(i, v.getAsDouble(), Double::sum);
}
} else {
// Daily value is stored under separate key
map.mergeDouble(24, volumes.aggregateDaily(), Double::sum);
}
}

return map;
}

private void writeErrorMetrics(Table byHour, Path path) {

byHour.addColumns(
byHour.doubleColumn("simulated_traffic_volume").subtract(byHour.doubleColumn("observed_traffic_volume")).setName("error")
byHour.doubleColumn("simulated_traffic_volume").subtract(byHour.doubleColumn("observed_traffic_volume")).setName("error")
);

byHour.addColumns(
byHour.doubleColumn("error").abs().setName("abs_error")
byHour.doubleColumn("error").abs().setName("abs_error")
);

DoubleColumn relError = byHour.doubleColumn("abs_error")
.multiply(100)
.divide(byHour.doubleColumn("observed_traffic_volume"))
.setName("rel_error");
.multiply(100)
.divide(byHour.doubleColumn("observed_traffic_volume"))
.setName("rel_error");

// Cut-off at Max error
relError = relError.set(relError.isMissing(), 1000d);
Expand Down
Loading

0 comments on commit de7b2a7

Please sign in to comment.