diff --git a/input/v2024.2/lausitz-v2024.2-10pct.config.xml b/input/v2024.2/lausitz-v2024.2-10pct.config.xml index 7074570..4af2bce 100644 --- a/input/v2024.2/lausitz-v2024.2-10pct.config.xml +++ b/input/v2024.2/lausitz-v2024.2-10pct.config.xml @@ -11,7 +11,7 @@ - + @@ -26,28 +26,28 @@ + value="https://svn.vsp.tu-berlin.de/repos/public-svn/matsim/scenarios/countries/de/lausitz/lausitz-v2024.2/input/lausitz-v2024.2-network-with-pt.xml.gz"/> + value="https://svn.vsp.tu-berlin.de/repos/public-svn/matsim/scenarios/countries/de/lausitz/lausitz-v2024.2/input/lausitz-v2024.2-10pct.plans-initial.xml.gz"/> - + - - + + - + diff --git a/src/main/java/org/matsim/dashboards/LausitzDashboardProvider.java b/src/main/java/org/matsim/dashboards/LausitzDashboardProvider.java index e4bd385..4bcf8c2 100644 --- a/src/main/java/org/matsim/dashboards/LausitzDashboardProvider.java +++ b/src/main/java/org/matsim/dashboards/LausitzDashboardProvider.java @@ -25,7 +25,8 @@ public List getDashboards(Config config, SimWrapper simWrapper) { .setAnalysisArgs("--person-filter", "subpopulation=person"); return List.of(trips, - new EmissionsDashboard(config.global().getCoordinateSystem()) + new EmissionsDashboard(config.global().getCoordinateSystem()), + new PtLineDashboard("https://svn.vsp.tu-berlin.de/repos/public-svn/matsim/scenarios/countries/de/lausitz/output/v2024.2/") // the NoiseAnalysis is not run here because it needs more RAM than the entire simulation, // which leads to VM crashes and prevents other analysis to run. We have to run it separately (e.g. with LausitzSimWrapperRunner) ); diff --git a/src/main/java/org/matsim/dashboards/LausitzSimWrapperRunner.java b/src/main/java/org/matsim/dashboards/LausitzSimWrapperRunner.java index 4c0df2b..df2fa03 100644 --- a/src/main/java/org/matsim/dashboards/LausitzSimWrapperRunner.java +++ b/src/main/java/org/matsim/dashboards/LausitzSimWrapperRunner.java @@ -43,6 +43,7 @@ import java.io.IOException; import java.io.InterruptedIOException; +import java.nio.file.Files; import java.nio.file.Path; import java.util.List; @@ -66,16 +67,24 @@ public final class LausitzSimWrapperRunner implements MATSimAppCommand { private boolean trips; @CommandLine.Option(names = "--emissions", defaultValue = "false", description = "create emission dashboard") private boolean emissions; + @CommandLine.Option(names = "--pt-line-base-dir", description = "create pt line dashboard with base run dir as input") + private String baseDir; + + private static final String FILE_TYPE = "_before_emissions.xml"; public LausitzSimWrapperRunner(){ // public constructor needed for testing purposes. } + public static void main(String[] args) { + new LausitzSimWrapperRunner().execute(args); + } + @Override public Integer call() throws Exception { - if (!noise && !trips && !emissions){ + if (!noise && !trips && !emissions && baseDir == null){ throw new IllegalArgumentException("you have not configured any dashboard to be created! Please use command line parameters!"); } @@ -112,30 +121,40 @@ public Integer call() throws Exception { sw.addDashboard(Dashboard.customize(new EmissionsDashboard(config.global().getCoordinateSystem())).context("emissions")); LausitzScenario.setEmissionsConfigs(config); - ConfigUtils.writeConfig(config, configPath); - - Config dummyConfig = new Config(); String networkPath = ApplicationUtils.matchInput("output_network.xml.gz", runDirectory).toString(); String vehiclesPath = ApplicationUtils.matchInput("output_vehicles.xml.gz", runDirectory).toString(); String transitVehiclesPath = ApplicationUtils.matchInput("output_transitVehicles.xml.gz", runDirectory).toString(); + String populationPath = ApplicationUtils.matchInput("output_plans.xml.gz", runDirectory).toString(); - dummyConfig.network().setInputFile(networkPath); - dummyConfig.vehicles().setVehiclesFile(vehiclesPath); - dummyConfig.transit().setVehiclesFile(transitVehiclesPath); + config.network().setInputFile(networkPath); + config.vehicles().setVehiclesFile(vehiclesPath); + config.transit().setVehiclesFile(transitVehiclesPath); + config.plans().setInputFile(populationPath); - Scenario scenario = ScenarioUtils.loadScenario(dummyConfig); + Scenario scenario = ScenarioUtils.loadScenario(config); // adapt network and veh types for emissions analysis like in LausitzScenario base run class PrepareNetwork.prepareEmissionsAttributes(scenario.getNetwork()); LausitzScenario.prepareVehicleTypesForEmissionAnalysis(scenario); -// overwrite outputs with adapted files +// write outputs with adapted files. +// original output files need to be overwritten as AirPollutionAnalysis searches for "config.xml". +// copy old files to separate files + Files.copy(Path.of(configPath), getUniqueTargetPath(Path.of(configPath.split(".xml")[0] + FILE_TYPE))); + Files.copy(Path.of(networkPath), getUniqueTargetPath(Path.of(networkPath.split(".xml")[0] + FILE_TYPE + ".gz"))); + Files.copy(Path.of(vehiclesPath), getUniqueTargetPath(Path.of(vehiclesPath.split(".xml")[0] + FILE_TYPE + ".gz"))); + Files.copy(Path.of(transitVehiclesPath), getUniqueTargetPath(Path.of(transitVehiclesPath.split(".xml")[0] + FILE_TYPE + ".gz"))); + + ConfigUtils.writeConfig(config, configPath); NetworkUtils.writeNetwork(scenario.getNetwork(), networkPath); new MatsimVehicleWriter(scenario.getVehicles()).writeFile(vehiclesPath); new MatsimVehicleWriter(scenario.getTransitVehicles()).writeFile(transitVehiclesPath); } + if (baseDir != null) { + sw.addDashboard(new PtLineDashboard(baseDir)); + } try { sw.generate(runDirectory, true); @@ -148,9 +167,22 @@ public Integer call() throws Exception { return 0; } - public static void main(String[] args) { - new LausitzSimWrapperRunner().execute(args); + private static Path getUniqueTargetPath(Path targetPath) { + int counter = 1; + Path uniquePath = targetPath; + + // Add a suffix if the file already exists + while (Files.exists(uniquePath)) { + String originalPath = targetPath.toString(); + int dotIndex = originalPath.lastIndexOf("."); + if (dotIndex == -1) { + uniquePath = Path.of(originalPath + "_" + counter); + } else { + uniquePath = Path.of(originalPath.substring(0, dotIndex) + "_" + counter + originalPath.substring(dotIndex)); + } + counter++; + } + return uniquePath; } - } diff --git a/src/main/java/org/matsim/dashboards/PtLineDashboard.java b/src/main/java/org/matsim/dashboards/PtLineDashboard.java new file mode 100644 index 0000000..3c14534 --- /dev/null +++ b/src/main/java/org/matsim/dashboards/PtLineDashboard.java @@ -0,0 +1,153 @@ +package org.matsim.dashboards; + +import org.matsim.run.analysis.PtLineAnalysis; +import org.matsim.run.scenarios.LausitzScenario; +import org.matsim.simwrapper.Dashboard; +import org.matsim.simwrapper.Header; +import org.matsim.simwrapper.Layout; +import org.matsim.simwrapper.viz.*; +import tech.tablesaw.plotly.traces.BarTrace; + +import java.util.ArrayList; +import java.util.List; + +/** + * Shows information about an optional policy case, which implements a pt line between Cottbus and Hoyerswerda. + * It also compares the agents and their trips using the new pt line with their respective trips in the base case. + */ +public class PtLineDashboard implements Dashboard { + private final String basePath; + private static final String SHARE = "share"; + private static final String ABSOLUTE = "Count [person]"; + private static final String INCOME_GROUP = "incomeGroup"; + private static final String DESCRIPTION = "... in base and policy case"; + + PtLineDashboard(String basePath) { + if (!basePath.endsWith("/")) { + basePath += "/"; + } + this.basePath = basePath; + } + + @Override + public void configure(Header header, Layout layout) { + header.title = "Pt Line Dashboard"; + header.description = "Shows statistics about agents, who used the newly implemented pt line between Cottbus and Hoyerswerda " + + "and compares to the trips of those agents in the base case."; + + String[] args = new ArrayList<>(List.of("--base-path", basePath)).toArray(new String[0]); + + layout.row("first") + .el(Tile.class, (viz, data) -> { + viz.dataset = data.compute(PtLineAnalysis.class, "mean_travel_stats.csv", args); + viz.height = 0.1; + }); + + layout.row("income") + .el(Bar.class, (viz, data) -> { + viz.title = "Agents per income group"; + viz.stacked = false; + viz.dataset = data.compute(PtLineAnalysis.class, "pt_persons_income_groups.csv", args); + viz.x = INCOME_GROUP; + viz.xAxisName = INCOME_GROUP; + viz.yAxisName = SHARE; + viz.columns = List.of(SHARE); + }) + .el(Bar.class, (viz, data) -> { + viz.title = "Agents per income group"; + viz.stacked = false; + viz.dataset = data.compute(PtLineAnalysis.class, "pt_persons_income_groups.csv", args); + viz.x = INCOME_GROUP; + viz.xAxisName = INCOME_GROUP; + viz.yAxisName = ABSOLUTE; + viz.columns = List.of(ABSOLUTE); + }) + .el(Bar.class, (viz, data) -> { + viz.title = "Mean score per income group"; + viz.stacked = false; + viz.dataset = data.compute(PtLineAnalysis.class, "pt_persons_mean_score_per_income_group.csv", args); + viz.x = INCOME_GROUP; + viz.xAxisName = INCOME_GROUP; + viz.yAxisName = "mean score"; + viz.columns = List.of("mean_score_base", "mean_score_policy"); + }); + + layout.row("age") + .el(Bar.class, (viz, data) -> { + viz.title = "Agents per age group"; + viz.stacked = false; + viz.dataset = data.compute(PtLineAnalysis.class, "pt_persons_age_groups.csv", args); + viz.x = "ageGroup"; + viz.xAxisName = "age group"; + viz.yAxisName = SHARE; + viz.columns = List.of(SHARE); + }) + .el(Bar.class, (viz, data) -> { + viz.title = "Agents per age group"; + viz.stacked = false; + viz.dataset = data.compute(PtLineAnalysis.class, "pt_persons_age_groups.csv", args); + viz.x = "ageGroup"; + viz.xAxisName = "age group"; + viz.yAxisName = ABSOLUTE; + viz.columns = List.of(ABSOLUTE); + }); + + layout.row("third") + .el(Plotly.class, (viz, data) -> { + viz.title = "Modal split (base case)"; + viz.description = "Shows mode of agents in base case, which used the new pt line in the policy case."; + + viz.layout = tech.tablesaw.plotly.components.Layout.builder() + .barMode(tech.tablesaw.plotly.components.Layout.BarMode.STACK) + .build(); + + Plotly.DataSet ds = viz.addDataset(data.compute(PtLineAnalysis.class, "pt_persons_base_modal_share.csv", args)) + .constant("source", "Base Case Mode") + .aggregate(List.of("main_mode"), SHARE, Plotly.AggrFunc.SUM); + + viz.mergeDatasets = true; + viz.addTrace(BarTrace.builder(Plotly.OBJ_INPUT, Plotly.INPUT).orientation(BarTrace.Orientation.HORIZONTAL).build(), + ds.mapping() + .name("main_mode") + .y("source") + .x(SHARE) + ); + }) + .el(Hexagons.class, (viz, data) -> { + + viz.title = "Pt line agents home locations"; + viz.center = data.context().getCenter(); + viz.zoom = data.context().mapZoomLevel; + viz.height = 7.5; + viz.width = 2.0; + + viz.file = data.compute(PtLineAnalysis.class, "pt_persons_home_locations.csv"); + viz.projection = LausitzScenario.CRS; + viz.addAggregation("home locations", "person", "home_x", "home_y"); + }); + + createTableLayouts(layout); + } + + private static void createTableLayouts(Layout layout) { + layout.row("fourth") + .el(Table.class, (viz, data) -> { + viz.title = "Executed scores"; + viz.description = DESCRIPTION; + viz.dataset = data.compute(PtLineAnalysis.class, "pt_persons_executed_score.csv"); + viz.showAllRows = true; + }) + .el(Table.class, (viz, data) -> { + viz.title = "Travel times"; + viz.description = DESCRIPTION; + viz.dataset = data.compute(PtLineAnalysis.class, "pt_persons_trav_time.csv"); + viz.showAllRows = true; + }) + .el(Table.class, (viz, data) -> { + viz.title = "Travel distances"; + viz.description = DESCRIPTION; + viz.dataset = data.compute(PtLineAnalysis.class, "pt_persons_traveled_distance.csv"); + viz.showAllRows = true; + }); + } +} diff --git a/src/main/java/org/matsim/run/analysis/PtLineAnalysis.java b/src/main/java/org/matsim/run/analysis/PtLineAnalysis.java index 7a1cf21..18d5212 100644 --- a/src/main/java/org/matsim/run/analysis/PtLineAnalysis.java +++ b/src/main/java/org/matsim/run/analysis/PtLineAnalysis.java @@ -1,48 +1,80 @@ package org.matsim.run.analysis; +import it.unimi.dsi.fastutil.ints.IntArrayList; +import it.unimi.dsi.fastutil.ints.IntList; import org.apache.commons.csv.CSVFormat; import org.apache.commons.csv.CSVPrinter; -import org.matsim.api.core.v01.Coord; -import org.matsim.api.core.v01.Id; -import org.matsim.api.core.v01.Scenario; +import org.apache.commons.lang3.Range; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.jetbrains.annotations.NotNull; import org.matsim.api.core.v01.events.PersonEntersVehicleEvent; -import org.matsim.api.core.v01.events.PersonLeavesVehicleEvent; import org.matsim.api.core.v01.events.handler.PersonEntersVehicleEventHandler; -import org.matsim.api.core.v01.events.handler.PersonLeavesVehicleEventHandler; -import org.matsim.api.core.v01.population.Person; +import org.matsim.application.CommandSpec; import org.matsim.application.MATSimAppCommand; -import org.matsim.core.api.experimental.events.AgentWaitingForPtEvent; +import org.matsim.application.options.CsvOptions; +import org.matsim.application.options.InputOptions; +import org.matsim.application.options.OutputOptions; import org.matsim.core.api.experimental.events.EventsManager; -import org.matsim.core.api.experimental.events.handler.AgentWaitingForPtEventHandler; -import org.matsim.core.config.ConfigUtils; import org.matsim.core.events.EventsUtils; import org.matsim.core.events.MatsimEventsReader; -import org.matsim.core.scenario.ScenarioUtils; -import org.matsim.pt.transitSchedule.api.TransitSchedule; -import org.matsim.pt.transitSchedule.api.TransitScheduleReader; +import org.matsim.core.utils.io.IOUtils; import picocli.CommandLine; +import tech.tablesaw.api.*; +import tech.tablesaw.columns.Column; +import tech.tablesaw.io.csv.CsvReadOptions; +import tech.tablesaw.selection.Selection; +import java.io.FileWriter; +import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; +import java.text.DecimalFormat; +import java.text.DecimalFormatSymbols; +import java.time.LocalTime; import java.util.*; -import java.util.stream.Collectors; import static org.matsim.application.ApplicationUtils.globFile; +import static tech.tablesaw.aggregate.AggregateFunctions.*; -@CommandLine.Command( - name = "pt-line", - description = "Get all agents who use the newly created pt line." +@CommandLine.Command(name = "pt-line", description = "Analyze and compare agents who use new pt connection from " + + " policy case and the respective trips in the base case..") +@CommandSpec(requireRunDirectory = true, + produces = {"pt_persons.csv", "pt_persons_home_locations.csv", "pt_persons_income_groups.csv", "pt_persons_age_groups.csv", + "mean_travel_stats.csv", "pt_persons_trav_time.csv", "pt_persons_traveled_distance.csv", "pt_persons_base_modal_share.csv", + "pt_persons_mean_score_per_income_group.csv", "pt_persons_executed_score.csv" + } ) + public class PtLineAnalysis implements MATSimAppCommand { + private static final Logger log = LogManager.getLogger(PtLineAnalysis.class); - @CommandLine.Option(names = "--dir", description = "Run directory with necessary data.", required = true) - private Path dir; + @CommandLine.Mixin + private final InputOptions input = InputOptions.ofCommand(PtLineAnalysis.class); + @CommandLine.Mixin + private OutputOptions output = OutputOptions.ofCommand(PtLineAnalysis.class); + @CommandLine.Option(names = "--income-groups", split = ",", description = "List of income for binning", defaultValue = "0,500,900,1500,2000,3000,4000,5000,6000,7000") + private List incomeGroups; + @CommandLine.Option(names = "--age-groups", split = ",", description = "List of age for binning", defaultValue = "0,18,30,50,70") + private List ageGroups; + @CommandLine.Option(names = "--base-path", description = "Path to run directory of base case.", required = true) + private Path basePath; - @CommandLine.Option(names = "--output", description = "Output path", required = true) - private String outputPath; + private final Map> ptPersons = new HashMap<>(); - private final Set> ptPersons = new HashSet<>(); - private final Map, List> eventMap = new HashMap<>(); + private static final String INCOME_GROUP = "incomeGroup"; + private static final String PERSON = "person"; + private static final String SHARE = "share"; + private static final String AGE_GROUP = "ageGroup"; + private static final String SCORE = "executed_score"; + private static final String INCOME = "income"; + private static final String TRAV_TIME = "trav_time"; + private static final String TRAV_DIST = "traveled_distance"; + private static final String EUCL_DIST = "euclidean_distance"; + private static final String MAIN_MODE = "main_mode"; + private static final String TRIP_ID = "trip_id"; + private static final String BASE_SUFFIX = "_base"; + private static final String COUNT_PERSON = "Count [person]"; public static void main(String[] args) { new PtLineAnalysis().execute(args); @@ -50,72 +82,439 @@ public static void main(String[] args) { @Override public Integer call() throws Exception { - String eventsFile = globFile(dir, "*output_events.xml.gz").toString(); - String transitScheduleFile = globFile(dir, "*output_transitSchedule.xml.gz").toString(); - - Scenario scenario = ScenarioUtils.createScenario(ConfigUtils.createConfig()); - TransitScheduleReader transitScheduleReader = new TransitScheduleReader(scenario); - transitScheduleReader.readFile(transitScheduleFile); + String eventsFile = globFile(input.getRunDirectory(), "*output_events.xml.gz").toString(); EventsManager manager = EventsUtils.createEventsManager(); - manager.addHandler(new NewPtLineEventHandler(scenario.getTransitSchedule())); + manager.addHandler(new NewPtLineEventHandler()); manager.initProcessing(); MatsimEventsReader reader = new MatsimEventsReader(manager); reader.readFile(eventsFile); manager.finishProcessing(); -// only keep agents which used new pt line - Map, List> relevantEvents = eventMap.entrySet().stream() - .filter(entry -> ptPersons.contains(entry.getKey())) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); +// write persons, who use new pt line and their entry time to csv file + writePtPersons(); + +// all necessary file input paths are defined here + String personsPath = globFile(input.getRunDirectory(), "*output_persons.csv.gz").toString(); + String tripsPath = globFile(input.getRunDirectory(), "*output_trips.csv.gz").toString(); + String basePersonsPath = globFile(basePath, "*output_persons.csv.gz").toString(); + String baseTripsPath = globFile(basePath, "*output_trips.csv.gz").toString(); + + Table persons = Table.read().csv(CsvReadOptions.builder(IOUtils.getBufferedReader(personsPath)) + .columnTypesPartial(Map.of(PERSON, ColumnType.TEXT, SCORE, ColumnType.DOUBLE, INCOME, ColumnType.DOUBLE)) + .sample(false) + .separator(CsvOptions.detectDelimiter(personsPath)).build()); + + Map columnTypes = new HashMap<>(Map.of(PERSON, ColumnType.TEXT, + TRAV_TIME, ColumnType.STRING, "dep_time", ColumnType.STRING, MAIN_MODE, ColumnType.STRING, + TRAV_DIST, ColumnType.DOUBLE, EUCL_DIST, ColumnType.DOUBLE, TRIP_ID, ColumnType.STRING)); + +// filter for persons, which used the new pt line in pt policy case + TextColumn personColumn = persons.textColumn(PERSON); + persons = persons.where(personColumn.isIn(ptPersons.keySet())); + + // read base persons and filter them + Table basePersons = Table.read().csv(CsvReadOptions.builder(IOUtils.getBufferedReader(basePersonsPath)) + .columnTypesPartial(Map.of(PERSON, ColumnType.TEXT, SCORE, ColumnType.DOUBLE, INCOME, ColumnType.DOUBLE)) + .sample(false) + .separator(CsvOptions.detectDelimiter(basePersonsPath)).build()); + + TextColumn basePersonColumn = basePersons.textColumn(PERSON); + basePersons = basePersons.where(basePersonColumn.isIn(ptPersons.keySet())); + + writeComparisonTable(persons, basePersons, SCORE, PERSON); + +// print csv file with home coords of new pt line agents + writeHomeLocations(persons); + + Map> incomeLabels = getLabels(incomeGroups); + incomeLabels.put(incomeGroups.getLast() + "+", Range.of(incomeGroups.getLast(), 9999999)); + incomeGroups.add(Integer.MAX_VALUE); + + +// add income group column to persons table for further analysis + persons = addIncomeGroupColumnToTable(persons, incomeLabels); + +// write income distr of new pt line agents + writeIncomeDistr(persons, incomeLabels); + +// write age distr of new pt line agents + writeAgeDistr(persons); + + for (int i = 0; i < basePersons.columnCount(); i++) { + Column column = basePersons.column(i); + if (!column.name().equals(PERSON)) { + column.setName(column.name() + BASE_SUFFIX); + } + } + Table basePersonsIncomeGroup = basePersons.joinOn(PERSON).inner(persons).retainColumns(PERSON, INCOME_GROUP, SCORE + BASE_SUFFIX); + +// calc mean score for every income group in base and policy and save to table + Table scoresPerIncomeGroup = persons.summarize(SCORE, mean).by(INCOME_GROUP) + .joinOn(INCOME_GROUP).inner(basePersonsIncomeGroup.summarize(SCORE + BASE_SUFFIX, mean).by(INCOME_GROUP)); + +// write scores per income group + writeScorePerIncomeGroupDistr(scoresPerIncomeGroup); + + Table trips = Table.read().csv(CsvReadOptions.builder(IOUtils.getBufferedReader(tripsPath)) + .columnTypesPartial(columnTypes) + .sample(false) + .separator(CsvOptions.detectDelimiter(tripsPath)).build()); + + Table baseTrips = Table.read().csv(CsvReadOptions.builder(IOUtils.getBufferedReader(baseTripsPath)) + .columnTypesPartial(columnTypes) + .sample(false) + .separator(CsvOptions.detectDelimiter(baseTripsPath)).build()); + +// filter for trips with new pt line only + TextColumn personTripsColumn = trips.textColumn(PERSON); + trips = trips.where(personTripsColumn.isIn(ptPersons.keySet())); + + IntList idx = new IntArrayList(); + + for (int i = 0; i < trips.rowCount(); i++) { + Row row = trips.row(i); - try (CSVPrinter printer = new CSVPrinter(Files.newBufferedWriter(Path.of(outputPath)), CSVFormat.DEFAULT)) { - printer.printRecord("person", "eventType", "time", "x", "y"); - for (Map.Entry, List> e : relevantEvents.entrySet()) { - for (EventData eventData : e.getValue()) { - printer.printRecord(e.getKey().toString(), eventData.eventType, eventData.time, eventData.coord.getX(), eventData.coord.getY()); + Double tripStart = parseTimeManually(row.getString("dep_time")); +// waiting time already included in travel time + Double travelTime = parseTimeManually(row.getString(TRAV_TIME)); + + List enterTimes = ptPersons.get(row.getString(PERSON)); + + for (Double enterTime : enterTimes) { + if (Range.of(tripStart, tripStart + travelTime).contains(enterTime)) { + idx.add(i); } } } + trips = trips.where(Selection.with(idx.toIntArray())); + +// filter trips of base case for comparison + StringColumn tripIdColumn = trips.stringColumn(TRIP_ID); + StringColumn baseTripIdColumn = baseTrips.stringColumn(TRIP_ID); + + baseTrips = baseTrips.where(baseTripIdColumn.isIn(tripIdColumn)); + +// the number of trips in both filtered tables should be the same + if (baseTrips.rowCount() != trips.rowCount()) { + log.fatal("Number of trips in filtered base case trips table ({}) and pt policy case trips table ({}) is not equal!" + + " Analysis cannot be continued.", baseTrips.rowCount(), trips.rowCount()); + return 2; + } + +// calc and write mean stats for policy and base case + calcAndWriteMeanStats(trips, persons, baseTrips, basePersons); + +// write tables for comparison of travel time and distance + writeComparisonTable(trips, baseTrips, TRAV_TIME, TRIP_ID); + writeComparisonTable(trips, baseTrips, TRAV_DIST, TRIP_ID); + +// write mode shares to csv + writeBaseModeShares(baseTrips); return 0; } + private void calcAndWriteMeanStats(Table trips, Table persons, Table baseTrips, Table basePersons) throws IOException { + double meanTravelTimePolicy = calcMean(trips.column(TRAV_TIME)); + double meanTravelDistancePolicy = calcMean(trips.column(TRAV_DIST)); + double meanEuclideanDistancePolicy = calcMean(trips.column(EUCL_DIST)); + double meanScorePolicy = calcMean(persons.column(SCORE)); + double meanTravelTimeBase = calcMean(baseTrips.column(TRAV_TIME)); + double meanTravelDistanceBase = calcMean(baseTrips.column(TRAV_DIST)); + double meanEuclideanDistanceBase = calcMean(baseTrips.column(EUCL_DIST)); + double meanScoreBase = calcMean(basePersons.column(SCORE + BASE_SUFFIX)); - private final class NewPtLineEventHandler implements PersonEntersVehicleEventHandler, PersonLeavesVehicleEventHandler, AgentWaitingForPtEventHandler { - TransitSchedule schedule; - NewPtLineEventHandler(TransitSchedule schedule) { - this.schedule = schedule; + if (meanTravelTimePolicy <= 0 || meanTravelTimeBase <= 0) { + log.fatal("Mean travel time for either base ({}) or policy case ({}) are zero. Mean travel velocity cannot" + + "be calculated! Divison by 0 not possible!", meanTravelTimeBase, meanTravelTimePolicy); + throw new IllegalArgumentException(); } - @Override - public void handleEvent(PersonEntersVehicleEvent event) { - if (event.getVehicleId().toString().contains("RE-VSP1") && !event.getPersonId().toString().contains("pt_")) { - eventMap.get(event.getPersonId()).add(new EventData(event.getEventType(), event.getTime(), new Coord(0, 0))); - ptPersons.add(event.getPersonId()); + + double meanVelocityPolicy = meanTravelDistancePolicy / meanTravelTimePolicy; + double meanVelocityBase = meanTravelDistanceBase / meanTravelTimeBase; + +// write mean stats to csv + DecimalFormat f = new DecimalFormat("0.00", new DecimalFormatSymbols(Locale.ENGLISH)); + + try (CSVPrinter printer = new CSVPrinter(new FileWriter(output.getPath("mean_travel_stats.csv").toString()), getCsvFormat())) { + printer.printRecord("\"mean travel time policy case\"", f.format(meanTravelTimePolicy)); + printer.printRecord("\"mean travel time base case\"", f.format(meanTravelTimeBase)); + printer.printRecord("\"mean travel distance policy case\"", f.format(meanTravelDistancePolicy)); + printer.printRecord("\"mean travel distance base case\"", f.format(meanTravelDistanceBase)); + printer.printRecord("\"mean trip velocity policy case\"", f.format(meanVelocityPolicy)); + printer.printRecord("\"mean trip velocity base case\"", f.format(meanVelocityBase)); + printer.printRecord("\"mean euclidean distance policy case\"", f.format(meanEuclideanDistancePolicy)); + printer.printRecord("\"mean euclidean distance base case\"", f.format(meanEuclideanDistanceBase)); + printer.printRecord("\"mean score policy case\"", f.format(meanScorePolicy)); + printer.printRecord("\"mean score base case\"", f.format(meanScoreBase)); + } + } + + private void writeBaseModeShares(Table baseTrips) { + // calc shares for new pt line trips in base case + StringColumn mainModeColumn = baseTrips.stringColumn(MAIN_MODE); + + Table counts = baseTrips.countBy(mainModeColumn); + + counts.addColumns( + counts.intColumn("Count") + .divide(mainModeColumn.size()) + .setName(SHARE) + ); + + try (CSVPrinter printer = new CSVPrinter(new FileWriter(output.getPath("pt_persons_base_modal_share.csv").toString()), getCsvFormat())) { + printer.printRecord(MAIN_MODE, SHARE); + for (int i = 0; i < counts.rowCount(); i++) { + Row row = counts.row(i); + printer.printRecord(row.getString(MAIN_MODE), row.getDouble(SHARE)); } + } catch (IOException e) { + throw new IllegalArgumentException(); } + } - @Override - public void handleEvent(AgentWaitingForPtEvent event) { - if (!event.getPersonId().toString().contains("pt_")) { - if (!eventMap.containsKey(event.getPersonId())) { - eventMap.put(event.getPersonId(), new ArrayList<>()); + private void writePtPersons() throws IOException { + try (CSVPrinter printer = new CSVPrinter(Files.newBufferedWriter(output.getPath("pt_persons.csv")), getCsvFormat())) { + printer.printRecord(PERSON, "time"); + for (Map.Entry> e : ptPersons.entrySet()) { + for (Double time : e.getValue()) { + printer.printRecord(e.getKey(), time); } - eventMap.get(event.getPersonId()) - .add(new EventData(event.getEventType(), event.getTime(), new Coord( - schedule.getFacilities().get(event.waitingAtStopId).getCoord().getX(), - schedule.getFacilities().get(event.waitingAtStopId).getCoord().getY()))); + } + } + } + + private void writeScorePerIncomeGroupDistr(Table scoresPerIncomeGroup) { + + try (CSVPrinter printer = new CSVPrinter(new FileWriter(output.getPath("pt_persons_mean_score_per_income_group.csv").toString()), getCsvFormat())) { + printer.printRecord(INCOME_GROUP, "mean_score_base", "mean_score_policy"); + + for (int i = 0; i < scoresPerIncomeGroup.rowCount(); i++) { + Row row = scoresPerIncomeGroup.row(i); + + printer.printRecord(row.getString(INCOME_GROUP), row.getDouble(2), row.getDouble(1)); } + } catch (IOException e) { + throw new IllegalArgumentException(); } + } - @Override - public void handleEvent(PersonLeavesVehicleEvent event) { - if (ptPersons.contains(event.getPersonId())) { - eventMap.get(event.getPersonId()).add(new EventData(event.getEventType(), event.getTime(), new Coord(0, 0))); + private void writeComparisonTable(Table policy, Table base, String paramName, String id) { + try (CSVPrinter printer = new CSVPrinter(new FileWriter(output.getPath("pt_persons_" + paramName + ".csv").toString()), getCsvFormat())) { + printer.printRecord(id, paramName + "_policy", paramName + BASE_SUFFIX); + for (int i = 0; i < policy.rowCount(); i++) { + Row row = policy.row(i); + Row baseRow = base.row(i); + + String policyValue = null; + String baseValue = null; + + if (policy.column(paramName) instanceof StringColumn) { + policyValue = row.getString(paramName); + baseValue = baseRow.getString(paramName); + } else if (policy.column(paramName) instanceof DoubleColumn) { + policyValue = String.valueOf(row.getDouble(paramName)); + baseValue = String.valueOf(baseRow.getDouble(paramName)); + } + printer.printRecord(row.getText(id), policyValue, baseValue); + } + } catch (IOException e) { + throw new IllegalArgumentException(); + } + } + + private void writeHomeLocations(Table persons) throws IOException { + // y think about adding first act coords here or even act before / after pt trip + try (CSVPrinter printer = new CSVPrinter(Files.newBufferedWriter(output.getPath("pt_persons_home_locations.csv")), getCsvFormat())) { + printer.printRecord(PERSON, "home_x", "home_y"); + + for (int i = 0; i < persons.rowCount(); i++) { + Row row = persons.row(i); + printer.printRecord(row.getText(PERSON), row.getDouble("home_x"), row.getDouble("home_y")); } } } - private record EventData(String eventType, double time, Coord coord) {} + private void writeIncomeDistr(Table persons, Map> labels) { + List incomeDistr = getDistr(persons, INCOME_GROUP, labels); + +// print income distr + try (CSVPrinter printer = new CSVPrinter(new FileWriter(output.getPath("pt_persons_income_groups.csv").toString()), getCsvFormat())) { + printer.printRecord(INCOME_GROUP, COUNT_PERSON, SHARE); + for (String s : incomeDistr) { + printer.printRecord(s); + } + } catch (IOException e) { + throw new IllegalArgumentException(); + } + } + + private void writeAgeDistr(Table persons) { + Map> labels = getLabels(ageGroups); + labels.put(ageGroups.getLast() + "+", Range.of(ageGroups.getLast(), 120)); + ageGroups.add(Integer.MAX_VALUE); + + persons.addColumns(StringColumn.create(AGE_GROUP)); + + for (int i = 0; i < persons.rowCount(); i++) { + Row row = persons.row(i); + + int age = row.getInt("age"); + String p = row.getText(PERSON); + + if (age < 0) { + log.error("age {} of person {} is negative. This should not happen!", age, p); + throw new IllegalArgumentException(); + } + + for (Map.Entry> e : labels.entrySet()) { + Range range = e.getValue(); + if (range.contains(age)) { + row.setString(AGE_GROUP, e.getKey()); + break; + } + } + } + + List ageDistr = getDistr(persons, AGE_GROUP, labels); + +// print age distr + try (CSVPrinter printer = new CSVPrinter(new FileWriter(output.getPath("pt_persons_age_groups.csv").toString()), getCsvFormat())) { + printer.printRecord(AGE_GROUP, COUNT_PERSON, SHARE); + for (String s : ageDistr) { + printer.printRecord(s); + } + } catch (IOException e) { + throw new IllegalArgumentException(); + } + } + + private Double calcMean(Column column) { + double total = 0; + + for (int i = 0; i < column.size(); i++) { + double value = 0; + if (column instanceof StringColumn stringColumn) { +// travel time is saved in hh:mm:ss format, thus read as string + value = LocalTime.parse(stringColumn.get(i)).toSecondOfDay(); + } else if (column instanceof DoubleColumn doubleColumn) { +// distances / scores are saved as doubles + value = doubleColumn.get(i); + } + total += value; + } + return total / column.size(); + } + + private Table addIncomeGroupColumnToTable(Table persons, Map> incomeLabels) { + persons.addColumns(StringColumn.create(INCOME_GROUP)); + + for (int i = 0; i < persons.rowCount(); i++) { + Row row = persons.row(i); + + int income = (int) Math.round(row.getDouble(INCOME)); + String p = row.getText(PERSON); + + if (income < 0) { + log.error("income {} of person {} is negative. This should not happen!", income, p); + throw new IllegalArgumentException(); + } + + for (Map.Entry> e : incomeLabels.entrySet()) { + Range range = e.getValue(); + if (range.contains(income)) { + row.setString(INCOME_GROUP, e.getKey()); + break; + } + } + } + return persons; + } + + private Map> getLabels(List groups) { + Map> labels = new HashMap<>(); + for (int i = 0; i < groups.size() - 1; i++) { + labels.put(String.format("%d - %d", groups.get(i), groups.get(i + 1) - 1), + Range.of(groups.get(i), groups.get(i + 1) - 1)); + } + return labels; + } + + private @NotNull List getDistr(Table persons, String group, Map> labels) { + Table aggr = persons.summarize(PERSON, count).by(group); + +// how to sort rows here? agg.sortOn does not work! Using workaround instead. -sme0324 + DoubleColumn shareCol = aggr.numberColumn(1).divide(aggr.numberColumn(1).sum()).setName(SHARE); + aggr.addColumns(shareCol); + + List distr = new ArrayList<>(); + + for (String k : labels.keySet()) { + boolean labelFound = false; + for (int i = 0; i < aggr.rowCount(); i++) { + Row row = aggr.row(i); + if (row.getString(group).equals(k)) { + distr.add(k + "," + row.getDouble(COUNT_PERSON) + "," + row.getDouble(SHARE)); + labelFound = true; + break; + } + } + if (!labelFound) { + distr.add(k + "," + 0 + "," + 0); + } + } + + distr.sort(Comparator.comparingInt(PtLineAnalysis::getLowerBound)); + return distr; + } + + private static CSVFormat getCsvFormat() { + return CSVFormat.DEFAULT.builder() + .setQuote(null) + .setDelimiter(',') + .setRecordSeparator("\r\n") + .build(); + } + + private static int getLowerBound(String s) { + String regex = " - "; + if (s.contains("+")) { + regex = "\\+"; + } + return Integer.parseInt(s.split(regex)[0]); + } + + private double parseTimeManually(String time) { + String[] parts = time.split(":"); + if (parts.length != 3) { + throw new IllegalArgumentException("Invalid time format: " + time); + } + + double hours = Double.parseDouble(parts[0]); + double minutes = Double.parseDouble(parts[1]); + double seconds = Double.parseDouble(parts[2]); + + // Validate minutes and seconds + if (minutes < 0 || minutes > 59 || seconds < 0 || seconds > 59) { + throw new IllegalArgumentException("Invalid minutes or seconds in: " + time); + } + + return hours * 3600 + minutes * 60 + seconds; + } + + + private final class NewPtLineEventHandler implements PersonEntersVehicleEventHandler { + + @Override + public void handleEvent(PersonEntersVehicleEvent event) { + if (event.getVehicleId().toString().contains("RE-VSP1") && !event.getPersonId().toString().contains("pt_")) { + if (!ptPersons.containsKey(event.getPersonId().toString())) { + ptPersons.put(event.getPersonId().toString(), new ArrayList<>()); + } + ptPersons.get(event.getPersonId().toString()).add(event.getTime()); + } + } + } } diff --git a/src/main/java/org/matsim/run/scenarios/LausitzScenario.java b/src/main/java/org/matsim/run/scenarios/LausitzScenario.java index 69a1853..41f9022 100644 --- a/src/main/java/org/matsim/run/scenarios/LausitzScenario.java +++ b/src/main/java/org/matsim/run/scenarios/LausitzScenario.java @@ -70,6 +70,7 @@ public class LausitzScenario extends MATSimApplication { public static final String HEAVY_MODE = "truck40t"; public static final String MEDIUM_MODE = "truck18t"; public static final String LIGHT_MODE = "truck8t"; + public static final String CRS = "EPSG:25832"; // To decrypt hbefa input files set MATSIM_DECRYPTION_PASSWORD as environment variable. ask VSP for access. private static final String HBEFA_2020_PATH = "https://svn.vsp.tu-berlin.de/repos/public-svn/3507bb3997e5657ab9da76dbedbb13c9b5991d3e/0e73947443d68f95202b71a156b337f7f71604ae/"; diff --git a/src/test/java/org/matsim/run/RunIntegrationTest.java b/src/test/java/org/matsim/run/RunIntegrationTest.java index cf7cd3b..64515d9 100644 --- a/src/test/java/org/matsim/run/RunIntegrationTest.java +++ b/src/test/java/org/matsim/run/RunIntegrationTest.java @@ -222,6 +222,8 @@ private void createSinglePersonTestPopulation(Config config, String mode) { Population population = PopulationUtils.createPopulation(config); PopulationFactory fac = population.getFactory(); Person person = fac.createPerson(personId); + person.getAttributes().putAttribute("home_x", 863538.13); + person.getAttributes().putAttribute("home_y", 5711028.24); Plan plan = PopulationUtils.createPlan(person); // home in hoyerswerda @@ -243,6 +245,7 @@ private void createSinglePersonTestPopulation(Config config, String mode) { person.addPlan(plan); PersonUtils.setIncome(person, 1000.); + PersonUtils.setAge(person, 30); person.getAttributes().putAttribute("subpopulation", "person"); population.addPerson(person);