Additional transit schedule validations (#3524)

* add transit schedule validation for stops and travel times, update version of gtfs converter

* fix duplicated argument

* update gtfs converter

* update gtfs version

* improved validator, added class to fix departure time issues

* fix departures that are the same at the end of a route

* check for routes with empty departures

contribs/application/pom.xml

@@ -87,7 +87,7 @@
 		<dependency>
 			<groupId>com.github.matsim-org</groupId>
 			<artifactId>gtfs2matsim</artifactId>
-			<version>fc8b13954d</version>
+			<version>0bd5850fd6</version>
 			<exclusions>
 				<!-- Exclude unneeded dependencies and these with known CVE -->
 				<exclusion>

contribs/application/src/main/java/org/matsim/application/prepare/pt/AdjustSameDepartureTimes.java

@@ -0,0 +1,154 @@
+package org.matsim.application.prepare.pt;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.matsim.core.utils.geometry.CoordUtils;
+import org.matsim.core.utils.misc.OptionalTime;
+import org.matsim.pt.transitSchedule.api.*;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Consumer;
+
+/**
+ * Some providers set the same departure and arrival times for multiple stops.
+ * This usually happens with on-demand buses, but can also happen on night buses where multiple stops can have a departure at the same minute.
+ * These departure times will lead to artifacts when using a pseudo network.
+ * The purpose of this class is to spread these departures more evently.
+ * However, in the case of on demand buses, the travel times will likely still be too optimistic.
+ */
+public class AdjustSameDepartureTimes implements Consumer<TransitSchedule> {
+
+	private static Logger log = LogManager.getLogger(AdjustSameDepartureTimes.class);
+
+	private static OptionalTime add(OptionalTime x, double t) {
+		if (!x.isDefined()) {
+			return x;
+		}
+		return OptionalTime.defined(x.seconds() + t);
+	}
+
+	@Override
+	public void accept(TransitSchedule schedule) {
+		for (TransitLine line : schedule.getTransitLines().values()) {
+
+			List<TransitRoute> routes = new ArrayList<>(line.getRoutes().values());
+			for (TransitRoute route : routes) {
+				List<TransitRouteStop> newStops = adjustDepartures(schedule.getFactory(), route);
+
+				if (newStops == null) {
+					continue;
+				}
+
+				log.info("Adjusted departures for route {} in line {}", route.getId(), line.getId());
+
+				line.removeRoute(route);
+
+				TransitRoute newRoute = schedule.getFactory().createTransitRoute(route.getId(), route.getRoute(), newStops, route.getTransportMode());
+				newRoute.setDescription(route.getDescription());
+				for (Map.Entry<String, Object> e : route.getAttributes().getAsMap().entrySet()) {
+					newRoute.getAttributes().putAttribute(e.getKey(), e.getValue());
+				}
+				route.getDepartures().values().forEach(newRoute::addDeparture);
+
+				line.addRoute(newRoute);
+			}
+		}
+
+	}
+
+	private List<TransitRouteStop> adjustDepartures(TransitScheduleFactory f, TransitRoute route) {
+
+		List<TransitRouteStop> stops = new ArrayList<>(route.getStops());
+
+		boolean adjusted = false;
+
+		// Check if the times at the end of the schedule are the same
+		// These need to be calculated and can not be interpolated
+		// The arrival at the last stop is shifted by small travel time
+		if (stops.size() > 1) {
+			TransitRouteStop last = stops.getLast();
+			TransitRouteStop secondLast = stops.get(stops.size() - 2);
+
+			OptionalTime lastDep = last.getDepartureOffset().or(last.getArrivalOffset());
+			OptionalTime secondLastDep = secondLast.getDepartureOffset().or(secondLast.getArrivalOffset());
+
+			if (lastDep.isDefined() && secondLastDep.isDefined() && lastDep.equals(secondLastDep)) {
+				// Calculate the time between the last two stops
+				double dist = Math.max(20, CoordUtils.calcEuclideanDistance(last.getStopFacility().getCoord(), secondLast.getStopFacility().getCoord()));
+				double time = dist / 10; // 10 m/s
+
+				// Calculate the time for the last stop
+				TransitRouteStop newStop = f.createTransitRouteStop(
+					last.getStopFacility(),
+					add(last.getArrivalOffset(), time),
+					add(last.getDepartureOffset(), time)
+				);
+
+				newStop.setAwaitDepartureTime(last.isAwaitDepartureTime());
+				newStop.setAllowAlighting(last.isAllowAlighting());
+				newStop.setAllowBoarding(last.isAllowBoarding());
+
+				stops.set(stops.size() - 1, newStop);
+				adjusted = true;
+			}
+		}
+
+		for (int i = 0; i < stops.size() - 1; ) {
+
+			TransitRouteStop stop = stops.get(i);
+			OptionalTime dep = stop.getDepartureOffset().or(stop.getArrivalOffset());
+
+			if (!dep.isDefined()) {
+				i++;
+				continue;
+			}
+
+			OptionalTime arr = null;
+			int j = i + 1;
+			for (; j < stops.size(); j++) {
+				TransitRouteStop nextStop = stops.get(j);
+				arr = nextStop.getArrivalOffset().or(nextStop.getDepartureOffset());
+				if (!dep.equals(arr)) {
+					break;
+				}
+			}
+
+			if (arr == null) {
+				i++;
+				continue;
+			}
+
+			if (j > i + 1) {
+				double time = dep.seconds();
+				double diff = (arr.seconds() - time) / (j - i);
+
+				for (int k = i + 1; k < j; k++) {
+					TransitRouteStop stopToAdjust = stops.get(k);
+					int add = (int) (diff * (k - i));
+
+					TransitRouteStop newStop = f.createTransitRouteStop(
+						stopToAdjust.getStopFacility(),
+						add(stopToAdjust.getArrivalOffset(), add),
+						add(stopToAdjust.getDepartureOffset(), add)
+					);
+
+					newStop.setAwaitDepartureTime(stopToAdjust.isAwaitDepartureTime());
+					newStop.setAllowAlighting(stopToAdjust.isAllowAlighting());
+					newStop.setAllowBoarding(stopToAdjust.isAllowBoarding());
+
+					stops.set(k, newStop);
+					adjusted = true;
+				}
+			}
+
+			i = j;
+		}
+
+		if (adjusted)
+			return stops;
+
+		return null;
+	}
+}

contribs/application/src/main/java/org/matsim/application/prepare/pt/CreateTransitScheduleFromGtfs.java

@@ -1,5 +1,6 @@
 package org.matsim.application.prepare.pt;
 
+import com.conveyal.gtfs.model.Route;
 import com.conveyal.gtfs.model.Stop;
 import org.apache.commons.io.FilenameUtils;
 import org.apache.logging.log4j.LogManager;
@@ -31,10 +32,8 @@
 import java.io.File;
 import java.nio.file.Path;
 import java.time.LocalDate;
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
+import java.util.*;
+import java.util.function.Consumer;
 import java.util.function.Predicate;
 
 
@@ -77,6 +76,15 @@ public class CreateTransitScheduleFromGtfs implements MATSimAppCommand {
 	@CommandLine.Option(names = "--include-stops", description = "Fully qualified class name to a Predicate<Stop> for filtering certain stops")
 	private Class<?> includeStops;
 
+	@CommandLine.Option(names = "--transform-stops", description = "Fully qualified class name to a Consumer<Stop> for transforming stops before usage")
+	private Class<?> transformStops;
+
+	@CommandLine.Option(names = "--transform-routes", description = "Fully qualified class name to a Consumer<Route> for transforming routes before usage")
+	private Class<?> transformRoutes;
+
+	@CommandLine.Option(names = "--transform-schedule", description = "Fully qualified class name to a Consumer<TransitSchedule> to be executed after the schedule was created", arity = "0..*", split = ",")
+	private List<Class<?>> transformSchedule;
+
 	@CommandLine.Option(names = "--merge-stops", description = "Whether stops should be merged by coordinate")
 	private boolean mergeStops;
 
@@ -134,6 +142,15 @@ public Integer call() throws Exception {
 				log.info("Using prefix: {}", prefix);
 			}
 
+			if (transformStops != null) {
+				converter.setTransformStop(createConsumer(transformStops, Stop.class));
+			}
+
+			if (transformRoutes != null) {
+				converter.setTransformRoute(createConsumer(transformRoutes, Route.class));
+			}
+
+
 			converter.build().convert();
 			i++;
 		}
@@ -144,18 +161,39 @@ public Integer call() throws Exception {
 			TransitSchedulePostProcessTools.copyEarlyDeparturesToFollowingNight(scenario.getTransitSchedule(), 6 * 3600, "copied");
 		}
 
+		if (transformSchedule != null && !transformSchedule.isEmpty()) {
+			for (Class<?> c : transformSchedule) {
+				Consumer<TransitSchedule> f = createConsumer(c, TransitSchedule.class);
+				log.info("Applying {} to created schedule", c.getName());
+				f.accept(scenario.getTransitSchedule());
+			}
+		}
+
+		for (TransitLine line : scenario.getTransitSchedule().getTransitLines().values()) {
+			List<TransitRoute> routes = new ArrayList<>(line.getRoutes().values());
+			for (TransitRoute route : routes) {
+				if (route.getDepartures().isEmpty()) {
+					log.warn("Route {} in line {} with no departures removed.", route.getId(), line.getId());
+					line.removeRoute(route);
+				}
+			}
+		}
+
 		Network network = NetworkUtils.readNetwork(networkFile);
 
 		Scenario ptScenario = getScenarioWithPseudoPtNetworkAndTransitVehicles(network, scenario.getTransitSchedule(), "pt_");
 
 		if (validate) {
 			//Check schedule and network
 			TransitScheduleValidator.ValidationResult checkResult = TransitScheduleValidator.validateAll(ptScenario.getTransitSchedule(), ptScenario.getNetwork());
+			List<String> warnings = checkResult.getWarnings();
+			if (!warnings.isEmpty())
+				log.warn("TransitScheduleValidator warnings: {}", String.join("\n", warnings));
+
 			if (checkResult.isValid()) {
 				log.info("TransitSchedule and Network valid according to TransitScheduleValidator");
-				log.warn("TransitScheduleValidator warnings: {}", checkResult.getWarnings());
 			} else {
-				log.error(checkResult.getErrors());
+				log.error("TransitScheduleValidator errors: {}", String.join("\n", checkResult.getErrors()));
 				throw new RuntimeException("TransitSchedule and/or Network invalid");
 			}
 		}
@@ -170,6 +208,12 @@ public Integer call() throws Exception {
 		return 0;
 	}
 
+	@SuppressWarnings({"unchecked", "unused"})
+	private <T> Consumer<T> createConsumer(Class<?> consumer, Class<T> type) throws ReflectiveOperationException {
+		return (Consumer<T>) consumer.getDeclaredConstructor().newInstance();
+	}
+
+	@SuppressWarnings("unchecked")
 	private Predicate<Stop> createFilter(int i) throws Exception {
 
 		Predicate<Stop> filter = (stop) -> true;