Merge branch 'master' into imc-pruning

.github/workflows/code-coverage.yaml

@@ -26,7 +26,7 @@ jobs:
         run: mvn verify -P jacoco --batch-mode --also-make --projects matsim -Dmaven.test.redirectTestOutputToFile -Dmatsim.preferLocalDtds=true
 
       - name: Push coverage to CodeCov
-        uses: codecov/codecov-action@v3
+        uses: codecov/codecov-action@v4
         with:
           files: ./matsim/target/site/jacoco/jacoco.xml
 

.github/workflows/deploy-on-pr-merge.yaml

@@ -46,7 +46,7 @@ jobs:
       - name: Submit Dependency Graph
         # Generate a complete dependency graph and submit the graph to the GitHub repository.
         # The goal is to improve security alerts from dependabot, because dependabot is not able to compute the complete dependency graph.
-        uses: advanced-security/maven-dependency-submission-action@v3
+        uses: advanced-security/maven-dependency-submission-action@v4
 
     env:
       MAVEN_OPTS: -Xmx2g

.github/workflows/verify-push.yaml

@@ -75,7 +75,7 @@ jobs:
       - name: Detect changes against master
         # we only want to build matsim (module) if changes are not limited to contribs
         id: detect-changes
-        uses: dorny/paths-filter@v2
+        uses: dorny/paths-filter@v3
         if: ${{matrix.module == 'matsim'}}
         with:
           filters: |

contribs/analysis/pom.xml

@@ -32,7 +32,7 @@
         <dependency>
             <groupId>org.apache.httpcomponents.client5</groupId>
             <artifactId>httpclient5</artifactId>
-            <version>5.3</version>
+            <version>5.3.1</version>
         </dependency>
     </dependencies>
 </project>

contribs/application/src/main/java/org/matsim/application/ApplicationUtils.java

@@ -31,10 +31,25 @@ public class ApplicationUtils {
 	private ApplicationUtils() {
 	}
 
+
+	/**
+	 * Merge given arguments with custom ones.
+	 *
+	 * @param args        given args, usually input from command line / main method
+	 * @param defaultArgs default arguments that will be added to existing ones.
+	 */
+	public static String[] mergeArgs(String[] args, String... defaultArgs) {
+		String[] mergedArgs = new String[args.length + defaultArgs.length];
+		System.arraycopy(args, 0, mergedArgs, 0, args.length);
+		System.arraycopy(defaultArgs, 0, mergedArgs, args.length, defaultArgs.length);
+		return mergedArgs;
+	}
+
 	/**
 	 * Extends a context (usually config location) with an relative filename.
 	 * If the results is a local file, the path will be returned. Otherwise, it will be an url.
 	 * The results can be used as input for command line parameter or {@link IOUtils#resolveFileOrResource(String)}.
+	 *
 	 * @return string with path or URL
 	 */
 	public static String resolve(URL context, String filename) {
@@ -61,9 +76,9 @@ public static Path globFile(Path path, String pattern) {
 
 		try {
 			return Files.list(path)
-					.filter(p -> m.matches(p.getFileName()))
-					.findFirst()
-					.orElseThrow(() -> new IllegalStateException("No " + pattern + " file found."));
+				.filter(p -> m.matches(p.getFileName()))
+				.findFirst()
+				.orElseThrow(() -> new IllegalStateException("No " + pattern + " file found."));
 		} catch (IOException e) {
 			throw new RuntimeException(e);
 		}
@@ -154,11 +169,19 @@ public static void checkCommand(Class<? extends MATSimAppCommand> command) {
 		boolean input = false;
 		boolean output = false;
 		for (Field field : fields) {
-			if (field.getType().equals(InputOptions.class))
+			if (field.getType().equals(InputOptions.class)) {
 				input = true;
+				CommandLine.Mixin mixin = field.getAnnotation(CommandLine.Mixin.class);
+				if (mixin == null)
+					throw new IllegalArgumentException(String.format("The command %s has no @Mixin annotation for InputOptions %s.", command, field.getName()));
+			}
 
-			if (field.getType().equals(OutputOptions.class))
+			if (field.getType().equals(OutputOptions.class)) {
 				output = true;
+				CommandLine.Mixin mixin = field.getAnnotation(CommandLine.Mixin.class);
+				if (mixin == null)
+					throw new IllegalArgumentException(String.format("The command %s has no @Mixin annotation for OutputOptions %s.", command, field.getName()));
+			}
 		}
 
 		if (!input) {
@@ -213,7 +236,7 @@ public static Path matchInput(String name, Path dir) {
 			return path.get();
 
 		// Match more general pattern at last
-		path = matchPattern( ".+\\.[a-zA-Z0-9]*_" + name + "\\..+", dir);
+		path = matchPattern(".+\\.[a-zA-Z0-9]*_" + name + "\\..+", dir);
 		if (path.isPresent())
 			return path.get();
 

contribs/application/src/main/java/org/matsim/application/CommandRunner.java

@@ -247,6 +247,22 @@ public Path getRequiredPath(Class<? extends MATSimAppCommand> command, String fi
 		return buildPath(spec, command).resolve(file);
 	}
 
+	/**
+	 * Return the output of a command with a placeholder.
+	 * @param file file name, which must contain a %s, which will be replaced by the placeholder
+	 */
+	public Path getRequiredPath(Class<? extends MATSimAppCommand> command, String file, String placeholder) {
+		CommandSpec spec = ApplicationUtils.getSpec(command);
+		if (!ArrayUtils.contains(spec.produces(), file))
+			throw new IllegalArgumentException(String.format("Command %s does not declare output %s", command, file));
+		if (!file.contains("%s"))
+			throw new IllegalArgumentException(String.format("File %s does not contain placeholder %%s", file));
+
+		file = String.format(file, placeholder);
+
+		return buildPath(spec, command).resolve(file);
+	}
+
 	/**
 	 * Base path for the runner.
 	 */
@@ -279,8 +295,8 @@ public void add(Class<? extends MATSimAppCommand> command, String... args) {
 		if (args.length != 0) {
 			String[] existing = this.args.get(command);
 			if (existing != null && existing.length > 0 && !Arrays.equals(existing, args)) {
-				throw new IllegalArgumentException(String.format("Command %s already registered with args %s, can not define different args as %s",
-					command.toString(), Arrays.toString(existing), Arrays.toString(args)));
+				throw new IllegalArgumentException(String.format("Command %s already registered with args %s, can not define different args as %s (name '%s').",
+					command.toString(), Arrays.toString(existing), Arrays.toString(args), name));
 			}
 		}
 
@@ -298,6 +314,22 @@ public void add(Class<? extends MATSimAppCommand> command, String... args) {
 		}
 	}
 
+	/**
+	 * Insert args for an already existing command. If the command was not added, this does nothing.
+	 */
+	public void insertArgs(Class<? extends MATSimAppCommand> command, String... args) {
+
+		if (!this.args.containsKey(command))
+			return;
+
+		String[] existing = this.args.get(command);
+		String[] newArgs = new String[existing.length + args.length];
+		System.arraycopy(args, 0, newArgs, 0, args.length);
+		System.arraycopy(existing, 0, newArgs, args.length, existing.length);
+
+		this.args.put(command, newArgs);
+	}
+
 	/**
 	 * Set specific shape file for certain command.
 	 */

contribs/application/src/main/java/org/matsim/application/analysis/LogFileAnalysis.java

@@ -171,7 +171,6 @@ private void renderWarnings(BufferedWriter writer, Set<Warning> warnings) throws
 		} else {
 
 			Map<String, List<Warning>> grouped = warnings.stream().collect(Collectors.groupingBy(w -> w.module, Collectors.toList()));
-			writer.write(String.format("<h3 class=\"found-warnings\">Warnings found in %d module%s ❌</h3>\n\n", grouped.size(), grouped.size() > 1 ? "s" : ""));
 
 			for (Map.Entry<String, List<Warning>> e : grouped.entrySet()) {
 
@@ -188,9 +187,7 @@ private void renderWarnings(BufferedWriter writer, Set<Warning> warnings) throws
 		writer.write("""
 			<style>
 			.dash-row.row-warnings .dash-card-frame {
-				margin-top: 0;
-				margin-bottom: 0;
-				padding: 0 0.4em;
+			    background: none;
 			}
 			.dash-row.row-warnings .no-warnings {
 				color: #4BB543;

contribs/application/src/main/java/org/matsim/application/analysis/emissions/AirPollutionAnalysis.java

@@ -53,9 +53,9 @@
 )
 @CommandSpec(requireRunDirectory = true,
 	produces = {
-		"emissions_total.csv", "emissions_grid_per_day.xyt.csv", "emissions_per_link.csv",
+		"emissions_total.csv", "emissions_grid_per_day.csv", "emissions_per_link.csv",
 		"emissions_per_link_per_m.csv",
-		"emissions_grid_per_hour.xyt.csv",
+		"emissions_grid_per_hour.csv",
 		"emissions_vehicle_info.csv",
 	}
 )
@@ -113,7 +113,7 @@ public void install() {
 
 		String eventsFile = ApplicationUtils.matchInput("events", input.getRunDirectory()).toString();
 
-		EmissionsOnLinkEventHandler emissionsEventHandler = new EmissionsOnLinkEventHandler(3600);
+		EmissionsOnLinkEventHandler emissionsEventHandler = new EmissionsOnLinkEventHandler(3600, 86400);
 		eventsManager.addHandler(emissionsEventHandler);
 		eventsManager.initProcessing();
 		MatsimEventsReader matsimEventsReader = new MatsimEventsReader(eventsManager);
@@ -129,7 +129,7 @@ public void install() {
 
 		writeRaster(filteredNetwork, config, emissionsEventHandler);
 
-		// writeTimeDependentRaster(filteredNetwork, config, emissionsEventHandler);
+		writeTimeDependentRaster(filteredNetwork, config, emissionsEventHandler);
 
 		return 0;
 	}
@@ -253,7 +253,7 @@ private void writeRaster(Network network, Config config, EmissionsOnLinkEventHan
 
 		Raster raster = rasterMap.values().stream().findFirst().orElseThrow();
 
-		try (CSVPrinter printer = new CSVPrinter(Files.newBufferedWriter(output.getPath("emissions_grid_per_day.xyt.csv")),
+		try (CSVPrinter printer = new CSVPrinter(Files.newBufferedWriter(output.getPath("emissions_grid_per_day.csv")),
 			CSVFormat.DEFAULT.builder().setCommentMarker('#').build())) {
 
 			String crs = ProjectionUtils.getCRS(network);
@@ -263,7 +263,7 @@ private void writeRaster(Network network, Config config, EmissionsOnLinkEventHan
 				crs = config.global().getCoordinateSystem();
 
 			// print coordinate system
-			printer.printComment(crs);
+//			printer.printComment(crs);
 
 			// print header
 			printer.print("time");
@@ -278,12 +278,15 @@ private void writeRaster(Network network, Config config, EmissionsOnLinkEventHan
 				for (int yi = 0; yi < yLength.get(0); yi++) {
 
 					Coord coord = raster.getCoordForIndex(xi, yi);
+					double value = rasterMap.get(Pollutant.CO2_TOTAL).getValueByIndex(xi, yi);
+
+//					if (value == 0)
+//						continue;
 
 					printer.print(0.0);
 					printer.print(coord.getX());
 					printer.print(coord.getY());
 
-					double value = rasterMap.get(Pollutant.CO2_TOTAL).getValueByIndex(xi, yi);
 					printer.print(value);
 
 					printer.println();
@@ -306,7 +309,7 @@ private void writeTimeDependentRaster(Network network, Config config, EmissionsO
 
 		Raster raster = firstBin.values().stream().findFirst().orElseThrow();
 
-		try (CSVPrinter printer = new CSVPrinter(IOUtils.getBufferedWriter(output.getPath("emissions_grid_per_hour.xyt.csv").toString()),
+		try (CSVPrinter printer = new CSVPrinter(IOUtils.getBufferedWriter(output.getPath("emissions_grid_per_hour.csv").toString()),
 			CSVFormat.DEFAULT.builder().setCommentMarker('#').build())) {
 
 			String crs = ProjectionUtils.getCRS(network);
@@ -332,12 +335,15 @@ private void writeTimeDependentRaster(Network network, Config config, EmissionsO
 					for (TimeBinMap.TimeBin<Map<Pollutant, Raster>> timeBin : timeBinMap.getTimeBins()) {
 
 						Coord coord = raster.getCoordForIndex(xi, yi);
+						double value = timeBin.getValue().get(Pollutant.CO2_TOTAL).getValueByIndex(xi, yi);
+
+//						if (value == 0)
+//							continue;
 
 						printer.print(timeBin.getStartTime());
 						printer.print(coord.getX());
 						printer.print(coord.getY());
 
-						double value = timeBin.getValue().get(Pollutant.CO2_TOTAL).getValueByIndex(xi, yi);
 						printer.print(value);
 
 						printer.println();

contribs/application/src/main/java/org/matsim/application/analysis/population/FilterTripModes.java

@@ -0,0 +1,52 @@
+package org.matsim.application.analysis.population;
+import org.matsim.application.CommandSpec;
+import org.matsim.application.MATSimAppCommand;
+import org.matsim.application.options.InputOptions;
+import org.matsim.application.options.OutputOptions;
+import org.matsim.application.options.ShpOptions;
+import org.matsim.core.utils.io.IOUtils;
+import picocli.CommandLine;
+import tech.tablesaw.api.*;
+import tech.tablesaw.io.csv.CsvReadOptions;
+
+import java.util.Map;
+
+@CommandLine.Command(name = "trip-filter", description = "Extracts trips and separate them by mode.")
+@CommandSpec(
+	requires = {"trips.csv"},
+	produces = {"trips_per_mode_%s.csv"}
+)
+public class FilterTripModes implements MATSimAppCommand {
+
+	@CommandLine.Mixin
+	private final InputOptions input = InputOptions.ofCommand(FilterTripModes.class);
+
+	@CommandLine.Mixin
+	private final OutputOptions output = OutputOptions.ofCommand(FilterTripModes.class);
+
+	@CommandLine.Mixin
+	private ShpOptions shp;
+
+	public static void main(String[] args) {
+		new FilterTripModes().execute(args);
+	}
+
+
+	@Override
+	public Integer call() throws Exception {
+
+		Table trips = Table.read().csv(CsvReadOptions.builder(IOUtils.getBufferedReader(input.getPath()))
+			.columnTypesPartial(Map.of("person", ColumnType.TEXT, "main_mode", ColumnType.STRING))
+			.sample(false)
+			.separator(';').build());
+
+		for (String mainMode : trips.stringColumn("main_mode").unique()) {
+
+			Table table = trips.where(trips.stringColumn("main_mode").isEqualTo(mainMode));
+
+			table.write().csv(output.getPath("trips_per_mode_%s.csv", mainMode).toFile());
+		}
+
+		return 0;
+	}
+}

contribs/application/src/main/java/org/matsim/application/analysis/population/StuckAgentAnalysis.java

@@ -35,6 +35,10 @@
 public class StuckAgentAnalysis implements MATSimAppCommand, PersonStuckEventHandler, ActivityStartEventHandler {
 	private static final Logger log = LogManager.getLogger(StuckAgentAnalysis.class);
 	private final Object2IntMap<String> stuckAgentsPerMode = new Object2IntOpenHashMap<>();
+
+	/**
+	 * Per mode, per hour -> number of stuck agents
+	 */
 	private final Map<String, Int2DoubleMap> stuckAgentsPerHour = new HashMap<>();
 	private final Map<String, Object2DoubleOpenHashMap<String>> stuckAgentsPerLink = new HashMap<>();
 	private final Object2DoubleOpenHashMap<String> allStuckLinks = new Object2DoubleOpenHashMap<>();
@@ -96,12 +100,13 @@ public Integer call() throws Exception {
 			// Sort Map
 			List<String> sorted = new ArrayList<>(allStuckLinks.keySet());
 			sorted.sort((o1, o2) -> -Double.compare(allStuckLinks.getDouble(o1), allStuckLinks.getDouble(o2)));
+
 			List<String> header = new ArrayList<>(stuckAgentsPerLink.keySet());
 			header.add(0, "link");
 			header.add(1, "Agents");
 			// Write to .csv
 			printer.printRecord(header);
-			for (int i = 0; i < 20; i++) {
+			for (int i = 0; i < 20 && i < sorted.size(); i++) {
 				String[] result = new String[header.size()];
 				result[0] = sorted.get(i);
 				result[1] = String.valueOf(this.summarizeStuckAgentsPerLink(sorted.get(i)));
@@ -146,8 +151,7 @@ private double summarizeStuckAgentsPerHour(int hour) {
 	@Override
 	public void handleEvent(PersonStuckEvent event) {
 		// Pie chart
-		if (!stuckAgentsPerMode.containsKey(event.getLegMode())) stuckAgentsPerMode.put(event.getLegMode(), 1);
-		else stuckAgentsPerMode.put(event.getLegMode(), stuckAgentsPerMode.getInt(event.getLegMode()) + 1);
+		stuckAgentsPerMode.mergeInt(event.getLegMode(), 1, Integer::sum);
 
 		// Stuck Agents per Hour
 		Int2DoubleMap perHour = stuckAgentsPerHour.computeIfAbsent(event.getLegMode(), (k) -> new Int2DoubleOpenHashMap());
@@ -159,7 +163,7 @@ public void handleEvent(PersonStuckEvent event) {
 		Object2DoubleMap<String> perLink = stuckAgentsPerLink.computeIfAbsent(event.getLegMode(), (k) -> new Object2DoubleOpenHashMap<>());
 
 		Id<Link> link = Objects.requireNonNullElseGet(event.getLinkId(), () -> Id.createLinkId("unknown"));
-		allStuckLinks.merge(link.toString(), 1., Double::sum);
+		allStuckLinks.mergeDouble(link.toString(), 1., Double::sum);
 		perLink.mergeDouble(link.toString(), 1, Double::sum);
 	}