App contrib maintenance (#3191)

* improve csv and shp options

* improve count options file format

* add log message

* update bast counts test

* remove lane functionality from sumo converter, use DisallowedLinks instead

contribs/application/src/main/java/org/matsim/application/options/CountsOptions.java

@@ -0,0 +1,155 @@
+package org.matsim.application.options;
+
+import org.apache.commons.csv.CSVFormat;
+import org.apache.commons.csv.CSVParser;
+import org.apache.commons.csv.CSVRecord;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.matsim.api.core.v01.Id;
+import org.matsim.api.core.v01.network.Link;
+import org.matsim.core.utils.io.IOUtils;
+import picocli.CommandLine;
+
+import javax.annotation.Nullable;
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.*;
+
+/**
+ * Common options when working with counts data.
+ */
+public final class CountsOptions {
+
+	private static final Logger log = LogManager.getLogger(CountsOptions.class);
+
+	@CommandLine.Option(names = "--counts-mapping", description = "Path to csv with count station ids to ignore")
+	private String input;
+
+	private Map<String, Id<Link>> manualMatchedCounts = null;
+	private Set<String> ignoredCounts = null;
+
+	public CountsOptions() {
+	}
+
+	public CountsOptions(@Nullable String input) {
+		this.input = input;
+	}
+
+	/**
+	 * Get list of ignored count ids.
+	 */
+	public Set<String> getIgnored() {
+		readMapping();
+		return ignoredCounts;
+	}
+
+	/**
+	 * Return mapping of count id to specified link id.
+	 */
+	public Map<String, Id<Link>> getManualMatched() {
+		readMapping();
+		return manualMatchedCounts;
+	}
+
+	private synchronized void readMapping() {
+
+		// Already read
+		if (manualMatchedCounts != null)
+			return;
+
+		manualMatchedCounts = new HashMap<>();
+		ignoredCounts = new HashSet<>();
+
+		// No input file
+		if (input == null)
+			return;
+
+		try (var reader = IOUtils.getBufferedReader(input)) {
+			CSVFormat format = CSVFormat.Builder.create()
+				.setAllowMissingColumnNames(true)
+				.setDelimiter(CsvOptions.detectDelimiter(input))
+				.setHeader()
+				.setSkipHeaderRecord(true)
+				.build();
+
+			try (CSVParser csv = new CSVParser(reader, format)) {
+				Schema schema = parseSchema(csv.getHeaderNames());
+
+				log.info("Using schema for counts mapping: {}", schema);
+
+				for (CSVRecord row : csv) {
+
+					String stationId = row.get(schema.stationColumn);
+					manualMatchedCounts.put(stationId, Id.createLinkId(row.get(schema.linkColumn)));
+
+					if (schema.usingColumn != null) {
+
+						String value = row.get(schema.usingColumn).strip().toLowerCase();
+						boolean val = value.equals("y") || value.equals("x") || value.equals("true");
+
+						if (schema.isWhiteList && !val)
+							ignoredCounts.add(stationId);
+						else if (!schema.isWhiteList && val)
+							ignoredCounts.add(stationId);
+					}
+				}
+			}
+		} catch (IOException e) {
+			throw new UncheckedIOException(e);
+		}
+
+	}
+
+	/**
+	 * Check whether station id should be ignored.
+	 */
+	public boolean isIgnored(String stationId) {
+		readMapping();
+		return ignoredCounts.contains(stationId);
+	}
+
+	/**
+	 * Return manually matched link id.
+	 *
+	 * @return null if not matched
+	 */
+	public Id<Link> isManuallyMatched(String stationId) {
+		readMapping();
+
+		if (manualMatchedCounts.isEmpty() || !manualMatchedCounts.containsKey(stationId))
+			return null;
+
+		return manualMatchedCounts.get(stationId);
+	}
+
+	private Schema parseSchema(List<String> header) {
+
+		List<String> names = header.stream()
+			.map(String::toLowerCase)
+			.map(String::strip)
+			.map(s -> s.replace("_", ""))
+			.toList();
+
+		int linkId = names.indexOf("linkid");
+
+		if (linkId < 0)
+			throw new IllegalArgumentException("Link id column not found in csv: " + header);
+
+		int using = names.indexOf("using");
+		int ignore = names.indexOf("ignore");
+
+		// first or second column for station id
+		int station = linkId == 0 ? 1 : 0;
+
+		if (using > 0)
+			return new Schema(header.get(station), header.get(linkId), header.get(using), true);
+
+		if (ignore > 0)
+			return new Schema(header.get(station), header.get(linkId), header.get(ignore), false);
+
+		return new Schema(header.get(station), header.get(linkId), null, false);
+	}
+
+	private record Schema(String stationColumn, String linkColumn, String usingColumn, boolean isWhiteList) {
+	}
+}

contribs/application/src/main/java/org/matsim/application/options/CsvOptions.java

@@ -80,7 +80,7 @@ public static Character detectDelimiter(String path) throws IOException {
 	 * Get the CSV format defined by the options.
 	 */
 	public CSVFormat getFormat() {
-		CSVFormat.Builder format = this.csvFormat.getFormat().builder().setSkipHeaderRecord(true);
+		CSVFormat.Builder format = this.csvFormat.getFormat().builder().setHeader().setSkipHeaderRecord(true);
 		if (csvDelimiter != null)
 			format = format.setDelimiter(csvDelimiter);
 

contribs/application/src/main/java/org/matsim/application/options/ShpOptions.java

@@ -369,8 +369,15 @@ public SimpleFeature queryFeature(Coord coord) {
 			List<SimpleFeature> result = index.query(new Envelope(p));
 			for (SimpleFeature ft : result) {
 				Geometry geom = (Geometry) ft.getDefaultGeometry();
-				if (geom.contains(MGC.coordinate2Point(p)))
-					return ft;
+
+				// Catch Exception for invalid, too complex geometries
+				try {
+					if (geom.contains(MGC.coordinate2Point(p)))
+						return ft;
+				} catch (TopologyException e) {
+					if (geom.convexHull().contains(MGC.coordinate2Point(p)))
+						return ft;
+				}
 			}
 
 			return null;
@@ -381,17 +388,7 @@ public SimpleFeature queryFeature(Coord coord) {
 		 */
 		@SuppressWarnings("unchecked")
 		public boolean contains(Coord coord) {
-
-			Coordinate p = MGC.coord2Coordinate(ct.transform(coord));
-
-			List<SimpleFeature> result = index.query(new Envelope(p));
-			for (SimpleFeature ft : result) {
-				Geometry geom = (Geometry) ft.getDefaultGeometry();
-				if (geom.contains(MGC.coordinate2Point(p)))
-					return true;
-			}
-
-			return false;
+			return queryFeature(coord) != null;
 		}
 
 		/**

contribs/application/src/main/java/org/matsim/application/prepare/counts/CreateCountsFromBAStData.java

@@ -11,7 +11,7 @@
 import org.matsim.api.core.v01.network.Link;
 import org.matsim.api.core.v01.network.Network;
 import org.matsim.application.MATSimAppCommand;
-import org.matsim.application.options.CountsOption;
+import org.matsim.application.options.CountsOptions;
 import org.matsim.application.options.CrsOptions;
 import org.matsim.application.options.ShpOptions;
 import org.matsim.core.config.groups.NetworkConfigGroup;
@@ -63,7 +63,7 @@ public class CreateCountsFromBAStData implements MATSimAppCommand {
 	@CommandLine.Mixin
 	private final ShpOptions shp = new ShpOptions();
 	@CommandLine.Mixin
-	private final CountsOption counts = new CountsOption();
+	private final CountsOptions counts = new CountsOptions();
 	@CommandLine.Mixin
 	private final CrsOptions crs = new CrsOptions("EPSG:25832");
 	@CommandLine.Option(names = "--network", description = "path to MATSim network", required = true)
@@ -284,7 +284,7 @@ private void readHourlyTrafficVolume(Path pathToDisaggregatedData, Map<String, B
 		}
 	}
 
-	private void match(Network network, NetworkIndex<BAStCountStation> index, BAStCountStation station, CountsOption counts) {
+	private void match(Network network, NetworkIndex<BAStCountStation> index, BAStCountStation station, CountsOptions counts) {
 
 		Id<Link> manuallyMatched = counts.isManuallyMatched(station.getId());
 		Link matched;
@@ -330,7 +330,7 @@ private List<Predicate<Link>> createRoadTypeFilter(List<String> types) {
 		return filter;
 	}
 
-	private void matchBAStWithNetwork(String pathToNetwork, Map<String, BAStCountStation> stations, CountsOption countsOption, CrsOptions crs) {
+	private void matchBAStWithNetwork(String pathToNetwork, Map<String, BAStCountStation> stations, CountsOptions countsOption, CrsOptions crs) {
 
 		if (crs.getTargetCRS() != null)
 			throw new RuntimeException("Please don't specify --target-crs. Only use --input-crs to determinate the network crs!");
@@ -366,7 +366,7 @@ private void matchBAStWithNetwork(String pathToNetwork, Map<String, BAStCountSta
 			match(filteredNetwork, index, station, countsOption);
 	}
 
-	private Map<String, BAStCountStation> readBAStCountStations(Path pathToAggregatedData, ShpOptions shp, CountsOption counts) {
+	private Map<String, BAStCountStation> readBAStCountStations(Path pathToAggregatedData, ShpOptions shp, CountsOptions counts) {
 
 		List<BAStCountStation> stations = new ArrayList<>();