diff --git a/.github/dependabot.yml b/.github/dependabot.yml index b76b8957033..749aa797fa1 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -4,7 +4,14 @@ updates: directory: "/" schedule: interval: "daily" + groups: + maven: + patterns: ["*"] + - package-ecosystem: "github-actions" directory: "/" schedule: interval: "daily" + groups: + github-actions: + patterns: ["*"] diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml new file mode 100644 index 00000000000..021bae1c67c --- /dev/null +++ b/.github/workflows/codeql.yml @@ -0,0 +1,49 @@ +name: CodeQL + +on: + push: + branches: 'master' + +jobs: + analyze: + name: Analyze + runs-on: 'ubuntu-latest' + timeout-minutes: 360 + permissions: + actions: read + contents: read + security-events: write + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Setup Java + uses: actions/setup-java@v3 + with: + java-version: 17 + distribution: 'zulu' + cache: 'maven' + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v2 + with: + languages: 'java' + # If you wish to specify custom queries, you can do so here or in a config file. + # By default, queries listed here will override any specified in a config file. + # Prefix the list here with "+" to use these queries and those in the config file. + + # For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs + # queries: security-extended,security-and-quality + + + # Autobuild attempts to build any compiled languages (C/C++, C#, Go, Java, or Swift). + # If this step fails, then you should remove it and run the build manually (see below) + - name: Autobuild + uses: github/codeql-action/autobuild@v2 + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v2 + with: + category: "/language:java" diff --git a/.github/workflows/deploy-on-pr-merge.yaml b/.github/workflows/deploy-on-pr-merge.yaml index deed3a54d24..332a3dfe178 100644 --- a/.github/workflows/deploy-on-pr-merge.yaml +++ b/.github/workflows/deploy-on-pr-merge.yaml @@ -10,8 +10,8 @@ on: jobs: deploy-snapshot: name: deploy PR-labelled version - # for PR-labelled deployment -- only if closed by merging - if: github.event_name == 'push' || github.event.pull_request.merged == true + # only if PR closed by merging + if: github.event.pull_request.merged == true runs-on: ubuntu-latest @@ -25,7 +25,7 @@ jobs: java-version: 17 distribution: 'zulu' cache: 'maven' - server-id: ${{ github.event_name == 'push' && 'matsim-snapshots' || 'matsim-releases' }} #choose mvn repo + server-id: 'matsim-releases' server-username: MAVEN_USERNAME server-password: MAVEN_PASSWORD @@ -43,5 +43,10 @@ jobs: MAVEN_USERNAME: ${{ secrets.REPOMATSIM_USERNAME }} MAVEN_PASSWORD: ${{ secrets.REPOMATSIM_TOKEN }} + - name: Submit Dependency Graph + # Generate a complete dependency graph and submit the graph to the GitHub repository. + # The goal is to improve security alerts from dependabot, because dependabot is not able to compute the complete dependency graph. + uses: advanced-security/maven-dependency-submission-action@v3 + env: MAVEN_OPTS: -Xmx2g diff --git a/contribs/drt-extensions/src/main/java/org/matsim/contrib/drt/extension/dashboards/DrtDashboard.java b/contribs/drt-extensions/src/main/java/org/matsim/contrib/drt/extension/dashboards/DrtDashboard.java index 291c0fbaf92..b7e1aeda0f9 100644 --- a/contribs/drt-extensions/src/main/java/org/matsim/contrib/drt/extension/dashboards/DrtDashboard.java +++ b/contribs/drt-extensions/src/main/java/org/matsim/contrib/drt/extension/dashboards/DrtDashboard.java @@ -140,11 +140,6 @@ public void configure(Header header, Layout layout) { viz.center = data.context().getCenter(); viz.zoom = data.context().mapZoomLevel; }) - //TODO group rejections per time bin. better put it into the plot with wait stats over daty time. -// .el(Line.class, (viz, data) -> { -// viz.dataset = data.output("ITERS/it." + lastIteration + "/*rejections_" + drtConfigGroup.mode + ".csv"); -// viz.x = "time"; -// }) ; // This plot is not absolutely necesarry given the hex plots @@ -163,12 +158,13 @@ public void configure(Header header, Layout layout) { viz.title = "Final Demand and Wait Stats over day time"; viz.description = "Number of rides (customers) is displayed in bars, wait statistics in lines"; - Plotly.DataSet dataset = viz.addDataset(data.output("*_waitStats_" + drtConfigGroup.mode + ".csv")); + Plotly.DataSet waitStats = viz.addDataset(data.output("*_waitStats_" + drtConfigGroup.mode + ".csv")); + Plotly.DataSet rejections = viz.addDataset(data.output("*drt_rejections_perTimeBin_" + drtConfigGroup.mode + ".csv")); viz.layout = tech.tablesaw.plotly.components.Layout.builder() .xAxis(Axis.builder().title("Time Bin").build()) .yAxis(Axis.builder().title("Wait Time [s]").build()) - .yAxis2(Axis.builder().title("Nr of Rides") + .yAxis2(Axis.builder().title("Nr of Rides/Rejections") .side(Axis.Side.right) .overlaying(ScatterTrace.YAxis.Y) .build()) @@ -179,7 +175,7 @@ public void configure(Header header, Layout layout) { .mode(ScatterTrace.Mode.LINE) .name("Average") .build(), - dataset.mapping() + waitStats.mapping() .x("timebin") .y("average_wait") ); @@ -188,7 +184,7 @@ public void configure(Header header, Layout layout) { .mode(ScatterTrace.Mode.LINE) .name("P5") .build(), - dataset.mapping() + waitStats.mapping() .x("timebin") .y("p_5") ); @@ -197,7 +193,7 @@ public void configure(Header header, Layout layout) { .mode(ScatterTrace.Mode.LINE) .name("P95") .build(), - dataset.mapping() + waitStats.mapping() .x("timebin") .y("p_95") ); @@ -207,11 +203,21 @@ public void configure(Header header, Layout layout) { .yAxis(ScatterTrace.YAxis.Y2.toString()) .name("Rides") .build(), - dataset.mapping() + waitStats.mapping() .x("timebin") .y("legs") ); + viz.addTrace(BarTrace.builder(Plotly.OBJ_INPUT, Plotly.INPUT) + .opacity(0.3) + .yAxis(ScatterTrace.YAxis.Y2.toString()) + .name("Rejections") + .build(), + rejections.mapping() + .x("timebin") + .y("rejections") + ); + }) .el(Area.class, (viz, data) -> { viz.title = "Vehicle occupancy"; //actually, without title the area plot won't work @@ -248,7 +254,7 @@ public void configure(Header header, Layout layout) { dataset.mapping() .x("iteration") .y("rejections") - .color(Plotly.ColorScheme.RdBu) +// .color(Plotly.ColorScheme.RdBu) ); viz.addTrace(BarTrace.builder(Plotly.OBJ_INPUT, Plotly.INPUT) @@ -257,7 +263,7 @@ public void configure(Header header, Layout layout) { dataset.mapping() .x("iteration") .y("rides") - .color(Plotly.ColorScheme.RdBu) +// .color(Plotly.ColorScheme.RdBu) ); }) @@ -294,7 +300,7 @@ public void configure(Header header, Layout layout) { dataset.mapping() .x("iteration") .y("inVehicleTravelTime_mean") - .color(Plotly.ColorScheme.RdBu) +// .color(Plotly.ColorScheme.RdBu) ); viz.addTrace(BarTrace.builder(Plotly.OBJ_INPUT, Plotly.INPUT) @@ -304,7 +310,7 @@ public void configure(Header header, Layout layout) { dataset.mapping() .x("iteration") .y("wait_average") - .color(Plotly.ColorScheme.RdBu) +// .color(Plotly.ColorScheme.RdBu) ); }) .el(Line.class, (viz, data) -> { diff --git a/contribs/drt/src/main/java/org/matsim/contrib/drt/analysis/DrtAnalysisControlerListener.java b/contribs/drt/src/main/java/org/matsim/contrib/drt/analysis/DrtAnalysisControlerListener.java index 7fd5197abe8..17f55751141 100644 --- a/contribs/drt/src/main/java/org/matsim/contrib/drt/analysis/DrtAnalysisControlerListener.java +++ b/contribs/drt/src/main/java/org/matsim/contrib/drt/analysis/DrtAnalysisControlerListener.java @@ -19,31 +19,7 @@ package org.matsim.contrib.drt.analysis; -import static java.util.stream.Collectors.toList; - -import java.awt.BasicStroke; -import java.awt.Color; -import java.io.BufferedWriter; -import java.io.FileOutputStream; -import java.io.IOException; -import java.text.DecimalFormat; -import java.text.DecimalFormatSymbols; -import java.text.ParseException; -import java.text.SimpleDateFormat; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Comparator; -import java.util.HashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.Optional; -import java.util.TreeMap; -import java.util.function.Function; -import java.util.stream.Collectors; - +import com.google.common.base.Preconditions; import org.apache.commons.lang3.tuple.Pair; import org.apache.commons.math3.stat.descriptive.DescriptiveStatistics; import org.apache.logging.log4j.LogManager; @@ -70,12 +46,13 @@ import org.matsim.contrib.drt.passenger.events.DrtRequestSubmittedEvent; import org.matsim.contrib.drt.run.DrtConfigGroup; import org.matsim.contrib.drt.schedule.DrtStayTask; +import org.matsim.contrib.dvrp.analysis.VehicleOccupancyProfileCalculator; import org.matsim.contrib.dvrp.fleet.DvrpVehicle; import org.matsim.contrib.dvrp.fleet.DvrpVehicleSpecification; import org.matsim.contrib.dvrp.fleet.FleetSpecification; import org.matsim.contrib.dvrp.optimizer.Request; import org.matsim.contrib.dvrp.passenger.PassengerPickedUpEvent; -import org.matsim.contrib.dvrp.analysis.VehicleOccupancyProfileCalculator; +import org.matsim.contrib.dvrp.passenger.PassengerRequestRejectedEvent; import org.matsim.core.config.Config; import org.matsim.core.config.groups.QSimConfigGroup; import org.matsim.core.controler.MatsimServices; @@ -87,7 +64,20 @@ import org.matsim.core.utils.misc.Time; import org.matsim.vehicles.Vehicle; -import com.google.common.base.Preconditions; +import java.awt.*; +import java.io.BufferedWriter; +import java.io.FileOutputStream; +import java.io.IOException; +import java.text.DecimalFormat; +import java.text.DecimalFormatSymbols; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.List; +import java.util.*; +import java.util.function.Function; +import java.util.stream.Collectors; + +import static java.util.stream.Collectors.toList; /** * @author jbischoff @@ -176,6 +166,13 @@ public void notifyIterationEnds(IterationEndsEvent event) { .sorted(Comparator.comparing(leg -> leg.departureTime)) .collect(toList()); + List rejectionEvents = drtEventSequenceCollector.getRejectedRequestSequences() + .values() + .stream() + .map(eventSequence -> eventSequence.getRejected().get()) + .sorted(Comparator.comparing(rejectionEvent -> rejectionEvent.getTime())) + .collect(toList()); + collection2Text(drtEventSequenceCollector.getRejectedRequestSequences().values(), filename(event, "drt_rejections", ".csv"), String.join(delimiter, "time", "personId", "fromLinkId", "toLinkId", "fromX", "fromY", "toX", "toY"), seq -> { DrtRequestSubmittedEvent submission = seq.getSubmitted(); @@ -252,6 +249,7 @@ public void notifyIterationEnds(IterationEndsEvent event) { writeVehicleDistances(drtVehicleStats.getVehicleStates(), filename(event, "vehicleDistanceStats", ".csv"), delimiter); analyseDetours(network, legs, drtVehicleStats.getTravelDistances(), drtCfg, filename(event, "drt_detours"), createGraphs, delimiter); analyseWaitTimes(filename(event, "waitStats"), legs, 1800, createGraphs, delimiter); + analyseRejections(filename(event,"drt_rejections_perTimeBin"), rejectionEvents,1800, createGraphs, delimiter); analyseConstraints(filename(event, "constraints"), legs, createGraphs); double endTime = qSimCfg.getEndTime() @@ -399,6 +397,7 @@ public void notifyShutdown(ShutdownEvent event) { dumpOutput(event.getIteration(), "waitTimeComparison", ".png"); dumpOutput(event.getIteration(), "waitTimeComparison", ".csv"); dumpOutput(event.getIteration(), "drt_rejections", ".csv"); + dumpOutput(event.getIteration(), "drt_rejections_perTimeBin", ".csv"); dumpOutput(event.getIteration(), "drt_legs", ".csv"); dumpOutput(event.getIteration(), "vehicleDistanceStats", ".csv"); dumpOutput(event.getIteration(), "drt_detours", ".csv"); @@ -453,6 +452,33 @@ private static Map> splitLegsIntoBins(Collection le return splitLegs; } + private static Map> splitEventsIntoBins(List rejectionEvents, int binSize_s) { + Map> rejections = new TreeMap<>(); + + int startTime = ((int)(rejectionEvents.get(0).getTime() / binSize_s)) * binSize_s; + int endTime = ((int)(rejectionEvents.get(rejectionEvents.size() - 1).getTime() / binSize_s) + 1) * binSize_s; + + for (int time = startTime; time < endTime; time = time + binSize_s) { + + // rejection list in this timebin + List rejectionList = new ArrayList<>(); + + //Iterate through each rejection + for (PassengerRequestRejectedEvent rejectedEvent : rejectionEvents){ + double rejectionTime = rejectedEvent.getTime(); + if (rejectionTime > endTime || rejectionTime < startTime) { + LogManager.getLogger(DrtAnalysisControlerListener.class).error("wrong end / start Times for analysis"); + } + + if (rejectionTime > time && rejectionTime < time + binSize_s) { + rejectionList.add(rejectedEvent); + } + } + rejections.put((double)time, rejectionList); + } + return rejections; + } + private static void analyzeBoardingsAndDeboardings(List legs, String delimiter, double startTime, double endTime, double timeBinSize, String boardingsFile, String deboardingsFile, Network network) { if (endTime < startTime) { @@ -708,6 +734,55 @@ private static void analyseWaitTimes(String fileName, List legs, int bin } + private static void analyseRejections(String fileName, List rejectionEvents, int binsize_s, boolean createGraphs, String delimiter) { + if (rejectionEvents.size() == 0) + return; + + Map> splitEvents = splitEventsIntoBins(rejectionEvents, binsize_s); + + DecimalFormat format = new DecimalFormat(); + format.setDecimalFormatSymbols(new DecimalFormatSymbols(Locale.US)); + format.setMinimumIntegerDigits(1); + format.setMaximumFractionDigits(2); + format.setGroupingUsed(false); + + SimpleDateFormat sdf2 = new SimpleDateFormat("HH:mm:ss"); + + BufferedWriter bw = IOUtils.getBufferedWriter(fileName + ".csv"); + TimeSeriesCollection dataset = new TimeSeriesCollection(); + TimeSeries rejections = new TimeSeries("number of rejections"); + + try { + bw.write(String.join(delimiter, "timebin", "rejections")); + + for(Map.Entry> e : splitEvents.entrySet()){ + int drt_numOfRejection = 0; + if (!e.getValue().isEmpty()) { + drt_numOfRejection = e.getValue().size(); + } + + Minute h = new Minute(sdf2.parse(Time.writeTime(e.getKey()))); + + rejections.addOrUpdate(h, Double.valueOf(drt_numOfRejection)); + bw.newLine(); + bw.write(String.join(delimiter, Time.writeTime(e.getKey()) + "",// + format.format(drt_numOfRejection) +"")); + } + + bw.flush(); + bw.close(); + if (createGraphs) { + dataset.addSeries(rejections); + JFreeChart chart = chartProfile(splitEvents.size(), dataset, "Number of rejections", "Number"); + ChartSaveUtils.saveAsPNG(chart, fileName, 1500, 1000); + } + + } catch (IOException | ParseException e) { + + e.printStackTrace(); + } + } + private static JFreeChart chartProfile(int length, TimeSeriesCollection dataset, String descriptor, String yax) { JFreeChart chart = ChartFactory.createTimeSeriesChart(descriptor, "Time", yax, dataset); diff --git a/contribs/vsp/pom.xml b/contribs/vsp/pom.xml index f7d6c310905..96a0cd8ca37 100644 --- a/contribs/vsp/pom.xml +++ b/contribs/vsp/pom.xml @@ -140,7 +140,7 @@ org.apache.poi poi-ooxml - 5.2.3 + 5.2.4 @@ -179,7 +179,7 @@ com.graphhopper graphhopper-core - 6.2 + 7.0 diff --git a/matsim/pom.xml b/matsim/pom.xml index b7feb34e61d..4dc0348e536 100644 --- a/matsim/pom.xml +++ b/matsim/pom.xml @@ -203,7 +203,7 @@ com.github.luben zstd-jni - 1.5.5-5 + 1.5.5-6 jakarta.validation diff --git a/pom.xml b/pom.xml index e8637f7a05c..867da53e42b 100644 --- a/pom.xml +++ b/pom.xml @@ -185,7 +185,7 @@ com.google.errorprone error_prone_annotations - 2.21.1 + 2.22.0 @@ -264,7 +264,7 @@ org.checkerframework checker-qual - 3.38.0 + 3.39.0