From c1e3335f0a987a4c375137d06f166b604e5a475b Mon Sep 17 00:00:00 2001 From: Joschka Bischoff Date: Thu, 7 Mar 2024 14:29:23 +0100 Subject: [PATCH 1/8] fully support subpopulations in ReplanningAnnealer --- .../annealing/ReplanningAnnealer.java | 62 +++++++++++-------- .../ReplanningAnnealerConfigGroup.java | 54 +++++++++------- .../annealing/ReplanningAnnealerTest.java | 49 +++++++++++++-- 3 files changed, 111 insertions(+), 54 deletions(-) diff --git a/matsim/src/main/java/org/matsim/core/replanning/annealing/ReplanningAnnealer.java b/matsim/src/main/java/org/matsim/core/replanning/annealing/ReplanningAnnealer.java index b576833d9fd..2522bd2a96e 100644 --- a/matsim/src/main/java/org/matsim/core/replanning/annealing/ReplanningAnnealer.java +++ b/matsim/src/main/java/org/matsim/core/replanning/annealing/ReplanningAnnealer.java @@ -57,7 +57,7 @@ public class ReplanningAnnealer implements IterationStartsListener, StartupListe private final ReplanningAnnealerConfigGroup saConfig; private final int innovationStop; private final String sep; - private final EnumMap currentValues; + private final EnumMap> currentValuesPerSubpopulation; private int currentIter; private List header; @Inject @@ -67,7 +67,7 @@ public class ReplanningAnnealer implements IterationStartsListener, StartupListe public ReplanningAnnealer(Config config) { this.config = config; this.saConfig = ConfigUtils.addOrGetModule(config, ReplanningAnnealerConfigGroup.class); - this.currentValues = new EnumMap<>(AnnealParameterOption.class); + this.currentValuesPerSubpopulation = new EnumMap<>(AnnealParameterOption.class); this.innovationStop = getInnovationStop(config); this.sep = config.global().getDefaultDelimiter(); } @@ -83,17 +83,19 @@ private static boolean isInnovationStrategy(String strategyName) { @Override public void notifyStartup(StartupEvent event) { header = new ArrayList<>(); - for (AnnealingVariable av : this.saConfig.getAnnealingVariables().values()) { + for (AnnealingVariable av : this.saConfig.getAllAnnealingVariables()) { if (!av.getAnnealType().equals(AnnealOption.disabled)) { // check and fix initial value if needed checkAndFixStartValue(av, event); - this.currentValues.put(av.getAnnealParameter(), av.getStartValue()); - header.add(av.getAnnealParameter().name()); + var mapPerSubpopulation = this.currentValuesPerSubpopulation.computeIfAbsent(av.getAnnealParameter(),a-> new HashMap<>()); + mapPerSubpopulation.put(av.getSubpopulation(),av.getStartValue()); + String subpopulationString = av.getSubpopulation()!=null? "_"+av.getSubpopulation() :""; + header.add(av.getAnnealParameter().name()+subpopulationString); if (av.getAnnealParameter().equals(AnnealParameterOption.globalInnovationRate)) { header.addAll(this.config.replanning().getStrategySettings().stream() - .filter(s -> Objects.equals(av.getDefaultSubpopulation(), s.getSubpopulation())) - .map(ReplanningConfigGroup.StrategySettings::getStrategyName) + .filter(s -> Objects.equals(av.getSubpopulation(), s.getSubpopulation())) + .map(strategySettings -> strategySettings.getStrategyName()+subpopulationString) .collect(Collectors.toList())); } } else { // if disabled, better remove it @@ -114,34 +116,35 @@ public void notifyStartup(StartupEvent event) { public void notifyIterationStarts(IterationStartsEvent event) { this.currentIter = event.getIteration() - this.config.controller().getFirstIteration(); Map annealStats = new HashMap<>(); - for (AnnealingVariable av : this.saConfig.getAnnealingVariables().values()) { + List allVariables = this.saConfig.getAllAnnealingVariables(); + for (AnnealingVariable av : allVariables) { if (this.currentIter > 0) { switch (av.getAnnealType()) { case geometric: - this.currentValues.compute(av.getAnnealParameter(), (k, v) -> + this.currentValuesPerSubpopulation.get(av.getAnnealParameter()).compute(av.getSubpopulation(), (k, v) -> v * av.getShapeFactor()); break; case exponential: int halfLifeIter = av.getHalfLife() <= 1.0 ? (int) (av.getHalfLife() * this.innovationStop) : (int) av.getHalfLife(); - this.currentValues.compute(av.getAnnealParameter(), (k, v) -> + this.currentValuesPerSubpopulation.get(av.getAnnealParameter()).compute(av.getSubpopulation(), (k, v) -> av.getStartValue() / Math.exp((double) this.currentIter / halfLifeIter)); break; case msa: - this.currentValues.compute(av.getAnnealParameter(), (k, v) -> + this.currentValuesPerSubpopulation.get(av.getAnnealParameter()).compute(av.getSubpopulation(), (k, v) -> av.getStartValue() / Math.pow(this.currentIter, av.getShapeFactor())); break; case sigmoid: halfLifeIter = av.getHalfLife() <= 1.0 ? (int) (av.getHalfLife() * this.innovationStop) : (int) av.getHalfLife(); - this.currentValues.compute(av.getAnnealParameter(), (k, v) -> + this.currentValuesPerSubpopulation.get(av.getAnnealParameter()).compute(av.getSubpopulation(), (k, v) -> av.getEndValue() + (av.getStartValue() - av.getEndValue()) / (1 + Math.exp(av.getShapeFactor() * (this.currentIter - halfLifeIter)))); break; case linear: double slope = (av.getStartValue() - av.getEndValue()) / (this.config.controller().getFirstIteration() - this.innovationStop); - this.currentValues.compute(av.getAnnealParameter(), (k, v) -> + this.currentValuesPerSubpopulation.get(av.getAnnealParameter()).compute(av.getSubpopulation(), (k, v) -> this.currentIter * slope + av.getStartValue()); break; case disabled: @@ -150,14 +153,16 @@ public void notifyIterationStarts(IterationStartsEvent event) { throw new IllegalArgumentException(); } - log.info("Annealling will be performed on parameter " + av.getAnnealParameter() + - ". Value: " + this.currentValues.get(av.getAnnealParameter())); + log.info("Annealling will be performed on parameter " + av.getAnnealParameter() +". Subpopulation: "+av.getSubpopulation()+ + ". Value: " +this.currentValuesPerSubpopulation.get(av.getAnnealParameter()).get(av.getSubpopulation())); - this.currentValues.compute(av.getAnnealParameter(), (k, v) -> + this.currentValuesPerSubpopulation.get(av.getAnnealParameter()).compute(av.getSubpopulation(), (k, v) -> Math.max(v, av.getEndValue())); } - double annealValue = this.currentValues.get(av.getAnnealParameter()); - annealStats.put(av.getAnnealParameter().name(), String.format(Locale.US, "%.4f", annealValue)); + double annealValue = this.currentValuesPerSubpopulation.get(av.getAnnealParameter()).get(av.getSubpopulation()); + String subpopulationString = av.getSubpopulation()!=null? "_"+av.getSubpopulation() :""; + + annealStats.put(av.getAnnealParameter().name()+subpopulationString, String.format(Locale.US, "%.4f", annealValue)); anneal(event, av, annealValue, annealStats); } @@ -178,6 +183,8 @@ private void writeIterationstats(int currentIter, Map annealStat } private void anneal(IterationStartsEvent event, AnnealingVariable av, double annealValue, Map annealStats) { + String subpopulationString = av.getSubpopulation()!=null? "_"+av.getSubpopulation() :""; + switch (av.getAnnealParameter()) { case BrainExpBeta: this.config.scoring().setBrainExpBeta(annealValue); @@ -193,16 +200,17 @@ private void anneal(IterationStartsEvent event, AnnealingVariable av, double ann annealValue = 0.0; } List annealValues = annealReplanning(annealValue, - event.getServices().getStrategyManager(), av.getDefaultSubpopulation()); + event.getServices().getStrategyManager(), av.getSubpopulation()); int i = 0; for (ReplanningConfigGroup.StrategySettings ss : this.config.replanning().getStrategySettings()) { - if (Objects.equals(ss.getSubpopulation(), av.getDefaultSubpopulation())) { - annealStats.put(ss.getStrategyName(), String.format(Locale.US, "%.4f", annealValues.get(i))); + if (Objects.equals(ss.getSubpopulation(), av.getSubpopulation())) { + annealStats.put(ss.getStrategyName()+subpopulationString, String.format(Locale.US, "%.4f", annealValues.get(i))); i++; } } - annealStats.put(av.getAnnealParameter().name(), String.format(Locale.US, "%.4f", // update value in case of switchoff - getStrategyWeights(event.getServices().getStrategyManager(), av.getDefaultSubpopulation(), StratType.allInnovation))); + + annealStats.put(av.getAnnealParameter().name()+subpopulationString, String.format(Locale.US, "%.4f", // update value in case of switchoff + getStrategyWeights(event.getServices().getStrategyManager(), av.getSubpopulation(), StratType.allInnovation))); break; default: throw new IllegalArgumentException(); @@ -328,14 +336,14 @@ private void checkAndFixStartValue(ReplanningAnnealerConfigGroup.AnnealingVariab configValue = this.config.scoring().getLearningRate(); break; case globalInnovationRate: - double innovationWeights = getStrategyWeights(this.config, av.getDefaultSubpopulation(), StratType.allInnovation); - double selectorWeights = getStrategyWeights(this.config, av.getDefaultSubpopulation(), StratType.allSelectors); + double innovationWeights = getStrategyWeights(this.config, av.getSubpopulation(), StratType.allInnovation); + double selectorWeights = getStrategyWeights(this.config, av.getSubpopulation(), StratType.allSelectors); if (innovationWeights + selectorWeights != 1.0) { log.warn("Initial sum of strategy weights different from 1.0. Rescaling."); double innovationStartValue = av.getStartValue() == null ? innovationWeights : av.getStartValue(); - rescaleStartupWeights(innovationStartValue, this.config, event.getServices().getStrategyManager(), av.getDefaultSubpopulation()); + rescaleStartupWeights(innovationStartValue, this.config, event.getServices().getStrategyManager(), av.getSubpopulation()); } - configValue = getStrategyWeights(this.config, av.getDefaultSubpopulation(), StratType.allInnovation); + configValue = getStrategyWeights(this.config, av.getSubpopulation(), StratType.allInnovation); break; default: throw new IllegalArgumentException(); diff --git a/matsim/src/main/java/org/matsim/core/replanning/annealing/ReplanningAnnealerConfigGroup.java b/matsim/src/main/java/org/matsim/core/replanning/annealing/ReplanningAnnealerConfigGroup.java index c3a9421862f..3c13879d7ca 100644 --- a/matsim/src/main/java/org/matsim/core/replanning/annealing/ReplanningAnnealerConfigGroup.java +++ b/matsim/src/main/java/org/matsim/core/replanning/annealing/ReplanningAnnealerConfigGroup.java @@ -20,7 +20,11 @@ package org.matsim.core.replanning.annealing; import java.util.EnumMap; +import java.util.HashMap; +import java.util.List; import java.util.Map; +import java.util.stream.Collectors; + import org.matsim.core.config.ConfigGroup; import org.matsim.core.config.ReflectiveConfigGroup; @@ -81,27 +85,35 @@ public void addParameterSet(final ConfigGroup set) { addAnnealingVariable((AnnealingVariable) set); } - public Map getAnnealingVariables() { - final EnumMap map = - new EnumMap<>(AnnealParameterOption.class); - for (ConfigGroup pars : getParameterSets(AnnealingVariable.GROUP_NAME)) { - final AnnealParameterOption name = ((AnnealingVariable) pars).getAnnealParameter(); - final AnnealingVariable old = map.put(name, (AnnealingVariable) pars); - if (old != null) { - throw new IllegalStateException("several parameter sets for variable " + name); - } - } - return map; - } + public List getAllAnnealingVariables(){ + return getAnnealingVariablesPerSubpopulation().values().stream().flatMap(a->a.values().stream()).collect(Collectors.toList()); + } + public Map> getAnnealingVariablesPerSubpopulation() { + final EnumMap> map = + new EnumMap<>(AnnealParameterOption.class); + for (ConfigGroup pars : getParameterSets(AnnealingVariable.GROUP_NAME)) { + AnnealParameterOption name = ((AnnealingVariable) pars).getAnnealParameter(); + String subpopulation = ((AnnealingVariable) pars).getSubpopulation(); + var paramsPerSubpopulation = map.computeIfAbsent(name,a->new HashMap<>()); + final AnnealingVariable old = paramsPerSubpopulation.put(subpopulation, (AnnealingVariable) pars); + if (old != null) { + throw new IllegalStateException("several parameter sets for variable " + name + " and subpopulation "+subpopulation); + } + } + return map; + } public void addAnnealingVariable(final AnnealingVariable params) { - final AnnealingVariable previous = this.getAnnealingVariables().get(params.getAnnealParameter()); + var previousMap = this.getAnnealingVariablesPerSubpopulation().get(params.getAnnealParameter()); + if (previousMap!=null){ + AnnealingVariable previous = previousMap.get(params.getSubpopulation()); if (previous != null) { final boolean removed = removeParameterSet(previous); if (!removed) { throw new RuntimeException("problem replacing annealing variable"); } } + } super.addParameterSet(params); } @@ -117,11 +129,11 @@ public static class AnnealingVariable extends ReflectiveConfigGroup { private static final String START_VALUE = "startValue"; private static final String END_VALUE = "endValue"; private static final String ANNEAL_TYPE = "annealType"; - private static final String DEFAULT_SUBPOP = "defaultSubpopulation"; + private static final String SUBPOPULATION = "subpopulation"; private static final String ANNEAL_PARAM = "annealParameter"; private static final String HALFLIFE = "halfLife"; private static final String SHAPE_FACTOR = "shapeFactor"; - private String defaultSubpop = null; + private String subpopulation = null; private Double startValue = null; private double endValue = 0.0001; private double shapeFactor = 0.9; @@ -167,14 +179,14 @@ public void setAnnealType(AnnealOption annealType) { this.annealType = annealType; } - @StringGetter(DEFAULT_SUBPOP) - public String getDefaultSubpopulation() { - return this.defaultSubpop; + @StringGetter(SUBPOPULATION) + public String getSubpopulation() { + return this.subpopulation; } - @StringSetter(DEFAULT_SUBPOP) + @StringSetter(SUBPOPULATION) public void setDefaultSubpopulation(String defaultSubpop) { - this.defaultSubpop = defaultSubpop; + this.subpopulation = defaultSubpop; } @StringGetter(ANNEAL_PARAM) @@ -220,7 +232,7 @@ public Map getComments() { map.put(ANNEAL_TYPE, "options: linear, exponential, geometric, msa, sigmoid and disabled (no annealing)."); map.put(ANNEAL_PARAM, "list of config parameters that shall be annealed. Currently supported: globalInnovationRate, BrainExpBeta, PathSizeLogitBeta, learningRate. Default is globalInnovationRate"); - map.put(DEFAULT_SUBPOP, "subpopulation to have the global innovation rate adjusted. Not applicable when annealing with other parameters."); + map.put(SUBPOPULATION, "subpopulation to have the global innovation rate adjusted. Not applicable when annealing with other parameters."); map.put(START_VALUE, "start value for annealing."); map.put(END_VALUE, "final annealing value. When the annealing function reaches this value, further results remain constant."); return map; diff --git a/matsim/src/test/java/org/matsim/core/replanning/annealing/ReplanningAnnealerTest.java b/matsim/src/test/java/org/matsim/core/replanning/annealing/ReplanningAnnealerTest.java index d16b44098a6..bc83393f8cb 100644 --- a/matsim/src/test/java/org/matsim/core/replanning/annealing/ReplanningAnnealerTest.java +++ b/matsim/src/test/java/org/matsim/core/replanning/annealing/ReplanningAnnealerTest.java @@ -39,6 +39,19 @@ public class ReplanningAnnealerTest { "8;0.1000;0.0500;0.0500;0.9000\n" + "9;0.0500;0.0250;0.0250;0.9500\n" + "10;0.0000;0.0000;0.0000;1.0000\n"; + + private String getExpectedLinearAnnealMultipleSubpopulations = "it;globalInnovationRate_otherAnnealer;ChangeExpBeta_otherAnnealer;TimeAllocationMutator_otherAnnealer;globalInnovationRate_subpop;ReRoute_subpop;SubtourModeChoice_subpop;ChangeExpBeta_subpop\n" + + "0;0.8000;0.2000;0.8000;0.5000;0.2500;0.2500;0.5000\n" + + "1;0.7200;0.2800;0.7200;0.4500;0.2250;0.2250;0.5500\n" + + "2;0.6400;0.3600;0.6400;0.4000;0.2000;0.2000;0.6000\n" + + "3;0.5600;0.4400;0.5600;0.3500;0.1750;0.1750;0.6500\n" + + "4;0.4800;0.5200;0.4800;0.3000;0.1500;0.1500;0.7000\n" + + "5;0.4000;0.6000;0.4000;0.2500;0.1250;0.1250;0.7500\n" + + "6;0.3200;0.6800;0.3200;0.2000;0.1000;0.1000;0.8000\n" + + "7;0.2400;0.7600;0.2400;0.1500;0.0750;0.0750;0.8500\n" + + "8;0.1600;0.8400;0.1600;0.1000;0.0500;0.0500;0.9000\n" + + "9;0.0800;0.9200;0.0800;0.0500;0.0250;0.0250;0.9500\n" + + "10;0.0000;1.0000;0.0000;0.0000;0.0000;0.0000;1.0000\n"; private String expectedMsaAnneal = "it;globalInnovationRate;ReRoute;SubtourModeChoice;ChangeExpBeta\n" + "0;0.5000;0.2500;0.2500;0.5000\n" + @@ -360,21 +373,45 @@ void testSubpopulationAnneal() throws IOException { this.saConfigVar.setStartValue(0.5); this.saConfigVar.setDefaultSubpopulation(targetSubpop); this.config.replanning().getStrategySettings().forEach(s -> s.setSubpopulation(targetSubpop)); - ReplanningConfigGroup.StrategySettings s = new ReplanningConfigGroup.StrategySettings(); - s.setStrategyName("TimeAllocationMutator"); - s.setWeight(0.25); - s.setSubpopulation("noAnneal"); - this.config.replanning().addStrategySettings(s); + + String otherAnnealerSubpopulation = "otherAnnealer"; + String othertargetSubpop = otherAnnealerSubpopulation; + ReplanningAnnealerConfigGroup.AnnealingVariable saConfigVar2 = new ReplanningAnnealerConfigGroup.AnnealingVariable(); + saConfigVar2.setAnnealType("linear"); + saConfigVar2.setEndValue(0.0); + saConfigVar2.setStartValue(0.8); + saConfigVar2.setDefaultSubpopulation(othertargetSubpop); + this.config.replanningAnnealer().addParameterSet(saConfigVar2); + + ReplanningConfigGroup.StrategySettings s = new ReplanningConfigGroup.StrategySettings(); + s.setStrategyName("TimeAllocationMutator"); + s.setWeight(0.25); + s.setSubpopulation(otherAnnealerSubpopulation); + ReplanningConfigGroup.StrategySettings s2 = new ReplanningConfigGroup.StrategySettings(); + s2.setStrategyName("ChangeExpBeta"); // shouldn't be affected + s2.setWeight(0.5); + s2.setSubpopulation(otherAnnealerSubpopulation); + this.config.replanning().addStrategySettings(s2); + this.config.replanning().addStrategySettings(s); + + + ReplanningConfigGroup.StrategySettings noAnnealSettings = new ReplanningConfigGroup.StrategySettings(); + noAnnealSettings.setStrategyName("TimeAllocationMutator"); + noAnnealSettings.setWeight(0.25); + noAnnealSettings.setSubpopulation("noAnneal"); + this.config.replanning().addStrategySettings(noAnnealSettings); Controler controler = new Controler(this.scenario); controler.run(); - Assertions.assertEquals(expectedLinearAnneal, readResult(controler.getControlerIO().getOutputFilename(FILENAME_ANNEAL))); + Assertions.assertEquals(getExpectedLinearAnnealMultipleSubpopulations, readResult(controler.getControlerIO().getOutputFilename(FILENAME_ANNEAL))); StrategyManager sm = controler.getInjector().getInstance(StrategyManager.class); List weights = sm.getWeights(targetSubpop); + List weights2 = sm.getWeights(otherAnnealerSubpopulation); Assertions.assertEquals(1.0, weights.stream().mapToDouble(Double::doubleValue).sum(), 1e-4); + Assertions.assertEquals(1.0, weights2.stream().mapToDouble(Double::doubleValue).sum(), 1e-4); } } From 2644bf9f4835945a315fc3bb5a7bd928c706ed68 Mon Sep 17 00:00:00 2001 From: Joschka Bischoff Date: Thu, 7 Mar 2024 14:53:17 +0100 Subject: [PATCH 2/8] add test with explicitly set null subpopulation --- .../annealing/ReplanningAnnealerTest.java | 24 +++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/matsim/src/test/java/org/matsim/core/replanning/annealing/ReplanningAnnealerTest.java b/matsim/src/test/java/org/matsim/core/replanning/annealing/ReplanningAnnealerTest.java index bc83393f8cb..760d3e430e2 100644 --- a/matsim/src/test/java/org/matsim/core/replanning/annealing/ReplanningAnnealerTest.java +++ b/matsim/src/test/java/org/matsim/core/replanning/annealing/ReplanningAnnealerTest.java @@ -40,7 +40,7 @@ public class ReplanningAnnealerTest { "9;0.0500;0.0250;0.0250;0.9500\n" + "10;0.0000;0.0000;0.0000;1.0000\n"; - private String getExpectedLinearAnnealMultipleSubpopulations = "it;globalInnovationRate_otherAnnealer;ChangeExpBeta_otherAnnealer;TimeAllocationMutator_otherAnnealer;globalInnovationRate_subpop;ReRoute_subpop;SubtourModeChoice_subpop;ChangeExpBeta_subpop\n" + + private String expectedLinearAnnealMultipleSubpopulations = "it;globalInnovationRate_otherAnnealer;ChangeExpBeta_otherAnnealer;TimeAllocationMutator_otherAnnealer;globalInnovationRate_subpop;ReRoute_subpop;SubtourModeChoice_subpop;ChangeExpBeta_subpop\n" + "0;0.8000;0.2000;0.8000;0.5000;0.2500;0.2500;0.5000\n" + "1;0.7200;0.2800;0.7200;0.4500;0.2250;0.2250;0.5500\n" + "2;0.6400;0.3600;0.6400;0.4000;0.2000;0.2000;0.6000\n" + @@ -404,7 +404,7 @@ void testSubpopulationAnneal() throws IOException { Controler controler = new Controler(this.scenario); controler.run(); - Assertions.assertEquals(getExpectedLinearAnnealMultipleSubpopulations, readResult(controler.getControlerIO().getOutputFilename(FILENAME_ANNEAL))); + Assertions.assertEquals(expectedLinearAnnealMultipleSubpopulations, readResult(controler.getControlerIO().getOutputFilename(FILENAME_ANNEAL))); StrategyManager sm = controler.getInjector().getInstance(StrategyManager.class); List weights = sm.getWeights(targetSubpop); @@ -414,4 +414,24 @@ void testSubpopulationAnneal() throws IOException { Assertions.assertEquals(1.0, weights2.stream().mapToDouble(Double::doubleValue).sum(), 1e-4); } + @Test + void testNullSubpopulationAnneal() throws IOException { + String targetSubpop = null; + this.saConfigVar.setAnnealType("linear"); + this.saConfigVar.setEndValue(0.0); + this.saConfigVar.setStartValue(0.5); + this.saConfigVar.setDefaultSubpopulation(targetSubpop); + this.config.replanning().getStrategySettings().forEach(s -> s.setSubpopulation(targetSubpop)); + + Controler controler = new Controler(this.scenario); + controler.run(); + + Assertions.assertEquals(expectedLinearAnneal, readResult(controler.getControlerIO().getOutputFilename(FILENAME_ANNEAL))); + + StrategyManager sm = controler.getInjector().getInstance(StrategyManager.class); + List weights = sm.getWeights(targetSubpop); + + Assertions.assertEquals(1.0, weights.stream().mapToDouble(Double::doubleValue).sum(), 1e-4); + } + } From 95bdede72e0ca054756e73d40475bc9b20789170 Mon Sep 17 00:00:00 2001 From: rakow Date: Mon, 18 Mar 2024 11:25:42 +0100 Subject: [PATCH 3/8] Event fingerprints and comparison (#3165) * started work on fingerprint comparator * added compare function, fixed hashtoadd * fixed compare function, added printFingerprint * changed structure, added compression, changed hash logic * added tests for comparison, added folders for input files * added hash order logic, also javadoc, also fixed minor errors * cleaned up tests, improved API * remove extra new line from error messages * update test * add deprecation message to old enum * handle file errors as separate comparison result * fix capital I in folder name --------- Co-authored-by: zlukich --- .../contrib/bicycle/run/BicycleTest.java | 2 +- .../events/VehicleLeavesTrafficEventTest.java | 6 +- ...unAverageEmissionToolOfflineExampleIT.java | 18 +- .../contrib/ev/example/RunEvExampleTest.java | 10 +- ...nEvExampleWithLTHConsumptionModelTest.java | 10 +- .../usecases/chessboard/RunChessboardIT.java | 6 +- .../run/RunParkingSearchScenarioIT.java | 10 +- .../run/RunWithParkingProxyIT.java | 6 +- .../run/RoadPricingByConfigfileTest.java | 4 +- .../builder/TravelTimeFourWaysTest.java | 5 +- .../signals/integration/SignalSystemsIT.java | 9 +- ...nerateSmallScaleCommercialTrafficTest.java | 6 +- .../contrib/etaxi/run/RunETaxiScenarioIT.java | 6 +- .../taxi/optimizer/TaxiOptimizerTests.java | 6 +- .../java/playground/vsp/ev/UrbanEVIT.java | 6 +- matsim/.gitignore | 2 +- .../org/matsim/core/events/EventsUtils.java | 81 ++++++- .../ComparisonResult.java | 6 + .../EventFingerprint.java | 165 ++++++++++++++ .../EventsFileComparator.java | 38 ++-- .../EventsFileFingerprintComparator.java | 124 ++++++++++ .../FingerprintEventHandler.java | 212 ++++++++++++++++++ .../utils/eventsfilecomparison/Worker.java | 2 + .../java/org/matsim/examples/EquilTest.java | 5 +- .../examples/OnePercentBerlin10sIT.java | 5 +- .../org/matsim/testcases/MatsimTestUtils.java | 3 +- .../EventsFileComparatorTest.java | 2 +- .../EventsFileFingerprintComparatorTest.java | 143 ++++++++++++ .../correct.fp.zst | Bin 0 -> 196 bytes .../events.attribute-order.xml | 14 ++ .../events.diff-hash.xml | 14 ++ .../events.diff-num-timestamps.xml | 12 + .../events.diff-type-count.fp.zst | Bin 0 -> 194 bytes .../events.diff-type-count.xml | 14 ++ .../events.event-order-wrong_logic.xml | 14 ++ .../events.event-order.xml | 14 ++ .../events.one-more-event.xml | 15 ++ .../events_correct.xml | 14 ++ 38 files changed, 918 insertions(+), 91 deletions(-) create mode 100644 matsim/src/main/java/org/matsim/utils/eventsfilecomparison/ComparisonResult.java create mode 100644 matsim/src/main/java/org/matsim/utils/eventsfilecomparison/EventFingerprint.java create mode 100644 matsim/src/main/java/org/matsim/utils/eventsfilecomparison/EventsFileFingerprintComparator.java create mode 100644 matsim/src/main/java/org/matsim/utils/eventsfilecomparison/FingerprintEventHandler.java create mode 100644 matsim/src/test/java/org/matsim/utils/eventsfilecomparison/EventsFileFingerprintComparatorTest.java create mode 100644 matsim/test/input/org/matsim/utils/eventsfilecomparison/EventsFileFingerprintComparatorTest/correct.fp.zst create mode 100644 matsim/test/input/org/matsim/utils/eventsfilecomparison/EventsFileFingerprintComparatorTest/events.attribute-order.xml create mode 100644 matsim/test/input/org/matsim/utils/eventsfilecomparison/EventsFileFingerprintComparatorTest/events.diff-hash.xml create mode 100644 matsim/test/input/org/matsim/utils/eventsfilecomparison/EventsFileFingerprintComparatorTest/events.diff-num-timestamps.xml create mode 100644 matsim/test/input/org/matsim/utils/eventsfilecomparison/EventsFileFingerprintComparatorTest/events.diff-type-count.fp.zst create mode 100644 matsim/test/input/org/matsim/utils/eventsfilecomparison/EventsFileFingerprintComparatorTest/events.diff-type-count.xml create mode 100644 matsim/test/input/org/matsim/utils/eventsfilecomparison/EventsFileFingerprintComparatorTest/events.event-order-wrong_logic.xml create mode 100644 matsim/test/input/org/matsim/utils/eventsfilecomparison/EventsFileFingerprintComparatorTest/events.event-order.xml create mode 100644 matsim/test/input/org/matsim/utils/eventsfilecomparison/EventsFileFingerprintComparatorTest/events.one-more-event.xml create mode 100644 matsim/test/input/org/matsim/utils/eventsfilecomparison/EventsFileFingerprintComparatorTest/events_correct.xml diff --git a/contribs/bicycle/src/test/java/org/matsim/contrib/bicycle/run/BicycleTest.java b/contribs/bicycle/src/test/java/org/matsim/contrib/bicycle/run/BicycleTest.java index 961e43f0182..222834c746b 100644 --- a/contribs/bicycle/src/test/java/org/matsim/contrib/bicycle/run/BicycleTest.java +++ b/contribs/bicycle/src/test/java/org/matsim/contrib/bicycle/run/BicycleTest.java @@ -65,7 +65,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.matsim.utils.eventsfilecomparison.EventsFileComparator.Result.FILES_ARE_EQUAL; +import static org.matsim.utils.eventsfilecomparison.ComparisonResult.FILES_ARE_EQUAL; /** * @author dziemke diff --git a/contribs/emissions/src/test/java/org/matsim/contrib/emissions/events/VehicleLeavesTrafficEventTest.java b/contribs/emissions/src/test/java/org/matsim/contrib/emissions/events/VehicleLeavesTrafficEventTest.java index 2858e572d58..1eacd47330d 100644 --- a/contribs/emissions/src/test/java/org/matsim/contrib/emissions/events/VehicleLeavesTrafficEventTest.java +++ b/contribs/emissions/src/test/java/org/matsim/contrib/emissions/events/VehicleLeavesTrafficEventTest.java @@ -19,7 +19,7 @@ import org.matsim.core.utils.io.IOUtils; import org.matsim.examples.ExamplesUtils; import org.matsim.testcases.MatsimTestUtils; -import org.matsim.utils.eventsfilecomparison.EventsFileComparator; +import org.matsim.utils.eventsfilecomparison.ComparisonResult; import java.net.URL; @@ -81,8 +81,8 @@ public void install(){ throw new RuntimeException(e) ; } final String expected = utils.getClassInputDirectory() + emissionEventsFileName; - EventsFileComparator.Result result = EventsUtils.compareEventsFiles(expected, resultingEvents); - Assertions.assertEquals( EventsFileComparator.Result.FILES_ARE_EQUAL, result); + ComparisonResult result = EventsUtils.compareEventsFiles(expected, resultingEvents); + Assertions.assertEquals( ComparisonResult.FILES_ARE_EQUAL, result); } } diff --git a/contribs/emissions/src/test/java/org/matsim/contrib/emissions/example/RunAverageEmissionToolOfflineExampleIT.java b/contribs/emissions/src/test/java/org/matsim/contrib/emissions/example/RunAverageEmissionToolOfflineExampleIT.java index 951ed2fb237..259b8472549 100644 --- a/contribs/emissions/src/test/java/org/matsim/contrib/emissions/example/RunAverageEmissionToolOfflineExampleIT.java +++ b/contribs/emissions/src/test/java/org/matsim/contrib/emissions/example/RunAverageEmissionToolOfflineExampleIT.java @@ -30,7 +30,7 @@ import org.matsim.core.utils.io.IOUtils; import org.matsim.examples.ExamplesUtils; import org.matsim.testcases.MatsimTestUtils; -import org.matsim.utils.eventsfilecomparison.EventsFileComparator.Result; +import org.matsim.utils.eventsfilecomparison.ComparisonResult; import java.net.URL; @@ -58,8 +58,8 @@ final void testAverage_vehTypeV1() { String expected = utils.getInputDirectory() + RunAverageEmissionToolOfflineExample.emissionEventsFilename; String actual = utils.getOutputDirectory() + RunAverageEmissionToolOfflineExample.emissionEventsFilename; - Result result = EventsUtils.compareEventsFiles( expected, actual ); - Assertions.assertEquals( Result.FILES_ARE_EQUAL, result); + ComparisonResult result = EventsUtils.compareEventsFiles( expected, actual ); + Assertions.assertEquals( ComparisonResult.FILES_ARE_EQUAL, result); } @Test @@ -80,8 +80,8 @@ final void testAverage_vehTypeV2() { String expected = utils.getInputDirectory() + RunAverageEmissionToolOfflineExample.emissionEventsFilename; String actual = utils.getOutputDirectory() + RunAverageEmissionToolOfflineExample.emissionEventsFilename; - Result result = EventsUtils.compareEventsFiles( expected, actual ); - Assertions.assertEquals( Result.FILES_ARE_EQUAL, result); + ComparisonResult result = EventsUtils.compareEventsFiles( expected, actual ); + Assertions.assertEquals( ComparisonResult.FILES_ARE_EQUAL, result); } /** @@ -106,8 +106,8 @@ final void testAverage_vehTypeV2b() { String expected = utils.getInputDirectory() + RunAverageEmissionToolOfflineExample.emissionEventsFilename; String actual = utils.getOutputDirectory() + RunAverageEmissionToolOfflineExample.emissionEventsFilename; - Result result = EventsUtils.compareEventsFiles( expected, actual ); - Assertions.assertEquals( Result.FILES_ARE_EQUAL, result); + ComparisonResult result = EventsUtils.compareEventsFiles( expected, actual ); + Assertions.assertEquals( ComparisonResult.FILES_ARE_EQUAL, result); } @@ -130,7 +130,7 @@ final void testAverage_vehTypeV2_HBEFA4() { String expected = utils.getInputDirectory() + RunAverageEmissionToolOfflineExample.emissionEventsFilename; String actual = utils.getOutputDirectory() + RunAverageEmissionToolOfflineExample.emissionEventsFilename; - Result result = EventsUtils.compareEventsFiles( expected, actual ); - Assertions.assertEquals( Result.FILES_ARE_EQUAL, result); + ComparisonResult result = EventsUtils.compareEventsFiles( expected, actual ); + Assertions.assertEquals( ComparisonResult.FILES_ARE_EQUAL, result); } } diff --git a/contribs/ev/src/test/java/org/matsim/contrib/ev/example/RunEvExampleTest.java b/contribs/ev/src/test/java/org/matsim/contrib/ev/example/RunEvExampleTest.java index e8bb29fea1c..87644405c7d 100644 --- a/contribs/ev/src/test/java/org/matsim/contrib/ev/example/RunEvExampleTest.java +++ b/contribs/ev/src/test/java/org/matsim/contrib/ev/example/RunEvExampleTest.java @@ -10,7 +10,7 @@ import org.matsim.core.events.EventsUtils; import org.matsim.core.population.PopulationUtils; import org.matsim.testcases.MatsimTestUtils; -import org.matsim.utils.eventsfilecomparison.EventsFileComparator; +import org.matsim.utils.eventsfilecomparison.ComparisonResult; public class RunEvExampleTest{ @@ -39,8 +39,8 @@ public class RunEvExampleTest{ { String expected = utils.getInputDirectory() + "/output_events.xml.gz" ; String actual = utils.getOutputDirectory() + "/output_events.xml.gz" ; - EventsFileComparator.Result result = EventsUtils.compareEventsFiles( expected, actual ); - Assertions.assertEquals(EventsFileComparator.Result.FILES_ARE_EQUAL, result); + ComparisonResult result = EventsUtils.compareEventsFiles( expected, actual ); + Assertions.assertEquals(ComparisonResult.FILES_ARE_EQUAL, result); } } catch ( Exception ee ) { @@ -72,8 +72,8 @@ public class RunEvExampleTest{ { String expected = utils.getInputDirectory() + "/output_events.xml.gz" ; String actual = utils.getOutputDirectory() + "/output_events.xml.gz" ; - EventsFileComparator.Result result = EventsUtils.compareEventsFiles( expected, actual ); - Assertions.assertEquals(EventsFileComparator.Result.FILES_ARE_EQUAL, result); + ComparisonResult result = EventsUtils.compareEventsFiles( expected, actual ); + Assertions.assertEquals(ComparisonResult.FILES_ARE_EQUAL, result); } } catch ( Exception ee ) { diff --git a/contribs/ev/src/test/java/org/matsim/contrib/ev/example/RunEvExampleWithLTHConsumptionModelTest.java b/contribs/ev/src/test/java/org/matsim/contrib/ev/example/RunEvExampleWithLTHConsumptionModelTest.java index ffc5f81e296..447b79966ad 100644 --- a/contribs/ev/src/test/java/org/matsim/contrib/ev/example/RunEvExampleWithLTHConsumptionModelTest.java +++ b/contribs/ev/src/test/java/org/matsim/contrib/ev/example/RunEvExampleWithLTHConsumptionModelTest.java @@ -10,11 +10,7 @@ import org.matsim.core.events.EventsUtils; import org.matsim.core.population.PopulationUtils; import org.matsim.testcases.MatsimTestUtils; -import org.matsim.utils.eventsfilecomparison.EventsFileComparator; - -import java.io.File; -import java.net.MalformedURLException; -import java.net.URL; +import org.matsim.utils.eventsfilecomparison.ComparisonResult; public class RunEvExampleWithLTHConsumptionModelTest{ @@ -43,8 +39,8 @@ void runTest(){ { String expected = utils.getInputDirectory() + "/output_events.xml.gz" ; String actual = utils.getOutputDirectory() + "/output_events.xml.gz" ; - EventsFileComparator.Result result = EventsUtils.compareEventsFiles( expected, actual ); - Assertions.assertEquals( EventsFileComparator.Result.FILES_ARE_EQUAL, result ); + ComparisonResult result = EventsUtils.compareEventsFiles( expected, actual ); + Assertions.assertEquals( ComparisonResult.FILES_ARE_EQUAL, result ); } } catch ( Exception ee ) { diff --git a/contribs/freight/src/test/java/org/matsim/freight/carriers/usecases/chessboard/RunChessboardIT.java b/contribs/freight/src/test/java/org/matsim/freight/carriers/usecases/chessboard/RunChessboardIT.java index ced181d3172..b195b3b89d7 100644 --- a/contribs/freight/src/test/java/org/matsim/freight/carriers/usecases/chessboard/RunChessboardIT.java +++ b/contribs/freight/src/test/java/org/matsim/freight/carriers/usecases/chessboard/RunChessboardIT.java @@ -32,7 +32,7 @@ import org.matsim.core.utils.io.IOUtils; import org.matsim.examples.ExamplesUtils; import org.matsim.testcases.MatsimTestUtils; -import org.matsim.utils.eventsfilecomparison.EventsFileComparator; +import org.matsim.utils.eventsfilecomparison.ComparisonResult; public class RunChessboardIT { @@ -60,8 +60,8 @@ void runChessboard() { { String expected = utils.getInputDirectory() + "/output_events.xml.gz" ; String actual = utils.getOutputDirectory() + "/output_events.xml.gz" ; - EventsFileComparator.Result result = EventsUtils.compareEventsFiles( expected, actual ); - Assertions.assertEquals( EventsFileComparator.Result.FILES_ARE_EQUAL, result ); + ComparisonResult result = EventsUtils.compareEventsFiles( expected, actual ); + Assertions.assertEquals( ComparisonResult.FILES_ARE_EQUAL, result ); } } catch (Exception ee ) { ee.printStackTrace(); diff --git a/contribs/parking/src/test/java/org/matsim/contrib/parking/parkingchoice/run/RunParkingSearchScenarioIT.java b/contribs/parking/src/test/java/org/matsim/contrib/parking/parkingchoice/run/RunParkingSearchScenarioIT.java index fa0dc4f5ffd..e3959961195 100644 --- a/contribs/parking/src/test/java/org/matsim/contrib/parking/parkingchoice/run/RunParkingSearchScenarioIT.java +++ b/contribs/parking/src/test/java/org/matsim/contrib/parking/parkingchoice/run/RunParkingSearchScenarioIT.java @@ -33,7 +33,7 @@ import org.matsim.core.events.EventsUtils; import org.matsim.core.population.PopulationUtils; import org.matsim.testcases.MatsimTestUtils; -import org.matsim.utils.eventsfilecomparison.EventsFileComparator; +import org.matsim.utils.eventsfilecomparison.ComparisonResult; /** * @author jbischoff @@ -108,8 +108,8 @@ void testRunParkingDistanceMemoryStrategy() { { String expected = utils.getInputDirectory() + "/output_events.xml.gz"; String actual = utils.getOutputDirectory() + "/output_events.xml.gz"; - EventsFileComparator.Result result = EventsUtils.compareEventsFiles(expected, actual); - Assertions.assertEquals(EventsFileComparator.Result.FILES_ARE_EQUAL, result); + ComparisonResult result = EventsUtils.compareEventsFiles(expected, actual); + Assertions.assertEquals(ComparisonResult.FILES_ARE_EQUAL, result); } } catch (Exception e) { e.printStackTrace(); @@ -146,8 +146,8 @@ void testRunParkingNearestParkingSpotStrategy() { { String expected = utils.getInputDirectory() + "/output_events.xml.gz"; String actual = utils.getOutputDirectory() + "/output_events.xml.gz"; - EventsFileComparator.Result result = EventsUtils.compareEventsFiles(expected, actual); - Assertions.assertEquals(EventsFileComparator.Result.FILES_ARE_EQUAL, result); + ComparisonResult result = EventsUtils.compareEventsFiles(expected, actual); + Assertions.assertEquals(ComparisonResult.FILES_ARE_EQUAL, result); } } catch (Exception e) { e.printStackTrace(); diff --git a/contribs/parking/src/test/java/org/matsim/contrib/parking/parkingproxy/run/RunWithParkingProxyIT.java b/contribs/parking/src/test/java/org/matsim/contrib/parking/parkingproxy/run/RunWithParkingProxyIT.java index 6d17839c44c..e87a4afbaba 100644 --- a/contribs/parking/src/test/java/org/matsim/contrib/parking/parkingproxy/run/RunWithParkingProxyIT.java +++ b/contribs/parking/src/test/java/org/matsim/contrib/parking/parkingproxy/run/RunWithParkingProxyIT.java @@ -30,7 +30,7 @@ import org.matsim.core.utils.io.IOUtils; import org.matsim.examples.ExamplesUtils; import org.matsim.testcases.MatsimTestUtils; -import org.matsim.utils.eventsfilecomparison.EventsFileComparator.Result; +import org.matsim.utils.eventsfilecomparison.ComparisonResult; public class RunWithParkingProxyIT { private static final Logger log = LogManager.getLogger(RunWithParkingProxyIT.class); @@ -50,8 +50,8 @@ void testMain(){ { String expected = utils.getInputDirectory() + "/output_events.xml.gz" ; String actual = utils.getOutputDirectory() + "/output_events.xml.gz" ; - Result result = EventsUtils.compareEventsFiles( expected, actual ); - if(!result.equals(Result.FILES_ARE_EQUAL)) { + ComparisonResult result = EventsUtils.compareEventsFiles( expected, actual ); + if(!result.equals(ComparisonResult.FILES_ARE_EQUAL)) { throw new RuntimeException("Events comparison ended with result " + result.name()); } } diff --git a/contribs/roadpricing/src/test/java/org/matsim/contrib/roadpricing/run/RoadPricingByConfigfileTest.java b/contribs/roadpricing/src/test/java/org/matsim/contrib/roadpricing/run/RoadPricingByConfigfileTest.java index 70b4bc9344d..e59e74dd90b 100644 --- a/contribs/roadpricing/src/test/java/org/matsim/contrib/roadpricing/run/RoadPricingByConfigfileTest.java +++ b/contribs/roadpricing/src/test/java/org/matsim/contrib/roadpricing/run/RoadPricingByConfigfileTest.java @@ -32,7 +32,7 @@ import org.matsim.core.utils.io.IOUtils; import org.matsim.examples.ExamplesUtils; import org.matsim.testcases.MatsimTestUtils; -import org.matsim.utils.eventsfilecomparison.EventsFileComparator; +import org.matsim.utils.eventsfilecomparison.ComparisonResult; /** * @author vsp-gleich @@ -55,7 +55,7 @@ final void testMain() { { String expected = utils.getInputDirectory() + "/output_events.xml.gz" ; String actual = utils.getOutputDirectory() + "/output_events.xml.gz" ; - Assertions.assertEquals(EventsFileComparator.Result.FILES_ARE_EQUAL, EventsUtils.compareEventsFiles( expected, actual )); + Assertions.assertEquals(ComparisonResult.FILES_ARE_EQUAL, EventsUtils.compareEventsFiles( expected, actual )); } { final Population expected = PopulationUtils.createPopulation( ConfigUtils.createConfig() ); diff --git a/contribs/signals/src/test/java/org/matsim/contrib/signals/builder/TravelTimeFourWaysTest.java b/contribs/signals/src/test/java/org/matsim/contrib/signals/builder/TravelTimeFourWaysTest.java index 9e341d444a6..ba97489f89e 100644 --- a/contribs/signals/src/test/java/org/matsim/contrib/signals/builder/TravelTimeFourWaysTest.java +++ b/contribs/signals/src/test/java/org/matsim/contrib/signals/builder/TravelTimeFourWaysTest.java @@ -38,8 +38,7 @@ import org.matsim.core.scenario.ScenarioUtils; import org.matsim.testcases.MatsimTestUtils; import org.matsim.utils.eventsfilecomparison.EventsFileComparator; - -import static org.matsim.utils.eventsfilecomparison.EventsFileComparator.*; +import org.matsim.utils.eventsfilecomparison.ComparisonResult; /** * @author aneumann @@ -125,7 +124,7 @@ private void runQSimWithSignals(final Scenario scenario) { eventsXmlWriter.closeFile(); // Assert.assertEquals("different events files", EventsFileComparator.compareAndReturnInt(this.testUtils.getInputDirectory() + EVENTSFILE, eventsOut), 0); - Assertions.assertEquals( Result.FILES_ARE_EQUAL, new EventsFileComparator().setIgnoringCoordinates( true ).runComparison( this.testUtils.getInputDirectory() + EVENTSFILE, eventsOut ) ); + Assertions.assertEquals( ComparisonResult.FILES_ARE_EQUAL, new EventsFileComparator().setIgnoringCoordinates( true ).runComparison( this.testUtils.getInputDirectory() + EVENTSFILE, eventsOut ) ); } } diff --git a/contribs/signals/src/test/java/org/matsim/contrib/signals/integration/SignalSystemsIT.java b/contribs/signals/src/test/java/org/matsim/contrib/signals/integration/SignalSystemsIT.java index 49e8af1f140..b44a5cf35ea 100644 --- a/contribs/signals/src/test/java/org/matsim/contrib/signals/integration/SignalSystemsIT.java +++ b/contribs/signals/src/test/java/org/matsim/contrib/signals/integration/SignalSystemsIT.java @@ -36,6 +36,7 @@ import org.matsim.core.scenario.ScenarioUtils; import org.matsim.testcases.MatsimTestUtils; import org.matsim.utils.eventsfilecomparison.EventsFileComparator; +import org.matsim.utils.eventsfilecomparison.ComparisonResult; import java.io.File; @@ -88,7 +89,7 @@ void testSignalSystems() { //iteration 0 String iterationOutput = controlerOutputDir + "ITERS/it.0/"; - Assertions.assertEquals(EventsFileComparator.Result.FILES_ARE_EQUAL, + Assertions.assertEquals(ComparisonResult.FILES_ARE_EQUAL, new EventsFileComparator().setIgnoringCoordinates( true ).runComparison( inputDirectory + "0.events.xml.gz", iterationOutput + "0.events.xml.gz"), "different events files after iteration 0 " @@ -113,7 +114,7 @@ void testSignalSystems() { //iteration 10 String iterationOutput = controlerOutputDir + "ITERS/it.10/"; - Assertions.assertEquals(EventsFileComparator.Result.FILES_ARE_EQUAL, + Assertions.assertEquals(ComparisonResult.FILES_ARE_EQUAL, new EventsFileComparator().setIgnoringCoordinates( true ).runComparison( inputDirectory + "10.events.xml.gz", iterationOutput + "10.events.xml.gz" ), "different event files after iteration 10" ); @@ -181,7 +182,7 @@ void testSignalSystemsWTryEndTimeThenDuration() { //iteration 0 String iterationOutput = controlerOutputDir + "ITERS/it.0/"; - Assertions.assertEquals(EventsFileComparator.Result.FILES_ARE_EQUAL, + Assertions.assertEquals(ComparisonResult.FILES_ARE_EQUAL, new EventsFileComparator().setIgnoringCoordinates( true ).runComparison( inputDirectory + "0.events.xml.gz", iterationOutput + "0.events.xml.gz"), "different events files after iteration 0 " @@ -206,7 +207,7 @@ void testSignalSystemsWTryEndTimeThenDuration() { //iteration 10 String iterationOutput = controlerOutputDir + "ITERS/it.10/"; - Assertions.assertEquals(EventsFileComparator.Result.FILES_ARE_EQUAL, + Assertions.assertEquals(ComparisonResult.FILES_ARE_EQUAL, new EventsFileComparator().setIgnoringCoordinates( true ).runComparison( inputDirectory + "10.events.xml.gz", iterationOutput + "10.events.xml.gz"), "different event files after iteration 10" ); diff --git a/contribs/small-scale-traffic-generation/src/test/java/org/matsim/smallScaleCommercialTrafficGeneration/RunGenerateSmallScaleCommercialTrafficTest.java b/contribs/small-scale-traffic-generation/src/test/java/org/matsim/smallScaleCommercialTrafficGeneration/RunGenerateSmallScaleCommercialTrafficTest.java index d88d2318fd1..b5e92503851 100644 --- a/contribs/small-scale-traffic-generation/src/test/java/org/matsim/smallScaleCommercialTrafficGeneration/RunGenerateSmallScaleCommercialTrafficTest.java +++ b/contribs/small-scale-traffic-generation/src/test/java/org/matsim/smallScaleCommercialTrafficGeneration/RunGenerateSmallScaleCommercialTrafficTest.java @@ -34,7 +34,7 @@ import org.matsim.freight.carriers.CarriersUtils; import org.matsim.freight.carriers.FreightCarriersConfigGroup; import org.matsim.testcases.MatsimTestUtils; -import org.matsim.utils.eventsfilecomparison.EventsFileComparator; +import org.matsim.utils.eventsfilecomparison.ComparisonResult; import java.io.File; import java.util.Objects; @@ -119,7 +119,7 @@ void testMainRunAndResults() { // compare events String expected = utils.getPackageInputDirectory() + "test.output_events.xml.gz" ; String actual = utils.getOutputDirectory() + "test.output_events.xml.gz" ; - EventsFileComparator.Result result = EventsUtils.compareEventsFiles( expected, actual ); - Assertions.assertEquals( EventsFileComparator.Result.FILES_ARE_EQUAL, result ); + ComparisonResult result = EventsUtils.compareEventsFiles( expected, actual ); + Assertions.assertEquals( ComparisonResult.FILES_ARE_EQUAL, result ); } } diff --git a/contribs/taxi/src/test/java/org/matsim/contrib/etaxi/run/RunETaxiScenarioIT.java b/contribs/taxi/src/test/java/org/matsim/contrib/etaxi/run/RunETaxiScenarioIT.java index c086adedc50..5e029c8a29e 100644 --- a/contribs/taxi/src/test/java/org/matsim/contrib/etaxi/run/RunETaxiScenarioIT.java +++ b/contribs/taxi/src/test/java/org/matsim/contrib/etaxi/run/RunETaxiScenarioIT.java @@ -29,7 +29,7 @@ import org.matsim.core.utils.io.IOUtils; import org.matsim.examples.ExamplesUtils; import org.matsim.testcases.MatsimTestUtils; -import org.matsim.utils.eventsfilecomparison.EventsFileComparator; +import org.matsim.utils.eventsfilecomparison.ComparisonResult; /** * @author michalm @@ -73,8 +73,8 @@ private void runScenario(String configPath) { { String expected = utils.getInputDirectory() + "/output_events.xml.gz"; String actual = utils.getOutputDirectory() + "/output_events.xml.gz"; - EventsFileComparator.Result result = EventsUtils.compareEventsFiles(expected, actual); - Assertions.assertEquals(EventsFileComparator.Result.FILES_ARE_EQUAL, result); + ComparisonResult result = EventsUtils.compareEventsFiles(expected, actual); + Assertions.assertEquals(ComparisonResult.FILES_ARE_EQUAL, result); } } } diff --git a/contribs/taxi/src/test/java/org/matsim/contrib/taxi/optimizer/TaxiOptimizerTests.java b/contribs/taxi/src/test/java/org/matsim/contrib/taxi/optimizer/TaxiOptimizerTests.java index ca985585f82..3a3800467af 100644 --- a/contribs/taxi/src/test/java/org/matsim/contrib/taxi/optimizer/TaxiOptimizerTests.java +++ b/contribs/taxi/src/test/java/org/matsim/contrib/taxi/optimizer/TaxiOptimizerTests.java @@ -35,7 +35,7 @@ import org.matsim.core.utils.io.IOUtils; import org.matsim.examples.ExamplesUtils; import org.matsim.testcases.MatsimTestUtils; -import org.matsim.utils.eventsfilecomparison.EventsFileComparator; +import org.matsim.utils.eventsfilecomparison.ComparisonResult; public class TaxiOptimizerTests { public static void runBenchmark(boolean vehicleDiversion, AbstractTaxiOptimizerParams taxiOptimizerParams, MatsimTestUtils utils) { @@ -70,8 +70,8 @@ public static void runBenchmark(boolean vehicleDiversion, AbstractTaxiOptimizerP { String expected = utils.getInputDirectory() + "/output_events.xml.gz"; String actual = utils.getOutputDirectory() + "/output_events.xml.gz"; - EventsFileComparator.Result result = EventsUtils.compareEventsFiles(expected, actual); - Assertions.assertEquals(EventsFileComparator.Result.FILES_ARE_EQUAL, result); + ComparisonResult result = EventsUtils.compareEventsFiles(expected, actual); + Assertions.assertEquals(ComparisonResult.FILES_ARE_EQUAL, result); } } } diff --git a/contribs/vsp/src/test/java/playground/vsp/ev/UrbanEVIT.java b/contribs/vsp/src/test/java/playground/vsp/ev/UrbanEVIT.java index 823fb89bf83..b96c04fb079 100644 --- a/contribs/vsp/src/test/java/playground/vsp/ev/UrbanEVIT.java +++ b/contribs/vsp/src/test/java/playground/vsp/ev/UrbanEVIT.java @@ -9,7 +9,7 @@ import org.matsim.core.events.EventsUtils; import org.matsim.core.population.PopulationUtils; import org.matsim.testcases.MatsimTestUtils; -import org.matsim.utils.eventsfilecomparison.EventsFileComparator; +import org.matsim.utils.eventsfilecomparison.ComparisonResult; public class UrbanEVIT { @RegisterExtension private MatsimTestUtils utils = new MatsimTestUtils(); @@ -38,8 +38,8 @@ void run() { { String expected = utils.getInputDirectory() + "/output_events.xml.gz" ; String actual = utils.getOutputDirectory() + "/output_events.xml.gz" ; - EventsFileComparator.Result result = EventsUtils.compareEventsFiles( expected, actual ); - Assertions.assertEquals( EventsFileComparator.Result.FILES_ARE_EQUAL, result ); + ComparisonResult result = EventsUtils.compareEventsFiles( expected, actual ); + Assertions.assertEquals( ComparisonResult.FILES_ARE_EQUAL, result ); } } catch ( Exception ee ) { diff --git a/matsim/.gitignore b/matsim/.gitignore index 029e60cc779..a35b8f59ee1 100644 --- a/matsim/.gitignore +++ b/matsim/.gitignore @@ -3,7 +3,6 @@ test/output bin .settings -bin target output out @@ -11,3 +10,4 @@ src/main/java/.gitignore /output_fastCapacityUpdate_false/ /output_fastCapacityUpdate_true/ /nullevents.xml.gz +*.zst \ No newline at end of file diff --git a/matsim/src/main/java/org/matsim/core/events/EventsUtils.java b/matsim/src/main/java/org/matsim/core/events/EventsUtils.java index 1f2ba7e3d33..7601a9091ad 100644 --- a/matsim/src/main/java/org/matsim/core/events/EventsUtils.java +++ b/matsim/src/main/java/org/matsim/core/events/EventsUtils.java @@ -21,19 +21,26 @@ package org.matsim.core.events; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.matsim.core.api.experimental.events.EventsManager; import org.matsim.core.config.Config; import org.matsim.core.controler.Injector; -import org.matsim.utils.eventsfilecomparison.EventsFileComparator; +import org.matsim.utils.eventsfilecomparison.*; + +import java.io.File; public final class EventsUtils { + private static final Logger log = LogManager.getLogger(EventsUtils.class); + + /** * Create a events manager instance that guarantees causality of processed events across all handlers. */ - public static EventsManager createEventsManager() { + public static EventsManager createEventsManager() { return new EventsManagerImpl(); - } + } /** * Creates a parallel events manager, with no guarantees for the order of processed events between multiple handlers. @@ -43,11 +50,16 @@ public static EventsManager createParallelEventsManager() { } public static EventsManager createEventsManager(Config config) { - final EventsManager events = Injector.createInjector( config, new EventsManagerModule() ).getInstance( EventsManager.class ); + final EventsManager events = Injector.createInjector(config, new EventsManagerModule()).getInstance(EventsManager.class); // events.initProcessing(); return events; } + public static void readEvents(EventsManager events, String filename) { + new MatsimEventsReader(events).readFile(filename); + } + + /** * The SimStepParallelEventsManagerImpl can handle events from multiple threads. * The (Parallel)EventsMangerImpl cannot, therefore it has to be wrapped into a @@ -58,21 +70,68 @@ public static EventsManager getParallelFeedableInstance(EventsManager events) { return events; } else if (events instanceof ParallelEventsManager) { return events; - } - else if (events instanceof SynchronizedEventsManagerImpl) { + } else if (events instanceof SynchronizedEventsManagerImpl) { return events; } else { return new SynchronizedEventsManagerImpl(events); } } - public static void readEvents( EventsManager events, String filename ) { - new MatsimEventsReader(events).readFile(filename) ; + /** + * Create and write fingerprint file for events. + */ + public static FingerprintEventHandler createEventsFingerprint(String eventFile, String outputFingerprintFile) { + + FingerprintEventHandler handler = EventsFileFingerprintComparator.createFingerprintHandler(eventFile, null); + + EventFingerprint.write(outputFingerprintFile, handler.getEventFingerprint()); + + return handler; + } + + + /** + * Compares existing event file against fingerprint file. This will also create new fingerprint file along the input events. + * + * @param eventFile local events file + * @param compareFingerprintFile path or uri to fingerprint file + * + * @return comparison results + */ + public static ComparisonResult createAndCompareEventsFingerprint(File eventFile, String compareFingerprintFile) { + + String path = eventFile.getPath().replaceFirst("\\.xml[.a-zA-z0-9]*$", ""); + + FingerprintEventHandler handler = EventsFileFingerprintComparator.createFingerprintHandler(eventFile.toString(), compareFingerprintFile); + EventFingerprint.write(path + ".fp.zst", handler.getEventFingerprint()); + + if (handler.getComparisonMessage() != null) + log.warn(handler.getComparisonMessage()); + + return handler.getComparisonResult(); + } + + /** + * Compares existing event file against fingerprint file. This will also create new fingerprint file along the input events. + * + * @throws AssertionError if comparison fails + * @see #createAndCompareEventsFingerprint(File, String) + */ + public static void assertEqualEventsFingerprint(File eventFile, String compareFingerprintFile) { + + String path = eventFile.getPath().replaceFirst("\\.xml[.a-zA-z0-9]*$", ""); + + FingerprintEventHandler handler = EventsFileFingerprintComparator.createFingerprintHandler(eventFile.toString(), compareFingerprintFile); + EventFingerprint.write(path + ".fp.zst", handler.getEventFingerprint()); + + + if (handler.getComparisonResult() != ComparisonResult.FILES_ARE_EQUAL) + throw new AssertionError(handler.getComparisonMessage()); + } - public static EventsFileComparator.Result compareEventsFiles( String filename1, String filename2 ) { - EventsFileComparator.Result result = EventsFileComparator.compare( filename1, filename2 ); - return result ; + public static ComparisonResult compareEventsFiles(String filename1, String filename2) { + return EventsFileComparator.compare(filename1, filename2); } } diff --git a/matsim/src/main/java/org/matsim/utils/eventsfilecomparison/ComparisonResult.java b/matsim/src/main/java/org/matsim/utils/eventsfilecomparison/ComparisonResult.java new file mode 100644 index 00000000000..b5e10fa2d65 --- /dev/null +++ b/matsim/src/main/java/org/matsim/utils/eventsfilecomparison/ComparisonResult.java @@ -0,0 +1,6 @@ +package org.matsim.utils.eventsfilecomparison; + +/** + * Result of event file comparison. + */ +public enum ComparisonResult {FILES_ARE_EQUAL, DIFFERENT_NUMBER_OF_TIMESTEPS, DIFFERENT_TIMESTEPS, DIFFERENT_EVENT_ATTRIBUTES, MISSING_EVENT, WRONG_EVENT_COUNT, FILE_ERROR} diff --git a/matsim/src/main/java/org/matsim/utils/eventsfilecomparison/EventFingerprint.java b/matsim/src/main/java/org/matsim/utils/eventsfilecomparison/EventFingerprint.java new file mode 100644 index 00000000000..74f254f746d --- /dev/null +++ b/matsim/src/main/java/org/matsim/utils/eventsfilecomparison/EventFingerprint.java @@ -0,0 +1,165 @@ +package org.matsim.utils.eventsfilecomparison; + +import it.unimi.dsi.fastutil.floats.FloatArrayList; +import it.unimi.dsi.fastutil.floats.FloatList; +import it.unimi.dsi.fastutil.objects.Object2IntMap; +import it.unimi.dsi.fastutil.objects.Object2IntOpenHashMap; +import org.matsim.core.utils.io.IOUtils; + +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.Arrays; +import java.util.Map; + +/** + * Class holding reduced information about an events file. + * If two such fingerprint are different, one can conclude that the event files are semantically different. + *

+ * The fingerprint is based on the following information: + * - Array of all timestamps + * - Counts of each event type + * - Hash of string concatenation of all event strings + * + * Note: Events with the same timestamp are allowed to occur in any order. + */ +public final class EventFingerprint { + + /** + * Header for version 1, FP/1 + */ + static final int HEADER_V1 = 0x46502f31; + + final FloatList timeArray; + final Object2IntMap eventTypeCounter; + final byte[] hash; + + /** + * Builder for the hash. + */ + private final MessageDigest digest; + + private EventFingerprint(FloatList timeArray, Object2IntMap eventTypeCounter, byte[] hash) { + this.timeArray = timeArray; + this.eventTypeCounter = eventTypeCounter; + this.hash = hash; + this.digest = null; + } + + EventFingerprint() { + this.timeArray = new FloatArrayList(); + this.eventTypeCounter = new Object2IntOpenHashMap<>(); + this.hash = new byte[20]; + + try { + digest = MessageDigest.getInstance("SHA-1"); + } catch (NoSuchAlgorithmException e) { + throw new IllegalStateException("Hashing not supported;"); + } + } + + public static void write(String filePath, EventFingerprint eventFingerprint) { + try (DataOutputStream dataOutputStream = new DataOutputStream(IOUtils.getOutputStream(IOUtils.getFileUrl(filePath), false))) { + // Write header and version + dataOutputStream.writeInt(EventFingerprint.HEADER_V1); + + // Write time array size and elements + dataOutputStream.writeInt(eventFingerprint.timeArray.size()); + for (float time : eventFingerprint.timeArray) { + dataOutputStream.writeFloat(time); + } + + // Write event type counter map size and elements + dataOutputStream.writeInt(eventFingerprint.eventTypeCounter.size()); + for (Map.Entry entry : eventFingerprint.eventTypeCounter.entrySet()) { + dataOutputStream.writeUTF(entry.getKey()); + dataOutputStream.writeInt(entry.getValue()); + } + + // Hash should always be computed at this point + assert !Arrays.equals(eventFingerprint.hash, new byte[20]) : "Hash was not computed"; + + // Write byte hash + dataOutputStream.write(eventFingerprint.hash); + + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + public static EventFingerprint read(String fingerprintPath) throws IOException { + EventFingerprint eventFingerprint; + + try (DataInputStream dataInputStream = new DataInputStream(IOUtils.getInputStream(IOUtils.getFileUrl(fingerprintPath)))) { + // Read header and version + int fileHeader = dataInputStream.readInt(); + + if (fileHeader != EventFingerprint.HEADER_V1) { + throw new IllegalArgumentException("Invalid fingerprint file header"); + } + + // Read time array + int timeArraySize = dataInputStream.readInt(); + FloatList timeArray = new FloatArrayList(); + for (int i = 0; i < timeArraySize; i++) { + timeArray.add(dataInputStream.readFloat()); + } + + // Read event type counter map + int eventTypeCounterSize = dataInputStream.readInt(); + Object2IntMap eventTypeCounter = new Object2IntOpenHashMap<>(); + for (int i = 0; i < eventTypeCounterSize; i++) { + String eventType = dataInputStream.readUTF(); + int count = dataInputStream.readInt(); + eventTypeCounter.put(eventType, count); + } + + // Read string hash + byte[] hash = dataInputStream.readNBytes(20); + + // Create EventFingerprint object + eventFingerprint = new EventFingerprint(timeArray, eventTypeCounter, hash); + } + + return eventFingerprint; + } + + void addTimeStamp(double timestamp) { + timeArray.add((float) timestamp); + } + + void addEventType(String str) { + // Increment the count for the given string + eventTypeCounter.mergeInt(str, 1, Integer::sum); + } + + void addHashCode(String stringToAdd) { + if (stringToAdd == null) { + return; + } + + digest.update(stringToAdd.getBytes(StandardCharsets.UTF_8)); + } + + byte[] computeHash() { + if (this.digest == null) + throw new IllegalStateException("Hash was from from input and can not be computed"); + + byte[] digest = this.digest.digest(); + System.arraycopy(digest, 0, hash, 0, hash.length); + return hash; + } + + @Override + public String toString() { + return "EventFingerprint{" + + "timeArray=" + timeArray.size() + + ", eventTypeCounter=" + eventTypeCounter + + ", hash=" + Arrays.toString(hash) + + '}'; + } +} diff --git a/matsim/src/main/java/org/matsim/utils/eventsfilecomparison/EventsFileComparator.java b/matsim/src/main/java/org/matsim/utils/eventsfilecomparison/EventsFileComparator.java index 66c44705ca7..0b759f11e95 100644 --- a/matsim/src/main/java/org/matsim/utils/eventsfilecomparison/EventsFileComparator.java +++ b/matsim/src/main/java/org/matsim/utils/eventsfilecomparison/EventsFileComparator.java @@ -37,8 +37,6 @@ public final class EventsFileComparator { private static final Logger log = LogManager.getLogger(EventsFileComparator.class); - public enum Result { FILES_ARE_EQUAL, DIFFERENT_NUMBER_OF_TIMESTEPS, DIFFERENT_TIMESTEPS, MISSING_EVENT, WRONG_EVENT_COUNT } - private boolean ignoringCoordinates = false; public EventsFileComparator setIgnoringCoordinates( boolean ignoringCoordinates ){ this.ignoringCoordinates = ignoringCoordinates; @@ -62,13 +60,13 @@ public static void main(String[] args) { * * @param filename1 name of the first event file * @param filename2 name of the second event file - * @return Result.FILES_ARE_EQUAL if the events files are equal, or some error code (see {@link Result}) if not. + * @return Result.FILES_ARE_EQUAL if the events files are equal, or some error code (see {@link ComparisonResult}) if not. */ - public static Result compare(final String filename1, final String filename2) { + public static ComparisonResult compare(final String filename1, final String filename2) { return new EventsFileComparator().runComparison( filename1, filename2 ); } - public Result runComparison( final String filename1, final String filename2 ) { + public ComparisonResult runComparison(final String filename1, final String filename2 ) { // (need method name different from pre-existing static method. kai, feb'20) EventsComparator comparator = new EventsComparator( ); @@ -90,8 +88,8 @@ public Result runComparison( final String filename1, final String filename2 ) { e.printStackTrace(); } - Result retCode = comparator.retCode; - if (retCode == Result.FILES_ARE_EQUAL) { + ComparisonResult retCode = comparator.retCode; + if (retCode == ComparisonResult.FILES_ARE_EQUAL) { log.info("Event files are semantically equivalent."); } else { log.warn("Event files differ."); @@ -103,7 +101,7 @@ private static class EventsComparator implements Runnable { private Worker worker1 = null; private Worker worker2 = null; - private volatile Result retCode = null ; + private volatile ComparisonResult retCode = null ; /*package*/ void setWorkers(final Worker w1, final Worker w2) { this.worker1 = w1; @@ -114,13 +112,13 @@ private static class EventsComparator implements Runnable { public void run() { if (this.worker1.getCurrentTime() != this.worker2.getCurrentTime()) { log.warn("Differnt time steps in event files!"); - setExitCode(Result.DIFFERENT_TIMESTEPS); + setExitCode(ComparisonResult.DIFFERENT_TIMESTEPS); return; } if (this.worker1.isFinished() != this.worker2.isFinished()) { log.warn("Events files have different number of time steps!"); - setExitCode(Result.DIFFERENT_NUMBER_OF_TIMESTEPS); + setExitCode(ComparisonResult.DIFFERENT_NUMBER_OF_TIMESTEPS); return; } @@ -141,7 +139,7 @@ public void run() { log.warn("The event:"); log.warn(entry.getKey()); log.warn("is missing in events file:" + worker2.getEventsFile()); - setExitCode(Result.MISSING_EVENT); + setExitCode(ComparisonResult.MISSING_EVENT); problem = true; if (logCounter == 50) { log.warn(Gbl.FUTURE_SUPPRESSED); @@ -152,7 +150,7 @@ public void run() { log.warn( "Wrong event count for: " + entry.getKey() + "\n" + entry.getValue().getCount() + " times in file:" + worker1.getEventsFile() + "\n" + counter.getCount() + " times in file:" + worker2.getEventsFile() ); - setExitCode( Result.WRONG_EVENT_COUNT ); + setExitCode( ComparisonResult.WRONG_EVENT_COUNT ); problem = true; } } @@ -168,7 +166,7 @@ public void run() { log.warn("The event:"); log.warn(e.getKey()); log.warn("is missing in events file:" + worker1.getEventsFile()); - setExitCode(Result.MISSING_EVENT); + setExitCode(ComparisonResult.MISSING_EVENT); problem = true; if (logCounter == 50) { log.warn(Gbl.FUTURE_SUPPRESSED); @@ -182,17 +180,25 @@ public void run() { } if (this.worker1.isFinished()) { - setExitCode(Result.FILES_ARE_EQUAL); + setExitCode(ComparisonResult.FILES_ARE_EQUAL); } } - private void setExitCode(final Result errCode) { + private void setExitCode(final ComparisonResult errCode) { this.retCode= errCode; - if (errCode != Result.FILES_ARE_EQUAL) { + if (errCode != ComparisonResult.FILES_ARE_EQUAL) { this.worker1.interrupt(); this.worker2.interrupt(); } } } + /** + * Don't use this enum. See deprecation message. + * @deprecated Use {@link ComparisonResult} instead. This enum is not used anymore and empty now. + */ + @Deprecated + public enum Result { + } + } diff --git a/matsim/src/main/java/org/matsim/utils/eventsfilecomparison/EventsFileFingerprintComparator.java b/matsim/src/main/java/org/matsim/utils/eventsfilecomparison/EventsFileFingerprintComparator.java new file mode 100644 index 00000000000..2152758b215 --- /dev/null +++ b/matsim/src/main/java/org/matsim/utils/eventsfilecomparison/EventsFileFingerprintComparator.java @@ -0,0 +1,124 @@ +package org.matsim.utils.eventsfilecomparison; + +import it.unimi.dsi.fastutil.objects.Object2IntMap; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.matsim.core.api.experimental.events.EventsManager; +import org.matsim.core.events.EventsUtils; + +import javax.annotation.Nullable; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.Arrays; + +/** + * Utility class for comparing events and fingerprints. + */ +public final class EventsFileFingerprintComparator { + + private static final Logger log = LogManager.getLogger(EventsFileComparator.class); + + private EventsFileFingerprintComparator() { + } + + + /** + * Create and compare event fingerprints and return the handler holding resulting information. + */ + public static FingerprintEventHandler createFingerprintHandler(final String eventsfile, @Nullable String compareFingerprint) { + + EventFingerprint fp = null; + Exception err = null; + if (compareFingerprint != null) { + try { + fp = EventFingerprint.read(compareFingerprint); + } catch (Exception e) { + log.warn("Could not read compare fingerprint from file: {}", compareFingerprint, e); + fp = new EventFingerprint(); + err = e; + } + } + + FingerprintEventHandler handler = new FingerprintEventHandler(fp); + + EventsManager manager = EventsUtils.createEventsManager(); + + manager.addHandler(handler); + + EventsUtils.readEvents(manager, eventsfile); + + manager.finishProcessing(); + handler.finishProcessing(); + + // File error overwrite any other error + if (err != null) { + handler.setComparisonResult(ComparisonResult.FILE_ERROR); + handler.setComparisonMessage(err.getMessage()); + } + + return handler; + } + + public static ComparisonResult compareFingerprints(final String fp1, final String fp2) { + + EventFingerprint fingerprint1; + EventFingerprint fingerprint2; + try { + fingerprint1 = EventFingerprint.read(fp1); + fingerprint2 = EventFingerprint.read(fp2); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + + String logMessage = ""; + //Check if time array size is the same + if (fingerprint1.timeArray.size() != fingerprint2.timeArray.size()) { + logMessage = "Different number of timesteps"; + log.warn(logMessage); + return ComparisonResult.DIFFERENT_NUMBER_OF_TIMESTEPS; + } + + //Check if both time arrays have the same timesteps + if (!Arrays.equals(fingerprint1.timeArray.toFloatArray(), fingerprint2.timeArray.toFloatArray())) { + logMessage = "Different timesteps"; + log.warn(logMessage); + return ComparisonResult.DIFFERENT_TIMESTEPS; + } + + + //Check which event type counts are different among 2 fingerprints + boolean countDiffers = false; + for (Object2IntMap.Entry entry1 : fingerprint1.eventTypeCounter.object2IntEntrySet()) { + String key = entry1.getKey(); + int count1 = entry1.getIntValue(); + int count2 = fingerprint2.eventTypeCounter.getInt(key); + if (count1 != count2) { + countDiffers = true; + + if (!logMessage.isEmpty()) + logMessage += "\n"; + + logMessage += "Count for event type '%s' differs: %d != %d".formatted(key, count1, count2); + } + } + if (countDiffers) { + log.warn(logMessage); + return ComparisonResult.WRONG_EVENT_COUNT; + } + + + //Check if total hash is the same + byte[] hash1 = fingerprint1.hash; + byte[] hash2 = fingerprint2.hash; + if (!Arrays.equals(hash1, hash2)) { + + logMessage = String.format("Difference occurred hash codes hash of first file is %s, hash of second is %s", Arrays.toString(hash1), Arrays.toString(hash2)); + + log.warn(logMessage); + return ComparisonResult.DIFFERENT_EVENT_ATTRIBUTES; + } + + return ComparisonResult.FILES_ARE_EQUAL; + + } +} diff --git a/matsim/src/main/java/org/matsim/utils/eventsfilecomparison/FingerprintEventHandler.java b/matsim/src/main/java/org/matsim/utils/eventsfilecomparison/FingerprintEventHandler.java new file mode 100644 index 00000000000..8e12c10a53c --- /dev/null +++ b/matsim/src/main/java/org/matsim/utils/eventsfilecomparison/FingerprintEventHandler.java @@ -0,0 +1,212 @@ +package org.matsim.utils.eventsfilecomparison; + +import it.unimi.dsi.fastutil.floats.FloatListIterator; +import it.unimi.dsi.fastutil.objects.Object2IntMap; +import org.matsim.api.core.v01.events.Event; +import org.matsim.core.events.handler.BasicEventHandler; + +import java.util.*; + +/** + * Handler for creating and comparing {@link EventFingerprint}. + */ +public final class FingerprintEventHandler implements BasicEventHandler { + + /** + * Generated finger print. + */ + private final EventFingerprint eventFingerprint = new EventFingerprint(); + + /** + * Precision for timestamp comparison. + */ + private static final float EPS = 1e-8f; + + /** + * Accumulate all event strings for same timestamps. + */ + private final List hashAccumulationList = new ArrayList<>(); + + /** + * Existing fingerprint for comparison against event file. Can be null, then no comparison is performed. + */ + private final EventFingerprint compareFingerprint; + + private FloatListIterator iterator = null; + + /** + * Result of the comparison. + */ + private ComparisonResult comparisonResult; + private String comparisonMessage; + + public FingerprintEventHandler() { + this.compareFingerprint = null; + } + + public FingerprintEventHandler(EventFingerprint compareFingerprint) { + this.compareFingerprint = compareFingerprint; + this.comparisonResult = null; + } + + public EventFingerprint getEventFingerprint() { + return eventFingerprint; + } + + public ComparisonResult getComparisonResult() { + return comparisonResult; + } + + void setComparisonResult(ComparisonResult comparisonResult) { + this.comparisonResult = comparisonResult; + } + + public String getComparisonMessage() { + return comparisonMessage; + } + + void setComparisonMessage(String comparisonMessage) { + this.comparisonMessage = comparisonMessage; + } + + @Override + public void handleEvent(Event event) { + + + String lexicographicSortedString = toLexicographicSortedString(event); + + if (compareFingerprint != null) { + if (iterator == null) { + this.iterator = compareFingerprint.timeArray.iterator(); + } + + if (this.comparisonResult == null) { + if (iterator.hasNext()) { + float entry = iterator.nextFloat(); + //Comparing floats with precision + if (Math.abs((float) event.getTime() - entry) >= EPS) { + this.comparisonResult = ComparisonResult.DIFFERENT_TIMESTEPS; + this.comparisonMessage = "Difference occurred in this event time=" + event.getTime() + lexicographicSortedString; + } + } else { + this.comparisonResult = ComparisonResult.DIFFERENT_TIMESTEPS; + this.comparisonMessage = "Additional event time=" + event.getTime() + lexicographicSortedString; + } + } + } + + eventFingerprint.addEventType(event.getEventType()); + + + //First timestep, nothing to accumulate + if (eventFingerprint.timeArray.isEmpty()) { + hashAccumulationList.add(lexicographicSortedString); + } else { + float lastTime = eventFingerprint.timeArray.getFloat(eventFingerprint.timeArray.size() - 1); + //If new time is the same as previous, add to accumulation list, event hash calculation is not ready + if (lastTime == event.getTime()) { + hashAccumulationList.add(lexicographicSortedString); + } + //if new time differs from previous, all hash can be calculated + else { + accumulateHash(); + hashAccumulationList.add(lexicographicSortedString); + } + } + + eventFingerprint.addTimeStamp(event.getTime()); + + //eventFingerprint.addHashCode(lexicographicSortedString); + } + + private void accumulateHash() { + + Collections.sort(hashAccumulationList); + + for (String str : hashAccumulationList) { + eventFingerprint.addHashCode(str); + } + + hashAccumulationList.clear(); + } + + /** + *

+ * Finish processing of the events file and return comparison result (if compare fingerprint was present). + * If the result is not equal it will generate a {@link #comparisonMessage}. + */ + void finishProcessing() { + + if (!hashAccumulationList.isEmpty()) { + accumulateHash(); + } + + byte[] hash = eventFingerprint.computeHash(); + + //hash = eventFingerprint.computeHash(); + + if (compareFingerprint == null) + return; + + //Handling EventTypeCounter differences + for (Object2IntMap.Entry entry1 : compareFingerprint.eventTypeCounter.object2IntEntrySet()) { + String key = entry1.getKey(); + int count1 = entry1.getIntValue(); + int count2 = eventFingerprint.eventTypeCounter.getInt(key); + if (count1 != count2) { + comparisonMessage = comparisonMessage == null ? "" : comparisonMessage; + + comparisonResult = (comparisonResult == null ? ComparisonResult.WRONG_EVENT_COUNT : comparisonResult); + + if (!comparisonMessage.isEmpty()) + comparisonMessage += "\n"; + + comparisonMessage += "Count for event type '%s' differs: %d (in fingerprint) != %d (in events)".formatted(key, count1, count2); + } + } + + // Difference was found in {@link EventFingerprint#eventTypeCounter} + if (comparisonResult != null) { + return; + } + + // only check hash if there was no difference up until here + if (!Arrays.equals(hash, compareFingerprint.hash)) { + comparisonResult = ComparisonResult.DIFFERENT_EVENT_ATTRIBUTES; + comparisonMessage = "Difference occurred in this hash of 2 files"; + return; + } + + comparisonResult = ComparisonResult.FILES_ARE_EQUAL; + } + + private String toLexicographicSortedString(Event event) { + List strings = new ArrayList(); + for (Map.Entry e : event.getAttributes().entrySet()) { + StringBuilder tmp = new StringBuilder(); + final String key = e.getKey(); + + // don't look at certain attributes + switch (key) { + case Event.ATTRIBUTE_X: + case Event.ATTRIBUTE_Y: + case Event.ATTRIBUTE_TIME: + continue; + } + + tmp.append(key); + tmp.append("="); + tmp.append(e.getValue()); + strings.add(tmp.toString()); + } + Collections.sort(strings); + StringBuilder eventStr = new StringBuilder(); + for (String str : strings) { + eventStr.append(" | "); + eventStr.append(str); + } + + eventStr.append(" | "); + return eventStr.toString(); + } +} diff --git a/matsim/src/main/java/org/matsim/utils/eventsfilecomparison/Worker.java b/matsim/src/main/java/org/matsim/utils/eventsfilecomparison/Worker.java index 004a0a5e347..81c9cddf32a 100644 --- a/matsim/src/main/java/org/matsim/utils/eventsfilecomparison/Worker.java +++ b/matsim/src/main/java/org/matsim/utils/eventsfilecomparison/Worker.java @@ -144,6 +144,7 @@ private String toLexicographicSortedString(Event event) { switch( key ){ case Event.ATTRIBUTE_X: case Event.ATTRIBUTE_Y: + case Event.ATTRIBUTE_TIME: continue; } } @@ -159,6 +160,7 @@ private String toLexicographicSortedString(Event event) { eventStr.append(" | "); eventStr.append(str); } + eventStr.append(" | ") ; return eventStr.toString(); } diff --git a/matsim/src/test/java/org/matsim/examples/EquilTest.java b/matsim/src/test/java/org/matsim/examples/EquilTest.java index 96794010292..31f714c3d34 100644 --- a/matsim/src/test/java/org/matsim/examples/EquilTest.java +++ b/matsim/src/test/java/org/matsim/examples/EquilTest.java @@ -41,6 +41,7 @@ import org.matsim.core.scenario.ScenarioUtils; import org.matsim.testcases.MatsimTestUtils; import org.matsim.utils.eventsfilecomparison.EventsFileComparator; +import org.matsim.utils.eventsfilecomparison.ComparisonResult; public class EquilTest { private static final Logger log = LogManager.getLogger( EquilTest.class ) ; @@ -84,7 +85,7 @@ void testEquil(boolean isUsingFastCapacityUpdate) { writer.closeFile(); - final EventsFileComparator.Result result = new EventsFileComparator().setIgnoringCoordinates( true ).runComparison( referenceFileName , eventsFileName ); - Assertions.assertEquals(EventsFileComparator.Result.FILES_ARE_EQUAL, result, "different event files." ); + final ComparisonResult result = new EventsFileComparator().setIgnoringCoordinates( true ).runComparison( referenceFileName , eventsFileName ); + Assertions.assertEquals(ComparisonResult.FILES_ARE_EQUAL, result, "different event files." ); } } diff --git a/matsim/src/test/java/org/matsim/examples/OnePercentBerlin10sIT.java b/matsim/src/test/java/org/matsim/examples/OnePercentBerlin10sIT.java index 89237ec5b1a..f6d862c5785 100644 --- a/matsim/src/test/java/org/matsim/examples/OnePercentBerlin10sIT.java +++ b/matsim/src/test/java/org/matsim/examples/OnePercentBerlin10sIT.java @@ -41,6 +41,7 @@ import org.matsim.core.scenario.ScenarioUtils; import org.matsim.testcases.MatsimTestUtils; import org.matsim.utils.eventsfilecomparison.EventsFileComparator; +import org.matsim.utils.eventsfilecomparison.ComparisonResult; public class OnePercentBerlin10sIT { @@ -92,7 +93,7 @@ void testOnePercent10sQSim() { writer.closeFile(); System.out.println("reffile: " + referenceEventsFileName); - assertEquals( EventsFileComparator.Result.FILES_ARE_EQUAL, + assertEquals( ComparisonResult.FILES_ARE_EQUAL, new EventsFileComparator().setIgnoringCoordinates( true ).runComparison( referenceEventsFileName, eventsFileName ), "different event files" ); @@ -137,7 +138,7 @@ void testOnePercent10sQSimTryEndTimeThenDuration() { writer.closeFile(); - assertEquals( EventsFileComparator.Result.FILES_ARE_EQUAL, + assertEquals( ComparisonResult.FILES_ARE_EQUAL, new EventsFileComparator().setIgnoringCoordinates( true ).runComparison( referenceEventsFileName, eventsFileName ), "different event files" ); diff --git a/matsim/src/test/java/org/matsim/testcases/MatsimTestUtils.java b/matsim/src/test/java/org/matsim/testcases/MatsimTestUtils.java index 87696593899..112bcaaddaf 100644 --- a/matsim/src/test/java/org/matsim/testcases/MatsimTestUtils.java +++ b/matsim/src/test/java/org/matsim/testcases/MatsimTestUtils.java @@ -32,6 +32,7 @@ import org.matsim.core.utils.io.IOUtils; import org.matsim.core.utils.misc.CRCChecksum; import org.matsim.utils.eventsfilecomparison.EventsFileComparator; +import org.matsim.utils.eventsfilecomparison.ComparisonResult; import java.io.BufferedReader; import java.io.File; @@ -316,7 +317,7 @@ public static void assertEqualFilesLineByLine(String inputFilename, String outpu } public static void assertEqualEventsFiles( String filename1, String filename2 ) { - Assertions.assertEquals(EventsFileComparator.Result.FILES_ARE_EQUAL ,EventsFileComparator.compare(filename1, filename2) ); + Assertions.assertEquals(ComparisonResult.FILES_ARE_EQUAL ,EventsFileComparator.compare(filename1, filename2) ); } public static void assertEqualFilesBasedOnCRC( String filename1, String filename2 ) { diff --git a/matsim/src/test/java/org/matsim/utils/eventsfilecomparison/EventsFileComparatorTest.java b/matsim/src/test/java/org/matsim/utils/eventsfilecomparison/EventsFileComparatorTest.java index 6618b85dc17..9a52ac8593e 100644 --- a/matsim/src/test/java/org/matsim/utils/eventsfilecomparison/EventsFileComparatorTest.java +++ b/matsim/src/test/java/org/matsim/utils/eventsfilecomparison/EventsFileComparatorTest.java @@ -20,7 +20,7 @@ package org.matsim.utils.eventsfilecomparison; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.matsim.utils.eventsfilecomparison.EventsFileComparator.Result.*; +import static org.matsim.utils.eventsfilecomparison.ComparisonResult.*; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; diff --git a/matsim/src/test/java/org/matsim/utils/eventsfilecomparison/EventsFileFingerprintComparatorTest.java b/matsim/src/test/java/org/matsim/utils/eventsfilecomparison/EventsFileFingerprintComparatorTest.java new file mode 100644 index 00000000000..a14100b52d1 --- /dev/null +++ b/matsim/src/test/java/org/matsim/utils/eventsfilecomparison/EventsFileFingerprintComparatorTest.java @@ -0,0 +1,143 @@ +package org.matsim.utils.eventsfilecomparison; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; +import org.matsim.core.events.EventsUtils; +import org.matsim.testcases.MatsimTestUtils; + +import java.io.File; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.*; + +class EventsFileFingerprintComparatorTest { + + @RegisterExtension + MatsimTestUtils utils = new MatsimTestUtils(); + + @Test + void testEqual() { + + assertThat(EventsUtils.createAndCompareEventsFingerprint( + new File(utils.getClassInputDirectory(), "events_correct.xml"), + new File(utils.getClassInputDirectory(), "correct.fp.zst").toString() + )).isEqualTo(ComparisonResult.FILES_ARE_EQUAL); + + // Check if file has been created + assertThat(new File(utils.getClassInputDirectory(), "events_correct.fp.zst")) + .exists() + .size().isGreaterThan(0); + + } + + @Test + void testDiffTimesteps() { + + assertThat(EventsUtils.createAndCompareEventsFingerprint( + new File(utils.getClassInputDirectory(), "events.diff-num-timestamps.xml"), + new File(utils.getClassInputDirectory(), "correct.fp.zst").toString() + )).isEqualTo(ComparisonResult.DIFFERENT_TIMESTEPS); + + + } + + @Test + void testDiffCounts() { + + assertThat(EventsUtils.createAndCompareEventsFingerprint( + new File(utils.getClassInputDirectory(), "events.diff-type-count.xml"), + new File(utils.getClassInputDirectory(), "correct.fp.zst").toString() + )).isEqualTo(ComparisonResult.WRONG_EVENT_COUNT); + + } + + @Test + void testDiffContent() { + assertThat(EventsUtils.createAndCompareEventsFingerprint( + new File(utils.getClassInputDirectory(), "events.diff-hash.xml"), + new File(utils.getClassInputDirectory(), "correct.fp.zst").toString() + )).isEqualTo(ComparisonResult.DIFFERENT_EVENT_ATTRIBUTES); + } + + @Test + void testAdditionalEvent() { + assertThat(EventsUtils.createAndCompareEventsFingerprint( + new File(utils.getClassInputDirectory(), "events.one-more-event.xml"), + new File(utils.getClassInputDirectory(), "correct.fp.zst").toString() + )).isEqualTo(ComparisonResult.DIFFERENT_TIMESTEPS); + } + + @Test + void testAttributeOrder() { + assertThat(EventsUtils.createAndCompareEventsFingerprint( + new File(utils.getClassInputDirectory(), "events.attribute-order.xml"), + new File(utils.getClassInputDirectory(), "correct.fp.zst").toString() + )).isEqualTo(ComparisonResult.FILES_ARE_EQUAL); + } + + @Test + void testEventOrder() { + assertThat(EventsUtils.compareEventsFiles( + new File(utils.getClassInputDirectory(), "events_correct.xml").toString(), + new File(utils.getClassInputDirectory(), "events.event-order-wrong_logic.xml").toString() + )).isEqualTo(ComparisonResult.FILES_ARE_EQUAL); + + assertThat(EventsUtils.createAndCompareEventsFingerprint( + new File(utils.getClassInputDirectory(), "events.event-order-wrong_logic.xml"), + new File(utils.getClassInputDirectory(), "correct.fp.zst").toString() + )).isEqualTo(ComparisonResult.FILES_ARE_EQUAL); + + } + + + @Test + void testEqualFingerprints() { + + EventsUtils.createEventsFingerprint(new File(utils.getClassInputDirectory(), "events_correct.xml").toString(), new File(utils.getClassInputDirectory(), "events.fp.zst").toString()); + + assertThat(EventsFileFingerprintComparator.compareFingerprints( + new File(utils.getClassInputDirectory(), "events.fp.zst").toString(), + new File(utils.getClassInputDirectory(), "correct.fp.zst").toString() + )).isEqualTo(ComparisonResult.FILES_ARE_EQUAL); + + + } + + @Test + void testDiffTimestepsFingerprints() { + + assertThat(EventsFileFingerprintComparator.compareFingerprints( + new File(utils.getClassInputDirectory(), "events.diff-num-timestamps.fp.zst").toString(), + new File(utils.getClassInputDirectory(), "correct.fp.zst").toString() + )).isEqualTo(ComparisonResult.DIFFERENT_NUMBER_OF_TIMESTEPS); + + + } + + @Test + void testDiffCountsFingerprints() { + + assertThat(EventsFileFingerprintComparator.compareFingerprints( + new File(utils.getClassInputDirectory(), "events.diff-type-count.fp.zst").toString(), + new File(utils.getClassInputDirectory(), "correct.fp.zst").toString() + )).isEqualTo(ComparisonResult.WRONG_EVENT_COUNT); + + } + + @Test + void assertion() { + + AssertionError err = assertThrows(AssertionError.class, () -> { + EventsUtils.assertEqualEventsFingerprint( + new File(utils.getClassInputDirectory(), "events.diff-num-timestamps.xml"), + new File(utils.getClassInputDirectory(), "correct.fp.zst").toString() + ); + }); + + assertThat(err).message().isEqualTo(""" + Difference occurred in this event time=48067.0 | link=1 | type=entered link | vehicle=5 |\s + Count for event type 'entered link' differs: 2 (in fingerprint) != 1 (in events) + Count for event type 'vehicle leaves traffic' differs: 1 (in fingerprint) != 0 (in events)"""); + + } +} diff --git a/matsim/test/input/org/matsim/utils/eventsfilecomparison/EventsFileFingerprintComparatorTest/correct.fp.zst b/matsim/test/input/org/matsim/utils/eventsfilecomparison/EventsFileFingerprintComparatorTest/correct.fp.zst new file mode 100644 index 0000000000000000000000000000000000000000..7bf62ca48e26975c0b7ed02c29daada8427db755 GIT binary patch literal 196 zcmdPcs{fZE;s`543a49uz99nx1Gn1>9tJ*l>%$E0wpu_w7XwdfUP)?EYKlTmW?nW> zgo%MYv8X7sEHMYjVPxP;Ni9e$Dk&{WWlKyhNzF?U4oEF3&d+lN=_(FO&B#p75i3*3 zNlh$EEmkNgN=!@3O!nc-NlhzZWZ(d5DJ}tO2N|WY!Q$}xlNm+}6d&AQ7IO5A#^z7U oc^OokHBQWsU_9V_{@bCh@EMC;v)-J}J)Oh7gH=4b3}^%c07D-^;{X5v literal 0 HcmV?d00001 diff --git a/matsim/test/input/org/matsim/utils/eventsfilecomparison/EventsFileFingerprintComparatorTest/events.attribute-order.xml b/matsim/test/input/org/matsim/utils/eventsfilecomparison/EventsFileFingerprintComparatorTest/events.attribute-order.xml new file mode 100644 index 00000000000..6885fa8af5d --- /dev/null +++ b/matsim/test/input/org/matsim/utils/eventsfilecomparison/EventsFileFingerprintComparatorTest/events.attribute-order.xml @@ -0,0 +1,14 @@ + + + + + + + + + + + + + + \ No newline at end of file diff --git a/matsim/test/input/org/matsim/utils/eventsfilecomparison/EventsFileFingerprintComparatorTest/events.diff-hash.xml b/matsim/test/input/org/matsim/utils/eventsfilecomparison/EventsFileFingerprintComparatorTest/events.diff-hash.xml new file mode 100644 index 00000000000..4ae06c2369f --- /dev/null +++ b/matsim/test/input/org/matsim/utils/eventsfilecomparison/EventsFileFingerprintComparatorTest/events.diff-hash.xml @@ -0,0 +1,14 @@ + + + + + + + + + + + + + + \ No newline at end of file diff --git a/matsim/test/input/org/matsim/utils/eventsfilecomparison/EventsFileFingerprintComparatorTest/events.diff-num-timestamps.xml b/matsim/test/input/org/matsim/utils/eventsfilecomparison/EventsFileFingerprintComparatorTest/events.diff-num-timestamps.xml new file mode 100644 index 00000000000..422b58d651d --- /dev/null +++ b/matsim/test/input/org/matsim/utils/eventsfilecomparison/EventsFileFingerprintComparatorTest/events.diff-num-timestamps.xml @@ -0,0 +1,12 @@ + + + + + + + + + + + + \ No newline at end of file diff --git a/matsim/test/input/org/matsim/utils/eventsfilecomparison/EventsFileFingerprintComparatorTest/events.diff-type-count.fp.zst b/matsim/test/input/org/matsim/utils/eventsfilecomparison/EventsFileFingerprintComparatorTest/events.diff-type-count.fp.zst new file mode 100644 index 0000000000000000000000000000000000000000..6437b813ac0fd615f3a74dea6c4492d7f5cca149 GIT binary patch literal 194 zcmdPcs{fZEVhbyS2&Y?sz99nx1Gn1>9tJ*l>%$E0wpu_w7XwdfUP)?EYKlTmW?nW> zgpq+gv8X7sEHQ^OCAA>2sHC(gl`S#3BsDKZI3TsCI6u!7q@g%0H6t@QN32XCCpEDw zwOFB~C^0Q9GuekTCpE2viGc&CrMLvB9b{yoZfn9FuFS{f&vgpd+TJXxYGnS-$Drb@ nabku9;{oUM-wt(!&sgl5_2zW$OkUpLf(JY?x literal 0 HcmV?d00001 diff --git a/matsim/test/input/org/matsim/utils/eventsfilecomparison/EventsFileFingerprintComparatorTest/events.diff-type-count.xml b/matsim/test/input/org/matsim/utils/eventsfilecomparison/EventsFileFingerprintComparatorTest/events.diff-type-count.xml new file mode 100644 index 00000000000..31662fae9c0 --- /dev/null +++ b/matsim/test/input/org/matsim/utils/eventsfilecomparison/EventsFileFingerprintComparatorTest/events.diff-type-count.xml @@ -0,0 +1,14 @@ + + + + + + + + + + + + + + \ No newline at end of file diff --git a/matsim/test/input/org/matsim/utils/eventsfilecomparison/EventsFileFingerprintComparatorTest/events.event-order-wrong_logic.xml b/matsim/test/input/org/matsim/utils/eventsfilecomparison/EventsFileFingerprintComparatorTest/events.event-order-wrong_logic.xml new file mode 100644 index 00000000000..1112364be0c --- /dev/null +++ b/matsim/test/input/org/matsim/utils/eventsfilecomparison/EventsFileFingerprintComparatorTest/events.event-order-wrong_logic.xml @@ -0,0 +1,14 @@ + + + + + + + + + + + + + + \ No newline at end of file diff --git a/matsim/test/input/org/matsim/utils/eventsfilecomparison/EventsFileFingerprintComparatorTest/events.event-order.xml b/matsim/test/input/org/matsim/utils/eventsfilecomparison/EventsFileFingerprintComparatorTest/events.event-order.xml new file mode 100644 index 00000000000..c4b9bbf33de --- /dev/null +++ b/matsim/test/input/org/matsim/utils/eventsfilecomparison/EventsFileFingerprintComparatorTest/events.event-order.xml @@ -0,0 +1,14 @@ + + + + + + + + + + + + + + \ No newline at end of file diff --git a/matsim/test/input/org/matsim/utils/eventsfilecomparison/EventsFileFingerprintComparatorTest/events.one-more-event.xml b/matsim/test/input/org/matsim/utils/eventsfilecomparison/EventsFileFingerprintComparatorTest/events.one-more-event.xml new file mode 100644 index 00000000000..663754db63f --- /dev/null +++ b/matsim/test/input/org/matsim/utils/eventsfilecomparison/EventsFileFingerprintComparatorTest/events.one-more-event.xml @@ -0,0 +1,15 @@ + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/matsim/test/input/org/matsim/utils/eventsfilecomparison/EventsFileFingerprintComparatorTest/events_correct.xml b/matsim/test/input/org/matsim/utils/eventsfilecomparison/EventsFileFingerprintComparatorTest/events_correct.xml new file mode 100644 index 00000000000..4b2bbbd1b92 --- /dev/null +++ b/matsim/test/input/org/matsim/utils/eventsfilecomparison/EventsFileFingerprintComparatorTest/events_correct.xml @@ -0,0 +1,14 @@ + + + + + + + + + + + + + + \ No newline at end of file From 4180bcd90c5daa59688a6ad7d2edef9b8308ad2b Mon Sep 17 00:00:00 2001 From: Chengqi Lu <43133404+luchengqi7@users.noreply.github.com> Date: Mon, 18 Mar 2024 11:58:17 +0100 Subject: [PATCH 4/8] Remove the need of binding to use DRT detour constraints (#3171) * Refactoring work * Update DrtOfferAcceptor.java * Remove original DEFAULT_ACCEPTOR * Improve binding --- .../edrt/run/EDrtModeOptimizerQSimModule.java | 12 +++++++----- .../drt/optimizer/DrtModeOptimizerQSimModule.java | 8 +++++--- ...ferAcceptor.java => DefaultOfferAcceptor.java} | 15 +++++++++++++-- .../contrib/drt/passenger/DrtOfferAcceptor.java | 3 --- .../matsim/contrib/drt/run/DrtConfigGroup.java | 8 +++++++- .../drt/optimizer/MaxDetourConstraintTest.java | 13 ------------- .../DefaultUnplannedRequestInserterTest.java | 3 ++- 7 files changed, 34 insertions(+), 28 deletions(-) rename contribs/drt/src/main/java/org/matsim/contrib/drt/passenger/{MaxDetourOfferAcceptor.java => DefaultOfferAcceptor.java} (62%) diff --git a/contribs/drt-extensions/src/main/java/org/matsim/contrib/drt/extension/edrt/run/EDrtModeOptimizerQSimModule.java b/contribs/drt-extensions/src/main/java/org/matsim/contrib/drt/extension/edrt/run/EDrtModeOptimizerQSimModule.java index 71ee5abe1c1..b617a93a0d3 100644 --- a/contribs/drt-extensions/src/main/java/org/matsim/contrib/drt/extension/edrt/run/EDrtModeOptimizerQSimModule.java +++ b/contribs/drt-extensions/src/main/java/org/matsim/contrib/drt/extension/edrt/run/EDrtModeOptimizerQSimModule.java @@ -44,6 +44,7 @@ import org.matsim.contrib.drt.optimizer.insertion.UnplannedRequestInserter; import org.matsim.contrib.drt.optimizer.rebalancing.RebalancingStrategy; import org.matsim.contrib.drt.passenger.DrtOfferAcceptor; +import org.matsim.contrib.drt.passenger.DefaultOfferAcceptor; import org.matsim.contrib.drt.prebooking.PrebookingActionCreator; import org.matsim.contrib.drt.run.DrtConfigGroup; import org.matsim.contrib.drt.schedule.DrtTaskFactory; @@ -178,7 +179,8 @@ public EmptyVehicleRelocator get() { getter.getModal(StopTimeCalculator.class), scheduleWaitBeforeDrive))) .asEagerSingleton(); - bindModal(DrtOfferAcceptor.class).toInstance(DrtOfferAcceptor.DEFAULT_ACCEPTOR); + bindModal(DefaultOfferAcceptor.class).toProvider(modalProvider(getter -> new DefaultOfferAcceptor(drtCfg.maxAllowedPickupDelay))); + bindModal(DrtOfferAcceptor.class).to(modalKey(DefaultOfferAcceptor.class)); bindModal(ScheduleTimingUpdater.class).toProvider(modalProvider( getter -> new ScheduleTimingUpdater(getter.get(MobsimTimer.class), @@ -187,11 +189,11 @@ public EmptyVehicleRelocator get() { bindModal(VrpLegFactory.class).toProvider(modalProvider(getter -> { DvrpConfigGroup dvrpCfg = getter.get(DvrpConfigGroup.class); MobsimTimer timer = getter.get(MobsimTimer.class); - + // Makes basic DrtActionCreator create legs with consumption tracker return v -> EDrtActionCreator.createLeg(dvrpCfg.mobsimMode, v, timer); })).in(Singleton.class); - + bindModal(EDrtActionCreator.class).toProvider(modalProvider(getter -> { VrpAgentLogic.DynActionCreator delegate = drtCfg.getPrebookingParams().isPresent() ? getter.getModal(PrebookingActionCreator.class) @@ -201,9 +203,9 @@ public EmptyVehicleRelocator get() { // + adds ChargingActivity return new EDrtActionCreator(delegate, getter.get(MobsimTimer.class)); })).asEagerSingleton(); - + bindModal(VrpAgentLogic.DynActionCreator.class).to(modalKey(EDrtActionCreator.class)); - + bindModal(VrpOptimizer.class).to(modalKey(DrtOptimizer.class)); } } diff --git a/contribs/drt/src/main/java/org/matsim/contrib/drt/optimizer/DrtModeOptimizerQSimModule.java b/contribs/drt/src/main/java/org/matsim/contrib/drt/optimizer/DrtModeOptimizerQSimModule.java index 8534468ce81..92336f544c4 100644 --- a/contribs/drt/src/main/java/org/matsim/contrib/drt/optimizer/DrtModeOptimizerQSimModule.java +++ b/contribs/drt/src/main/java/org/matsim/contrib/drt/optimizer/DrtModeOptimizerQSimModule.java @@ -37,6 +37,7 @@ import org.matsim.contrib.drt.optimizer.insertion.selective.SelectiveInsertionSearchQSimModule; import org.matsim.contrib.drt.optimizer.rebalancing.RebalancingStrategy; import org.matsim.contrib.drt.passenger.DrtOfferAcceptor; +import org.matsim.contrib.drt.passenger.DefaultOfferAcceptor; import org.matsim.contrib.drt.prebooking.PrebookingActionCreator; import org.matsim.contrib.drt.run.DrtConfigGroup; import org.matsim.contrib.drt.schedule.DrtStayTaskEndTimeCalculator; @@ -138,7 +139,7 @@ public EmptyVehicleRelocator get() { }).asEagerSingleton(); bindModal(DrtScheduleInquiry.class).to(DrtScheduleInquiry.class).asEagerSingleton(); - + boolean scheduleWaitBeforeDrive = drtCfg.getPrebookingParams().map(p -> p.scheduleWaitBeforeDrive).orElse(false); bindModal(RequestInsertionScheduler.class).toProvider(modalProvider( getter -> new DefaultRequestInsertionScheduler(getter.getModal(Fleet.class), @@ -147,7 +148,8 @@ public EmptyVehicleRelocator get() { getter.getModal(StopTimeCalculator.class), scheduleWaitBeforeDrive))) .asEagerSingleton(); - bindModal(DrtOfferAcceptor.class).toInstance(DrtOfferAcceptor.DEFAULT_ACCEPTOR); + bindModal(DefaultOfferAcceptor.class).toProvider(modalProvider(getter -> new DefaultOfferAcceptor(drtCfg.maxAllowedPickupDelay))); + bindModal(DrtOfferAcceptor.class).to(modalKey(DefaultOfferAcceptor.class)); bindModal(ScheduleTimingUpdater.class).toProvider(modalProvider( getter -> new ScheduleTimingUpdater(getter.get(MobsimTimer.class), @@ -160,7 +162,7 @@ public EmptyVehicleRelocator get() { return v -> VrpLegFactory.createWithOnlineTracker(dvrpCfg.mobsimMode, v, OnlineTrackerListener.NO_LISTENER, timer); })).in(Singleton.class); - + if (drtCfg.getPrebookingParams().isEmpty()) { bindModal(VrpAgentLogic.DynActionCreator.class).to(modalKey(DrtActionCreator.class)); } else { diff --git a/contribs/drt/src/main/java/org/matsim/contrib/drt/passenger/MaxDetourOfferAcceptor.java b/contribs/drt/src/main/java/org/matsim/contrib/drt/passenger/DefaultOfferAcceptor.java similarity index 62% rename from contribs/drt/src/main/java/org/matsim/contrib/drt/passenger/MaxDetourOfferAcceptor.java rename to contribs/drt/src/main/java/org/matsim/contrib/drt/passenger/DefaultOfferAcceptor.java index fe2dfb25834..9eeed624139 100644 --- a/contribs/drt/src/main/java/org/matsim/contrib/drt/passenger/MaxDetourOfferAcceptor.java +++ b/contribs/drt/src/main/java/org/matsim/contrib/drt/passenger/DefaultOfferAcceptor.java @@ -2,13 +2,24 @@ import java.util.Optional; -public class MaxDetourOfferAcceptor implements DrtOfferAcceptor{ +public class DefaultOfferAcceptor implements DrtOfferAcceptor{ private final double maxAllowedPickupDelay; - public MaxDetourOfferAcceptor(double maxAllowedPickupDelay) { + /** + * Generate Default offer acceptor with max allowed pickup delay. + * @param maxAllowedPickupDelay: maximum allowed delay since the initially assigned pickup time. + */ + public DefaultOfferAcceptor(double maxAllowedPickupDelay) { this.maxAllowedPickupDelay = maxAllowedPickupDelay; } + /** + * Generate Default offer acceptor. + */ + public DefaultOfferAcceptor() { + this.maxAllowedPickupDelay = Double.POSITIVE_INFINITY; + } + @Override public Optional acceptDrtOffer(DrtRequest request, double departureTime, double arrivalTime) { double updatedLatestStartTime = Math.min(departureTime diff --git a/contribs/drt/src/main/java/org/matsim/contrib/drt/passenger/DrtOfferAcceptor.java b/contribs/drt/src/main/java/org/matsim/contrib/drt/passenger/DrtOfferAcceptor.java index 4118169fe70..68ab4751459 100644 --- a/contribs/drt/src/main/java/org/matsim/contrib/drt/passenger/DrtOfferAcceptor.java +++ b/contribs/drt/src/main/java/org/matsim/contrib/drt/passenger/DrtOfferAcceptor.java @@ -26,9 +26,6 @@ * @author Michal Maciejewski (michalm) */ public interface DrtOfferAcceptor { - DrtOfferAcceptor DEFAULT_ACCEPTOR = (request, departureTime, arrivalTime) -> Optional.of( - AcceptedDrtRequest.createFromOriginalRequest(request)); - /** * @param request drt request * @param departureTime offered departure time for the new request diff --git a/contribs/drt/src/main/java/org/matsim/contrib/drt/run/DrtConfigGroup.java b/contribs/drt/src/main/java/org/matsim/contrib/drt/run/DrtConfigGroup.java index 7fb685bd92e..93597ab6271 100644 --- a/contribs/drt/src/main/java/org/matsim/contrib/drt/run/DrtConfigGroup.java +++ b/contribs/drt/src/main/java/org/matsim/contrib/drt/run/DrtConfigGroup.java @@ -133,7 +133,7 @@ public static DrtConfigGroup getSingleModeDrtConfig(Config config) { @Comment( "Defines the maximum delay allowed from the initial scheduled pick up time. Once the initial pickup time is offered, the latest promised" + "pickup time is calculated based on initial scheduled pickup time + maxAllowedPickupDelay. " - + "By default, this limit is disabled. If enabled, a value between 120 and 240 is a good choice.") + + "By default, this limit is disabled. If enabled, a value between 0 and 240 is a good choice.") @PositiveOrZero public double maxAllowedPickupDelay = Double.POSITIVE_INFINITY;// [s] @@ -323,6 +323,12 @@ protected void checkConsistency(Config config) { if (useModeFilteredSubnetwork) { DvrpModeRoutingNetworkModule.checkUseModeFilteredSubnetworkAllowed(config, mode); } + + if ((maxDetourAlpha != Double.POSITIVE_INFINITY && maxDetourBeta != Double.POSITIVE_INFINITY) || maxAbsoluteDetour != Double.POSITIVE_INFINITY) { + Verify.verify(maxAllowedPickupDelay != Double.POSITIVE_INFINITY, "Detour constraints are activated, " + + "maxAllowedPickupDelay must be specified! A value between 0 and 240 seconds can be a good choice for maxAllowedPickupDelay."); + } + } @Override diff --git a/contribs/drt/src/test/java/org/matsim/contrib/drt/optimizer/MaxDetourConstraintTest.java b/contribs/drt/src/test/java/org/matsim/contrib/drt/optimizer/MaxDetourConstraintTest.java index 895f3a096f2..83ffce4ddb5 100644 --- a/contribs/drt/src/test/java/org/matsim/contrib/drt/optimizer/MaxDetourConstraintTest.java +++ b/contribs/drt/src/test/java/org/matsim/contrib/drt/optimizer/MaxDetourConstraintTest.java @@ -2,12 +2,9 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; -import org.matsim.contrib.drt.passenger.DrtOfferAcceptor; -import org.matsim.contrib.drt.passenger.MaxDetourOfferAcceptor; import org.matsim.contrib.drt.run.DrtConfigGroup; import org.matsim.contrib.drt.run.DrtControlerCreator; import org.matsim.contrib.drt.run.MultiModeDrtConfigGroup; -import org.matsim.contrib.dvrp.run.AbstractDvrpModeQSimModule; import org.matsim.contrib.dvrp.run.DvrpConfigGroup; import org.matsim.core.config.Config; import org.matsim.core.config.ConfigUtils; @@ -29,7 +26,6 @@ public void testMaxDetourConstraint() { URL configUrl = IOUtils.extendUrl(ExamplesUtils.getTestScenarioURL("mielec"), "mielec_drt_config.xml"); Config config = ConfigUtils.loadConfig(configUrl, new MultiModeDrtConfigGroup(), new DvrpConfigGroup(), new OTFVisConfigGroup()); - MultiModeDrtConfigGroup multiModeDrtConfigGroup = MultiModeDrtConfigGroup.get(config); DrtConfigGroup drtConfigGroup = DrtConfigGroup.getSingleModeDrtConfig(config); // Max wait time @@ -50,15 +46,6 @@ public void testMaxDetourConstraint() { Controler controler = DrtControlerCreator.createControler(config, false); - for (DrtConfigGroup drtCfg : multiModeDrtConfigGroup.getModalElements()) { - controler.addOverridingQSimModule(new AbstractDvrpModeQSimModule(drtCfg.mode) { - @Override - protected void configureQSim() { - bindModal(DrtOfferAcceptor.class).toProvider(modalProvider(getter -> new MaxDetourOfferAcceptor(drtCfg.maxAllowedPickupDelay))); - } - }); - } - controler.run(); } } diff --git a/contribs/drt/src/test/java/org/matsim/contrib/drt/optimizer/insertion/DefaultUnplannedRequestInserterTest.java b/contribs/drt/src/test/java/org/matsim/contrib/drt/optimizer/insertion/DefaultUnplannedRequestInserterTest.java index 58637acaddc..2396c4da810 100644 --- a/contribs/drt/src/test/java/org/matsim/contrib/drt/optimizer/insertion/DefaultUnplannedRequestInserterTest.java +++ b/contribs/drt/src/test/java/org/matsim/contrib/drt/optimizer/insertion/DefaultUnplannedRequestInserterTest.java @@ -42,6 +42,7 @@ import org.matsim.contrib.drt.optimizer.DrtRequestInsertionRetryQueue; import org.matsim.contrib.drt.optimizer.VehicleEntry; import org.matsim.contrib.drt.passenger.AcceptedDrtRequest; +import org.matsim.contrib.drt.passenger.DefaultOfferAcceptor; import org.matsim.contrib.drt.passenger.DrtOfferAcceptor; import org.matsim.contrib.drt.passenger.DrtRequest; import org.matsim.contrib.drt.schedule.DefaultDrtStopTask; @@ -285,7 +286,7 @@ private DefaultUnplannedRequestInserter newInserter(Fleet fleet, double now, VehicleEntry.EntryFactory vehicleEntryFactory, DrtRequestInsertionRetryQueue insertionRetryQueue, DrtInsertionSearch insertionSearch, RequestInsertionScheduler insertionScheduler) { return new DefaultUnplannedRequestInserter(mode, fleet, () -> now, eventsManager, insertionScheduler, - vehicleEntryFactory, insertionRetryQueue, insertionSearch, DrtOfferAcceptor.DEFAULT_ACCEPTOR, + vehicleEntryFactory, insertionRetryQueue, insertionSearch, new DefaultOfferAcceptor(), forkJoinPoolExtension.forkJoinPool, StaticPassengerStopDurationProvider.of(10.0, 0.0)); } From b739ee890cf833fecae596ab5c898a04e4885706 Mon Sep 17 00:00:00 2001 From: nkuehnel Date: Mon, 18 Mar 2024 12:01:34 +0100 Subject: [PATCH 5/8] install drt zonalsystem module independently of the rebalancing module --- .../contrib/drt/optimizer/rebalancing/RebalancingModule.java | 1 - .../src/main/java/org/matsim/contrib/drt/run/DrtModeModule.java | 2 ++ 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/contribs/drt/src/main/java/org/matsim/contrib/drt/optimizer/rebalancing/RebalancingModule.java b/contribs/drt/src/main/java/org/matsim/contrib/drt/optimizer/rebalancing/RebalancingModule.java index b9c008bc3a1..ef148776efd 100644 --- a/contribs/drt/src/main/java/org/matsim/contrib/drt/optimizer/rebalancing/RebalancingModule.java +++ b/contribs/drt/src/main/java/org/matsim/contrib/drt/optimizer/rebalancing/RebalancingModule.java @@ -46,7 +46,6 @@ public RebalancingModule(DrtConfigGroup drtCfg) { public void install() { if (drtCfg.getRebalancingParams().isPresent()) { RebalancingParams rebalancingParams = drtCfg.getRebalancingParams().get(); - install(new DrtModeZonalSystemModule(drtCfg)); if (rebalancingParams.getRebalancingStrategyParams() instanceof MinCostFlowRebalancingStrategyParams) { install(new DrtModeMinCostFlowRebalancingModule(drtCfg)); diff --git a/contribs/drt/src/main/java/org/matsim/contrib/drt/run/DrtModeModule.java b/contribs/drt/src/main/java/org/matsim/contrib/drt/run/DrtModeModule.java index a0cb1fc007c..f667841b104 100644 --- a/contribs/drt/src/main/java/org/matsim/contrib/drt/run/DrtModeModule.java +++ b/contribs/drt/src/main/java/org/matsim/contrib/drt/run/DrtModeModule.java @@ -25,6 +25,7 @@ import com.google.inject.name.Names; import org.matsim.api.core.v01.network.Network; import org.matsim.contrib.drt.analysis.DrtEventSequenceCollector; +import org.matsim.contrib.drt.analysis.zonal.DrtModeZonalSystemModule; import org.matsim.contrib.drt.fare.DrtFareHandler; import org.matsim.contrib.drt.optimizer.rebalancing.RebalancingModule; import org.matsim.contrib.drt.prebooking.analysis.PrebookingModeAnalysisModule; @@ -65,6 +66,7 @@ public void install() { null : ConfigGroup.getInputFileURL(getConfig().getContext(), drtCfg.vehiclesFile), drtCfg.changeStartLinkToLastLinkInSchedule)); + install(new DrtModeZonalSystemModule(drtCfg)); install(new RebalancingModule(drtCfg)); install(new DrtModeRoutingModule(drtCfg)); From b89ad74a799c2eb37b1af14d09796e5c2eb66731 Mon Sep 17 00:00:00 2001 From: marecabo <23156476+marecabo@users.noreply.github.com> Date: Mon, 18 Mar 2024 12:28:36 +0100 Subject: [PATCH 6/8] Move estimateWaitingTime(List elements) to PTWaitingTimeEstimator interface --- .../utils/PTWaitingTimeEstimator.java | 19 +++++++++++++++++++ .../utils/ScheduleWaitingTimeEstimator.java | 19 ------------------- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/contribs/discrete_mode_choice/src/main/java/org/matsim/contribs/discrete_mode_choice/components/utils/PTWaitingTimeEstimator.java b/contribs/discrete_mode_choice/src/main/java/org/matsim/contribs/discrete_mode_choice/components/utils/PTWaitingTimeEstimator.java index 2b4da256f0b..a253763488a 100644 --- a/contribs/discrete_mode_choice/src/main/java/org/matsim/contribs/discrete_mode_choice/components/utils/PTWaitingTimeEstimator.java +++ b/contribs/discrete_mode_choice/src/main/java/org/matsim/contribs/discrete_mode_choice/components/utils/PTWaitingTimeEstimator.java @@ -1,5 +1,10 @@ package org.matsim.contribs.discrete_mode_choice.components.utils; +import java.util.List; + +import org.matsim.api.core.v01.TransportMode; +import org.matsim.api.core.v01.population.Leg; +import org.matsim.api.core.v01.population.PlanElement; import org.matsim.pt.routes.TransitPassengerRoute; /** @@ -11,5 +16,19 @@ * @author sebhoerl */ public interface PTWaitingTimeEstimator { + double estimateWaitingTime(double departureTime, TransitPassengerRoute route); + + default double estimateWaitingTime(List elements) { + double totalWaitingTime = 0.0; + + for (PlanElement element : elements) { + if (element instanceof Leg leg && leg.getMode().equals(TransportMode.pt)) { + TransitPassengerRoute route = (TransitPassengerRoute) leg.getRoute(); + totalWaitingTime += this.estimateWaitingTime(leg.getDepartureTime().seconds(), route); + } + } + + return totalWaitingTime; + } } diff --git a/contribs/discrete_mode_choice/src/main/java/org/matsim/contribs/discrete_mode_choice/components/utils/ScheduleWaitingTimeEstimator.java b/contribs/discrete_mode_choice/src/main/java/org/matsim/contribs/discrete_mode_choice/components/utils/ScheduleWaitingTimeEstimator.java index 2ab5bfe10ca..78e9d45ee97 100644 --- a/contribs/discrete_mode_choice/src/main/java/org/matsim/contribs/discrete_mode_choice/components/utils/ScheduleWaitingTimeEstimator.java +++ b/contribs/discrete_mode_choice/src/main/java/org/matsim/contribs/discrete_mode_choice/components/utils/ScheduleWaitingTimeEstimator.java @@ -8,8 +8,6 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.matsim.api.core.v01.Id; -import org.matsim.api.core.v01.population.Leg; -import org.matsim.api.core.v01.population.PlanElement; import org.matsim.core.utils.collections.Tuple; import org.matsim.core.utils.misc.OptionalTime; import org.matsim.core.utils.misc.Time; @@ -54,23 +52,6 @@ private Tuple, Id> createId(TransitLine transitLin return new Tuple<>(transitLine.getId(), transitRoute.getId()); } - public double estimateWaitingTime(List elements) { - double totalWaitingTime = 0.0; - - for (PlanElement element : elements) { - if (element instanceof Leg) { - Leg leg = (Leg) element; - - if (leg.getMode().equals("pt")) { - TransitPassengerRoute route = (TransitPassengerRoute) leg.getRoute(); - totalWaitingTime += estimateWaitingTime(leg.getDepartureTime().seconds(), route); - } - } - } - - return totalWaitingTime; - } - @Override public double estimateWaitingTime(double agentDepartureTime, TransitPassengerRoute route) { TransitLine transitLine = transitSchedule.getTransitLines().get(route.getLineId()); From 4258f24f15a85523a9cf9e9d7b5136ab9682d1f7 Mon Sep 17 00:00:00 2001 From: nkuehnel Date: Mon, 18 Mar 2024 12:31:46 +0100 Subject: [PATCH 7/8] ensure zonal params are present --- .../zonal/DrtModeZonalSystemModule.java | 78 ++++++++++--------- 1 file changed, 40 insertions(+), 38 deletions(-) diff --git a/contribs/drt/src/main/java/org/matsim/contrib/drt/analysis/zonal/DrtModeZonalSystemModule.java b/contribs/drt/src/main/java/org/matsim/contrib/drt/analysis/zonal/DrtModeZonalSystemModule.java index 8d81ba21749..986f75177a5 100644 --- a/contribs/drt/src/main/java/org/matsim/contrib/drt/analysis/zonal/DrtModeZonalSystemModule.java +++ b/contribs/drt/src/main/java/org/matsim/contrib/drt/analysis/zonal/DrtModeZonalSystemModule.java @@ -53,53 +53,55 @@ public DrtModeZonalSystemModule(DrtConfigGroup drtCfg) { @Override public void install() { - DrtZonalSystemParams params = drtCfg.getZonalSystemParams().orElseThrow(); - - bindModal(DrtZonalSystem.class).toProvider(modalProvider(getter -> { - Network network = getter.getModal(Network.class); - switch (params.zonesGeneration) { - case ShapeFile: - final List preparedGeometries = loadPreparedGeometries( + if (drtCfg.getZonalSystemParams().isPresent()) { + DrtZonalSystemParams params = drtCfg.getZonalSystemParams().get(); + + bindModal(DrtZonalSystem.class).toProvider(modalProvider(getter -> { + Network network = getter.getModal(Network.class); + switch (params.zonesGeneration) { + case ShapeFile: + final List preparedGeometries = loadPreparedGeometries( ConfigGroup.getInputFileURL(getConfig().getContext(), params.zonesShapeFile)); - return DrtZonalSystem.createFromPreparedGeometries(network, + return DrtZonalSystem.createFromPreparedGeometries(network, EntryStream.of(preparedGeometries).mapKeys(i -> (i + 1) + "").toMap()); - case GridFromNetwork: - Preconditions.checkNotNull(params.cellSize); - var gridZones = drtCfg.operationalScheme == OperationalScheme.serviceAreaBased ? + case GridFromNetwork: + Preconditions.checkNotNull(params.cellSize); + var gridZones = drtCfg.operationalScheme == OperationalScheme.serviceAreaBased ? createGridFromNetworkWithinServiceArea(network, params.cellSize, - loadPreparedGeometries(ConfigGroup.getInputFileURL(getConfig().getContext(), - drtCfg.drtServiceAreaShapeFile))) : + loadPreparedGeometries(ConfigGroup.getInputFileURL(getConfig().getContext(), + drtCfg.drtServiceAreaShapeFile))) : createGridFromNetwork(network, params.cellSize); - return DrtZonalSystem.createFromPreparedGeometries(network, gridZones); - - default: - throw new RuntimeException("Unsupported zone generation"); - } - })).asEagerSingleton(); - - bindModal(DrtZoneTargetLinkSelector.class).toProvider(modalProvider(getter -> { - switch (params.targetLinkSelection) { - case mostCentral: - return new MostCentralDrtZoneTargetLinkSelector(getter.getModal(DrtZonalSystem.class)); - case random: - return new RandomDrtZoneTargetLinkSelector(); - default: - throw new RuntimeException( + return DrtZonalSystem.createFromPreparedGeometries(network, gridZones); + + default: + throw new RuntimeException("Unsupported zone generation"); + } + })).asEagerSingleton(); + + bindModal(DrtZoneTargetLinkSelector.class).toProvider(modalProvider(getter -> { + switch (params.targetLinkSelection) { + case mostCentral: + return new MostCentralDrtZoneTargetLinkSelector(getter.getModal(DrtZonalSystem.class)); + case random: + return new RandomDrtZoneTargetLinkSelector(); + default: + throw new RuntimeException( "Unsupported target link selection = " + params.targetLinkSelection); - } - })).asEagerSingleton(); + } + })).asEagerSingleton(); - //zonal analysis - bindModal(ZonalIdleVehicleXYVisualiser.class).toProvider(modalProvider( + //zonal analysis + bindModal(ZonalIdleVehicleXYVisualiser.class).toProvider(modalProvider( getter -> new ZonalIdleVehicleXYVisualiser(getter.get(MatsimServices.class), drtCfg.getMode(), - getter.getModal(DrtZonalSystem.class)))).asEagerSingleton(); - addControlerListenerBinding().to(modalKey(ZonalIdleVehicleXYVisualiser.class)); - addEventHandlerBinding().to(modalKey(ZonalIdleVehicleXYVisualiser.class)); + getter.getModal(DrtZonalSystem.class)))).asEagerSingleton(); + addControlerListenerBinding().to(modalKey(ZonalIdleVehicleXYVisualiser.class)); + addEventHandlerBinding().to(modalKey(ZonalIdleVehicleXYVisualiser.class)); - bindModal(DrtZonalWaitTimesAnalyzer.class).toProvider(modalProvider( + bindModal(DrtZonalWaitTimesAnalyzer.class).toProvider(modalProvider( getter -> new DrtZonalWaitTimesAnalyzer(drtCfg, getter.getModal(DrtEventSequenceCollector.class), - getter.getModal(DrtZonalSystem.class)))).asEagerSingleton(); - addControlerListenerBinding().to(modalKey(DrtZonalWaitTimesAnalyzer.class)); + getter.getModal(DrtZonalSystem.class)))).asEagerSingleton(); + addControlerListenerBinding().to(modalKey(DrtZonalWaitTimesAnalyzer.class)); + } } } From e1ba68d7b9d8a70ae14dae4f7d34bd8dbf322643 Mon Sep 17 00:00:00 2001 From: marecabo <23156476+marecabo@users.noreply.github.com> Date: Mon, 18 Mar 2024 12:44:56 +0100 Subject: [PATCH 8/8] Do not use default method in interface --- .../utils/NullWaitingTimeEstimator.java | 10 ++++++++++ .../utils/PTWaitingTimeEstimator.java | 14 +------------- .../utils/ScheduleWaitingTimeEstimator.java | 17 +++++++++++++++++ 3 files changed, 28 insertions(+), 13 deletions(-) diff --git a/contribs/discrete_mode_choice/src/main/java/org/matsim/contribs/discrete_mode_choice/components/utils/NullWaitingTimeEstimator.java b/contribs/discrete_mode_choice/src/main/java/org/matsim/contribs/discrete_mode_choice/components/utils/NullWaitingTimeEstimator.java index 445451713ca..e003a9f09e9 100644 --- a/contribs/discrete_mode_choice/src/main/java/org/matsim/contribs/discrete_mode_choice/components/utils/NullWaitingTimeEstimator.java +++ b/contribs/discrete_mode_choice/src/main/java/org/matsim/contribs/discrete_mode_choice/components/utils/NullWaitingTimeEstimator.java @@ -1,5 +1,8 @@ package org.matsim.contribs.discrete_mode_choice.components.utils; +import java.util.List; + +import org.matsim.api.core.v01.population.PlanElement; import org.matsim.pt.routes.TransitPassengerRoute; /** @@ -9,8 +12,15 @@ * @author sebhoerl */ public class NullWaitingTimeEstimator implements PTWaitingTimeEstimator { + @Override public double estimateWaitingTime(double agentDepartureTime, TransitPassengerRoute route) { return 0.0; } + + @Override + public double estimateWaitingTime(List elements) { + return 0.0; + } + } diff --git a/contribs/discrete_mode_choice/src/main/java/org/matsim/contribs/discrete_mode_choice/components/utils/PTWaitingTimeEstimator.java b/contribs/discrete_mode_choice/src/main/java/org/matsim/contribs/discrete_mode_choice/components/utils/PTWaitingTimeEstimator.java index a253763488a..25804cddb95 100644 --- a/contribs/discrete_mode_choice/src/main/java/org/matsim/contribs/discrete_mode_choice/components/utils/PTWaitingTimeEstimator.java +++ b/contribs/discrete_mode_choice/src/main/java/org/matsim/contribs/discrete_mode_choice/components/utils/PTWaitingTimeEstimator.java @@ -2,8 +2,6 @@ import java.util.List; -import org.matsim.api.core.v01.TransportMode; -import org.matsim.api.core.v01.population.Leg; import org.matsim.api.core.v01.population.PlanElement; import org.matsim.pt.routes.TransitPassengerRoute; @@ -19,16 +17,6 @@ public interface PTWaitingTimeEstimator { double estimateWaitingTime(double departureTime, TransitPassengerRoute route); - default double estimateWaitingTime(List elements) { - double totalWaitingTime = 0.0; + double estimateWaitingTime(List elements); - for (PlanElement element : elements) { - if (element instanceof Leg leg && leg.getMode().equals(TransportMode.pt)) { - TransitPassengerRoute route = (TransitPassengerRoute) leg.getRoute(); - totalWaitingTime += this.estimateWaitingTime(leg.getDepartureTime().seconds(), route); - } - } - - return totalWaitingTime; - } } diff --git a/contribs/discrete_mode_choice/src/main/java/org/matsim/contribs/discrete_mode_choice/components/utils/ScheduleWaitingTimeEstimator.java b/contribs/discrete_mode_choice/src/main/java/org/matsim/contribs/discrete_mode_choice/components/utils/ScheduleWaitingTimeEstimator.java index 78e9d45ee97..6fc60d7ad73 100644 --- a/contribs/discrete_mode_choice/src/main/java/org/matsim/contribs/discrete_mode_choice/components/utils/ScheduleWaitingTimeEstimator.java +++ b/contribs/discrete_mode_choice/src/main/java/org/matsim/contribs/discrete_mode_choice/components/utils/ScheduleWaitingTimeEstimator.java @@ -8,6 +8,9 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.matsim.api.core.v01.Id; +import org.matsim.api.core.v01.TransportMode; +import org.matsim.api.core.v01.population.Leg; +import org.matsim.api.core.v01.population.PlanElement; import org.matsim.core.utils.collections.Tuple; import org.matsim.core.utils.misc.OptionalTime; import org.matsim.core.utils.misc.Time; @@ -52,6 +55,20 @@ private Tuple, Id> createId(TransitLine transitLin return new Tuple<>(transitLine.getId(), transitRoute.getId()); } + @Override + public double estimateWaitingTime(List elements) { + double totalWaitingTime = 0.0; + + for (PlanElement element : elements) { + if (element instanceof Leg leg && leg.getMode().equals(TransportMode.pt)) { + TransitPassengerRoute route = (TransitPassengerRoute) leg.getRoute(); + totalWaitingTime += this.estimateWaitingTime(leg.getDepartureTime().seconds(), route); + } + } + + return totalWaitingTime; + } + @Override public double estimateWaitingTime(double agentDepartureTime, TransitPassengerRoute route) { TransitLine transitLine = transitSchedule.getTransitLines().get(route.getLineId());