From bdeba2f1c6c459d431d7739e759dffff0909c9bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20H=C3=B6rl?= Date: Wed, 6 Mar 2024 07:16:45 +0100 Subject: [PATCH 1/4] feat: transit routing by transport mode utilities --- .../routing/pt/raptor/RaptorParameters.java | 10 +++++ .../pt/raptor/SwissRailRaptorCore.java | 4 +- .../pt/raptor/SwissRailRaptorTest.java | 45 ++++++++++++++++++- 3 files changed, 57 insertions(+), 2 deletions(-) diff --git a/matsim/src/main/java/ch/sbb/matsim/routing/pt/raptor/RaptorParameters.java b/matsim/src/main/java/ch/sbb/matsim/routing/pt/raptor/RaptorParameters.java index ba79214ff73..3aa0443b9e0 100644 --- a/matsim/src/main/java/ch/sbb/matsim/routing/pt/raptor/RaptorParameters.java +++ b/matsim/src/main/java/ch/sbb/matsim/routing/pt/raptor/RaptorParameters.java @@ -65,6 +65,8 @@ public class RaptorParameters { private double transferPenaltyPerTravelTimeHour = 0.0; private double transferPenaltyMinimum = Double.NEGATIVE_INFINITY; private double transferPenaltyMaximum = Double.POSITIVE_INFINITY; + + private boolean useTransportModeUtilities = false; private final SwissRailRaptorConfigGroup config; @@ -159,5 +161,13 @@ public double getTransferPenaltyMaximum() { public void setTransferPenaltyMaximum(double transferPenaltyMaximum) { this.transferPenaltyMaximum = transferPenaltyMaximum; } + + public boolean isUseTransportModeUtilities() { + return useTransportModeUtilities; + } + + public void setUseTransportModeUtilities(boolean useTransportModeUtilities) { + this.useTransportModeUtilities = useTransportModeUtilities; + } } diff --git a/matsim/src/main/java/ch/sbb/matsim/routing/pt/raptor/SwissRailRaptorCore.java b/matsim/src/main/java/ch/sbb/matsim/routing/pt/raptor/SwissRailRaptorCore.java index 09cef257e09..0242f5e91ab 100644 --- a/matsim/src/main/java/ch/sbb/matsim/routing/pt/raptor/SwissRailRaptorCore.java +++ b/matsim/src/main/java/ch/sbb/matsim/routing/pt/raptor/SwissRailRaptorCore.java @@ -588,6 +588,7 @@ private void exploreRoutes(RaptorParameters parameters, Person person) { CachingTransferProvider transferProvider = this.data.new CachingTransferProvider(); double marginalUtilityOfWaitingPt_utl_s = parameters.getMarginalUtilityOfWaitingPt_utl_s(); + boolean useTransportModeUtilities = parameters.isUseTransportModeUtilities(); int routeIndex = -1; for (int firstRouteStopIndex = this.improvedRouteStopIndices.nextSetBit(0); firstRouteStopIndex >= 0; firstRouteStopIndex = this.improvedRouteStopIndices.nextSetBit(firstRouteStopIndex+1)) { @@ -627,7 +628,8 @@ private void exploreRoutes(RaptorParameters parameters, Person person) { routeIndex = tmpRouteIndex; int firstDepartureTime = (boardingPE.firstDepartureTime == TIME_UNDEFINED) ? currentAgentBoardingTime : boardingPE.firstDepartureTime; - double marginalUtilityOfTravelTime_utl_s = parameters.getMarginalUtilityOfTravelTime_utl_s(boardingPE.toRouteStop.mode); + double marginalUtilityOfTravelTime_utl_s = parameters.getMarginalUtilityOfTravelTime_utl_s( + !useTransportModeUtilities ? boardingPE.toRouteStop.mode : boardingPE.toRouteStop.route.getTransportMode()); transferProvider.reset(boardingPE.transfer); for (int toRouteStopIndex = firstRouteStopIndex + 1; toRouteStopIndex < route.indexFirstRouteStop + route.countRouteStops; toRouteStopIndex++) { diff --git a/matsim/src/test/java/ch/sbb/matsim/routing/pt/raptor/SwissRailRaptorTest.java b/matsim/src/test/java/ch/sbb/matsim/routing/pt/raptor/SwissRailRaptorTest.java index 98431e35cc7..567c6ace568 100644 --- a/matsim/src/test/java/ch/sbb/matsim/routing/pt/raptor/SwissRailRaptorTest.java +++ b/matsim/src/test/java/ch/sbb/matsim/routing/pt/raptor/SwissRailRaptorTest.java @@ -33,6 +33,7 @@ import org.matsim.api.core.v01.network.Node; import org.matsim.api.core.v01.population.Activity; import org.matsim.api.core.v01.population.Leg; +import org.matsim.api.core.v01.population.Person; import org.matsim.api.core.v01.population.PlanElement; import org.matsim.core.config.Config; import org.matsim.core.config.ConfigUtils; @@ -337,7 +338,7 @@ void testLineChange() { assertEquals(Math.ceil(expectedTravelTime), actualTravelTime, MatsimTestUtils.EPSILON); } @Test - void testLineChangeWithDifferentUtils() { + void testLineChangeWithDifferentModeChangeUtils() { Fixture f = new Fixture(); f.init(); f.blueLine.getRoutes().values().forEach(transitRoute -> transitRoute.setTransportMode(TransportMode.ship)); @@ -355,6 +356,48 @@ void testLineChangeWithDifferentUtils() { //changing from train to ship is so expensive that direct walk is cheaper assertNull(legs); } + + @Test + void testLineChangeWithDifferentTravelTimeUtils() { + Fixture f = new Fixture(); + f.init(); + f.blueLine.getRoutes().values().forEach(transitRoute -> transitRoute.setTransportMode("bus")); + SwissRailRaptorConfigGroup swissRailRaptorConfigGroup = ConfigUtils.addOrGetModule(f.config, SwissRailRaptorConfigGroup.class); + swissRailRaptorConfigGroup.setTransferWalkMargin(0); + RaptorParameters raptorParams = RaptorUtils.createParameters(f.config); + SwissRailRaptorData data = SwissRailRaptorData.create(f.schedule, null, RaptorUtils.createStaticConfig(f.config), f.network, null); + TransitRouter router = new SwissRailRaptor.Builder(data, f.config).with(new RaptorParametersForPerson() { + @Override + public RaptorParameters getRaptorParameters(Person person) { + return raptorParams; + } + }).build(); + + // from C to G (see Fixture), competing between red line (express) and blue line (regular) + Coord fromCoord = new Coord(12000, 5000); + Coord toCoord = new Coord(28000, 5000); + + // default case + List legs = router.calcRoute(DefaultRoutingRequest.withoutAttributes(new FakeFacility(fromCoord), new FakeFacility(toCoord), 6.0*3600 - 60.0, null)); + assertEquals(3, legs.size()); + assertEquals("red", ((TransitPassengerRoute) ((Leg) legs.get(1)).getRoute()).getLineId().toString()); + + // routing by transport mode, same costs, choose red (train) again + raptorParams.setUseTransportModeUtilities(true); + raptorParams.setMarginalUtilityOfTravelTime_utl_s("train", -1e-3); + raptorParams.setMarginalUtilityOfTravelTime_utl_s("bus", -1e-3); + legs = router.calcRoute(DefaultRoutingRequest.withoutAttributes(new FakeFacility(fromCoord), new FakeFacility(toCoord), 6.0*3600 - 60.0, null)); + assertEquals(3, legs.size()); + assertEquals("red", ((TransitPassengerRoute) ((Leg) legs.get(1)).getRoute()).getLineId().toString()); + + // routing by transport mode, train is quicker, but more costly now, so choose blue (bus) + raptorParams.setUseTransportModeUtilities(true); + raptorParams.setMarginalUtilityOfTravelTime_utl_s("train", -1e-2); + raptorParams.setMarginalUtilityOfTravelTime_utl_s("bus", -1e-3); + legs = router.calcRoute(DefaultRoutingRequest.withoutAttributes(new FakeFacility(fromCoord), new FakeFacility(toCoord), 6.0*3600 - 60.0, null)); + assertEquals(3, legs.size()); + assertEquals("blue", ((TransitPassengerRoute) ((Leg) legs.get(1)).getRoute()).getLineId().toString()); + } @Test void testFasterAlternative() { From c1e3335f0a987a4c375137d06f166b604e5a475b Mon Sep 17 00:00:00 2001 From: Joschka Bischoff Date: Thu, 7 Mar 2024 14:29:23 +0100 Subject: [PATCH 2/4] fully support subpopulations in ReplanningAnnealer --- .../annealing/ReplanningAnnealer.java | 62 +++++++++++-------- .../ReplanningAnnealerConfigGroup.java | 54 +++++++++------- .../annealing/ReplanningAnnealerTest.java | 49 +++++++++++++-- 3 files changed, 111 insertions(+), 54 deletions(-) diff --git a/matsim/src/main/java/org/matsim/core/replanning/annealing/ReplanningAnnealer.java b/matsim/src/main/java/org/matsim/core/replanning/annealing/ReplanningAnnealer.java index b576833d9fd..2522bd2a96e 100644 --- a/matsim/src/main/java/org/matsim/core/replanning/annealing/ReplanningAnnealer.java +++ b/matsim/src/main/java/org/matsim/core/replanning/annealing/ReplanningAnnealer.java @@ -57,7 +57,7 @@ public class ReplanningAnnealer implements IterationStartsListener, StartupListe private final ReplanningAnnealerConfigGroup saConfig; private final int innovationStop; private final String sep; - private final EnumMap currentValues; + private final EnumMap> currentValuesPerSubpopulation; private int currentIter; private List header; @Inject @@ -67,7 +67,7 @@ public class ReplanningAnnealer implements IterationStartsListener, StartupListe public ReplanningAnnealer(Config config) { this.config = config; this.saConfig = ConfigUtils.addOrGetModule(config, ReplanningAnnealerConfigGroup.class); - this.currentValues = new EnumMap<>(AnnealParameterOption.class); + this.currentValuesPerSubpopulation = new EnumMap<>(AnnealParameterOption.class); this.innovationStop = getInnovationStop(config); this.sep = config.global().getDefaultDelimiter(); } @@ -83,17 +83,19 @@ private static boolean isInnovationStrategy(String strategyName) { @Override public void notifyStartup(StartupEvent event) { header = new ArrayList<>(); - for (AnnealingVariable av : this.saConfig.getAnnealingVariables().values()) { + for (AnnealingVariable av : this.saConfig.getAllAnnealingVariables()) { if (!av.getAnnealType().equals(AnnealOption.disabled)) { // check and fix initial value if needed checkAndFixStartValue(av, event); - this.currentValues.put(av.getAnnealParameter(), av.getStartValue()); - header.add(av.getAnnealParameter().name()); + var mapPerSubpopulation = this.currentValuesPerSubpopulation.computeIfAbsent(av.getAnnealParameter(),a-> new HashMap<>()); + mapPerSubpopulation.put(av.getSubpopulation(),av.getStartValue()); + String subpopulationString = av.getSubpopulation()!=null? "_"+av.getSubpopulation() :""; + header.add(av.getAnnealParameter().name()+subpopulationString); if (av.getAnnealParameter().equals(AnnealParameterOption.globalInnovationRate)) { header.addAll(this.config.replanning().getStrategySettings().stream() - .filter(s -> Objects.equals(av.getDefaultSubpopulation(), s.getSubpopulation())) - .map(ReplanningConfigGroup.StrategySettings::getStrategyName) + .filter(s -> Objects.equals(av.getSubpopulation(), s.getSubpopulation())) + .map(strategySettings -> strategySettings.getStrategyName()+subpopulationString) .collect(Collectors.toList())); } } else { // if disabled, better remove it @@ -114,34 +116,35 @@ public void notifyStartup(StartupEvent event) { public void notifyIterationStarts(IterationStartsEvent event) { this.currentIter = event.getIteration() - this.config.controller().getFirstIteration(); Map annealStats = new HashMap<>(); - for (AnnealingVariable av : this.saConfig.getAnnealingVariables().values()) { + List allVariables = this.saConfig.getAllAnnealingVariables(); + for (AnnealingVariable av : allVariables) { if (this.currentIter > 0) { switch (av.getAnnealType()) { case geometric: - this.currentValues.compute(av.getAnnealParameter(), (k, v) -> + this.currentValuesPerSubpopulation.get(av.getAnnealParameter()).compute(av.getSubpopulation(), (k, v) -> v * av.getShapeFactor()); break; case exponential: int halfLifeIter = av.getHalfLife() <= 1.0 ? (int) (av.getHalfLife() * this.innovationStop) : (int) av.getHalfLife(); - this.currentValues.compute(av.getAnnealParameter(), (k, v) -> + this.currentValuesPerSubpopulation.get(av.getAnnealParameter()).compute(av.getSubpopulation(), (k, v) -> av.getStartValue() / Math.exp((double) this.currentIter / halfLifeIter)); break; case msa: - this.currentValues.compute(av.getAnnealParameter(), (k, v) -> + this.currentValuesPerSubpopulation.get(av.getAnnealParameter()).compute(av.getSubpopulation(), (k, v) -> av.getStartValue() / Math.pow(this.currentIter, av.getShapeFactor())); break; case sigmoid: halfLifeIter = av.getHalfLife() <= 1.0 ? (int) (av.getHalfLife() * this.innovationStop) : (int) av.getHalfLife(); - this.currentValues.compute(av.getAnnealParameter(), (k, v) -> + this.currentValuesPerSubpopulation.get(av.getAnnealParameter()).compute(av.getSubpopulation(), (k, v) -> av.getEndValue() + (av.getStartValue() - av.getEndValue()) / (1 + Math.exp(av.getShapeFactor() * (this.currentIter - halfLifeIter)))); break; case linear: double slope = (av.getStartValue() - av.getEndValue()) / (this.config.controller().getFirstIteration() - this.innovationStop); - this.currentValues.compute(av.getAnnealParameter(), (k, v) -> + this.currentValuesPerSubpopulation.get(av.getAnnealParameter()).compute(av.getSubpopulation(), (k, v) -> this.currentIter * slope + av.getStartValue()); break; case disabled: @@ -150,14 +153,16 @@ public void notifyIterationStarts(IterationStartsEvent event) { throw new IllegalArgumentException(); } - log.info("Annealling will be performed on parameter " + av.getAnnealParameter() + - ". Value: " + this.currentValues.get(av.getAnnealParameter())); + log.info("Annealling will be performed on parameter " + av.getAnnealParameter() +". Subpopulation: "+av.getSubpopulation()+ + ". Value: " +this.currentValuesPerSubpopulation.get(av.getAnnealParameter()).get(av.getSubpopulation())); - this.currentValues.compute(av.getAnnealParameter(), (k, v) -> + this.currentValuesPerSubpopulation.get(av.getAnnealParameter()).compute(av.getSubpopulation(), (k, v) -> Math.max(v, av.getEndValue())); } - double annealValue = this.currentValues.get(av.getAnnealParameter()); - annealStats.put(av.getAnnealParameter().name(), String.format(Locale.US, "%.4f", annealValue)); + double annealValue = this.currentValuesPerSubpopulation.get(av.getAnnealParameter()).get(av.getSubpopulation()); + String subpopulationString = av.getSubpopulation()!=null? "_"+av.getSubpopulation() :""; + + annealStats.put(av.getAnnealParameter().name()+subpopulationString, String.format(Locale.US, "%.4f", annealValue)); anneal(event, av, annealValue, annealStats); } @@ -178,6 +183,8 @@ private void writeIterationstats(int currentIter, Map annealStat } private void anneal(IterationStartsEvent event, AnnealingVariable av, double annealValue, Map annealStats) { + String subpopulationString = av.getSubpopulation()!=null? "_"+av.getSubpopulation() :""; + switch (av.getAnnealParameter()) { case BrainExpBeta: this.config.scoring().setBrainExpBeta(annealValue); @@ -193,16 +200,17 @@ private void anneal(IterationStartsEvent event, AnnealingVariable av, double ann annealValue = 0.0; } List annealValues = annealReplanning(annealValue, - event.getServices().getStrategyManager(), av.getDefaultSubpopulation()); + event.getServices().getStrategyManager(), av.getSubpopulation()); int i = 0; for (ReplanningConfigGroup.StrategySettings ss : this.config.replanning().getStrategySettings()) { - if (Objects.equals(ss.getSubpopulation(), av.getDefaultSubpopulation())) { - annealStats.put(ss.getStrategyName(), String.format(Locale.US, "%.4f", annealValues.get(i))); + if (Objects.equals(ss.getSubpopulation(), av.getSubpopulation())) { + annealStats.put(ss.getStrategyName()+subpopulationString, String.format(Locale.US, "%.4f", annealValues.get(i))); i++; } } - annealStats.put(av.getAnnealParameter().name(), String.format(Locale.US, "%.4f", // update value in case of switchoff - getStrategyWeights(event.getServices().getStrategyManager(), av.getDefaultSubpopulation(), StratType.allInnovation))); + + annealStats.put(av.getAnnealParameter().name()+subpopulationString, String.format(Locale.US, "%.4f", // update value in case of switchoff + getStrategyWeights(event.getServices().getStrategyManager(), av.getSubpopulation(), StratType.allInnovation))); break; default: throw new IllegalArgumentException(); @@ -328,14 +336,14 @@ private void checkAndFixStartValue(ReplanningAnnealerConfigGroup.AnnealingVariab configValue = this.config.scoring().getLearningRate(); break; case globalInnovationRate: - double innovationWeights = getStrategyWeights(this.config, av.getDefaultSubpopulation(), StratType.allInnovation); - double selectorWeights = getStrategyWeights(this.config, av.getDefaultSubpopulation(), StratType.allSelectors); + double innovationWeights = getStrategyWeights(this.config, av.getSubpopulation(), StratType.allInnovation); + double selectorWeights = getStrategyWeights(this.config, av.getSubpopulation(), StratType.allSelectors); if (innovationWeights + selectorWeights != 1.0) { log.warn("Initial sum of strategy weights different from 1.0. Rescaling."); double innovationStartValue = av.getStartValue() == null ? innovationWeights : av.getStartValue(); - rescaleStartupWeights(innovationStartValue, this.config, event.getServices().getStrategyManager(), av.getDefaultSubpopulation()); + rescaleStartupWeights(innovationStartValue, this.config, event.getServices().getStrategyManager(), av.getSubpopulation()); } - configValue = getStrategyWeights(this.config, av.getDefaultSubpopulation(), StratType.allInnovation); + configValue = getStrategyWeights(this.config, av.getSubpopulation(), StratType.allInnovation); break; default: throw new IllegalArgumentException(); diff --git a/matsim/src/main/java/org/matsim/core/replanning/annealing/ReplanningAnnealerConfigGroup.java b/matsim/src/main/java/org/matsim/core/replanning/annealing/ReplanningAnnealerConfigGroup.java index c3a9421862f..3c13879d7ca 100644 --- a/matsim/src/main/java/org/matsim/core/replanning/annealing/ReplanningAnnealerConfigGroup.java +++ b/matsim/src/main/java/org/matsim/core/replanning/annealing/ReplanningAnnealerConfigGroup.java @@ -20,7 +20,11 @@ package org.matsim.core.replanning.annealing; import java.util.EnumMap; +import java.util.HashMap; +import java.util.List; import java.util.Map; +import java.util.stream.Collectors; + import org.matsim.core.config.ConfigGroup; import org.matsim.core.config.ReflectiveConfigGroup; @@ -81,27 +85,35 @@ public void addParameterSet(final ConfigGroup set) { addAnnealingVariable((AnnealingVariable) set); } - public Map getAnnealingVariables() { - final EnumMap map = - new EnumMap<>(AnnealParameterOption.class); - for (ConfigGroup pars : getParameterSets(AnnealingVariable.GROUP_NAME)) { - final AnnealParameterOption name = ((AnnealingVariable) pars).getAnnealParameter(); - final AnnealingVariable old = map.put(name, (AnnealingVariable) pars); - if (old != null) { - throw new IllegalStateException("several parameter sets for variable " + name); - } - } - return map; - } + public List getAllAnnealingVariables(){ + return getAnnealingVariablesPerSubpopulation().values().stream().flatMap(a->a.values().stream()).collect(Collectors.toList()); + } + public Map> getAnnealingVariablesPerSubpopulation() { + final EnumMap> map = + new EnumMap<>(AnnealParameterOption.class); + for (ConfigGroup pars : getParameterSets(AnnealingVariable.GROUP_NAME)) { + AnnealParameterOption name = ((AnnealingVariable) pars).getAnnealParameter(); + String subpopulation = ((AnnealingVariable) pars).getSubpopulation(); + var paramsPerSubpopulation = map.computeIfAbsent(name,a->new HashMap<>()); + final AnnealingVariable old = paramsPerSubpopulation.put(subpopulation, (AnnealingVariable) pars); + if (old != null) { + throw new IllegalStateException("several parameter sets for variable " + name + " and subpopulation "+subpopulation); + } + } + return map; + } public void addAnnealingVariable(final AnnealingVariable params) { - final AnnealingVariable previous = this.getAnnealingVariables().get(params.getAnnealParameter()); + var previousMap = this.getAnnealingVariablesPerSubpopulation().get(params.getAnnealParameter()); + if (previousMap!=null){ + AnnealingVariable previous = previousMap.get(params.getSubpopulation()); if (previous != null) { final boolean removed = removeParameterSet(previous); if (!removed) { throw new RuntimeException("problem replacing annealing variable"); } } + } super.addParameterSet(params); } @@ -117,11 +129,11 @@ public static class AnnealingVariable extends ReflectiveConfigGroup { private static final String START_VALUE = "startValue"; private static final String END_VALUE = "endValue"; private static final String ANNEAL_TYPE = "annealType"; - private static final String DEFAULT_SUBPOP = "defaultSubpopulation"; + private static final String SUBPOPULATION = "subpopulation"; private static final String ANNEAL_PARAM = "annealParameter"; private static final String HALFLIFE = "halfLife"; private static final String SHAPE_FACTOR = "shapeFactor"; - private String defaultSubpop = null; + private String subpopulation = null; private Double startValue = null; private double endValue = 0.0001; private double shapeFactor = 0.9; @@ -167,14 +179,14 @@ public void setAnnealType(AnnealOption annealType) { this.annealType = annealType; } - @StringGetter(DEFAULT_SUBPOP) - public String getDefaultSubpopulation() { - return this.defaultSubpop; + @StringGetter(SUBPOPULATION) + public String getSubpopulation() { + return this.subpopulation; } - @StringSetter(DEFAULT_SUBPOP) + @StringSetter(SUBPOPULATION) public void setDefaultSubpopulation(String defaultSubpop) { - this.defaultSubpop = defaultSubpop; + this.subpopulation = defaultSubpop; } @StringGetter(ANNEAL_PARAM) @@ -220,7 +232,7 @@ public Map getComments() { map.put(ANNEAL_TYPE, "options: linear, exponential, geometric, msa, sigmoid and disabled (no annealing)."); map.put(ANNEAL_PARAM, "list of config parameters that shall be annealed. Currently supported: globalInnovationRate, BrainExpBeta, PathSizeLogitBeta, learningRate. Default is globalInnovationRate"); - map.put(DEFAULT_SUBPOP, "subpopulation to have the global innovation rate adjusted. Not applicable when annealing with other parameters."); + map.put(SUBPOPULATION, "subpopulation to have the global innovation rate adjusted. Not applicable when annealing with other parameters."); map.put(START_VALUE, "start value for annealing."); map.put(END_VALUE, "final annealing value. When the annealing function reaches this value, further results remain constant."); return map; diff --git a/matsim/src/test/java/org/matsim/core/replanning/annealing/ReplanningAnnealerTest.java b/matsim/src/test/java/org/matsim/core/replanning/annealing/ReplanningAnnealerTest.java index d16b44098a6..bc83393f8cb 100644 --- a/matsim/src/test/java/org/matsim/core/replanning/annealing/ReplanningAnnealerTest.java +++ b/matsim/src/test/java/org/matsim/core/replanning/annealing/ReplanningAnnealerTest.java @@ -39,6 +39,19 @@ public class ReplanningAnnealerTest { "8;0.1000;0.0500;0.0500;0.9000\n" + "9;0.0500;0.0250;0.0250;0.9500\n" + "10;0.0000;0.0000;0.0000;1.0000\n"; + + private String getExpectedLinearAnnealMultipleSubpopulations = "it;globalInnovationRate_otherAnnealer;ChangeExpBeta_otherAnnealer;TimeAllocationMutator_otherAnnealer;globalInnovationRate_subpop;ReRoute_subpop;SubtourModeChoice_subpop;ChangeExpBeta_subpop\n" + + "0;0.8000;0.2000;0.8000;0.5000;0.2500;0.2500;0.5000\n" + + "1;0.7200;0.2800;0.7200;0.4500;0.2250;0.2250;0.5500\n" + + "2;0.6400;0.3600;0.6400;0.4000;0.2000;0.2000;0.6000\n" + + "3;0.5600;0.4400;0.5600;0.3500;0.1750;0.1750;0.6500\n" + + "4;0.4800;0.5200;0.4800;0.3000;0.1500;0.1500;0.7000\n" + + "5;0.4000;0.6000;0.4000;0.2500;0.1250;0.1250;0.7500\n" + + "6;0.3200;0.6800;0.3200;0.2000;0.1000;0.1000;0.8000\n" + + "7;0.2400;0.7600;0.2400;0.1500;0.0750;0.0750;0.8500\n" + + "8;0.1600;0.8400;0.1600;0.1000;0.0500;0.0500;0.9000\n" + + "9;0.0800;0.9200;0.0800;0.0500;0.0250;0.0250;0.9500\n" + + "10;0.0000;1.0000;0.0000;0.0000;0.0000;0.0000;1.0000\n"; private String expectedMsaAnneal = "it;globalInnovationRate;ReRoute;SubtourModeChoice;ChangeExpBeta\n" + "0;0.5000;0.2500;0.2500;0.5000\n" + @@ -360,21 +373,45 @@ void testSubpopulationAnneal() throws IOException { this.saConfigVar.setStartValue(0.5); this.saConfigVar.setDefaultSubpopulation(targetSubpop); this.config.replanning().getStrategySettings().forEach(s -> s.setSubpopulation(targetSubpop)); - ReplanningConfigGroup.StrategySettings s = new ReplanningConfigGroup.StrategySettings(); - s.setStrategyName("TimeAllocationMutator"); - s.setWeight(0.25); - s.setSubpopulation("noAnneal"); - this.config.replanning().addStrategySettings(s); + + String otherAnnealerSubpopulation = "otherAnnealer"; + String othertargetSubpop = otherAnnealerSubpopulation; + ReplanningAnnealerConfigGroup.AnnealingVariable saConfigVar2 = new ReplanningAnnealerConfigGroup.AnnealingVariable(); + saConfigVar2.setAnnealType("linear"); + saConfigVar2.setEndValue(0.0); + saConfigVar2.setStartValue(0.8); + saConfigVar2.setDefaultSubpopulation(othertargetSubpop); + this.config.replanningAnnealer().addParameterSet(saConfigVar2); + + ReplanningConfigGroup.StrategySettings s = new ReplanningConfigGroup.StrategySettings(); + s.setStrategyName("TimeAllocationMutator"); + s.setWeight(0.25); + s.setSubpopulation(otherAnnealerSubpopulation); + ReplanningConfigGroup.StrategySettings s2 = new ReplanningConfigGroup.StrategySettings(); + s2.setStrategyName("ChangeExpBeta"); // shouldn't be affected + s2.setWeight(0.5); + s2.setSubpopulation(otherAnnealerSubpopulation); + this.config.replanning().addStrategySettings(s2); + this.config.replanning().addStrategySettings(s); + + + ReplanningConfigGroup.StrategySettings noAnnealSettings = new ReplanningConfigGroup.StrategySettings(); + noAnnealSettings.setStrategyName("TimeAllocationMutator"); + noAnnealSettings.setWeight(0.25); + noAnnealSettings.setSubpopulation("noAnneal"); + this.config.replanning().addStrategySettings(noAnnealSettings); Controler controler = new Controler(this.scenario); controler.run(); - Assertions.assertEquals(expectedLinearAnneal, readResult(controler.getControlerIO().getOutputFilename(FILENAME_ANNEAL))); + Assertions.assertEquals(getExpectedLinearAnnealMultipleSubpopulations, readResult(controler.getControlerIO().getOutputFilename(FILENAME_ANNEAL))); StrategyManager sm = controler.getInjector().getInstance(StrategyManager.class); List weights = sm.getWeights(targetSubpop); + List weights2 = sm.getWeights(otherAnnealerSubpopulation); Assertions.assertEquals(1.0, weights.stream().mapToDouble(Double::doubleValue).sum(), 1e-4); + Assertions.assertEquals(1.0, weights2.stream().mapToDouble(Double::doubleValue).sum(), 1e-4); } } From 2644bf9f4835945a315fc3bb5a7bd928c706ed68 Mon Sep 17 00:00:00 2001 From: Joschka Bischoff Date: Thu, 7 Mar 2024 14:53:17 +0100 Subject: [PATCH 3/4] add test with explicitly set null subpopulation --- .../annealing/ReplanningAnnealerTest.java | 24 +++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/matsim/src/test/java/org/matsim/core/replanning/annealing/ReplanningAnnealerTest.java b/matsim/src/test/java/org/matsim/core/replanning/annealing/ReplanningAnnealerTest.java index bc83393f8cb..760d3e430e2 100644 --- a/matsim/src/test/java/org/matsim/core/replanning/annealing/ReplanningAnnealerTest.java +++ b/matsim/src/test/java/org/matsim/core/replanning/annealing/ReplanningAnnealerTest.java @@ -40,7 +40,7 @@ public class ReplanningAnnealerTest { "9;0.0500;0.0250;0.0250;0.9500\n" + "10;0.0000;0.0000;0.0000;1.0000\n"; - private String getExpectedLinearAnnealMultipleSubpopulations = "it;globalInnovationRate_otherAnnealer;ChangeExpBeta_otherAnnealer;TimeAllocationMutator_otherAnnealer;globalInnovationRate_subpop;ReRoute_subpop;SubtourModeChoice_subpop;ChangeExpBeta_subpop\n" + + private String expectedLinearAnnealMultipleSubpopulations = "it;globalInnovationRate_otherAnnealer;ChangeExpBeta_otherAnnealer;TimeAllocationMutator_otherAnnealer;globalInnovationRate_subpop;ReRoute_subpop;SubtourModeChoice_subpop;ChangeExpBeta_subpop\n" + "0;0.8000;0.2000;0.8000;0.5000;0.2500;0.2500;0.5000\n" + "1;0.7200;0.2800;0.7200;0.4500;0.2250;0.2250;0.5500\n" + "2;0.6400;0.3600;0.6400;0.4000;0.2000;0.2000;0.6000\n" + @@ -404,7 +404,7 @@ void testSubpopulationAnneal() throws IOException { Controler controler = new Controler(this.scenario); controler.run(); - Assertions.assertEquals(getExpectedLinearAnnealMultipleSubpopulations, readResult(controler.getControlerIO().getOutputFilename(FILENAME_ANNEAL))); + Assertions.assertEquals(expectedLinearAnnealMultipleSubpopulations, readResult(controler.getControlerIO().getOutputFilename(FILENAME_ANNEAL))); StrategyManager sm = controler.getInjector().getInstance(StrategyManager.class); List weights = sm.getWeights(targetSubpop); @@ -414,4 +414,24 @@ void testSubpopulationAnneal() throws IOException { Assertions.assertEquals(1.0, weights2.stream().mapToDouble(Double::doubleValue).sum(), 1e-4); } + @Test + void testNullSubpopulationAnneal() throws IOException { + String targetSubpop = null; + this.saConfigVar.setAnnealType("linear"); + this.saConfigVar.setEndValue(0.0); + this.saConfigVar.setStartValue(0.5); + this.saConfigVar.setDefaultSubpopulation(targetSubpop); + this.config.replanning().getStrategySettings().forEach(s -> s.setSubpopulation(targetSubpop)); + + Controler controler = new Controler(this.scenario); + controler.run(); + + Assertions.assertEquals(expectedLinearAnneal, readResult(controler.getControlerIO().getOutputFilename(FILENAME_ANNEAL))); + + StrategyManager sm = controler.getInjector().getInstance(StrategyManager.class); + List weights = sm.getWeights(targetSubpop); + + Assertions.assertEquals(1.0, weights.stream().mapToDouble(Double::doubleValue).sum(), 1e-4); + } + } From 95bdede72e0ca054756e73d40475bc9b20789170 Mon Sep 17 00:00:00 2001 From: rakow Date: Mon, 18 Mar 2024 11:25:42 +0100 Subject: [PATCH 4/4] Event fingerprints and comparison (#3165) * started work on fingerprint comparator * added compare function, fixed hashtoadd * fixed compare function, added printFingerprint * changed structure, added compression, changed hash logic * added tests for comparison, added folders for input files * added hash order logic, also javadoc, also fixed minor errors * cleaned up tests, improved API * remove extra new line from error messages * update test * add deprecation message to old enum * handle file errors as separate comparison result * fix capital I in folder name --------- Co-authored-by: zlukich --- .../contrib/bicycle/run/BicycleTest.java | 2 +- .../events/VehicleLeavesTrafficEventTest.java | 6 +- ...unAverageEmissionToolOfflineExampleIT.java | 18 +- .../contrib/ev/example/RunEvExampleTest.java | 10 +- ...nEvExampleWithLTHConsumptionModelTest.java | 10 +- .../usecases/chessboard/RunChessboardIT.java | 6 +- .../run/RunParkingSearchScenarioIT.java | 10 +- .../run/RunWithParkingProxyIT.java | 6 +- .../run/RoadPricingByConfigfileTest.java | 4 +- .../builder/TravelTimeFourWaysTest.java | 5 +- .../signals/integration/SignalSystemsIT.java | 9 +- ...nerateSmallScaleCommercialTrafficTest.java | 6 +- .../contrib/etaxi/run/RunETaxiScenarioIT.java | 6 +- .../taxi/optimizer/TaxiOptimizerTests.java | 6 +- .../java/playground/vsp/ev/UrbanEVIT.java | 6 +- matsim/.gitignore | 2 +- .../org/matsim/core/events/EventsUtils.java | 81 ++++++- .../ComparisonResult.java | 6 + .../EventFingerprint.java | 165 ++++++++++++++ .../EventsFileComparator.java | 38 ++-- .../EventsFileFingerprintComparator.java | 124 ++++++++++ .../FingerprintEventHandler.java | 212 ++++++++++++++++++ .../utils/eventsfilecomparison/Worker.java | 2 + .../java/org/matsim/examples/EquilTest.java | 5 +- .../examples/OnePercentBerlin10sIT.java | 5 +- .../org/matsim/testcases/MatsimTestUtils.java | 3 +- .../EventsFileComparatorTest.java | 2 +- .../EventsFileFingerprintComparatorTest.java | 143 ++++++++++++ .../correct.fp.zst | Bin 0 -> 196 bytes .../events.attribute-order.xml | 14 ++ .../events.diff-hash.xml | 14 ++ .../events.diff-num-timestamps.xml | 12 + .../events.diff-type-count.fp.zst | Bin 0 -> 194 bytes .../events.diff-type-count.xml | 14 ++ .../events.event-order-wrong_logic.xml | 14 ++ .../events.event-order.xml | 14 ++ .../events.one-more-event.xml | 15 ++ .../events_correct.xml | 14 ++ 38 files changed, 918 insertions(+), 91 deletions(-) create mode 100644 matsim/src/main/java/org/matsim/utils/eventsfilecomparison/ComparisonResult.java create mode 100644 matsim/src/main/java/org/matsim/utils/eventsfilecomparison/EventFingerprint.java create mode 100644 matsim/src/main/java/org/matsim/utils/eventsfilecomparison/EventsFileFingerprintComparator.java create mode 100644 matsim/src/main/java/org/matsim/utils/eventsfilecomparison/FingerprintEventHandler.java create mode 100644 matsim/src/test/java/org/matsim/utils/eventsfilecomparison/EventsFileFingerprintComparatorTest.java create mode 100644 matsim/test/input/org/matsim/utils/eventsfilecomparison/EventsFileFingerprintComparatorTest/correct.fp.zst create mode 100644 matsim/test/input/org/matsim/utils/eventsfilecomparison/EventsFileFingerprintComparatorTest/events.attribute-order.xml create mode 100644 matsim/test/input/org/matsim/utils/eventsfilecomparison/EventsFileFingerprintComparatorTest/events.diff-hash.xml create mode 100644 matsim/test/input/org/matsim/utils/eventsfilecomparison/EventsFileFingerprintComparatorTest/events.diff-num-timestamps.xml create mode 100644 matsim/test/input/org/matsim/utils/eventsfilecomparison/EventsFileFingerprintComparatorTest/events.diff-type-count.fp.zst create mode 100644 matsim/test/input/org/matsim/utils/eventsfilecomparison/EventsFileFingerprintComparatorTest/events.diff-type-count.xml create mode 100644 matsim/test/input/org/matsim/utils/eventsfilecomparison/EventsFileFingerprintComparatorTest/events.event-order-wrong_logic.xml create mode 100644 matsim/test/input/org/matsim/utils/eventsfilecomparison/EventsFileFingerprintComparatorTest/events.event-order.xml create mode 100644 matsim/test/input/org/matsim/utils/eventsfilecomparison/EventsFileFingerprintComparatorTest/events.one-more-event.xml create mode 100644 matsim/test/input/org/matsim/utils/eventsfilecomparison/EventsFileFingerprintComparatorTest/events_correct.xml diff --git a/contribs/bicycle/src/test/java/org/matsim/contrib/bicycle/run/BicycleTest.java b/contribs/bicycle/src/test/java/org/matsim/contrib/bicycle/run/BicycleTest.java index 961e43f0182..222834c746b 100644 --- a/contribs/bicycle/src/test/java/org/matsim/contrib/bicycle/run/BicycleTest.java +++ b/contribs/bicycle/src/test/java/org/matsim/contrib/bicycle/run/BicycleTest.java @@ -65,7 +65,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.matsim.utils.eventsfilecomparison.EventsFileComparator.Result.FILES_ARE_EQUAL; +import static org.matsim.utils.eventsfilecomparison.ComparisonResult.FILES_ARE_EQUAL; /** * @author dziemke diff --git a/contribs/emissions/src/test/java/org/matsim/contrib/emissions/events/VehicleLeavesTrafficEventTest.java b/contribs/emissions/src/test/java/org/matsim/contrib/emissions/events/VehicleLeavesTrafficEventTest.java index 2858e572d58..1eacd47330d 100644 --- a/contribs/emissions/src/test/java/org/matsim/contrib/emissions/events/VehicleLeavesTrafficEventTest.java +++ b/contribs/emissions/src/test/java/org/matsim/contrib/emissions/events/VehicleLeavesTrafficEventTest.java @@ -19,7 +19,7 @@ import org.matsim.core.utils.io.IOUtils; import org.matsim.examples.ExamplesUtils; import org.matsim.testcases.MatsimTestUtils; -import org.matsim.utils.eventsfilecomparison.EventsFileComparator; +import org.matsim.utils.eventsfilecomparison.ComparisonResult; import java.net.URL; @@ -81,8 +81,8 @@ public void install(){ throw new RuntimeException(e) ; } final String expected = utils.getClassInputDirectory() + emissionEventsFileName; - EventsFileComparator.Result result = EventsUtils.compareEventsFiles(expected, resultingEvents); - Assertions.assertEquals( EventsFileComparator.Result.FILES_ARE_EQUAL, result); + ComparisonResult result = EventsUtils.compareEventsFiles(expected, resultingEvents); + Assertions.assertEquals( ComparisonResult.FILES_ARE_EQUAL, result); } } diff --git a/contribs/emissions/src/test/java/org/matsim/contrib/emissions/example/RunAverageEmissionToolOfflineExampleIT.java b/contribs/emissions/src/test/java/org/matsim/contrib/emissions/example/RunAverageEmissionToolOfflineExampleIT.java index 951ed2fb237..259b8472549 100644 --- a/contribs/emissions/src/test/java/org/matsim/contrib/emissions/example/RunAverageEmissionToolOfflineExampleIT.java +++ b/contribs/emissions/src/test/java/org/matsim/contrib/emissions/example/RunAverageEmissionToolOfflineExampleIT.java @@ -30,7 +30,7 @@ import org.matsim.core.utils.io.IOUtils; import org.matsim.examples.ExamplesUtils; import org.matsim.testcases.MatsimTestUtils; -import org.matsim.utils.eventsfilecomparison.EventsFileComparator.Result; +import org.matsim.utils.eventsfilecomparison.ComparisonResult; import java.net.URL; @@ -58,8 +58,8 @@ final void testAverage_vehTypeV1() { String expected = utils.getInputDirectory() + RunAverageEmissionToolOfflineExample.emissionEventsFilename; String actual = utils.getOutputDirectory() + RunAverageEmissionToolOfflineExample.emissionEventsFilename; - Result result = EventsUtils.compareEventsFiles( expected, actual ); - Assertions.assertEquals( Result.FILES_ARE_EQUAL, result); + ComparisonResult result = EventsUtils.compareEventsFiles( expected, actual ); + Assertions.assertEquals( ComparisonResult.FILES_ARE_EQUAL, result); } @Test @@ -80,8 +80,8 @@ final void testAverage_vehTypeV2() { String expected = utils.getInputDirectory() + RunAverageEmissionToolOfflineExample.emissionEventsFilename; String actual = utils.getOutputDirectory() + RunAverageEmissionToolOfflineExample.emissionEventsFilename; - Result result = EventsUtils.compareEventsFiles( expected, actual ); - Assertions.assertEquals( Result.FILES_ARE_EQUAL, result); + ComparisonResult result = EventsUtils.compareEventsFiles( expected, actual ); + Assertions.assertEquals( ComparisonResult.FILES_ARE_EQUAL, result); } /** @@ -106,8 +106,8 @@ final void testAverage_vehTypeV2b() { String expected = utils.getInputDirectory() + RunAverageEmissionToolOfflineExample.emissionEventsFilename; String actual = utils.getOutputDirectory() + RunAverageEmissionToolOfflineExample.emissionEventsFilename; - Result result = EventsUtils.compareEventsFiles( expected, actual ); - Assertions.assertEquals( Result.FILES_ARE_EQUAL, result); + ComparisonResult result = EventsUtils.compareEventsFiles( expected, actual ); + Assertions.assertEquals( ComparisonResult.FILES_ARE_EQUAL, result); } @@ -130,7 +130,7 @@ final void testAverage_vehTypeV2_HBEFA4() { String expected = utils.getInputDirectory() + RunAverageEmissionToolOfflineExample.emissionEventsFilename; String actual = utils.getOutputDirectory() + RunAverageEmissionToolOfflineExample.emissionEventsFilename; - Result result = EventsUtils.compareEventsFiles( expected, actual ); - Assertions.assertEquals( Result.FILES_ARE_EQUAL, result); + ComparisonResult result = EventsUtils.compareEventsFiles( expected, actual ); + Assertions.assertEquals( ComparisonResult.FILES_ARE_EQUAL, result); } } diff --git a/contribs/ev/src/test/java/org/matsim/contrib/ev/example/RunEvExampleTest.java b/contribs/ev/src/test/java/org/matsim/contrib/ev/example/RunEvExampleTest.java index e8bb29fea1c..87644405c7d 100644 --- a/contribs/ev/src/test/java/org/matsim/contrib/ev/example/RunEvExampleTest.java +++ b/contribs/ev/src/test/java/org/matsim/contrib/ev/example/RunEvExampleTest.java @@ -10,7 +10,7 @@ import org.matsim.core.events.EventsUtils; import org.matsim.core.population.PopulationUtils; import org.matsim.testcases.MatsimTestUtils; -import org.matsim.utils.eventsfilecomparison.EventsFileComparator; +import org.matsim.utils.eventsfilecomparison.ComparisonResult; public class RunEvExampleTest{ @@ -39,8 +39,8 @@ public class RunEvExampleTest{ { String expected = utils.getInputDirectory() + "/output_events.xml.gz" ; String actual = utils.getOutputDirectory() + "/output_events.xml.gz" ; - EventsFileComparator.Result result = EventsUtils.compareEventsFiles( expected, actual ); - Assertions.assertEquals(EventsFileComparator.Result.FILES_ARE_EQUAL, result); + ComparisonResult result = EventsUtils.compareEventsFiles( expected, actual ); + Assertions.assertEquals(ComparisonResult.FILES_ARE_EQUAL, result); } } catch ( Exception ee ) { @@ -72,8 +72,8 @@ public class RunEvExampleTest{ { String expected = utils.getInputDirectory() + "/output_events.xml.gz" ; String actual = utils.getOutputDirectory() + "/output_events.xml.gz" ; - EventsFileComparator.Result result = EventsUtils.compareEventsFiles( expected, actual ); - Assertions.assertEquals(EventsFileComparator.Result.FILES_ARE_EQUAL, result); + ComparisonResult result = EventsUtils.compareEventsFiles( expected, actual ); + Assertions.assertEquals(ComparisonResult.FILES_ARE_EQUAL, result); } } catch ( Exception ee ) { diff --git a/contribs/ev/src/test/java/org/matsim/contrib/ev/example/RunEvExampleWithLTHConsumptionModelTest.java b/contribs/ev/src/test/java/org/matsim/contrib/ev/example/RunEvExampleWithLTHConsumptionModelTest.java index ffc5f81e296..447b79966ad 100644 --- a/contribs/ev/src/test/java/org/matsim/contrib/ev/example/RunEvExampleWithLTHConsumptionModelTest.java +++ b/contribs/ev/src/test/java/org/matsim/contrib/ev/example/RunEvExampleWithLTHConsumptionModelTest.java @@ -10,11 +10,7 @@ import org.matsim.core.events.EventsUtils; import org.matsim.core.population.PopulationUtils; import org.matsim.testcases.MatsimTestUtils; -import org.matsim.utils.eventsfilecomparison.EventsFileComparator; - -import java.io.File; -import java.net.MalformedURLException; -import java.net.URL; +import org.matsim.utils.eventsfilecomparison.ComparisonResult; public class RunEvExampleWithLTHConsumptionModelTest{ @@ -43,8 +39,8 @@ void runTest(){ { String expected = utils.getInputDirectory() + "/output_events.xml.gz" ; String actual = utils.getOutputDirectory() + "/output_events.xml.gz" ; - EventsFileComparator.Result result = EventsUtils.compareEventsFiles( expected, actual ); - Assertions.assertEquals( EventsFileComparator.Result.FILES_ARE_EQUAL, result ); + ComparisonResult result = EventsUtils.compareEventsFiles( expected, actual ); + Assertions.assertEquals( ComparisonResult.FILES_ARE_EQUAL, result ); } } catch ( Exception ee ) { diff --git a/contribs/freight/src/test/java/org/matsim/freight/carriers/usecases/chessboard/RunChessboardIT.java b/contribs/freight/src/test/java/org/matsim/freight/carriers/usecases/chessboard/RunChessboardIT.java index ced181d3172..b195b3b89d7 100644 --- a/contribs/freight/src/test/java/org/matsim/freight/carriers/usecases/chessboard/RunChessboardIT.java +++ b/contribs/freight/src/test/java/org/matsim/freight/carriers/usecases/chessboard/RunChessboardIT.java @@ -32,7 +32,7 @@ import org.matsim.core.utils.io.IOUtils; import org.matsim.examples.ExamplesUtils; import org.matsim.testcases.MatsimTestUtils; -import org.matsim.utils.eventsfilecomparison.EventsFileComparator; +import org.matsim.utils.eventsfilecomparison.ComparisonResult; public class RunChessboardIT { @@ -60,8 +60,8 @@ void runChessboard() { { String expected = utils.getInputDirectory() + "/output_events.xml.gz" ; String actual = utils.getOutputDirectory() + "/output_events.xml.gz" ; - EventsFileComparator.Result result = EventsUtils.compareEventsFiles( expected, actual ); - Assertions.assertEquals( EventsFileComparator.Result.FILES_ARE_EQUAL, result ); + ComparisonResult result = EventsUtils.compareEventsFiles( expected, actual ); + Assertions.assertEquals( ComparisonResult.FILES_ARE_EQUAL, result ); } } catch (Exception ee ) { ee.printStackTrace(); diff --git a/contribs/parking/src/test/java/org/matsim/contrib/parking/parkingchoice/run/RunParkingSearchScenarioIT.java b/contribs/parking/src/test/java/org/matsim/contrib/parking/parkingchoice/run/RunParkingSearchScenarioIT.java index fa0dc4f5ffd..e3959961195 100644 --- a/contribs/parking/src/test/java/org/matsim/contrib/parking/parkingchoice/run/RunParkingSearchScenarioIT.java +++ b/contribs/parking/src/test/java/org/matsim/contrib/parking/parkingchoice/run/RunParkingSearchScenarioIT.java @@ -33,7 +33,7 @@ import org.matsim.core.events.EventsUtils; import org.matsim.core.population.PopulationUtils; import org.matsim.testcases.MatsimTestUtils; -import org.matsim.utils.eventsfilecomparison.EventsFileComparator; +import org.matsim.utils.eventsfilecomparison.ComparisonResult; /** * @author jbischoff @@ -108,8 +108,8 @@ void testRunParkingDistanceMemoryStrategy() { { String expected = utils.getInputDirectory() + "/output_events.xml.gz"; String actual = utils.getOutputDirectory() + "/output_events.xml.gz"; - EventsFileComparator.Result result = EventsUtils.compareEventsFiles(expected, actual); - Assertions.assertEquals(EventsFileComparator.Result.FILES_ARE_EQUAL, result); + ComparisonResult result = EventsUtils.compareEventsFiles(expected, actual); + Assertions.assertEquals(ComparisonResult.FILES_ARE_EQUAL, result); } } catch (Exception e) { e.printStackTrace(); @@ -146,8 +146,8 @@ void testRunParkingNearestParkingSpotStrategy() { { String expected = utils.getInputDirectory() + "/output_events.xml.gz"; String actual = utils.getOutputDirectory() + "/output_events.xml.gz"; - EventsFileComparator.Result result = EventsUtils.compareEventsFiles(expected, actual); - Assertions.assertEquals(EventsFileComparator.Result.FILES_ARE_EQUAL, result); + ComparisonResult result = EventsUtils.compareEventsFiles(expected, actual); + Assertions.assertEquals(ComparisonResult.FILES_ARE_EQUAL, result); } } catch (Exception e) { e.printStackTrace(); diff --git a/contribs/parking/src/test/java/org/matsim/contrib/parking/parkingproxy/run/RunWithParkingProxyIT.java b/contribs/parking/src/test/java/org/matsim/contrib/parking/parkingproxy/run/RunWithParkingProxyIT.java index 6d17839c44c..e87a4afbaba 100644 --- a/contribs/parking/src/test/java/org/matsim/contrib/parking/parkingproxy/run/RunWithParkingProxyIT.java +++ b/contribs/parking/src/test/java/org/matsim/contrib/parking/parkingproxy/run/RunWithParkingProxyIT.java @@ -30,7 +30,7 @@ import org.matsim.core.utils.io.IOUtils; import org.matsim.examples.ExamplesUtils; import org.matsim.testcases.MatsimTestUtils; -import org.matsim.utils.eventsfilecomparison.EventsFileComparator.Result; +import org.matsim.utils.eventsfilecomparison.ComparisonResult; public class RunWithParkingProxyIT { private static final Logger log = LogManager.getLogger(RunWithParkingProxyIT.class); @@ -50,8 +50,8 @@ void testMain(){ { String expected = utils.getInputDirectory() + "/output_events.xml.gz" ; String actual = utils.getOutputDirectory() + "/output_events.xml.gz" ; - Result result = EventsUtils.compareEventsFiles( expected, actual ); - if(!result.equals(Result.FILES_ARE_EQUAL)) { + ComparisonResult result = EventsUtils.compareEventsFiles( expected, actual ); + if(!result.equals(ComparisonResult.FILES_ARE_EQUAL)) { throw new RuntimeException("Events comparison ended with result " + result.name()); } } diff --git a/contribs/roadpricing/src/test/java/org/matsim/contrib/roadpricing/run/RoadPricingByConfigfileTest.java b/contribs/roadpricing/src/test/java/org/matsim/contrib/roadpricing/run/RoadPricingByConfigfileTest.java index 70b4bc9344d..e59e74dd90b 100644 --- a/contribs/roadpricing/src/test/java/org/matsim/contrib/roadpricing/run/RoadPricingByConfigfileTest.java +++ b/contribs/roadpricing/src/test/java/org/matsim/contrib/roadpricing/run/RoadPricingByConfigfileTest.java @@ -32,7 +32,7 @@ import org.matsim.core.utils.io.IOUtils; import org.matsim.examples.ExamplesUtils; import org.matsim.testcases.MatsimTestUtils; -import org.matsim.utils.eventsfilecomparison.EventsFileComparator; +import org.matsim.utils.eventsfilecomparison.ComparisonResult; /** * @author vsp-gleich @@ -55,7 +55,7 @@ final void testMain() { { String expected = utils.getInputDirectory() + "/output_events.xml.gz" ; String actual = utils.getOutputDirectory() + "/output_events.xml.gz" ; - Assertions.assertEquals(EventsFileComparator.Result.FILES_ARE_EQUAL, EventsUtils.compareEventsFiles( expected, actual )); + Assertions.assertEquals(ComparisonResult.FILES_ARE_EQUAL, EventsUtils.compareEventsFiles( expected, actual )); } { final Population expected = PopulationUtils.createPopulation( ConfigUtils.createConfig() ); diff --git a/contribs/signals/src/test/java/org/matsim/contrib/signals/builder/TravelTimeFourWaysTest.java b/contribs/signals/src/test/java/org/matsim/contrib/signals/builder/TravelTimeFourWaysTest.java index 9e341d444a6..ba97489f89e 100644 --- a/contribs/signals/src/test/java/org/matsim/contrib/signals/builder/TravelTimeFourWaysTest.java +++ b/contribs/signals/src/test/java/org/matsim/contrib/signals/builder/TravelTimeFourWaysTest.java @@ -38,8 +38,7 @@ import org.matsim.core.scenario.ScenarioUtils; import org.matsim.testcases.MatsimTestUtils; import org.matsim.utils.eventsfilecomparison.EventsFileComparator; - -import static org.matsim.utils.eventsfilecomparison.EventsFileComparator.*; +import org.matsim.utils.eventsfilecomparison.ComparisonResult; /** * @author aneumann @@ -125,7 +124,7 @@ private void runQSimWithSignals(final Scenario scenario) { eventsXmlWriter.closeFile(); // Assert.assertEquals("different events files", EventsFileComparator.compareAndReturnInt(this.testUtils.getInputDirectory() + EVENTSFILE, eventsOut), 0); - Assertions.assertEquals( Result.FILES_ARE_EQUAL, new EventsFileComparator().setIgnoringCoordinates( true ).runComparison( this.testUtils.getInputDirectory() + EVENTSFILE, eventsOut ) ); + Assertions.assertEquals( ComparisonResult.FILES_ARE_EQUAL, new EventsFileComparator().setIgnoringCoordinates( true ).runComparison( this.testUtils.getInputDirectory() + EVENTSFILE, eventsOut ) ); } } diff --git a/contribs/signals/src/test/java/org/matsim/contrib/signals/integration/SignalSystemsIT.java b/contribs/signals/src/test/java/org/matsim/contrib/signals/integration/SignalSystemsIT.java index 49e8af1f140..b44a5cf35ea 100644 --- a/contribs/signals/src/test/java/org/matsim/contrib/signals/integration/SignalSystemsIT.java +++ b/contribs/signals/src/test/java/org/matsim/contrib/signals/integration/SignalSystemsIT.java @@ -36,6 +36,7 @@ import org.matsim.core.scenario.ScenarioUtils; import org.matsim.testcases.MatsimTestUtils; import org.matsim.utils.eventsfilecomparison.EventsFileComparator; +import org.matsim.utils.eventsfilecomparison.ComparisonResult; import java.io.File; @@ -88,7 +89,7 @@ void testSignalSystems() { //iteration 0 String iterationOutput = controlerOutputDir + "ITERS/it.0/"; - Assertions.assertEquals(EventsFileComparator.Result.FILES_ARE_EQUAL, + Assertions.assertEquals(ComparisonResult.FILES_ARE_EQUAL, new EventsFileComparator().setIgnoringCoordinates( true ).runComparison( inputDirectory + "0.events.xml.gz", iterationOutput + "0.events.xml.gz"), "different events files after iteration 0 " @@ -113,7 +114,7 @@ void testSignalSystems() { //iteration 10 String iterationOutput = controlerOutputDir + "ITERS/it.10/"; - Assertions.assertEquals(EventsFileComparator.Result.FILES_ARE_EQUAL, + Assertions.assertEquals(ComparisonResult.FILES_ARE_EQUAL, new EventsFileComparator().setIgnoringCoordinates( true ).runComparison( inputDirectory + "10.events.xml.gz", iterationOutput + "10.events.xml.gz" ), "different event files after iteration 10" ); @@ -181,7 +182,7 @@ void testSignalSystemsWTryEndTimeThenDuration() { //iteration 0 String iterationOutput = controlerOutputDir + "ITERS/it.0/"; - Assertions.assertEquals(EventsFileComparator.Result.FILES_ARE_EQUAL, + Assertions.assertEquals(ComparisonResult.FILES_ARE_EQUAL, new EventsFileComparator().setIgnoringCoordinates( true ).runComparison( inputDirectory + "0.events.xml.gz", iterationOutput + "0.events.xml.gz"), "different events files after iteration 0 " @@ -206,7 +207,7 @@ void testSignalSystemsWTryEndTimeThenDuration() { //iteration 10 String iterationOutput = controlerOutputDir + "ITERS/it.10/"; - Assertions.assertEquals(EventsFileComparator.Result.FILES_ARE_EQUAL, + Assertions.assertEquals(ComparisonResult.FILES_ARE_EQUAL, new EventsFileComparator().setIgnoringCoordinates( true ).runComparison( inputDirectory + "10.events.xml.gz", iterationOutput + "10.events.xml.gz"), "different event files after iteration 10" ); diff --git a/contribs/small-scale-traffic-generation/src/test/java/org/matsim/smallScaleCommercialTrafficGeneration/RunGenerateSmallScaleCommercialTrafficTest.java b/contribs/small-scale-traffic-generation/src/test/java/org/matsim/smallScaleCommercialTrafficGeneration/RunGenerateSmallScaleCommercialTrafficTest.java index d88d2318fd1..b5e92503851 100644 --- a/contribs/small-scale-traffic-generation/src/test/java/org/matsim/smallScaleCommercialTrafficGeneration/RunGenerateSmallScaleCommercialTrafficTest.java +++ b/contribs/small-scale-traffic-generation/src/test/java/org/matsim/smallScaleCommercialTrafficGeneration/RunGenerateSmallScaleCommercialTrafficTest.java @@ -34,7 +34,7 @@ import org.matsim.freight.carriers.CarriersUtils; import org.matsim.freight.carriers.FreightCarriersConfigGroup; import org.matsim.testcases.MatsimTestUtils; -import org.matsim.utils.eventsfilecomparison.EventsFileComparator; +import org.matsim.utils.eventsfilecomparison.ComparisonResult; import java.io.File; import java.util.Objects; @@ -119,7 +119,7 @@ void testMainRunAndResults() { // compare events String expected = utils.getPackageInputDirectory() + "test.output_events.xml.gz" ; String actual = utils.getOutputDirectory() + "test.output_events.xml.gz" ; - EventsFileComparator.Result result = EventsUtils.compareEventsFiles( expected, actual ); - Assertions.assertEquals( EventsFileComparator.Result.FILES_ARE_EQUAL, result ); + ComparisonResult result = EventsUtils.compareEventsFiles( expected, actual ); + Assertions.assertEquals( ComparisonResult.FILES_ARE_EQUAL, result ); } } diff --git a/contribs/taxi/src/test/java/org/matsim/contrib/etaxi/run/RunETaxiScenarioIT.java b/contribs/taxi/src/test/java/org/matsim/contrib/etaxi/run/RunETaxiScenarioIT.java index c086adedc50..5e029c8a29e 100644 --- a/contribs/taxi/src/test/java/org/matsim/contrib/etaxi/run/RunETaxiScenarioIT.java +++ b/contribs/taxi/src/test/java/org/matsim/contrib/etaxi/run/RunETaxiScenarioIT.java @@ -29,7 +29,7 @@ import org.matsim.core.utils.io.IOUtils; import org.matsim.examples.ExamplesUtils; import org.matsim.testcases.MatsimTestUtils; -import org.matsim.utils.eventsfilecomparison.EventsFileComparator; +import org.matsim.utils.eventsfilecomparison.ComparisonResult; /** * @author michalm @@ -73,8 +73,8 @@ private void runScenario(String configPath) { { String expected = utils.getInputDirectory() + "/output_events.xml.gz"; String actual = utils.getOutputDirectory() + "/output_events.xml.gz"; - EventsFileComparator.Result result = EventsUtils.compareEventsFiles(expected, actual); - Assertions.assertEquals(EventsFileComparator.Result.FILES_ARE_EQUAL, result); + ComparisonResult result = EventsUtils.compareEventsFiles(expected, actual); + Assertions.assertEquals(ComparisonResult.FILES_ARE_EQUAL, result); } } } diff --git a/contribs/taxi/src/test/java/org/matsim/contrib/taxi/optimizer/TaxiOptimizerTests.java b/contribs/taxi/src/test/java/org/matsim/contrib/taxi/optimizer/TaxiOptimizerTests.java index ca985585f82..3a3800467af 100644 --- a/contribs/taxi/src/test/java/org/matsim/contrib/taxi/optimizer/TaxiOptimizerTests.java +++ b/contribs/taxi/src/test/java/org/matsim/contrib/taxi/optimizer/TaxiOptimizerTests.java @@ -35,7 +35,7 @@ import org.matsim.core.utils.io.IOUtils; import org.matsim.examples.ExamplesUtils; import org.matsim.testcases.MatsimTestUtils; -import org.matsim.utils.eventsfilecomparison.EventsFileComparator; +import org.matsim.utils.eventsfilecomparison.ComparisonResult; public class TaxiOptimizerTests { public static void runBenchmark(boolean vehicleDiversion, AbstractTaxiOptimizerParams taxiOptimizerParams, MatsimTestUtils utils) { @@ -70,8 +70,8 @@ public static void runBenchmark(boolean vehicleDiversion, AbstractTaxiOptimizerP { String expected = utils.getInputDirectory() + "/output_events.xml.gz"; String actual = utils.getOutputDirectory() + "/output_events.xml.gz"; - EventsFileComparator.Result result = EventsUtils.compareEventsFiles(expected, actual); - Assertions.assertEquals(EventsFileComparator.Result.FILES_ARE_EQUAL, result); + ComparisonResult result = EventsUtils.compareEventsFiles(expected, actual); + Assertions.assertEquals(ComparisonResult.FILES_ARE_EQUAL, result); } } } diff --git a/contribs/vsp/src/test/java/playground/vsp/ev/UrbanEVIT.java b/contribs/vsp/src/test/java/playground/vsp/ev/UrbanEVIT.java index 823fb89bf83..b96c04fb079 100644 --- a/contribs/vsp/src/test/java/playground/vsp/ev/UrbanEVIT.java +++ b/contribs/vsp/src/test/java/playground/vsp/ev/UrbanEVIT.java @@ -9,7 +9,7 @@ import org.matsim.core.events.EventsUtils; import org.matsim.core.population.PopulationUtils; import org.matsim.testcases.MatsimTestUtils; -import org.matsim.utils.eventsfilecomparison.EventsFileComparator; +import org.matsim.utils.eventsfilecomparison.ComparisonResult; public class UrbanEVIT { @RegisterExtension private MatsimTestUtils utils = new MatsimTestUtils(); @@ -38,8 +38,8 @@ void run() { { String expected = utils.getInputDirectory() + "/output_events.xml.gz" ; String actual = utils.getOutputDirectory() + "/output_events.xml.gz" ; - EventsFileComparator.Result result = EventsUtils.compareEventsFiles( expected, actual ); - Assertions.assertEquals( EventsFileComparator.Result.FILES_ARE_EQUAL, result ); + ComparisonResult result = EventsUtils.compareEventsFiles( expected, actual ); + Assertions.assertEquals( ComparisonResult.FILES_ARE_EQUAL, result ); } } catch ( Exception ee ) { diff --git a/matsim/.gitignore b/matsim/.gitignore index 029e60cc779..a35b8f59ee1 100644 --- a/matsim/.gitignore +++ b/matsim/.gitignore @@ -3,7 +3,6 @@ test/output bin .settings -bin target output out @@ -11,3 +10,4 @@ src/main/java/.gitignore /output_fastCapacityUpdate_false/ /output_fastCapacityUpdate_true/ /nullevents.xml.gz +*.zst \ No newline at end of file diff --git a/matsim/src/main/java/org/matsim/core/events/EventsUtils.java b/matsim/src/main/java/org/matsim/core/events/EventsUtils.java index 1f2ba7e3d33..7601a9091ad 100644 --- a/matsim/src/main/java/org/matsim/core/events/EventsUtils.java +++ b/matsim/src/main/java/org/matsim/core/events/EventsUtils.java @@ -21,19 +21,26 @@ package org.matsim.core.events; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.matsim.core.api.experimental.events.EventsManager; import org.matsim.core.config.Config; import org.matsim.core.controler.Injector; -import org.matsim.utils.eventsfilecomparison.EventsFileComparator; +import org.matsim.utils.eventsfilecomparison.*; + +import java.io.File; public final class EventsUtils { + private static final Logger log = LogManager.getLogger(EventsUtils.class); + + /** * Create a events manager instance that guarantees causality of processed events across all handlers. */ - public static EventsManager createEventsManager() { + public static EventsManager createEventsManager() { return new EventsManagerImpl(); - } + } /** * Creates a parallel events manager, with no guarantees for the order of processed events between multiple handlers. @@ -43,11 +50,16 @@ public static EventsManager createParallelEventsManager() { } public static EventsManager createEventsManager(Config config) { - final EventsManager events = Injector.createInjector( config, new EventsManagerModule() ).getInstance( EventsManager.class ); + final EventsManager events = Injector.createInjector(config, new EventsManagerModule()).getInstance(EventsManager.class); // events.initProcessing(); return events; } + public static void readEvents(EventsManager events, String filename) { + new MatsimEventsReader(events).readFile(filename); + } + + /** * The SimStepParallelEventsManagerImpl can handle events from multiple threads. * The (Parallel)EventsMangerImpl cannot, therefore it has to be wrapped into a @@ -58,21 +70,68 @@ public static EventsManager getParallelFeedableInstance(EventsManager events) { return events; } else if (events instanceof ParallelEventsManager) { return events; - } - else if (events instanceof SynchronizedEventsManagerImpl) { + } else if (events instanceof SynchronizedEventsManagerImpl) { return events; } else { return new SynchronizedEventsManagerImpl(events); } } - public static void readEvents( EventsManager events, String filename ) { - new MatsimEventsReader(events).readFile(filename) ; + /** + * Create and write fingerprint file for events. + */ + public static FingerprintEventHandler createEventsFingerprint(String eventFile, String outputFingerprintFile) { + + FingerprintEventHandler handler = EventsFileFingerprintComparator.createFingerprintHandler(eventFile, null); + + EventFingerprint.write(outputFingerprintFile, handler.getEventFingerprint()); + + return handler; + } + + + /** + * Compares existing event file against fingerprint file. This will also create new fingerprint file along the input events. + * + * @param eventFile local events file + * @param compareFingerprintFile path or uri to fingerprint file + * + * @return comparison results + */ + public static ComparisonResult createAndCompareEventsFingerprint(File eventFile, String compareFingerprintFile) { + + String path = eventFile.getPath().replaceFirst("\\.xml[.a-zA-z0-9]*$", ""); + + FingerprintEventHandler handler = EventsFileFingerprintComparator.createFingerprintHandler(eventFile.toString(), compareFingerprintFile); + EventFingerprint.write(path + ".fp.zst", handler.getEventFingerprint()); + + if (handler.getComparisonMessage() != null) + log.warn(handler.getComparisonMessage()); + + return handler.getComparisonResult(); + } + + /** + * Compares existing event file against fingerprint file. This will also create new fingerprint file along the input events. + * + * @throws AssertionError if comparison fails + * @see #createAndCompareEventsFingerprint(File, String) + */ + public static void assertEqualEventsFingerprint(File eventFile, String compareFingerprintFile) { + + String path = eventFile.getPath().replaceFirst("\\.xml[.a-zA-z0-9]*$", ""); + + FingerprintEventHandler handler = EventsFileFingerprintComparator.createFingerprintHandler(eventFile.toString(), compareFingerprintFile); + EventFingerprint.write(path + ".fp.zst", handler.getEventFingerprint()); + + + if (handler.getComparisonResult() != ComparisonResult.FILES_ARE_EQUAL) + throw new AssertionError(handler.getComparisonMessage()); + } - public static EventsFileComparator.Result compareEventsFiles( String filename1, String filename2 ) { - EventsFileComparator.Result result = EventsFileComparator.compare( filename1, filename2 ); - return result ; + public static ComparisonResult compareEventsFiles(String filename1, String filename2) { + return EventsFileComparator.compare(filename1, filename2); } } diff --git a/matsim/src/main/java/org/matsim/utils/eventsfilecomparison/ComparisonResult.java b/matsim/src/main/java/org/matsim/utils/eventsfilecomparison/ComparisonResult.java new file mode 100644 index 00000000000..b5e10fa2d65 --- /dev/null +++ b/matsim/src/main/java/org/matsim/utils/eventsfilecomparison/ComparisonResult.java @@ -0,0 +1,6 @@ +package org.matsim.utils.eventsfilecomparison; + +/** + * Result of event file comparison. + */ +public enum ComparisonResult {FILES_ARE_EQUAL, DIFFERENT_NUMBER_OF_TIMESTEPS, DIFFERENT_TIMESTEPS, DIFFERENT_EVENT_ATTRIBUTES, MISSING_EVENT, WRONG_EVENT_COUNT, FILE_ERROR} diff --git a/matsim/src/main/java/org/matsim/utils/eventsfilecomparison/EventFingerprint.java b/matsim/src/main/java/org/matsim/utils/eventsfilecomparison/EventFingerprint.java new file mode 100644 index 00000000000..74f254f746d --- /dev/null +++ b/matsim/src/main/java/org/matsim/utils/eventsfilecomparison/EventFingerprint.java @@ -0,0 +1,165 @@ +package org.matsim.utils.eventsfilecomparison; + +import it.unimi.dsi.fastutil.floats.FloatArrayList; +import it.unimi.dsi.fastutil.floats.FloatList; +import it.unimi.dsi.fastutil.objects.Object2IntMap; +import it.unimi.dsi.fastutil.objects.Object2IntOpenHashMap; +import org.matsim.core.utils.io.IOUtils; + +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.Arrays; +import java.util.Map; + +/** + * Class holding reduced information about an events file. + * If two such fingerprint are different, one can conclude that the event files are semantically different. + *

+ * The fingerprint is based on the following information: + * - Array of all timestamps + * - Counts of each event type + * - Hash of string concatenation of all event strings + * + * Note: Events with the same timestamp are allowed to occur in any order. + */ +public final class EventFingerprint { + + /** + * Header for version 1, FP/1 + */ + static final int HEADER_V1 = 0x46502f31; + + final FloatList timeArray; + final Object2IntMap eventTypeCounter; + final byte[] hash; + + /** + * Builder for the hash. + */ + private final MessageDigest digest; + + private EventFingerprint(FloatList timeArray, Object2IntMap eventTypeCounter, byte[] hash) { + this.timeArray = timeArray; + this.eventTypeCounter = eventTypeCounter; + this.hash = hash; + this.digest = null; + } + + EventFingerprint() { + this.timeArray = new FloatArrayList(); + this.eventTypeCounter = new Object2IntOpenHashMap<>(); + this.hash = new byte[20]; + + try { + digest = MessageDigest.getInstance("SHA-1"); + } catch (NoSuchAlgorithmException e) { + throw new IllegalStateException("Hashing not supported;"); + } + } + + public static void write(String filePath, EventFingerprint eventFingerprint) { + try (DataOutputStream dataOutputStream = new DataOutputStream(IOUtils.getOutputStream(IOUtils.getFileUrl(filePath), false))) { + // Write header and version + dataOutputStream.writeInt(EventFingerprint.HEADER_V1); + + // Write time array size and elements + dataOutputStream.writeInt(eventFingerprint.timeArray.size()); + for (float time : eventFingerprint.timeArray) { + dataOutputStream.writeFloat(time); + } + + // Write event type counter map size and elements + dataOutputStream.writeInt(eventFingerprint.eventTypeCounter.size()); + for (Map.Entry entry : eventFingerprint.eventTypeCounter.entrySet()) { + dataOutputStream.writeUTF(entry.getKey()); + dataOutputStream.writeInt(entry.getValue()); + } + + // Hash should always be computed at this point + assert !Arrays.equals(eventFingerprint.hash, new byte[20]) : "Hash was not computed"; + + // Write byte hash + dataOutputStream.write(eventFingerprint.hash); + + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + public static EventFingerprint read(String fingerprintPath) throws IOException { + EventFingerprint eventFingerprint; + + try (DataInputStream dataInputStream = new DataInputStream(IOUtils.getInputStream(IOUtils.getFileUrl(fingerprintPath)))) { + // Read header and version + int fileHeader = dataInputStream.readInt(); + + if (fileHeader != EventFingerprint.HEADER_V1) { + throw new IllegalArgumentException("Invalid fingerprint file header"); + } + + // Read time array + int timeArraySize = dataInputStream.readInt(); + FloatList timeArray = new FloatArrayList(); + for (int i = 0; i < timeArraySize; i++) { + timeArray.add(dataInputStream.readFloat()); + } + + // Read event type counter map + int eventTypeCounterSize = dataInputStream.readInt(); + Object2IntMap eventTypeCounter = new Object2IntOpenHashMap<>(); + for (int i = 0; i < eventTypeCounterSize; i++) { + String eventType = dataInputStream.readUTF(); + int count = dataInputStream.readInt(); + eventTypeCounter.put(eventType, count); + } + + // Read string hash + byte[] hash = dataInputStream.readNBytes(20); + + // Create EventFingerprint object + eventFingerprint = new EventFingerprint(timeArray, eventTypeCounter, hash); + } + + return eventFingerprint; + } + + void addTimeStamp(double timestamp) { + timeArray.add((float) timestamp); + } + + void addEventType(String str) { + // Increment the count for the given string + eventTypeCounter.mergeInt(str, 1, Integer::sum); + } + + void addHashCode(String stringToAdd) { + if (stringToAdd == null) { + return; + } + + digest.update(stringToAdd.getBytes(StandardCharsets.UTF_8)); + } + + byte[] computeHash() { + if (this.digest == null) + throw new IllegalStateException("Hash was from from input and can not be computed"); + + byte[] digest = this.digest.digest(); + System.arraycopy(digest, 0, hash, 0, hash.length); + return hash; + } + + @Override + public String toString() { + return "EventFingerprint{" + + "timeArray=" + timeArray.size() + + ", eventTypeCounter=" + eventTypeCounter + + ", hash=" + Arrays.toString(hash) + + '}'; + } +} diff --git a/matsim/src/main/java/org/matsim/utils/eventsfilecomparison/EventsFileComparator.java b/matsim/src/main/java/org/matsim/utils/eventsfilecomparison/EventsFileComparator.java index 66c44705ca7..0b759f11e95 100644 --- a/matsim/src/main/java/org/matsim/utils/eventsfilecomparison/EventsFileComparator.java +++ b/matsim/src/main/java/org/matsim/utils/eventsfilecomparison/EventsFileComparator.java @@ -37,8 +37,6 @@ public final class EventsFileComparator { private static final Logger log = LogManager.getLogger(EventsFileComparator.class); - public enum Result { FILES_ARE_EQUAL, DIFFERENT_NUMBER_OF_TIMESTEPS, DIFFERENT_TIMESTEPS, MISSING_EVENT, WRONG_EVENT_COUNT } - private boolean ignoringCoordinates = false; public EventsFileComparator setIgnoringCoordinates( boolean ignoringCoordinates ){ this.ignoringCoordinates = ignoringCoordinates; @@ -62,13 +60,13 @@ public static void main(String[] args) { * * @param filename1 name of the first event file * @param filename2 name of the second event file - * @return Result.FILES_ARE_EQUAL if the events files are equal, or some error code (see {@link Result}) if not. + * @return Result.FILES_ARE_EQUAL if the events files are equal, or some error code (see {@link ComparisonResult}) if not. */ - public static Result compare(final String filename1, final String filename2) { + public static ComparisonResult compare(final String filename1, final String filename2) { return new EventsFileComparator().runComparison( filename1, filename2 ); } - public Result runComparison( final String filename1, final String filename2 ) { + public ComparisonResult runComparison(final String filename1, final String filename2 ) { // (need method name different from pre-existing static method. kai, feb'20) EventsComparator comparator = new EventsComparator( ); @@ -90,8 +88,8 @@ public Result runComparison( final String filename1, final String filename2 ) { e.printStackTrace(); } - Result retCode = comparator.retCode; - if (retCode == Result.FILES_ARE_EQUAL) { + ComparisonResult retCode = comparator.retCode; + if (retCode == ComparisonResult.FILES_ARE_EQUAL) { log.info("Event files are semantically equivalent."); } else { log.warn("Event files differ."); @@ -103,7 +101,7 @@ private static class EventsComparator implements Runnable { private Worker worker1 = null; private Worker worker2 = null; - private volatile Result retCode = null ; + private volatile ComparisonResult retCode = null ; /*package*/ void setWorkers(final Worker w1, final Worker w2) { this.worker1 = w1; @@ -114,13 +112,13 @@ private static class EventsComparator implements Runnable { public void run() { if (this.worker1.getCurrentTime() != this.worker2.getCurrentTime()) { log.warn("Differnt time steps in event files!"); - setExitCode(Result.DIFFERENT_TIMESTEPS); + setExitCode(ComparisonResult.DIFFERENT_TIMESTEPS); return; } if (this.worker1.isFinished() != this.worker2.isFinished()) { log.warn("Events files have different number of time steps!"); - setExitCode(Result.DIFFERENT_NUMBER_OF_TIMESTEPS); + setExitCode(ComparisonResult.DIFFERENT_NUMBER_OF_TIMESTEPS); return; } @@ -141,7 +139,7 @@ public void run() { log.warn("The event:"); log.warn(entry.getKey()); log.warn("is missing in events file:" + worker2.getEventsFile()); - setExitCode(Result.MISSING_EVENT); + setExitCode(ComparisonResult.MISSING_EVENT); problem = true; if (logCounter == 50) { log.warn(Gbl.FUTURE_SUPPRESSED); @@ -152,7 +150,7 @@ public void run() { log.warn( "Wrong event count for: " + entry.getKey() + "\n" + entry.getValue().getCount() + " times in file:" + worker1.getEventsFile() + "\n" + counter.getCount() + " times in file:" + worker2.getEventsFile() ); - setExitCode( Result.WRONG_EVENT_COUNT ); + setExitCode( ComparisonResult.WRONG_EVENT_COUNT ); problem = true; } } @@ -168,7 +166,7 @@ public void run() { log.warn("The event:"); log.warn(e.getKey()); log.warn("is missing in events file:" + worker1.getEventsFile()); - setExitCode(Result.MISSING_EVENT); + setExitCode(ComparisonResult.MISSING_EVENT); problem = true; if (logCounter == 50) { log.warn(Gbl.FUTURE_SUPPRESSED); @@ -182,17 +180,25 @@ public void run() { } if (this.worker1.isFinished()) { - setExitCode(Result.FILES_ARE_EQUAL); + setExitCode(ComparisonResult.FILES_ARE_EQUAL); } } - private void setExitCode(final Result errCode) { + private void setExitCode(final ComparisonResult errCode) { this.retCode= errCode; - if (errCode != Result.FILES_ARE_EQUAL) { + if (errCode != ComparisonResult.FILES_ARE_EQUAL) { this.worker1.interrupt(); this.worker2.interrupt(); } } } + /** + * Don't use this enum. See deprecation message. + * @deprecated Use {@link ComparisonResult} instead. This enum is not used anymore and empty now. + */ + @Deprecated + public enum Result { + } + } diff --git a/matsim/src/main/java/org/matsim/utils/eventsfilecomparison/EventsFileFingerprintComparator.java b/matsim/src/main/java/org/matsim/utils/eventsfilecomparison/EventsFileFingerprintComparator.java new file mode 100644 index 00000000000..2152758b215 --- /dev/null +++ b/matsim/src/main/java/org/matsim/utils/eventsfilecomparison/EventsFileFingerprintComparator.java @@ -0,0 +1,124 @@ +package org.matsim.utils.eventsfilecomparison; + +import it.unimi.dsi.fastutil.objects.Object2IntMap; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.matsim.core.api.experimental.events.EventsManager; +import org.matsim.core.events.EventsUtils; + +import javax.annotation.Nullable; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.Arrays; + +/** + * Utility class for comparing events and fingerprints. + */ +public final class EventsFileFingerprintComparator { + + private static final Logger log = LogManager.getLogger(EventsFileComparator.class); + + private EventsFileFingerprintComparator() { + } + + + /** + * Create and compare event fingerprints and return the handler holding resulting information. + */ + public static FingerprintEventHandler createFingerprintHandler(final String eventsfile, @Nullable String compareFingerprint) { + + EventFingerprint fp = null; + Exception err = null; + if (compareFingerprint != null) { + try { + fp = EventFingerprint.read(compareFingerprint); + } catch (Exception e) { + log.warn("Could not read compare fingerprint from file: {}", compareFingerprint, e); + fp = new EventFingerprint(); + err = e; + } + } + + FingerprintEventHandler handler = new FingerprintEventHandler(fp); + + EventsManager manager = EventsUtils.createEventsManager(); + + manager.addHandler(handler); + + EventsUtils.readEvents(manager, eventsfile); + + manager.finishProcessing(); + handler.finishProcessing(); + + // File error overwrite any other error + if (err != null) { + handler.setComparisonResult(ComparisonResult.FILE_ERROR); + handler.setComparisonMessage(err.getMessage()); + } + + return handler; + } + + public static ComparisonResult compareFingerprints(final String fp1, final String fp2) { + + EventFingerprint fingerprint1; + EventFingerprint fingerprint2; + try { + fingerprint1 = EventFingerprint.read(fp1); + fingerprint2 = EventFingerprint.read(fp2); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + + String logMessage = ""; + //Check if time array size is the same + if (fingerprint1.timeArray.size() != fingerprint2.timeArray.size()) { + logMessage = "Different number of timesteps"; + log.warn(logMessage); + return ComparisonResult.DIFFERENT_NUMBER_OF_TIMESTEPS; + } + + //Check if both time arrays have the same timesteps + if (!Arrays.equals(fingerprint1.timeArray.toFloatArray(), fingerprint2.timeArray.toFloatArray())) { + logMessage = "Different timesteps"; + log.warn(logMessage); + return ComparisonResult.DIFFERENT_TIMESTEPS; + } + + + //Check which event type counts are different among 2 fingerprints + boolean countDiffers = false; + for (Object2IntMap.Entry entry1 : fingerprint1.eventTypeCounter.object2IntEntrySet()) { + String key = entry1.getKey(); + int count1 = entry1.getIntValue(); + int count2 = fingerprint2.eventTypeCounter.getInt(key); + if (count1 != count2) { + countDiffers = true; + + if (!logMessage.isEmpty()) + logMessage += "\n"; + + logMessage += "Count for event type '%s' differs: %d != %d".formatted(key, count1, count2); + } + } + if (countDiffers) { + log.warn(logMessage); + return ComparisonResult.WRONG_EVENT_COUNT; + } + + + //Check if total hash is the same + byte[] hash1 = fingerprint1.hash; + byte[] hash2 = fingerprint2.hash; + if (!Arrays.equals(hash1, hash2)) { + + logMessage = String.format("Difference occurred hash codes hash of first file is %s, hash of second is %s", Arrays.toString(hash1), Arrays.toString(hash2)); + + log.warn(logMessage); + return ComparisonResult.DIFFERENT_EVENT_ATTRIBUTES; + } + + return ComparisonResult.FILES_ARE_EQUAL; + + } +} diff --git a/matsim/src/main/java/org/matsim/utils/eventsfilecomparison/FingerprintEventHandler.java b/matsim/src/main/java/org/matsim/utils/eventsfilecomparison/FingerprintEventHandler.java new file mode 100644 index 00000000000..8e12c10a53c --- /dev/null +++ b/matsim/src/main/java/org/matsim/utils/eventsfilecomparison/FingerprintEventHandler.java @@ -0,0 +1,212 @@ +package org.matsim.utils.eventsfilecomparison; + +import it.unimi.dsi.fastutil.floats.FloatListIterator; +import it.unimi.dsi.fastutil.objects.Object2IntMap; +import org.matsim.api.core.v01.events.Event; +import org.matsim.core.events.handler.BasicEventHandler; + +import java.util.*; + +/** + * Handler for creating and comparing {@link EventFingerprint}. + */ +public final class FingerprintEventHandler implements BasicEventHandler { + + /** + * Generated finger print. + */ + private final EventFingerprint eventFingerprint = new EventFingerprint(); + + /** + * Precision for timestamp comparison. + */ + private static final float EPS = 1e-8f; + + /** + * Accumulate all event strings for same timestamps. + */ + private final List hashAccumulationList = new ArrayList<>(); + + /** + * Existing fingerprint for comparison against event file. Can be null, then no comparison is performed. + */ + private final EventFingerprint compareFingerprint; + + private FloatListIterator iterator = null; + + /** + * Result of the comparison. + */ + private ComparisonResult comparisonResult; + private String comparisonMessage; + + public FingerprintEventHandler() { + this.compareFingerprint = null; + } + + public FingerprintEventHandler(EventFingerprint compareFingerprint) { + this.compareFingerprint = compareFingerprint; + this.comparisonResult = null; + } + + public EventFingerprint getEventFingerprint() { + return eventFingerprint; + } + + public ComparisonResult getComparisonResult() { + return comparisonResult; + } + + void setComparisonResult(ComparisonResult comparisonResult) { + this.comparisonResult = comparisonResult; + } + + public String getComparisonMessage() { + return comparisonMessage; + } + + void setComparisonMessage(String comparisonMessage) { + this.comparisonMessage = comparisonMessage; + } + + @Override + public void handleEvent(Event event) { + + + String lexicographicSortedString = toLexicographicSortedString(event); + + if (compareFingerprint != null) { + if (iterator == null) { + this.iterator = compareFingerprint.timeArray.iterator(); + } + + if (this.comparisonResult == null) { + if (iterator.hasNext()) { + float entry = iterator.nextFloat(); + //Comparing floats with precision + if (Math.abs((float) event.getTime() - entry) >= EPS) { + this.comparisonResult = ComparisonResult.DIFFERENT_TIMESTEPS; + this.comparisonMessage = "Difference occurred in this event time=" + event.getTime() + lexicographicSortedString; + } + } else { + this.comparisonResult = ComparisonResult.DIFFERENT_TIMESTEPS; + this.comparisonMessage = "Additional event time=" + event.getTime() + lexicographicSortedString; + } + } + } + + eventFingerprint.addEventType(event.getEventType()); + + + //First timestep, nothing to accumulate + if (eventFingerprint.timeArray.isEmpty()) { + hashAccumulationList.add(lexicographicSortedString); + } else { + float lastTime = eventFingerprint.timeArray.getFloat(eventFingerprint.timeArray.size() - 1); + //If new time is the same as previous, add to accumulation list, event hash calculation is not ready + if (lastTime == event.getTime()) { + hashAccumulationList.add(lexicographicSortedString); + } + //if new time differs from previous, all hash can be calculated + else { + accumulateHash(); + hashAccumulationList.add(lexicographicSortedString); + } + } + + eventFingerprint.addTimeStamp(event.getTime()); + + //eventFingerprint.addHashCode(lexicographicSortedString); + } + + private void accumulateHash() { + + Collections.sort(hashAccumulationList); + + for (String str : hashAccumulationList) { + eventFingerprint.addHashCode(str); + } + + hashAccumulationList.clear(); + } + + /** + *

+ * Finish processing of the events file and return comparison result (if compare fingerprint was present). + * If the result is not equal it will generate a {@link #comparisonMessage}. + */ + void finishProcessing() { + + if (!hashAccumulationList.isEmpty()) { + accumulateHash(); + } + + byte[] hash = eventFingerprint.computeHash(); + + //hash = eventFingerprint.computeHash(); + + if (compareFingerprint == null) + return; + + //Handling EventTypeCounter differences + for (Object2IntMap.Entry entry1 : compareFingerprint.eventTypeCounter.object2IntEntrySet()) { + String key = entry1.getKey(); + int count1 = entry1.getIntValue(); + int count2 = eventFingerprint.eventTypeCounter.getInt(key); + if (count1 != count2) { + comparisonMessage = comparisonMessage == null ? "" : comparisonMessage; + + comparisonResult = (comparisonResult == null ? ComparisonResult.WRONG_EVENT_COUNT : comparisonResult); + + if (!comparisonMessage.isEmpty()) + comparisonMessage += "\n"; + + comparisonMessage += "Count for event type '%s' differs: %d (in fingerprint) != %d (in events)".formatted(key, count1, count2); + } + } + + // Difference was found in {@link EventFingerprint#eventTypeCounter} + if (comparisonResult != null) { + return; + } + + // only check hash if there was no difference up until here + if (!Arrays.equals(hash, compareFingerprint.hash)) { + comparisonResult = ComparisonResult.DIFFERENT_EVENT_ATTRIBUTES; + comparisonMessage = "Difference occurred in this hash of 2 files"; + return; + } + + comparisonResult = ComparisonResult.FILES_ARE_EQUAL; + } + + private String toLexicographicSortedString(Event event) { + List strings = new ArrayList(); + for (Map.Entry e : event.getAttributes().entrySet()) { + StringBuilder tmp = new StringBuilder(); + final String key = e.getKey(); + + // don't look at certain attributes + switch (key) { + case Event.ATTRIBUTE_X: + case Event.ATTRIBUTE_Y: + case Event.ATTRIBUTE_TIME: + continue; + } + + tmp.append(key); + tmp.append("="); + tmp.append(e.getValue()); + strings.add(tmp.toString()); + } + Collections.sort(strings); + StringBuilder eventStr = new StringBuilder(); + for (String str : strings) { + eventStr.append(" | "); + eventStr.append(str); + } + + eventStr.append(" | "); + return eventStr.toString(); + } +} diff --git a/matsim/src/main/java/org/matsim/utils/eventsfilecomparison/Worker.java b/matsim/src/main/java/org/matsim/utils/eventsfilecomparison/Worker.java index 004a0a5e347..81c9cddf32a 100644 --- a/matsim/src/main/java/org/matsim/utils/eventsfilecomparison/Worker.java +++ b/matsim/src/main/java/org/matsim/utils/eventsfilecomparison/Worker.java @@ -144,6 +144,7 @@ private String toLexicographicSortedString(Event event) { switch( key ){ case Event.ATTRIBUTE_X: case Event.ATTRIBUTE_Y: + case Event.ATTRIBUTE_TIME: continue; } } @@ -159,6 +160,7 @@ private String toLexicographicSortedString(Event event) { eventStr.append(" | "); eventStr.append(str); } + eventStr.append(" | ") ; return eventStr.toString(); } diff --git a/matsim/src/test/java/org/matsim/examples/EquilTest.java b/matsim/src/test/java/org/matsim/examples/EquilTest.java index 96794010292..31f714c3d34 100644 --- a/matsim/src/test/java/org/matsim/examples/EquilTest.java +++ b/matsim/src/test/java/org/matsim/examples/EquilTest.java @@ -41,6 +41,7 @@ import org.matsim.core.scenario.ScenarioUtils; import org.matsim.testcases.MatsimTestUtils; import org.matsim.utils.eventsfilecomparison.EventsFileComparator; +import org.matsim.utils.eventsfilecomparison.ComparisonResult; public class EquilTest { private static final Logger log = LogManager.getLogger( EquilTest.class ) ; @@ -84,7 +85,7 @@ void testEquil(boolean isUsingFastCapacityUpdate) { writer.closeFile(); - final EventsFileComparator.Result result = new EventsFileComparator().setIgnoringCoordinates( true ).runComparison( referenceFileName , eventsFileName ); - Assertions.assertEquals(EventsFileComparator.Result.FILES_ARE_EQUAL, result, "different event files." ); + final ComparisonResult result = new EventsFileComparator().setIgnoringCoordinates( true ).runComparison( referenceFileName , eventsFileName ); + Assertions.assertEquals(ComparisonResult.FILES_ARE_EQUAL, result, "different event files." ); } } diff --git a/matsim/src/test/java/org/matsim/examples/OnePercentBerlin10sIT.java b/matsim/src/test/java/org/matsim/examples/OnePercentBerlin10sIT.java index 89237ec5b1a..f6d862c5785 100644 --- a/matsim/src/test/java/org/matsim/examples/OnePercentBerlin10sIT.java +++ b/matsim/src/test/java/org/matsim/examples/OnePercentBerlin10sIT.java @@ -41,6 +41,7 @@ import org.matsim.core.scenario.ScenarioUtils; import org.matsim.testcases.MatsimTestUtils; import org.matsim.utils.eventsfilecomparison.EventsFileComparator; +import org.matsim.utils.eventsfilecomparison.ComparisonResult; public class OnePercentBerlin10sIT { @@ -92,7 +93,7 @@ void testOnePercent10sQSim() { writer.closeFile(); System.out.println("reffile: " + referenceEventsFileName); - assertEquals( EventsFileComparator.Result.FILES_ARE_EQUAL, + assertEquals( ComparisonResult.FILES_ARE_EQUAL, new EventsFileComparator().setIgnoringCoordinates( true ).runComparison( referenceEventsFileName, eventsFileName ), "different event files" ); @@ -137,7 +138,7 @@ void testOnePercent10sQSimTryEndTimeThenDuration() { writer.closeFile(); - assertEquals( EventsFileComparator.Result.FILES_ARE_EQUAL, + assertEquals( ComparisonResult.FILES_ARE_EQUAL, new EventsFileComparator().setIgnoringCoordinates( true ).runComparison( referenceEventsFileName, eventsFileName ), "different event files" ); diff --git a/matsim/src/test/java/org/matsim/testcases/MatsimTestUtils.java b/matsim/src/test/java/org/matsim/testcases/MatsimTestUtils.java index 87696593899..112bcaaddaf 100644 --- a/matsim/src/test/java/org/matsim/testcases/MatsimTestUtils.java +++ b/matsim/src/test/java/org/matsim/testcases/MatsimTestUtils.java @@ -32,6 +32,7 @@ import org.matsim.core.utils.io.IOUtils; import org.matsim.core.utils.misc.CRCChecksum; import org.matsim.utils.eventsfilecomparison.EventsFileComparator; +import org.matsim.utils.eventsfilecomparison.ComparisonResult; import java.io.BufferedReader; import java.io.File; @@ -316,7 +317,7 @@ public static void assertEqualFilesLineByLine(String inputFilename, String outpu } public static void assertEqualEventsFiles( String filename1, String filename2 ) { - Assertions.assertEquals(EventsFileComparator.Result.FILES_ARE_EQUAL ,EventsFileComparator.compare(filename1, filename2) ); + Assertions.assertEquals(ComparisonResult.FILES_ARE_EQUAL ,EventsFileComparator.compare(filename1, filename2) ); } public static void assertEqualFilesBasedOnCRC( String filename1, String filename2 ) { diff --git a/matsim/src/test/java/org/matsim/utils/eventsfilecomparison/EventsFileComparatorTest.java b/matsim/src/test/java/org/matsim/utils/eventsfilecomparison/EventsFileComparatorTest.java index 6618b85dc17..9a52ac8593e 100644 --- a/matsim/src/test/java/org/matsim/utils/eventsfilecomparison/EventsFileComparatorTest.java +++ b/matsim/src/test/java/org/matsim/utils/eventsfilecomparison/EventsFileComparatorTest.java @@ -20,7 +20,7 @@ package org.matsim.utils.eventsfilecomparison; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.matsim.utils.eventsfilecomparison.EventsFileComparator.Result.*; +import static org.matsim.utils.eventsfilecomparison.ComparisonResult.*; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; diff --git a/matsim/src/test/java/org/matsim/utils/eventsfilecomparison/EventsFileFingerprintComparatorTest.java b/matsim/src/test/java/org/matsim/utils/eventsfilecomparison/EventsFileFingerprintComparatorTest.java new file mode 100644 index 00000000000..a14100b52d1 --- /dev/null +++ b/matsim/src/test/java/org/matsim/utils/eventsfilecomparison/EventsFileFingerprintComparatorTest.java @@ -0,0 +1,143 @@ +package org.matsim.utils.eventsfilecomparison; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; +import org.matsim.core.events.EventsUtils; +import org.matsim.testcases.MatsimTestUtils; + +import java.io.File; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.*; + +class EventsFileFingerprintComparatorTest { + + @RegisterExtension + MatsimTestUtils utils = new MatsimTestUtils(); + + @Test + void testEqual() { + + assertThat(EventsUtils.createAndCompareEventsFingerprint( + new File(utils.getClassInputDirectory(), "events_correct.xml"), + new File(utils.getClassInputDirectory(), "correct.fp.zst").toString() + )).isEqualTo(ComparisonResult.FILES_ARE_EQUAL); + + // Check if file has been created + assertThat(new File(utils.getClassInputDirectory(), "events_correct.fp.zst")) + .exists() + .size().isGreaterThan(0); + + } + + @Test + void testDiffTimesteps() { + + assertThat(EventsUtils.createAndCompareEventsFingerprint( + new File(utils.getClassInputDirectory(), "events.diff-num-timestamps.xml"), + new File(utils.getClassInputDirectory(), "correct.fp.zst").toString() + )).isEqualTo(ComparisonResult.DIFFERENT_TIMESTEPS); + + + } + + @Test + void testDiffCounts() { + + assertThat(EventsUtils.createAndCompareEventsFingerprint( + new File(utils.getClassInputDirectory(), "events.diff-type-count.xml"), + new File(utils.getClassInputDirectory(), "correct.fp.zst").toString() + )).isEqualTo(ComparisonResult.WRONG_EVENT_COUNT); + + } + + @Test + void testDiffContent() { + assertThat(EventsUtils.createAndCompareEventsFingerprint( + new File(utils.getClassInputDirectory(), "events.diff-hash.xml"), + new File(utils.getClassInputDirectory(), "correct.fp.zst").toString() + )).isEqualTo(ComparisonResult.DIFFERENT_EVENT_ATTRIBUTES); + } + + @Test + void testAdditionalEvent() { + assertThat(EventsUtils.createAndCompareEventsFingerprint( + new File(utils.getClassInputDirectory(), "events.one-more-event.xml"), + new File(utils.getClassInputDirectory(), "correct.fp.zst").toString() + )).isEqualTo(ComparisonResult.DIFFERENT_TIMESTEPS); + } + + @Test + void testAttributeOrder() { + assertThat(EventsUtils.createAndCompareEventsFingerprint( + new File(utils.getClassInputDirectory(), "events.attribute-order.xml"), + new File(utils.getClassInputDirectory(), "correct.fp.zst").toString() + )).isEqualTo(ComparisonResult.FILES_ARE_EQUAL); + } + + @Test + void testEventOrder() { + assertThat(EventsUtils.compareEventsFiles( + new File(utils.getClassInputDirectory(), "events_correct.xml").toString(), + new File(utils.getClassInputDirectory(), "events.event-order-wrong_logic.xml").toString() + )).isEqualTo(ComparisonResult.FILES_ARE_EQUAL); + + assertThat(EventsUtils.createAndCompareEventsFingerprint( + new File(utils.getClassInputDirectory(), "events.event-order-wrong_logic.xml"), + new File(utils.getClassInputDirectory(), "correct.fp.zst").toString() + )).isEqualTo(ComparisonResult.FILES_ARE_EQUAL); + + } + + + @Test + void testEqualFingerprints() { + + EventsUtils.createEventsFingerprint(new File(utils.getClassInputDirectory(), "events_correct.xml").toString(), new File(utils.getClassInputDirectory(), "events.fp.zst").toString()); + + assertThat(EventsFileFingerprintComparator.compareFingerprints( + new File(utils.getClassInputDirectory(), "events.fp.zst").toString(), + new File(utils.getClassInputDirectory(), "correct.fp.zst").toString() + )).isEqualTo(ComparisonResult.FILES_ARE_EQUAL); + + + } + + @Test + void testDiffTimestepsFingerprints() { + + assertThat(EventsFileFingerprintComparator.compareFingerprints( + new File(utils.getClassInputDirectory(), "events.diff-num-timestamps.fp.zst").toString(), + new File(utils.getClassInputDirectory(), "correct.fp.zst").toString() + )).isEqualTo(ComparisonResult.DIFFERENT_NUMBER_OF_TIMESTEPS); + + + } + + @Test + void testDiffCountsFingerprints() { + + assertThat(EventsFileFingerprintComparator.compareFingerprints( + new File(utils.getClassInputDirectory(), "events.diff-type-count.fp.zst").toString(), + new File(utils.getClassInputDirectory(), "correct.fp.zst").toString() + )).isEqualTo(ComparisonResult.WRONG_EVENT_COUNT); + + } + + @Test + void assertion() { + + AssertionError err = assertThrows(AssertionError.class, () -> { + EventsUtils.assertEqualEventsFingerprint( + new File(utils.getClassInputDirectory(), "events.diff-num-timestamps.xml"), + new File(utils.getClassInputDirectory(), "correct.fp.zst").toString() + ); + }); + + assertThat(err).message().isEqualTo(""" + Difference occurred in this event time=48067.0 | link=1 | type=entered link | vehicle=5 |\s + Count for event type 'entered link' differs: 2 (in fingerprint) != 1 (in events) + Count for event type 'vehicle leaves traffic' differs: 1 (in fingerprint) != 0 (in events)"""); + + } +} diff --git a/matsim/test/input/org/matsim/utils/eventsfilecomparison/EventsFileFingerprintComparatorTest/correct.fp.zst b/matsim/test/input/org/matsim/utils/eventsfilecomparison/EventsFileFingerprintComparatorTest/correct.fp.zst new file mode 100644 index 0000000000000000000000000000000000000000..7bf62ca48e26975c0b7ed02c29daada8427db755 GIT binary patch literal 196 zcmdPcs{fZE;s`543a49uz99nx1Gn1>9tJ*l>%$E0wpu_w7XwdfUP)?EYKlTmW?nW> zgo%MYv8X7sEHMYjVPxP;Ni9e$Dk&{WWlKyhNzF?U4oEF3&d+lN=_(FO&B#p75i3*3 zNlh$EEmkNgN=!@3O!nc-NlhzZWZ(d5DJ}tO2N|WY!Q$}xlNm+}6d&AQ7IO5A#^z7U oc^OokHBQWsU_9V_{@bCh@EMC;v)-J}J)Oh7gH=4b3}^%c07D-^;{X5v literal 0 HcmV?d00001 diff --git a/matsim/test/input/org/matsim/utils/eventsfilecomparison/EventsFileFingerprintComparatorTest/events.attribute-order.xml b/matsim/test/input/org/matsim/utils/eventsfilecomparison/EventsFileFingerprintComparatorTest/events.attribute-order.xml new file mode 100644 index 00000000000..6885fa8af5d --- /dev/null +++ b/matsim/test/input/org/matsim/utils/eventsfilecomparison/EventsFileFingerprintComparatorTest/events.attribute-order.xml @@ -0,0 +1,14 @@ + + + + + + + + + + + + + + \ No newline at end of file diff --git a/matsim/test/input/org/matsim/utils/eventsfilecomparison/EventsFileFingerprintComparatorTest/events.diff-hash.xml b/matsim/test/input/org/matsim/utils/eventsfilecomparison/EventsFileFingerprintComparatorTest/events.diff-hash.xml new file mode 100644 index 00000000000..4ae06c2369f --- /dev/null +++ b/matsim/test/input/org/matsim/utils/eventsfilecomparison/EventsFileFingerprintComparatorTest/events.diff-hash.xml @@ -0,0 +1,14 @@ + + + + + + + + + + + + + + \ No newline at end of file diff --git a/matsim/test/input/org/matsim/utils/eventsfilecomparison/EventsFileFingerprintComparatorTest/events.diff-num-timestamps.xml b/matsim/test/input/org/matsim/utils/eventsfilecomparison/EventsFileFingerprintComparatorTest/events.diff-num-timestamps.xml new file mode 100644 index 00000000000..422b58d651d --- /dev/null +++ b/matsim/test/input/org/matsim/utils/eventsfilecomparison/EventsFileFingerprintComparatorTest/events.diff-num-timestamps.xml @@ -0,0 +1,12 @@ + + + + + + + + + + + + \ No newline at end of file diff --git a/matsim/test/input/org/matsim/utils/eventsfilecomparison/EventsFileFingerprintComparatorTest/events.diff-type-count.fp.zst b/matsim/test/input/org/matsim/utils/eventsfilecomparison/EventsFileFingerprintComparatorTest/events.diff-type-count.fp.zst new file mode 100644 index 0000000000000000000000000000000000000000..6437b813ac0fd615f3a74dea6c4492d7f5cca149 GIT binary patch literal 194 zcmdPcs{fZEVhbyS2&Y?sz99nx1Gn1>9tJ*l>%$E0wpu_w7XwdfUP)?EYKlTmW?nW> zgpq+gv8X7sEHQ^OCAA>2sHC(gl`S#3BsDKZI3TsCI6u!7q@g%0H6t@QN32XCCpEDw zwOFB~C^0Q9GuekTCpE2viGc&CrMLvB9b{yoZfn9FuFS{f&vgpd+TJXxYGnS-$Drb@ nabku9;{oUM-wt(!&sgl5_2zW$OkUpLf(JY?x literal 0 HcmV?d00001 diff --git a/matsim/test/input/org/matsim/utils/eventsfilecomparison/EventsFileFingerprintComparatorTest/events.diff-type-count.xml b/matsim/test/input/org/matsim/utils/eventsfilecomparison/EventsFileFingerprintComparatorTest/events.diff-type-count.xml new file mode 100644 index 00000000000..31662fae9c0 --- /dev/null +++ b/matsim/test/input/org/matsim/utils/eventsfilecomparison/EventsFileFingerprintComparatorTest/events.diff-type-count.xml @@ -0,0 +1,14 @@ + + + + + + + + + + + + + + \ No newline at end of file diff --git a/matsim/test/input/org/matsim/utils/eventsfilecomparison/EventsFileFingerprintComparatorTest/events.event-order-wrong_logic.xml b/matsim/test/input/org/matsim/utils/eventsfilecomparison/EventsFileFingerprintComparatorTest/events.event-order-wrong_logic.xml new file mode 100644 index 00000000000..1112364be0c --- /dev/null +++ b/matsim/test/input/org/matsim/utils/eventsfilecomparison/EventsFileFingerprintComparatorTest/events.event-order-wrong_logic.xml @@ -0,0 +1,14 @@ + + + + + + + + + + + + + + \ No newline at end of file diff --git a/matsim/test/input/org/matsim/utils/eventsfilecomparison/EventsFileFingerprintComparatorTest/events.event-order.xml b/matsim/test/input/org/matsim/utils/eventsfilecomparison/EventsFileFingerprintComparatorTest/events.event-order.xml new file mode 100644 index 00000000000..c4b9bbf33de --- /dev/null +++ b/matsim/test/input/org/matsim/utils/eventsfilecomparison/EventsFileFingerprintComparatorTest/events.event-order.xml @@ -0,0 +1,14 @@ + + + + + + + + + + + + + + \ No newline at end of file diff --git a/matsim/test/input/org/matsim/utils/eventsfilecomparison/EventsFileFingerprintComparatorTest/events.one-more-event.xml b/matsim/test/input/org/matsim/utils/eventsfilecomparison/EventsFileFingerprintComparatorTest/events.one-more-event.xml new file mode 100644 index 00000000000..663754db63f --- /dev/null +++ b/matsim/test/input/org/matsim/utils/eventsfilecomparison/EventsFileFingerprintComparatorTest/events.one-more-event.xml @@ -0,0 +1,15 @@ + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/matsim/test/input/org/matsim/utils/eventsfilecomparison/EventsFileFingerprintComparatorTest/events_correct.xml b/matsim/test/input/org/matsim/utils/eventsfilecomparison/EventsFileFingerprintComparatorTest/events_correct.xml new file mode 100644 index 00000000000..4b2bbbd1b92 --- /dev/null +++ b/matsim/test/input/org/matsim/utils/eventsfilecomparison/EventsFileFingerprintComparatorTest/events_correct.xml @@ -0,0 +1,14 @@ + + + + + + + + + + + + + + \ No newline at end of file