From 14776bd71e85efba88e1d9b5db0e84796fc52867 Mon Sep 17 00:00:00 2001 From: Martijn van Laar Date: Fri, 16 Apr 2021 15:52:50 +0200 Subject: [PATCH 001/122] Update env.yml with current MONGO_HOST instead of Mongo_URI Fixes: #370 Based on comment of @landonreed --- configurations/default/env.yml.tmp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configurations/default/env.yml.tmp b/configurations/default/env.yml.tmp index eb5769962..63eb77d12 100644 --- a/configurations/default/env.yml.tmp +++ b/configurations/default/env.yml.tmp @@ -15,5 +15,5 @@ SPARKPOST_EMAIL: email@example.com GTFS_DATABASE_URL: jdbc:postgresql://localhost/catalogue # GTFS_DATABASE_USER: # GTFS_DATABASE_PASSWORD: -#MONGO_URI: mongodb://mongo-host:27017 +#MONGO_HOST: mongodb://mongo-host:27017 MONGO_DB_NAME: catalogue From ba0ffa74c66bdbf04cae6b95adcbb3e05601429c Mon Sep 17 00:00:00 2001 From: Martijn van Laar Date: Fri, 16 Apr 2021 19:04:10 +0200 Subject: [PATCH 002/122] remove protocol --- configurations/default/env.yml.tmp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configurations/default/env.yml.tmp b/configurations/default/env.yml.tmp index 63eb77d12..119b399a6 100644 --- a/configurations/default/env.yml.tmp +++ b/configurations/default/env.yml.tmp @@ -15,5 +15,5 @@ SPARKPOST_EMAIL: email@example.com GTFS_DATABASE_URL: jdbc:postgresql://localhost/catalogue # GTFS_DATABASE_USER: # GTFS_DATABASE_PASSWORD: -#MONGO_HOST: mongodb://mongo-host:27017 +#MONGO_HOST: mongo-host:27017 MONGO_DB_NAME: catalogue From 011129352aadb2f00e6abf3c3ebdd899c34ad291 Mon Sep 17 00:00:00 2001 From: Landon Reed Date: Tue, 27 Apr 2021 08:40:25 -0400 Subject: [PATCH 003/122] fix(MergeFeedsJob): add trip handling strategies for MTC merges --- pom.xml | 2 +- .../datatools/manager/jobs/FeedToMerge.java | 24 ++ .../datatools/manager/jobs/MergeFeedsJob.java | 277 +++++++++++------- .../manager/jobs/MergeFeedsResult.java | 3 + .../datatools/manager/jobs/MergeStrategy.java | 46 +++ .../manager/utils/MergeFeedUtils.java | 144 +++++++++ .../manager/jobs/MergeFeedsJobTest.java | 248 ++++++++++++---- .../datatools/gtfs/merge-data-base/agency.txt | 2 + .../gtfs/merge-data-base/calendar.txt | 3 + .../gtfs/merge-data-base/feed_info.txt | 2 + .../datatools/gtfs/merge-data-base/routes.txt | 3 + .../gtfs/merge-data-base/stop_attributes.txt | 3 + .../gtfs/merge-data-base/stop_times.txt | 7 + .../datatools/gtfs/merge-data-base/stops.txt | 6 + .../datatools/gtfs/merge-data-base/trips.txt | 4 + .../gtfs/merge-data-future/agency.txt | 2 + .../gtfs/merge-data-future/calendar.txt | 3 + .../gtfs/merge-data-future/feed_info.txt | 2 + .../gtfs/merge-data-future/routes.txt | 3 + .../merge-data-future/stop_attributes.txt | 3 + .../gtfs/merge-data-future/stop_times.txt | 5 + .../gtfs/merge-data-future/stops.txt | 6 + .../gtfs/merge-data-future/trips.txt | 3 + .../gtfs/merge-data-mod-services/agency.txt | 2 + .../gtfs/merge-data-mod-services/calendar.txt | 3 + .../merge-data-mod-services/feed_info.txt | 2 + .../gtfs/merge-data-mod-services/routes.txt | 3 + .../stop_attributes.txt | 3 + .../merge-data-mod-services/stop_times.txt | 5 + .../gtfs/merge-data-mod-services/stops.txt | 6 + .../gtfs/merge-data-mod-services/trips.txt | 3 + .../gtfs/merge-data-mod-trips/agency.txt | 2 + .../gtfs/merge-data-mod-trips/calendar.txt | 3 + .../gtfs/merge-data-mod-trips/feed_info.txt | 2 + .../gtfs/merge-data-mod-trips/routes.txt | 3 + .../merge-data-mod-trips/stop_attributes.txt | 3 + .../gtfs/merge-data-mod-trips/stop_times.txt | 5 + .../gtfs/merge-data-mod-trips/stops.txt | 6 + .../gtfs/merge-data-mod-trips/trips.txt | 4 + 39 files changed, 702 insertions(+), 154 deletions(-) create mode 100644 src/main/java/com/conveyal/datatools/manager/jobs/FeedToMerge.java create mode 100644 src/main/java/com/conveyal/datatools/manager/jobs/MergeStrategy.java create mode 100644 src/main/java/com/conveyal/datatools/manager/utils/MergeFeedUtils.java create mode 100755 src/test/resources/com/conveyal/datatools/gtfs/merge-data-base/agency.txt create mode 100755 src/test/resources/com/conveyal/datatools/gtfs/merge-data-base/calendar.txt create mode 100644 src/test/resources/com/conveyal/datatools/gtfs/merge-data-base/feed_info.txt create mode 100755 src/test/resources/com/conveyal/datatools/gtfs/merge-data-base/routes.txt create mode 100644 src/test/resources/com/conveyal/datatools/gtfs/merge-data-base/stop_attributes.txt create mode 100755 src/test/resources/com/conveyal/datatools/gtfs/merge-data-base/stop_times.txt create mode 100755 src/test/resources/com/conveyal/datatools/gtfs/merge-data-base/stops.txt create mode 100755 src/test/resources/com/conveyal/datatools/gtfs/merge-data-base/trips.txt create mode 100755 src/test/resources/com/conveyal/datatools/gtfs/merge-data-future/agency.txt create mode 100755 src/test/resources/com/conveyal/datatools/gtfs/merge-data-future/calendar.txt create mode 100644 src/test/resources/com/conveyal/datatools/gtfs/merge-data-future/feed_info.txt create mode 100755 src/test/resources/com/conveyal/datatools/gtfs/merge-data-future/routes.txt create mode 100644 src/test/resources/com/conveyal/datatools/gtfs/merge-data-future/stop_attributes.txt create mode 100755 src/test/resources/com/conveyal/datatools/gtfs/merge-data-future/stop_times.txt create mode 100755 src/test/resources/com/conveyal/datatools/gtfs/merge-data-future/stops.txt create mode 100755 src/test/resources/com/conveyal/datatools/gtfs/merge-data-future/trips.txt create mode 100755 src/test/resources/com/conveyal/datatools/gtfs/merge-data-mod-services/agency.txt create mode 100755 src/test/resources/com/conveyal/datatools/gtfs/merge-data-mod-services/calendar.txt create mode 100644 src/test/resources/com/conveyal/datatools/gtfs/merge-data-mod-services/feed_info.txt create mode 100755 src/test/resources/com/conveyal/datatools/gtfs/merge-data-mod-services/routes.txt create mode 100644 src/test/resources/com/conveyal/datatools/gtfs/merge-data-mod-services/stop_attributes.txt create mode 100755 src/test/resources/com/conveyal/datatools/gtfs/merge-data-mod-services/stop_times.txt create mode 100755 src/test/resources/com/conveyal/datatools/gtfs/merge-data-mod-services/stops.txt create mode 100755 src/test/resources/com/conveyal/datatools/gtfs/merge-data-mod-services/trips.txt create mode 100755 src/test/resources/com/conveyal/datatools/gtfs/merge-data-mod-trips/agency.txt create mode 100755 src/test/resources/com/conveyal/datatools/gtfs/merge-data-mod-trips/calendar.txt create mode 100644 src/test/resources/com/conveyal/datatools/gtfs/merge-data-mod-trips/feed_info.txt create mode 100755 src/test/resources/com/conveyal/datatools/gtfs/merge-data-mod-trips/routes.txt create mode 100644 src/test/resources/com/conveyal/datatools/gtfs/merge-data-mod-trips/stop_attributes.txt create mode 100755 src/test/resources/com/conveyal/datatools/gtfs/merge-data-mod-trips/stop_times.txt create mode 100755 src/test/resources/com/conveyal/datatools/gtfs/merge-data-mod-trips/stops.txt create mode 100755 src/test/resources/com/conveyal/datatools/gtfs/merge-data-mod-trips/trips.txt diff --git a/pom.xml b/pom.xml index 3bd530acb..07d0b5656 100644 --- a/pom.xml +++ b/pom.xml @@ -263,7 +263,7 @@ com.github.conveyal gtfs-lib - 6.2.2 + add-stoptime-hash-SNAPSHOT diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/FeedToMerge.java b/src/main/java/com/conveyal/datatools/manager/jobs/FeedToMerge.java new file mode 100644 index 000000000..753730a5d --- /dev/null +++ b/src/main/java/com/conveyal/datatools/manager/jobs/FeedToMerge.java @@ -0,0 +1,24 @@ +package com.conveyal.datatools.manager.jobs; + +import com.conveyal.datatools.manager.models.FeedVersion; +import com.conveyal.gtfs.loader.Table; +import com.google.common.collect.HashMultimap; +import com.google.common.collect.SetMultimap; + +import java.io.IOException; +import java.util.zip.ZipFile; + +/** + * Helper class that collects the feed version and its zip file. Note: this class helps with sorting versions to + * merge in a list collection. + */ +public class FeedToMerge { + public FeedVersion version; + public ZipFile zipFile; + public SetMultimap idsForTable = HashMultimap.create(); + + public FeedToMerge(FeedVersion version) throws IOException { + this.version = version; + this.zipFile = new ZipFile(version.retrieveGtfsFile()); + } +} \ No newline at end of file diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java b/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java index 9a4ec6463..8ba6a930d 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java @@ -13,11 +13,15 @@ import com.conveyal.datatools.manager.persistence.Persistence; import com.conveyal.gtfs.error.NewGTFSError; import com.conveyal.gtfs.error.NewGTFSErrorType; +import com.conveyal.gtfs.loader.Feed; import com.conveyal.gtfs.loader.Field; import com.conveyal.gtfs.loader.ReferenceTracker; import com.conveyal.gtfs.loader.Table; +import com.conveyal.gtfs.model.StopTime; import com.csvreader.CsvReader; import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.supercsv.io.CsvListWriter; @@ -32,24 +36,23 @@ import java.time.temporal.ChronoUnit; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collection; -import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.Set; import java.util.UUID; import java.util.stream.Collectors; import java.util.zip.ZipEntry; -import java.util.zip.ZipFile; import java.util.zip.ZipOutputStream; import static com.conveyal.datatools.manager.jobs.MergeFeedsType.SERVICE_PERIOD; import static com.conveyal.datatools.manager.jobs.MergeFeedsType.REGIONAL; +import static com.conveyal.datatools.manager.jobs.MergeStrategy.CHECK_STOP_TIMES; +import static com.conveyal.datatools.manager.jobs.MergeStrategy.EXTEND_FUTURE; import static com.conveyal.datatools.manager.models.FeedRetrievalMethod.REGIONAL_MERGE; import static com.conveyal.datatools.manager.models.FeedRetrievalMethod.SERVICE_PERIOD_MERGE; +import static com.conveyal.datatools.manager.utils.MergeFeedUtils.*; import static com.conveyal.datatools.manager.utils.StringUtils.getCleanName; import static com.conveyal.gtfs.loader.DateField.GTFS_DATE_FORMATTER; import static com.conveyal.gtfs.loader.Field.getFieldIndex; @@ -127,6 +130,9 @@ public class MergeFeedsJob extends MonitorableJob { private static final Logger LOG = LoggerFactory.getLogger(MergeFeedsJob.class); public static final ObjectMapper mapper = new ObjectMapper(); + private static Set intersectingTripIds = new HashSet<>(); + private static Set tripsOnlyInCurrentFeed = new HashSet<>(); + private static Set tripsOnlyInFutureFeed = new HashSet<>(); private final Set feedVersions; private final FeedSource feedSource; private final ReferenceTracker referenceTracker = new ReferenceTracker(); @@ -140,7 +146,11 @@ public class MergeFeedsJob extends MonitorableJob { * dataset. Otherwise, this will be null throughout the life of the job. */ final FeedVersion mergedVersion; - public boolean failOnDuplicateTripId = true; + public MergeStrategy mergeStrategy = MergeStrategy.DEFAULT; + private Set tripIdsToModifyForCurrentFeed = new HashSet<>(); + private Set tripIdsToSkipForCurrentFeed = new HashSet<>(); + private Set serviceIdsToExtend = new HashSet<>(); + private Set serviceIdsToCloneAndRename = new HashSet<>(); public MergeFeedsJob(Auth0UserProfile owner, Set feedVersions, String file, MergeFeedsType mergeType) { this(owner, feedVersions, file, mergeType, true); @@ -233,6 +243,58 @@ public void jobFinished() { tablesToMerge.addAll(Arrays.asList(GtfsPlusTable.tables)); } int numberOfTables = tablesToMerge.size(); + // Before initiating the merge process, run some pre-processing to check for id conflicts for certain tables + if (mergeType.equals(SERVICE_PERIOD)) { + mergeStrategy = getMergeStrategy(feedsToMerge); + if (mergeStrategy == CHECK_STOP_TIMES) { + Feed futureFeed = new Feed(DataManager.GTFS_DATA_SOURCE, feedsToMerge.get(0).version.namespace); + Feed pastFeed = new Feed(DataManager.GTFS_DATA_SOURCE, feedsToMerge.get(1).version.namespace); + for (String tripId : intersectingTripIds) { + // Fetch all ordered stop_times for each common trip_id, hash them, and compare the two sets for the + // future and current feed. If the stop_times are an exact match, include one instance of the trip + // (ignoring the other identical one). If they do not match, modify the current trip_id and include. + List futureStopTimes = Lists.newArrayList(futureFeed.stopTimes.getOrdered(tripId)); + List pastStopTimes = Lists.newArrayList(pastFeed.stopTimes.getOrdered(tripId)); + String futureServiceId = futureFeed.trips.get(tripId).service_id; + String pastServiceId = pastFeed.trips.get(tripId).service_id; + // FIXME: what if service_ids do not match! Perhaps the right approach would be to just return + // FAIL_DUE_TO_MATCHING_TRIP_IDS in that case. It might be too complicated otherwise. + if (!stopTimesMatch(futureStopTimes, pastStopTimes)) { + // If stop_times or services do not match, the trip will be cloned. Also, track the service_id + // (it will need to be cloned and renamed for both current/past feeds). + tripIdsToModifyForCurrentFeed.add(tripId); + serviceIdsToCloneAndRename.add(futureServiceId); + } else { + // If the trip's stop_times are an exact match, we can safely include just the + // future trip and exclude the current/past one. Also, track the service_id (it will need to be + // extended to the full time range). + tripIdsToSkipForCurrentFeed.add(tripId); + serviceIdsToExtend.add(futureServiceId); + } + } + for (String tripId : tripsOnlyInCurrentFeed) { + String serviceId = pastFeed.trips.get(tripId).service_id; + // If a trip in the current feed contains a service_id that has been modified above, it needs to be + // cloned/renamed so as to avoid having it operate on an extended service. + if (serviceIdsToExtend.contains(serviceId)) { + serviceIdsToCloneAndRename.add(serviceId); + } + } + for (String tripId : tripsOnlyInFutureFeed) { + String serviceId = futureFeed.trips.get(tripId).service_id; + // If a trip in the future feed contains a service_id that has been modified above, it needs to be + // cloned/renamed so as to avoid having it operate on an extended service. + if (serviceIdsToExtend.contains(serviceId)) { + serviceIdsToCloneAndRename.add(serviceId); + } + } + } + } + // Skip merging process altogether if the failing condition is met. + if (MergeStrategy.FAIL_DUE_TO_MATCHING_TRIP_IDS.equals(mergeStrategy)) { + status.fail("Feed merge failed because the trip_ids are identical in the future and current feeds. A new service requires unique trip_ids for merging."); + return; + } // Loop over GTFS tables and merge each feed one table at a time. for (int i = 0; i < numberOfTables; i++) { Table table = tablesToMerge.get(i); @@ -280,25 +342,16 @@ public void jobFinished() { } } - /** - * Collect zipFiles for each feed version before merging tables. - * Note: feed versions are sorted by first calendar date so that future dataset is iterated over first. This is - * required for the MTC merge strategy which prefers entities from the future dataset over past entities. - */ - private List collectAndSortFeeds(Set feedVersions) { - return feedVersions.stream().map(version -> { - try { - return new FeedToMerge(version); - } catch (Exception e) { - LOG.error("Could not create zip file for version: {}", version.version); - return null; + private boolean stopTimesMatch(List futureStopTimes, List pastStopTimes) { + if (futureStopTimes.size() != pastStopTimes.size()) { + return false; + } + for (int i = 0; i < pastStopTimes.size(); i++) { + if (pastStopTimes.get(i).hashCode() != futureStopTimes.get(i).hashCode()) { + return false; } - }).filter(Objects::nonNull).filter(entry -> entry.version.validationResult != null - && entry.version.validationResult.firstCalendarDate != null) - // MTC-specific sort mentioned in above comment. - // TODO: If another merge strategy requires a different sort order, a merge type check should be added. - .sorted(Comparator.comparing(entry -> entry.version.validationResult.firstCalendarDate, - Comparator.reverseOrder())).collect(Collectors.toList()); + } + return true; } /** @@ -376,16 +429,22 @@ private int constructMergedTable(Table table, List feedsToMerge, // Get shared fields between all feeds being merged. This is used to filter the spec fields so that only // fields found in the collection of feeds are included in the merged table. Set sharedFields = getSharedFields(feedsToMerge, table); - // Initialize future feed's first date to the first calendar date from the validation result. + // Initialize future and past/current feed's first date to the first calendar date from validation result. // This is equivalent to either the earliest date of service defined for a calendar_date record or the // earliest start_date value for a calendars.txt record. For MTC, however, they require that GTFS // providers use calendars.txt entries and prefer that this value (which is used to determine cutoff // dates for the active feed when merging with the future) be strictly assigned the earliest // calendar#start_date (unless that table for some reason does not exist). LocalDate futureFeedFirstDate = feedsToMerge.get(0).version.validationResult.firstCalendarDate; + LocalDate pastFeedFirstDate = feedsToMerge.get(1).version.validationResult.firstCalendarDate; LocalDate futureFirstCalendarStartDate = LocalDate.MAX; // Iterate over each zip file. For service period merge, the first feed is the future GTFS. for (int feedIndex = 0; feedIndex < feedsToMerge.size(); feedIndex++) { + if (EXTEND_FUTURE.equals(mergeStrategy) && feedIndex > 0) { + // No need to iterate over second (current) file if strategy is to simply extend the future GTFS + // service to start earlier. + continue; + } boolean keyFieldMissing = false; // Use for a new agency ID for use if the feed does not contain one. Initialize to // null. If the value becomes non-null, the agency_id is missing and needs to be @@ -465,7 +524,7 @@ private int constructMergedTable(Table table, List feedsToMerge, futureFirstCalendarStartDate.isBefore(LocalDate.MAX) && futureFeedFirstDate.isBefore(futureFirstCalendarStartDate) ) { - // If the future feed's first date is before the feed's first calendar start date, + // If the future feed's first date is before the its first calendar start date, // override the future feed first date with the calendar start date for use when checking // MTC calendar_dates and calendar records for modification/exclusion. futureFeedFirstDate = futureFirstCalendarStartDate; @@ -651,14 +710,26 @@ private int constructMergedTable(Table table, List feedsToMerge, int startDateIndex = getFieldIndex(fieldsFoundInZip, "start_date"); LocalDate startDate = LocalDate - .parse(csvReader.get(startDateIndex), - GTFS_DATE_FORMATTER); + .parse(csvReader.get(startDateIndex), GTFS_DATE_FORMATTER); if (feedIndex == 0) { // For the future feed, check if the calendar's start date is earlier than the // previous earliest value and update if so. if (futureFirstCalendarStartDate.isAfter(startDate)) { futureFirstCalendarStartDate = startDate; } + // FIXME: Move this below so that a cloned service doesn't get prematurely + // modified? (do we want the cloned record to have the original values?) + if (index == startDateIndex) { + if (EXTEND_FUTURE == mergeStrategy || + (CHECK_STOP_TIMES == mergeStrategy && + // TODO: Need to ensure serviceIds are being extended. + serviceIdsToExtend.contains(keyValue)) + ) { + // Update start_date to extend service through the past/current feed's + // start date if the merge strategy dictates. + val = valueToWrite = pastFeedFirstDate.format(GTFS_DATE_FORMATTER); + } + } } else { // If a service_id from the active calendar has both the // start_date and end_date in the future, the service will be @@ -679,8 +750,7 @@ private int constructMergedTable(Table table, List feedsToMerge, // end_date in the future, the end_date shall be set to one // day prior to the earliest start_date in future dataset // before appending the calendar record to the merged file. - int endDateIndex = - getFieldIndex(fieldsFoundInZip, "end_date"); + int endDateIndex = getFieldIndex(fieldsFoundInZip, "end_date"); if (index == endDateIndex) { LocalDate endDate = LocalDate .parse(csvReader.get(endDateIndex), GTFS_DATE_FORMATTER); @@ -750,18 +820,31 @@ private int constructMergedTable(Table table, List feedsToMerge, if (hasDuplicateError(idErrors)) skipRecord = true; break; case "trips": - // trip_ids between active and future datasets must not match. If any trip_id is found - // to be matching, the merge should fail with appropriate notification to user with the - // cause of the failure. Merge result should include all conflicting trip_ids. + // trip_ids between active and future datasets must not match. The MergeStrategy + // determines behavior when matching trip_ids (or service_ids) are found between + if (feedIndex > 0) { + // Handling past/current feed. + if (tripIdsToSkipForCurrentFeed.contains(keyValue)) { + skipRecord = true; + } else if (tripIdsToModifyForCurrentFeed.contains(keyValue)) { + valueToWrite = String.join(":", idScope, val); + // Update key value for subsequent ID conflict checks for this row. + keyValue = valueToWrite; + mergeFeedsResult.remappedIds.put( + getTableScopedValue(table, idScope, val), + valueToWrite + ); + } + } for (NewGTFSError error : idErrors) { if (error.errorType.equals(NewGTFSErrorType.DUPLICATE_ID)) { - mergeFeedsResult.failureReasons - .add("Trip ID conflict caused merge failure."); - mergeFeedsResult.idConflicts.add(error.badValue); - mergeFeedsResult.errorCount++; - if (failOnDuplicateTripId) - mergeFeedsResult.failed = true; - skipRecord = true; + valueToWrite = String.join(":", idScope, val); + // Update key value for subsequent ID conflict checks for this row. + keyValue = valueToWrite; + mergeFeedsResult.remappedIds.put( + getTableScopedValue(table, idScope, val), + valueToWrite + ); } } break; @@ -965,10 +1048,26 @@ private int constructMergedTable(Table table, List feedsToMerge, .toArray(String[]::new); writer.write(headers); } - // Write line to table (plus new line char). + // Write line to table. writer.write(rowValues); lineNumber++; mergedLineNumber++; + if ((table.name.equals("calendar")) && serviceIdsToCloneAndRename.contains(rowValues[keyFieldIndex])) { + // FIXME: Do we need to worry about calendar_dates? + String[] clonedValues = rowValues.clone(); + String newServiceId = clonedValues[keyFieldIndex] = String.join(":", idScope, rowValues[keyFieldIndex]);; + // Modify start/end date. + int startDateIndex = Table.CALENDAR.getFieldIndex("start_date"); + int endDateIndex = Table.CALENDAR.getFieldIndex("end_date"); + clonedValues[startDateIndex] = + feed.version.validationResult.firstCalendarDate.format(GTFS_DATE_FORMATTER); + clonedValues[endDateIndex] = + feed.version.validationResult.lastCalendarDate.format(GTFS_DATE_FORMATTER); + referenceTracker.checkReferencesAndUniqueness(keyValue, lineNumber, table.fields[0], newServiceId, table, keyField, orderField); + writer.write(clonedValues); + lineNumber++; + mergedLineNumber++; + } } // End of iteration over each row. } writer.flush(); @@ -985,71 +1084,45 @@ private int constructMergedTable(Table table, List feedsToMerge, return mergedLineNumber; } - private static String stopCodeFailureMessage(int stopsMissingStopCodeCount, int stopsCount, int specialStopsCount) { - return String.format( - "If stop_code is provided for some stops (for those with location_type = " + - "empty or 0), all stops must have stop_code values. The merge process " + - "found %d of %d total stops that were incorrectly missing stop_code values. " + - "Note: \"special\" stops with location_type > 0 need not specify this value " + - "(%d special stops found in feed).", - stopsMissingStopCodeCount, - stopsCount, - specialStopsCount - ); - } - - /** Get the set of shared fields for all feeds being merged for a specific table. */ - private Set getSharedFields(List feedsToMerge, Table table) throws IOException { - Set sharedFields = new HashSet<>(); - // First, iterate over each feed to collect the shared fields that need to be output in the merged table. - for (FeedToMerge feed : feedsToMerge) { - CsvReader csvReader = table.getCsvReader(feed.zipFile, null); - // If csv reader is null, the table was not found in the zip file. - if (csvReader == null) { - continue; + private static MergeStrategy getMergeStrategy(List feedsToMerge) throws IOException { + // Iterate over both feeds (future feed is first). + for (int i = 0; i < feedsToMerge.size(); i++) { + FeedToMerge feed = feedsToMerge.get(i); + Set tablesToCheck = Sets.newHashSet(Table.TRIPS, Table.CALENDAR, Table.CALENDAR_DATES); + for (Table table : tablesToCheck) { + feed.idsForTable.get(table).addAll(getIdsForTable(feed.zipFile, table)); } - // Get fields found from headers and add them to the shared fields set. - Field[] fieldsFoundInZip = table.getFieldsFromFieldHeaders(csvReader.getHeaders(), null); - sharedFields.addAll(Arrays.asList(fieldsFoundInZip)); } - return sharedFields; - } - - /** - * Checks whether a collection of fields contains a field with the provided name. - */ - private boolean containsField(Collection fields, String fieldName) { - for (Field field : fields) if (field.name.equals(fieldName)) return true; - return false; - } - - /** Checks that any of a set of errors is of the type {@link NewGTFSErrorType#DUPLICATE_ID}. */ - private boolean hasDuplicateError(Set errors) { - for (NewGTFSError error : errors) { - if (error.errorType.equals(NewGTFSErrorType.DUPLICATE_ID)) return true; + // We are on the last (second) feed to merge. Check for conflicts with future feed. + FeedToMerge futureFeed = feedsToMerge.get(0); + FeedToMerge currentFeed = feedsToMerge.get(1); + Set pastTripIds = currentFeed.idsForTable.get(Table.TRIPS); + Set futureTripIds = futureFeed.idsForTable.get(Table.TRIPS); + Set pastServiceIds = new HashSet<>(); + pastServiceIds.addAll(currentFeed.idsForTable.get(Table.CALENDAR)); + pastServiceIds.addAll(currentFeed.idsForTable.get(Table.CALENDAR_DATES)); + Set futureServiceIds = new HashSet<>(); + futureServiceIds.addAll(futureFeed.idsForTable.get(Table.CALENDAR)); + futureServiceIds.addAll(futureFeed.idsForTable.get(Table.CALENDAR_DATES)); + boolean serviceIdsMatch = pastServiceIds.equals(futureServiceIds); + intersectingTripIds = Sets.intersection(pastTripIds, futureTripIds); + tripsOnlyInCurrentFeed = Sets.difference(pastTripIds, futureTripIds); + tripsOnlyInFutureFeed = Sets.difference(futureTripIds, pastTripIds); + boolean tripIdsMatch = pastTripIds.equals(futureTripIds); + if (serviceIdsMatch && tripIdsMatch) { + // Effectively this exact match condition means that the future feed will be used as is + // (including stops, routes, etc.), the only modification being service date ranges. + // This is Condition 2 in the docs. + return EXTEND_FUTURE; } - return false; - } - - /** Get table-scoped value used for key when remapping references for a particular feed. */ - private static String getTableScopedValue(Table table, String prefix, String id) { - return String.join(":", - table.name, - prefix, - id); - } - - /** - * Helper class that collects the feed version and its zip file. Note: this class helps with sorting versions to - * merge in a list collection. - */ - private class FeedToMerge { - public FeedVersion version; - public ZipFile zipFile; - - FeedToMerge(FeedVersion version) throws IOException { - this.version = version; - this.zipFile = new ZipFile(version.retrieveGtfsFile()); + if (serviceIdsMatch) { + // If just the service_ids are an exact match, do the trip/stoptimes checking thing for matching trip_ids + return CHECK_STOP_TIMES; + } + if (tripIdsMatch) { + // Do not permit merge to continue. + return MergeStrategy.FAIL_DUE_TO_MATCHING_TRIP_IDS; } + return MergeStrategy.DEFAULT; } } diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsResult.java b/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsResult.java index 86b8aa643..43cea68ae 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsResult.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsResult.java @@ -1,5 +1,7 @@ package com.conveyal.datatools.manager.jobs; +import org.eclipse.jetty.http.HttpFields; + import java.io.Serializable; import java.util.Date; import java.util.HashMap; @@ -37,6 +39,7 @@ public class MergeFeedsResult implements Serializable { public boolean failed; /** Set of reasons explaining why merge operation failed */ public Set failureReasons = new HashSet<>(); + public Set tripIdsToCheck = new HashSet<>(); public MergeFeedsResult (MergeFeedsType type) { this.type = type; diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/MergeStrategy.java b/src/main/java/com/conveyal/datatools/manager/jobs/MergeStrategy.java new file mode 100644 index 000000000..185de9c86 --- /dev/null +++ b/src/main/java/com/conveyal/datatools/manager/jobs/MergeStrategy.java @@ -0,0 +1,46 @@ +package com.conveyal.datatools.manager.jobs; + +/** + * This enum defines the different strategies for merging, which is currently dependent on whether trip_ids and/or + * service_ids between the two feeds are exactly matching. + */ +public enum MergeStrategy { + /** + * If service_ids and trip_ids between active and future feed are all unique, all IDs shall be included + * in merged feed. If a service_id from the active calendar has end_date in the future, the end_date shall be + * set to one day prior to the earliest start_date in future dataset before appending the calendar record to + * the merged file. It shall be ensured that trip_ids between active and future datasets must not match. + */ + DEFAULT, + /** + * If service_ids and trip_ids in active feed are the same as future feed then the service end date for the + * merged feed shall match with future feed’s service end date and the service start date for the merged feed + * should be the merged date. All files from the future feed only shall be used in the merged feed. + */ + EXTEND_FUTURE, + /** + * If trip_ids provided in active and future feeds are the same but the service_ids are unique then merge + * functionality shall reject feeds from merging. The user shall be notified that a new service requires unique + * trip_ids for merging. + */ + FAIL_DUE_TO_MATCHING_TRIP_IDS, + /** + * If service_ids in active and future feed exactly match but only some of the trip_ids match then the merge + * strategy shall handle the following three cases: + * - *trip_id in both feeds*: The service shall start from the data merge date and end at the future feed’s service + * end date. + * Note: The merge process shall validate records in stop_times.txt file for same trip signature (same set of + * stops with same sequence). Trips with matching stop_times will be included as is (but not duplicated of course). + * Trips that do not match on stop_times will be handled with the below approaches. + * Note: Same service IDs shall be used (but extended to account for the full range of dates from past to future). + * - *trip_id in past feed*: A new service shall be created starting from the merge date and expiring at the end + * of active service period. + * Note: a new service_id will be generated for these current/past trips in the merged feed (rather than using the + * service_id with extended range). + * - *trip_id in future feed*: A new service shall be created for these trips with service period defined in future + * feed + * Note: a new service_id will be generated for these future trips in the merged feed (rather than using the + * service_id with extended range). + */ + CHECK_STOP_TIMES +} \ No newline at end of file diff --git a/src/main/java/com/conveyal/datatools/manager/utils/MergeFeedUtils.java b/src/main/java/com/conveyal/datatools/manager/utils/MergeFeedUtils.java new file mode 100644 index 000000000..638025c19 --- /dev/null +++ b/src/main/java/com/conveyal/datatools/manager/utils/MergeFeedUtils.java @@ -0,0 +1,144 @@ +package com.conveyal.datatools.manager.utils; + +import com.conveyal.datatools.manager.jobs.FeedToMerge; +import com.conveyal.datatools.manager.jobs.MergeStrategy; +import com.conveyal.datatools.manager.models.FeedVersion; +import com.conveyal.gtfs.error.NewGTFSError; +import com.conveyal.gtfs.error.NewGTFSErrorType; +import com.conveyal.gtfs.loader.Field; +import com.conveyal.gtfs.loader.Table; +import com.csvreader.CsvReader; +import com.google.common.collect.Sets; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Collection; +import java.util.Comparator; +import java.util.HashSet; +import java.util.List; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.zip.ZipFile; + +import static com.conveyal.gtfs.loader.Field.getFieldIndex; + +public class MergeFeedUtils { + private static final Logger LOG = LoggerFactory.getLogger(MergeFeedUtils.class); + + /** + * Get the ids (e.g., trip_id, service_id) for the provided table from the zipfile. + */ + public static Set getIdsForTable(ZipFile zipFile, Table table) throws IOException { + Set ids = new HashSet<>(); + String keyField = table.getKeyFieldName(); + CsvReader csvReader = table.getCsvReader(zipFile, null); + if (csvReader == null) { + LOG.warn("Table {} not found in zip file: {}", table.name, zipFile.getName()); + return ids; + } + Field[] fieldsFoundInZip = table.getFieldsFromFieldHeaders(csvReader.getHeaders(), null); + // Get the key field (id value) for each row. + int keyFieldIndex = getFieldIndex(fieldsFoundInZip, keyField); + while (csvReader.readRecord()) ids.add(csvReader.get(keyFieldIndex)); + csvReader.close(); + return ids; + } + + public static String stopCodeFailureMessage(int stopsMissingStopCodeCount, int stopsCount, int specialStopsCount) { + return String.format( + "If stop_code is provided for some stops (for those with location_type = " + + "empty or 0), all stops must have stop_code values. The merge process " + + "found %d of %d total stops that were incorrectly missing stop_code values. " + + "Note: \"special\" stops with location_type > 0 need not specify this value " + + "(%d special stops found in feed).", + stopsMissingStopCodeCount, + stopsCount, + specialStopsCount + ); + } + + /** + * Collect zipFiles for each feed version before merging tables. + * Note: feed versions are sorted by first calendar date so that future dataset is iterated over first. This is + * required for the MTC merge strategy which prefers entities from the future dataset over past entities. + */ + public static List collectAndSortFeeds(Set feedVersions) { + return feedVersions.stream().map(version -> { + try { + return new FeedToMerge(version); + } catch (Exception e) { + LOG.error("Could not create zip file for version: {}", version.version); + return null; + } + }).filter(Objects::nonNull).filter(entry -> entry.version.validationResult != null + && entry.version.validationResult.firstCalendarDate != null) + // MTC-specific sort mentioned in above comment. + // TODO: If another merge strategy requires a different sort order, a merge type check should be added. + .sorted(Comparator.comparing(entry -> entry.version.validationResult.firstCalendarDate, + Comparator.reverseOrder())).collect(Collectors.toList()); + } + + /** Get the set of shared fields for all feeds being merged for a specific table. */ + public static Set getSharedFields(List feedsToMerge, Table table) throws IOException { + Set sharedFields = new HashSet<>(); + // First, iterate over each feed to collect the shared fields that need to be output in the merged table. + for (FeedToMerge feed : feedsToMerge) { + CsvReader csvReader = table.getCsvReader(feed.zipFile, null); + // If csv reader is null, the table was not found in the zip file. + if (csvReader == null) { + continue; + } + // Get fields found from headers and add them to the shared fields set. + Field[] fieldsFoundInZip = table.getFieldsFromFieldHeaders(csvReader.getHeaders(), null); + sharedFields.addAll(Arrays.asList(fieldsFoundInZip)); + } + return sharedFields; + } + + /** + * Checks whether a collection of fields contains a field with the provided name. + */ + public static boolean containsField(Collection fields, String fieldName) { + for (Field field : fields) if (field.name.equals(fieldName)) return true; + return false; + } + + /** Checks that any of a set of errors is of the type {@link NewGTFSErrorType#DUPLICATE_ID}. */ + public static boolean hasDuplicateError(Set errors) { + for (NewGTFSError error : errors) { + if (error.errorType.equals(NewGTFSErrorType.DUPLICATE_ID)) return true; + } + return false; + } + + /** Get table-scoped value used for key when remapping references for a particular feed. */ + public static String getTableScopedValue(Table table, String prefix, String id) { + return String.join(":", + table.name, + prefix, + id); + } + + public static boolean preferFutureTable(Table table, MergeStrategy mergeStrategy) { + Set tablesToNotPreferFuture = Sets.newHashSet( + Table.CALENDAR.name, + Table.CALENDAR_DATES.name + ); + if (tablesToNotPreferFuture.contains(table.name)) return false; + // Always prefer the "future" file for the feed_info table, which means + // we can skip any iterations following the first one. If merging the agency + // table, we should only skip the following feeds if performing an MTC merge + // because that logic assumes the two feeds share the same agency (or + // agencies). NOTE: feed_info file is skipped by default (outside of this + // method) for a regional merge), which is why this block is exclusively + // for an MTC merge. Also, this statement may print multiple log + // statements, but it is deliberately nested in the csv while block in + // order to detect agency_id mismatches and fail the merge if found. + if (table.name.equals("feed_info")) return true; + // If merge strategy is to extend the future calendar files to the past, all other files should prefer future. + return MergeStrategy.EXTEND_FUTURE.equals(mergeStrategy); + } +} diff --git a/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java b/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java index 7c6084007..a7a74eff2 100644 --- a/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java +++ b/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java @@ -8,6 +8,7 @@ import com.conveyal.datatools.manager.models.Project; import com.conveyal.datatools.manager.persistence.Persistence; import com.conveyal.gtfs.error.NewGTFSErrorType; +import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.slf4j.Logger; @@ -26,6 +27,7 @@ import static com.conveyal.datatools.TestUtils.zipFolderFiles; import static com.conveyal.datatools.manager.models.FeedRetrievalMethod.MANUALLY_UPLOADED; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; @@ -36,7 +38,7 @@ public class MergeFeedsJobTest extends UnitTest { private static final Logger LOG = LoggerFactory.getLogger(MergeFeedsJobTest.class); private static Auth0UserProfile user = Auth0UserProfile.createTestAdminUser(); private static FeedVersion bartVersion1; - private static FeedVersion bartVersion2; + private static FeedVersion bartVersion2SameTrips; private static FeedVersion calTrainVersion; private static Project project; private static FeedVersion napaVersion; @@ -45,6 +47,10 @@ public class MergeFeedsJobTest extends UnitTest { private static FeedVersion bothCalendarFilesVersion3; private static FeedVersion onlyCalendarVersion; private static FeedVersion onlyCalendarDatesVersion; + private static FeedVersion fakeTransitBase; + private static FeedVersion fakeTransitFuture; + private static FeedVersion fakeTransitModService; + private static FeedVersion fakeTransitModTrips; /** * Prepare and start a testing-specific web server @@ -63,7 +69,7 @@ public static void setUp() throws IOException { FeedSource bart = new FeedSource("BART", project.id, MANUALLY_UPLOADED); Persistence.feedSources.create(bart); bartVersion1 = createFeedVersionFromGtfsZip(bart, "bart_old.zip"); - bartVersion2 = createFeedVersionFromGtfsZip(bart, "bart_new.zip"); + bartVersion2SameTrips = createFeedVersionFromGtfsZip(bart, "bart_new.zip"); // Caltrain FeedSource caltrain = new FeedSource("Caltrain", project.id, MANUALLY_UPLOADED); @@ -98,6 +104,23 @@ public static void setUp() throws IOException { fakeAgency, zipFolderFiles("fake-agency-with-calendar-and-calendar-dates-3") ); + + // Other fake feeds for testing MTC MergeStrategy types. + FeedSource fakeTransit = new FeedSource("Fake Transit", project.id, MANUALLY_UPLOADED); + Persistence.feedSources.create(fakeTransit); + fakeTransitBase = createFeedVersion(fakeTransit, zipFolderFiles("merge-data-base")); + fakeTransitFuture = createFeedVersion(fakeTransit, zipFolderFiles("merge-data-future")); + fakeTransitModService = createFeedVersion(fakeTransit, zipFolderFiles("merge-data-mod-services")); + fakeTransitModTrips = createFeedVersion(fakeTransit, zipFolderFiles("merge-data-mod-trips")); + } + + /** + * Prepare and start a testing-specific web server + */ + @AfterAll + public static void tearDown() throws IOException { + // Delete project (feed sources/versions will also be deleted). + if (project != null) project.delete(); } /** @@ -249,84 +272,207 @@ public void canMergeRegionalWithOnlyCalendarFeed () throws SQLException { } /** - * Ensures that an MTC merge of feeds with duplicate trip IDs will fail. + * Ensures that an MTC merge of feeds that has exactly matching trips but mismatched services fails according to the + * strategy {@link MergeStrategy#FAIL_DUE_TO_MATCHING_TRIP_IDS}. */ @Test - public void mergeMTCShouldFailOnDuplicateTrip() { + public void mergeMTCShouldFailOnDuplicateTripsButMismatchedServices() { Set versions = new HashSet<>(); - versions.add(bartVersion1); - versions.add(bartVersion2); + versions.add(fakeTransitBase); + versions.add(fakeTransitModService); MergeFeedsJob mergeFeedsJob = new MergeFeedsJob(user, versions, "merged_output", MergeFeedsType.SERVICE_PERIOD); // Run the job in this thread (we're not concerned about concurrency here). mergeFeedsJob.run(); // Result should fail. assertTrue( mergeFeedsJob.mergeFeedsResult.failed, - "Merge feeds job should fail due to duplicate trip IDs." + "Merge feeds job should fail if feeds have exactly matching trips but mismatched services." + ); + assertEquals( + MergeStrategy.FAIL_DUE_TO_MATCHING_TRIP_IDS, + mergeFeedsJob.mergeStrategy ); } /** - * Tests that the MTC merge strategy will successfully merge BART feeds. Note: this test turns off - * {@link MergeFeedsJob#failOnDuplicateTripId} in order to force the merge to succeed even though there are duplicate - * trips contained within. + * Ensures that an MTC merge of feeds with exact matches of service_ids and trip_ids will utilize the + * {@link MergeStrategy#EXTEND_FUTURE} strategy correctly. */ @Test - public void canMergeBARTFeeds() throws SQLException { + public void mergeMTCShouldHandleExtendFutureStrategy() throws SQLException { Set versions = new HashSet<>(); - versions.add(bartVersion1); - versions.add(bartVersion2); + versions.add(fakeTransitBase); + versions.add(fakeTransitFuture); MergeFeedsJob mergeFeedsJob = new MergeFeedsJob(user, versions, "merged_output", MergeFeedsType.SERVICE_PERIOD); - // This time, turn off the failOnDuplicateTripId flag. - mergeFeedsJob.failOnDuplicateTripId = false; - // Result should succeed this time. + // Run the job in this thread (we're not concerned about concurrency here). mergeFeedsJob.run(); - assertFeedMergeSucceeded(mergeFeedsJob); - // Check GTFS+ line numbers. - assertEquals( - 2, // Magic number represents expected number of lines after merge. - mergeFeedsJob.mergeFeedsResult.linesPerTable.get("directions").intValue(), - "Merged directions count should equal expected value." + // Result should fail. + assertFalse( + mergeFeedsJob.mergeFeedsResult.failed, + "Merge feeds job should succeed with EXTEND_FEED strategy." ); assertEquals( - 2, // Magic number represents the number of stop_attributes in the merged BART feed. - mergeFeedsJob.mergeFeedsResult.linesPerTable.get("stop_attributes").intValue(), - "Merged feed stop_attributes count should equal expected value." + MergeStrategy.EXTEND_FUTURE, + mergeFeedsJob.mergeStrategy ); - // Check GTFS file line numbers. - assertEquals( - 4552, // Magic number represents the number of trips in the merged BART feed. - mergeFeedsJob.mergedVersion.feedLoadResult.trips.rowCount, - "Merged feed trip count should equal expected value." + // assert service_ids start_dates have been extended to the start_date of the base feed. + String mergedNamespace = mergeFeedsJob.mergedVersion.namespace; + + // - calendar table + // expect a total of 2 records in calendar table + assertThatSqlCountQueryYieldsExpectedCount( + String.format("SELECT count(*) FROM %s.calendar", mergedNamespace), + 2 ); - assertEquals( - 9, // Magic number represents the number of routes in the merged BART feed. - mergeFeedsJob.mergedVersion.feedLoadResult.routes.rowCount, - "Merged feed route count should equal expected value." + // expect that both records in calendar table have the correct start_date + assertThatSqlCountQueryYieldsExpectedCount( + String.format("SELECT count(*) FROM %s.calendar where start_date = '20170918' and monday = 1", mergedNamespace), + 2 + ); + } + + /** + * Ensures that an MTC merge of feeds with exact matches of service_ids and trip_ids will utilize the + * {@link MergeStrategy#CHECK_STOP_TIMES} strategy correctly. + */ + @Test + public void mergeMTCShouldHandleCheckStopTimesStrategy() throws SQLException { + Set versions = new HashSet<>(); + versions.add(fakeTransitBase); + versions.add(fakeTransitModTrips); + MergeFeedsJob mergeFeedsJob = new MergeFeedsJob(user, versions, "merged_output", MergeFeedsType.SERVICE_PERIOD); + // Run the job in this thread (we're not concerned about concurrency here). + mergeFeedsJob.run(); + // Result should fail. + assertFalse( + mergeFeedsJob.mergeFeedsResult.failed, + "Merge feeds job should succeed with CHECK_STOP_TIMES strategy." ); assertEquals( - // During merge, if identical shape_id is found in both feeds, active feed shape_id should be feed-scoped. - bartVersion1.feedLoadResult.shapes.rowCount + bartVersion2.feedLoadResult.shapes.rowCount, - mergeFeedsJob.mergedVersion.feedLoadResult.shapes.rowCount, - "Merged feed shapes count should equal expected value." - ); - // Expect that two calendar dates are excluded from the past feed (because they occur after the first date of - // the future feed) . - int expectedCalendarDatesCount = bartVersion1.feedLoadResult.calendarDates.rowCount + bartVersion2.feedLoadResult.calendarDates.rowCount - 2; + MergeStrategy.CHECK_STOP_TIMES, + mergeFeedsJob.mergeStrategy + ); + // assert service_ids start_dates have been extended to the start_date of the base feed. + String mergedNamespace = mergeFeedsJob.mergedVersion.namespace; + + // - calendar table + // expect a total of 6 records in calendar table: + // - 2 original (common_id start date extended) + // - 2 cloned for past/current feed + assertThatSqlCountQueryYieldsExpectedCount( + String.format("SELECT count(*) FROM %s.calendar", mergedNamespace), + 4 + ); + // expect that both records in calendar table have the correct start_date + assertThatSqlCountQueryYieldsExpectedCount( + String.format("SELECT count(*) FROM %s.calendar where start_date = '20170918' and end_date = '20170925'", mergedNamespace), + 1 + ); + // expect 3 trips (one trip is omitted from the past/current feed because it is an exact match of a future trip + assertThatSqlCountQueryYieldsExpectedCount( + String.format("SELECT count(*) FROM %s.trips", mergedNamespace), + 3 + ); + } + + /** + * Ensures that an MTC merge of feeds with exact matches of service_ids and trip_ids will utilize the + * {@link MergeStrategy#DEFAULT} strategy correctly. + */ + @Test + public void mergeMTCShouldHandleDefaultStrategy() throws SQLException { + Set versions = new HashSet<>(); + versions.add(fakeTransitBase); + versions.add(fakeTransitFuture); + MergeFeedsJob mergeFeedsJob = new MergeFeedsJob(user, versions, "merged_output", MergeFeedsType.SERVICE_PERIOD); + // Run the job in this thread (we're not concerned about concurrency here). + mergeFeedsJob.run(); + // Result should fail. + assertFalse( + mergeFeedsJob.mergeFeedsResult.failed, + "Merge feeds job should utilize EXTEND_FEED strategy." + ); assertEquals( - // During merge, if identical shape_id is found in both feeds, active feed shape_id should be feed-scoped. - expectedCalendarDatesCount, - mergeFeedsJob.mergedVersion.feedLoadResult.calendarDates.rowCount, - "Merged feed calendar_dates count should equal expected value." + MergeStrategy.DEFAULT, + mergeFeedsJob.mergeStrategy ); - // Ensure there are no referential integrity errors or duplicate ID errors. - assertThatFeedHasNoErrorsOfType( - mergeFeedsJob.mergedVersion.namespace, - NewGTFSErrorType.REFERENTIAL_INTEGRITY.toString(), - NewGTFSErrorType.DUPLICATE_ID.toString() + // assert service_ids start_dates have been extended to the start_date of the base feed. + String mergedNamespace = mergeFeedsJob.mergedVersion.namespace; + + // - calendar table + // expect a total of 2 records in calendar table + assertThatSqlCountQueryYieldsExpectedCount( + String.format("SELECT count(*) FROM %s.calendar", mergedNamespace), + 2 + ); + // expect that both records in calendar table have the correct start_date + assertThatSqlCountQueryYieldsExpectedCount( + String.format("SELECT count(*) FROM %s.calendar where start_date = '20170918'", mergedNamespace), + 2 ); } +// /** +// * Tests that the MTC merge strategy will successfully merge BART feeds. Note: this test turns off +// * {@link MergeFeedsJob#failOnDuplicateTripId} in order to force the merge to succeed even though there are duplicate +// * trips contained within. +// */ +// @Test +// public void canMergeBARTFeeds() throws SQLException { +// Set versions = new HashSet<>(); +// versions.add(bartVersion1); +// versions.add(bartVersion2SameTrips); +// MergeFeedsJob mergeFeedsJob = new MergeFeedsJob(user, versions, "merged_output", MergeFeedsType.SERVICE_PERIOD); +// // This time, turn off the failOnDuplicateTripId flag. +// mergeFeedsJob.failOnDuplicateTripId = false; +// // Result should succeed this time. +// mergeFeedsJob.run(); +// assertFeedMergeSucceeded(mergeFeedsJob); +// // Check GTFS+ line numbers. +// assertEquals( +// 2, // Magic number represents expected number of lines after merge. +// mergeFeedsJob.mergeFeedsResult.linesPerTable.get("directions").intValue(), +// "Merged directions count should equal expected value." +// ); +// assertEquals( +// 2, // Magic number represents the number of stop_attributes in the merged BART feed. +// mergeFeedsJob.mergeFeedsResult.linesPerTable.get("stop_attributes").intValue(), +// "Merged feed stop_attributes count should equal expected value." +// ); +// // Check GTFS file line numbers. +// assertEquals( +// 4552, // Magic number represents the number of trips in the merged BART feed. +// mergeFeedsJob.mergedVersion.feedLoadResult.trips.rowCount, +// "Merged feed trip count should equal expected value." +// ); +// assertEquals( +// 9, // Magic number represents the number of routes in the merged BART feed. +// mergeFeedsJob.mergedVersion.feedLoadResult.routes.rowCount, +// "Merged feed route count should equal expected value." +// ); +// assertEquals( +// // During merge, if identical shape_id is found in both feeds, active feed shape_id should be feed-scoped. +// bartVersion1.feedLoadResult.shapes.rowCount + bartVersion2SameTrips.feedLoadResult.shapes.rowCount, +// mergeFeedsJob.mergedVersion.feedLoadResult.shapes.rowCount, +// "Merged feed shapes count should equal expected value." +// ); +// // Expect that two calendar dates are excluded from the past feed (because they occur after the first date of +// // the future feed) . +// int expectedCalendarDatesCount = bartVersion1.feedLoadResult.calendarDates.rowCount + bartVersion2SameTrips.feedLoadResult.calendarDates.rowCount - 2; +// assertEquals( +// // During merge, if identical shape_id is found in both feeds, active feed shape_id should be feed-scoped. +// expectedCalendarDatesCount, +// mergeFeedsJob.mergedVersion.feedLoadResult.calendarDates.rowCount, +// "Merged feed calendar_dates count should equal expected value." +// ); +// // Ensure there are no referential integrity errors or duplicate ID errors. +// assertThatFeedHasNoErrorsOfType( +// mergeFeedsJob.mergedVersion.namespace, +// NewGTFSErrorType.REFERENTIAL_INTEGRITY.toString(), +// NewGTFSErrorType.DUPLICATE_ID.toString() +// ); +// } + /** * Tests whether a MTC feed merge of two feed versions correctly feed scopes the service_id's of the feed that is * chronologically before the other one. This tests two feeds where one of them has both calendar files, and the diff --git a/src/test/resources/com/conveyal/datatools/gtfs/merge-data-base/agency.txt b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-base/agency.txt new file mode 100755 index 000000000..a916ce91b --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-base/agency.txt @@ -0,0 +1,2 @@ +agency_id,agency_name,agency_url,agency_lang,agency_phone,agency_email,agency_timezone,agency_fare_url,agency_branding_url +1,Fake Transit,,,,,America/Los_Angeles,, diff --git a/src/test/resources/com/conveyal/datatools/gtfs/merge-data-base/calendar.txt b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-base/calendar.txt new file mode 100755 index 000000000..0e75afb73 --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-base/calendar.txt @@ -0,0 +1,3 @@ +service_id,monday,tuesday,wednesday,thursday,friday,saturday,sunday,start_date,end_date +common_id,1,1,1,1,1,1,1,20170918,20170920 +only_calendar_id,1,1,1,1,1,1,1,20170921,20170922 diff --git a/src/test/resources/com/conveyal/datatools/gtfs/merge-data-base/feed_info.txt b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-base/feed_info.txt new file mode 100644 index 000000000..ceac60810 --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-base/feed_info.txt @@ -0,0 +1,2 @@ +feed_id,feed_publisher_name,feed_publisher_url,feed_lang,feed_version +fake_transit,Conveyal,http://www.conveyal.com,en,1.0 \ No newline at end of file diff --git a/src/test/resources/com/conveyal/datatools/gtfs/merge-data-base/routes.txt b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-base/routes.txt new file mode 100755 index 000000000..b13480efa --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-base/routes.txt @@ -0,0 +1,3 @@ +agency_id,route_id,route_short_name,route_long_name,route_desc,route_type,route_url,route_color,route_text_color,route_branding_url +1,1,1,Route 1,,3,,7CE6E7,FFFFFF, +1,2,2,Route 2,,3,,7CE6E7,FFFFFF, diff --git a/src/test/resources/com/conveyal/datatools/gtfs/merge-data-base/stop_attributes.txt b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-base/stop_attributes.txt new file mode 100644 index 000000000..b77c473e0 --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-base/stop_attributes.txt @@ -0,0 +1,3 @@ +stop_id,accessibility_id,cardinal_direction,relative_position,stop_city +4u6g,0,SE,FS,Scotts Valley +johv,0,SE,FS,Scotts Valley diff --git a/src/test/resources/com/conveyal/datatools/gtfs/merge-data-base/stop_times.txt b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-base/stop_times.txt new file mode 100755 index 000000000..081b2fad0 --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-base/stop_times.txt @@ -0,0 +1,7 @@ +trip_id,arrival_time,departure_time,stop_id,stop_sequence,stop_headsign,pickup_type,drop_off_type,shape_dist_traveled,timepoint +only-calendar-trip1,07:00:00,07:00:00,4u6g,1,,0,0,0.0000000, +only-calendar-trip1,07:01:00,07:01:00,johv,2,,0,0,341.4491961, +only-calendar-trip2,07:00:00,07:00:00,johv,1,,0,0,0.0000000, +only-calendar-trip2,07:01:00,07:01:00,4u6g,2,,0,0,341.4491961, +trip3,07:00:00,07:00:00,4u6g,1,,0,0,0.0000000, +trip3,07:01:00,07:01:00,johv,2,,0,0,341.4491961, \ No newline at end of file diff --git a/src/test/resources/com/conveyal/datatools/gtfs/merge-data-base/stops.txt b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-base/stops.txt new file mode 100755 index 000000000..0db5a6d40 --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-base/stops.txt @@ -0,0 +1,6 @@ +stop_id,stop_code,stop_name,stop_desc,stop_lat,stop_lon,zone_id,stop_url,location_type,parent_station,stop_timezone,wheelchair_boarding +4u6g,4u6g,Butler Ln,,37.0612132,-122.0074332,,,0,,, +johv,johv,Scotts Valley Dr & Victor Sq,,37.0590172,-122.0096058,,,0,,, +123,,Parent Station,,37.0666,-122.0777,,,1,,, +1234,1234,Child Stop,,37.06662,-122.07772,,,0,123,, +1234567,1234567,Unused stop,,37.06668,-122.07781,,,0,123,, \ No newline at end of file diff --git a/src/test/resources/com/conveyal/datatools/gtfs/merge-data-base/trips.txt b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-base/trips.txt new file mode 100755 index 000000000..1a8e2c972 --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-base/trips.txt @@ -0,0 +1,4 @@ +route_id,trip_id,trip_headsign,trip_short_name,direction_id,block_id,shape_id,bikes_allowed,wheelchair_accessible,service_id +1,only-calendar-trip1,,,0,,,0,0,common_id +2,only-calendar-trip2,,,0,,,0,0,common_id +2,trip3,,,0,,,0,0,common_id \ No newline at end of file diff --git a/src/test/resources/com/conveyal/datatools/gtfs/merge-data-future/agency.txt b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-future/agency.txt new file mode 100755 index 000000000..a916ce91b --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-future/agency.txt @@ -0,0 +1,2 @@ +agency_id,agency_name,agency_url,agency_lang,agency_phone,agency_email,agency_timezone,agency_fare_url,agency_branding_url +1,Fake Transit,,,,,America/Los_Angeles,, diff --git a/src/test/resources/com/conveyal/datatools/gtfs/merge-data-future/calendar.txt b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-future/calendar.txt new file mode 100755 index 000000000..e97e3d013 --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-future/calendar.txt @@ -0,0 +1,3 @@ +service_id,monday,tuesday,wednesday,thursday,friday,saturday,sunday,start_date,end_date +common_id,1,1,1,1,1,1,1,20170923,20170925 +only_calendar_id,1,1,1,1,1,1,1,20170924,20170927 diff --git a/src/test/resources/com/conveyal/datatools/gtfs/merge-data-future/feed_info.txt b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-future/feed_info.txt new file mode 100644 index 000000000..ceac60810 --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-future/feed_info.txt @@ -0,0 +1,2 @@ +feed_id,feed_publisher_name,feed_publisher_url,feed_lang,feed_version +fake_transit,Conveyal,http://www.conveyal.com,en,1.0 \ No newline at end of file diff --git a/src/test/resources/com/conveyal/datatools/gtfs/merge-data-future/routes.txt b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-future/routes.txt new file mode 100755 index 000000000..b13480efa --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-future/routes.txt @@ -0,0 +1,3 @@ +agency_id,route_id,route_short_name,route_long_name,route_desc,route_type,route_url,route_color,route_text_color,route_branding_url +1,1,1,Route 1,,3,,7CE6E7,FFFFFF, +1,2,2,Route 2,,3,,7CE6E7,FFFFFF, diff --git a/src/test/resources/com/conveyal/datatools/gtfs/merge-data-future/stop_attributes.txt b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-future/stop_attributes.txt new file mode 100644 index 000000000..b77c473e0 --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-future/stop_attributes.txt @@ -0,0 +1,3 @@ +stop_id,accessibility_id,cardinal_direction,relative_position,stop_city +4u6g,0,SE,FS,Scotts Valley +johv,0,SE,FS,Scotts Valley diff --git a/src/test/resources/com/conveyal/datatools/gtfs/merge-data-future/stop_times.txt b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-future/stop_times.txt new file mode 100755 index 000000000..646706c5c --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-future/stop_times.txt @@ -0,0 +1,5 @@ +trip_id,arrival_time,departure_time,stop_id,stop_sequence,stop_headsign,pickup_type,drop_off_type,shape_dist_traveled,timepoint +only-calendar-trip1,07:00:00,07:00:00,4u6g,1,,0,0,0.0000000, +only-calendar-trip1,07:01:00,07:01:00,johv,2,,0,0,341.4491961, +only-calendar-trip2,07:00:00,07:00:00,johv,1,,0,0,0.0000000, +only-calendar-trip2,07:01:00,07:01:00,4u6g,2,,0,0,341.4491961, \ No newline at end of file diff --git a/src/test/resources/com/conveyal/datatools/gtfs/merge-data-future/stops.txt b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-future/stops.txt new file mode 100755 index 000000000..0db5a6d40 --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-future/stops.txt @@ -0,0 +1,6 @@ +stop_id,stop_code,stop_name,stop_desc,stop_lat,stop_lon,zone_id,stop_url,location_type,parent_station,stop_timezone,wheelchair_boarding +4u6g,4u6g,Butler Ln,,37.0612132,-122.0074332,,,0,,, +johv,johv,Scotts Valley Dr & Victor Sq,,37.0590172,-122.0096058,,,0,,, +123,,Parent Station,,37.0666,-122.0777,,,1,,, +1234,1234,Child Stop,,37.06662,-122.07772,,,0,123,, +1234567,1234567,Unused stop,,37.06668,-122.07781,,,0,123,, \ No newline at end of file diff --git a/src/test/resources/com/conveyal/datatools/gtfs/merge-data-future/trips.txt b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-future/trips.txt new file mode 100755 index 000000000..387b076cd --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-future/trips.txt @@ -0,0 +1,3 @@ +route_id,trip_id,trip_headsign,trip_short_name,direction_id,block_id,shape_id,bikes_allowed,wheelchair_accessible,service_id +1,only-calendar-trip1,,,0,,,0,0,common_id +2,only-calendar-trip2,,,0,,,0,0,common_id \ No newline at end of file diff --git a/src/test/resources/com/conveyal/datatools/gtfs/merge-data-mod-services/agency.txt b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-mod-services/agency.txt new file mode 100755 index 000000000..a916ce91b --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-mod-services/agency.txt @@ -0,0 +1,2 @@ +agency_id,agency_name,agency_url,agency_lang,agency_phone,agency_email,agency_timezone,agency_fare_url,agency_branding_url +1,Fake Transit,,,,,America/Los_Angeles,, diff --git a/src/test/resources/com/conveyal/datatools/gtfs/merge-data-mod-services/calendar.txt b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-mod-services/calendar.txt new file mode 100755 index 000000000..0e75afb73 --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-mod-services/calendar.txt @@ -0,0 +1,3 @@ +service_id,monday,tuesday,wednesday,thursday,friday,saturday,sunday,start_date,end_date +common_id,1,1,1,1,1,1,1,20170918,20170920 +only_calendar_id,1,1,1,1,1,1,1,20170921,20170922 diff --git a/src/test/resources/com/conveyal/datatools/gtfs/merge-data-mod-services/feed_info.txt b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-mod-services/feed_info.txt new file mode 100644 index 000000000..ceac60810 --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-mod-services/feed_info.txt @@ -0,0 +1,2 @@ +feed_id,feed_publisher_name,feed_publisher_url,feed_lang,feed_version +fake_transit,Conveyal,http://www.conveyal.com,en,1.0 \ No newline at end of file diff --git a/src/test/resources/com/conveyal/datatools/gtfs/merge-data-mod-services/routes.txt b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-mod-services/routes.txt new file mode 100755 index 000000000..b13480efa --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-mod-services/routes.txt @@ -0,0 +1,3 @@ +agency_id,route_id,route_short_name,route_long_name,route_desc,route_type,route_url,route_color,route_text_color,route_branding_url +1,1,1,Route 1,,3,,7CE6E7,FFFFFF, +1,2,2,Route 2,,3,,7CE6E7,FFFFFF, diff --git a/src/test/resources/com/conveyal/datatools/gtfs/merge-data-mod-services/stop_attributes.txt b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-mod-services/stop_attributes.txt new file mode 100644 index 000000000..b77c473e0 --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-mod-services/stop_attributes.txt @@ -0,0 +1,3 @@ +stop_id,accessibility_id,cardinal_direction,relative_position,stop_city +4u6g,0,SE,FS,Scotts Valley +johv,0,SE,FS,Scotts Valley diff --git a/src/test/resources/com/conveyal/datatools/gtfs/merge-data-mod-services/stop_times.txt b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-mod-services/stop_times.txt new file mode 100755 index 000000000..646706c5c --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-mod-services/stop_times.txt @@ -0,0 +1,5 @@ +trip_id,arrival_time,departure_time,stop_id,stop_sequence,stop_headsign,pickup_type,drop_off_type,shape_dist_traveled,timepoint +only-calendar-trip1,07:00:00,07:00:00,4u6g,1,,0,0,0.0000000, +only-calendar-trip1,07:01:00,07:01:00,johv,2,,0,0,341.4491961, +only-calendar-trip2,07:00:00,07:00:00,johv,1,,0,0,0.0000000, +only-calendar-trip2,07:01:00,07:01:00,4u6g,2,,0,0,341.4491961, \ No newline at end of file diff --git a/src/test/resources/com/conveyal/datatools/gtfs/merge-data-mod-services/stops.txt b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-mod-services/stops.txt new file mode 100755 index 000000000..0db5a6d40 --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-mod-services/stops.txt @@ -0,0 +1,6 @@ +stop_id,stop_code,stop_name,stop_desc,stop_lat,stop_lon,zone_id,stop_url,location_type,parent_station,stop_timezone,wheelchair_boarding +4u6g,4u6g,Butler Ln,,37.0612132,-122.0074332,,,0,,, +johv,johv,Scotts Valley Dr & Victor Sq,,37.0590172,-122.0096058,,,0,,, +123,,Parent Station,,37.0666,-122.0777,,,1,,, +1234,1234,Child Stop,,37.06662,-122.07772,,,0,123,, +1234567,1234567,Unused stop,,37.06668,-122.07781,,,0,123,, \ No newline at end of file diff --git a/src/test/resources/com/conveyal/datatools/gtfs/merge-data-mod-services/trips.txt b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-mod-services/trips.txt new file mode 100755 index 000000000..387b076cd --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-mod-services/trips.txt @@ -0,0 +1,3 @@ +route_id,trip_id,trip_headsign,trip_short_name,direction_id,block_id,shape_id,bikes_allowed,wheelchair_accessible,service_id +1,only-calendar-trip1,,,0,,,0,0,common_id +2,only-calendar-trip2,,,0,,,0,0,common_id \ No newline at end of file diff --git a/src/test/resources/com/conveyal/datatools/gtfs/merge-data-mod-trips/agency.txt b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-mod-trips/agency.txt new file mode 100755 index 000000000..a916ce91b --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-mod-trips/agency.txt @@ -0,0 +1,2 @@ +agency_id,agency_name,agency_url,agency_lang,agency_phone,agency_email,agency_timezone,agency_fare_url,agency_branding_url +1,Fake Transit,,,,,America/Los_Angeles,, diff --git a/src/test/resources/com/conveyal/datatools/gtfs/merge-data-mod-trips/calendar.txt b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-mod-trips/calendar.txt new file mode 100755 index 000000000..e97e3d013 --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-mod-trips/calendar.txt @@ -0,0 +1,3 @@ +service_id,monday,tuesday,wednesday,thursday,friday,saturday,sunday,start_date,end_date +common_id,1,1,1,1,1,1,1,20170923,20170925 +only_calendar_id,1,1,1,1,1,1,1,20170924,20170927 diff --git a/src/test/resources/com/conveyal/datatools/gtfs/merge-data-mod-trips/feed_info.txt b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-mod-trips/feed_info.txt new file mode 100644 index 000000000..ceac60810 --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-mod-trips/feed_info.txt @@ -0,0 +1,2 @@ +feed_id,feed_publisher_name,feed_publisher_url,feed_lang,feed_version +fake_transit,Conveyal,http://www.conveyal.com,en,1.0 \ No newline at end of file diff --git a/src/test/resources/com/conveyal/datatools/gtfs/merge-data-mod-trips/routes.txt b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-mod-trips/routes.txt new file mode 100755 index 000000000..b13480efa --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-mod-trips/routes.txt @@ -0,0 +1,3 @@ +agency_id,route_id,route_short_name,route_long_name,route_desc,route_type,route_url,route_color,route_text_color,route_branding_url +1,1,1,Route 1,,3,,7CE6E7,FFFFFF, +1,2,2,Route 2,,3,,7CE6E7,FFFFFF, diff --git a/src/test/resources/com/conveyal/datatools/gtfs/merge-data-mod-trips/stop_attributes.txt b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-mod-trips/stop_attributes.txt new file mode 100644 index 000000000..b77c473e0 --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-mod-trips/stop_attributes.txt @@ -0,0 +1,3 @@ +stop_id,accessibility_id,cardinal_direction,relative_position,stop_city +4u6g,0,SE,FS,Scotts Valley +johv,0,SE,FS,Scotts Valley diff --git a/src/test/resources/com/conveyal/datatools/gtfs/merge-data-mod-trips/stop_times.txt b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-mod-trips/stop_times.txt new file mode 100755 index 000000000..646706c5c --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-mod-trips/stop_times.txt @@ -0,0 +1,5 @@ +trip_id,arrival_time,departure_time,stop_id,stop_sequence,stop_headsign,pickup_type,drop_off_type,shape_dist_traveled,timepoint +only-calendar-trip1,07:00:00,07:00:00,4u6g,1,,0,0,0.0000000, +only-calendar-trip1,07:01:00,07:01:00,johv,2,,0,0,341.4491961, +only-calendar-trip2,07:00:00,07:00:00,johv,1,,0,0,0.0000000, +only-calendar-trip2,07:01:00,07:01:00,4u6g,2,,0,0,341.4491961, \ No newline at end of file diff --git a/src/test/resources/com/conveyal/datatools/gtfs/merge-data-mod-trips/stops.txt b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-mod-trips/stops.txt new file mode 100755 index 000000000..0db5a6d40 --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-mod-trips/stops.txt @@ -0,0 +1,6 @@ +stop_id,stop_code,stop_name,stop_desc,stop_lat,stop_lon,zone_id,stop_url,location_type,parent_station,stop_timezone,wheelchair_boarding +4u6g,4u6g,Butler Ln,,37.0612132,-122.0074332,,,0,,, +johv,johv,Scotts Valley Dr & Victor Sq,,37.0590172,-122.0096058,,,0,,, +123,,Parent Station,,37.0666,-122.0777,,,1,,, +1234,1234,Child Stop,,37.06662,-122.07772,,,0,123,, +1234567,1234567,Unused stop,,37.06668,-122.07781,,,0,123,, \ No newline at end of file diff --git a/src/test/resources/com/conveyal/datatools/gtfs/merge-data-mod-trips/trips.txt b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-mod-trips/trips.txt new file mode 100755 index 000000000..d745f3502 --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-mod-trips/trips.txt @@ -0,0 +1,4 @@ +route_id,trip_id,trip_headsign,trip_short_name,direction_id,block_id,shape_id,bikes_allowed,wheelchair_accessible,service_id +1,only-calendar-trip999,,,0,,,0,0,common_id +2,only-calendar-trip2,,,0,,,0,0,common_id +2,trip3,,,0,,,0,0,only_calendar_id \ No newline at end of file From c59dbb303777ccd864816556e1e94d13c36424dc Mon Sep 17 00:00:00 2001 From: Landon Reed Date: Mon, 3 May 2021 09:05:43 -0400 Subject: [PATCH 004/122] refactor(MergeFeedsJob): use variables for feedIndex checks --- .../datatools/manager/jobs/MergeFeedsJob.java | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java b/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java index 8ba6a930d..f101b484a 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java @@ -440,7 +440,9 @@ private int constructMergedTable(Table table, List feedsToMerge, LocalDate futureFirstCalendarStartDate = LocalDate.MAX; // Iterate over each zip file. For service period merge, the first feed is the future GTFS. for (int feedIndex = 0; feedIndex < feedsToMerge.size(); feedIndex++) { - if (EXTEND_FUTURE.equals(mergeStrategy) && feedIndex > 0) { + boolean handlingPastFeed = feedIndex > 0; + boolean handlingFutureFeed = feedIndex == 0; + if (EXTEND_FUTURE.equals(mergeStrategy) && handlingPastFeed) { // No need to iterate over second (current) file if strategy is to simply extend the future GTFS // service to start earlier. continue; @@ -482,7 +484,7 @@ private int constructMergedTable(Table table, List feedsToMerge, // Iterate over rows in table, writing them to the out file. while (csvReader.readRecord()) { String keyValue = csvReader.get(keyFieldIndex); - if (feedIndex > 0 && mergeType.equals(SERVICE_PERIOD)) { + if (handlingPastFeed && mergeType.equals(SERVICE_PERIOD)) { // Always prefer the "future" file for the feed_info table, which means // we can skip any iterations following the first one. If merging the agency // table, we should only skip the following feeds if performing an MTC merge @@ -591,10 +593,10 @@ private int constructMergedTable(Table table, List feedsToMerge, // When all stops missing stop_code for the first feed, there's nothing to do (i.e., // no failure condition has been triggered yet). Just indicate this in the flag and // proceed with the merge. - if (feedIndex == 0) stopCodeMissingFromFirstFeed = true; + if (handlingFutureFeed) stopCodeMissingFromFirstFeed = true; // However... if the second feed was missing stop_codes and the first feed was not, // fail the merge job. - if (feedIndex == 1 && !stopCodeMissingFromFirstFeed) { + if (handlingPastFeed && !stopCodeMissingFromFirstFeed) { mergeFeedsResult.failed = true; mergeFeedsResult.errorCount++; mergeFeedsResult.failureReasons.add( @@ -673,7 +675,7 @@ private int constructMergedTable(Table table, List feedsToMerge, Set idErrors; // If analyzing the second feed (non-future feed), the service_id always gets feed scoped. // See https://github.com/ibi-group/datatools-server/issues/244 - if (feedIndex == 1 && field.name.equals("service_id")) { + if (handlingPastFeed && field.name.equals("service_id")) { valueToWrite = String.join(":", idScope, val); mergeFeedsResult.remappedIds.put( getTableScopedValue(table, idScope, val), @@ -711,7 +713,7 @@ private int constructMergedTable(Table table, List feedsToMerge, getFieldIndex(fieldsFoundInZip, "start_date"); LocalDate startDate = LocalDate .parse(csvReader.get(startDateIndex), GTFS_DATE_FORMATTER); - if (feedIndex == 0) { + if (handlingFutureFeed) { // For the future feed, check if the calendar's start date is earlier than the // previous earliest value and update if so. if (futureFirstCalendarStartDate.isAfter(startDate)) { @@ -771,7 +773,7 @@ private int constructMergedTable(Table table, List feedsToMerge, // not before the first date of the future feed. int dateIndex = getFieldIndex(fieldsFoundInZip, "date"); LocalDate date = LocalDate.parse(csvReader.get(dateIndex), GTFS_DATE_FORMATTER); - if (feedIndex > 0) { + if (handlingPastFeed) { if (!date.isBefore(futureFeedFirstDate)) { LOG.warn( "Skipping calendar_dates entry {} because it operates in the time span of future feed (i.e., after or on {}).", @@ -795,7 +797,7 @@ private int constructMergedTable(Table table, List feedsToMerge, // sequences 1,2,3,10 and active contains 1,2,7,9,10; the merged set will contain // 1,2,3,7,9,10). if (field.name.equals("shape_id")) { - if (feedIndex == 0) { + if (handlingFutureFeed) { // Track shape_id if working on future feed. shapeIdsInFutureFeed.add(val); } else if (shapeIdsInFutureFeed.contains(val)) { @@ -822,7 +824,7 @@ private int constructMergedTable(Table table, List feedsToMerge, case "trips": // trip_ids between active and future datasets must not match. The MergeStrategy // determines behavior when matching trip_ids (or service_ids) are found between - if (feedIndex > 0) { + if (handlingPastFeed) { // Handling past/current feed. if (tripIdsToSkipForCurrentFeed.contains(keyValue)) { skipRecord = true; From 206724041483c02e0c798d257055cd671548bbd2 Mon Sep 17 00:00:00 2001 From: Landon Reed Date: Mon, 3 May 2021 12:25:03 -0400 Subject: [PATCH 005/122] refactor: address PR #376 comments --- .../datatools/manager/jobs/MergeFeedsJob.java | 155 ++++++++++-------- .../manager/jobs/MergeFeedsResult.java | 2 - .../datatools/manager/jobs/MergeStrategy.java | 6 +- .../manager/utils/MergeFeedUtils.java | 62 ++++--- .../manager/jobs/MergeFeedsJobTest.java | 37 +++-- 5 files changed, 135 insertions(+), 127 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java b/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java index f101b484a..19f0ca25e 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java @@ -59,21 +59,21 @@ /** * This job handles merging two or more feed versions according to logic specific to the specified merge type. - * The current merge types handled here are: + * The merge types handled here are: * - {@link MergeFeedsType#REGIONAL}: this is essentially a "dumb" merge. For each feed version, each primary key is * scoped so that there is no possibility that it will conflict with other IDs * found in any other feed version. Note: There is absolutely no attempt to merge * entities based on either expected shared IDs or entity location (e.g., stop * coordinates). * - {@link MergeFeedsType#SERVICE_PERIOD}: this strategy is defined in detail at https://github.com/conveyal/datatools-server/issues/185, - * but in essence, this strategy attempts to merge a current and future feed into + * but in essence, this strategy attempts to merge an active and future feed into * a combined file. For certain entities (specifically stops and routes) it uses * alternate fields as primary keys (stop_code and route_short_name) if they are * available. There is some complexity related to this in {@link #constructMergedTable(Table, List, ZipOutputStream)}. * Another defining characteristic is to prefer entities defined in the "future" - * file if there are matching entities in the current file. + * file if there are matching entities in the active file. * Future merge strategies could be added here. For example, some potential customers have mentioned a desire to - * prefer entities from the "current" version, so that entities edited in Data Tools would override the values found + * prefer entities from the active version, so that entities edited in Data Tools would override the values found * in the "future" file, which may have limited data attributes due to being exported from scheduling software with * limited GTFS support. * @@ -86,9 +86,9 @@ * most recent active version or a selected one in order to further process the feed. * 3. Use the chosen version to merge the future feed. The merging process needs to be efficient so * that the user doesn’t need to wait more than a tolerable time. - * 4. The merge process shall compare the current and future datasets, validate the following rules + * 4. The merge process shall compare the active and future datasets, validate the following rules * and generate the Merge Validation Report: - * i. Merging will be based on route_short_name in the current and future datasets. All matching + * i. Merging will be based on route_short_name in the active and future datasets. All matching * route_short_names between the datasets shall be considered same route. Any route_short_name * in active data not present in the future will be appended to the future routes file. * ii. Future feed_info.txt file should get priority over active feed file when difference is @@ -113,15 +113,15 @@ * matching, the merge should fail with appropriate notification to user with the cause of the * failure. Notification should include all matched trip_ids. * vi. New shape_ids in the future datasets should be appended in the merged feed. - * vii. Merging fare_attributes will be based on fare_id in the current and future datasets. All + * vii. Merging fare_attributes will be based on fare_id in the active and future datasets. All * matching fare_ids between the datasets shall be considered same fare. Any fare_id in active * data not present in the future will be appended to the future fare_attributes file. * viii. All fare rules from the future dataset will be included. Any identical fare rules from - * the current dataset will be discarded. Any fare rules unique to the current dataset will be + * the active dataset will be discarded. Any fare rules unique to the active dataset will be * appended to the future file. * ix. All transfers.txt entries with unique stop pairs (from - to) from both the future and - * current datasets will be included in the merged file. Entries with duplicate stop pairs from - * the current dataset will be discarded. + * active datasets will be included in the merged file. Entries with duplicate stop pairs from + * the active dataset will be discarded. * x. All GTFS+ files should be merged based on how the associated base GTFS file is merged. For * example, directions for routes that are not in the future routes.txt file should be appended * to the future directions.txt file in the merged feed. @@ -131,7 +131,7 @@ public class MergeFeedsJob extends MonitorableJob { private static final Logger LOG = LoggerFactory.getLogger(MergeFeedsJob.class); public static final ObjectMapper mapper = new ObjectMapper(); private static Set intersectingTripIds = new HashSet<>(); - private static Set tripsOnlyInCurrentFeed = new HashSet<>(); + private static Set tripsOnlyInActiveFeed = new HashSet<>(); private static Set tripsOnlyInFutureFeed = new HashSet<>(); private final Set feedVersions; private final FeedSource feedSource; @@ -147,8 +147,8 @@ public class MergeFeedsJob extends MonitorableJob { */ final FeedVersion mergedVersion; public MergeStrategy mergeStrategy = MergeStrategy.DEFAULT; - private Set tripIdsToModifyForCurrentFeed = new HashSet<>(); - private Set tripIdsToSkipForCurrentFeed = new HashSet<>(); + private Set tripIdsToModifyForActiveFeed = new HashSet<>(); + private Set tripIdsToSkipForActiveFeed = new HashSet<>(); private Set serviceIdsToExtend = new HashSet<>(); private Set serviceIdsToCloneAndRename = new HashSet<>(); @@ -248,43 +248,45 @@ public void jobFinished() { mergeStrategy = getMergeStrategy(feedsToMerge); if (mergeStrategy == CHECK_STOP_TIMES) { Feed futureFeed = new Feed(DataManager.GTFS_DATA_SOURCE, feedsToMerge.get(0).version.namespace); - Feed pastFeed = new Feed(DataManager.GTFS_DATA_SOURCE, feedsToMerge.get(1).version.namespace); + Feed activeFeed = new Feed(DataManager.GTFS_DATA_SOURCE, feedsToMerge.get(1).version.namespace); for (String tripId : intersectingTripIds) { - // Fetch all ordered stop_times for each common trip_id, hash them, and compare the two sets for the - // future and current feed. If the stop_times are an exact match, include one instance of the trip - // (ignoring the other identical one). If they do not match, modify the current trip_id and include. + // Fetch all ordered stop_times for each common trip_id and compare the two sets for the + // future and active feed. If the stop_times are an exact match, include one instance of the trip + // (ignoring the other identical one). If they do not match, modify the active trip_id and include. List futureStopTimes = Lists.newArrayList(futureFeed.stopTimes.getOrdered(tripId)); - List pastStopTimes = Lists.newArrayList(pastFeed.stopTimes.getOrdered(tripId)); + List activeStopTimes = Lists.newArrayList(activeFeed.stopTimes.getOrdered(tripId)); String futureServiceId = futureFeed.trips.get(tripId).service_id; - String pastServiceId = pastFeed.trips.get(tripId).service_id; + String activeServiceId = activeFeed.trips.get(tripId).service_id; // FIXME: what if service_ids do not match! Perhaps the right approach would be to just return // FAIL_DUE_TO_MATCHING_TRIP_IDS in that case. It might be too complicated otherwise. - if (!stopTimesMatch(futureStopTimes, pastStopTimes)) { + if (!stopTimesMatch(futureStopTimes, activeStopTimes)) { // If stop_times or services do not match, the trip will be cloned. Also, track the service_id - // (it will need to be cloned and renamed for both current/past feeds). - tripIdsToModifyForCurrentFeed.add(tripId); + // (it will need to be cloned and renamed for both active feeds). + tripIdsToModifyForActiveFeed.add(tripId); serviceIdsToCloneAndRename.add(futureServiceId); } else { // If the trip's stop_times are an exact match, we can safely include just the - // future trip and exclude the current/past one. Also, track the service_id (it will need to be + // future trip and exclude the active one. Also, track the service_id (it will need to be // extended to the full time range). - tripIdsToSkipForCurrentFeed.add(tripId); + tripIdsToSkipForActiveFeed.add(tripId); serviceIdsToExtend.add(futureServiceId); } } - for (String tripId : tripsOnlyInCurrentFeed) { - String serviceId = pastFeed.trips.get(tripId).service_id; - // If a trip in the current feed contains a service_id that has been modified above, it needs to be - // cloned/renamed so as to avoid having it operate on an extended service. + for (String tripId : tripsOnlyInActiveFeed) { + String serviceId = activeFeed.trips.get(tripId).service_id; if (serviceIdsToExtend.contains(serviceId)) { + // If a trip only in the active feed references a service_id that is set to be extended, that + // service_id needs to be cloned and renamed to differentiate it from the same service_id in + // the future feed. (The trip in question will be linked to the cloned service_id.) serviceIdsToCloneAndRename.add(serviceId); } } for (String tripId : tripsOnlyInFutureFeed) { String serviceId = futureFeed.trips.get(tripId).service_id; - // If a trip in the future feed contains a service_id that has been modified above, it needs to be - // cloned/renamed so as to avoid having it operate on an extended service. if (serviceIdsToExtend.contains(serviceId)) { + // If a trip only in the future feed references a service_id that is set to be extended, that + // service_id needs to be cloned and renamed to differentiate it from the same service_id in + // the future feed. (The trip in question will be linked to the cloned service_id.) serviceIdsToCloneAndRename.add(serviceId); } } @@ -292,7 +294,7 @@ public void jobFinished() { } // Skip merging process altogether if the failing condition is met. if (MergeStrategy.FAIL_DUE_TO_MATCHING_TRIP_IDS.equals(mergeStrategy)) { - status.fail("Feed merge failed because the trip_ids are identical in the future and current feeds. A new service requires unique trip_ids for merging."); + status.fail("Feed merge failed because the trip_ids are identical in the future and active feeds. A new service requires unique trip_ids for merging."); return; } // Loop over GTFS tables and merge each feed one table at a time. @@ -342,12 +344,15 @@ public void jobFinished() { } } - private boolean stopTimesMatch(List futureStopTimes, List pastStopTimes) { - if (futureStopTimes.size() != pastStopTimes.size()) { + /** + * Checks whether the future and active stop_times for a particular trip_id are an exact match. + */ + private boolean stopTimesMatch(List futureStopTimes, List activeStopTimes) { + if (futureStopTimes.size() != activeStopTimes.size()) { return false; } - for (int i = 0; i < pastStopTimes.size(); i++) { - if (pastStopTimes.get(i).hashCode() != futureStopTimes.get(i).hashCode()) { + for (int i = 0; i < activeStopTimes.size(); i++) { + if (activeStopTimes.get(i).equals(futureStopTimes.get(i))) { return false; } } @@ -428,22 +433,22 @@ private int constructMergedTable(Table table, List feedsToMerge, try { // Get shared fields between all feeds being merged. This is used to filter the spec fields so that only // fields found in the collection of feeds are included in the merged table. - Set sharedFields = getSharedFields(feedsToMerge, table); - // Initialize future and past/current feed's first date to the first calendar date from validation result. + Set allFields = getAllFields(feedsToMerge, table); + // Initialize future and active feed's first date to the first calendar date from validation result. // This is equivalent to either the earliest date of service defined for a calendar_date record or the // earliest start_date value for a calendars.txt record. For MTC, however, they require that GTFS // providers use calendars.txt entries and prefer that this value (which is used to determine cutoff // dates for the active feed when merging with the future) be strictly assigned the earliest // calendar#start_date (unless that table for some reason does not exist). LocalDate futureFeedFirstDate = feedsToMerge.get(0).version.validationResult.firstCalendarDate; - LocalDate pastFeedFirstDate = feedsToMerge.get(1).version.validationResult.firstCalendarDate; + LocalDate activeFeedFirstDate = feedsToMerge.get(1).version.validationResult.firstCalendarDate; LocalDate futureFirstCalendarStartDate = LocalDate.MAX; // Iterate over each zip file. For service period merge, the first feed is the future GTFS. for (int feedIndex = 0; feedIndex < feedsToMerge.size(); feedIndex++) { - boolean handlingPastFeed = feedIndex > 0; + boolean handlingActiveFeed = feedIndex > 0; boolean handlingFutureFeed = feedIndex == 0; - if (EXTEND_FUTURE.equals(mergeStrategy) && handlingPastFeed) { - // No need to iterate over second (current) file if strategy is to simply extend the future GTFS + if (EXTEND_FUTURE.equals(mergeStrategy) && handlingActiveFeed) { + // No need to iterate over second (active) file if strategy is to simply extend the future GTFS // service to start earlier. continue; } @@ -460,7 +465,7 @@ private int constructMergedTable(Table table, List feedsToMerge, String idScope = getCleanName(feedSource.name) + version.version; CsvReader csvReader = table.getCsvReader(feed.zipFile, null); // If csv reader is null, the table was not found in the zip file. There is no need - // to handle merging this table for the current zip file. + // to handle merging this table for this zip file. if (csvReader == null) { LOG.warn("Table {} not found in the zip file for {}{}", table.name, feedSource.name, version.version); @@ -484,7 +489,7 @@ private int constructMergedTable(Table table, List feedsToMerge, // Iterate over rows in table, writing them to the out file. while (csvReader.readRecord()) { String keyValue = csvReader.get(keyFieldIndex); - if (handlingPastFeed && mergeType.equals(SERVICE_PERIOD)) { + if (handlingActiveFeed && mergeType.equals(SERVICE_PERIOD)) { // Always prefer the "future" file for the feed_info table, which means // we can skip any iterations following the first one. If merging the agency // table, we should only skip the following feeds if performing an MTC merge @@ -545,7 +550,7 @@ private int constructMergedTable(Table table, List feedsToMerge, List fieldsList = new ArrayList<>(Arrays.asList(fieldsFoundInZip)); fieldsList.add(Table.AGENCY.fields[0]); fieldsFoundInZip = fieldsList.toArray(fieldsFoundInZip); - sharedFields.add(Table.AGENCY.fields[0]); + allFields.add(Table.AGENCY.fields[0]); } fieldsFoundList = Arrays.asList(fieldsFoundInZip); } @@ -596,7 +601,7 @@ private int constructMergedTable(Table table, List feedsToMerge, if (handlingFutureFeed) stopCodeMissingFromFirstFeed = true; // However... if the second feed was missing stop_codes and the first feed was not, // fail the merge job. - if (handlingPastFeed && !stopCodeMissingFromFirstFeed) { + if (handlingActiveFeed && !stopCodeMissingFromFirstFeed) { mergeFeedsResult.failed = true; mergeFeedsResult.errorCount++; mergeFeedsResult.failureReasons.add( @@ -614,9 +619,9 @@ private int constructMergedTable(Table table, List feedsToMerge, } } } - // Filter the spec fields on the set of shared fields found in all feeds to be merged. + // Filter the spec fields on the set of fields found in all feeds to be merged. List sharedSpecFields = specFields.stream() - .filter(field -> containsField(sharedFields, field.name)) + .filter(field -> containsField(allFields, field.name)) .collect(Collectors.toList()); Field[] sharedSpecFieldsArray = sharedSpecFields.toArray(new Field[0]); boolean skipRecord = false; @@ -638,7 +643,7 @@ private int constructMergedTable(Table table, List feedsToMerge, String val = csvReader.get(index); // Default value to write is unchanged from value found in csv (i.e. val). Note: if looking to // modify the value that is written in the merged file, you must update valueToWrite (e.g., - // updating the current feed's end_date or accounting for cases where IDs conflict). + // updating this feed's end_date or accounting for cases where IDs conflict). String valueToWrite = val; // Handle filling in agency_id if missing when merging regional feeds. if (newAgencyId != null && field.name.equals("agency_id") && mergeType @@ -675,7 +680,7 @@ private int constructMergedTable(Table table, List feedsToMerge, Set idErrors; // If analyzing the second feed (non-future feed), the service_id always gets feed scoped. // See https://github.com/ibi-group/datatools-server/issues/244 - if (handlingPastFeed && field.name.equals("service_id")) { + if (handlingActiveFeed && field.name.equals("service_id")) { valueToWrite = String.join(":", idScope, val); mergeFeedsResult.remappedIds.put( getTableScopedValue(table, idScope, val), @@ -723,13 +728,15 @@ private int constructMergedTable(Table table, List feedsToMerge, // modified? (do we want the cloned record to have the original values?) if (index == startDateIndex) { if (EXTEND_FUTURE == mergeStrategy || - (CHECK_STOP_TIMES == mergeStrategy && + ( + CHECK_STOP_TIMES == mergeStrategy && // TODO: Need to ensure serviceIds are being extended. - serviceIdsToExtend.contains(keyValue)) + serviceIdsToExtend.contains(keyValue) + ) ) { - // Update start_date to extend service through the past/current feed's + // Update start_date to extend service through the active feed's // start date if the merge strategy dictates. - val = valueToWrite = pastFeedFirstDate.format(GTFS_DATE_FORMATTER); + val = valueToWrite = activeFeedFirstDate.format(GTFS_DATE_FORMATTER); } } } else { @@ -773,7 +780,7 @@ private int constructMergedTable(Table table, List feedsToMerge, // not before the first date of the future feed. int dateIndex = getFieldIndex(fieldsFoundInZip, "date"); LocalDate date = LocalDate.parse(csvReader.get(dateIndex), GTFS_DATE_FORMATTER); - if (handlingPastFeed) { + if (handlingActiveFeed) { if (!date.isBefore(futureFeedFirstDate)) { LOG.warn( "Skipping calendar_dates entry {} because it operates in the time span of future feed (i.e., after or on {}).", @@ -824,11 +831,11 @@ private int constructMergedTable(Table table, List feedsToMerge, case "trips": // trip_ids between active and future datasets must not match. The MergeStrategy // determines behavior when matching trip_ids (or service_ids) are found between - if (handlingPastFeed) { - // Handling past/current feed. - if (tripIdsToSkipForCurrentFeed.contains(keyValue)) { + if (handlingActiveFeed) { + // Handling active feed. + if (tripIdsToSkipForActiveFeed.contains(keyValue)) { skipRecord = true; - } else if (tripIdsToModifyForCurrentFeed.contains(keyValue)) { + } else if (tripIdsToModifyForActiveFeed.contains(keyValue)) { valueToWrite = String.join(":", idScope, val); // Update key value for subsequent ID conflict checks for this row. keyValue = valueToWrite; @@ -871,7 +878,7 @@ private int constructMergedTable(Table table, List feedsToMerge, Set primaryKeyErrors = referenceTracker .checkReferencesAndUniqueness(primaryKeyValue, lineNumber, field, val, table); - // Merging will be based on route_short_name/stop_code in the current and future datasets. All + // Merging will be based on route_short_name/stop_code in the active and future datasets. All // matching route_short_names/stop_codes between the datasets shall be considered same route/stop. Any // route_short_name/stop_code in active data not present in the future will be appended to the // future routes/stops file. @@ -1054,6 +1061,9 @@ private int constructMergedTable(Table table, List feedsToMerge, writer.write(rowValues); lineNumber++; mergedLineNumber++; + // If the current row is for a calendar service_id that is marked for cloning/renaming, clone the + // values, change the ID, extend the start/end dates to the feed's full range, and write the + // additional line to the file. if ((table.name.equals("calendar")) && serviceIdsToCloneAndRename.contains(rowValues[keyFieldIndex])) { // FIXME: Do we need to worry about calendar_dates? String[] clonedValues = rowValues.clone(); @@ -1086,8 +1096,12 @@ private int constructMergedTable(Table table, List feedsToMerge, return mergedLineNumber; } + /** + * Get the merge strategy to use for MTC service period merges by checking the active and future feeds for various + * combinations of matching trip and service IDs. + */ private static MergeStrategy getMergeStrategy(List feedsToMerge) throws IOException { - // Iterate over both feeds (future feed is first). + // Iterate over both feeds to collect all trip and service IDs. for (int i = 0; i < feedsToMerge.size(); i++) { FeedToMerge feed = feedsToMerge.get(i); Set
tablesToCheck = Sets.newHashSet(Table.TRIPS, Table.CALENDAR, Table.CALENDAR_DATES); @@ -1095,22 +1109,21 @@ private static MergeStrategy getMergeStrategy(List feedsToMerge) th feed.idsForTable.get(table).addAll(getIdsForTable(feed.zipFile, table)); } } - // We are on the last (second) feed to merge. Check for conflicts with future feed. FeedToMerge futureFeed = feedsToMerge.get(0); - FeedToMerge currentFeed = feedsToMerge.get(1); - Set pastTripIds = currentFeed.idsForTable.get(Table.TRIPS); + FeedToMerge activeFeed = feedsToMerge.get(1); + Set activeTripIds = activeFeed.idsForTable.get(Table.TRIPS); Set futureTripIds = futureFeed.idsForTable.get(Table.TRIPS); - Set pastServiceIds = new HashSet<>(); - pastServiceIds.addAll(currentFeed.idsForTable.get(Table.CALENDAR)); - pastServiceIds.addAll(currentFeed.idsForTable.get(Table.CALENDAR_DATES)); + Set activeServiceIds = new HashSet<>(); + activeServiceIds.addAll(activeFeed.idsForTable.get(Table.CALENDAR)); + activeServiceIds.addAll(activeFeed.idsForTable.get(Table.CALENDAR_DATES)); Set futureServiceIds = new HashSet<>(); futureServiceIds.addAll(futureFeed.idsForTable.get(Table.CALENDAR)); futureServiceIds.addAll(futureFeed.idsForTable.get(Table.CALENDAR_DATES)); - boolean serviceIdsMatch = pastServiceIds.equals(futureServiceIds); - intersectingTripIds = Sets.intersection(pastTripIds, futureTripIds); - tripsOnlyInCurrentFeed = Sets.difference(pastTripIds, futureTripIds); - tripsOnlyInFutureFeed = Sets.difference(futureTripIds, pastTripIds); - boolean tripIdsMatch = pastTripIds.equals(futureTripIds); + boolean serviceIdsMatch = activeServiceIds.equals(futureServiceIds); + intersectingTripIds = Sets.intersection(activeTripIds, futureTripIds); + tripsOnlyInActiveFeed = Sets.difference(activeTripIds, futureTripIds); + tripsOnlyInFutureFeed = Sets.difference(futureTripIds, activeTripIds); + boolean tripIdsMatch = activeTripIds.equals(futureTripIds); if (serviceIdsMatch && tripIdsMatch) { // Effectively this exact match condition means that the future feed will be used as is // (including stops, routes, etc.), the only modification being service date ranges. diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsResult.java b/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsResult.java index 43cea68ae..aeae3381a 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsResult.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsResult.java @@ -1,7 +1,5 @@ package com.conveyal.datatools.manager.jobs; -import org.eclipse.jetty.http.HttpFields; - import java.io.Serializable; import java.util.Date; import java.util.HashMap; diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/MergeStrategy.java b/src/main/java/com/conveyal/datatools/manager/jobs/MergeStrategy.java index 185de9c86..c4dbbc6d1 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/MergeStrategy.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/MergeStrategy.java @@ -32,10 +32,10 @@ public enum MergeStrategy { * Note: The merge process shall validate records in stop_times.txt file for same trip signature (same set of * stops with same sequence). Trips with matching stop_times will be included as is (but not duplicated of course). * Trips that do not match on stop_times will be handled with the below approaches. - * Note: Same service IDs shall be used (but extended to account for the full range of dates from past to future). - * - *trip_id in past feed*: A new service shall be created starting from the merge date and expiring at the end + * Note: Same service IDs shall be used (but extended to account for the full range of dates from active to future). + * - *trip_id in active feed*: A new service shall be created starting from the merge date and expiring at the end * of active service period. - * Note: a new service_id will be generated for these current/past trips in the merged feed (rather than using the + * Note: a new service_id will be generated for these active trips in the merged feed (rather than using the * service_id with extended range). * - *trip_id in future feed*: A new service shall be created for these trips with service period defined in future * feed diff --git a/src/main/java/com/conveyal/datatools/manager/utils/MergeFeedUtils.java b/src/main/java/com/conveyal/datatools/manager/utils/MergeFeedUtils.java index 638025c19..84047627e 100644 --- a/src/main/java/com/conveyal/datatools/manager/utils/MergeFeedUtils.java +++ b/src/main/java/com/conveyal/datatools/manager/utils/MergeFeedUtils.java @@ -47,6 +47,10 @@ public static Set getIdsForTable(ZipFile zipFile, Table table) throws IO return ids; } + /** + * Construct stop_code failure message for {@link com.conveyal.datatools.manager.jobs.MergeFeedsJob} in the case of + * incomplete stop_code values for all records. + */ public static String stopCodeFailureMessage(int stopsMissingStopCodeCount, int stopsCount, int specialStopsCount) { return String.format( "If stop_code is provided for some stops (for those with location_type = " + @@ -63,26 +67,36 @@ public static String stopCodeFailureMessage(int stopsMissingStopCodeCount, int s /** * Collect zipFiles for each feed version before merging tables. * Note: feed versions are sorted by first calendar date so that future dataset is iterated over first. This is - * required for the MTC merge strategy which prefers entities from the future dataset over past entities. + * required for the MTC merge strategy which prefers entities from the future dataset over active feed entities. */ public static List collectAndSortFeeds(Set feedVersions) { - return feedVersions.stream().map(version -> { - try { - return new FeedToMerge(version); - } catch (Exception e) { - LOG.error("Could not create zip file for version: {}", version.version); - return null; - } - }).filter(Objects::nonNull).filter(entry -> entry.version.validationResult != null - && entry.version.validationResult.firstCalendarDate != null) + return feedVersions.stream() + .map(version -> { + try { + return new FeedToMerge(version); + } catch (Exception e) { + LOG.error("Could not create zip file for version: {}", version.version); + return null; + } + }) + // Filter out any feeds that do not have zip files (see above try/catch) and feeds that were never fully + // validated (which suggests that they would break things during validation). + .filter(Objects::nonNull) + .filter( + entry -> entry.version.validationResult != null + && entry.version.validationResult.firstCalendarDate != null + ) // MTC-specific sort mentioned in above comment. // TODO: If another merge strategy requires a different sort order, a merge type check should be added. - .sorted(Comparator.comparing(entry -> entry.version.validationResult.firstCalendarDate, - Comparator.reverseOrder())).collect(Collectors.toList()); + .sorted( + Comparator.comparing( + entry -> entry.version.validationResult.firstCalendarDate, + Comparator.reverseOrder()) + ).collect(Collectors.toList()); } - /** Get the set of shared fields for all feeds being merged for a specific table. */ - public static Set getSharedFields(List feedsToMerge, Table table) throws IOException { + /** Get all fields found in the feeds being merged for a specific table. */ + public static Set getAllFields(List feedsToMerge, Table table) throws IOException { Set sharedFields = new HashSet<>(); // First, iterate over each feed to collect the shared fields that need to be output in the merged table. for (FeedToMerge feed : feedsToMerge) { @@ -121,24 +135,4 @@ public static String getTableScopedValue(Table table, String prefix, String id) prefix, id); } - - public static boolean preferFutureTable(Table table, MergeStrategy mergeStrategy) { - Set tablesToNotPreferFuture = Sets.newHashSet( - Table.CALENDAR.name, - Table.CALENDAR_DATES.name - ); - if (tablesToNotPreferFuture.contains(table.name)) return false; - // Always prefer the "future" file for the feed_info table, which means - // we can skip any iterations following the first one. If merging the agency - // table, we should only skip the following feeds if performing an MTC merge - // because that logic assumes the two feeds share the same agency (or - // agencies). NOTE: feed_info file is skipped by default (outside of this - // method) for a regional merge), which is why this block is exclusively - // for an MTC merge. Also, this statement may print multiple log - // statements, but it is deliberately nested in the csv while block in - // order to detect agency_id mismatches and fail the merge if found. - if (table.name.equals("feed_info")) return true; - // If merge strategy is to extend the future calendar files to the past, all other files should prefer future. - return MergeStrategy.EXTEND_FUTURE.equals(mergeStrategy); - } } diff --git a/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java b/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java index a7a74eff2..f5d6ca0e0 100644 --- a/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java +++ b/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java @@ -47,9 +47,13 @@ public class MergeFeedsJobTest extends UnitTest { private static FeedVersion bothCalendarFilesVersion3; private static FeedVersion onlyCalendarVersion; private static FeedVersion onlyCalendarDatesVersion; + /** The base feed for testing the MTC merge strategies. */ private static FeedVersion fakeTransitBase; + /** The base feed but with calendar start/end dates that have been transposed to the future. */ private static FeedVersion fakeTransitFuture; + /** The base feed but with differing service_ids. */ private static FeedVersion fakeTransitModService; + /** The base feed (transposed to the future dates) but with differing trip_ids. */ private static FeedVersion fakeTransitModTrips; /** @@ -115,11 +119,10 @@ public static void setUp() throws IOException { } /** - * Prepare and start a testing-specific web server + * Delete project on tear down (feed sources/versions will also be deleted). */ @AfterAll public static void tearDown() throws IOException { - // Delete project (feed sources/versions will also be deleted). if (project != null) project.delete(); } @@ -356,23 +359,23 @@ public void mergeMTCShouldHandleCheckStopTimesStrategy() throws SQLException { String mergedNamespace = mergeFeedsJob.mergedVersion.namespace; // - calendar table - // expect a total of 6 records in calendar table: + // expect a total of 4 records in calendar table: // - 2 original (common_id start date extended) - // - 2 cloned for past/current feed + // - 2 cloned for active feed assertThatSqlCountQueryYieldsExpectedCount( String.format("SELECT count(*) FROM %s.calendar", mergedNamespace), 4 ); - // expect that both records in calendar table have the correct start_date - assertThatSqlCountQueryYieldsExpectedCount( - String.format("SELECT count(*) FROM %s.calendar where start_date = '20170918' and end_date = '20170925'", mergedNamespace), - 1 - ); - // expect 3 trips (one trip is omitted from the past/current feed because it is an exact match of a future trip - assertThatSqlCountQueryYieldsExpectedCount( - String.format("SELECT count(*) FROM %s.trips", mergedNamespace), - 3 - ); +// // expect that both records in calendar table have the correct start_date +// assertThatSqlCountQueryYieldsExpectedCount( +// String.format("SELECT count(*) FROM %s.calendar where start_date = '20170918' and end_date = '20170925'", mergedNamespace), +// 1 +// ); +// // expect 3 trips (one trip is omitted from the active feed because it is an exact match of a future trip +// assertThatSqlCountQueryYieldsExpectedCount( +// String.format("SELECT count(*) FROM %s.trips", mergedNamespace), +// 3 +// ); } /** @@ -390,7 +393,7 @@ public void mergeMTCShouldHandleDefaultStrategy() throws SQLException { // Result should fail. assertFalse( mergeFeedsJob.mergeFeedsResult.failed, - "Merge feeds job should utilize EXTEND_FEED strategy." + "Merge feeds job should utilize DEFAULT strategy." ); assertEquals( MergeStrategy.DEFAULT, @@ -456,7 +459,7 @@ public void mergeMTCShouldHandleDefaultStrategy() throws SQLException { // mergeFeedsJob.mergedVersion.feedLoadResult.shapes.rowCount, // "Merged feed shapes count should equal expected value." // ); -// // Expect that two calendar dates are excluded from the past feed (because they occur after the first date of +// // Expect that two calendar dates are excluded from the active feed (because they occur after the first date of // // the future feed) . // int expectedCalendarDatesCount = bartVersion1.feedLoadResult.calendarDates.rowCount + bartVersion2SameTrips.feedLoadResult.calendarDates.rowCount - 2; // assertEquals( @@ -726,7 +729,7 @@ public void canMergeFeedsWithMTCForServiceIds3 () throws SQLException { 1 ); // Modified cal_to_remove should still exist in calendar_dates. It is modified even though it does not exist in - // the future feed due to the MTC requirement to update all service_ids in the past feed. + // the future feed due to the MTC requirement to update all service_ids in the active feed. // See https://github.com/ibi-group/datatools-server/issues/244 assertThatSqlCountQueryYieldsExpectedCount( String.format( From 53169121a09277c91b13d7800d90bad9e8ef72e2 Mon Sep 17 00:00:00 2001 From: Landon Reed Date: Mon, 3 May 2021 17:27:03 -0400 Subject: [PATCH 006/122] build(deps): update gtfs-lib dep --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 07d0b5656..759b72b44 100644 --- a/pom.xml +++ b/pom.xml @@ -263,7 +263,7 @@ com.github.conveyal gtfs-lib - add-stoptime-hash-SNAPSHOT + add-stoptime-hash-v3.3.0-g736541a-721 From 35025b33295d841d816e8f818023b2a9d4928bc1 Mon Sep 17 00:00:00 2001 From: Landon Reed Date: Mon, 10 May 2021 09:44:38 -0400 Subject: [PATCH 007/122] refactor(merge-feeds): refactor fail method, fix tests --- .../datatools/manager/jobs/MergeFeedsJob.java | 52 +++--- .../manager/jobs/MergeFeedsResult.java | 2 +- .../manager/jobs/MergeFeedsJobTest.java | 164 +++++++++--------- .../gtfs/merge-data-base/stop_times.txt | 4 +- .../datatools/gtfs/merge-data-base/trips.txt | 3 +- .../merge-data-future-unique-ids/agency.txt | 2 + .../merge-data-future-unique-ids/calendar.txt | 3 + .../feed_info.txt | 2 + .../merge-data-future-unique-ids/routes.txt | 3 + .../stop_attributes.txt | 3 + .../stop_times.txt | 5 + .../merge-data-future-unique-ids/stops.txt | 6 + .../merge-data-future-unique-ids/trips.txt | 3 + .../gtfs/merge-data-mod-services/calendar.txt | 1 + 14 files changed, 141 insertions(+), 112 deletions(-) create mode 100755 src/test/resources/com/conveyal/datatools/gtfs/merge-data-future-unique-ids/agency.txt create mode 100755 src/test/resources/com/conveyal/datatools/gtfs/merge-data-future-unique-ids/calendar.txt create mode 100644 src/test/resources/com/conveyal/datatools/gtfs/merge-data-future-unique-ids/feed_info.txt create mode 100755 src/test/resources/com/conveyal/datatools/gtfs/merge-data-future-unique-ids/routes.txt create mode 100644 src/test/resources/com/conveyal/datatools/gtfs/merge-data-future-unique-ids/stop_attributes.txt create mode 100755 src/test/resources/com/conveyal/datatools/gtfs/merge-data-future-unique-ids/stop_times.txt create mode 100755 src/test/resources/com/conveyal/datatools/gtfs/merge-data-future-unique-ids/stops.txt create mode 100755 src/test/resources/com/conveyal/datatools/gtfs/merge-data-future-unique-ids/trips.txt diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java b/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java index 19f0ca25e..d6a4421f7 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java @@ -294,7 +294,7 @@ public void jobFinished() { } // Skip merging process altogether if the failing condition is met. if (MergeStrategy.FAIL_DUE_TO_MATCHING_TRIP_IDS.equals(mergeStrategy)) { - status.fail("Feed merge failed because the trip_ids are identical in the future and active feeds. A new service requires unique trip_ids for merging."); + failMergeJob("Feed merge failed because the trip_ids are identical in the future and active feeds. A new service requires unique trip_ids for merging."); return; } // Loop over GTFS tables and merge each feed one table at a time. @@ -326,10 +326,7 @@ public void jobFinished() { } // Close output stream for zip file. out.close(); - if (mergeFeedsResult.failed) { - // Fail job if the merge result indicates something went wrong. - status.fail("Merging feed versions failed."); - } else { + if (!mergeFeedsResult.failed) { // Store feed locally and (if applicable) upload regional feed to S3. storeMergedFeed(); status.completeSuccessfully("Merged feed created successfully."); @@ -344,6 +341,19 @@ public void jobFinished() { } } + /** + * Handle updating {@link MergeFeedsResult} and the overall job status when a failure condition is triggered while + * merging feeds. + */ + private void failMergeJob(String failureMessage) { + LOG.error(failureMessage); + mergeFeedsResult.failed = true; + mergeFeedsResult.errorCount++; + mergeFeedsResult.failureReasons.add(failureMessage); + // Use generic message for overall job status. + status.fail("Merging feed versions failed."); + } + /** * Checks whether the future and active stop_times for a particular trip_id are an exact match. */ @@ -352,7 +362,7 @@ private boolean stopTimesMatch(List futureStopTimes, List ac return false; } for (int i = 0; i < activeStopTimes.size(); i++) { - if (activeStopTimes.get(i).equals(futureStopTimes.get(i))) { + if (activeStopTimes.get(i).hashCode() == futureStopTimes.get(i).hashCode()) { return false; } } @@ -512,15 +522,12 @@ private int constructMergedTable(Table table, List feedsToMerge, .filter(transitId -> transitId.startsWith("agency_id")) .findAny() .orElse(null); - String message = String.format( + failMergeJob(String.format( "MTC merge detected mismatching agency_id values between two " + "feeds (%s and %s). Failing merge operation.", agencyId, otherAgencyId - ); - LOG.error(message); - mergeFeedsResult.failed = true; - mergeFeedsResult.failureReasons.add(message); + )); return -1; } LOG.warn("Skipping {} file for feed {}/{} (future file preferred)", @@ -602,17 +609,13 @@ private int constructMergedTable(Table table, List feedsToMerge, // However... if the second feed was missing stop_codes and the first feed was not, // fail the merge job. if (handlingActiveFeed && !stopCodeMissingFromFirstFeed) { - mergeFeedsResult.failed = true; - mergeFeedsResult.errorCount++; - mergeFeedsResult.failureReasons.add( + failMergeJob( stopCodeFailureMessage(stopsMissingStopCodeCount, stopsCount, specialStopsCount) ); } } else if (stopsMissingStopCodeCount > 0) { // If some, but not all, stops are missing stop_code, the merge feeds job must fail. - mergeFeedsResult.failed = true; - mergeFeedsResult.errorCount++; - mergeFeedsResult.failureReasons.add( + failMergeJob( stopCodeFailureMessage(stopsMissingStopCodeCount, stopsCount, specialStopsCount) ); } @@ -646,17 +649,14 @@ private int constructMergedTable(Table table, List feedsToMerge, // updating this feed's end_date or accounting for cases where IDs conflict). String valueToWrite = val; // Handle filling in agency_id if missing when merging regional feeds. - if (newAgencyId != null && field.name.equals("agency_id") && mergeType - .equals(REGIONAL)) { + if (newAgencyId != null && field.name.equals("agency_id") && mergeType.equals(REGIONAL)) { if (val.equals("") && table.name.equals("agency") && lineNumber > 0) { // If there is no agency_id value for a second (or greater) agency // record, fail the merge feed job. - String message = String.format( + failMergeJob(String.format( "Feed %s has multiple agency records but no agency_id values.", - feed.version.id); - mergeFeedsResult.failed = true; - mergeFeedsResult.failureReasons.add(message); - LOG.error(message); + feed.version.id + )); return -1; } LOG.info("Updating {}#agency_id to (auto-generated) {} for ID {}", @@ -664,8 +664,7 @@ private int constructMergedTable(Table table, List feedsToMerge, val = newAgencyId; } // Determine if field is a GTFS identifier. - boolean isKeyField = - field.isForeignReference() || keyField.equals(field.name); + boolean isKeyField = field.isForeignReference() || keyField.equals(field.name); if (this.mergeType.equals(REGIONAL) && isKeyField && !val.isEmpty()) { // For regional merge, if field is a GTFS identifier (e.g., route_id, // stop_id, etc.), add scoped prefix. @@ -1138,6 +1137,7 @@ private static MergeStrategy getMergeStrategy(List feedsToMerge) th // Do not permit merge to continue. return MergeStrategy.FAIL_DUE_TO_MATCHING_TRIP_IDS; } + // If neither the trips or services are exact matches, use the default merge strategy. return MergeStrategy.DEFAULT; } } diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsResult.java b/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsResult.java index aeae3381a..0f847874b 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsResult.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsResult.java @@ -15,7 +15,6 @@ public class MergeFeedsResult implements Serializable { /** Number of feeds merged */ public int feedCount; - public int errorCount; /** Type of merge operation performed */ public MergeFeedsType type; /** Contains a set of strings for which there were error-causing duplicate values */ @@ -35,6 +34,7 @@ public class MergeFeedsResult implements Serializable { public int recordsSkipCount; public Date startTime; public boolean failed; + public int errorCount; /** Set of reasons explaining why merge operation failed */ public Set failureReasons = new HashSet<>(); public Set tripIdsToCheck = new HashSet<>(); diff --git a/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java b/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java index f5d6ca0e0..e445d648f 100644 --- a/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java +++ b/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java @@ -51,10 +51,14 @@ public class MergeFeedsJobTest extends UnitTest { private static FeedVersion fakeTransitBase; /** The base feed but with calendar start/end dates that have been transposed to the future. */ private static FeedVersion fakeTransitFuture; + /** The base feed with start/end dates that have been transposed to the future AND unique trip and service IDs. */ + private static FeedVersion fakeTransitFutureUnique; /** The base feed but with differing service_ids. */ private static FeedVersion fakeTransitModService; /** The base feed (transposed to the future dates) but with differing trip_ids. */ private static FeedVersion fakeTransitModTrips; + private static FeedSource napa; + private static FeedSource caltrain; /** * Prepare and start a testing-specific web server @@ -76,14 +80,12 @@ public static void setUp() throws IOException { bartVersion2SameTrips = createFeedVersionFromGtfsZip(bart, "bart_new.zip"); // Caltrain - FeedSource caltrain = new FeedSource("Caltrain", project.id, MANUALLY_UPLOADED); + caltrain = new FeedSource("Caltrain", project.id, MANUALLY_UPLOADED); Persistence.feedSources.create(caltrain); - calTrainVersion = createFeedVersionFromGtfsZip(caltrain, "caltrain_gtfs.zip"); // Napa - FeedSource napa = new FeedSource("Napa", project.id, MANUALLY_UPLOADED); + napa = new FeedSource("Napa", project.id, MANUALLY_UPLOADED); Persistence.feedSources.create(napa); - napaVersion = createFeedVersionFromGtfsZip(napa, "napa-no-agency-id.zip"); // Fake agencies (for testing calendar service_id merges with MTC strategy). FeedSource fakeAgency = new FeedSource("Fake Agency", project.id, MANUALLY_UPLOADED); @@ -114,6 +116,7 @@ public static void setUp() throws IOException { Persistence.feedSources.create(fakeTransit); fakeTransitBase = createFeedVersion(fakeTransit, zipFolderFiles("merge-data-base")); fakeTransitFuture = createFeedVersion(fakeTransit, zipFolderFiles("merge-data-future")); + fakeTransitFutureUnique = createFeedVersion(fakeTransit, zipFolderFiles("merge-data-future-unique-ids")); fakeTransitModService = createFeedVersion(fakeTransit, zipFolderFiles("merge-data-mod-services")); fakeTransitModTrips = createFeedVersion(fakeTransit, zipFolderFiles("merge-data-mod-trips")); } @@ -133,6 +136,8 @@ public static void tearDown() throws IOException { public void canMergeRegional() throws SQLException { // Set up list of feed versions to merge. Set versions = new HashSet<>(); + napaVersion = createFeedVersionFromGtfsZip(napa, "napa-no-agency-id.zip"); + calTrainVersion = createFeedVersionFromGtfsZip(caltrain, "caltrain_gtfs.zip"); versions.add(bartVersion1); versions.add(calTrainVersion); versions.add(napaVersion); @@ -286,15 +291,16 @@ public void mergeMTCShouldFailOnDuplicateTripsButMismatchedServices() { MergeFeedsJob mergeFeedsJob = new MergeFeedsJob(user, versions, "merged_output", MergeFeedsType.SERVICE_PERIOD); // Run the job in this thread (we're not concerned about concurrency here). mergeFeedsJob.run(); + // Check that correct strategy was used. + assertEquals( + MergeStrategy.FAIL_DUE_TO_MATCHING_TRIP_IDS, + mergeFeedsJob.mergeStrategy + ); // Result should fail. assertTrue( mergeFeedsJob.mergeFeedsResult.failed, "Merge feeds job should fail if feeds have exactly matching trips but mismatched services." ); - assertEquals( - MergeStrategy.FAIL_DUE_TO_MATCHING_TRIP_IDS, - mergeFeedsJob.mergeStrategy - ); } /** @@ -346,15 +352,16 @@ public void mergeMTCShouldHandleCheckStopTimesStrategy() throws SQLException { MergeFeedsJob mergeFeedsJob = new MergeFeedsJob(user, versions, "merged_output", MergeFeedsType.SERVICE_PERIOD); // Run the job in this thread (we're not concerned about concurrency here). mergeFeedsJob.run(); - // Result should fail. - assertFalse( - mergeFeedsJob.mergeFeedsResult.failed, - "Merge feeds job should succeed with CHECK_STOP_TIMES strategy." - ); + // Check that correct strategy was used. assertEquals( MergeStrategy.CHECK_STOP_TIMES, mergeFeedsJob.mergeStrategy ); + // Result should succeed. + assertFalse( + mergeFeedsJob.mergeFeedsResult.failed, + "Merge feeds job should succeed with CHECK_STOP_TIMES strategy." + ); // assert service_ids start_dates have been extended to the start_date of the base feed. String mergedNamespace = mergeFeedsJob.mergedVersion.namespace; @@ -379,26 +386,27 @@ public void mergeMTCShouldHandleCheckStopTimesStrategy() throws SQLException { } /** - * Ensures that an MTC merge of feeds with exact matches of service_ids and trip_ids will utilize the + * Ensures that an MTC merge of feeds with non-matching service_ids and trip_ids will utilize the * {@link MergeStrategy#DEFAULT} strategy correctly. */ @Test public void mergeMTCShouldHandleDefaultStrategy() throws SQLException { Set versions = new HashSet<>(); versions.add(fakeTransitBase); - versions.add(fakeTransitFuture); + versions.add(fakeTransitFutureUnique); MergeFeedsJob mergeFeedsJob = new MergeFeedsJob(user, versions, "merged_output", MergeFeedsType.SERVICE_PERIOD); // Run the job in this thread (we're not concerned about concurrency here). mergeFeedsJob.run(); - // Result should fail. - assertFalse( - mergeFeedsJob.mergeFeedsResult.failed, - "Merge feeds job should utilize DEFAULT strategy." - ); + // Check that correct strategy was used. assertEquals( MergeStrategy.DEFAULT, mergeFeedsJob.mergeStrategy ); + // Result should succeed. + assertFalse( + mergeFeedsJob.mergeFeedsResult.failed, + "Merge feeds job should utilize DEFAULT strategy." + ); // assert service_ids start_dates have been extended to the start_date of the base feed. String mergedNamespace = mergeFeedsJob.mergedVersion.namespace; @@ -415,66 +423,62 @@ public void mergeMTCShouldHandleDefaultStrategy() throws SQLException { ); } -// /** -// * Tests that the MTC merge strategy will successfully merge BART feeds. Note: this test turns off -// * {@link MergeFeedsJob#failOnDuplicateTripId} in order to force the merge to succeed even though there are duplicate -// * trips contained within. -// */ -// @Test -// public void canMergeBARTFeeds() throws SQLException { -// Set versions = new HashSet<>(); -// versions.add(bartVersion1); -// versions.add(bartVersion2SameTrips); -// MergeFeedsJob mergeFeedsJob = new MergeFeedsJob(user, versions, "merged_output", MergeFeedsType.SERVICE_PERIOD); -// // This time, turn off the failOnDuplicateTripId flag. -// mergeFeedsJob.failOnDuplicateTripId = false; -// // Result should succeed this time. -// mergeFeedsJob.run(); -// assertFeedMergeSucceeded(mergeFeedsJob); -// // Check GTFS+ line numbers. -// assertEquals( -// 2, // Magic number represents expected number of lines after merge. -// mergeFeedsJob.mergeFeedsResult.linesPerTable.get("directions").intValue(), -// "Merged directions count should equal expected value." -// ); -// assertEquals( -// 2, // Magic number represents the number of stop_attributes in the merged BART feed. -// mergeFeedsJob.mergeFeedsResult.linesPerTable.get("stop_attributes").intValue(), -// "Merged feed stop_attributes count should equal expected value." -// ); -// // Check GTFS file line numbers. -// assertEquals( -// 4552, // Magic number represents the number of trips in the merged BART feed. -// mergeFeedsJob.mergedVersion.feedLoadResult.trips.rowCount, -// "Merged feed trip count should equal expected value." -// ); -// assertEquals( -// 9, // Magic number represents the number of routes in the merged BART feed. -// mergeFeedsJob.mergedVersion.feedLoadResult.routes.rowCount, -// "Merged feed route count should equal expected value." -// ); -// assertEquals( -// // During merge, if identical shape_id is found in both feeds, active feed shape_id should be feed-scoped. -// bartVersion1.feedLoadResult.shapes.rowCount + bartVersion2SameTrips.feedLoadResult.shapes.rowCount, -// mergeFeedsJob.mergedVersion.feedLoadResult.shapes.rowCount, -// "Merged feed shapes count should equal expected value." -// ); -// // Expect that two calendar dates are excluded from the active feed (because they occur after the first date of -// // the future feed) . -// int expectedCalendarDatesCount = bartVersion1.feedLoadResult.calendarDates.rowCount + bartVersion2SameTrips.feedLoadResult.calendarDates.rowCount - 2; -// assertEquals( -// // During merge, if identical shape_id is found in both feeds, active feed shape_id should be feed-scoped. -// expectedCalendarDatesCount, -// mergeFeedsJob.mergedVersion.feedLoadResult.calendarDates.rowCount, -// "Merged feed calendar_dates count should equal expected value." -// ); -// // Ensure there are no referential integrity errors or duplicate ID errors. -// assertThatFeedHasNoErrorsOfType( -// mergeFeedsJob.mergedVersion.namespace, -// NewGTFSErrorType.REFERENTIAL_INTEGRITY.toString(), -// NewGTFSErrorType.DUPLICATE_ID.toString() -// ); -// } + /** + * Tests that the MTC merge strategy will successfully merge BART feeds. + */ + @Test + public void canMergeBARTFeeds() throws SQLException { + Set versions = new HashSet<>(); + versions.add(bartVersion1); + versions.add(bartVersion2SameTrips); + MergeFeedsJob mergeFeedsJob = new MergeFeedsJob(user, versions, "merged_output", MergeFeedsType.SERVICE_PERIOD); + // Result should succeed this time. + mergeFeedsJob.run(); + assertFeedMergeSucceeded(mergeFeedsJob); + // Check GTFS+ line numbers. + assertEquals( + 2, // Magic number represents expected number of lines after merge. + mergeFeedsJob.mergeFeedsResult.linesPerTable.get("directions").intValue(), + "Merged directions count should equal expected value." + ); + assertEquals( + 2, // Magic number represents the number of stop_attributes in the merged BART feed. + mergeFeedsJob.mergeFeedsResult.linesPerTable.get("stop_attributes").intValue(), + "Merged feed stop_attributes count should equal expected value." + ); + // Check GTFS file line numbers. + assertEquals( + 5149, // Magic number represents the number of trips in the merged BART feed. + mergeFeedsJob.mergedVersion.feedLoadResult.trips.rowCount, + "Merged feed trip count should equal expected value." + ); + assertEquals( + 9, // Magic number represents the number of routes in the merged BART feed. + mergeFeedsJob.mergedVersion.feedLoadResult.routes.rowCount, + "Merged feed route count should equal expected value." + ); + assertEquals( + // During merge, if identical shape_id is found in both feeds, active feed shape_id should be feed-scoped. + bartVersion1.feedLoadResult.shapes.rowCount + bartVersion2SameTrips.feedLoadResult.shapes.rowCount, + mergeFeedsJob.mergedVersion.feedLoadResult.shapes.rowCount, + "Merged feed shapes count should equal expected value." + ); + // Expect that two calendar dates are excluded from the active feed (because they occur after the first date of + // the future feed) . + int expectedCalendarDatesCount = bartVersion1.feedLoadResult.calendarDates.rowCount + bartVersion2SameTrips.feedLoadResult.calendarDates.rowCount - 2; + assertEquals( + // During merge, if identical shape_id is found in both feeds, active feed shape_id should be feed-scoped. + expectedCalendarDatesCount, + mergeFeedsJob.mergedVersion.feedLoadResult.calendarDates.rowCount, + "Merged feed calendar_dates count should equal expected value." + ); + // Ensure there are no referential integrity errors or duplicate ID errors. + assertThatFeedHasNoErrorsOfType( + mergeFeedsJob.mergedVersion.namespace, + NewGTFSErrorType.REFERENTIAL_INTEGRITY.toString(), + NewGTFSErrorType.DUPLICATE_ID.toString() + ); + } /** * Tests whether a MTC feed merge of two feed versions correctly feed scopes the service_id's of the feed that is diff --git a/src/test/resources/com/conveyal/datatools/gtfs/merge-data-base/stop_times.txt b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-base/stop_times.txt index 081b2fad0..646706c5c 100755 --- a/src/test/resources/com/conveyal/datatools/gtfs/merge-data-base/stop_times.txt +++ b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-base/stop_times.txt @@ -2,6 +2,4 @@ trip_id,arrival_time,departure_time,stop_id,stop_sequence,stop_headsign,pickup_t only-calendar-trip1,07:00:00,07:00:00,4u6g,1,,0,0,0.0000000, only-calendar-trip1,07:01:00,07:01:00,johv,2,,0,0,341.4491961, only-calendar-trip2,07:00:00,07:00:00,johv,1,,0,0,0.0000000, -only-calendar-trip2,07:01:00,07:01:00,4u6g,2,,0,0,341.4491961, -trip3,07:00:00,07:00:00,4u6g,1,,0,0,0.0000000, -trip3,07:01:00,07:01:00,johv,2,,0,0,341.4491961, \ No newline at end of file +only-calendar-trip2,07:01:00,07:01:00,4u6g,2,,0,0,341.4491961, \ No newline at end of file diff --git a/src/test/resources/com/conveyal/datatools/gtfs/merge-data-base/trips.txt b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-base/trips.txt index 1a8e2c972..387b076cd 100755 --- a/src/test/resources/com/conveyal/datatools/gtfs/merge-data-base/trips.txt +++ b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-base/trips.txt @@ -1,4 +1,3 @@ route_id,trip_id,trip_headsign,trip_short_name,direction_id,block_id,shape_id,bikes_allowed,wheelchair_accessible,service_id 1,only-calendar-trip1,,,0,,,0,0,common_id -2,only-calendar-trip2,,,0,,,0,0,common_id -2,trip3,,,0,,,0,0,common_id \ No newline at end of file +2,only-calendar-trip2,,,0,,,0,0,common_id \ No newline at end of file diff --git a/src/test/resources/com/conveyal/datatools/gtfs/merge-data-future-unique-ids/agency.txt b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-future-unique-ids/agency.txt new file mode 100755 index 000000000..a916ce91b --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-future-unique-ids/agency.txt @@ -0,0 +1,2 @@ +agency_id,agency_name,agency_url,agency_lang,agency_phone,agency_email,agency_timezone,agency_fare_url,agency_branding_url +1,Fake Transit,,,,,America/Los_Angeles,, diff --git a/src/test/resources/com/conveyal/datatools/gtfs/merge-data-future-unique-ids/calendar.txt b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-future-unique-ids/calendar.txt new file mode 100755 index 000000000..5d9454336 --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-future-unique-ids/calendar.txt @@ -0,0 +1,3 @@ +service_id,monday,tuesday,wednesday,thursday,friday,saturday,sunday,start_date,end_date +future_id,1,1,1,1,1,1,1,20170923,20170925 +future_id_other,1,1,1,1,1,1,1,20170924,20170927 diff --git a/src/test/resources/com/conveyal/datatools/gtfs/merge-data-future-unique-ids/feed_info.txt b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-future-unique-ids/feed_info.txt new file mode 100644 index 000000000..ceac60810 --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-future-unique-ids/feed_info.txt @@ -0,0 +1,2 @@ +feed_id,feed_publisher_name,feed_publisher_url,feed_lang,feed_version +fake_transit,Conveyal,http://www.conveyal.com,en,1.0 \ No newline at end of file diff --git a/src/test/resources/com/conveyal/datatools/gtfs/merge-data-future-unique-ids/routes.txt b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-future-unique-ids/routes.txt new file mode 100755 index 000000000..b13480efa --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-future-unique-ids/routes.txt @@ -0,0 +1,3 @@ +agency_id,route_id,route_short_name,route_long_name,route_desc,route_type,route_url,route_color,route_text_color,route_branding_url +1,1,1,Route 1,,3,,7CE6E7,FFFFFF, +1,2,2,Route 2,,3,,7CE6E7,FFFFFF, diff --git a/src/test/resources/com/conveyal/datatools/gtfs/merge-data-future-unique-ids/stop_attributes.txt b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-future-unique-ids/stop_attributes.txt new file mode 100644 index 000000000..b77c473e0 --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-future-unique-ids/stop_attributes.txt @@ -0,0 +1,3 @@ +stop_id,accessibility_id,cardinal_direction,relative_position,stop_city +4u6g,0,SE,FS,Scotts Valley +johv,0,SE,FS,Scotts Valley diff --git a/src/test/resources/com/conveyal/datatools/gtfs/merge-data-future-unique-ids/stop_times.txt b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-future-unique-ids/stop_times.txt new file mode 100755 index 000000000..cc847bc94 --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-future-unique-ids/stop_times.txt @@ -0,0 +1,5 @@ +trip_id,arrival_time,departure_time,stop_id,stop_sequence,stop_headsign,pickup_type,drop_off_type,shape_dist_traveled,timepoint +future-trip1,07:00:00,07:00:00,4u6g,1,,0,0,0.0000000, +future-trip1,07:01:00,07:01:00,johv,2,,0,0,341.4491961, +future-trip2,07:00:00,07:00:00,johv,1,,0,0,0.0000000, +future-trip2,07:01:00,07:01:00,4u6g,2,,0,0,341.4491961, \ No newline at end of file diff --git a/src/test/resources/com/conveyal/datatools/gtfs/merge-data-future-unique-ids/stops.txt b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-future-unique-ids/stops.txt new file mode 100755 index 000000000..0db5a6d40 --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-future-unique-ids/stops.txt @@ -0,0 +1,6 @@ +stop_id,stop_code,stop_name,stop_desc,stop_lat,stop_lon,zone_id,stop_url,location_type,parent_station,stop_timezone,wheelchair_boarding +4u6g,4u6g,Butler Ln,,37.0612132,-122.0074332,,,0,,, +johv,johv,Scotts Valley Dr & Victor Sq,,37.0590172,-122.0096058,,,0,,, +123,,Parent Station,,37.0666,-122.0777,,,1,,, +1234,1234,Child Stop,,37.06662,-122.07772,,,0,123,, +1234567,1234567,Unused stop,,37.06668,-122.07781,,,0,123,, \ No newline at end of file diff --git a/src/test/resources/com/conveyal/datatools/gtfs/merge-data-future-unique-ids/trips.txt b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-future-unique-ids/trips.txt new file mode 100755 index 000000000..71fc760c8 --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-future-unique-ids/trips.txt @@ -0,0 +1,3 @@ +route_id,trip_id,trip_headsign,trip_short_name,direction_id,block_id,shape_id,bikes_allowed,wheelchair_accessible,service_id +1,future-trip1,,,0,,,0,0,common_id +2,future-trip2,,,0,,,0,0,common_id \ No newline at end of file diff --git a/src/test/resources/com/conveyal/datatools/gtfs/merge-data-mod-services/calendar.txt b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-mod-services/calendar.txt index 0e75afb73..201a95e65 100755 --- a/src/test/resources/com/conveyal/datatools/gtfs/merge-data-mod-services/calendar.txt +++ b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-mod-services/calendar.txt @@ -1,3 +1,4 @@ service_id,monday,tuesday,wednesday,thursday,friday,saturday,sunday,start_date,end_date common_id,1,1,1,1,1,1,1,20170918,20170920 only_calendar_id,1,1,1,1,1,1,1,20170921,20170922 +new_cal_id,1,1,1,1,1,0,0,20170921,20170922 \ No newline at end of file From 08ba2db2ada298992680961e0ec02a1efba75c63 Mon Sep 17 00:00:00 2001 From: Landon Reed Date: Mon, 10 May 2021 11:09:04 -0400 Subject: [PATCH 008/122] test(MergeFeedsJobTest): fix input data and test assertions --- .../datatools/manager/jobs/MergeFeedsJob.java | 12 ++++++------ .../manager/jobs/MergeFeedsJobTest.java | 16 +++++++++------- .../gtfs/merge-data-future-unique-ids/trips.txt | 4 ++-- .../gtfs/merge-data-mod-trips/stop_times.txt | 10 +++++++--- .../gtfs/merge-data-mod-trips/trips.txt | 1 + 5 files changed, 25 insertions(+), 18 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java b/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java index d6a4421f7..1585d2dbf 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java @@ -130,9 +130,6 @@ public class MergeFeedsJob extends MonitorableJob { private static final Logger LOG = LoggerFactory.getLogger(MergeFeedsJob.class); public static final ObjectMapper mapper = new ObjectMapper(); - private static Set intersectingTripIds = new HashSet<>(); - private static Set tripsOnlyInActiveFeed = new HashSet<>(); - private static Set tripsOnlyInFutureFeed = new HashSet<>(); private final Set feedVersions; private final FeedSource feedSource; private final ReferenceTracker referenceTracker = new ReferenceTracker(); @@ -147,6 +144,9 @@ public class MergeFeedsJob extends MonitorableJob { */ final FeedVersion mergedVersion; public MergeStrategy mergeStrategy = MergeStrategy.DEFAULT; + private Set intersectingTripIds = new HashSet<>(); + private Set tripsOnlyInActiveFeed = new HashSet<>(); + private Set tripsOnlyInFutureFeed = new HashSet<>(); private Set tripIdsToModifyForActiveFeed = new HashSet<>(); private Set tripIdsToSkipForActiveFeed = new HashSet<>(); private Set serviceIdsToExtend = new HashSet<>(); @@ -362,7 +362,7 @@ private boolean stopTimesMatch(List futureStopTimes, List ac return false; } for (int i = 0; i < activeStopTimes.size(); i++) { - if (activeStopTimes.get(i).hashCode() == futureStopTimes.get(i).hashCode()) { + if (!activeStopTimes.get(i).equals(futureStopTimes.get(i))) { return false; } } @@ -677,7 +677,7 @@ private int constructMergedTable(Table table, List feedsToMerge, // York State). if (mergeType.equals(SERVICE_PERIOD)) { Set idErrors; - // If analyzing the second feed (non-future feed), the service_id always gets feed scoped. + // If analyzing the second feed (active feed), the service_id always gets feed scoped. // See https://github.com/ibi-group/datatools-server/issues/244 if (handlingActiveFeed && field.name.equals("service_id")) { valueToWrite = String.join(":", idScope, val); @@ -1099,7 +1099,7 @@ private int constructMergedTable(Table table, List feedsToMerge, * Get the merge strategy to use for MTC service period merges by checking the active and future feeds for various * combinations of matching trip and service IDs. */ - private static MergeStrategy getMergeStrategy(List feedsToMerge) throws IOException { + private MergeStrategy getMergeStrategy(List feedsToMerge) throws IOException { // Iterate over both feeds to collect all trip and service IDs. for (int i = 0; i < feedsToMerge.size(); i++) { FeedToMerge feed = feedsToMerge.get(i); diff --git a/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java b/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java index e445d648f..f0e74e33e 100644 --- a/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java +++ b/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java @@ -366,12 +366,13 @@ public void mergeMTCShouldHandleCheckStopTimesStrategy() throws SQLException { String mergedNamespace = mergeFeedsJob.mergedVersion.namespace; // - calendar table - // expect a total of 4 records in calendar table: + // expect a total of 5 records in calendar table: // - 2 original (common_id start date extended) // - 2 cloned for active feed + // - 1 cloned and modified for future feed assertThatSqlCountQueryYieldsExpectedCount( String.format("SELECT count(*) FROM %s.calendar", mergedNamespace), - 4 + 5 ); // // expect that both records in calendar table have the correct start_date // assertThatSqlCountQueryYieldsExpectedCount( @@ -411,15 +412,16 @@ public void mergeMTCShouldHandleDefaultStrategy() throws SQLException { String mergedNamespace = mergeFeedsJob.mergedVersion.namespace; // - calendar table - // expect a total of 2 records in calendar table + // expect a total of 4 records in calendar table (all records from original files are included). assertThatSqlCountQueryYieldsExpectedCount( String.format("SELECT count(*) FROM %s.calendar", mergedNamespace), - 2 + 4 ); - // expect that both records in calendar table have the correct start_date + // - trips table + // expect a total of 4 records in trips table (all records from original files are included). assertThatSqlCountQueryYieldsExpectedCount( - String.format("SELECT count(*) FROM %s.calendar where start_date = '20170918'", mergedNamespace), - 2 + String.format("SELECT count(*) FROM %s.trips", mergedNamespace), + 4 ); } diff --git a/src/test/resources/com/conveyal/datatools/gtfs/merge-data-future-unique-ids/trips.txt b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-future-unique-ids/trips.txt index 71fc760c8..221d7a051 100755 --- a/src/test/resources/com/conveyal/datatools/gtfs/merge-data-future-unique-ids/trips.txt +++ b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-future-unique-ids/trips.txt @@ -1,3 +1,3 @@ route_id,trip_id,trip_headsign,trip_short_name,direction_id,block_id,shape_id,bikes_allowed,wheelchair_accessible,service_id -1,future-trip1,,,0,,,0,0,common_id -2,future-trip2,,,0,,,0,0,common_id \ No newline at end of file +1,future-trip1,,,0,,,0,0,future_id +2,future-trip2,,,0,,,0,0,future_id \ No newline at end of file diff --git a/src/test/resources/com/conveyal/datatools/gtfs/merge-data-mod-trips/stop_times.txt b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-mod-trips/stop_times.txt index 646706c5c..29b88cefb 100755 --- a/src/test/resources/com/conveyal/datatools/gtfs/merge-data-mod-trips/stop_times.txt +++ b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-mod-trips/stop_times.txt @@ -1,5 +1,9 @@ trip_id,arrival_time,departure_time,stop_id,stop_sequence,stop_headsign,pickup_type,drop_off_type,shape_dist_traveled,timepoint -only-calendar-trip1,07:00:00,07:00:00,4u6g,1,,0,0,0.0000000, -only-calendar-trip1,07:01:00,07:01:00,johv,2,,0,0,341.4491961, +trip3,07:00:00,07:00:00,4u6g,1,,0,0,0.0000000, +trip3,07:01:00,07:01:00,johv,2,,0,0,341.4491961, +only-calendar-trip1,08:00:00,08:00:00,4u6g,1,,0,0,0.0000000, +only-calendar-trip1,08:01:00,08:01:00,johv,2,,0,0,341.4491961, only-calendar-trip2,07:00:00,07:00:00,johv,1,,0,0,0.0000000, -only-calendar-trip2,07:01:00,07:01:00,4u6g,2,,0,0,341.4491961, \ No newline at end of file +only-calendar-trip2,07:01:00,07:01:00,4u6g,2,,0,0,341.4491961, +only-calendar-trip999,07:00:00,07:00:00,johv,1,,0,0,0.0000000, +only-calendar-trip999,07:01:00,07:01:00,4u6g,2,,0,0,341.4491961, \ No newline at end of file diff --git a/src/test/resources/com/conveyal/datatools/gtfs/merge-data-mod-trips/trips.txt b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-mod-trips/trips.txt index d745f3502..095ff16d5 100755 --- a/src/test/resources/com/conveyal/datatools/gtfs/merge-data-mod-trips/trips.txt +++ b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-mod-trips/trips.txt @@ -1,4 +1,5 @@ route_id,trip_id,trip_headsign,trip_short_name,direction_id,block_id,shape_id,bikes_allowed,wheelchair_accessible,service_id 1,only-calendar-trip999,,,0,,,0,0,common_id +1,only-calendar-trip1,,,0,,,0,0,common_id 2,only-calendar-trip2,,,0,,,0,0,common_id 2,trip3,,,0,,,0,0,only_calendar_id \ No newline at end of file From 65e2df6b720c1f45742469d45a9262f928589073 Mon Sep 17 00:00:00 2001 From: Landon Reed Date: Mon, 10 May 2021 11:28:43 -0400 Subject: [PATCH 009/122] test: fix bart feed merge assertion --- .../com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java b/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java index f0e74e33e..bb0d79d22 100644 --- a/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java +++ b/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java @@ -450,7 +450,7 @@ public void canMergeBARTFeeds() throws SQLException { ); // Check GTFS file line numbers. assertEquals( - 5149, // Magic number represents the number of trips in the merged BART feed. + 4629, // Magic number represents the number of trips in the merged BART feed. mergeFeedsJob.mergedVersion.feedLoadResult.trips.rowCount, "Merged feed trip count should equal expected value." ); From 6f73b63279362055609b8f1513ec6f6c6fcaf5eb Mon Sep 17 00:00:00 2001 From: Landon Reed Date: Tue, 1 Jun 2021 17:27:41 -0400 Subject: [PATCH 010/122] refactor: move mergeStrategy to MergeFeedsResult --- .../datatools/manager/jobs/MergeFeedsJob.java | 13 ++++++------- .../datatools/manager/jobs/MergeFeedsResult.java | 1 + .../datatools/manager/jobs/MergeFeedsJobTest.java | 8 ++++---- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java b/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java index 1585d2dbf..507b30300 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java @@ -143,7 +143,6 @@ public class MergeFeedsJob extends MonitorableJob { * dataset. Otherwise, this will be null throughout the life of the job. */ final FeedVersion mergedVersion; - public MergeStrategy mergeStrategy = MergeStrategy.DEFAULT; private Set intersectingTripIds = new HashSet<>(); private Set tripsOnlyInActiveFeed = new HashSet<>(); private Set tripsOnlyInFutureFeed = new HashSet<>(); @@ -245,8 +244,8 @@ public void jobFinished() { int numberOfTables = tablesToMerge.size(); // Before initiating the merge process, run some pre-processing to check for id conflicts for certain tables if (mergeType.equals(SERVICE_PERIOD)) { - mergeStrategy = getMergeStrategy(feedsToMerge); - if (mergeStrategy == CHECK_STOP_TIMES) { + mergeFeedsResult.mergeStrategy = getMergeStrategy(feedsToMerge); + if (mergeFeedsResult.mergeStrategy == CHECK_STOP_TIMES) { Feed futureFeed = new Feed(DataManager.GTFS_DATA_SOURCE, feedsToMerge.get(0).version.namespace); Feed activeFeed = new Feed(DataManager.GTFS_DATA_SOURCE, feedsToMerge.get(1).version.namespace); for (String tripId : intersectingTripIds) { @@ -293,7 +292,7 @@ public void jobFinished() { } } // Skip merging process altogether if the failing condition is met. - if (MergeStrategy.FAIL_DUE_TO_MATCHING_TRIP_IDS.equals(mergeStrategy)) { + if (MergeStrategy.FAIL_DUE_TO_MATCHING_TRIP_IDS.equals(mergeFeedsResult.mergeStrategy)) { failMergeJob("Feed merge failed because the trip_ids are identical in the future and active feeds. A new service requires unique trip_ids for merging."); return; } @@ -457,7 +456,7 @@ private int constructMergedTable(Table table, List feedsToMerge, for (int feedIndex = 0; feedIndex < feedsToMerge.size(); feedIndex++) { boolean handlingActiveFeed = feedIndex > 0; boolean handlingFutureFeed = feedIndex == 0; - if (EXTEND_FUTURE.equals(mergeStrategy) && handlingActiveFeed) { + if (EXTEND_FUTURE.equals(mergeFeedsResult.mergeStrategy) && handlingActiveFeed) { // No need to iterate over second (active) file if strategy is to simply extend the future GTFS // service to start earlier. continue; @@ -726,9 +725,9 @@ private int constructMergedTable(Table table, List feedsToMerge, // FIXME: Move this below so that a cloned service doesn't get prematurely // modified? (do we want the cloned record to have the original values?) if (index == startDateIndex) { - if (EXTEND_FUTURE == mergeStrategy || + if (EXTEND_FUTURE == mergeFeedsResult.mergeStrategy || ( - CHECK_STOP_TIMES == mergeStrategy && + CHECK_STOP_TIMES == mergeFeedsResult.mergeStrategy && // TODO: Need to ensure serviceIds are being extended. serviceIdsToExtend.contains(keyValue) ) diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsResult.java b/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsResult.java index 0f847874b..e29666323 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsResult.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsResult.java @@ -17,6 +17,7 @@ public class MergeFeedsResult implements Serializable { public int feedCount; /** Type of merge operation performed */ public MergeFeedsType type; + public MergeStrategy mergeStrategy = MergeStrategy.DEFAULT; /** Contains a set of strings for which there were error-causing duplicate values */ public Set idConflicts = new HashSet<>(); /** Contains the set of IDs for records that were excluded in the merged feed */ diff --git a/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java b/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java index bb0d79d22..8ebdb3fd5 100644 --- a/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java +++ b/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java @@ -294,7 +294,7 @@ public void mergeMTCShouldFailOnDuplicateTripsButMismatchedServices() { // Check that correct strategy was used. assertEquals( MergeStrategy.FAIL_DUE_TO_MATCHING_TRIP_IDS, - mergeFeedsJob.mergeStrategy + mergeFeedsJob.mergeFeedsResult.mergeStrategy ); // Result should fail. assertTrue( @@ -322,7 +322,7 @@ public void mergeMTCShouldHandleExtendFutureStrategy() throws SQLException { ); assertEquals( MergeStrategy.EXTEND_FUTURE, - mergeFeedsJob.mergeStrategy + mergeFeedsJob.mergeFeedsResult.mergeStrategy ); // assert service_ids start_dates have been extended to the start_date of the base feed. String mergedNamespace = mergeFeedsJob.mergedVersion.namespace; @@ -355,7 +355,7 @@ public void mergeMTCShouldHandleCheckStopTimesStrategy() throws SQLException { // Check that correct strategy was used. assertEquals( MergeStrategy.CHECK_STOP_TIMES, - mergeFeedsJob.mergeStrategy + mergeFeedsJob.mergeFeedsResult.mergeStrategy ); // Result should succeed. assertFalse( @@ -401,7 +401,7 @@ public void mergeMTCShouldHandleDefaultStrategy() throws SQLException { // Check that correct strategy was used. assertEquals( MergeStrategy.DEFAULT, - mergeFeedsJob.mergeStrategy + mergeFeedsJob.mergeFeedsResult.mergeStrategy ); // Result should succeed. assertFalse( From 7ca798e4a643d10842ccef6d6d8b751e66e1ffa4 Mon Sep 17 00:00:00 2001 From: Landon Reed Date: Fri, 11 Jun 2021 10:23:58 -0400 Subject: [PATCH 011/122] refactor(MergeFeedsJob): remove outdated comment --- .../java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java b/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java index 507b30300..5ec2657ae 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java @@ -728,7 +728,6 @@ private int constructMergedTable(Table table, List feedsToMerge, if (EXTEND_FUTURE == mergeFeedsResult.mergeStrategy || ( CHECK_STOP_TIMES == mergeFeedsResult.mergeStrategy && - // TODO: Need to ensure serviceIds are being extended. serviceIdsToExtend.contains(keyValue) ) ) { From 04195e6836664fb4776486dd379ca39db7e80bc7 Mon Sep 17 00:00:00 2001 From: Landon Reed Date: Mon, 14 Jun 2021 11:07:44 -0400 Subject: [PATCH 012/122] refactor(MeregeFeedsJob): add getter for future/active feeds --- .../datatools/manager/jobs/MergeFeedsJob.java | 25 +++++++++++++------ 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java b/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java index 5ec2657ae..e3044b6bf 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java @@ -150,11 +150,22 @@ public class MergeFeedsJob extends MonitorableJob { private Set tripIdsToSkipForActiveFeed = new HashSet<>(); private Set serviceIdsToExtend = new HashSet<>(); private Set serviceIdsToCloneAndRename = new HashSet<>(); + private List feedsToMerge; public MergeFeedsJob(Auth0UserProfile owner, Set feedVersions, String file, MergeFeedsType mergeType) { this(owner, feedVersions, file, mergeType, true); } + /** Shorthand method to get the future feed during a service period merge */ + private FeedToMerge getFutureFeed() { + return feedsToMerge.get(0); + } + + /** Shorthand method to get the active feed during a service period merge */ + private FeedToMerge getActiveFeed() { + return feedsToMerge.get(1); + } + /** * @param owner user ID that initiated job * @param feedVersions set of feed versions to merge @@ -229,7 +240,7 @@ public void jobFinished() { // Create the zipfile. ZipOutputStream out = new ZipOutputStream(new FileOutputStream(mergedTempFile)); LOG.info("Created merge file: " + mergedTempFile.getAbsolutePath()); - List feedsToMerge = collectAndSortFeeds(feedVersions); + feedsToMerge = collectAndSortFeeds(feedVersions); // Determine which tables to merge (only merge GTFS+ tables for MTC extension). final List
tablesToMerge = @@ -246,8 +257,8 @@ public void jobFinished() { if (mergeType.equals(SERVICE_PERIOD)) { mergeFeedsResult.mergeStrategy = getMergeStrategy(feedsToMerge); if (mergeFeedsResult.mergeStrategy == CHECK_STOP_TIMES) { - Feed futureFeed = new Feed(DataManager.GTFS_DATA_SOURCE, feedsToMerge.get(0).version.namespace); - Feed activeFeed = new Feed(DataManager.GTFS_DATA_SOURCE, feedsToMerge.get(1).version.namespace); + Feed futureFeed = new Feed(DataManager.GTFS_DATA_SOURCE, getFutureFeed().version.namespace); + Feed activeFeed = new Feed(DataManager.GTFS_DATA_SOURCE, getActiveFeed().version.namespace); for (String tripId : intersectingTripIds) { // Fetch all ordered stop_times for each common trip_id and compare the two sets for the // future and active feed. If the stop_times are an exact match, include one instance of the trip @@ -449,8 +460,8 @@ private int constructMergedTable(Table table, List feedsToMerge, // providers use calendars.txt entries and prefer that this value (which is used to determine cutoff // dates for the active feed when merging with the future) be strictly assigned the earliest // calendar#start_date (unless that table for some reason does not exist). - LocalDate futureFeedFirstDate = feedsToMerge.get(0).version.validationResult.firstCalendarDate; - LocalDate activeFeedFirstDate = feedsToMerge.get(1).version.validationResult.firstCalendarDate; + LocalDate futureFeedFirstDate = getFutureFeed().version.validationResult.firstCalendarDate; + LocalDate activeFeedFirstDate = getActiveFeed().version.validationResult.firstCalendarDate; LocalDate futureFirstCalendarStartDate = LocalDate.MAX; // Iterate over each zip file. For service period merge, the first feed is the future GTFS. for (int feedIndex = 0; feedIndex < feedsToMerge.size(); feedIndex++) { @@ -1106,8 +1117,8 @@ private MergeStrategy getMergeStrategy(List feedsToMerge) throws IO feed.idsForTable.get(table).addAll(getIdsForTable(feed.zipFile, table)); } } - FeedToMerge futureFeed = feedsToMerge.get(0); - FeedToMerge activeFeed = feedsToMerge.get(1); + FeedToMerge futureFeed = getFutureFeed(); + FeedToMerge activeFeed = getActiveFeed(); Set activeTripIds = activeFeed.idsForTable.get(Table.TRIPS); Set futureTripIds = futureFeed.idsForTable.get(Table.TRIPS); Set activeServiceIds = new HashSet<>(); From aab5dd78d16b99258460e532f3268e0217758099 Mon Sep 17 00:00:00 2001 From: Landon Reed Date: Mon, 14 Jun 2021 11:19:34 -0400 Subject: [PATCH 013/122] build(deps): bump gtfs-lib to 6.2.4 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 1c0a3facb..428595344 100644 --- a/pom.xml +++ b/pom.xml @@ -270,7 +270,7 @@ com.github.conveyal gtfs-lib - add-stoptime-hash-v3.3.0-g736541a-721 + 6.2.4 From 91fe4c9ffacac5eb93f991f7b905ce8a5c1ff920 Mon Sep 17 00:00:00 2001 From: Landon Reed Date: Mon, 14 Jun 2021 11:21:08 -0400 Subject: [PATCH 014/122] refactor(MergeFeedsJob): fix comment --- .../java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java b/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java index e3044b6bf..3cb0c1d87 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java @@ -548,7 +548,7 @@ private int constructMergedTable(Table table, List feedsToMerge, futureFirstCalendarStartDate.isBefore(LocalDate.MAX) && futureFeedFirstDate.isBefore(futureFirstCalendarStartDate) ) { - // If the future feed's first date is before the its first calendar start date, + // If the future feed's first date is before its first calendar start date, // override the future feed first date with the calendar start date for use when checking // MTC calendar_dates and calendar records for modification/exclusion. futureFeedFirstDate = futureFirstCalendarStartDate; From 383c635c0697828361784d92a93be1c2c87a4d81 Mon Sep 17 00:00:00 2001 From: Landon Reed Date: Mon, 14 Jun 2021 11:27:07 -0400 Subject: [PATCH 015/122] refactor(MergeFeedsJob): update unclear comment --- .../conveyal/datatools/manager/jobs/MergeFeedsJob.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java b/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java index 3cb0c1d87..9a2c03488 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java @@ -837,10 +837,12 @@ private int constructMergedTable(Table table, List feedsToMerge, if (hasDuplicateError(idErrors)) skipRecord = true; break; case "trips": - // trip_ids between active and future datasets must not match. The MergeStrategy - // determines behavior when matching trip_ids (or service_ids) are found between + // trip_ids between active and future datasets must not match. The tripIdsToSkip and + // tripIdsToModify sets below are determined based on the MergeStrategy used for MTC + // service period merges. if (handlingActiveFeed) { - // Handling active feed. + // Handling active feed. Skip or modify trip id if found in one of the + // respective sets. if (tripIdsToSkipForActiveFeed.contains(keyValue)) { skipRecord = true; } else if (tripIdsToModifyForActiveFeed.contains(keyValue)) { From 1cfd32e5362ab68b6d7b440379db98110c80299a Mon Sep 17 00:00:00 2001 From: Landon Reed Date: Mon, 14 Jun 2021 12:15:39 -0400 Subject: [PATCH 016/122] refactor(MergeFeedsJob): clean up get strategy code --- .../datatools/manager/jobs/FeedToMerge.java | 15 ++ .../datatools/manager/jobs/MergeFeedsJob.java | 164 ++++++++++-------- 2 files changed, 102 insertions(+), 77 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/FeedToMerge.java b/src/main/java/com/conveyal/datatools/manager/jobs/FeedToMerge.java index 753730a5d..2a23bef0b 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/FeedToMerge.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/FeedToMerge.java @@ -4,10 +4,15 @@ import com.conveyal.gtfs.loader.Table; import com.google.common.collect.HashMultimap; import com.google.common.collect.SetMultimap; +import com.google.common.collect.Sets; import java.io.IOException; +import java.util.HashSet; +import java.util.Set; import java.util.zip.ZipFile; +import static com.conveyal.datatools.manager.utils.MergeFeedUtils.getIdsForTable; + /** * Helper class that collects the feed version and its zip file. Note: this class helps with sorting versions to * merge in a list collection. @@ -16,9 +21,19 @@ public class FeedToMerge { public FeedVersion version; public ZipFile zipFile; public SetMultimap idsForTable = HashMultimap.create(); + public Set serviceIds = new HashSet<>(); + private static final Set
tablesToCheck = Sets.newHashSet(Table.TRIPS, Table.CALENDAR, Table.CALENDAR_DATES); public FeedToMerge(FeedVersion version) throws IOException { this.version = version; this.zipFile = new ZipFile(version.retrieveGtfsFile()); } + + public void collectTripAndServiceIds() throws IOException { + for (Table table : tablesToCheck) { + idsForTable.get(table).addAll(getIdsForTable(zipFile, table)); + } + serviceIds.addAll(idsForTable.get(Table.CALENDAR)); + serviceIds.addAll(idsForTable.get(Table.CALENDAR_DATES)); + } } \ No newline at end of file diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java b/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java index 9a2c03488..f5e196129 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java @@ -143,9 +143,6 @@ public class MergeFeedsJob extends MonitorableJob { * dataset. Otherwise, this will be null throughout the life of the job. */ final FeedVersion mergedVersion; - private Set intersectingTripIds = new HashSet<>(); - private Set tripsOnlyInActiveFeed = new HashSet<>(); - private Set tripsOnlyInFutureFeed = new HashSet<>(); private Set tripIdsToModifyForActiveFeed = new HashSet<>(); private Set tripIdsToSkipForActiveFeed = new HashSet<>(); private Set serviceIdsToExtend = new HashSet<>(); @@ -255,52 +252,7 @@ public void jobFinished() { int numberOfTables = tablesToMerge.size(); // Before initiating the merge process, run some pre-processing to check for id conflicts for certain tables if (mergeType.equals(SERVICE_PERIOD)) { - mergeFeedsResult.mergeStrategy = getMergeStrategy(feedsToMerge); - if (mergeFeedsResult.mergeStrategy == CHECK_STOP_TIMES) { - Feed futureFeed = new Feed(DataManager.GTFS_DATA_SOURCE, getFutureFeed().version.namespace); - Feed activeFeed = new Feed(DataManager.GTFS_DATA_SOURCE, getActiveFeed().version.namespace); - for (String tripId : intersectingTripIds) { - // Fetch all ordered stop_times for each common trip_id and compare the two sets for the - // future and active feed. If the stop_times are an exact match, include one instance of the trip - // (ignoring the other identical one). If they do not match, modify the active trip_id and include. - List futureStopTimes = Lists.newArrayList(futureFeed.stopTimes.getOrdered(tripId)); - List activeStopTimes = Lists.newArrayList(activeFeed.stopTimes.getOrdered(tripId)); - String futureServiceId = futureFeed.trips.get(tripId).service_id; - String activeServiceId = activeFeed.trips.get(tripId).service_id; - // FIXME: what if service_ids do not match! Perhaps the right approach would be to just return - // FAIL_DUE_TO_MATCHING_TRIP_IDS in that case. It might be too complicated otherwise. - if (!stopTimesMatch(futureStopTimes, activeStopTimes)) { - // If stop_times or services do not match, the trip will be cloned. Also, track the service_id - // (it will need to be cloned and renamed for both active feeds). - tripIdsToModifyForActiveFeed.add(tripId); - serviceIdsToCloneAndRename.add(futureServiceId); - } else { - // If the trip's stop_times are an exact match, we can safely include just the - // future trip and exclude the active one. Also, track the service_id (it will need to be - // extended to the full time range). - tripIdsToSkipForActiveFeed.add(tripId); - serviceIdsToExtend.add(futureServiceId); - } - } - for (String tripId : tripsOnlyInActiveFeed) { - String serviceId = activeFeed.trips.get(tripId).service_id; - if (serviceIdsToExtend.contains(serviceId)) { - // If a trip only in the active feed references a service_id that is set to be extended, that - // service_id needs to be cloned and renamed to differentiate it from the same service_id in - // the future feed. (The trip in question will be linked to the cloned service_id.) - serviceIdsToCloneAndRename.add(serviceId); - } - } - for (String tripId : tripsOnlyInFutureFeed) { - String serviceId = futureFeed.trips.get(tripId).service_id; - if (serviceIdsToExtend.contains(serviceId)) { - // If a trip only in the future feed references a service_id that is set to be extended, that - // service_id needs to be cloned and renamed to differentiate it from the same service_id in - // the future feed. (The trip in question will be linked to the cloned service_id.) - serviceIdsToCloneAndRename.add(serviceId); - } - } - } + mergeFeedsResult.mergeStrategy = getMergeStrategy(); } // Skip merging process altogether if the failing condition is met. if (MergeStrategy.FAIL_DUE_TO_MATCHING_TRIP_IDS.equals(mergeFeedsResult.mergeStrategy)) { @@ -1110,29 +1062,16 @@ private int constructMergedTable(Table table, List feedsToMerge, * Get the merge strategy to use for MTC service period merges by checking the active and future feeds for various * combinations of matching trip and service IDs. */ - private MergeStrategy getMergeStrategy(List feedsToMerge) throws IOException { - // Iterate over both feeds to collect all trip and service IDs. - for (int i = 0; i < feedsToMerge.size(); i++) { - FeedToMerge feed = feedsToMerge.get(i); - Set
tablesToCheck = Sets.newHashSet(Table.TRIPS, Table.CALENDAR, Table.CALENDAR_DATES); - for (Table table : tablesToCheck) { - feed.idsForTable.get(table).addAll(getIdsForTable(feed.zipFile, table)); - } - } - FeedToMerge futureFeed = getFutureFeed(); - FeedToMerge activeFeed = getActiveFeed(); - Set activeTripIds = activeFeed.idsForTable.get(Table.TRIPS); - Set futureTripIds = futureFeed.idsForTable.get(Table.TRIPS); - Set activeServiceIds = new HashSet<>(); - activeServiceIds.addAll(activeFeed.idsForTable.get(Table.CALENDAR)); - activeServiceIds.addAll(activeFeed.idsForTable.get(Table.CALENDAR_DATES)); - Set futureServiceIds = new HashSet<>(); - futureServiceIds.addAll(futureFeed.idsForTable.get(Table.CALENDAR)); - futureServiceIds.addAll(futureFeed.idsForTable.get(Table.CALENDAR_DATES)); - boolean serviceIdsMatch = activeServiceIds.equals(futureServiceIds); - intersectingTripIds = Sets.intersection(activeTripIds, futureTripIds); - tripsOnlyInActiveFeed = Sets.difference(activeTripIds, futureTripIds); - tripsOnlyInFutureFeed = Sets.difference(futureTripIds, activeTripIds); + private MergeStrategy getMergeStrategy() throws IOException { + boolean shouldFailJob = false; + FeedToMerge futureFeedToMerge = getFutureFeed(); + FeedToMerge activeFeedToMerge = getActiveFeed(); + futureFeedToMerge.collectTripAndServiceIds(); + activeFeedToMerge.collectTripAndServiceIds(); + Set activeTripIds = activeFeedToMerge.idsForTable.get(Table.TRIPS); + Set futureTripIds = futureFeedToMerge.idsForTable.get(Table.TRIPS); + // Determine whether service and trip IDs are exact matches. + boolean serviceIdsMatch = activeFeedToMerge.serviceIds.equals(futureFeedToMerge.serviceIds); boolean tripIdsMatch = activeTripIds.equals(futureTripIds); if (serviceIdsMatch && tripIdsMatch) { // Effectively this exact match condition means that the future feed will be used as is @@ -1140,15 +1079,86 @@ private MergeStrategy getMergeStrategy(List feedsToMerge) throws IO // This is Condition 2 in the docs. return EXTEND_FUTURE; } - if (serviceIdsMatch) { - // If just the service_ids are an exact match, do the trip/stoptimes checking thing for matching trip_ids - return CHECK_STOP_TIMES; - } if (tripIdsMatch) { - // Do not permit merge to continue. + // If only trip IDs match, do not permit merge to continue. return MergeStrategy.FAIL_DUE_TO_MATCHING_TRIP_IDS; } + if (serviceIdsMatch) { + // If just the service_ids are an exact match, check the that the stoptimes having matching signatures + // between the two feeds (i.e., each stop time in the ordered list is identical between the two feeds). + Feed futureFeed = new Feed(DataManager.GTFS_DATA_SOURCE, futureFeedToMerge.version.namespace); + Feed activeFeed = new Feed(DataManager.GTFS_DATA_SOURCE, activeFeedToMerge.version.namespace); + for (String tripId : Sets.intersection(activeTripIds, futureTripIds)) { + if (compareStopTimes(tripId, futureFeed, activeFeed)) { + shouldFailJob = true; + } + } + Set tripsOnlyInActiveFeed = Sets.difference(activeTripIds, futureTripIds); + for (String tripId : tripsOnlyInActiveFeed) { + String serviceId = activeFeed.trips.get(tripId).service_id; + if (serviceIdsToExtend.contains(serviceId)) { + // If a trip only in the active feed references a service_id that is set to be extended, that + // service_id needs to be cloned and renamed to differentiate it from the same service_id in + // the future feed. (The trip in question will be linked to the cloned service_id.) + serviceIdsToCloneAndRename.add(serviceId); + } + } + Set tripsOnlyInFutureFeed = Sets.difference(futureTripIds, activeTripIds); + for (String tripId : tripsOnlyInFutureFeed) { + String serviceId = futureFeed.trips.get(tripId).service_id; + if (serviceIdsToExtend.contains(serviceId)) { + // If a trip only in the future feed references a service_id that is set to be extended, that + // service_id needs to be cloned and renamed to differentiate it from the same service_id in + // the future feed. (The trip in question will be linked to the cloned service_id.) + serviceIdsToCloneAndRename.add(serviceId); + } + } + // If a failure was encountered above, use failure strategy. Otherwise, use check stop times to proceed with + // feed merge. + return shouldFailJob ? MergeStrategy.FAIL_DUE_TO_MATCHING_TRIP_IDS : CHECK_STOP_TIMES; + } // If neither the trips or services are exact matches, use the default merge strategy. return MergeStrategy.DEFAULT; } + + /** + * Compare stop times for the given tripId between the future and active feeds. The comparison will inform whether + * trip and/or service IDs should be modified in the output merged feed. + * @return true if an error was encountered during the check. false if no error was encountered. + */ + private boolean compareStopTimes(String tripId, Feed futureFeed, Feed activeFeed) { + // Fetch all ordered stop_times for each shared trip_id and compare the two sets for the + // future and active feed. If the stop_times are an exact match, include one instance of the trip + // (ignoring the other identical one). If they do not match, modify the active trip_id and include. + List futureStopTimes = Lists.newArrayList(futureFeed.stopTimes.getOrdered(tripId)); + List activeStopTimes = Lists.newArrayList(activeFeed.stopTimes.getOrdered(tripId)); + String futureServiceId = futureFeed.trips.get(tripId).service_id; + String activeServiceId = activeFeed.trips.get(tripId).service_id; + if (!futureServiceId.equals(activeServiceId)) { + // We cannot account for the case where service_ids do not match! It would be a bit too complicated + // to handle this unique case, so instead just include in the failure reasons and use failure + // strategy. + failMergeJob( + String.format("Shared trip_id (%s) had mismatched service id between two feeds (active: %s, future: %s)", + tripId, + activeServiceId, + futureServiceId + ) + ); + return true; + } + if (!stopTimesMatch(futureStopTimes, activeStopTimes)) { + // If stop_times or services do not match, the trip will be cloned. Also, track the service_id + // (it will need to be cloned and renamed for both active feeds). + tripIdsToModifyForActiveFeed.add(tripId); + serviceIdsToCloneAndRename.add(futureServiceId); + } else { + // If the trip's stop_times are an exact match, we can safely include just the + // future trip and exclude the active one. Also, track the service_id (it will need to be + // extended to the full time range). + tripIdsToSkipForActiveFeed.add(tripId); + serviceIdsToExtend.add(futureServiceId); + } + return false; + } } From 352fde2845c1c58a7b8cd8fd27fc3058a93e270e Mon Sep 17 00:00:00 2001 From: Landon Reed Date: Mon, 14 Jun 2021 12:19:15 -0400 Subject: [PATCH 017/122] refactor(FeedsToMerge): add javadoc --- .../java/com/conveyal/datatools/manager/jobs/FeedToMerge.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/FeedToMerge.java b/src/main/java/com/conveyal/datatools/manager/jobs/FeedToMerge.java index 2a23bef0b..56ddcf413 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/FeedToMerge.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/FeedToMerge.java @@ -29,6 +29,7 @@ public FeedToMerge(FeedVersion version) throws IOException { this.zipFile = new ZipFile(version.retrieveGtfsFile()); } + /** Collects all trip/service IDs (tables noted in {@link #tablesToCheck}) for comparing feeds during merge. */ public void collectTripAndServiceIds() throws IOException { for (Table table : tablesToCheck) { idsForTable.get(table).addAll(getIdsForTable(zipFile, table)); From 4ddf697dbb58950872fcc5523ef8ee3ecb410e92 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Jun 2021 17:21:01 +0000 Subject: [PATCH 018/122] build(deps): bump snakeyaml from 1.23 to 1.26 Bumps [snakeyaml](https://bitbucket.org/asomov/snakeyaml) from 1.23 to 1.26. - [Commits](https://bitbucket.org/asomov/snakeyaml/branches/compare/snakeyaml-1.26..snakeyaml-1.23) --- updated-dependencies: - dependency-name: org.yaml:snakeyaml dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 428595344..3ac5ab3b1 100644 --- a/pom.xml +++ b/pom.xml @@ -408,7 +408,7 @@ org.yaml snakeyaml - 1.23 + 1.26 From ca6bfd862c473a89613fa544ac88432fd6d023b6 Mon Sep 17 00:00:00 2001 From: binh-dam-ibigroup <56846598+binh-dam-ibigroup@users.noreply.github.com> Date: Tue, 19 Oct 2021 15:39:53 -0400 Subject: [PATCH 035/122] fix(GtfsPlusValidation): Use CsvReader instead of own code. --- .../manager/gtfsplus/GtfsPlusValidation.java | 26 ++++++++++++------- .../gtfsplus/GtfsPlusValidationTest.java | 15 ++++++++++- 2 files changed, 30 insertions(+), 11 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/gtfsplus/GtfsPlusValidation.java b/src/main/java/com/conveyal/datatools/manager/gtfsplus/GtfsPlusValidation.java index c6adecb10..74408e5eb 100644 --- a/src/main/java/com/conveyal/datatools/manager/gtfsplus/GtfsPlusValidation.java +++ b/src/main/java/com/conveyal/datatools/manager/gtfsplus/GtfsPlusValidation.java @@ -1,23 +1,22 @@ package com.conveyal.datatools.manager.gtfsplus; -import com.conveyal.datatools.common.utils.Consts; import com.conveyal.datatools.manager.DataManager; import com.conveyal.datatools.manager.models.FeedVersion; import com.conveyal.datatools.manager.persistence.FeedStore; import com.conveyal.datatools.manager.persistence.Persistence; import com.conveyal.gtfs.GTFSFeed; +import com.csvreader.CsvReader; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ArrayNode; import org.apache.commons.io.input.BOMInputStream; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.BufferedReader; import java.io.File; import java.io.IOException; import java.io.InputStream; -import java.io.InputStreamReader; import java.io.Serializable; +import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.Collection; import java.util.Enumeration; @@ -124,10 +123,15 @@ private static void validateTable( GTFSFeed gtfsFeed ) throws IOException { String tableId = specTable.get("id").asText(); + // Read in table data from input stream. - BufferedReader in = new BufferedReader(new InputStreamReader(inputStreamToValidate)); - String line = in.readLine(); - String[] inputHeaders = line.split(","); + CsvReader csvReader = new CsvReader(inputStreamToValidate, ',', StandardCharsets.UTF_8); + // Don't skip empty records (this is set to true by default on CsvReader. We want to check for empty records + // during table load, so that they are logged as validation issues (rows with wrong number of columns). + csvReader.setSkipEmptyRecords(false); + csvReader.readHeaders(); + + String[] inputHeaders = csvReader.getHeaders(); List fieldList = Arrays.asList(inputHeaders); JsonNode[] fieldsFound = new JsonNode[inputHeaders.length]; JsonNode specFields = specTable.get("fields"); @@ -144,24 +148,26 @@ private static void validateTable( issues.add(new ValidationIssue(tableId, fieldName, -1, "Required column missing.")); } } + // Iterate over each row and validate each field value. int rowIndex = 0; int rowsWithWrongNumberOfColumns = 0; - while ((line = in.readLine()) != null) { - String[] values = line.split(Consts.COLUMN_SPLIT, -1); + while (csvReader.readRecord()) { // First, check that row has the correct number of fields. - if (values.length != fieldsFound.length) { + int recordColumnCount = csvReader.getColumnCount(); + if (recordColumnCount != fieldsFound.length) { rowsWithWrongNumberOfColumns++; } // Validate each value in row. Note: we iterate over the fields and not values because a row may be missing // columns, but we still want to validate that missing value (e.g., if it is missing a required field). for (int f = 0; f < fieldsFound.length; f++) { // If value exists for index, use that. Otherwise, default to null to avoid out of bounds exception. - String val = f < values.length ? values[f] : null; + String val = f < recordColumnCount ? csvReader.get(f) : null; validateTableValue(issues, tableId, rowIndex, val, fieldsFound[f], gtfsFeed); } rowIndex++; } + // Add issue for wrong number of columns after processing all rows. // Note: We considered adding an issue for each row, but opted for the single error approach because there's no // concept of a row-level issue in the UI right now. So we would potentially need to add that to the UI diff --git a/src/test/java/com/conveyal/datatools/manager/gtfsplus/GtfsPlusValidationTest.java b/src/test/java/com/conveyal/datatools/manager/gtfsplus/GtfsPlusValidationTest.java index 52d77ae65..abbebb9bf 100644 --- a/src/test/java/com/conveyal/datatools/manager/gtfsplus/GtfsPlusValidationTest.java +++ b/src/test/java/com/conveyal/datatools/manager/gtfsplus/GtfsPlusValidationTest.java @@ -23,6 +23,7 @@ public class GtfsPlusValidationTest extends UnitTest { private static final Logger LOG = LoggerFactory.getLogger(MergeFeedsJobTest.class); private static FeedVersion bartVersion1; + private static FeedVersion bartVersion1WithQuotedValues; private static Project project; /** @@ -40,13 +41,25 @@ public static void setUp() throws IOException { bart.projectId = project.id; Persistence.feedSources.create(bart); bartVersion1 = createFeedVersionFromGtfsZip(bart, "bart_new.zip"); + bartVersion1WithQuotedValues = createFeedVersionFromGtfsZip(bart, "bart_new_with_quoted_values.zip"); } @Test - public void canValidateCleanGtfsPlus() throws Exception { + void canValidateCleanGtfsPlus() throws Exception { LOG.info("Validation BART GTFS+"); GtfsPlusValidation validation = GtfsPlusValidation.validate(bartVersion1.id); // Expect issues to be zero. assertThat("Issues count for clean BART feed is zero", validation.issues.size(), equalTo(0)); } + + @Test + void canValidateGtfsPlusWithQuotedValues() throws Exception { + LOG.info("Validation BART GTFS+ with quoted values"); + GtfsPlusValidation validation = GtfsPlusValidation.validate(bartVersion1WithQuotedValues.id); + // Expect issues to be zero. + assertThat( + "Issues count for clean BART feed (quoted values) is zero", + validation.issues.size(), equalTo(0) + ); + } } From 8608aec1d5cd4c381c757425bd95934552405276 Mon Sep 17 00:00:00 2001 From: binh-dam-ibigroup <56846598+binh-dam-ibigroup@users.noreply.github.com> Date: Tue, 19 Oct 2021 16:05:07 -0400 Subject: [PATCH 036/122] test(bart_new_with_quoted_values.zip): Add resource for new test. --- .../gtfs/bart_new_with_quoted_values.zip | Bin 0 -> 517957 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 src/test/resources/com/conveyal/datatools/gtfs/bart_new_with_quoted_values.zip diff --git a/src/test/resources/com/conveyal/datatools/gtfs/bart_new_with_quoted_values.zip b/src/test/resources/com/conveyal/datatools/gtfs/bart_new_with_quoted_values.zip new file mode 100644 index 0000000000000000000000000000000000000000..233231a7aec69a489b2fb5ad4327a45c9489a67f GIT binary patch literal 517957 zcmcF~bzGF+*6#oV3_bMFC0#=Z2uMpeh@^BQT?!II4c*e+C>>JLEvYCBoq{xigw!2= z=e+0q?!D)IKlh*e%-YYhYwi7Ad+lbSse%S10f3OdX%$yBz(1dGMr4wouC4P7dW*RsRK9R+#HCLwaly1~Uqoti3`AaUdTU&ddCBd9! z%D*JoM5YiSC9pJivUavIcQ>^%_q6u-mn6f}RL1{}B%JEp9G_LyV<%Oom6W)YzA05H zs&liAj7>)KR&Xh9ZDD)kPV@0Y8wH{KjZ=J^Ti6%gk#Zmi3Tfyc>C002ALZCmsKeM} zc&eJDLBIr3z_{FV0FjionVnteE=Db&9+innM;DzQg$aoq6J$nX|G6Q(EXrC=3C5vwRicgJYg z#;qsy7bab%^e5@&%yf`IL((`G7nrq&!1#DUQ^Va;U%H2Ycel3mw0Cj-w?Y_+ zUCi13>Hd$d#W~8$t@M{VM!1yKc#xWqU2|op;P?sIWxLuUPqktTevKij+#{tXMf(>J z(!S|NZe&#t)%|B6LC9`Pfdpb|swOR~p`*{sWBNZ}@_l}e4H=UyjDs>9O9ybv+Ea4=RR9#{T= zmw4TIJt;^#aLxUf3Xbk*-cchP`Jtizt%ARIYe8H9fEcL)8*_JSQ*%#GcY6!3zmL%W zp?5!j*X~Ru4F-Ck-6e-~zg;@qyhH2blU5}8AVr|5o8RSgfj+3q2nv&Ur4gC^Y5ViR zn}FBz63)Ewvj-=8-tC=F1M-em=w=T#e%-!#+n#4ybJ2CAv2g4C`r5y9S>pcQJK#^7 z$szBNi)3KTS3SD3jmaZZBZN?}1(c{9)lXe8Bzp%{fYm!VwMTF`o-24b5~_%mGe@h= zUPm61U5?7knklXB8JtPnM~}?(j+kLe$D&PvOmK+_C-X27UF~8RR_bRZ{q?DhdW(XP zP?@wkSFn>)o4ru!kXTHWMFR12dMcxFde$~6p;By`!S6f?u~qg7#47pQa*$%V7JF|> z=gge&ESs{mQiT0~4w79_Cm5;^05dqW6YN=rJrkqGpwc$v_QR^gR__!QSXA9hTRgrS zMTrquExuM5%<(~f7&j(}DSTcIx17oh=AcpdgxgjiezuYkTUCO@XKi4ruDRQ&K`aJ%bNNNAYgO_Dk zm(cfc9GJm_I;k!2v-ODac<~>__@!@-x!U^DB8%zES$o|gAWncRW-Yu{bP=o5Tqlo$T|!dhYYT`@4QQ{`vY# zCBtW|BE2fF0@NA&P|S9%XAab$iluUf#{{7>{JV++!S2eRjqLz5&l@TY<6aW)>^weM zopxmK$_72jtR@+q_7;+G|q~0_`6ikON8-`KHn=le%|WP*6XuTS>v-N zW_voIqeV61eUDy5I?EUTUkI4AT?XLUF-ygf~Pxg+i?=FtIZci>u1FtU@ zy6$hvtQWhk56kkF-yYuIUTJ*fJio4K|77d}q6_d%L#NGK4qU>?<|PUUxZS%p(HFup z4fMUz?=|S%yU44YuueSQD7#i#q$sVM7BtM#17A%eY*mCZk_}1&2tT4S;xJ2DY4*Z3QRVcMA%vK?a$u=VnINft zuOPTe&~PdsczA#iM{e_~FQtiI4xUfAM#NPAL?RaMN5F`aLBjaiz6767<~9DdGc3Wyo)mD0Fsj>ZlCAJr|v?X+i1a*@yOhxYVtm(+Bz?1JljVWL6SbXINnJ?@40W_18AvY-Lf+H6-EeRW?BeW4p7&V>Obc3q(e z4W`c-s4-l^*HwvI@-Hb24vl+<>u0q-iZFe?v#t!Yu^uC`uvRDn>Lbt$VRTy6L^tqy z#h_H;v`fM*uAX={&NO<>2z`XmqMfeLA`$yn+R0`SM7c)#qMK66eqcr03SJ}{ll-9w zo7~|glgvI{P&PaY;1;&t*PN#ezR`tso4LO%olHIc?ND$9Iir-o2o8_89xS;MS!=m; zRq*BL36D!vl#W2(sRhuGQQb(E%!J)VG($B;- z1XHBNiVIP^Fy^NawBrY>V!%df;}e33SwgPy_`SWAgK!mQtu!l!^@YTRDRi+ZaF|sw zDEKH`Qor9~QTWLl2H~a!#XLu~+RuFlwo0L~lt%>It-|Pw?kKFz&@rj_F=z!Tt}i<3 zpA9w+HE{D+pAqtxYgjEF6Y%fwnU)J765I;Jt?1JqfK>NbfvuCJvsgkn=r=sGUP|_t z?PG+LW5tMVp%1?D(51v3MZ=&4VB`(mJV)>aely58eX(?x-Q{o}1jEKdmxhwqVui1- zs|u2IUf`lL&1_~Mi8UsP^^l#G&?JS} zBV_PxPY(vRQ+_uF_QD2&jD-9RJ)5`UCUWceuR`Xp41u} z+CZcb`oIrJPrqmm56eVzbfj1g3~RKn|C=;BOSL~f@q~qdo7=kqppdFrAM!OPIsWsD zgh!iPx7=z_q<%IAp>!(PjnyuoIbWN&=in9NaA@Ql<^U@r84&J@1#eM-621?;LG_|v z|1H(;2sY2cJd(z_Lf@6@-wTG%q7l|$GQLcPXF0+1(ZIoYgqLW9wg|BFEemRoT+1^; zX*{L=Jz~N}EJ6|}p{1ya+?cu`Avp(_FeL)s5b{MX;gWv+apbB8JYNFd9S(AW*LIz9 z4>=O`yd5;2VOfyM&PGR(j>iK&hk$@+(yX{ZEhyMUX2}LncW>{2LT`*pUrmJD&&UwO zCiX9qhx5<{rGTXxF^S>zXz=Et20XAQ7X2kPeJu)%7Og*FT#n?9k1*^NEsr1@ZF#r2 zX1DkvJkhrs{S+)A>De{nOuNG*E~2zr9?t7x4t&7 z&Dn;JhKIL1+2!}<7wqR3e5P|+HpAt>rQFSvmPP729K&hawRMyX?+_sPp}rt^?}1Pj z1g9*gU70LOgHe@ahwQ#g@i|a7UwcwOtf;~}1A3>PI6$hiWWPYCN`eYJM9g>g(9sxB zF`Uf0F)*CoMXpLhH6@|Qw{Z?EKx@rUnM*?fsv!4apvFQ?YW1cMg(ZkS4U zZvp^E7*Sh0LWc8Vv=4PFB&x}aBNZDZujh`Yj>*-2jI940*6K8ByX z$|7JcPkdIKx1aR8!_l|pxXk)svRq^!LY2}T0xmG0zK9t5IO^6oDhd?mT%?nwN#*2m z#nF2!b7}*K0@+|T3)aqn^{*1^V zdSte(>ZcAg$mY97WVY>)<<}x9pCmmDU%z&W21GW$VOC&*P4=e~7WOg!8+YQV68B#! zRYT=+4bz{a6Z%YsBP>CV^FksbOihSOQ<5v&8WU&f4vFP2r8dml|E1qMb~2}i7&Od* zUQ#2j#c~a<#kzrBd0-f!Ci6Z7T_JpX<}L*4r$C$sDGqysM^MP8?KZHrNuM0)kuQmFh%IX! zI;f_E8Jn1youBvuA5Y3bYM>{**$(aGV>N3uG#UW1|BHOaz(B>oO~J&F5XO}ogK8F; zx2UJltuhvvc0U;|4CeY}^Tti{4?4AUL^zdFHZRqmH)>Vu0G;>hQKfoy8HP=Lg~92v zXm?lEXf(%*ySrR)%*xP+Z*PMzVHUJ@-;Rp0LROTHRNO8a{h9`0-D37>K@nL%Ur7Dh z8i$eirH60JXEK884?c+Xg6{`MthFmufRtxqaRHNrI`qa3z zyLsz9&Au&Pq)%L#q}JEs`s(^?sGMkTT4$kaZq??y;5u!~3<=4WCy)Gp1nhDApe$AY zV%%4p?~A8{s0*)1E+_xenVU*!HNyEA+@X!ugZ#LkINC;~eUatvQ3kMYZ)IWmEt*oA`+=vU7*X$?r0X&cI<&k&kS{)`n0aPhxMAvT~jQrO-%_%g|SLe zdvxj3v4J{kx{*;~n!eU_&goGN$qXFjf)zWtf}}nd)Ewaab=4Ks0V7||R?RvB7H+99 z6{!IeRM`OzDvlN@-Tf0@Dh?q;p~bz5L38=s>DMi5k5A=uU723jY6EzIas$Zqe|(*J zew&&4Hzw)Q+7s=RgV;)4i0&DBa#Vdtl(Vb$G@e5_y_x13wbZ~Wb~$9NVRlmyOe!+s za7=8D5l!)~*P;l|d<`0cZjKfm))5khURw(wBkQ*4VYif)j6rvX4W$3H4or64oI%uWkvy(^O#GsIz@)jQ`~cq0&2v6o z;^d|!R$bZAvihc_WnId{ZHU2*4>{{LET;ak&@x4+K5iX4!E^X{N;Tu$K;*A!?y7Z< z%hM+PRC^U1Pag9pKh`5;JO{8|G4!%G zgAvG$`D%*}-NK{G>{KfJzR;rmprU<8O!@F_%Nw-Nzg{boa;z_TsS# z^CllXvv2&;GuWgEglKHY1a{TphK1^dj6T4$rCv>+6sd*cqh9Zq>m)e&PlRJh5Gan~ zKTz8ZwbbYj>w)|%E*D16@X)E(MKo8(@?t=j#&*@RTLrCN+ZBHk%!Ji&0^#)F=_|VT zz`T3u##7gj>X0h$vjL+6aK|_N9;1WBw@`+?!BF3z5emuf%??r4c?X4(We0^I)Z}w& zRl9Ey)S8TWq8!hzW~ODty&Az9G|c@CzKD|meNlWN>@oi7FX}l*(^$Z+SE?tlrnxpa>p0C#x-x|#`| zyq$4tfaZy|#rf{;2#%NGhYcVrIEAXGU%GDzA~RwkVMIoj#ho+ciq5O#io#7Kyi!)n z@E3>6@JlQuygV=0D~kLI**Px6kr`@`r5MFbTbf39*fJ@^lUbbEG04Xgp+?tx9J4WC z#uNR*t@z8iAMdDG{1xTRG?Bo;FqzrCV13cfWx``J!0{~P+v z9W9wS+MEK&)A@L$`wDp~Y8^xCyE`4yxw%tNY3sD~Y3kxn{}FN)Qcv2oK_(qad~I*xc}eZ*+WO(^sR9|8yBYoS zSmv^?F?ANLdO7c;!QO|#U^Y)1!a^0!!oSmBPv*vHq+QaA3zeZ z33@lFjs*tlOvXQ7qq!Fl25z~RLo~ksyke>WNwtMzDV90%?egL ztp+O)0plNmXqEN_nXsmRE6POOwsa~TE4sVO2quYmulHe{M24PmU)ptL##6oOM`)3L z7ZIy`l|xhV8%VIzvSLLQ_YAhyqIgX2o@6Fc-WLLj^#xNH6L}ahcMs7B_hSb1MIh(M zECNF)T?&-WbIf0}*I!d%;{2TXIM_@A6nB-6e=3a?xW*`Q752V}RG_ov&e!3)X$vT? z%QX>p(^~>~d+dBlBpw4MU}>7rE%16i_u+l7jR~pHUS+L;rQ$8>raY@ib@K6$TaI!> zJmY>h%8)!kP#^8-<7gG}i~;5+vqsWq#z!;^Ki(L@TbGIGtv)Fq8zn>o3FxAMFGHq4 zMVQa|-+V|3JeqB{mctq} z!Fuv-VI;C**_)yC&B9Rn?t%ZnUy(*K>c2VYJSsoRV$z1viTgVa%}ob9rt#r9rs}=j zo2M1hFn>BwoJ7YHf) zdBKEp`52_OX{o^M04wN*#MHlj23Huf(8}w{59*PELO>X3JvAV#YEwcY(Jsc#fL~nS zm#!|)?w)w^wJtnG!S(sn)AB?RZ%`f&!%{NxfLUF%JY@IfdU1#WDpU^(19|NTtwq^* zlr^%H<P z_IPJvo^qzLnzq^G=(sPNg+={76$_r%iRcy`JJ=Sxm8XKOn#N<#5Mzh#oZVrkWBWX$ zZ@kHS8rng*OV5t?d3qj``Pg9A zl@(QOhEJCBh?e&-E~=tDu%3&YK;M3gOBVetKPl*8!zJ=Xb4~YTBE)atNWA4q2jJw; z$^Xqgwdis>@mrm>&S>nFpf|%JYvHq%T}_5=!%e%~wR%g;>%eWIn?e8NZgrOZxehHM z{cNj-*OsZ{Kk$srSc&ll)l0pl_T1|X(j1Z7b2ooO#D~Vhw`9qyl0#)V)*Pb&Ke z*Pmm;;x&rU%kwel^5jvp>dd2~-uCZ->v()r^XMp&^9woCbv@&LXcnZ8HDs#5t5kFP zTlGJ2!UuMlH!Xgvt8nSL&{@>I5u=FyCZY2U!(MK=`yzZGzhIB+08P5?;URxe9&|6R%;FaL z&-U9DP8f@WEtZ*x5(qMC6AGzET)a&kfBB*sGrYmcOrK5C9C<$4IcLSS#>9Qt44AP| zLG0)twZXx6wFg;u7`P?@y_(JD^u-^FdxJv@{KoDiBG`o5y<;-Nx%~sIvbv?d%q=Ng z&FxjDpS^q`5sy;nQT~EF1RHq-+n<$04|&yNW(1v!Q$nbJ@sjv`;C&;Zis-Ht!TNNP z^{2x$1Scu%T;#m1S*32a$%U)#`1{Y@IQ_Y!n`UYAXREX3JVfHmVvl^4&p8;%tW@Ca zhsVCvTF+H=0bQHXi4!p3Eba@VcW^;zy3_oIw4dAhCupR&HWRC|zyE zSAscEu5Tgf`##kDomv{gi7l1`)Q&2HqJ-&@h13+%;#>8s<ZK+)yXvei*f{`7iHjGi?9zCuRK8J*zZ>j0xrdBk3Fz`Bpl! zMjEq5uvtpNe-YA8^FW?gSn>!I`F3azn^^b5VHD*dh`^Hf5$$GU=t0ts3Tg}yQH=BB z?j0Mnm_whS>0oTAs5UXKIP&^D)6A0;MZS!)Tho&T!VHAuOk&B;PMH~lYHY3T8N2Q1 zSRe_T51QsdhrEqfgN`l%iVgrpVSu7g;C~UIX#X$z ztN|Uxht7xn1rs@o;6DUM)uC-)QRK0(9>|pFK`77nDDuKoW}aPWG5ESc(}|g8t5+Kp z-+)E$K~PZ>l<-0Uq%hq({z%SnWLbP>o^O1O*8DY~^CMXD+m0|T?;s=vj-+!iEK^s* zv{Z){H;IMw_c6^zh8U~gih39}3u7ZNqD!HR1h`i>P9n&zI*znGo zD*t@g6N|?9j%AiRY2;1&L#(EdgqyGYQ0!wmUpxMmK2;1YD1{sf|1wN-Ty`)DkAf9A zoF6*Uq6nq%!D0!KR~=+T;dic=oc!JJHrR>7>=}u=G{zP~-@`O!E00L18xf))Ifk(r zj-hKYr^oa#Cj^`E+YMN0MvTzAx1u$pp;H+55|FQy{R3;=DW`Lr1;3ucb|0_Cr+DV3 z9L+>&DrtFyG_qlnrg@Nvhn`q@w)u9c)xcu@*d`P>rN_ikS*;d~&?mg)vdgpBc3iPE zRJndv<)Ci09~iqpMsZD^MwS^9i;7(29qq}H%QZ_6vblY^50p@?R=@2TzkVrXrHSH* zZlBZydswL{F87;Yvao4aYAMT9GeI_UL2C9XsbfQFLq5v0uQ>K@GdT8tz%KcYAB_#g&~dIN?>i6(8Q*=^@U6f?-dprOk4H@BnmBNEGch$rUU zA~#*rJ3D;}*~aE*W#bs?+bl7M!cLmcoO{`NoJ8K$2Ac`l4MdMsFbSJ`4w;8cZN}Gl ze_)~VMQ%VJljC}m)=J-u4=$}uQ+iNFuDm?RcHrZFiB4sWNtKf~f}B?nc66*%IW;55 zA?J{i6OEpV*D&oCjR#o@nM02MA(!fAjgzeRZm1Q zWCIghLT{d}6a6BOR2epXbf^L@XKFTUr1HR#N7SJgA$QH&WHChUIPzGfAw;D}b$G6p z{G4k6ih@-!tJ3jzhq)n3zZXvpr$?XGkX9*S=1S%o{>H~GOcR5O+P7{sZ3y@vrw?SH zsuxZ2H9{e^W54f*HJuReb#2&C?8`bdv@9CP+jr=A_2ezF+vCq8ZO zG+MtlIX4Vi?kCfFwO*F11#=71?{Q5B&BXhr(aY>!u&RI=euvaDOAX(Zl>tGBax|I4 zl%RxLiUMBgd~NTx<(AN#YGLKtuU^<2^iq-NY;GRx%M~R*5e&;5n)kxfRzJ-`Oy>k(F^#|>$Ky)k*6kPrM5eiw8l3R+^!JJNT?p^{*&pGW63$T^61|aG$>8jXcA6ym#oD2AOhBsB z+cngt7?k#>u7RP2uT@@<}o6l z9zp^PfthK#qG>J?g!GPEYfh-crQG&`qZys3qp1^DLBYT5gyth+0&^+OHl zZu~0n&3`t-DL~pF;yswT zSzB9~+B@60{J)rQK97-pFs6NW0rMF~fw2@&wL8XZwh=HBLvginhh} zh7eKx)>QfD;GeJKdj*M&0^ZPxdF%;Y=pX*Zam)vye9s>by+Oi$fc~F_;y|VlJP74( z?dFBF5Vrq+GcMv}ALW80&1%_sl(;w|c}K?7zA25X1Vj0?iFHPaIX9wplA~X@BJ0Ei zC0pA@up(=DsOjGrl#PVckr97A0%-p*QFeFv-^>QN{(+(j}5{%VrnUx+) zMaKcJz8*+(Njs#Z1b?(zIi`$~jQEMt5C7|u=LWokHXdH$p-1to($;v?ujTFVlOAtl z^KzT82`VV5>`=;#-24$k8*lp}_hl4GCGLyPzI7+(t;kd+Va@ZRm<~H=EBJMNd;tO#kU9QN134wc@9F#62 zY63w7eQ|+yS5K{qC6^w1^`$Q++B!cGa2OA(g(`h`#F^EG^+yavSI0iZ6N_rpLNIIq z<=BbrSmrZ+X$^y?yBFuPAmg0gIpDuWokX zB}B#{+?XPW?3|q|=fNVE2VH44yFheN0DuRL{99L!X8!*&#la{bk@CUEuxF{mIJR3Nmn8c7Jh6aDNdvet*1je~SDe^Ze<$-@gAH z$o^aM?&AK}%AM71g!b&atBbC0diQPz|m2#qE3YU?xyzpS(_FkBPE>@bmCkbEwoh zmdd{4_mLnz)Tw9p9Mg6yY|FjdD$KMVQHZ?{U3!<3oG;xkp59ZQv^-met-i5l8Wn=5qSdiwM zlt^PD!M)}{PT~wdhmXAW8fJ{I+OdLeGwd6AX##1R-*Qgtt4uU?IyLW_Nd>?=7cRe! z1Z5xfN!;WS#WwvQ4nNorR2J7`;s974biIUt7v%Fwc1&4mU(Le#R2QV*Y;;Zf10?)d zDc`@kLYX_St;W^oGbd+}wPkD{RN>0ms}V?iN(Vh)+a22&ZZ$Kidm8kb`ou?!{S{nw z!|4Ry==~jDsoMl(Jy(9ijPk{omIIG_ydNiT!?gM*x_h|hFL1hR1?<4l&4()pQ3v+l zE}j_u>|(XMn2C0hi3D@LfV#OyansXn21PG~-36v+*zyo=0G!-OzzJFe^z#oTWcQ>Mc z$`}afK(m>*q4eWXY~Ve>Qnidv_MiLV4EW)elCES!727o~9yhl%l-u0y!s0$ymR&Oc zRY>58AA!`z$+nCUD9gT9bJ#0nigAuN?L1ogFsV3V!m;7oIj#IKsJ{^}pRCkq-qn|>%T6;*FTXYn)TFxwo3?&`jU(z* z)-RB*jo^zjXB+~nR%{N@X~s42mvBqor8j^4KAJt~Eogh1X05z2m(j4%@La;j1_Jg_ z3h8af&&>c`^MtP!DOecgwaZuwN%-wMf#ihkHL2(%n7upg%R*1S&5_l|uF_eobqE5f zWH~z*I9U2|d%jP>Ljp2Dy^080g&JPeas8)PO`4DBtG17%qXojubr#=^oUUCCti=upG;1$4Js9EYXW=avxA4!a>H_l z0OXY`%Z$o#@k;~k#b>VUxl^jm&q>8Y9>))skvIIf-z|A<#oD`E`7%>sU0fekp!j8B z*_OybN>~}}xrRer3Cl8W4~PGdA)h0)=`(}y?eI>~m+yB=`V^q^S@+mDC%FxCo22ri z%Em8Lr5mpCy0sS@f8sz@#sZ2PG>QgKn&pLcO1-@uQZgoMGn_%*jwvE2vq4rLZrusC z(sI1rDPXs<75MLH5~XURxoOC8+UpqRMug%_GggZlvZ{va>fZ5|O;ihU^F4LV|31;# zOjhW_O}rfFRohTCaMkQT`W zo&l~UrV6V$ka(gk=M}f;AnX*PoK&tJD{D>;uU5_+#?7tmZZW;) z3C`Djp`}EJ_=QifO2!&ZlElz!VwP8G4v{N6cpV858BltJkWWGLB5q4kt;gezL%V!* z95VvHBqW_u5*=xtnQLDa6P8h8Lo_cE^Q}H*EwPfMUN~+Zrv+K)J{C`EzZOyC{C0-Z z{S5=aSIM2N7y#A%qU=pFNCYvfPSUG4pm|r$9>`@wq~cAh0w6m_LwbrMo?AJ9x}-hl zC5U+)espm^Pg1e*`yl6JJdT5dACHb;lCaqmevWqpkWCNn&G*(@xkM~THEo8l{fB(z6`#Zsj82;3Y?0b^@Q zr99$yU7J^ge&dJ5@%cmK?0;HfT8Poe*&}s@ijH1FSt-p`_ z^CQY~)r5uNPn@mOaECmsf;un)r#bN@hmCE2mQ=l9l8W`m+z@E=6xdW|{Ev75P#QCK zGiiM7D&U3K^}d=Cw|PT3N^R8wxW9}hIx^_hv?6zE>fW?Evt?JJVK4ub)a4Chldh`t zrhseW0y6E1MU;>#i7dFN!I4020QXU=)p>P{)a47NR;!TcCsCh8#6#F}p8t9=6O$+Z zX5@#;h-tFArLT>f z1iz@1+IhxA&$jqQQs{|7Gw$G8V$lBZoiAN&yrrbK3$%XwMFX4Hx#Df^Ib4Wb>NPZ&#!IL z;*07kJ)y<5CewH*jcBKmu$l{~u<<(-)5&|CjdePdJsobp_c76q<2bnkB)O+CE@&QJ zT^qL_jYZ1`nl|LR{`vK0Mpfzq;k(#Rp@`E161BL5bm{=ACH3*mAR@8~Jb>6xMMhA% z$F_hOTa%8!ll44D8n_8xjE(>bpEpFv26X}YkhP!(<8TDGAT%Kf1f*|Ni^+U_sjeNY zW(%^I`Jjmx`g8!?Z%1dY36FS(Ogxv+V-%HFPML4boa`RYA}O31%b|puE0mKIHr}_d zdIPg1wc%f0+Qgv|X=1xMsO;QwX|HXk%4An6Lv1R}DPbbt8mCGs<3mKAFS2&RU!L&v zSM2L^*>DKOq==*vex6S;>=S|OOatdaualKilvSYui3i|R!5`;x}t7TU91tAN~dYWxhA49`6GMO)T4EF&QHBXbnm&Y*`*~ZEy); ziA{QpoKs8)8~29Ps>Dl5t*8(YLXe!J9Q%2*6up%yGUt0nLeL{eQ{~y0NgGmU5sKz= z6_~xt&o~%7=_cnu8IGaDTHVAPjAs{Z>Y;D-wPcX175a~g;k2oyU(monE3n0;xJ z7sft@StY7xXRcAkJdRNkb>rZ?=)peW%n@k{9uQD<4kB155k&9p+^3f>QO-@Q^|S&P z*)VP@IlZgFWo)c^RAeEXEFC)UmfWY?1BGkCQ^XJXo?s9fUW|D(TjWv8*_o`JC-1H` z_uJBNuaHV-^1~}u!(cf?kJdIe)@ zAiU3|YrWl%lo>*Y%}rharC-Q2l3>q0!<`0c>9CpAQ^-K3Wk{r7KwuQ(vfshY4pw|c zB|mC#v&DYy?Xxv5Y#fh%?)hN0`LIo1g`cpMnlw%=Cul65}Q*;1Fl##^FW8kg+t5Z75)a z$9B3x0p$H~coM~3L zCE%Yex1$WmwL_0+$~CXZe#go69l7b74{_bc`F z$M<52)mo8!;@Tdg^VPJ%v)d5IS=K+5y0;oLMpb%Qp5|f0ILlOIeWV6d=dovqbw)v z!+->5G-iZgJ9Uf&BH4*w>Pg3|<{|cxQiKt$3yx4GQpU0&2w5!yeEOvyzV55rN@&sV zOF~tswY*Ah0#2q(Dg}4zMqj!xP{44oZk5A5E`8eFmsLzVJcWXLDD422NL11mUxPYb zRh(IB%7cL#pN>0)7W7T7(00t8r z#TljF-S)Rhk<+SrgQe_H8SA)JyIJSHJSDkODf=x{nd2RmQfXw-V>}FW803`)U8zP7 zl#D6Pk}Rt|$*NZ`s|#kt%RDV;B*xX_?37@W$$h~~tSN+|BryI4LZV2douvBpLyRtC zi@Ii#YV%52eV=shH*hhsw>>kN;dn_NWa?@EU1&B;P@Lj z$BM0z0D|0row!zqJ+jmQe?=Rmpg6-OgXz;3*FT*0EDg3_kM#4eQy=Rn(L`U3z$$h` zL#c)>dUOSI_|VlEtOs;A6I9{qdhb-3eH>(+aFciB1g$t!!4pXp)c7>OPt0qd&SYel&JeQtJ#vUuUy9l{cGY3{_-MJzi=J(Jf z3;xg_HLJg5l=vwMW6{iuldbsWp4>bVnOEeGO`Y|F)zZkS%J1-Lqg(JX+E%P6A@x?r zPbOR2;F+X~A!6!X$-u$nKA_hU=Vz`w^pk`?b_GZ0&0@5#nG~<8y`+Q+S`fv-n;ROJ z-q7=nT&&k&pP&Vpd3Y|Pp1V$%`BboYdw92Fm? zl!E4XE3#TxKvc`9XWd_|Q;gZlTXfIb#=Kw=e5FO-%#c1+IwauK^m{a=nb^#p+ru6B|I~laOGM6#CD$MRdMEaC_i#(y#EETcggquac zV-`WCk)ZAxXu(i*uxiP2`Xu8JlXyrD*3)KSsHms5ishkr|0gy%)+Q^sn|R-Ogt~+M z*eM}ffs6%mi1`v5`Q-F}Nq3}^w4kd^Hl#d`rvW|MAdo*J4t7(O-#jQ_fW{5LEqmtc zW|X)nR&UsvAGPrpdE~^`+Oi3V65Myu+nQ!fzZTghEcu$so)fV`jiLIY^lY&raS4&i z{#3}I|HRW;e#nh9-haM&Y)9mKhuMmzjUY{w)SQn_sb|V5g}lD4^0jDZ)@{2_X=5H! zQ?R#9Bc0!4)VFVJZflVWgEmT?MtS9EX zSAX;z%QU7yCI>mF16#*1{#t0j`Dyx>dIDw6m+#oQhbr~d z!#QVPAS$WgSi;mi>6C!;d0WURd}CEhyih=DlE6_X;ZZU}+`;@61}jR#ZJ)YYE6Ok1K0S3dln09ePCQ;@ zY8$4p$#Vhv0ge8?Dl{cgk9d}KzWVjYuSrL>bkI@Bs*B56cE zpyT~~O|8}LHeSof-9%J{?I4yawm1xQpcAjqy=y%%Gh|=1)=+2h%c`S?ea2E5qMi(Zde&t{?|FRIrp1`f+~--_iX{&$rx+CgNhMLlmu z`e;ytlVm=m%@g7U@e#aJIS_%?;^IpCHy0?#1#=G~q0$JcRQSI5uEI8&2+Cp&LDzXt zSHltz2C0fzp+V^krApKR`|Eg6I?x2#V?@HjQenU z^(D@8##LQft_=Q6;ES*Fm`bH_zes~?zgfYRK_m3nVXBMLNh9ZL8;6*z=k()N*T-(= zFd5Gf!DO|?jZ##UZlo{6==J9(tP(i(rMVf3FWnurcuyJE0L{KNt!;J>T^p1Xf#_@AY zXk6&NbY?}Gxb!QW?n9ZviZojO#+1e@Kh;gg@`#Qcf^zmp>5PnBbkth@@tQbX+G z$SNYn+w}H4JB2BrTOMOPgV#nXmqjy}5e?l`)S`TtrI(;E03CQBH}3p*L0co;H_IEyFG2m?xVt}C@;9NEHFD7 zcWNeJ(w(IZ=UO}cj`Xh{kt9^Sh400S(dhqE<5g`0ud+=xE_v7J^RPyMBA z*sTPL;~GIX<=*|uqa9ay-Q{fwrUiCap2J*?Ps-cha0Of##kC~weFCZfrA?In@q`1K z`$z0&Zu~Yioo+R&#VU!RHc0}{d5L+Hask|a4w=FztKaF1MHx;5#H?glBUz?pq=LQ9 zQi#&S#(Mr0BK26H*~Wo!-*x8dIHB1dTpy7oog6Bm6H}t^#VkqrdCY1xeh^SP80rW( zXM95A>!po9Zht4M@9BZHHFr@u zS++-nJjcglAeEsGPm@u{k{{0f*9i|M+L@)>%%-uvgkTwBh|;qtErFu=GDz{=U>Y}4 zoxgd|93s0#(E-yx&~fOZhAmMwm#(2sbBbgtnW|fwee+Ugr!4(CX}aVW){$Nhc9=4+ zvL?7$d-%Wiq2$$7!~`eu>Cp7DsrlJs_N03o$el(~PH5cKscS{>fez-fkr~|YNICqU z=L5TZX!elM@@qXtjz8%P>0$9^nr**50(H}LD*v4Ro=e>MrJ1Pts*&}ugQx(SgVjg} z46Qy+(@Cpmiya)o8~OznV|<>8>vP=RYz)B}xDs>^j)|95+>CtF?BN4?kTzy}Zg^_N zC4_13Ev^A=`GaaTe6?Nx$h%m1+Vcf9IYH zQK@;u3UCa;SyJ4$9+s?cc%<1342Rs3lRtPotL?)(lqZS2(F`Gj|FhP-%PB5<4INaT zDr@_m2Yn6o9_TXt%H4!UClGx=JdsErGF)ka5~$xoTOD&HMC5fiJmz++ z(PNVubCr0kwu&phgWsn8#fhrQ+blYmZhR&*g_ojO6KhJd`=L#9udenKfsXQ;LQCTh z3Q^flSmNkP(2+azU~gQx?EJS=I{TGKos3tDd-B!Kb!N31qoYsf1;S$hoNVo8sMV$B zyt2&Ck`VPK{okQ!Ca}CC77}s_y;&`kOq|rL!qkg2GuXc5g&pDzfy&E^2cst6`;5}f zU|h0qKaCK7!PPi05P=sU&ns?dKj(3NoB8`mVS>?$&?)PGC7MDC@Izt{#4*jxEuM9` zy@7{LTbSNWr!FrXcTIiKzt(4jYNV7ZFWubQqU0oTPpma=IGXL0g+yE+hgw44Y5_P@ z>G)CJN@+yawS4>B&z9(5P1AyZLS3HVx+MLbVLGsK@Zuq542$Rf8$wMsoF#^hlIUCJ zJ45$TP!{Wt^fJS<*1m8ud7P$I_gDcKSfZDvC7tMFMa3WjBciq7 zuBaqa)_-hBcD@=zm2rzR?a7*n5)|*Im!%aHt@Ax#%nadiE$@;jBDvRmVL7W6@M|0P znR^JB43v8OJ}N{g?#j<11rKILzZAmNi6+RtrVNVZ9p_HB^VO5H64FW2*26Dij*5;; z-=tf0kfq+_(@oPp7YmMWU(SKP_OPVD7h@Cf*PGQ2_(&KDyDEFF9}J@n=fT=Evr3;n z0b0svisp>CTnoIwajLtu#hr2NH0gf~3v%FOw!Nv3L&m2L^X&R7Gx7_{ zviEj;!&wQ6Vj1YkA8-Fh*@F~&A}V)E;Nf6a_8Ryszf^xNgSX%KZ&LuE#F3e-Vwnbp z?QKPh7Bkq>CBDJQ#;PW4G?_h)a3I9yIY=%%7Lb{iVs=GUA|Z-*a70f`EpP(J$i-ziK$17opSP%Vp{=gI^07>a=e5p6L_)5;fG} zGtB}VuqQF3Q*oWDPH_x7gNM=fU&mS87=#UVxWlwM78+%B63OuQH5Y#eA$SoZkYwup zfK*3;V*z3U_i!ALm+(^80RDuG8wYA8m2lmlQXS#Njj4b~IB#X1lOZAJI0~6IeziJm zH&SAtm0LB%`EAH9E=kvsPS@!oA5D0luhv{O^^>+E%Bn1PVv5iXWEha@0wV+=c33)6 zZK1ihZwc|On~{>miw-`Dw6)zx$sEIpQi>(1h$KXM0ek!yb3(mBWFyS6+5vTfVtmh(3~EcWr`2n8<1i2M*KB^ zr7($senN9wdFF*nsy$p`mUh-zJ~1(tnV*LtN1JE^A6JB7NVYA?*elNl znxp?&h!$gP4b9O)x$CY9;GTjF^n%#9Lt?>HTzHNdqhai*+t6SSzwr%9;1%gvFco73 z?GNf+Qr9&+g$-ePLrtKAwGB_<=4S;;(UHsNj5-uH$Dp~o?f)Y+;tkVW?-twPcls23 z>dQc(_P#_ij2LePE6K%DUz+P-Kk1!Wsemp6Vcf9km{Daq$Ya4ewXYBH#r zN&BwdUhC(*AKd)NkDiTwO7P#|0D3#x>OX19el#j{|5R+nI{P!@*PlCy(Y|8EIV?#g z_CX}{U4#1XTKb;izR{}ZkqGu^Iv7BsT6>5up4-8nV?#;QTCrb8|CS`tzAuac3IsT&?1QBbukHOKT5cF z|N4N9{gb+yr>fPNgbpuzaY0PCcoC~8@ps9%{!mnXk}VHw^tIK=^Akm19cQntmBZP! z>`y|>iBBPU%*oeN%6JTAO*!BxXPZXvVcGYwWXZ_PLti$_?9Z&1U{vCA-}^1O%rp_n z3CW=Vfnokvcm{-4@rH|Pg>{aYeSLCwhrj0&Oejg3wnMK*;i(;rJ8MI4Z!8&S8lDhN z!oLH!tSMm39tiO)zP4jVOq-x2CH#7NvEb~P2ij!Bj8PpQ%ND!!&%bZNgzTb7Zj-%~Ea3CgU!YY8(=XQmeI z5l7!x3^sK2^?X-0!sW5)0Xd;Kcq^Kb7_Hi6045-GvboU2K<`6KSjGD$ZpBw z$-`s$l7n9{5mVz{Lvy1RKZ#*x9^f%_NKBZEo-Z;B_^l^XEodv$s|e<@{9qk&N!dwE zy!Cw)!*Xb#-#rlH`aSvFFr~S(71A+}C+&;7wLOaRGyX8fi8CNmMU-c`oXM$)w*n z#jk&a{)h&TEN#zG)aoO1*A^@Ia0RpY>JO*RId~ihwc55e?MJ3`4|(&KN+qK&OolVc zeFxXj;n;XjCye`2?d#9E5^0&UAx0d#;aUVb6zoiCS z>?fYjO52n97ScgynQPw25`!kN4*(~qcA-2sM5PG>;4&WA(%Q#EP4{bNxv|klKz@u_ zrHV90C~gSN6wG$i94J70Vogh%@Vi&25M)Fc{lhZGIPT9C_-gg3npny95zhBc-Bi6~ z#J;j9BQc1ZQU+0d2NuxwhacUK2o+C^$y^-MvPd#R6+gQunwqI+(q<*TJi!jkssFK% z@?Gr-fEme$ymZ>IA-S~vDmcon>-d~)bGwb0=#YUTsfun7&-|Ebt~pmgqRlL?b)(eD zt5gxcd3A=kwXv|;K#JgN)e1QMZQx>p#SbBIKx$IJJzSt(TOa%yL`PG#CpuJ8t(C35 zw@y#8t^me6yjHBO-dH9XpIW2s0lRplEkC{a;M1G+f(qc4!nV0g8*P5B%$9Ay3=KzC zM>u7fND(MemF%3FnOQyw3y_I!470Thc=D4D@Fa8Q=v`yaz)xGuz&wp*KDg<;vK=H+ z+fY43>x5UNE7a(Qon7C8;Tf?f>65;~dl&W;k&Q-GC;?H@dLYhEfm3ufZE1SulSKnl z>ZD0-3!HLz%qHfo^*R{8%k@z!DHoYdm$XDr!If>v{IlV)DG8 z4BJMDv@PrKDt>3jbmx&NqYLK(6lEsp8H|FBo1 zHkPK+Wi)zPvxSBWmC(FiT5D2_ zzF|n&zpDICR5^Qf%(w=*IzrJz?rz{w4@sK@>v;V+f{mRHzKS*THxM7X7>tQs98WYQ zi#^&%Q$CqUtd=6N^D#lD z$34ONBO}v*P}SF7N##2UJ3HRfmd3T$>#@7E$HSI(x1awUgm+{IO(qj7&?*p&aZjj& z@>fF=nthd55%sV6lA0rY#wjDni%FBR& z+OQds>~9h~%|M_a-#fgAAeC86+dPSTuT7fJTHBdy$UgV@>@T@qJ&@K+I0z%2us%?z z&cOwSP_6-eSl@^-cy%X=;CeFEIHs!n-8Fi1njv3|8XPFK=f~!F^;>noLv5H`6{Ir( zMAX0gZzAg#XL0s#=GoIkf~r7jT<~qqMjVjg!foAVLDa}qL$Fph-c>G5C zH|&7dOYaPmQ1U0h3_JK+e>fs`FR>vZMQ31&DOhKPd-64qOu}+NJP#`$a|yOXt!RdW z)FvrbxoM$$vTTSj*>tg%FtOTXXY#=q4eXeM#Mqy6l>tgmBJ>C;_gTsGHH9ytw@sm~5u`8tivzjlgu{ z3ztozlE1CYyLaNjH?#bOixt5(82=?@nQqeA&wFd5r`*g=TFVr!^SV}aD2?eaab4E; zs_7uwo@H$LE>^UY&!b8dL7V1<)y0n0d6jc@EeWlPcJoED05Q(W0xHlrSn5k>j5_8y znUudjGpKKs+M2DQCp6C;IvJ;4>7eRiOp5L}*REpT@JdkcqTD*R0eo}Wkm} zBtlnC6L3>~Gylg~(x%S+EWNE`SjtIxZaJy%;h3~0-!Ju9j|@Jl?>-|k%9s^8`t~QQ zVT7mL?ME1p3$CwjBlFHVtyz4*-#BqbUbqMN#^wF$vT3*|Wp2j<@i8b8x-3IP_@5&! zG!XQ~J7x+qFV|f|QkLUFAX#jNwos zQcAxwB0II{&}E0E9~2b;19*qR%#v~8Ip)6L@ccV>x{|3vz|NTZ1m*mIePdsP3>~U# zK*znBZvAA`T<)xswDM@}sGCC)ROTI7V zEf)*AlG7k-3A9K{1)sYBut`g_T}Sgep?(>E9G?%9OO^ccpk9k1`%EFKEXVB%Y&m*Y ztb)vCEmi;%PGU21M#`;X=;>m(?J5ke8kTH+o#p!HxXlSjj7180l}$=&r!N=oJzq%n zXZ@IQ1y=JFjsdp9qL=H>StlKlx`2r}c64Q?kG7;ggs`=261dOluv^J&#l8zu>s^Zc zP~iccG={vU-TE4Onl2=vtr49SrLP|Y>=c$m75^>~MWs0{mx%BRxEe>0r{>9UT+~&2 zx0g*$!4j9g^FlnR;G1ogyGabvt8r3vC_AbWi~NUF(8+h|W~S_mhW9{j1lEU~wQP_8 zSJS!f;2vPnA}%$~dk`p6$370L^G*LAXX^&$NC2rhyqmrwOZLxM;L~se(%`w3P3jj3 zL;!ksDB*K)S8;FGcnNl4s&h}vr=9tRJVz`*6O<%AT-FsxXc+hbAgP|0C_&wdZ-n%{ z8qUZEWCUhBQORaf{g&;m9QvNy8ijK8h*7n|lb&_57%+FhI`d6=hB~o&`8`HS-|VwG zb`mR>=v+Lw&NwtGGR{GSpmkICHq-f3TQgXyrQ6>lGsj=uP?69UEtXA8eXdgJtNz4e z=+yiL_i6R+U*;;e%bgC^__4M0E75^nP*P49M59KK3*5wgQR*UHv z*UZ(Ky|e@R8Yt&d+CAn1e<^}*etdMGu7RWoxdB39Dh6zex+=4duoW$fu7`nT<3tLp z1RFLAU|pWlhrst8^s|m>nTJ%-`d(-d9W*T$zu%P?*9tE$G8J(R@WG2l0>Iz9p@s=_0G;LVM7>JM*gWXJP_H)qSbyp2dP(2z&{E*`Wzaf&X7*K{wTKq?g z&`<^uQJrwtPaEeTJH$WmD31@m*kG$7C##RE@FMHKheA7)5sJi~K9sWnh(ghZs7^=? z=RF2rNfBggpZ9w3iq!l(3fLld#$eIcf?Exsg0!VZIKtW9Mm|tRgFDwm&YvKoC#S|)_etN@ zV^?9lGpEjluF8HjU@d9zo5DGhbjfuO+f!d)o%tL-u&4#vrqkwjPf3tigI#pvgBAfJ z$g70EoVq3nZGs`EIiR{P4fmp835Ng;VL+m|XUn#z5v z%SRl*cT|X^W>H6VyhP!BXFgj$_dv7+TJfDnG@?i7f+R>gcGHBD`%y^CRc|%++rar0 z{z;WyNX-SGYBaf3<5lb*lf#f6;DxcIj>Fh>#;ya{uSH1;v;szWtrnHWh$;fBglQ}v z0q-d{OiEvus*0H-y>U%44`u=+ROOc2spLQGDyvZbLOs?3)YMgJ+@#6e210&6+Qm*2 z0ccG_$idI4z#|2qG~PADFTNOHKsnmh=S`I?$aW8Vm`kzzlRtKoVb<#z{|#da)oEkL%H@V>-rYZZ{C50zM$ zm{V_Mj=!Q2Z#77z0Tz*+L)y28)TI3WJa$CN#wEG4)#=o8)J$DI!0pHI`^jfUHro(y zS_QT0#WohDpB}k8czrpv101j|XRn0lSUNhhp*w{~Ee;h;D%v<`R0JM1r}bey1in?6 zYN@-(slHO%His``mOaS4uv>zao^c!$4@S$i><|`@_f{#y_xX$e0d44KK*``vSASf< zn>MbjZc(AaB(`#xDh!_WPv02{$XN1C!Uze*ZR$VL(2lKAkiP%TDlVYo$$&@Tczq<+ z8n)@8V6K+h(qaaqPX~b??TbgloV?54 z_Rrq&y^Ip-r-0FKoYmQ`Zhu3=<0Gx6woX|J4Jz*JMQB*Ob7K4vabUD6W5faksf1Qj2?x$# z3Q1qQH&Wqa@(~?czv}yps1@Hrm)0J5mHvbgz;&jRx5bcL9WQ%-IC=lWJA>J1zjES| z*ZSqu(bVW_il;qV_G&4hZ}zI}t)|3dBE|qj6Gr@`V?- zHdxiKiRfzeXCrYG9u*_JRr>*0P+Jg z3A$dtdDNsTtvbMsUrZgids>COVgDR81g;}mQIpgcHaM{w8?(o!7=2`a!39*9W z0k~JJ-2fi|{EtI!O@UG7;-AW+%FZwIMhVd8gh3amw|Q?%3~Wk0k2naA*srCap;wsj zhmrnqL-R$u9+g**P4&3Dv66o#&ffxCmnO=Jt8mMnhY3|}T>kuZ zIXOTQ$zj6xJZ^pV(0E+TE+#1G#oHBrNx0^i1XEW*G&7$KTRmJEvqlSkwbM$WUHj#n z0T<|u+&NP{_(0wu&~@+i@VDdGei#fet7Slv0QMGoDZ_lRPS;)egAI6qE)c2I@0W^; zW+R9McDB4PPu{XuNAul9UdCb)C3EYK+aI?ZJ86(ddjj`aC!dZ1 zY*tU!cC)GvK=n)F`&@P@=j;_Ym45J{pKPsd-sQUs$yddI z5XE93o=VAcX~@=_+3sdN`I{F=_VC8Qt}=njJ7!?UsSLn~UBTdBz9&xIv?6fJ=m!G{ zy3<;-pqO)DQys4gr1_9N1J*smXx3BUbXYJ>73&F@gNXyuyVVsPZwPjEynwvrSIm1; zhf&#m-vcWGd)wXxNd%~hwOkL32-e##AMfa5qxSkNSB6n$GgXxan{c80yxfLk!H}qSxq@}^Qy7Aki z$3oL4{!bW&3<&#q%^5r;hFYS{a)Qf`m`^zZYp0q?TBpZJIKP|cOrHz*!~}scA16H2 zk1@NVMM6STs8s<#SNjk4X0g0nRh2Dyet)q}rNZ0q7J6 z$|eU_P>9;O`;0R-T;AVfD2Xan%(8eOT)$!-=MP3{in-+yzllLn>FxROd{VAc!K7ao zGz(4+P{ExNO29LD8yaCyqZjFZ^NhoLsTXgrjj~rT9_hc&e;7o>gB893h$rr`^}|@G zX>Z2or`B2gUL+?3elY<{pEn=2`^P=?dM3Afsv*|8t_z&A`M_>lB1CSP&Ar2Z`qV)B z-H=h&-wBeW=7Sc>7j)@*xIR+@dD`1T z!J_z*|3AAOdp{SzT07z4#52@@M_SWNJ8akV?XBu!5f=9SqZJ1z=7~9tXoCvyFlmQ6 z4*Jih3UAC0Y=)Xxid8A>6m;xr74@h~C zr046fPK7pN_Jk|5?6rH6d-kC-FzICTgBdJI&L^A+GYbP~rM@X-QgwS--W3trD6(%b z$ZDd%@jNELoH^n3h%^Au!mh~}{SxvrHr704ub5^=0Mw!`kz^ztADiA*dPAk&85)DV)zyPqLY`+i=c-p~>y7~nvqdj|%iU)PhD$p~Piw^kJL^VhVa|g*GUkQFSPkXR z>l2{RweB-jy_B0=p-*lP5YIvqddE*|)j3+Hw%W7Ow^*y0NQPp+I0l=fsJ-hE*~0;@ z4#Uq=ln!=bagFDZ9kf#B#q)pKrjjd^D_4+Mx)-ORd-FObF5GUpi5j|u1I6SL)R3=@ zkv^2qGJq|;6HhK2V@+l)Ea!g$c?91-W#zYnM4UgfS4fh&GC~&%d^;nN6*F>$=S9!(%M3W zIPB@0D5IxB9>o$eNjQ83eLQ4rWGPOM&JgPP%-MMLTd~HpNYJ#$KE=~3;5{Z6huFpG zULWFI@re@t$>K{~krHwb$oG_F(H#W}Lr1xNf_l_u`E-e+(EJEzYBPkxGIR4jAR1vL6%*8& zJT9JOP%#7qq$8#c;0RC4m>`3tac~mbE{NphQ63Iiz^A!ZAJ+PBn7G|z|L#)4SEZ?F z{<{gWZ}Tzcx`+OXI{PzhR^>p&F*(xK9S#FN_#&*N&NJA&(TkGuB{!( zm^&Z)X3njxQ{0bbQ(HoUzX9QyONgcD(8QaYD**}RR}W9GUUg#3ldE$hKZHp#`|kI} zF>id#>cm*(93_qhl!z9q9pr-2B7>Y-J2BP?&F>?x`nxb|Oim)7KRI3dB&;nskhMa* zl!WcuTZ48@R&`;_O_rBCqei#o3D&Fq5B2(K40x7={Ej!;gVJH+I?p(ijKjc&JUIz@ zUk!G8o_1ttN?)A7J7y%%bPy454g#Di2AxW{o`ivoPlbODE!x{JfQ_qH*x3toWEJZ) zN9~=F_1AqZ(}A^oOWe+nskP3~B%#rPe;#Jf^m&h^P2G2 z&>K?qQZ2zz6$0!Ba*Dp=OQnoex@>xL_Z+j@A$XZwN^r8(*3j_>KGXI4{n=KVb5H)E zz6~kLfXDpPr0zLOfm*O5AVNEmH(5vvDq-ZjrM}~}iYu?9NEZOlX5LCY7b|7V3_Kag zP)DKkJlGgjV0_fKYPFTw1VesfASAkx6{jgrY97<)sOK(|=T8F3o(r;77X!q>8*Wrb zF=pS?G##-7vKB-|!deichIUTC*p!;NV<5o-2bts)LMP zRU9?(H9~Yw$H&|3f9v`o1}Glqj~X;V5&-d)(mPg@swL!aMjuv-r%2J%Sm#~L2f=}8QSL;opplH z`c#jQTj3}9vyzVu8|m9p0PUGwQ$!KN?Xt5@(gPn6UvGO1hDmn$ow%mK2JenwgpjE)VAj z*Ay8x+7L;kYOi^3f#c0>rjSr~Uh_DeIOfJ)#4yBs0j6tPlo*L?T&_>=O5Avuj2SYk zl@Yv>*!6nR9GO*IQ%mC5zw#c3qz;pg2Zw;^Km(&M?oBr?%qq(#4EYQsOO&#Or`axL z6U*dE3a4$yKsqN+^)5`wX90EXlG^D?-7>vSlQ~uhX7i}>>t}1|#PZ8yvzSMuI8IfX z>(K(Jkf{rq(VA0FWBiaLx>Xgdk&U#r=iFmo5#y7}8?pK^brLubAlT5fyk%`DD zD3?&&IQBrRmDF-N=O{`rz&Vg<&<*#WZz+*zU zgi_=CNycxbG9pb}FhKWEx0$l7orkD%=e}3;oX0vY=`z$=uYC}G;_co{zMA>iXG$Jx=Iu|li_b!F5PRW zmbq1XuM}PDpdO65=5dNUh+(#KXSJjtlh}+W`o})Liu)P9pmCNQxNQ223(H9f0Dmj2 zLZ`5=vuI8}|I{6zOo)=TI4_7nv!24v;(Pi=dF-~WJqS0*L(|1n+X5AdbJTGT{Jb%= z?fZhaA?gVi#nNL0P4H6Ll$*_8KB16MXG4?}VIJki6A5Ylmg?P>n9EV{P`$djL^d7J zJ1>LPEo955H}e8lHLnBlti2OgoGlHwk=zsUe64Ise@8OTQi0Dq9i}S#>v?g&^J6bM z6kowZRh4p_V_jvMKInOf-v=R+@DB<$MKX)pwBRD0>4ubQh10QUR=MOYqpXLYp;Mp5 z>Pd($x>Yc%Y%;x-Z32o`5avS^!A9${G?~P^p_l85o>e&Xhdp_{I)1{F4&^e)W^17E z>HpW&Ud$Hhx@Av=sY@BAPU|BnPPC_ixu4rvZhjMjWyT+wbFe{G^e# zmxiD2Gp4b(h(QT)8Gea5ua}xKpKV4#nP6Y5fCimD-e)W7_pfBu7t=eb7{ABot-OG> zN?CQF-fjf&7+F617P8%`lD%9_rt*&n$4k;4R_@Gim5H^wCYo+~1rm42)L`zBI3gIq)r*~|ayZ_;p2C2mL zig^7D(^0rO=@Idw!Mwg$+ux9uUOh1`UB!Wg^<(ve4&5G*JjkXq)O%ZcB8`g~_%XRR zXXV{INp(Hp!0NUKD=aMRzK~bRkWc?o%lxVM~RJI+){h+t6H^~&_(+J{(HtkN6H8pjJl$;4BXTiTv@#ZPhgL8 z6~h3Ov#e=Dfwf^)+;q_T2kP-`QbUet9wKRWm^EwKWa&>z_&=HGGK5T(ojTB~fi3F15vcWr9q_=o9eXuv1xeEGRLQri2swjjn7`? zV@M8s%y?jeGCeJy(Rlzw$xLwbd)eQ`XRr1AhHWq$NTX!h`^CFx+Qqw5#mfhkAet@B zvs%ixNZjPMhx&-?tZYGcUhPIV@(3Qw|s1hbY;otH$u=wF6(zX z*h!9N0QkSnJNbm;i)jP_m-2t zK6>u*=UJpwniaRa7Qu%6fT>?{>5q4x2;+64%ar=-=#>|kn<4Z>di;v3y4dg@?E4c_ zUsi~Wr%V@>0qe;%>&wlW4wCASR)@xWc{r9XPTGVNc%!yh-?tj}4hQ^d7bRDMVvE*K5Byo=8OuirLKKmTilvEB@Nz}}q_yU00OVK9`Et#%M%=FcJi zB?7akV6QtvP9VPp34HO6dDy(|xLl^GCbbq1V=t7qIDh72vjQFx$=@eC`P7kMts6mL z!mvHb6G)qv2b8&$X%aLb90Dr?JN;PLvv=Y^UbN4Al^eOdH!c>h#myofZnSZ3cf9s> zZ3VU;ABB=wQW#iDl+3pSlrhV4Db61%Yh$zPSbfB?NDly{E$bO503SY-#XXA5eNQcJ zSt3CI8ZLs*W$~5&Rw5!2w{#o-p0g71Yw`e2e1`sDUu8Svg#8)jGw6WP!ua=4e<#^D z^#fA}aC^Eex&Xn*5PmAdJAPZYuZhzBV%m4$0$_X`LY)W{0~+0Txh%?g$tSz=RF2J& z7Ln3P)KG)-u9-~bx=(iomoes>3Ka@8g7wlRfjBy6vaVwpV@WS=hfrEBk=_V=;8niW zTr0gHtx6dGRYH=;m%>|J`B=f%QyZJr{k6J}oIO?EO#qoON`M|yfbqoXYKE4mZw6{+ zDitFv7$K&mYW)7FL3xVQ8ow*yA73<7&qNV{BH)G=H=zAT!FMD>N~yqr(eshxduB`r z1SuC}U^N0pSX=IH(kYqL00VLw-GNrEIK_c`=TRx35S)?io~w;m?W)t7YRzSDE|8)w zb~P{6DTEjhX&WS1iC+DM>px^qxr48QZb(&3iEy5(Hp?CUnbYoEu#2Rb_a}c!f&TlF zNzlyA3J4=5HU>d5P(;#nT>HG={8;-rq3_#&efy^VtkBZ1?!)|Sk^kR!|9h|$wdftA zb(@fhT&qFlG!PD$X8*8zkG`3=B1ygFEX%^1C8H~tpo*5qsTDf=I&6^Vhwte-cgkyY*zIvlLOu z4uB0N@r)>zaE8?1xxtL)cTUKFY5)E_g`>IjooL2~pZF@b2|4eg9#!m6s8BblWPl8w zuYxdr;$yE(<&Px^NxRq;%ep@E!bJB|n^0KMCI?i9*o0?{nW1$yDeL1X9qAa4=s8Mz zGrswGfvS4(SDz)yKH(dZr6q{U&9`eATJL<#)pH%jb+g0ZqLgR!f*9kkI~upS^6P>L z^)qMR7z$--sp+hLnmH`5&GJl3Njkuu_7bSfxv39Bs%Ia<=iK7#$RNI!+{03vfFCN| zX37}z1V!a)o4hg5kOw;}UCe>5GG=KZt$q2{12in<_P{AoAQ?A|OT`SdJ{sZItWV8k zGgc7&1RC;8PMas;_wq>JW^+P8Z@oTg(|2-L_ZBA$qO7>4=Uc+^REQpDsF&0nF%7E8 zofB+Mdf6HT+8VT%;zNk4?7S%I+i=1Budm`li=(u(_?UIEkI?#_t^_bZ+9e0v507_b-b)>(DNc$3V=Vq& zPbKW!Pe8l75UF-*nIS-pW(rN7)LVkmW!$K%-7-4pPpL8`O-8|@U>XV?@<^WbC#kw8 z(+oV%HShnHumr@J1jP5OyQ$#SlV+3OPay38{X4kU4ZDQ~u`N+kI4A`0EAE(I84a4j zqrIG-A1xtr1smqR-c@N|6@^+Zgxu5dD2`F=%f?u-Z5Ub`gTdiG;Q1p=E;&DIK z@fK5i3htw#4~NMVDlih_QI2o%2xcuBjgP=YeggRF1*KFhWH1zB9f$W_r~uF+O^rq zB!qKpWUWhOW}Tu@y7>Jl6jmKb)lOS2du`OWIU(1)G02%FH{A@mDRil+1wz?U zxtfNZDaK(<>9B9CxJ7|BwYt1#$XEayCZ_}MyOx%4lfFq%yukePa6_lr5>L!R1iMHT zEb1{}>i^jOi>Z}~zX0>QdeXp&~+8zvzmI!7vb zxbj?4BOtYt)k(&796Gja=kz)j_Oz{>IT}&!%@8>txVzx(^ zHDN0b8`RBj?&U#4HIAD~x$XaRAc+zJ8#^WurD|BPxYapDk$P?|ss<24 zDI=?y0X`8MDf!Ls0-eymgl^||Jba(Zr`lN^}-u{I_mzbc|_lKxr0X8%I6(T~5EeW&zH ztE}v3mzYvf&^+x>Zq+>>X=~VTimE!cb(K?wrxFNfMAjdR|K3o+M35k?wFf2`T!LKi zmq#Goi`O+NnKXMNJ>n448WM9paIyqpWhjWCMN_~?zPT&gbW9pxBwx}foVc@2IG?mq zneV(pTE9|ev#P$^cBoK9HLzKK3$M(^4Yvz1^Z&rhK7NAfFO)VR^(h==fdQ_}z6M3} z2U_`AvmzVK{@FENu2Ytkawh;X*QsTEmJV!Gt|O8m`CS#kjBv8~z7((3+4ptUoL>AN z@&1683RxqQ@6_x3IkD|CEc5XLQ*92v|JD@(B=R!)9-bUZS`#EAxeQo*NC8pQ98 zMvzDiurvR{t~y;lPF>15zM`I0i36Y1!og!c|5sp|bB8K55y0ihuQ#t5oQoWdGdb(1 z+r9=!0u1)m{?ZYcAk~+01T~XLIRLwri6InZCu&GJ@;jktLiE*oNR6EH!g8AS7qYSb z3~oHz@?1Z@;{>+`Z<&W>@8U65mE#TqS4NoCEQq-Yu>&{t*fzDTm&~7r{8VSbJ_b(TV2=0jPRl=f z?1z>{CDM@O5V7tr-q11&Lo=p2>N{hL-I+B+ zB(ap;r12CRk4z=-e;I`vR=sp!KBMCy0SBghW9beKO28TmCGqC zQcEm@Dm@&qYS`dM!vQZlY~23QBa7dkHO!qIy(iI%`1PFeM*d6r_xH9hpWnzwygRx3 zd%pJT^62ZF`Jd;7VSoQ#x)XebM;_5t4t;OM2+#%z4{1@_Bz4loevQ*l7<4zPLTt2( zKe0bqUKN-L^2BBa^~jiSoP4WH7=O{i%M-!DLexQBAeyd3UH>?8|Ck+j^!fis>uQnv zpzlOa9hCcYzLCUA5;Ph>Y)iZb)iITm|ItDAV2x-@*GoxoiHNEm&Sb`bGyRu|vCn%V zcvmoO!wnJrVxBNId20eWg{@~|92wsX{p5yb1UU+qAbxT?XSno04S3l^&F2LPk+}gI zsq@y%_wr;z^^~C0rN94Jo`(9PQvJQ0>C6S5qf)m5otZS7XdowacQKlqq+%KeAuLn0 z-J&GVhD5NwO$e_H-x)-GHg5AfzhAOR+vU{Jp?%h|a7;ofxW4I`yA2}z&f#n$)ygA1 z7%IBv$hI^ilQ7co)`u?sb&W4w{6U@7qxr@4PRzihYZqp4@}vumzu9Ar*x16eLCkI; z{=dQ441L*}`u_&_9Wl3M{2g~UMsK}{p8mSOrwgzJdT`^}!VVte^;Is$;Q4=TdXM<_ zmHP#4WP{ZO51#GHBeDm)(MREhL@9*6)*yvzPTpvd;=(_#=sz%<>xGO)b$AUH9=xLO zH+9qV8%=oQ-pSkg6YqHXTl$hO!Bya~=D~l!uZ#}d^S;(bCbOLWpUgD_R%OaF9Y@2~ zEHt05K5xrJliL2pC0$PYaJ-9#=xaM>O84-xE}Z+^ zH=e{e?V)Lq3iq|I8Xpcm&rs#Q^NrUsrUO0iQR61|BlR>+13ljyP4DxC@P(A6Av?MB z1*$USUzVrobUN4x>}JTnsYuiBEcFxM&lGHan`YSgKZ>q95XwJ{+s?kTW!<^6N1V;i zKAfGsWt=^-qEsplXP-0ADti->6-Bl~c0wYQkxEoX^?Up0eLwfS-~HbEzTfZjeV)%# z5ienst3Fgqu0>XXl~E?U-Yk=!s8JMq4tFTqPcGg z&qjLA`Cn137XMh0h`k&+X^5qI(l+!oO1eqKtsV092Q(BAE9eoh9}v)zyJv z?6D4#5U*0Vs3F+(WZ#oWu2T1^B{-dDMWF)r43Ajrss!n-Rp!NFEVEqBvr_cK_Y5_; z#;XL4JP`RuLr8*2G10(t&oG2Hxk~UXEP*UK1JFk1*8KGTZr}U}yk*X9qr2vqG_>2Y zps%}jmWle9i7@s&F%%Jzi>ZkqxSwaS8K>+SdWwrz3$D~v7Q|L|5j@YcL`?{L(;dR& zC-Cd}i22OQ>jYCRWRel`H(E;s5GD};A!#mj zHv%Z5wqFk@gHDiu(%L7>Li7aE0f$LQme9H79FBd~l{1-YHgnDxfs0-671BAW7-OR= z@4d#pSJu14K3*nxT90P_Ef%96{>gAhilC=8aU+ZG-11fG_UlYt_sXPL8#cJcNdiot}W2+ z>|uBca0MTwt`qChUQ(?}}dsYDCoD6`sxPDF-e@2V0#G^)6FHgBo(qx0xlTD?<}DL1g@*QQ-5jtgeF5yL;2N7<2P}7=gm5aj!oJ%Z;(1cPB4wNw zzV2%&bW+bRZH#rJuGLR|-H_5D28qcXoPzK=ShsW-G~q9i_tljbUdZFox@o^ZF97!+|AR6?PDIO)o$5;5QEaGj%BK*C z8URj5{)0FEL4n)Ggij$)YbukELhczCehN7Q;QpfWH6yn5DcW3x*~ORFj9Ayrz`2Oo zbt6jve&+rM=l{WU*gxoM1wd(ufAHf!IEDNN!>oSJTkZMhNLym6zPB?j3ChAg}&ExcU@;&UXLcmkt0H z`Tm2Ho&VC_`UknX{-sU%2PM1zr7Z#q>(%NZgqAZO7}xI|%{EsS9vOTwZreNJY(W$q z<$X8q*$Z$6AnEH{)oWB+0EwT^E?V=IJX7*?hyjM|{j`;p*)ulZySB>X0AX-@jl11n zp~@ns>qhyXf8L^ks0rOv(ll;_MustGMe5e1RfypmF?#aFn+<<#b` zrmv40828jov6FC)?Y{VsfOdhImvO=?mi5*MwR7nAgRCRZFNnrLh2a0A0vw1zP)_}9e)G3qH8`YMRL&?ryU9WU~3Ct(#9Tr zjd-IhNd|tT{vkG4`^V3#B~ex#RM4cd>;Tp-kLyFI%Il(w|CnYd*%o}bBCX6Q9!;md zJne^Ff$bY({j&I4&yGpp}!SlbD*X-2p; zhsqDf<=K9$3SmrGX$OE;7yXd|B@b@_M8#qczs7L?V|i?$_y7>}`k6Q;DBQXDfher> zrfP@2TnD4-P(~oOHrpe?<|`(h{-l=Jj-~wBHz4yXCC5Y`zz+_G=FV z)UH$S)p<{za+uT`WyBlNlPb-D=Dj#&nOz$EPh=6(u5Nm%l$2hV9YE;TvnM99r^Tzd zvdV+5242sw$u>KB!*%#nVeg%h;($FjTwekU@b_}eSFF(ZKgvU8A1Zt+^89dwC=Al8 zJ4^Fe3ItemjS*SchoN@BT}#UE_mJR?6I(Z=XPN6%1KRuGBRmC;o9)w0DiltbAV(Db z_gM+tuZRygK5XNfB|Sq9<m*>3e zw#j!?Sr|G>boA^xKddYa5+8~|vN0tvq7{kW1-GLoGq%mk#~ELdh|XQYdqCUd_>+AK z_$rA|525(iN@i(s+^*8uN?Fx!jF5Y+HjM76?b$xac%umRHv?$o9?XaFfxYlJ#=zC* zNIEBW4?#S^O|?&rroA&KjGI0)<<~#96CGZV&wuAebsLw`0;cH2bOS-I(}74vL$6c43yPUfVL$7Z2DkpPiX)=m-heQ&l~T~MD(TQ|AEuB7dphFa_ZPlJE(uDOJ$#!>yewz(_o$Y>{-Jlc@ zZ||P%KIrE5{T7{padETcU)2eZAS=_1*9^1Q@)K3o9XF*on^fXd)_rym1 zz8qICynlBgRQ@3M1%toHfjyI!*51Qg5dparySpIvVb~OwVC=X!Is+D@^%uF$Z;O`H zz02Y+;@nREI>F-}THf)@j4y0WM75GAl`YOsW_t&XJbUruE+_j{74BNra%0Ua6TpCo z-E0s}MdfP^ARF*Nw>Z#%E^KIxbX0Gj7-j#q4{8py5wx@TM zZB@>h#YGyl<(5MehjmlvAfz)%3RpN=j_1!$9q7At2+F_8OqC+5GzpRY;yTCLTI)Ok zlVWw7g^ZjmiG99PYDtiy`Z*kDY0|I^;mw2^1Y@dXA5pz&1pb{Q`&0-x2@>D*$S?aY)Uw{F>=SaVJJM=+0D2 zEyvAwgmFmTB-1y1EkyYpuHH{khbh*ZwOoREuxST=FHi)7(M*bigXv?K<=!>~)N>=w zRDoW8>y16m9s~M2^J*M|&ecd#`Rph@1;I6H0w~Y>qF6mm?+RRML;t;uhHi1j^IxU1 z*h_n?ivEi+6g1-!6qX`lk&KJC!&&FNx(`Nwh#2w#)Ce#FK$o5&1=#{giNcIseLia6 z^ZhX_gxVV;Va0iE2Qt7_df^>i12pm0yzp>_i2f&EbH$tJYsQE_EE-UIL8oc8li@jH zCH_llgB^WvPWb-3xoK_%YOnY-O_df55NGvteKl9n=6-B6>JTCRQJdY}3FDKK-- zh1{*(r+(?KJuUO72(Va#lu2e!++xV#OVTcA*CwLj=IVHET;#68yoLyUBcq{g&CMspUG(_rV!QF+ zPCwpvc`EDD2VC$+U+-eTHR`Vd_`fyfz2(XBDG1Ds_XQ-@8GH{V6p$kD06xFoBY;`- z=}XtYikI|ZJ-(BUZxq2vklSrtR{FxfU}>*oX#P_`;^s?||I5(c#PE>(iH@SI5N~A7 zvdfLyRd}}Qy2c9^JyWddZe-1SZMr)f{2~#B!xelFC&ZD&`D_B_)$~oOh!`u!8+xal zM3Vm!G|u{|2Op`fGH1=w)B;HC#%GwuEgn(WVU1Tx%?_4>%S1HJ#yZAe~Bk!&1u`AlW9SZi0r|#)J%7V*WyDC~XZ$ zcR6%mOpYVCn3dXop(1X5;xF2VkzR(829wDA32kMgln#xbKF%d&9@N_>A= z#M2|LHEOVw0K&edORbTQ>N6hw_G)Vnqv~m>AVqCwA=)k^TK?jjr0HHG-Ls0X3Jsz* zU?bFXhK8E<<6kShNu5t()NCd0-9XR+qU>DjwIg|73kQu7TQcQZY{z_E;jx&|HkruN zko(v6*)rXAe>NH(l2fr29tjoHFxfsKzHW@#2E7>>oExt({XOxr|K*%`J(~vT*DSVb z-smSS_;G60{J0p`La?sJjXZP5cuq1tLSu2gQ~|vBhM{a#jml_S6qU7ve%&K8%{vZi z192u)alt@d$dpCqcPj_gT;9<0tOL&cC*f*N?Y=L4r5lT{aSB61)j^UqUd!^bNscN; zCbR8#3gV@T5vgM8LT}tR5#l0Bg#o|=cWpxv`5v2)jbY~m59G@UjV6b`7WCLeR4l^r zvR9>wD?`+yZuMgm*eIg@fFKWAHY^+ zV_J`sZ#mG)RybPtdpdtjcsW7$^HZ{Rx3tF|B9ET=ru(R&qniJD)<=3tHvm-on_(hV z9&j(JMT5I~`E=6uSI*#+7maO^a|?Epr!r(kN+MHbL+#kd)x0CB0I~e&5YGw65&wybp?1%TvFnjM{)HAQ; zoAZ8fNE?g4LenKbGBJ+i^xZnQcT{$~dH-wEu;|makZ}D+LC~VN5RzZ;(?5$}Z5@;y zIaP|d6JO9Bh^!5X4lX}!3qnqnP!2?C%a#?6{Nxkcu9GSrbv?_TQm+ZW>7!M|onkMIbk!9}wMeJWb0mlr1@})(fL6lOmRb=Hs>RrnF(& z8Z8e%mA}MIME?3EYd@}n7g*H!J{GBi17yxrOphLG%T|1(^U!%N^e#_0+xzxRa%rNu znu*Bd%kkvt1#Q`?oA&AkPiNf6G5Xx*^OF8&3ieO)604SnNHxWA_@wfli9(Bhq%hmsp_bJx)a| z0olOj1%2m_&MeiQmj6*HvRYK9;BYt{UCA+0p;UsM^Z!;Usx;VtYkvQ(nRT;aB(z}m zP)Xt1Cb(rV&F5#`!(aM<#8Mut!J{=%UgzsZ)eZBvmi*V+EL>eAU`f;nea4vh_!ft@TW~O2z}cwVhdC%N5$A zg?|$0^Ucx3FNGuZb$bzESSc~zbq7U{QFAI!&opng96#9I9qvhy4bubr*K{ei*@2yx zS+WfkKiDyq*By>k?set~>$1mU>XNrU-a zH<_;*$p@pc_D&za5}rTj6I(gDo*;Dk`r5bKT$U~XyY&P9scdQuV)2t{a`inWew7#! zM2lR#A(q-r?|`p0yM7F!qo%)K=IP$gAg~4UM#VbmEIfhZtk(;R>BE`Q;iqe0BXer{ z85Jvwc@`B!O7a!MegDM&C?96{@zs6A-OZ1`tN8(~F}16!&6t+MW1@Hf!W8n-Zm9~L zX>)NI%6k2jzh*d~0=AG3n}YlyPy6sSa(UEHOWN=fywnK(70QSEsr%9v1Dwt|HiFYX zd{t`q7^xPt}X*hj|cK0-i>+4W(>{;nQwy*g+?rI^$71V(h|ei zORN1EvEM!U67pk~$w9L6%!3i%xc$j<4i?u&+|G+#J zTmE&M-2?OxnK2mQEMjDcO9hJYA=|~_13p0io9cb{5PRh>6Z2q$!^?+juoF>?uKYU+ z(1zS+QBcd`p)Q@gVJO&5gjOyu0f06@$MtGfHq7;{^Ti?rcyc#uEtE1d}^jWsddkcOiSUrwz>fk?U zC_&ICa`7SfH|Dc!MW8GEbm}#;d{2~_;H}+5*{Y6D_6o26Gxg^gVa*LZye!@h*LY=H zInGk$%_Mb8&n#A~Bk=JXC#-mreL{lIAl6O&_6eP=hi;2TWSLGJ9a~RJ2NoU8CUT<2 zKdb*5V*05+PvKHXMC;jXy~W12y#$_hXBn&o5UJs3hk)hOd=Af4f&9Nu`EQStx6XKd z-}6`gOzhJd^Mb}Li`$n)f4Ga9R18<#<^{dlmx~_n_wHqO+zw{w+Ch37%$(YS&w?5O zw*mD~p_o|*HpQkYq-E%2tU}hd;Wws@ztcM9D?irB-#9ve(;W?0AAsdJQO)fU1@MuV z^v9&)GX7X=uMTUC#U!k0$D>a@82J?HD}nqQecvm#0%+DTE9pemtyKfET;UDS`1>jH z8yM|3pYv8}=))e*A7o~Wy}CbY`Q`@mChv#5B-uh&vH|6hWi)#*_S!1~EL@|KcKhS> zukPRokc^L|YB7|Dh8SDfOBr5e|M9#&qhRuAci zi8bbUwchd%rzx8WFvli_Y9>9eJBV0LGjQO5-q?RVsE~g{Z_|V785<^-W4l2rv4Lu~ zl9}3%Qg=bRK{12(?un?#zKjN0sU=?)S$kuDqH%~=p*jBMZ$?%*zBXR zxqlkiJemp<@?M6anwuEHJvFi^$C&U6?=y57)V9rzJ`oR6TsPJ8l)Jq!U69?VQ4CBF zJOSFKx4d=2FGMx#20oiR7KwiE)(8DG`DTo1dt{?=O4un}DV!?(P>PBwMp@~YYSW>1 zBl%RP`LSI4TL`!nJkF)?K1K#aHgx#(TyyLS1x9{o-`9P&Li0Co+uSe(s2{2tKQf`q zukF5{E$`PT)f%TER;_tE-r%N@K-kpL9f8>t=A49rU;N>;>9*C}(=?aNXYqGSpPrXM zKGLdUN7`I1biOtM?ye>2(obObVW`jJK@vd+51~G@t53jwnQz9#vE42#XXcd0^d4Nb zHST(mpfE|EpQph>t7Fm%Qui#FfdEL`oLBO*QXfQ^QXVs;r?8VD-WBla$t%s!3eE2S zHj{cQ%}()CUW~#7e^9Bn>Y)9OG;Nvxp!T+a-fj7V@1oJ#Cv|D9Vmin-f_TW6Aga36 zfTYKP;qZ?v z=GP^CmU=-xhN}Tbd@Q7Q9&|#8zRro#na=0*Yz|NXn0Tl4VxilRMTaN{_ia$2pJ4%> zgOmGqTs5XbCW)l7G0uVYG&#-Txx;rBSH0vqvf}5KoCp|*tw-Hg<~*i38;L$v{~{ot zzosK*X!G*JXH2iD<|TIoD67HIlH7yKk9iIK+q7eW10d|&BwP}6t@gp#4HfKI z6Ix803>qMv4H`+N=wNna9G65XJfptYW-oje_^^?q#Z-;1Dt%uZw-F5r3nVxlG(u|B z#jE^pE=f0v8iG6WPF-eWX+DOCHa&Hcbf((i!<*w{Xgn48WVx@|L6}($O?63yS-4&b z3xq#82U$t7z0}(~QB!njKT$tv>+in14QC`(4OaPL=h1O;_*LFBlxRgyai# z`0^NZOA zXr>xn4yW&#-kWm{#8i^BIkDur3xgWJv|q2V0~ayPd3x_(=yu8J?7zc2;*o!NvQ)_7 z6~|6>7dq$Mz-n$-oSp#cH<*rVs z*FIP%ddZ`+0z+2~)iE|b^P9Ek&3>06RLnx9tO)olNv5xA>_!7TOiLtg-m3VA&;Wb8 z=t#6_5UaCTohv9ZCkkZQ+x~Uy&Dfi0P_i_8@x5xc{Y;N1mCwK@|FOkwiPmQSO)L&C zzp67&l?3h(jp*P3Ey6w~kqIW;MU2zE41hNl*Uhc<|mA9jYqSq@oq+C71o) zX=g_#xt5zwE^x&^x^c`d5@Z=D*>RCl00k<4_MI`#1ZH5eV3!0-A{Ce;y7IgcwSis> z_BY3h-ns^#^Rd#9iObl`&eY9aVyP?H5GFb5KSP_;wx^_z6$GdTV;NAEX{7BCK>e}Qd@4S1(`UNq;oO16E3~a&wLY|JGu|iuP z`9Rgm+jp_*(ThJX)w3lU7OvV<;XcHpR_gp$-_KCT)%G31U99vzM9*RHHFU{uuQS5} zDqKl|R5j^(=xqM+X}x}{7uf`bm8_oM!Cj7IfN3J*PecEn_jwA$8;@>d$_>H&Fn{FJ z8<)H$oe`eaP&1lGL$OfF5EJ!eWR4v-3#L@I6rD5fl8HT8$=njcwa4gP3p;)27o-WW zTzS`KX#EW&%NEhQ-F?Y_MldO4N&Q*M z(jjvKD7L13Si}g)tIy2qO(hJtsw}84vn8^Fmev)|pQy)mY5Zp2#!{Qso4%gaggw5G z7t7wyIsz3eeCpK0Glm}G`jczGow?vo-~~`9DC|zqHj6a-41gs240@r)n|biDH%5FZ ze<^j>%436db;+3Lyg^qcm7PG)C{UJjv>?Bbv|HiW@V&Q`L>waATl?@lnujhqI0P@t zwwT*a-8u|5yn)(re;gv|vjcTa)P8h#>?KSd@K+<5n&&;dLd(2DH7=3SAH69XJyzs= zaxr$5YLp483-&j<|B}C)sx@ydkv0r3rFZZ5r8d*@>7UGWs7Zv_nD^KF``9PMFY~)U zVK)MPGb}yZbW((7L(iz;4?v*uXA8>aw}PbNF)PvangHi5FJjVtatizODzekneZry8TN^ZuPDUgo>B_zS>eS%+Z{snzcL8+;_ikK`s?N59t~ z1B}ld&jj%fx|Ngeq3o5T;UccP;BM#}5G|ej{h9x!z)>lC788ub+L=g*4HZBOqp`BX zRXSK;iI}=Jg}jdj^B+o-zR_PrqrxH z{ivAOk8H7vIuoXfCZuv@{O~ovk6E#-Je;DZ0D@pKD|RZNLy&B~Zw)f(q*kP0iB5uN zy1WTW7f|nAmhlq(AJJQY&r*?|pv+UZLG>*I;`YZifY{W^@%_K}=qLQDjmZvK2&3x;SyR%9spjj3WX4xT>wLL+J{>gqI~tIn&R0dTwOKOIeioo3c+2s{%|$YMgnvJpfoY7{ zirCNl=s-Aw4kosDH84q$BQCNc%7LM3h@&Toa@~1DQ+V8~f>p-mNE}V)$eL5%YnN7k z?(CxB53?s<7(Bww!k<#sT^lPnKP$WY5z_OrU^TL=;`iebN%>AL$^KVEUoJz7ij=N! zTm%t6C#|kItyK>-?FEsVsv|X}8^5g-tU0|FmDo_nj!*U4Cp{)>){FHCL3T3fM*(!R z)k&1Drv0zH!=p=|t6OF84<7pHKu-#a2N>Gb1!-?@&@P`ddqlOrPS(?PCruwdbJ6GR zoOvMIEA9Qopi@r!*348J%_qSI?XP)(s)R4&>K8M4MIxyOH&P#MN-ImcoqtTeHL-Fg z=Bz3mt+&UN^wlvBo}7}${JtV+x9YWS3>oxp;o{;P@}Y7Nv*y$Yo9QN=-}=s1|GZu$ zPo8hLY1q`m?@8%dvM0&5!)Jb09Vf_k%<%sga5B=!PYRl7+Zj}J`#5SacF$Y?x^R+g ztH#bKD~6%>c{UunwsC$<=8Gqbe@Rs#? zNN@22+3s#HcnuAM-|x_(2VHW7sdZDgR{Gtm$d7vk5nH{z(m>tbQ5fI%3Ci+C8Zi|x zg?B#k?b4s;`~$wR+c6QX)8e+|ij*2?X5$!LGDuVPWsI^|t<7{aK^3ucaT!ukIkqNd ztgqx+hL~;*#(kI}`H<-4@ySgQw5xw^ppMV}@{y z4CR^p8Vbrb0wH}K&k90)^E{T-J@6Ic_GyC#5>{KaDn{LjtB0v^Jn1{rK&+xYda2^8 z;-*bYyh-g7Fw#I3$J1$24ldcxj3b3Uvq>g5wfq-n zuy+3<26G_U;I3<^LpLR8nYWjGEwL-0NUL0Gz6EyEEAR~B?dVSy4I22HDKQz|)O8G! z|6-WCc=OYnE7dd&ZNw_83VxPaFI%N{|<_ia2 zg$*I$2JEjQaIy$mAnurkV@^{e6VnYB?s%yQyLHAWrg%)NnDd@MlJb@Oq*VPCwcAAx zdaqDoyIgr3AK)pNKw-TT(GRVx*?BsE!6Xo;l=rXS!i|K4b*8il{qkG0ZMs2U*_P_u(WkuCCY2N_|4R!JY|H;{vT-QRq}Gi*|?$HAy3!D>JKak z`sJ=Oi*378SxiV2R1JPSSKi^^-8g|C{j1-#1AEiK)qp$^G6NCdWws!~sR1ABL~yID z-ara%xgjtxo~oSb(7E9W(>5vwdLd)R{O+kA?qXKA<5YHk6vw^nZFMLZ2OOc!LU^S} zh!3ZHtlt__6NuZ-9^!e@fyT8Lg{2|_3}ri!;bOe@cds=u5{>S&KaB!KNE&?(X|-`A z8b#eS6#AU8;Sn%zo%yT&jksfBvV)a6Jjc6LAwFLo23iRRMHD^l4vxDzHpn==fIay& zCPjLGaSsYdI&m_{k4vH#@WFDd)ppT;z)M_E(wJ42fRk1YwYEWQX(Ah_vKtql&PyMa` z!0C}+h6$#}#w}&4VwzwRq!B=@@`1r4pq-85?Mvzlsc}TZ2sEH>*rDL9)EUKrWUaSR z;*Slw33mN&n1VL?X04Y#(h}p5(q(kHQrTjlwsAbN>9UMx>qfiCyi{7U=aVO*%7Jb5dkMQG zxR5uE<2C`wD48K2qv{pib*9)NIrev$nwV3Eu^MjimYk1cKj zR@=H~l|9yR@}y#btbfbL%!ioqmTEQ$SlfjWH1W9BcTy7o!Bhp$U~((CJ5ROiQ!N-I z#Kk1bCby;jLZA3OPj#57Z&Q2HImZ#Pa0z>x&o2D3agSYjo>?yLu<&i|#l^#q8Bp)e z@&?KNO=rR;eA!f++EIIiH*V-B7S**A8;DHGd8$%dOEo5ht0a$l5=POsfq$;$@7D!> zp|_B6TLUq;{Sz8s=;Nf!Zb}sqKbEflO z=bDWNif@{?t&~^Ya0pw3`dsLbgZr|y`?GfJJqozL6aE9?W50yfv0+viC`EmqSS$Ro zc=Zlz`v|!)J0({aB1OY4fWqt#Hj6da_Z-IQ-XH=i265 zm;pF=d@b&3=a)_q`AWjFwGgWhaI0X!$}hIAd^5OVOOrn%zAMyvV(zNMu7I(>BouYE z7b+Xl32wbuXcl1RN&>e!w{STeU)Fs6Ss1AF%;2RvLcVjkSdrza=GRi{Ol%>F7E)$I02Obv(x zKej0KWE0HprN8PK_eL=fw6bQfJs4kvZxIhcHG4m4{c-_$|0I1(QCz**koYiosM%Y6 zIjX-5b|F0p*09f(sh>Lip69J3f$A+A-lN){I{M^DH4Nj|ihS3`c(iN%D)<-KN9oCH zXLmoxH9nAv!#vi2-rD=VOqlI)_T{y{jidoSYp} zA7{+tRtfEzXqUxe^98$_kffKb@tjmtbq#Uz<<}^R6=SI2d`;RF=N2Sbl zZLKD#5C$z0%yE}slEPU`1w!4{^KAR0qo9yS8y9`@IRO6c3x3|^x7qxp9U-tE4Ix`rpWK;vyIll5VWypwng&wtm?)VrLiFknQ3 ziYit4FT6xiE%F=G34H*JREE&A^e;2ucKG~_)wZrEjmn8#(z*AiKLWlIb#1Eh0-R>M z(tyIryfyCqE-jBF)%aJzocX~oc(Ec|g=OV?2s$-vt!sB(3)?_%lEO;keqJoM!R05I zS)yANb@Jn4y~3cQtaq=d4@MxU&(k2fwTmHtG$H6*ypHZ{8VD279wV;|8snC>yfeX- z-gxDjNnwz2wS!{L4}hy%tcXx^l!AVsR~}^S@gXSYXSiKASvK-=KJqWV13@ zormwTQ>&b?o1DHi)|l3cmvF{S-+{K);YFMm?I*}BKkWj@cDBll%Af4GIx1a?=|Q<* zSW1?NbHyNCqWoM^tnKN` zO7IHI#|Y4BG-Qbw;}7Y!@MCv430|?!78rtmYbF$|bJWYZKO)y%3u#FH>UyLb6Xd-~ z_|Z`X=yXdP&v(JwKG55FPDf z9~&#i#Dq7ab~FCJz_>;?FSUXYdBsu*USRc;xTCzX zozPzEdb2c7@_rXB0$WuSghD*gd$(|;xetDerqsVj7Kgd1FXcO?2GdETX**@Vl&1$` z4#Bfm`8?$$jk3-HmjX1TeT2o^G>kok{uAJ(6-O>B`hTmSmoL>#VPs-IPQR2L0-F9g zyrd$Pu3w(6?VcTfAuErOZXDz0^E1foJu<-U|KLjSC&zE>KtIkU%|ALMSB|^bEFtG{ zo2+w~)d@)a8s%NTIZRieF$(jC!M|<3ja`DiYFNOakdZ5tT8CdHXY-bx%!^@M)LXG# z9lW(~u0FLW%ra4Z&JDb_PM?glKkV3^+G5L+P_KB@Hvigm0G0&(9v8W$1gb-7H4t2q zhLjBJ57Q3m-@W!yiN(Fq&!59$18vcH`gdNSw4wBXzP9~}Z))iTFo}vdJTIy36Gco; zlake6K)|~7jHjRjn7SJgvodbimZ8#mftOI{xyuAsm11S}uGxP@K@snY&MEQltn2ec z3>{Ya_@=%k(=9)2JWjV*{sp*Gwo$@WV2Myem}Ar$bAxi5iXb|#B)4zkH*b=jAv(Wg z5`o%+q(C4qgE+{JB4D;rQ6+Nn$B+4S*N!aT9yuRqqw~6(mtyF#4}qHR<;2~@s$t|EObbI@TYCqmVWyu6%rB=&pbIOE=PbXTxWQ$!anHib!Arnz6syDr;KwZL)yZC$JfkS(Adu?pKF}j|& z@iGzUql*VaEOMPSQ7Q<~y&ffzpDx=nZ%*D8xa<`Hv5%-j$vpvmkE1>to}4PXD7pIK zCqRu6)6|BOSQy!M`#4GI5q|m+MhaHyt_Qw!`IRwH1c}cx2?zH`oNNBc#i~uC*PQ$7 zHx7`NFBiwVSplI^+H!u#JLh1DdOv>ay-xtnhkvHE!JSriqPtPg+mYKYEdJkgO+z>v z?r~-@i$qU-cS}(#Sd$v&VYt#BF57+JcKrChY-j%B{42}q`0jPD>v%d)MDH?F@+`75 zm*;f-ainbL#k?>y}~p9ju?uZ9;1 z=DW{aEFL^q9be0X@80;nJia#3pNZ$DkC6O$4in=qF%8i-_{)z{Bf9kE-L1U8j|soI zW^o3^#zVfJ<8b$sZRMhS%PWusMA#CrIfXmQ#xq$cQXAmuN^xe)0@&(clRi zt-Ujei}=kvR3fvxbX4=6)xr)*>n_hTRlYa4d3~4y5{T2^IZn(5` z3-SsFBBk&28@{h;L+FWiF~nt(3K{zrWiB`mffdfhFI;`oM4{=`dG)+XzD;8OX7Iu6yp3b-9^oJwx|U;0%=62r_4PK} zo&QA;#0LXK{Y=2!o+6n3><~;^e3jA|Ge`-cAqzxK9m|7V!ZKP#=I>Z%z9k0|?jV;d zzUH=x=+%8^prB&AbA!iEaxGV%_>L5tZVs$=-)NwGvu7u`Snn1&2BIlc(-*vd7^8_+Lh|hVmSqYsB8+#re25$3h3oH%MA6%xFr+_%1u<9U2@u<~jBps=}NOB$2t#62x^9>Ve+fRiTP@H%KADZvSOK zKbH@4uCuH);j2(Iu-V2Lm@xt^>^&~!Vrj7)f zOUb7YUYLd;7T?KD2$3aV!lQ!2uP|AsA$STYUADLANoWF{8iETmVvTbd%S`>tEt7R{ za@{Lg)v%nAn*6Yk{&5D1h!$5TsgkH!y}N#Ol~PoU0C8PNvC?P1U{>DD-Q}f9^&;GL z$bo4~UUom2rAV87)*a7$xTQ$Axdx&1l(_XZ$@HjiQ56Hr8l-a%%e z_yOQaKuvkuC2ktkn;w>iLj(d4lT^38z57R=4YZM(A8Y+p`86#y8RT~8bZxYaB;h{} zlk0TMTWMacOMFr@>yWonHR0}{5;xXi0u^$32O%}N9(&#u99(|(4R#}><1~Cp zq4+VdKS<0k)i>?dW7d&q8sB*|qibW%TbaBobe3JeAG9f^y z0_Ov(n~yc&=brtrn%rJ3D^WwZM>T|WNlTO-(H$hsWQH$TtSzeptV{=P{&D_*LIb&9 z8YEOt*~ATMJ23ETc9^Opy)J!CAO7SFkBpdX91OvCQ-E6U^rD4O69D((kAtc(^#smA zD&lFCXo!gc+&}8{2L4zwP3-0anIVFnN zyhpxARadN8iSg(H7#*MYjl@m50Ss2g|k}mPRm)`2;`c^)nrYPZo{Pwa- zwQkTvCCUA8*&15U2h0VHLw=mGwwu@(#tc+r&aP$^k2=_F%2R3 zjB&mg^zqWkR%-vWU4(x#j4tY~ujS_&QGgB|b(`s`6!GaEmflVXxStR5m#3QG*d8hr z4WkjYq1H-DHrXOc|L9rQCUODWl?@J`ar@Cg5WtRHkj_=Wcw?>J_d%)wW5QG77rN!C zJzN=HNuE9I7Bq}^3%s>7;g#G^2?Y9Fyx;5;bM#ryb7Fl?3T%(k+{!0gxq<#=BRIxb z5^mD|0PU(^V00yEj2GF~z;>)GHBwGhXY4nICjZ^3KuI+)!GbA!F!QNXfnltEGBu&z z7ue+$S9fpB@T%U}V#F^V?1yX}j{!D0k==KpZ#ZJ$;r0pN&N<%!OW!ryQ<{=8whH0! zd{;dlIo7QxdvcS7xNtJLI~zfUsURSNY%gCr-hew$)F*nUD1&@7Uk|quyz_w@*B1(s zyW_JS>t#i$ODZp4rQDM{yCWJfqsY1a1TfSwZsAF~LIU(M_S_$|JEed{s^s%floS?F zv`JR8wD4NMqFN?43R*QgE}&6iMnH5{tMYFVH*n8-DpF{w0fOEZG|YjlfHD5fqU1__ zQ$aWY>N7!o>{_V9{hfNa)LM zAOGj+Dg&bUpS}`$AaNYsbsUX!N*{4_H!9sA-F?8(aCAwFNH<6cf^>H)(j_5{@W0>l z;(0Z*-`w4A&Cbqz=6lGi^MCM^%%-Ehb-z^~b5URknkJApMCfB*Wtw~Sz-6l}iL6`b z-w&N4TVoub29y9rz0_`iY4>&W!1to6^n;^qVSVvFj7*CC0`#|7hrSfDZtGWn;KuPl zVpVvD1&)0#9&o*R+ke4?+j{HeZHU+5IUQV8l<+mE#74L=q4&V-6xYwd+LT*)g(B;J!X4O z@1K8z0O@bXOYVE34%7qZA*@gYh^*Hu_TC?KbYM&GxH8sgV=^gWe-|^5f52p1e z{UHD)MZX639Y0)9ViZN3F+xusDn#)SMW8TQIQ^xVXt)Ty+wLRpOTlzGB?H9!ms+nuYtrr)FAr`Qd`^77 z(G7A6pv(;I&5L@x!{+{P{iZveqXKcZy z!RTT2ba?b{^S>p8V1(GFbrY>VgA=+YLBiM4!z0#T!ZXvlPjadCe|)yt1e)tv8pvp* zqgP!4&KGRJbdxc7^w!%GKk)$pMkwydStV4e9~BCAS>YQ7M%EC@Z0fS+>4Q|#yCa|# zA`jlz#ixk8WImCkI}!PQ0*ku!{4L|_m(g**yj{N(7yFBHq(xD~u96^j>II`wsgp!la^j z(usIdrl@kj_l!<001C1Kn6fIGq~M~M--CrmB(f-_#r^g}%m#b0bP-P1$yV_3AAF~< zxHpfUg=qCC;o@;v9i5lpTR0^!a%-O;i#0(*X4d8-;MxmLu=#qGiH=Y2ja~hj2R@$S zsyseQV`8A@9MvDgtSPvh-z`kX!nu_+1RT<*`3&9xZZ@}2DBtjCP(;uLZ$F}+o7#Yf zKgcFq?0=ZTo3<=|66k)_hj#@_i7%V-mYPg^8R15*1pp`9GB7&dtEK3OafPGw4_DgS z?O3an;b8L7Ix44E`{IfzJ+_18lOHILP?3)s!FY277Oj29H;3{Fv=m}|5K0wI2q#(s zl-;09JGRL}WT)Ssi9-;Ka*+^*bfC@`l-;eEcMx<*G4aEo8R7(v^Pjoc#TIKg(Qh9+ zE5sJQ9I+ru)J5791=FHbH7eDkC0uOSqVT0OKgF9jQQU1vH2I>|2HDKjZnD0R(bw>y zxvbZ4gr8KVi786GWQZ!!bo=VW3-xcXX$S3q{dw@R&Rt#WJmv7#$7T>H=&DeXPhUYso{H~|&;1&v zh_5Z3wVF3dEwpRSp>p!BW-?^(npNdQ6!n|=L$sO7i4x(#9=35$=2|yKLkL~g8vW?W zS0D)A%x&RPhEM*yf&H&GqBbF}ttvL`qKQ4e-1PV^D--L-tXh>&0;bET<2f6sEQzN^ z1p6J8M1_rop7FRn5kMpaSn>VbR?037#r%G)1W2OZJppS*mL?$p&6)A|yhABVFQ?5L z1CsJgj`j*9p8pDWRPMT%{i6|tSHGH51dnCBp{k&aC0tPd8BOn-R0vI*D1o66#~r1U zK14l*(RCtXzdQ;5y9rR2-7G+aFlH`yPhLnh{*hgL)?hT49JL1&{DveI$$8-}7Hf$l`1$7u5+G<{{x}G-}P4NsbeRFML z{R6)a(qQt3xQAno=)6kJtfbw^%o4VHDXDdWj944Aoo7w zR8D40k^d(e{542zeUF}z=VLD^R=b|UJ&o28xS{*%>bZ28Or~aluQ{gOM%^jr*k~O` zxVKe(;;8&M1RC{TsXmc|bU?(+51aX3hs$g;JlpKTl^~i{TH>?8O<1_&OPc3+ys&eDkz0pP72ZpgmIT zA5qtfS5HcpJC{OjDFj}g?rnSu1v8ABcqOjMnkMgXFNu@v4+kfa>;1I6UVH^I_o??+ z;r#dJ)waa5TPwi&(aPx|7}|9x)K_opULyT-;XPm^>wDB zma|daHD&~@2P_L!nb+s2NI&xRmwhB20JIMi-9O4EyS$>v!SNrN^)mgYHrw$c<%mc7 zND6-tYtm;Hf6aJC@Y?CI?_@ZE&ysP6D4M^-hs1^W)I_oy@u1nCYki$nlbgCyeXGB? za3m!w?DRIKphaU;_^R+Wp;IelQ;#RIKZrk;GV*1FHO2dnOq4}DT1}UOuiSz8K7cR2 zG!ZAXJoE-_ODU$m;%H&~s znT9j;YO-SAMR4iwnJ{ z!o#W(Zfk{~O$+@b8s*9$K8~f5>uLX0eN(Jc+_Q=nv*{f;zGsnA9PZS|`C3W)0;~4Q zJ+^4CO5```_RG?{xK{}@fX+E#r1#Nord<8{!z10m`PGBCG;Zq8P_6kSkBW%|a8L;D z5q@m_JRSeE3qxyq6jK|E`Apv=9RQrMv7h=ryJ_fNp=V zzuQ4yUCgfazA!u@5KbpStDZ-T5YUqOCY=X>H;xRf1Ep8S-tf8B$Eh>BF|JjsK?!tP z82pI!Y`Ui3853=iqU{q|_m0SHd{ubRsNW^QMOu)WT}_Ja6eETp@NVxD1!r|=;FZvK z025`qBVNgD;9bsUf=c`wp0$sNbRMP53;vo{gyr#yWkD9!VxDF{t#uM{D0>x6z3#p$ zgw5nwcoerehEm_&{vPAWp1;o9%C*&&L|A0h>v?h9F<2z}S>SDwrSvfFIfx*vbiQ*~ z40|%J#~A$B&mJDaH~L#c(N|x99iXPYb#U8GIHGu3eJv9#B;*rql{z!wfFtxv^R-Qv zVdO03g8Qjt{t{Mk;1(w+TkA%4H^&C!+_Y$m?z?%HU8KJMrx6SkM$+6a1CC>)jzhBa z-z=AGS9aHFj?=N?duJYev&p;Hr53T4KW;xQ;iq`6<bG{-0>}~`jwLFc^f-mfoIn9^K_LIKSWp$Yb zk<>ZXTxY-fMRGsJi#_-5&aX&wB`D?1l-d4Q8v;qE>zv;rjn%qBK_0- z1cigQvtu}<-7A1_BMMK8`Cr;%ecb7Dm&S?`k5omARRhHlhJ*2MToUwk20x!sNmfbj zs@78~?EOk3>sQ@3gopCA1-(-G*e z|C_G$;%;i}aUP(VAr*s-pYIa08DHy{cVatO2MwR@b@zrr}+v zxh8w9lgw!2{i+Yts!kq0t~(X-YGp}C{7gO_Ekrt!)j=WVo5$J9P7GJoniUtuCNg@(c#s>B;D@GS4 z>m|4=WGGLYvAF5GCQhNn>MSl1zdq=t`3{!3R=X|OqrTh+e={-MwwD*Je*z5ElvOD{ zD{3MxuW=tP{>)54k$fdu@20LPji%4M?YV{hGtT}X&tqXq^TMQMS>;fNzU5;MJ-qP;cnQ+Gi(6lJHIw2Cyc71^u;A z#lTiZHX;q{W`MmueK{J2jLda`EAc6nZkO6Fo7UXb;_L8XJK~YoQ#$Ej)R%4>jqA`J zBa?K*L7=*q$-SNdL_A{WB&%`0FimWT)whEqR4OA^&0J|x?_T<+*24@;$*w|zH<*)A zQL)^m^EQr{9ch_L8DS_}%jq$53@&Q45#&^oyK}#mt|_ASl1_6%wK;!?158=))GL4Qzo}#pPTHxr;0aBnd2?zveC=Crna)4&DpdL zpsxoQ)ts2v5=8FU#sk??B$Kf~|Lm{|+ps6o=$31{2#T@ii#mMxZ^6)}R=FlKIuiOX z<4}B&MHu1I=gpriVR!pr1Qp#SBdGHX@PW;goN~1+g(NKh6OiIJSYF{WE7YI;s~179 z8*AY?l^+(c__JLK!X+cC2f+rK?h-GCFn&33j>S6$hdJU^y<3`!h$PEvz%FPeG_^%c zw^azZ##Z=)loCFRa#tgYr_kxYP>EsdKuMp=>uir+Roy7%-Zg7jF&d>dS*uliBef5L zUb@K`5L3l_PQhNcnSA+56DWt}yy;;YQF4DfPYQv0o=ef#&Ei4eCKCv*(UPnWO^y+W zoo9Xf^_7MEKU&TuU9oHQ7#O}s>!dJ#fuAAX)1iRb{4b5}6KgG!ZPavbiS* zbzvoOEYB4X6+y0je^YE_plWHtbu85isUP*2WHrS0`4ss@iaxRW83w$FhO}q@C>q?h zNwV&Zs3I{}xFwf~vmblI_%@oKa5Om8h5atzzCls+^##3}E;O3}Azj33)Wamh zFqg^FiH*a#b%hI39sKrL;-~GAkRm@lXX&zF@Cz7(^j`PEGI7v=S)xe=&m&QOQsAsy z-#DC)VKwijhJuwlS;Sd*G*pWjgN~}Q)Ep`z=?R`K0yK1~qo7)A#=O)-E8;RFh*K-5maYY8=U~Cj4@h&lrx9P2-1Hcpd5~sJ6)m-O$ve1We0Z1&Dec7 zd8D_#PY5Sl%igZFNqwCK-79CaVJRRpqjZ5Qrcp;eUGb6d9DJXcc6k}7{)0ijVsZHm zckF;MZy~Kz+FuWmXdi?e!ZeK)Qy=#?XO`MVdQCl$^^)9WXlo^B%j}O9W>1vMkjEXN zB}Sk8#}yxC$ZGtSS@-0_9p5@~zo2gVVlDSI(W9@!r|=!#TbyYA`xaYKTUK5|fn<%Emf6WW954@9+zuHyL5Neyr2cR| zrNeMxhnCYCAOW%+rNB}*x0;Apnh83kiZXMeA3%n`S|fLxEZdn!dw8#M<^>|^twla@&_N4`tbU4CJZs}*6?0^gGiZ% z)4dtK^>g^d=ihV?VL|Zv-r6H!)#f^qn-Vn|1*_+#zqOn@m~yvI zM=l@m(LSq-lUFxO(M6dWCKCIR^BWg8()w$qeKl} zd@Yp(DhV+g^o*Jnvw@&k9id1U!1pxTVkBB|oJ+B|a26+2T#=Z&)g^fomG9Dq(V`;Q zAly2XWx+qkhPwY${g}s+mRH6YCVGlf^f&qw21D2R5tP*& zMgv`}xTwm|nCL=5#?DOyC_`?_&*9=9IskRr@qLOjCKX{uni>l5IrWBj_l#YUJ zGx@NYZi9LNOHs;F8X63x@<3E!k%>-Xw9C+4R5f_W8*S4`JBu#jnYL<7%OBkM|4i#E zA*_$5%x#zI{eD2*kdM(`J;<#+WHb zyQ9d1La;u?mEdEPjDmZ%QW%9C6eDoJm!9iL<(|22y!kvZgneW$uCF8#DHI&VYM}-U$+%>ZpQLcE4uH zsM|j|gEOl|zl#E;g<#x;CUy*vY*Q7Gih`T56uZd73m&#PTODSZb=2)J3Un4p`~-J; zSXwpU$;IZw+=lQ&`JX)5yj~-f6~M=RicL2vNM&35e3yC+Zx7^KnLa%{qOs?YA*NZ& zaaXQcv`lcuBxOfr;Y}rqX%ddAU|9TMzQp*n^62~HMpUM0TGDlj1 zawcdNlHptV>EIOg9vc^Fa||zTq>AFqQnb`@9SjMp=ys118!02RgN!woif69|L12)G zfgNc_qVha&mPxpTJm&$(f)6rQ!c6VHN0e|0&4yLfy^pfl2#(-S`SNC__!`iVS&}t7 zO+TN9tD+53Jq;dzNzY;`024GM9ed~cy9bL=bIX6%;@T^h;wDYV0uT}uWQ8G=0wqu~ z5T_09vw8q)!hO;MUi6KDRV|wYJThdR@i;(okvLJ%g1}h@O}LvYI|?hRvV{$KrStun zIfJ@$r*86-{_mx?qF-*WCj2TGJXXfd3W{fJI^~k{FOdxvJ<)#+T{JPYZw%D}IZL=u zRb{}F#Zjh`Jzb_Qx?48A0YtcLpDAA0!w0$8x&DxVKETZs?U3B4iiy~q` zwI@^?U`1+QiB9&{?~GEie}@J10s*4fTdr(x+;GjxOo=o%v2-_zp{LbQ>0TW|0P+K7cpH%V@dY`tx`I7 zUV7OZ`ny^>*T`kaj>yKyf@hOR>J%~dtV+MOsSd#Bb#o7Ogvp36$2wm#d0@CI-*7Fdzvcq#uEU0bYGMYAqse)CGTmZ*{TE5=*0W zm>vdm#_Mc?_)LyPHmeXA$Hmy8LcZ(_;A*4CNSB~T-3{a*!6m~hPRKliXrjbPkit(j zOHKxzY4!zEDO@^)Mit+D6(^XxlPS5?XQ*KeSNf>P1%^@gBy6eaL`u8O8I2Kfc}`xJ z)lHmCN54bcyHlcIz@*24g74BZr0 zbomr6(%=!?&ZyV8d@h~D$sB8TDFqko(?9U&IpT#k%y1@r_+mio6B%+V@8Wc@lH-5Y zivLvcbzq3*Xn*-@W`@8;a|p2V+7Jx@#^l z%PmKc0|Qaus7TBk>X(jaCBIvPi%k}cK||H8EW#?Ay3vzvPXz~39c%=moX>d!Nh)On zsyhT_?k$C`GJwYIUT!+PS9V-|^W^+Ww{6QU43Y~9%=xm!gJ@K1!I4-%st9ziHN^F zRGOl|dx>UkdFpdtGs%w#AiUIW>XqU?{?#e``MRqTRQ%x|6Ivb0$Zop9NE|&HAH}%G z6UnVnNvG2I!vWY8zZ4K!`wNWbZe*Hl7fa()j)09Zgw~SW0JPwcw@7DepC(d5W|1t8 zC9Sbz>KTACoS<~sOB~&7cLbrOxdijZ?TgeNR(~~-@|ISCWcu2r-Z@q)7ai;PyJ(8)AA7ilNb(2n&-4yQz z%?9JZHk&h6R^n*SpXbabsb?ey#JGSH=p^pTVH|@W7E2=O%0UE6VyHYu`zscbUWfo# z+@)$N6oZjT0LG`vYOg3?Z5zH|nL=`ImuJi~F8L-DAlH^Ei9um>f5gBEW^|F9 z!tcfHQZ-l9_{~VbBG?GSp(nh&{rh)exhY+^)AcFHH1%OV_+vSIMgP`aNsDKyBWQhYQ_W5Ip;77=*{JPLpl z2>M}uTR&O}n1hj=i}O~#$L&AdNWN;D_b=(GDiyqsVuh zmfg!X0vxe}qo4{jJwY+-z`nBaMnh|pPfkeLqWvy;Rx=e`q^#}X4N$Py_Y&2gk>Hggr!G$KzgNTX761B1$s_g?<}SbyGYb%TH!(MfJTs zhnD{4z&eR$+AENF;|BHGoGu9o)0ZIvaNvfeLr+CL443u12SmV(T~2@8)LE$Y?4uc+ zY@w!LaW}{xVeSQ^H`#Fpsjy5Xy~FMIrs4tAIh0U-Tu=>zxiaPhd_;s;Ava+ub-2L; zkp7Z2y?@x*&rpcFF(>5sn|%;i4A#f>fQ;6arIJj3rKvZ7)|aUA)B%d%@buWX zf7??+sP>u0v~B2y7 zK^UBo)V54$)>VQ(#N?qeY*Ww~j+%jaYdzHTJI^*R8u@>cBycGJ^O;j3s6!9uWW!(+ zUq6Oc~|+L zS}J=A>5TnU65!0AuD;JCa5bKyFjCb2@xRc}#zUMbKH|c zrIC-@(PUlb_*jsw{S)X-lq)Kv`I_s|XN0#RuOa(%GVb<8COD;O)ks5UXYlxCW&-;iHThFmm03LUl+&rF#6pROXe;cSW@uoWp*l1Zg` z@yb^9IU*!AqBokg?3zVnoZ9}HU zn6mKf`(D0G4dqx;;`)&HG7WNVj7b6?6uHTbX8r)K8sidFKKjV&uarYb=N`kgW6uIf&(F>87u+xH61og$y zTEe3KB2E3Mb(2EqB=`=DY!5Ke?#uQptJ$qqXg|efvH_{nvvi3u22e$eAxH`a` z0ehlXb4+;AOet|J@;us;F!Q00y$qpJv+MMAa5X!TaddJwMNax{4+vTvB5D?-M9%!o z18F2v-1C6DS|Sp51D8euN+xKL%6fI1=+Wels)}5O9?5sI1)vfX2(nn43fND93aK0^ z3XPW1PnUHYs!mZavU%{0rB59mPXh$1P#;;hh9ZdkFZM#(w{9l(KXt(A6aiW&<+yO7Tnv&$1A~Ak)?V@S`Nx#i)`zq7R z6}^o&x5&_C@UY~1M-+H1G}Vu|b434#j?%%|-e{zLPtY;>qcd)Q(O3{LI_j!N!&>Th;8PK+$2YiNlMWUu1PmSu(qT#+Rb`f6(BicDX$plW3PO{2)oIEkAe(m|f>Dn%Vinl&B$Z2i=_+ z0s|3XWXq%`hV70pgNMqx%m6`yp#5x!XCJt4oGF-&k^0QK@w)yGa0KzrS9@9Eh3kD( zzd&V2TEdlK6yEu8h>v@W^~uX@H<&uI4A#|8ag!p@4TuEz_>>(lT^_O($Vl z`&!~@H({}V^DicaAI2$7es~qU%Vg9#~ zl7}15{O_pt>sTjaQ64|u3BonNLuR^kTnq~JoH7@OL5ELmNFOYbgj3Xbz|gaND!?Aj zBB0s;@YQ%UTvt-i?07J`=1CAL{m?LrGBe8VC`~9TbmGE^&miejVLX~F)lZVC!KeKBbkp=*l(r^@9b0T84Nf!lVUw5<3&A z>Vjd!kF3Vk-r*Wd@J3H4x+%hg8(|(rn-0$dkm}~{@tPYrU2n5~AV%DJ{oX>Y=S0xI zy;CBoGn~4wTf0x>D4KCPxI&2VnmYzh44@Etvmr!azs(yryjCBr-hLA|i@(%YmUnkj z$g$1o?jcy^eh9Fq?%s!HU7*q7Lf3g;3Y;G-${Z9ht_KwPDY&^74WAkBzW=(l6L}>} z76md-eIar>Fl@XpY@Xg3>i9LW!!wRiuZpnJ(RYOGnPX>^E275XfT~1gguW}FtQ+_e zF$QQ~{H7q%w=jFCYo<)~X~elL0?Q9koyK}kKlsoKh{YO zF`-K+d@lYU)t6IjK@YoGJ@z9s8G1F8+qNs<3$)+@yYGz0eK&adXbRgMHTCB%%BPls zf1J0k8bhhV%SM)(cxa{hOC^rH$=&b`S7?9c)FD9O zATfzZCbWaMgF*t^j$Wa1eXoJ5lCcZ?kwBFe{< zaq@3!X7C+~g(peRR5-IzBxW3iNZHzr-HJU?OO2%s4+nMF+%?Yl%j=SnWgL>%X}h zwS?iHzwui-yr=3bi`Qh7BPu}xWSGiK0hRI~=>*|^GUxI%C~z8I$#>IP6^3d0uJjir zdo*2NX&P_;WjU(s#TTtX#JmntI^pxZSfe=#rM<8GHk{!^iFc5|$cX0Gj*{pgf!9Cm z9XhPSHFiI(qRv_*@Vxt-&@7SnEC zZoE%_S8KweyE3QU1N{t0zM^Qi@^{6o|0kUzh)6n>{kHXf%qHhgI@S82ZnI;l>(|08 zy=6J9X@|OW5D&gmw@&QrwF7IV?QFAF&iS#_VS(>J;o=H^R`OB{`C_SI}=;yVGZ&6H5jSH=63MJ#L<)xi^c z>enin+trLTE7UnCP{~)`6Ak157^_oW1^5~}C-fg=bl1wO9cym` znyK#8Z}Xj%W3+6(pFH6D%V^oj3Q#VQ7NS$*EfC9MCJ3^d?jVx7nRRr zhq|@;?ArGCerciJW=b_RJLaY^4*Hl)zmx=TUHQ0?4~aHpjMeNCCNEf{nG{UcN#Hqb zuWkKcNE7^px`5W|F&j$$^O=72 z>)EHj+3#YE2NNzU=CO>Qu;-sh-A=~>FK(L=#! zpHZ-}r`r!VBEOEIlw@pz^G3*taGk}WURrg|%aR<5@>1%$`iQ**4f))A32H_Gc@r8g zCVubP?PgB;TNH=g)TmM#Vr&H&|;U{m-j?(g(?rK`@Ze7C@_?geUE94W>+ zxY937;rG2{8rNr^WgNh1YF<^(zilijvYOiEf41JLfobhGvP)rV4=S?KYh3z4Uz+pu za$#gLlzJ?8w8F3WwUo+~k`-8RvuCgvw`5vP%}Y}8uscb5bw4F*$y}YspHf@!6lSP` z5fTmkap~KM=N1wzB-(6lj8xxJ4>;6S66O6>?6%*~GtM(F+$Gkyk2IV1r#VIFwc~<6 zUe+w?+Mdu$|4a#X*p2&J`<-l$4&3ZPYb-CkSRbAGgTNvD=vC_v%l)Ht!=E0YK9$9dwWcTb6~{rW4<@@XcewZ+#EdKc?oF3% zZ^_g!!A-23(iuD@PwjL{-(QP5B8d2pEq9r-o8Ios^z~EUYEFgZC*OHJ4&)ch>2nn zpM~zH5TKarGuT~R)cYaAoJR};>$N^5i`l><+u|n%y}K2Z`+zuBA?Tcds(mL@kAN-S z;-8n#!zo0kew@w*wAU<@Je^5S9mV0_ib3w@jQevif^~3mE|DIcFj2J&+m;X|?DlJby7aqdK6gCFshF>R-@nlylOtF#4;`?@lO63e*^_ZX8&# z@GSS&wmmw2BQI(h-{Y#8J}bMKXJFyJORltUs$=23JJsCjV&{0$*1G4(qHN(l!9w-g z)tDm3A3FGuM(Kj8s8?UR5Ifq!A+nG}LetjJa$(XY8~9^-X+<|NLWFiq)W1D}$MS{@ zmB5H&eZxGvwCw2tM<>oxwsBeTP3yL?cDbG!@vYkWuYeMffV#4c!^GwI{_lhL4v8On zB7~el&0e(14KtLz?;WnE>VDRKfejjQ?y~V>yOE;3PEq_?k4?dpoB>O~Tlf@~gp*E1 zpnu>kX6PC)#zJm^>__swx4&6Kwslin2ZMAJIic?q2|rZ+^q-Kqi2y4P$cG9kZN5&S z3S>&4VaTFTP8ekGCTwm#v1oZ7!9QZ@uzmBU8Gk&bN}ObrMDJ9kcb(?ts@IwGhx(QQ zqe|MVesf&q1RzBXr!k3t#pUX3?{(ec3S%|_W(d8R3t!p`FVGn@wsY+Fk42$Ne=~1m z$)d=B>DI+HRE$mE?k?2uzDc^wrqpPyvI^Gvq^=m$Y9M@OPXb0K$tn!9%(BC(n{ ztBk%f*R-~y^aA7ZQL7fq?;<<(%DfN6JQ@4L20Rns_sD_O-zw=XE5jyUowklM*;}61 z!demM18Hp>htjh34{twu`D0A`m>r-BG$|9E@eQqb@JZnn^0GKRDe`LJ3$!P-9wb{z z(am2aYwzEVCJgrdx<3Dl&He`yoznb{Cc4D(rt?i_FM%K8STY60jMDZlhr(#QZ61?S ztgX0be4%*e<+I?#cHhb|*+Z4{)|CsdPW@^eYsk>b<=Zyito<`7bVfxh-kHxoUk#lb z`?jxY6zs8Jqt!Ab%Fjn{o~HG1;HRjmqtIVT-ULKBRr$yI*E;S_j2^E|gj*>vM=GIU zmG(X;t}eYC;Tq~&lQ8i5TSloo0P$V?l^Sw!_}!|FGeo7D%gjr0T3QP&*w#8KO*4qn zmn&K9I(anJC;vWVY54mVAy&g;pz_*DRFcAT-Z@%S9381sdyK^&qaeB9KW^TBA zwJ^}B^C_yNealraG1nyrQ|erp@{nYZBc%EKgCUK(K(tU<)QBE5gpQGZc^K7kVdP(Q zTL6BB6`oMa&FME2h-NYo@)vSodCxED{WILqHk|$n*|0IMls6~i9oXX;#fL%Q0p0gp z?pc*-q_>C3StGpo@vKCoW*jNR4w!=k&+f@bwlp3BE%4K}KZ&1yI1%t$t)C28SfN9jIO~gO}7x z|A{_FkiFi@AI&30CmrL_Y#+TQsk~BQ30GQxaTQzJa6(QI9@``Gsh(zAgYNl&&L`gX zv;gap6~3YOux4SZnMnB9Sn3Mo>@2BnbcXSfxyOSNBq*fdlESZ;IpQ~NuMsfAbmWXj zH!|_9u!Bw6eAeWm#*{bfGR{sgT8I?V@X<_uWCl#{DhlIm|LmeD-FtHKAYTi{$7*kH z*f-C@2e~b+t*Cu(HIylwc%FLNwF=%7ziM znF)*WZWX>ftw?Hqoj1nfWh;KTv4(6BJ8-~CuYK@#cG0(v zXhkk!m<9QYag(bI#*w7{oC# z(%#lE0kgBfp9BEwmo{BhattZi$dic<12>bt4c=t(ao{J{0xo*)3lrujv_zHn&Gyya z*%&D#^xVdmwo8=C`&a7c&DMRh%OU#rHWtyJTr)o{ONuGG;VgB9uwz)iDt1+RU~qI= zi~5!ym4^!3I2gb8qpnGln;cXb9$nvpOsS2BCBYU~Oh);!e!SJZ(wjGC`|cOd;$OiT|6S8KrM4w|Cbsi@ zkN>BMh5H5FANhST#l_V)SplqcP57YWHq!&zR)u8&9Z0=bKdhc9>v9@Q+yzRlDpva@ zzsaWfLfL$24*XM)p5;5ZuB@bu$+B{b*&6hR7ck77tYYEHHvA)}7a7zV_DNmEI>rzm z({cak&@MNTE3b1*eH6TI<#DA4W;!sHw{dZgTMptF2Om|%HExjpC~at|pAMp36*JB$ z6DmAauy)X$4(vZDz-iV?w2oiHL_Lrnzr(xYjGpy_DM!tsez2o6Ny$=w^a3 z!MC_P`UMc7ytusDx`-)_l13a-s8qE6B`>Y0ypP?g^clmjdNDt6@qjB0wX&(Vp^6A| zg<4f?yG|aXc4asdBk?09L0T5WGuMS~Qm8>>L1YIpz7;_)Z%X6us7D&xTCm=9(lU|n z8jRZlYY{`@o`K~%`C-83PA89a7Hz#+w6{3aiv7E@%CGQW<~9pweYQjp)E+NqzR(?8 z+rTc?b3i~@zqn+Irw(jqj71GGaD^}<3!Wz)i6;ql1YHSwm#A(A3i2!+Y_}pN7yv^m zQ`5p={2CcWYh>0B-zX_)vmfWa@wFX0!IG$wofFXx9BRlr-Fm`j%fOqqG_v}AV@VpIH+wK7eo5=Wb^$b^sODHQjS zM8KTguLnCb*Tl)xqX!+)Xj|LG*Z`SZf(ww2rRMfO^lSu6@?W}==*qHTFJQb==U05L zESRInpY~gom;Q#}sQ}xFjMz>rq~%cojJuD8CkqxfboGr#Aw>rX-BwVXWMb!8dh5&k z4ZIksAI6)qb;=FoHv@Tj(_C-frx`7+nQZY6oudwH?@BDgm!ajkdS2cWvYYb+sY`ur zw8-Q6wkfGtnP7tvF2x`*EnsbS$!Wy43uWRI?1|4nB*a?+dPiw>6SUEg`!_(6}K!s6fV zmZX5`tGKw5N4+K$^+hOcySTFZ>B;I)(1Z5&>6%HrGZbg(+(LYXLid957)&m5YL1SN zltxmwi}*i!iVG$GigS}V`5S@J8I2CR z8Ax5c4OaIc%tF_CYvmKXGPuTg*c!O7E+*iHGH9ef(|zW&RlW70g&@-fn^&K&EiRB{ zj>)pxTD<*O#;q6wvK8_fBsqbyzD_6TahATXznuP)!MXq+4wsBLB^V1 z^NF6fxqnGOu{OUTJl=_uR-yAHm7`*ph8q$Z7CA}HrUV&)eLE6O<{P)+% zP!#X*Yj?=d$@sg2mnl9T)oQ5Ouo0!Odtm8pBXMyG>$P-Wm@>ed(fo>JO$Oqs*$1;>eI)w+jrZu;@5X)W1Zyx?J{tyPKzs`im z=T($i-Y_;avFB{bY`#Qo!47k&bb772S6;j5g$AK8cxSvWeM*Q~c2W!HFspRvpU&u7 za6889%jgIZBx?TF`Z;0OH_P>Uq}-GbLh_gnlM1};W!447PxM;eS=C!Yzh4alTe)D_ zRG=pus|GN0>&^Lby`@vml0j`1E|%|#L$l*I0`Qho#?j{KkhIMD)yK8rE2}$sTfKYN z*DLqa^0tvojU5?NrZPJW;{&roQXCo$s6NWzjss1Hfv zQQ5+5HFLj+rN4rs?eStdaXreQ*2J$!=?x!QLD{hAQ-WOT^9;og5JSM*ipMNvA4tc$Ne`IN=Jm3N`Y!Er|*83~J;}=!* zCN`GV2I$TCW|6oKLc_jU@XEC%L#4A%szzt&+`*BiG&N^iXt z-$OJp(B|^YD9p&!fIEQ?o~3}c?E6hfshO3&_(Jb~{>z3E zBdkP~2X>j9^%=+g#>cWJOE?859RurEYy|4OHewK~D&oyhCQ%L&ql08gM7?!9iQC2} z2uQpdYG`v!)!)WnUAFet2NvBPg;65& zkx#^Yte4{8QnsuRl9FFBm5GNeW;dgci&J@eiD&jtL|!~Swhj3|y+wne$1y%|7Ce4T zPb@HU{?$E&f(0?pHaf^l??SI56A$u*V<@sSV5&&}O7G&|LzUh2@Vb2^`_e+FRQlVc z72#8a4A&!w-JcTnU1d}AZD)OpW~b&{%4!yTY>s;YyeNEp$#dQE+X9prws+}zO^Wr& zVDU66XjQy=n)|>D_3f+jt+J+fDvs;OSxtX3Pr7GEK7mgA=2teCqqAd-%%+pt;;M2O zUdyW-N>3{KrAJxE6V@~LYtzM5z3KGsM3hvhz4<%?b*UWI_I_btH5)sw!j#sU6jQsE{mv%`v=te;btbsu8^eK~^k$s~8S}j4@V8S^(k2TAII)E%%$Rv|BTM3c z99?H199e?I4sJth6emU(znxVG!=-I!Dr1_;6s84oBSANxrNjRUI zS^TTvBjEFu14ZU-=75&yQ4x0xgQ+yiq2ifso#(b>Oi7Acq2T zib6pul4~}5^EToNdcLTxPbL=1bwBi^6&D2`y!fRnnOm`!S}@8C-DQf4#)iFxA8Ud7 z-$h$QaH(IN4*qbb$1n?|&klDufUsM#H$BD0sF2#g(pP#`R~A21G)+`QA-~HCqu95alK3$H z$W0ltM56FqPh*DHA#9KZ|6gP&j9;a@LIdAa{nR)(ISgA|sEHW}agfoRv+FE>ZS{$P z40V)_zda;@~8Nx)Cs@!LSx&VFjL%O!4u~5q!-I@xW z#Bj723`HQGbf*6x`2lW_fF?qdOdimXrF^bHeBS1F^^fHxjLE{byR&BbQp{8KaYWYO zEhlW8j8;_y-v5k4)g|kU9tQ$ET=JeDWj+?I0}8yq9MpEu2xTH6#!`W2#H0~n zRfiX`9kKEv53^oFj<5zkfp94oV?Xh5D0mT~kewTTFZL>-(1QuOPtT3la_F=W`=M&1 zw}1Z{?;HjdLqmT$&R!oTKTrHOk>qL?POY_{x&4zCw-)+yi$C1= zSCN%yEm_y}BeO`>g%9E8`f}~$T7FHOTTqAi@zFl(h}thilxt9j6PP}~#&USjv5map zh}J98H7J@7C@Rz*9&)^(IzFNej|$q%nH^L)PHU}C`CHOnpDeTQzQ?A$N`I}^csD2} z^h~5Bg(ea&2G5SJQw{QMK`FXof2r>(Z)l8BD$A8cwC(SKA9^`b z(l$t=hYK*wj|t9)Hdy&+-VQ07y`o7@Zh9%m#O8ElC!;crHR_jBqMUgo0V9&!@JxLa zF}MNY<4PQYnWbeXqnheOnK=)e>n~~iYiq`bh601#<-{;gf=)b-EZK6BdJN{n@B8Ut$FCRB|{C!YO`|v|11P+lkS5VLm3uc;)S+ z#CYeCr{TTIlgNm9m1mt9^Exj!8_y@NE+@t(Zy^uQH}Ai~7~eduVm!Y*ozfUTiIrV& z9w*|L0uyOx0P-inYQX!Sad!cAKM7a^+kVC=1rGlta15M`VO?iB{ehJ7Ibq{Tre;|B z`|&F!^YG~jF0Igi-6`>JpLw*g2~W}sk5HZBS_bFx{Kt|S5!kf%&*ENWL0SkSan&!k9!-32jnyr=FPDH@h2?Ac~ox5l(2?&(kz=!EVmF$k<)kL3Sn0 zLsJ1TB47eVKrs`@0guhZM7Ti*PVKFn@CB=DxsEc+$R5Ae~AS$655ViQ*ebk{xtYtqnTge5wfA&Yc*_E~6$JRU~c&+P` zrq!O%P#mOZnZ~%aabC2?#kly`Uhz=zZhAp$f}SODe{=KU^n&(;HA|$12OT!pB0!HD ze`*??q!62d$svVyopMBzvzCc>Mgq;?<^eWt+#9T<>Cr$8G(*@OpJE2xq6jo&g&AlD z2RQ{GC5i*26`J?b7*;^4M}04Cyq98>0V#vvy)^q?3Vag^i0ToE+=Zdp0O>DApdU{D z;)7?<*lIvOXhNZ5rwBF>v`oA>g$`E+Ypn23|(U>58 zp+!9{Gxh|&tuf<<1wE~N_5|5228)ITeXY%V>S}#Yxj6vZg}A5g_mtS_o;GrzWxIu& zM;ufS4LlkawbdVSqW3o=y&D#_Cd}^XH(%(t)9tU6MP=dy`Q)PASvQgIcj0*NBjMWW z3hd4SGaw$_8T7JFTs8(fPJQ9j!PKC?Ox&;-B-VY&lwP+Br>lsBdu+qlT?1x}d2D9T z5A_Mq7^Uoc=>{OZ*f?fx+Y3U4{UXt1CQv@ZElaK{380)&b=L2W+l4285Elq8F`1>$ zlsj&^myB56OWxMsOO{NjebV&iKhzcH{4AbUzd?4n#u(`AA|VwI#uI&+SO$EPsH>d`!}e@=$yX($sH zJ8enI7P0fR87CJu?fjgTrQsRQ`c}-sX$qr-hx?V9V2!wV@Fw>>8jZ2|8DKY~+ViTr zewS^x<6hGK+{5)s?XE^#EO-++Pmh)ZO}RK~NTR!aTGE8pT^v;u=(u7W2I92oa z{w;|=d9U;)sh{wJa6xfHcs%tEi)wy6o>UsO?<_WBB94K-0v_$?Ed49uM>;t8j3{aY z6)2wcXfxNVx?*2RqTD@BkD5#!^=Kjs+clsuDEGma}5Vq7J`l37T)KL$za3??Y}0nsMt&qhnF`!;gD` zl+L|?qZFX=20$sl%Pb+#=6W+5)%K4@PxiQscb%gc$AQ`KqE>=FL^e;GDwGbI(N+}% z*jFQSwWk`59zA*QW%dM3?R(a2L`3vH+`KZK+5or4&<1BV)g^r;h;X`jG{eiV(`ZqT zKXIPNeDv(h%wD5i&8pIsoBGNnow3v`48EXSBfb#HxK$K&PlN91aoj!ayQk13fb!rj zMv{~Uq{0`pYha7`nV9~J#5cRuN%s_lzNa$)%`?kS^xLg|ycGKmQ&bF4Zi^-ScbN6^ z#Fx9=1xvs=gS9I0#cuWHJ@u-)r`*c`?QgoL?f}grY)`b_t!`Mx&%o$)-EgFDVHh(r;k3|)B0^HtT*Oy-Sd6>uzEADxopV<(sPR@)!d zddf%RUcu2C>yf*5FUFkUQw4s+ve-18=Npc^e41!Wg`*486S3f5jm+|)Ke}|R|Mf7- zyU2|$oNxc#aD;dM;X6A91g4WV=f^YwJg#kfTlFFxQ|+Et)$fL;^=+!l?|;G+YolaW zr*gF|Hmx|Fv*|YS$Ek3z8btbyG~dw7YK@7!-uC$B1h1wP9$#VThbeVp3sPeH6S#Jr zj^^&jV23=*x}IFrw`oiW?e^l#j`(vr9IZ7RSzXmqX&B29ai~6L(zc76qY~rZ zJ@%v17eLd)SB5rrC$66%E#D2zFj~Ft8>s)X;q-QsvQYW1qpti(0e;HzoLr>r^9M#{ zA1PkhXtQkqkMq7YJ_+ow4V$G3naGtLzkO>O-9Dc#`&`gpEGm)g`QnWJl=wk}Y&X>) z2k?i6?n~yq-HJM=quz<{ub%9+OGTh+QV(&+XR~rncM4rct@k5FV99x)A0)YbyB*k| zf>*el&+SXGb_r;5YK+Y%B?Tm&Cz69{bNjfBqX!4)lpSAAF9_2vhpt~9s9HWqkjkwe z33#7H*z|OeBUiAPhCa|ew;oo$DcY4NB=*@xiqenL-g&y7Eb8@LZpFxUa>bvCj0zZ= zH38wP7EQR6?d^XL^m4OcszG&XNGu%sB}dB#c35nv^Nd>TN=$AF%v(6*BE#PiF&zsP ziGn*xQ=+yXy&&5Si+lgE=#NnY zo(DxuT6bG4Gz$R3+S9j{ce4;;wj)#{ZqgWU($i zA{=vNa#&h?OMhTE4jCJYhSh@{p_VvXr|La6Va_rXo1uA`$j1aJ~9!;PZPiX)w#a# zDJB|8y85Kq_!F%P%vd5Y<9b;)93#Tyko_TMea#Q)q}~xWn}5VIQ?)Cp#rYO5YrTzu zYR)_vFKh6}d$`|SB0xrs4=eU)vJebbRX-+s`^F$Ce@Cg$)~sKorW6nM!}9tjAw&4_ zEK#8m6|i-x;wO8?nc15=*c5`&nc0X<7ng^I1>CQZP4 z!Hj>W{yf=xf%p;L|HfND1SX&VIp6?T(2l^5>JE(N*i}`HC%FR(o@;-;ulhM@*ipZ= zL#m~I*bPBM7A!!Lx=pvGi7|s!-CsZ$7M=4f%%!e$e`kYyR4_BE8bo%&7xZ! zF84IPSs$8cruY=tDGl|X!@gli`G4b?aj^Aoolo{uX|j}(V0c<9Bq|&!GbEiv92_Yd z;ylN1>+`*`Z5}RKTZ5d2r0d-g0)iuFRXf0!ZSz#^5I z1vxgz>@SlnvsV2*Rmdj)3+Uf~peH7Y%*61X^L)Y*2R3bCL1HSkJC#Yq%%q6}rAY}yt-Wvoonv8D6v@dUaPmWO#M9@d}}wS^Szr~JkivF2l+6m%_(z;`7w+8wKS@B{i=wFa5ywaGsd~5Nt3d!Z7|Z@PtXr27S(tZnRQ}9$nZ>z5`pf0V z0SVFA>^*A8Q@fKdREQT$AHw_dA`3>$pjtE>kDT!oP|alw7>B27E2_~xvKW_7XinD# zqg?&?MOSA@Ab$0hxHQFkIe`66-7uy*h~RDR`+Y%8yd4QYPyb01XYZ(wEOpD+Fs@Q% zh=k8E@83|rC5vctYFf#nH*plg@m~^}#8A@o<#PD-e9Nv0G$?}GwxRb60F-LedC~hw^ z7s3@BTC?PTM8)*FDENGG#)BgkXCY--;8UOwXQ+f~&);~Z%t$!yqjm$KdTT*o?GZ*L zeXa(#)O>bd2yO$db`b~VH+=!PLbMr?+{#WHF0I*>oMl|f0K{1+Dlguiu%W4GLm#Tu z1^b4G58!g;%1P&&-O%X!W^L4B;N>M-%5u$R9#G{8C;ryhPoGhDq>8b}^!*_5mAl^(F+NViJ|FFO25oo8kt;P zGK%+w7|FWNILPQPehpyZZFyhTkiYex!F2QU>EYAT#IS6z^Nq7!&*OSQP~pQ)f9T=RVB$h0r-6{N*4WFBpQ!rdvzHY=J6xi4 zMgq_!)`%AZ%66Y+@(`Ky1L|eNxY1RQ%V?^Q-jt)*D4Z?7G#nh-CDJ#mng7Q7q9-io zR`TS68cJ0ko+y6sZYUjUyjokF^z0IaoS{)9j6(-@z}Kf-cLX{63;Ffy3Kf%=Ke8`; z^*kV2=U@1mm;YcwEIa&mZzzvR2cdO-VsbV>`z6%7K3_NNKf+VlgxfJ|jXlq$Mt&cW z2)>j~e||txr9vpLkIWa z9}&lYWdeKIXvXoUpQ#)!9TbN?#Qf(|SG?gzm9vaSGcpbpTi<$9w}>=kK*JD6QGE9W zjX~5cP0Yzf=j%o`8rMg>nFcle8RsXaRKBD4Kdvfmgyu`h^F?jI**-sc`iRZdN!lzk ztebmb+NVjtfWs`E&uyqRyos!`p>+SksP1DAkNJ|2L^Dh0&<9tiF9j1t^u7HV4wKZQ zR=ljGmgh&m>NQ*N)0gJhT*-uIl&ik^Fg+Jx;%)n?9B%;p-NAxTZ)wgdfNPY z0Bl(Ilhx9>NAX&pWyOo)X>&TQP93Xt9_7;?Tt_lBVb?E8CQsyCsVtQ=hp%5_Jup)*->f@+d9tbEvu!F0;a5&yq}$t;S*<)Hn8&9^Bx_ln_;#G07EhMB$9gGqUf zrWRo&)d5;c<28fPbH$$EcMHz0JM{`-2p!MYWB}OmYesak2soRc1;WQuFmZZ&0-fFLv=CtB9lpCt_>FO32yf ziiJcnEr6Yq@)O)`s6xX>7}ID#|UE6 z2*z3Ouuq2(jOli}eK@W=adittR@fR+6Ls0)8puMnm})@^`9}PVS7~kV6DNi{diH z$@k(1eV~5c6A=Di3nh)EB+I@@B6norwv8Nt&*R5asZ5^OiUO}dyoK3%OHl?Nfr_Xk zONgz%vYP1a#}p_&3vt%SVIF3wkIkT(r6h|Y7}iew_JO>hn8O5Goxfp<)2a~d48OTK z6dSG)UibJOyU0od<<1YP9p>%8k4BbXzp%{Irz9?{(&X7Idt6VhXHoi}Q{&@$q+d__ z4e6duToi&?BXNj6H#=KgUWqOBBZ{Tso6MG#9dHTzkrm228?P5V9jj^z_Uk zVMsw$`KLCC5zpSY_~dLIJq=UC7Zvh;jV(L@heTHCx4Zwyr%%yQ_5~3l4i}Z6XACWm zuD$?^5G;}m1z%T{ptJgd%EsLVEUFrk=h_siopDjf_iXY@w(slWqI61GG~W3yr?JZ^^rBqcXQ2wKC2mTpbVoSvSdpW**@YZnpHnM!%~et1HE72kSg zKw(R5A!^c4pBx{os~?mYp5uiU7@}&2+O0OMa@lC5WdHUJ4?cot>O+#}2r}lxY)I#R zkbNF>Lt4H58Bk7~Ru1WO-o{a(32)}>&(RC4c|njp#WBWh+OGXC>A+(>#|wM;JJlP4 z_hksL@Uq;GXnLqe)fcDlDOyVbB7Q7y$ z(SHFNR5|Mnwi{bfvrKQqA(j>&%q(K2;EDZ*5g*cIcqS7bA~1+yu#3-0Ef8K2|GcnP zd`ezc9fEW0QbXB_O?d3WB>j4UT^#39G?~qbF{|%=UpKMZF4J;zXV^X_y2CIq zrLfpuJJI4tU`qaNX5$4I)jjf+!Mm#44%9X>AzU(LT=ADfQ4~cGusz>BqCIYJp3Jp? z-b>3e{u3b;h{|28ekCtnC*JOl7!tz)jum%xgA*|vG4{V4(!z5+PjiG<0I`hK*<(cn z8~Bo2v!6Bx1*!BJPMhgjO6Eos_P z8||4M(|%%(?c$T%YFs=%H(%r(adcm>Y5^RQ8hHmtto&sx9>S%)+;66U*O(f08h@pN zl!q5tAMC7>(e*p0ehZ44zNIFVM(@scI0}gpNTb`qo!|dN!rWcv1$lg~osR_9s6aYVJtdi%BXX3r&d?GLR6;3chBLPnHVqOX#W$oA)U_T7e6K3S1t`=S~8 zbn(v>pxftMv<|6PzE30A1z)?hk*E!?WHK$g)?SRKHn*cD36n~)!*ksDu$s&j4#YR0 zUUMhEZeXqxXPjUcb4iFLe;2AzRUPp>l&=%jC^n7;AFY2X70*aZBfiU)maL?8Dy+OV znIAFTp9-sB_7owItn_aoUrzFi3a*D>OBC2+H5e6(6tZ*^VUAc_3~et>1+T**t7c_x zqxg8cKJ{U?xkGY4#Z)L@)N593|KfYRK(?kzs!&PBvw%I=?G&k0?EoSiZ4zT#!5>kT zz9QBDWH(eJn^TL$KQw&2RoGTnc~GgkJ|{G?l6`&ni+~so-Ja8ID#N#liX5xJX*6!~aNndvgdodYI4W9eTUGl6?J%zEohUi~fw1lXQq^8Si_KpCGTt zs7&M==a*EQ8EL^cDdfg%am3t_+x$bCzBHIABwaa`tCHvaV3JnY zv$5!gAo=EwOC{T#<_(`OYacau#f;OKRQ0?M;RJ)*qu++!8}0*1T{Rmx>AmJ)?-WS^ zs)aFSx^;8go*Q=@_MjX05sS?ZV%V93hGsOR6|C@!0NBN})kYatFH{Z095cL+c>S9} zX^Z6y^t9oZG!TTFfw~CJqqmw06G>bJ@lqNO359&V{pWME0pTCJeL?h9@>3fU;DUY|X1J*{pa7Wj>v3N5hCXI#|aUqyD| z!U`vxd`$pzk7pb=Ros)Ni%UVdl{8U+#a)$BI{NR8#W)emXMeS(ho#Y zR#)TCpqO8=_Bs>{=g%%*r8bF>!a|Ynlc@avv)GfH0>zF@KxEDhiSn`vfN4($GQw9F zfP~O$*`ZZioW~#`V}*uUf|%zb{p|NDFHjzO*|H&`zTtzL~?TbLEor+uPvMG9#*;rxcJuD59bw2%P zHJA-TPb+9`m3g5o#Vynv>_*vmrmhriKy@U|6la$TOXVVV24R=8Nu6bk$#7a9lGYgz zj1G<=mgbT;=ol1eu|$q7|MifC3ss?qC<}Xdft27e(MR4m0_Jg}Ds$1)@)@p#$39~l ztf4>~X}#DZ2g*=?@(acSrmYQsD<#~KX9eb{S6lLpKoB)g@%}-6-SXKi?hNg(Gopb^ zIQRvV{54uF8v&e8V-hv{-LNsfrR$pVGmNHzk`HTFn5o@cUBmpsAXifD8vazIABzog0N4i&)L8K0&(*06CEpOs1HQ_OhNj!Dh$t- z4p9b<0oFqCxHf5&HLGP@TSwZfmP>ai`+7fjY1_QmOkZ-;c9|W0&p5#A&LJgN@-W(eDHNW~6j^&b>(Mtnt*zC{ zIfl1ilx;xlurzCtFFF#SAzt^mLDI*X145eYo6{11E$p>2_@P`2t)lKfd4RQY7%r!S&6pt%g0kc5s!@qNl~eJESYjmyod;HIpr+lyK@bE>ZiQ#puN zj#ryEkKwdXEG#lF8am0WXrpH3PvKg1D<)ExMMqZg)iXJ6eE);Ty4~R7!u6j$bijUp zy|w*qYSJ4a@Or~wvtF1}4*5${qT78`meQm+!2{z}-ONvr*-(enqFXTsco|d1 zU%Zl`Abpw{`Q{4;wtuG|1UMI_6AHgxdq@klP6I*t)7q{@xgJv8LESg8T0e^@ zd>IH*cHi#zOuASn(TSIzWQk({AzrDNN~C_X8c!`v~Z+_vo@`rysRHs+aq zv5i5(e-QyL?O}xeX?A)M(DI8|_X22M(hI)K*pD6b@5xb2sKf@>C9)B>)zmBVi%*!s zyGarQJLNy@=i|4iUjI?5tybJPBzBayxHhf0>XpTS$pwDsNg8b3fM8wrhI;i`R8yoN zW0UUeZEqN13~9)kse-A`g4aXpRKoK#17>57B52CrwPUx%P+^t!xKW6w*8s%CS$xEe z-(mR*^$IdCpw&B`>z`J*7v0_tbN5Z5e{~1JA$Sjm9?TulpAf2x6W!?B<^(#G5;X1|N(+ z=g1QEz!`#SPQ#`hWg7p%L){PPLM6$7-04;*Mx~E|@>M=FUxSYd5BE2oqs6bRE_SVD z-wPgvIG`GLycNT-41p^s+h})H@2$cjVsD3zj91IOVPuiwjFQNw{ML7Pk)q9@PB!u9 zVlMll0k&fn96FKIp~w0!$l?rkA$g%c zUH7^$s%qSs4(Dhtgt|d&u9)bT?rW}L(~4eG`y-?@%D}5dC1fw$51eT=V!C%8g(3>EO|!WttB1Imgjj{dA#Vze)W$pNij@4tmWdqg%zhaVCMBR z+@~%)blohEimX>jEQ^z4>6^eiy@Bnk%D<}H)IV*PrKo{nujnn$sn9#E1}LQ}g0X^S%N4L!IFzy6aO8+E8DOxw+T`p3o6{P3T-k z?9V3~9xH}m7dM_p==8h0NjJAIfA){x9gLK1wKnrb91t~B_I02;_cRhW7I23ACryAo z7|VInt>{Yf-N)lVC~k9~6f!O{vhV&|FkNf|`TQZLp*t+0bTfRf-*1P$OA}=Jz&aGT zTU&KoTE|%kz3}ykcz0XNF86sVh|)NHC7o-~_JlsPKxNz3=#T!emH7*V`|gwr88>Ep z^}BtyMWl7QZm_?R#~&fs_4qHiUaQrXn6a_X-lzG&^fnDg2sN+TX56{*Wu=r&?+;bg;CDulZ$+X#9Vkal9PkOS z_sq!!?;6H+?h&|f)k(e2&1tSMUy1Xe;PK4rK3(I_;Er#7xGv;k0RrWO05{vsBN34U z#=yBqll|$mEQdb97xFh$6JlibM*TwcAAHq6{Jj~lr!#lQ_3<-YXy~F6BQfLN4FxV& z$&cb7Yy3`-IE&rk#!CKONX2C;HD@KUI{Xc)cL-L&5)t**&2C+e(r zlSME>zBdJYl|~)7$#5wt7KfdfgCU>JgVZN40N;@9Pj-k2(g-T_U@-)(zH)nrnmPf~o+4AKFD2FlE|obCJI4 z8@eubuEs3O3e{NKKLvT^EPtEwl`@c_3rMffDNxqZxrg1w1cE=&D?Go_QKo;~mSf-UC zWAP<}oB7tEzq^M9?i4j2#*EI!Q~~cSAG<=YYWYQ^ed*wV#BuSpA^sO)!eV_DT8|Yz z8(JQxR^-id;PRfr(_Zkt3Ap>tpnX`xl1c|UZh33M^QU7GiEQ0!Gp{OPqX2Uk1>mz* ze`9~wI^2gjX2p=Zb0{XfP3O)b7#zoFI;7@Q1-d(`@xY4y^Ev`%XT*mg2G|~^fOS(1 z;(R57zPXBLWI;dj_O0)V}DzJ2~@k>g34JLE>XV#+Fvk9yEMcC~qRH#5{6$1BO+vQ%j_POEZ zo|7=_xZV4M<*_hihI(_qo&_xN1^<4K^$H&C{{tjUN8gbb@0Ny_EsZeocnQs7CdbS1 zk=Sj^bpFfhT374a>Z!wNcyYZ&fI}^Xx$9|qX5^ULejDL0wnA1( z7$opfP$?4G+2N=@5HqWF;P84syqc@`xtDQr?fWYgjo+I!NsNc@FXEkkqu2#?$)BFO zfxL_Z#a6>37C9DqjzumU*}A2}@{!;a(}aE&ADeHZ!toJpI4lJ^y99p_)Cn zJE=aIcICq-=7!~>yhql3p%Zf}{y6V04TqONQSpSwHOtkf^k3d~iw08?Z*#O)f2M7< zQsm0>#z3H{OwXMvdaxHgc!gnU?Ss#4M#|SL=6*iS*Srl?n~U&S_TX>H_?pBa*mmTU^D;qjS3L8zVmw9bZ@h{?cJJG7 z+uC8^M-+w-Cx4P;;=S~)YW94*2}NMoQv5YTDn4-Zf*q~VV!tCkrMn$$kUI~3a{vlj z;Gl(EdblG}?I#M^S1cSrvy7=?H(~BBy125C--qtv+=H4?%m;@9_Myt@p{Zeu`kefV zWx@mmDj?JfhIOelYCv1}^?HNgo%x%X0m|F#O|i$zMPOQ!w?p+G%%^ACeyllj^3vb- z3S03jsDz@Bt-Y6R8Zw%dlo^RkB!RExou>)L>V7`zohiQ~4O{&4{!>17x+ENm*?t%Q zkG_W@Gx3g<=Xs%t{N8~5f~gt!h;d5GBATaoid+U>4;waJ^=}-XCzSGBoV;@Y9*>14 zYtH=&on5(u{39e7i%#lV-JbcZWi8f}=%+y9A3z=mOT7b$bqa#r)MR?vDYL*|j7j3~reCu*_7wfiIyuyb2>kBVQGHj?*nf4$ zmvx78-ifU%;inC`Bs*kL;1)%ZXD-%N9F2YY+aVQ38$JG#G)fn^?UVf`Tg%pyh#~0G z@oL@Hs7q#AX3t$9`ufl6JClJL=N?Z(9ou82naxWR^7&H zyc=~yDT(M^DfEr=m2+<*`6kE2sZyX=)AjFHv4fM|2Q>d;{LZcQQ_2Um-ELPJYWI7s zx|XZvpT=n|cKlnqbN?#`I2&oD$E4SBuk7&oA`z<-GCOww8YBfJ|sR1Vtgkcb$>3!|8* zg+PQhxjiddf+))<>zY|RKfl5?{FG7YzU=ve8{HsF*plbD^TVi+UxRMxpHaJ13Yv9F zg`m>xEJ8K+JG|VnnuOXGUC`lXjJ?9xyR-ZCW!t8a4V)!c)yUlZ{5bL3a(6pax#krjW1eTy z3STFAhd=zV?9-=k=PDe&r>R3wF`-J;nQHaxoRRyT#<&J}gF?s&rS^M;n}=yv2LCTM z(e}ywkMrK1NrTOVpmFGXN~x8_OK zLXguNm1rcvEYWhFL*PF1D7K~M7((3@9YZ~E?m{iB>F9BS6h27M`06P|MP?y{ zJbKI@{+^;Smr*z)`qsVoi5JsUnV@ykIUaSycM6e76~CpfG&;inQVIK%K@QWdb{ z6nSMkAT%+}ISN#!uEbXLq%#J=?Ed6xMWD_yvR?$s@L5>Y+q|R5a07jv)rsY2lK(V( zcX`g--YO`Wz6^Ns=rNsK2f-{WHS{agUjJe1D-^!0#flN5To|KLMr`@Af~{KjPB%$k z>Co{2;>#a8;isp1CH-awBA6p4VLvz*PVf@z?5VL2-)M4Ci2UG zxVT||bb9jzh~iPQ>D9}ny*qP5_mTP9)%e=ZN2x(FJ($#|%=9ohOSN8hD-WNQ)LIT` zOOQyja(psJr^krnGN-qEFKff!lR!wQeDB`!_d~g&lNDQ>yzD)W=Y>(xi0e~S}D ztYnY(j~&9jg_CCU%J?qwT|Zt%ea+XW*ws*Z3Z*eQNr(HIW9}A0lT_(s~ z?;k%NJYU{?%5cy+WM#7Ik=*;IYmF(?CZw0|pBev71cdoF>8;O&H%{>yX=qKyM6Wer{n z0Mksic!{;2x#GDs{94po+MH?RYaS_gbaBy_4r&=XciZgz+`>uVhjxDo`eU@#lX$bh z&l^ScHP5c1#7~U>i`(|lfupXTjQRVmmJ>VXmREW*==Lb$VBbh0$t(9Kj~h-`(fpgc zoqcf4^Lh;}rStk(Et&IrD{YbUgG4QuWyxMYF2R8O4>V#2JOWn!p)Fq#D;5#fTcq88vCU+7X35j4!n4XlX~zRV^MNiYvf$P3_rknR}y%@7au$`W36{Ll%^fD^X=eh+29Ek zil@O0KpxDMM5*$?spq!@TLvGRj2>et0Afb`JPDNuWCuNUj~5jhP7o z94br%R7GV!O$q_4j4EYR!LIBY7dHG53MCWyzF#Uc!Jq${UXE)>i#^o~ADZsPRIxTc5Y zf9)x|Y^`rr#!gcTp1C_*4STVOiv*Xble4Qzu7-2bUX{2aRg&cVnM+cNwU6tzcpDmY~J4^y+ zrqwS&m1{3i?{0FJyQgSn$n9E{ph~q#Y1<2T#}(OtiZ>hX)u8ozwdw24=(6N*GG2;+ zS4kCUn;Hf1GA4MFHuXLr^#}>vqo0iyv4awU;Ywa1EP=1A%I%^Nsm0o)Tv?ktC^dBs zptv{{mLbPojR5>Mmb_Vj$;Lv1!!)4$XDTdOj-UsTT9^aT(3~aHAX}A9_8(%dB*eA6|9Rsu%= zVsRE|zTVj{p!t^}XF&5y`T~IF`HH0GpYe3(Cwd@{$>}xT?JUe zR9>*#2F$2G0o-F~v$#wQ_PqM=sl$j7v`l=q1tipc*^!Ri9l5Lqq?niNhyYab6Yvb` zqeWb19>b3M$I}vBW2`c9>P{f)<;s0j!dh%i4+npC>wuY=Cl6-O8act- zweP)ez#SmvV=BPb)P8^kNE2wd9?a zpNk@7w+#4T!S!R>oLiCL9mb1Zz3p^v50JM0i2*(Bo>ffq+pTz31W2hV*)0Mz3h$-b z%7CZqug*1>zH+5xN-7a5X zDYml5J}*000MScsg(>sT8U*oq>elL-r$&@6z#>?Kiq7ersEastFqP^(8khjsZMd{V zqRgC*xyP;bCAc!P=+AqM`hAbHhxf>S3ZMsVkB{P%DtGTa%6_~@-M)LY@4rX?Pevot zWX*VW6=Mx@BfI~xvF%ktMPwCcCMdyWj z>AF@mPj#)jR&`EwOa70$_h_WyAx_ziGrv5pF2n#E8dfkpg3Gwx zBjU!j-qpY}?_(Y!)m%34xSJ!o2j)%!lNa~-ung&R<-g3QpYykQn}?WZl2;TI7FT&8 zemJIkb*S>W3?oMAB7Uf;G_j{+jZ?MSC0ze-F(m%zFBkA=fvrRPzs`$vczk}bt%ZXy z#%E)qiNH9K{xi5IbbpXhezeN+mJ5@<=kix>twAr`E0E zRa5Hx$G(`Bk!jDBitlxv%ZS|td2)(x1-J8Jrc%E>2|O6kyt!AuCvd33T#%)W<&5#q z<^i?S|2R6!ur``54C4ff1b0YqiWGNu*Wyy#9ZG>xG(gbe?#11$P+W_{i#rsG7cElS zZ~6Y+*UU3#pPghk+2qU|f_qK#kuD&>@Px5uDn^fVTJ795IY`7Uo6WSLg^4+K?%%C+ z+BqnksJJg8lx^cuj@FBHYCY+TU?MRl@=Q>!FG!Q9rr8R)odB2Y?NC|pAIr0MOpq+O zc&F`xpcnWcI^Jul`uLIJ!A?T7OL1nN{ax^>tV~mE){N%v@KB_n7}$Ad7XQg(aP*?1ZBBv4fj@2XK`2!zC?8(gv$Fe2RXZ+{=xAfE{O zsmD7$)f_+viEI-J$>x{V22H(# z6`3fr1i?g@D1W?n3y%yQbs!chOw^4+k+A*)ngcXMJpPm28ec}Zx+Jdx%A0Iyrdp|Z z9HH6}>V#nXVP3Gun9p>s@2J0J6sxatthy*nos-h^{s1G=*4cM?tHER~Dx%tNsNCAi z_(V%&2W+OyEb3EBfTNaEj)~g3E_^w2V{5$pzDm940gWcO>6J&^6{AR3WHr*R*hyUP{o~budMi>;Cx!Cd2dF zTdg7b8FOc*nZ**msF5!snpZ)gtIp!VZ|N5<}5*BFd! zVMOYU5S_`Tgu=^fjrxj3wKaD{g18y?<@u&7zGAn z7k?`r7rP;A?G8q}Ff{qSQHbg&r%f6@_lkbct_4$k=?Cf39^~H=L&HFo5{HU|s$06! z+ysY`AFq0rFfvXr9xTN~)S1kUvUCKo9(_nqR8{tVhO<;B{yPEC?(WMn+cTh3O~n2L zB`$OG@%~ZORydaz4tHot$|qP96hOUeHuZK3Vq_$yE=gNJ%;E%3Bknhov7qVik!o-Dr)1|vy)wk= zXUH+BDO|o&!tE~!FUFJzhi{>ituDEJ7$x-jM#0Hq*|v1D(EvdOO_Jqo@j&t;txtIoN`juQDtNcG?Y6z*N5$9|>aHoMj}8g`NK+ zfG~anbGm5A+!C0_ID$FP@DV+hJ$qen@M9&(s{8jC8CiFA8rnJw4T>{(>bTHc!?5Gt zlzepjR(N_S%n|ZdQD$YjMP}Ax&E33s`~{5=h7PHhrK;})mu}p^Tst=!Cdd?RVZkivZxUc=wh8$?r~I{&HxCe#)zM|j z=%`_t_T~mM0!+5a&%+V51~3^MMw2k?4X!=82sY;B8LIBk$*R|lF^PrRRwc`}Y6MFV zneL}&mb6J^7GqY%%^tiyThY5&ELuiNhR+`wk9%DbO6++GISn2J;2A#7xKhFLs=YAN z=|6ZH@8o-7*O;>n=7xU2e7N{wRAb`omVEjp+z6493Y1^q@&R$V;&E6o32<1nJ4Fz1 zxO_yG7^rzKBk2>rZSTa3s+W*$b!f_Kolc6P@=;!vD=D*IJ+$Omg>4s8_6$3;#AQHx z{SYAntB8S8=s!&eu?Lv_j3{?lQb7Qo#_#F!2gN{%2lXKeLy6fvAP~%P7E44zIZ^*X zx*bON=2ATYV~%kDC-r6Kg!XJIaPWO7=1{dy8BArwL;G@@ScjQ4(1KPlF`73}01Fza ztg6U3w~|$K&1ZR&6NnH1#neF#xJsvGg9zbFI|h=aubJFSEz}P;_QkPR7T8 z&d+Mh{23wuW#Y|3CnAo1L%{tppTANrUy#RSzf;dBqxnP$0Iq?8yaIDO;z5!Sar#8f z&}7xfV=W8uxm5%0FT@xH-?Yi-drjv`JDuMREA^CGrTMXB*;FyUYWq&Li?RYD#D-KT zv#|${wj-V;Xz9Im^;RDI*h;!wt}Z^#Hhc8~<{Dogk~hSHY|L%Hb)YH-ERcdGZ=VAi zk67?IGeapXCK7O8YqZ+2t}id-hfAzzU5SIQxiNfv&Mj6(9AqBZ{-}N4vhZP^ zxG}uXjz=yNytng~a9iXWf-4w~+H<-s_Gw!*_c1IlX9}E1;$36wIIsVtm4hi^62IPF z`bNZ;B%FbyWP3O%{E8(su*Qm|v1~H?eZ4Sojdz7d3k!^B_Q@O(p{J;LCT*!6af&dG zi1h+lL!8ynh`44+`PX#oF&cfvALUnlzzXnNw2WB|BT!k>doSt)E8|qBh^XRWHP3SOYy`K zY@;(av)P1-Lz$ST9C3#?@X#u?SUWp9n|Hk{=0P3MIv!2>@lMt(VP$ky_1PTrW}+#v zM68TT(I@l?qM^Y3WF~kcK;b8B`GEM5aoDv9TknrQnw)LWA51^tTaprs;?Tn^mxkRo zEK5vEzI>OaT?Jhuqn1T}5kKKIw@!OZZ_n;V^FbG0*}iG@v;!e!cIHB^uZ`Oz`&LNi zy{G#>DgE-PVkn!EzwoDmdcXwMn4aFLTi|lRNSH!qwLyhFDUq;HXXW5WhOd-ivkNIbPdqjN-IEqYB`-@>KRY9E{|q8r=q*klYZMGV7oWZUOw4H~Y< zC3}TNb+T=qO%!U_Ou|GrJR}D?gMa+yGUV#V?LDQmKEmlKhD12VPli$>?W5!+W6AkX z=1YUCcYj}tYzcVZ~7mU=q0OWdoWgmtqD z@|+2*km9?-hcjiyuw(bQ#}10VJN*gmz3lE-p1IAwugn+A1D1&~@dZAy>?-jkoPkz)^<(H=TFMIpfJwp@Lk{Ri}6>~!q zyiG*e{hh}{6NKo|ME~WTij!s9(u*T4ZOfS3W-+)m)A~4Y@x=WBcQKAwT9Y?SIv`bh zvB{S<#H&D-epp)fdrWCU$UV5Y1yV zeMZLBA2iH%Ea0;@m9)HDKAdG)bJjNv`WILT&Sv)7k$bs`liLP(@#@!76pSu|u{`en$ zYs<$(RgBP9`d9kt@ah#G8O(E{)}@|8O5=wvhI$vS4JSF)nd(xioy`ODytz1UMjhC9%OAOy`g<-3+V}FU<&~IdzB$^z?P!2O`xB ze|j4r7?a6@dua7ovly6>>5qbH9)dTIZ_7wS8iZNx_b2FEEt{ba)rbr87in6)^B^lZ z&d*9@;~%~RxdIh6#_Jgs>Dvu>q5rXi*YGABU~iaDz2bDz)J&CW6M`EI_;1M%U%68Z z)^V04{&o&U=p&DLuk&UFN3MO*a$}+2WD@M2V^E!K1T$h~8*`?$-ndC2@PY}4;wtG= zLVATJ9?@CLhz}zoeuYeqSA>7A>;L@feLfnU$hkt0>Rf%}k5N=ZIU;inR<{W!}J@~Z2f1ALj^T)Jqt#mUK+XI1M^~w}A-TtZ5 z8IwtR{n=RL;%jL}-C0S^$i+z`%SV#APe8FmC-H$Zy}S1-gnz;9i0kEx^0CV8nXIBQ zm#l;369MuHVvUpIEGIhUBzk|H)70_tihhM~00o`#fov^*@WWR_VWpV3lfk)fhC)&2 zD3Wq6dg@7bEzI?icHk1)hoiFeG^-tMuV!v(>e`;=s=jmpygw|v?VJP7Z!#RU z@956KQ20NhevlH!9V=U|WAVh%jbk*>DTekv^v?#+V!*|+%Q+i=k|l&=bU2J=-vnF; zt6T!7Z~3drj4Ug2%_N+f{9lX2&(GLhvIe}TffgKh&VdIV$UJ=FZt$Ln^fVaS46CH$ zT=X>X(yD$Dw%OMEO4Ga=yix#{Xdz*SFL7yj25-bC$jd8?r|bZ67Ajvj>lmlizO>e9y|w%flour;UW2-eC&Tta`|}K{kyvQnQe7? zfFjg&BftD99Hqzp6`ItK}cLaGo)wd}B8UEqf>$QHVTKLz#;wZ=#-P7WZzeJb!|VN?X!{jXYd zolmj)I^Y%w|2eua&y71Mfzp7#g_jUu*ETeBPBL+R$eONpgnW%>6#+C-W4Wy>#)wDE zd*d89U#5N2qnx$FH2c{c@5K|s9~L3{+P~v}-||2RXDmmXy7@qzU!??VM2apEGRsZA zyW+4{OJdDoDlawW%B+px?nKN&a#(2|Jc0|#O)X?1iQ|*8iTM9S!cB6g!6P;~p+R@v zGv!9+;C9x5?IZ~K8<&Lh@bD;8L%tx;f!Xo}xOikc5sAwgY^hKCLcWEURD>)SE$Mw6% znf+7lfpRj32yqh@7Yrj75dl#GE#+(Mqr{bk?WRKwsH^eRPghkJ(5~Lz9ktNL774AV z)4tnd@ZA>FbFXoQ3CG)I9N3&%g}?P4q41_8&%=se#-%L(D&2pR6bT98qm5h*(}d?_ zzLc#YW0fZ@!I(T9d>!MJ+mow`l%$N+XqD=_+fXjh!#4Yv9EdAE0zxDD@w@e8 z3;_v$f3}6pVwZm5UlOR^)YG||8`#okDrX_Rhf`#yFN{Ot@+uA7)U-byM_U^VZjxz| z@aH9cE!qPWa}=~pDeAO@w%GVv?C*yob)!KY83ujBhOqq!FlzPa5=J1CWk%X2?5{rZ zQG^n7B^-o4PvDF`5~p6EM13r+zrn39N?Z;`T9)L##L2>9RH< zW;z`S=X1Ffat;F@UtzpaEO8#c^2<+2kI=}5YWlr8B!^`PoqdcKoLC0U8MA43KQV?3 zrc=v|^8ef_1GkO-@dF;=ffUX;LoXVxo@9cfH@7LR5~KYNl~VBauC7pFBWTE6bu<1gY1iU?fYWtMH?xlDhUffF{zB3 zl`B;nb6As-|3C*%W4$qCwgY=1vh1hUNTL2;-FxWa2N%4QE~K#bTs&Pz@Vj2tDDF%= zJxpB7!!+4UyDl`87%v93hzVnC6rc59cmXoiUk_!##8%bdHo7Y!tWY-ou=)-!!hcd$ z4Z&~cteLNw5Gyu`6qA*RLqgtjm9{Hqsd66-LVj(@7el3NokJ*Y8ldns>cb7mdr1Z@ zTc@Mg&!8|vN%*=IyVF*lWY)yuX~uSz0;k~z%eB*e>|5dxh~vjNj6VKAaU7*2GJ0F* zQ|wtnA6hptKba_UNw|^e*qXbV-b$RPf!9aW<<4*a^71s%Jf!qiVnr`LfF)1!BR57* zxP;VvOZw2mpbEwj(_fqw&y@~wt@(JX^IH;2)gltnW;{e6Hv% z!gwP_FH_DXWWPN`8aaM%TLtF3#kKz0m=H$wZ-3$mq$bd{>{)&XkH5#zk1>g19yNV zmEa#-v~7tZi4mm)S$B&-_-NC?_e``ldXa;A$nTw{I9Qs5k3}J`x)QjnzEHlKwFzXf ziSR_kSP_3p;FckRBkwwivhaKy_=#yYGz6A)#ic+GgXph7WL+;jNaL7&v3T)qTOxge zcSL_OS=eqp`zm@pDrNNL@*A%O96&7eK2FAe{i@MUKT%D0DEqU57!9~T?l+R7CeX-pbC+2z1bub=rom z6(8)DnnJc7($vA3%?WDA9(HdH*5673{2!rPB^+#or;v#X7IiT9B7quM{r)aI2e&10 zHn~VmthZ`tMuIz935P^p<$$)Q?C)}|}YUD^ofJt=VVyEc8 zE4;tK8q?rSr~;IqB1?;(zkUaJE@|hw%t|GqX&%l;4X7d}R->sPTZU z9sYiwv6}U~*i?tzzp$8=(Av$Q>LN=|Zq3FNUNM<#kRM0|F~PxPB*8^V{1VLBq9w8h z3G8Ds!K+N6I(^ggfq(-;S9!|fRYw_`h3mjE!79R&`A(7MA+Adg8Br}J(VlzCZne8s z`Q9g0F>^oSTFl!}tbcG5V$4Ey+~Up!lG<0gv{fq*?BpJIi}ECHk8MXn@ImYFYaVk< zRFGHpN4{jzzquX!3$w-%>NGSaRIgbvNX9ine5W}hSfjmx>T5`9<;USB|C5e!6y zRZM=41y&?Pm_RF21ntSr)`&UqbmO=B%rH?INeeMTc!H&B%_v?GA+?4S%()+VhR@pu z*=XCME{fs~whH3x9J`A|o&9GB_0Xfe90b?Ohc}ZYr4a0*|CCcH!XQKWD-EavL4h+Qb4 z?=`XVytYI{r}`_ST05MFNA}*ljq7XH8!<43GuT$(G??7g@=_*uW%8sp1ph}c=T{-@ zBEklVaNxF4au8W6JvYH9E{0%mqL{hD&%G$T&^&|=T!4BgmbA(1ODx*946ou^5%X;% z;j37wJRpw#9X%=Vq+9#96A_lCP~|2>hnYv|aV&?}%S;k{d@SjO_%s^(K0jhU8&FY} z1dN6clqljDg}XmRBq+c&dXHW_vsL%5u*XX#{O{C7<;{oEnAHzln@1OaX3?H=d!7RR zJs%6l{GIA{o8RjAhaV)0@cQ2axim_91QgE~*T(W!9*!`wuQC+p!oh(%xQyTOGLJfH z*eROmAhFEBg3+&%#5O7Y6#!#+0I*vyI{YADOd`>+p!Gu}2EfCQ!0!yX`PY^MYhfzl z#ReSnUoJmyC02o2B0tivVPz3yd#}ys4YiWdU5r9f+4F89y(`d^xn4*8ptqVEMPIX-sgU**=2uJ**2jBQIABixux-g1 z7jVy&AuldTrbxnk6XeSlsu)kN_Cwi5*Ku=v~RbxmdsbG~1?}}ey#Pg$? zRinKUKFF5cfB#l6YV@St13z&o@drI)oz=9Gs+mGrOve8FlANp=dV37K3nzpG7()LB z7!qp+3@PY_kDh1-;3Y2gJ<`(F4LrOE|763vI57CR{7iYBEw4JOmk0OX?=(b>oy=Ji zBrgqpXJmXhwv3;2Mey0uE2TV|J}!Fya|8-}F^N!b#oP4|@nZi#lvjzT z4|pE3bBFOpv`5!;S5eK=pMSC7OGsGOD?a}L?bAM7(~-yCOn*+`#+Q`Xol`{IXmBYG z4jddw1U=$eE4==xP)t2mK(iBA92Pi;l?3kG!%+e~!%gWKZs-t_if2AEE6+)Ppl5U_ zq?yg3j|hapk|-bX^p!(@#;Y+%c}KP-(RLzPD}kTk4GeNObeie(v!CtX&B;ngM<@|) zSd7p|2f{d#*&gx!sdoO1FJv^mp~K0B&3-m_)s~l_?N=gy2C*`^+|cpo4gw`yKWHmR z6zwZfKZ9DBKHbon<)h4fE(p?Al;B2Jqs;+_8-z>Kf_I5sZlc6~4oRSOrpEpw$Mn$QJv6>>^28)|? z<H9?@|6`BaA2#Tm7-I?z7vAC;I#B_dn8if=ZQnxy@su3cZ8cCr?yo&R|UZSnpA zeicuZG*K{+Y%!aEP+%O7~#AN3oU)7>pOht zn0IzINU3xUDyPV8%E1bpHn$^QlB)CVOMe?F&@VLQ&O7d6L|2^Qx<>&W^3Fbd(FZFi za&It-1A2ZW{5jRO%!CBnPV6Wc@4Sm6eQ~C%6+JMc*{m22rE3LBiaZ;P8q!%u{LwGc zm6R806(IfL#{e8u^_vRd0>>OM%#TL^7{<#43{OtH7=|+f26XrU0~Add zET!Mq)D*ck?%V-sYsQOIEfJlY12ZNf&(P;>Ec_aybd6XWkS-=Y>Z!Jglzbza#|=~Y#9F#Wr?VA( z6ggS?(o&`VNu`v=C;u@`Rf0|CmOV3YIsVv~`?;Ruzc$-Pi=fI>(r&i5y70O05xU{b z-PI<#@cHjJdf|fIY&N>&bfal6SncxCO0ECW>hpr7`ZheF)q%QEIzF%T7?j@(Sm+kr zaI?SQp4ETDAZ`sk8|9T?!&K&O)g0ZZx$hvO_gtY9#V?p(M4>gFx48Nmd6d`~z^T>M z6h!N(ni{T9tZyxNU(%J@`E64IhFiD_opo#DFc{jrTqc+(N}cS)SG-L{2*!ucR--iU z0+(vBAWyz$ad_5Jb-5rLl8C(&TTc8jb&FTWpM&5X3`h6s( z!0@=)p&ZX0vKOp-K}5tZ!D zcz0sH=1)`C*BTz4J9>S(Y#{96GCDkd^!m+;fsiS~YYJf8gbgq*bsQMizG`dr!<*>T zxjLt|C_gTBURSWW!E<-@OI-N8?ziR!_fF*keqwbKxfTF18vxYjS{fdYI_Y~srIq3S zqt_~H27;y~E)=X~RidsywA5u6zz}zLTnT~xqYDt&vhO9ZdpqDQnpgnvwzD@p7v;qC zf)77lZ0@WBHi>?|*reY8Yz~#bL@Rjz;%&$k2+_a#5+dhogJ&mZfPkL5e&VkeG4JM! zc$|Wj>Gg+A0|8Uid5V+Ls-ON=8{R~&Zq?bfyBBfq36?&)v1Qld-^AI>>kfZwa9(BQ zPd1vBGvt#it#-T&=`T7$**@=Sux-ru1vfUla;wg~ixy#D8J%*i&U79hw=)m18T0O{ zV55}gO;c@ed4;m8L#|TVy?bgTs`RbFy74nH2*rpz^daW~YYn-uMAhC5CF;E9H z)mcNXn)QPJ4uo2(GxTE4P}trqfXCqwtd7xna2Lma8Hg^nzthZ4v`v(kdz>+jt262! za8O)Bg;h$)9FQFi*7m&9RJ{hB&A<;faHoo(#wx6ks<5e~v4}@3*t5CaP|BsbQNlla zkl8UE{}`B`^#idgkA{~^YPRIB10@*?VmVT6yT1L7VO*p`*JpfHkvpW;!Pn&eXYw*0 zd(l>8p}&9sII?KO>$8BXEJ>`qf`1*UtG7LNVk|P2_*tFlpF=?|jS4?0-ha4@=tXO9 zvK{B4YUmhW>tkYg?pWYPQNI;&Dy3d&i{KP<28pxG+7RFONmaym}4Sf->&)tN?v%VbE;ISk42X64HP_J2O4sgi#cgrEh&F64*d7ZG6- zQ%Op6pZ?Pm7`%C(rkb!4lEKLG_Z?0jVd-KgX zYejqB=eg5q&QBD{@Nvl|Io^DsRODmoNu!dFII@a$k83vlM{}tJX}j6CbZ-z05r2xY zWgUZriMwd9`j#fX9PmH_g=0!r^D_II0%JLe)%aLU+fjK@-tnA#0b_@B+X3Skg5CMGnn`wvDY0)|H&v1J#? z5?IicH90qq@dlx~l2m7@ejWPUyEP9cG;OqMATIVYOcy1jRy5n%bj zvRpVhKZ-}^Bc?i&C$@;faZO&1I&;@lqCe|7B5LBu-K~(eo7^Z5nAC%^!Vd169un{k z=&TxgD->x?^+T|W=;1>|uVK)kWEY>+;ADho$W(|gs|d7u1j~I$aL6-$-$FE0#NrNT z;wNQzo85#s{8gWZd+@w4I73hng8}I_>#kJj6 z3x4KA!9A{r@}MN;2rKNIaVkn$vZZ~DZ{Rt*?P8v&(+2~ zb|aRaLPin|E53nZhnTLuKlB8LOW3vxqkjim5*AM^5)op*(1+(E)`*V66e{UXYR_NP zd4!F=+_FxBOSr;?f|sTDdJbxfUPAA}&gI}uKZ-rBxXS-+7vuY)m<%{~M-v4RgCF=q zM#W?mN$WIQOL;DB@mgW~nv^)wQjw2R9wunm}MM{$O z+H&r+B=1c@4!w+h7f|z5A}lG%xg+L$X=BA)Qyagve-;O2l{A#JrlY9$24i}A`GVRh zzh=601FeMNY!74fk~}2^R&xbHQO#v$8Efre{WjJMT`4FoHl+*b`}T{&9)*CM0}THs z`;UOJ+4lFbqB`j~OnwvpF_GOUtTE{{{ew7iA2~o+c(eWxyNS#wS}4Bw-PB)(QDX9xCeR9rwfEH0^uISB`8$uNVyq!in19Tg6UW1{hu(AQvza-vL$ zpaG;6Z?}Em3&c70m}XI%QDC7P<4;@B%uhpi$PIt>h_2P3KwkVlt;}5l1k=aH7g>&< zknA1tcGuJXY#>vt^@S=YbwP#lCoiE2ABxuME(U5E@t(4B^s^_c>8$E*Q;Xn*18QPV zm{(HPVEih-W;{gNSO|>tB)Y3ZN2TxW$8?ZFAD1I2RbAeVQAS@TBtR8i=B2wS?8vc24W zC!)K$fFg)piO)3*VKcfoT+fILBNPkkN5bEUO{n_wzgN>cMN*QR-! zHE5U6W)EqW5j@C}BZa)f4ckxJqC+jRv{UoMNXN#N#KRC%WUsj?;)8uM&{h%bjglW- z{F_;vCYpPUSnyE|UuD!cSrP(rN;2r5GDZ$LEm5+>$WF7jZJF$VGVA)Borbsy@+h%S z?iI_kaHGGn*ueH^ z7^P8r{RkA#);ih;#`1S!I$~&V6dmXlp62FPe3RKY`9aPg1Pa%> zA3;Iti%7n~pYz|XDu6~nxz^T3 ziMrUhUW>SbQ1J31Aj(&(L@E8f%jYTvWaEX-PZAQ@;BTkZ+wW+H_K&D`| z;MOzwS5-6`IgDdZnlqtDMF#zxbWQ59nGtVxlR&!B%n!@ zyAGK|Bw37-R%Z|a4$gMo(WI?L1wDyeI(X^)TJbpHTIns6jp&BfKj+e?JMT#3S$J*R zC7CpK%bg;rDTi?+saV+^9vSpARxqR>M3Fdf!gGt$oZYgVS+aPQ*s)PifPLu4lW*oS z{Gx1%CT$m#LLDZ`56nSph7_SW#j5&Y-kYq5mD+Lx-^vJs4;xd8WQ;teTH*1)-&7^eos{7osu=(>*|tZP9Au32Laa! zDsg9qPNx;IRYT$0zAR$!T^>rM(%FGR4fi9*;$&-bcC1B|Z}$<9F<$MbRG0Z^M7XvA z6b*^QESZpiHt33sQzxr9V@}G@9MkPf6ydz&&1skcd?vFvBO*nRMc%#RQZCvdg_9D2 z_(@h3Dk|@tY-ghItCJtL5-_;&4_sZ(F!e;(C1?-T*vHd)_E{Gz!B~Il4LfBlc+#l z$3UmbI-8!33z968EU>4^{_L8jk+Xf<-52g-c@Jcn1jM!Ib{~42RRg0_djVSR=Gt(P ztf6=x_Ws2aazRfMXNSo73G~@EHQgBYGv4#a-rO48n41LoeG%KuO5XTWo@bU-nBLG?H^1Eq_&?$0|R5k z7_?*_C;Be^va>AOvfM>saS^A}a-oXyr_tDm(8gtybss$3FUUN{mbkyD5jD_))0Y79 z?w1Ne3qB3b7UwBI1P`|?Bbj^_Xn1%v36W+<3Ucm{{B~sm%4j$;ac}V}-A^dGJsH!&p+p-cO6WR|+Cmc_F=udyfhtf1t+# z`Q9Fg^y?icNcwMMSRpl|PJAe`r@Q!y87@d68QnvC33?yj8lU~*;zFVLr{XDYQP$Xw zi7)#cM*1C3_K0moyDJvjo+cfSZ0zlvXl8<0aY2>SZ}+>Yomd+;j8CQjb7|L7Q7m@J z<)fR(#n4cKKXYX?>DwnZ6YPrNDGF*XY(A~_KdRp_O`c~pIra(UZ;5U@eqq&m5zEn3 zP8Md!DS=bT9Hv_^LQzOCiU@6pEGiK+$J!aE&6@jH2FR8D%NG=Lf+Ko0eNUi@l?KNx zy-)AEQ4^s?SdR1Ha-C~p*@y&Wt*&TQI@wRsG?ftE!0y|GqgMC6ig#ar-!97V%gB3* zN9PDze#0YoxMl|Oo_O5Etk*`#oQ!YrSx#yqho?A|7Qp>&K_`V4Uk4TbTO(e#OK)3m zNOifpPd!K;F+ML)wmwyQ9m|My$v9y|-Hj|a5Z3?**!9;p-WFpot3ITODE2If+((Dh$Ui%4%OVjp7$y?sMT9wLK9Nm_rjdDp@WK`EG6acsD8fFW! zj^4u}1H)$6^Id#6Fem;|t)+8}W-Ngf{De@7@CajiyQYqr{ zgtj!Gdu80F$qoo#!QVM>-k7;5!758WoAJy3w(DY%dxNFyA=18;1JH)Wc_XGg`Fd+6-lNNBwyLh_;8^o_6Z$VsTt zBzX@w2AK}#BvXX2qyplJ5PCG%!hcel?x~>`IqL7u^K;B%^|2q;4jzK|blg8WxoL;v zt?;36Fnz_(r;E+NMs(?RvXb@Dx@a1G`5$3bCbxhqRx3Ck-=}8?c^q)Ns0e47S376}=kqy|6PcW?`4a6m z$zo6sRj_VCRHPxzUkmUS|}(#OC9(!*~HD%Xq&3tO+_+o%K)*W9#VwX@gy^_mxL zuS1D+Row*zL|usS3fz1H$Z^xyTik-j!`#%3SLMjwaSm>{>B5Rucs5Ju4t;xcXC)Jx%_q7Y z?u>~F<2~weRJflW-E^)0H4FxaF^~c8P-T}(to&&aEh8ILs$H5Mf)WPDB~m24qNkc@ zXS5i1Is?MO=bH;kn%e`feKnvA!FPeK{tFk<^%jm=JvA{`sf86^KbBY))ciY zP-!ANLMw=@h=8{8n7l}6SFJq$Jp@=zp+s2WK|X0vU%5daty!gr95UXXEt}{Q1zo|b z`~!q&7V}?5z;0oW(pdA9V|6cURpK!`^ydgp_z{h(k)`BfgNdTegCzx1z6X&aji6`q zCM$~<(KTBqTT4%<{aOJNb^(tr&bLTfT-nLCbM57!`8l8R)RHhVJEA1}=NZRfvC9SD zc_ajs7*`U_U%eASl?G{DHx)UiKsRHROVgn5NW36QjK-iNnuKKLD#6lNdZ;w)*N;0)g7yH>}Z`}1kDt0+Rg>#ASaCP67dur7RzN!gA_ zTYdTIGwlAtSsI4g2sOuy#RL+n)^);`55Nh^U@{CTd%h_>khaRq1+jMd_1_56AAAsl zL-4dxLV2u7mYwu&#vyhld*-!@w)4y;cytkcuE1sc&EJR{K}6mS#C&cbcf_!&s+V$g z3%<>Ayg^@4F_p}WYlv8au#J_zBw_!mD`6Gs zXG_Y~K6NR0H2ZgCR|h+pm|kzVjvzi`kG|( z#$Pe;cM#;hsX?V|K8YPTTEo9QB^SMBpnQiFY9bZ}Zu^z-5ie9hL+{4TZN_6_}uK^D#lG<-gMBc;|#gg7{ z^%1r4_Wmbznq`D@>?p%9%^hXB!2{j>eoXJfhfC^C13OT+)c>pnQ^SP})<#Ic*vrhq zyC#McmYE^+lWWqaj?=X#f1}XIu%hBZdynvdbj-vp&TNTQ4{dIdpslhbG2Dbm| zI<`G8L#wIewcD9)m3C^rW8pK_pgp)AZ}nH#nSL%azp8LEMf-i7uun8q9Q-y^wu97w z(gNkbHqH}fFv*18dK85$Mp9+fVuYIHr&mq!yleM`#~;rgiETgiB|e2f*cZTIV831M zjgw8tpvyG-E5cYAxbb}X^u_Y;p|l3#-ByQnU1EB^MWZO=4KY0C4NLIuVvOvw6k}r` zmel$IG+6B&II9J}B{A1oxYeq)CN-nRVldp==n}}sTjMs(ykOWnqQMs zlBKLyAu6`i{`fmN0R{6YAOq#8Eyxok-U3@{eKF}mMUY!6#%(_fNvWKEI{ei1zl)}l?lkBit|}3VwLjGb zNgOh!5#}Y>K{j(X?Z3z{E;SVUv3{(4>6mPbNrkP<@b4n?T~_47Ry5BZWNx442HCFy zY2X|*HbDNDL_)&s41r=M_7Sj870b&-vw?v}?@10@{)FXAsonKEnLCV5M|4PWK!DbU zf4%4)$WxpIR9YCvnA*}WiOoI88j(Md-WN|inmx#uJenwtV=kWT!4F?XX@U1tfHx~!e;#x1> zBr!HE$*gQjq^La)MS{l~(JrEjLgscwD70-AFw~W+37U>P@KmYpFBXCjPEnp>S9)8W_ zf|R+>a#`N`2VY+C|9kEE3=2x#g%)l$)BR@p%KiFaJHbG)H8X=X3)XtUlAhqH)2^6O zP$@-%@E<7rierTX#)x=ZYKlko;ZM<%r-gTymLP%R!FxK-Va;UC_i4-jI8_ItnPPI4!cWFj}2b5DNd zFULuqO?4Xg{OU;c2F_n#{j{Yjru?gt7$}`F4_(Y5#!$jyzzu@=WPddIyWT5f^N(TO z-MaB!TWPBPOSe%(+0(^KX*dM!)rVHs$fJ=Wl|0D5fn!2M7KhV>EM3qEST6ohM|mVY z@OZ6UTB?soFn=*iu5^fgx79Gy>p=8A!7H!RT>C=Vmw;KE)G0kHKEypfm?d~oKODJD zxGsNt5EEzegm>@p+SH!OY8zD#o%AR&@^AE@nCk+6ATE7)*X>}yKq*FZt+reZ_>@rs zh@`j4Kd3d>xnB!f*At0aA&<%+UMQ`jzixhew=K^yo8kU1m$O`Yt>A$U8HHJZT?(lr z6qV*odkiFy?}CZp3TnvKB7E_D>&Pz2jjID~y_{wj$%(2ZCd3PCE^l%dQ~fQar9@mH zsRW>KKKhIYcCoMWKid!;7rd%n4J=HUHvROiJiw#}px1?SN$u!MP?o>FXs!8b7jep@ z77|XAYky|D95AvM9x@8?D|E(uFpx$rz4|*F?(g~Da)=-4P}CFLWncL5H;RFfQh5(p zc^KnlMGLOLPKWvHHq0PKPtc1+>)L9c|ExP>Es`+S zl)+Dbt}{G;yvIYry>Di9;>N=rk27m;UC(*z$m*dKe9PjiG|F_{6HiEAfJ zySttHKi|GcM*o$t`zv7^G4&-qAn-N;?M)05jy3f{%P=xbj+C=9Voy4#5Sf0w?Q=N~ zNOgh8*It)~o1KrKaCDftF5_YR*HdezQKjex%v>*Xn5P~+#F}JZoAxbG`9X+_o~F=) zJWBXn5vDMOvFmbB@*buT8R`@EXhaPrqo4HGUYM~bD9~1zVceyYg(-~55X zlo4}&?h7*PCmS69{+{i)a{*W<%t3T7>i;p;y)J_qZ{u>N)i-9ncgosjSROKsadtIH zoT&Y=v#;fmmxJLgLdml`Hz_}N_GPdoeE?#J%pOJt`RyNbv&^#Ky>p`~#Hpq+58M$uMoJiKb)!NGQc?N(`JO}) zyk$M(MuQLDxd{5Fpv%^{1Nipzn>5I6c0t`n;>};Iotv-!>&l&#y zkcE=zTeFAO|MuouzLT+k2I=o7!U*!m@AZRad;C~I&!@;>FGk9#x9b`6TyK%zINs?9 zpYQ_rjX84*jda_e;cH_J-Mlw%5ZBV@=d^G`l)h?qHEQ@4fay`jf8mK*S`-LnH_v#K zQO2@YvK{6vs$TuD*0cf?JuIqT%dwOcaL{ZY?f=xfC%yJ?C#%_SIz2D;rDIBs{AF2<4Q*I zAxVOn?E{N0=V|i4Pd!5&DVh)}vN!s|5AzcAtbPamH}5!a-&qN=MCtC2pP>=1BLx?j zq@v6kZ5oSMS?Kuexcnqo2^BBf<4lEOj^o_Cu&_=lMx?()$MGsmO`y#%ojJVo}J67peT~xD~(3 zieOs#!rhfm%_>PsaU6z3R{=_RYlA}jiWX%zWpq(=0VVF;VT??F#Ki!*o5zhwYQv}l;+tw$F(YM?d@t1$amB82O?n$TRtidO6si|dx>$&O8r1*g zKxJRaR`*pV*LU+UK_toOPLU|2mUWC)dc807X2Cp3QGhE1gTm9O2lSba{p^V3O|d46 zGTj&jX3m@iqZAGL0Vn2|btFR~M3xcVg-gD)DCuz3MR#;hlOBHj7tG=yyG}twdH#-) zmEHRTT?(>+^;?(TvLU$p7R;k+*|W8uU8lT@97l*ZxtPFm8$) z6Vr7;WkUFYOg_fSTzyS~CQCG-*^@!Sk)hS9P$D)*@`lp zwd1hm`-YYjmAKcAEg-6iE$r~XQ0;n3rMJK&HGz}{N=)#JOja39I~$NPXe_ApzM(3t zs#YcmUGNo6W;SLDFh;Fy3P^<1=h~=ohS#2;D>KOc2nLFUGN~71o(sIX7G*d_n=*ht zILJkz`!kUnyEDrrp&OSlys09=vR~6E3dUqEiMn%+wD(;Z0!-P0lI4}S`-W(M_@~Y? zQDeY5Mge403g(jXH9>|nRdAo~!VcV@`2zmK=|inMXckGCij2tE8(T!L#)j-BU>O%Ee>L8 zb%#hesUla;gdHX^S7&m*dCLTDgEl7HGhJCg7j@LX8X)c71Ip-&cocVT$%-@ogzjxnk2l4!k5@uMCgfLxdS1UxT+JVO)xq_0a z!O<%WT;5i8F zq8Jv*GiV{FAytdWI|z}BI}eHZ2;R~)E^0>#c5vk>{$sC*WEstT`9~@vBVj1Td+NQk zc{AH_p1XD6`E*a7v3IB98(+a@9ZbxAy7A?Ne9t(w6op`D~*pw9ldjE zM4n^+Zel8f#vnSO zjWL|R@@@n>`_QjJT?u=5v}WX$1%{~8|Mf;shh4<{{<^{P-K`Irbo7$P1xLAo24mLq zKAcLEGt$!G4s5gnxbGCX=+8~Kz*Vj%*}K!mJ0a*W3X~6X{Brbi6sHK53lZHY$zU%V z(FCLtw;^P?ltCyW@%~7L_UT$&IJKq1`7Lb%15f&92Y{99UV3+NwP;0#tKgS7sg`wzpH6P*cHW-_D$yPttBBb+A)Vtw-eLy&SaP z5T-mBodjm%9{8d@lZD}C1z4(96!OrB8vYrs#Vd5N5%iuRp&W-5Viv^54F_d;@&Bxm zhBli@xdYsS`hpUjL9<#T5GVurYWtamf}OGeFuOREl=9wU;4k^Tu(3z5$=lq?6Cra# zFrrB2Ox^}-3v_NQ;PF7QmNs+JPSTt(*hVB$(`+Q;DbpJ-9{}+O3eh+GQ-P{XkoN4J zsh(KLrEoop*Z*0Wf_38sKOgClBznGm91SmxNkFprLj3pjmZ24YNgk@Qnerx>-nFNn-dp}LRw3So%r@z(e#Ts7(B(Ls8pD3>kiw)Ee(!-7F7(*my051ihBOY^gtp zDYsiJr)+5)wwi72SW{1q8&T(#kb`{!}G3wtcK@FsehVq#zMi9&ho_Z z8s1Uj=r-Ecb7rBWQ&Eo-NCH`0g~g2TmAoALV~DoU`VrLZ$9&`u{qc_*Qi?D`x)Uvo z=`o}cETgSy`yjByfIi2DL<=4{(N(EwbaVz;r8r~?b3r_{a3-|(k84Vt>@AI$2*)V% zsuKh9)W~e#$WM#)^qcWr31ScDH?@m26At4A{dh&}s0zNxlJBaQehxSESeAplqiuq~ z$ZABicMB7Z>o3hJ17O&-aK9AjMpYhQh&Kh-12n6nqgoo63!uQ`(@|x5@&qA}V9g>~ z9F!*-V#s!I7JSPewBHfdVIUe&aCCSqmBWC>UMo1F`}y(XHb2-GrIonw9ZVDuRDz+n zv?fvGB(aN0U_MFqeKfK%df7p61h}?7(O4$334*1Z7<-rm_U~pddPmy^fobuC_u8LL zo=#>iznwL<(@?BNF9rQ{2o>UuYO85Z~CB^GLCbB%tBK z^<7+aFN2)febOO*D8MiGNg!jvlmK<3QV@iZ)t7SocUuIb9Syx??GED57kIp& z#FMY<(>PFJAef}a^qvyXw`NZ}mO@yMC`C;-#Mg8b50aP#4wgdGq*2sgor^L`Dv|Pi z4Rq9`=z2}KNfZ7|JT5&hDZ%ZD7EJ)DQ&Bp?sN!y8O`>_0$K*5Dj(}Vj@FgaSKbT-4 zC3UM4boo8e>P-RTn=U)N^yfYzwUeP2Xwwc`5%bA z>rI>z2lke(&&7?OZ;cue-h5eT=9F-n535_87_yu4>4UCK z$ivTrKdMiUp3hJ?QFt`rBray`_t*UI@9mZPN3{MP%((7}ytYO5WEW zZYz&LE2^BC^y4*>&|#?nlVE=v2|Kl=1th0WHZmu-9P4v&(Q}#XLvkDEL-uhDB*_8o zrYY6G2@UhS+L+8wh6`3uA?K_$eQL16FK(}?Es0ti*5vix;Wrhg{GCZGx)-a z^~H<&AIGFa3c<~&%Q}qc7!~C+eB?M2gQ(29K#; z71PvZhc_&Ln%&H$RN>EHwQMgNjrXt+Z?5l*{mnz+t-|8KNAujU(4Ili)-=H%6r%;cF_~{LiMvbNkmvHR||Bo_Ih!>HD9}n_|gLO*S zyG3h61WEVU*F??Ud=0y`BJD{`z)#LJbrEYt?%n-gaAdcZ6$`E}>O_fB32%^(w!XT^ zV{q+hPlIFqA8VR`dYh#pICuz|!2}+${~G~OCzn4tj--}L2@!6t&f`k$%4f`!V#de` zS01uu@^KzSywCg(3afe57L6zrDd~elrAHZiF<1bc%Oi@90)Ul_51$=THNVov+>T1Hv|*;a&t}Vb?*m z0uQP@@Dt`E)B+FGuioi{z z^37{mEenk&oWvrObn4wuu&5P0hMqhsX%=3bv>d_@4i*50#?d*f$)n0lzIqV$9`8hP z=3T${?(8Hm9gw?*Gimc@UQM*4PFecXnkui0_`2mgxgNd+jcLa1RG;S#2y5-Yb&Hkp z7^+qFuc{1%z!XcfvYn(;U*zHWs!YC(NdKRImeodtq&_&npW0L=J{|~+EXk~x4pIKt zBezg@(W|irpU5-{F>Mv$FKReRkKi_fYdXKw-^h}s+h&#jH-i62fgpXCJV;V&U^p4o zAW4Trc?pA@o2k+vWN3xWd+?b~L_g5KrQUjs?trAO_7g>ACS5=cg*J9yN}MS_5sx2g zG=s3d7BX`HY6U|io5IU-8cAi=VFofTx2Pfi*yzkMY9PFV79>D7JM4z9mvmQVf?`ZLyo9`;(#Mz9(tOr@Vr1 ziD26}fymXxWyq^s6B+cS-(rfcD+j=r!t8=p4AY^4XG~TjsFycm-h0v=a70p>S@aMH zy1^=Gq=bTToN8^8yq(dU4eS9WjZ1~a%`c>I1xnU$R(J4p+Bbr0uluu3VTmME(t?J( zkyKt5e=W!8*55)4^3}?kL%Alk*fQnf^$`D2v1J}2e&+k!nAWP#t=kuJ4YSDxXS<~9U=x#!pF9rhsx~ zwn3tfG6#Avoy!-!!C1;jZ5Iy^DjaTmIpQ;*y$RZbkWZS*F>;8ap?^<|(~hf6r{9Ca z5-5im$4a#0W3T0S{=%zA>*`SI8=;)N_%UatYf%;|Mkf_Q=c%onIPNR2GCw$nwnwxa zQ$0x?N#GXk%?wb%o+qwjTDcy*_#pWfL&{o89B~0C$7H&PKX;2o)A;jq0Ny*iWZh)& zPdPR?UbvW27Cp~5Sr=Z83#nBdA=Y>0BT9YC7WyT+9-iXGy`NK<#iv^th|Y{M=l!wh z5rdekM(_y`Z26wvtjfX7B&Mh#MueDTTN!kiH5-``UFI)yMOv47I3U|!!NzL*S#4gG zDKl_dWLarm^>cM0?kfCfPqh!!6E3*+*>o9Nr&mIxwLOA=KS{=KR{Ps9N3B-L<|ky9k9I?~x`S{x@PxN$V<7iWb{-{ci#nlq?2 zud%b9C)km812}0XJYAf?ESUPj zb}0mHQ18`c__^xH=9@D!3{CUMsayLcpv>0y_EqS6L$O_wq0FByWUmYqR-`!I3B(Rd zxU~oL?!+*bjwz@dN35-pu^I!8?6JV9mm-U|SS9dwv$;>Rs$i1DMhWA0CetH{k+eV53_YGuj(Raqf?!7%1 z9Un^?DLg0}jxN+3#y$@!bZ^52;p#H3GR=7vBUUm5LwRC&)p58EN{B1d%N`D6p;N)V zi55GlSehrk(xo~#D-&Wxpvv+oOdhC**GU_rA3fh?m4EX|Fyw*eq-&vOU@?tUuz_H+ z-hkEE%N+`}ACuNomydhN%;F*$jAhefDOA^IU+iQ?ZkH_11R3!~;x)V z9I{;bvhlXNxbP*`HOd^ual;9hlxrb4VXJA{QA+RXk&#q&npQ$ZD+x&CQ>$# z!H$c-hB~N-)XT(}34!_lh9N33ObCVmhbY=4qH z7fx3$7oO$$RWNzb{Xnq&c5-invMANmXFY;|yu2+b$3#z;CCx&!s51nnGY{uZ! zQM>sg5`shy96WUobjtB=OcA0~{7r%vM}%II)?!+U-mb)v@1a<2sudZ1m~^%Hj{Lwi z(OAb9it^^kEc$1Xze|gu_pF~1=cx#dIoB5FyejIb?JqQN8zjj3Mef%jOjjpXFgyIKMa+ zy-HDu@X@V;yYR%72=35Xkw>mwSiznD@Wb>5&mr_ZjVDzf6v5;VCY?|2Fz zR^UsCRlrk)2}xMx3Isw)a_?h4_B%@b*J)Gl?+n&rMtz2QO+wLgN9K@{qL(ccM)I}X z<7-iQ*W+NsLgAHtF=u|gdzTgpM?(G|5b*s~3{B@a@l-{oYqLTWf%&xQ>?zlS*}(il zitmdsdPBN2XU`85Z;_;om0#;xo_d$$;R5a_*sHuOFrSHw3>Mri?TD_OB8qD`q|ELh zam52}N5O#~qKixZd8yNpy?%wswO^M@_F(Orq*A>#W0RF$6`c&eo( zDPDaTzKMoBavaP^^>&GdI+tkQkfZ_}RfQ#YL3K*(%;k(!03_g7vgL9Fivor^sIlZo z2S%1O)KUHAWd*C?Jm#5PiQ9xOTx*=Mhc!o@I1>EqMZJaj$q`SMi8UGE^pl1iIl2P5 zSKw135#Cewgu4Z&s53;a|H#d&rwIJ&iuDTG#j8~Yis>RNtwPX;EH zywsa(*dzn%aXt?E7knT@2P08LC;f^C7FJQ&1z`8UcYTr$kYD_>AqhX+=R)ot;CnDa z)x@9*fickB$8fCmok)$`kv_KWR|Z^Ur1X2OZG^#+=6fMZ7Z8tL+${gk8Fd-}zkFQs zMwB4EfDPrX(85<~ukQV6;Ey(C%O{BKr{lNf+ z@Q&S*lOtt86CC0KCHh6s(DT>V6DJO6gS~JZA;+fx@0@He9MkW2oBa{wr#VrlbifbS zYek-CZO-hI>F}sYDG%Y^oDSs^-4?5;GJizrm{gC=uQYMq$@9V5ia6_5I+n@v(y1X7 z9{@J1Mb_ihBjF^jb=h8OYbUYPcrAkHne{X0L~MW}>)ri;$-y5#;5s?qFB-tAJSCHb zgyfOJ*$_2Szo$b~Hn?z_WE77=KBvxeOcF<^5|9jIW_Qag*;sJH;}&w8C`=5FiYu~; zx|lpuKsaE)eTOr^0j+)`?WGd#2u@i+kkgN)AF=)@fS#hGKRJLO_5(}WG_{jiA&rwo zwY!zaJ)4-MA-{Vxkd*+mkLu#@clSr1N$SwjR@P?wPa0fO)(Lgqaf{q{xXv6ly#cf# zjO=K?c}H3sC)juRK*S^kpYfbEYj-eEU}CQ>H(0A~k#FSWrpVq@{DdfcqAC-}_`vZe_@R+j5m^T#?jinUq+&O2r` zM`5avXPi)Ag%h>9v@(6DS2zVN1_uBYY`!8=LSkA>4&gZ@*)ZR0W>YwPMDr0_1p}>) zo7A5QYKn=%50U5ga%~gF6E-hCGMm?=mzrgwh;31?fW{NF2o~VG^8K5~GAAZlvRq9g zhuiL^@fg}znP`xgYp$lx5T`nIsWy}FL%CgU7Qie-Y1{5&k?}H(nyW2>`t^tYMXGnK z=2O5)XHBy`6su^iIzJa~MZJHd43E&=fDsUJ!^k!4iJ>iH%7{vlGrScxg>GZ=}TcT@_{>E6w3}pM-@xzJSex;&&#AyAzUVl8fw&))?VrPzsi5@6p>;^^`87~~^*0qJM(OLO!WuL(~Hfa75wZkn8g#CQO# z4(F#An%c%x7waXvq#sVyF}+jZwn$|k#TibOv- z`~JrrTEal|25u^D#I5)uhjSoc9*6hpij~#uAL#gRZMS0aw#<{{5!bnto=PWh3n4#9 zt@%;(qb7q7BpBtElP_}v%X8Dbaso7Ysgms_ErqHh`~xU8?_~D*Rb@r0cnD#wJs_>n z36vc}^AX7yLsIrhN&uxO_Ak?R?hl4h(CO$^;s%`G^ONXVR)CMC7b&D+Q2TxHtA@qx znDW`r%kQ^voaHA58y6uu#Cnn>dYoN`HvspI6>Zm&*E>}o1%X7!ZPJSirWx6uGov{f zV_*=l!?o}fND%dyOR_~yN*9&0fqMQb8^A<3Qs9W$In(Clsas;3$QU-1Dvc^z@La_{ zwUDiPvA&o_e;>e5bWWR6E{b{Y=M=fQ)^JR!Q|DOxTF7JF5la@rV1y;JY2`);q)_~i zdUB7dSRh_M0@xBhYLyAV;)8!({Rz`jrEr#J`0+QzM@)HOR4CpLtVrX) z{c5HU+15(BDFjUtcECN}GqxnVuRxYA3~HVD%0Y<$%POxXNFm4gLSM)LSj{D)dbNUC z&VsPS`{><=O-z7iN0P|fNEuzQtkoc3mB9DO*&_^mHGd6c|wWIL3ZbpM`YL=Kv=?OhJHFFC8R`ic(h+*c`=qTT)Po1@m04)h3y?ewzpc{U_ zBC)o?&hniAcwbcQi=UmPr2zbpNOFo+#61_8i4x!EMrq&dZ_1krLFv-Z`dZd`&gRg5 zP$x>VVHW&sHQ;v0Sm96X`OicU%d}Ke(X#}kzRl&dx4}~x(is~U>!`OzxgXbJJaQ;2 zLcoS;FWWD{rSWftazx{9xiUy0baMR6ZOu6k&Oj(oU3+50?W6+lz1M%PFwYl zmMycH|G52bvX-i+$tUU)X_p9MeHhbhXzdQGzf=O7Xq_&-FSm`CZD1fw0L>(J|2+6p z^12u@@;lKx*7I<~s>%uAyC*5*+tkdDMp^L)`yD3BHiaVrPAt>#$u<~b$EL=+lopUe+NQ1+1K#;Z*0Z(5HY#-9 z(vKVKf))PK`wK+Z|BRCkVx1mmiLOt47e9B9nkme~U6>p%aF*ML6f~l~v$_8t+Vq8^uujO2O&9FUj5vopv^ z1hmv!K1eaJGzx02WtpIJ+b9MHFM|S%8)Q@c#?)kQEY-VtFpd!W?D$zbC<5Li>@{T7 z<_iG!1rwPDzq7qyi+^4Mtr)3y$GDETeiWjGsssyt$ z^=$EJSRtC0{~)fPyi8vYw@uS*#G{ie|6chDSh%u;{JI!1jxE{qf z!XE$)kGu{Q9K=q<2JyYHn?vD_6dgW?q8XV zEv0kqY`IR=I=*wu#*&PvS<^IIrFg8-4`+s152NP1Rfuk=@EW7vD7a}Eknbm-?{7fN zliw~j%x}e`@6vZnVh>oz;a`KFXLtMRLzXwj-L zCV0*A3Dn15D2DFXG;~JYm1Y!&|Q>tsGl?Ml_77RU0nc zn#X@>b#Y8NN)-2fnTIRQ`BFa_Z~@#|6JP8SE@@#)nUet+SQZFyM;Gyn3obzcfVnJicWS4EBrbvRH;n;&<`;6(>_cuQ4F#!TvCxp(@!~x=ajY2 zlM=PTeX`ZQLsSc!XMwoI2Ax*V4MvMBl^rxzSH{GLeW}bmFM*-ps>I4MlUidFi}q7ie~4I*sLLGEbr-+77eo<3tFW=Cj|pY*{AwOt~d z`Qb#xEJqnbM2{lP+ZNS4aMPtKMAFsF#nH*e56U$i)^3x{8*$p!>y0jOqK2g{L z?qwm6^Oe3R^%`11u+*y0X4g#ph+@oKK8MKzx7a}ir3$83i)aPi(mGL&Uhl8a<}s!j zQwqrT3t{KKb8&#q zA3tOAI%j(Sp8lmV=4NS|(0=f_CqfJE02CxfZW9_H`p+a2-1<1T5@hkO+|9<;^U?ez z<@n&BY7#*>aP&a55x>D@Iy2$a1?>8#axeJVG8%`*;DmSIJ@<#KJ+On!whc8{K1v#9 z7HfmiE?-+TjM-5A13!is9*hQHtcCM>~6Z^;JfuTTmzXKpB1&FJF1AR-Q zLelU@cH*f_Vw>688DdP|-hVUP-4=HA53#Q*U1vJua$99Wq9r#~utg|XdGu~i#mM@- zU0Bd=kO{L3*7DU$%ARI^C-=gyq(Z=ms*0cW4IrM>+2;#bg@XLF?*wW6OT=hwhsRf; z*(PS6%9_@nJW98SGGo@_2S?7~>l%_a@eiGw&QyhkDPrL_Ey%v}Fr8l+E+6$Zpn)+J zDU#u@Yfuo>i1|z9h`>6+Su*oc{8}?31GW!{Z!mksR@R!n-@sM{2mw3OB;^pCnrio7kW#Z;Qq0H2PH zOLfkZ8IyyJEpdt-jh80HW8{8 zh~u_xocRPd)25BMrf<6BVNxo3P`@ADI_sEWk>=`A88tf!ZcVsc^4{k`dGalAIL1pW zD`@?9e<}=^KyFY=x2TYzhj4_rn%-@q`$aEwR|`GjMUJ!m{LN}a=P#^^w2gH^yJO9p)6{@{Srgr@p|76-NE$gu$5Ybn3qxiL%sJXg{XjTk{rb*xZ;{U>ob zNR@f+f{X-js#==q6HgU2Jq^;+D}K$^rT5M<3$DnX55-v6p-Ywjdq<^8OZ?;|N?Nlo z(|SH3_3hTV2zaKj(n|Bj#V6h+2DS=)$vOIECt%ts#GXH`bF1pyw8bP_nL*jM5o&fx zOUp4}oX48Lr9lX}x?4|Vsk7MDFx5A3*^|UuIQ7*$p7s=PplqQ;wk5QQH4SlIU;%r6 zCytCjR{sRd)MGPlMMi8H)ltCH^nEkd_zCQ3cVC8j2H{g0-85RE?=!yv@l-UJy3KKY z0cM^qI)o#$Cb58hM!Jnt!ZO~{G>D`LhP{&t3hl*mINSOcI?gy(cxj7p;%oLNVJ;rPoCk+d24k9P&i5jG7kvSNT_ zqa|i1Jm-D(fvlqt4D`$j*+fath8*nw!WMQoAu#$HQe<19g54Gx%@Y}ma9Xij*41=H zYlg_LmBT`vYBI8RRGN;g4V1Glzc zqPM1B;BTd9dOV(Q>7u1+I1`4m%8+M35U<95U3n5-m__#OA8cFoDmbbEtzs7g82IwXxpOg|9$;5?K2i;Iiuo7yUxbev+nvsSroSZVO z#_sCI2Jp~s>RdwIu$r4Ge0DHe&Q}Mw(|!yHXe^gQyVxb@SH>8R`>gBRR>I;gW8yM5 z|1f3@;qk;|c0QbF``7OIF}QJ#HKq$Cw`}iesPdF09aepbz~9AEetc{_G@RjC+c+zn zboN$qV|x@hc2vrWd+J=>uxEdl_L-WM-*n_8@NQ)!ZORd&dc zfzs-_?^uK6yP2V7qWAWhe|EZRn^rz@rR&H$HsK{XbnR+XEu*o>h0ERwG*d|O#xB`@ zg3vU0biQ#`G!94xsP-GCJOmdd54KPpbCFRF+6wtm;+aTM58ARs&RC0C1M4#VukZQf zvDtoLS@EXIReZjy#kS)8v-(*^XrvxDZ&i(+-kBRf5i#}f!9s3(HRY(bEl;jIlx}y^ z($m@+5tGszEU@A!=0ou-v?F3G)%S;k<7+Wq9@8kG|5Mh&zBrqFt!*Di;Aa zbUX`)`Old31eBwq4{2C6Nd3;S3H}&@@4T`A=p5omnHr==%!MRDTy+rbi+6IvrAuM%8ILW0I zUNvJ?c+UN0(8?u;08I2|1_o{7xc6%5BqVAD)iiUt<;Z^)=+0R7$C5-OTu2lb!#r{M z=?1RCIte*B*T1@&(?`%tAT_@fssK8oo|*NPF1*n@q8eMxAey1jP5Jz|1dE|B7#-vZ zZxVW)=q~C-p!v1bV9)HYP#$!GLznX>o7xp2^mdi#BQmkK?ZP1(my8p8iVSW}7ok!i zn7Cf4?dHSM(#y-|=G^P*G}R%6*DL42PEb>km5h}xsRUo`56*)M zx(G$eO}~;bRNu#%-m}%DTfGMxkAs|W$w=^K;m38d<5ik}?bu>B>JX!36~UgwI|ezW zuRx%OZDx1gFKv!q7R!dHnW8gGnZ`H&av)bZD_w?Jox)?AYb#4TRxN?k!+7t1&rUzB z*1!#k1HX?K-kgV5og%ST+E|`R9YjQzvXk|gi5&)&%qk+^zULgeF4pxMhVu^!k8NE= z5Q0wO>aIO#b}44no{1e>(QYzUHSUQW;>-9q^7&nC9>jHR;J(2x93BQ@Xi8sr)~4l; z$NAT!WDQXKqBEjZL#X@`TMTC6vzt!9ZX5iRU*=mU<+qo4Uu?(L$#AQd%FU5neB$AEk=2? zz1ccZ)!EQ2wK5v1gQc!2O>eB7=ufvH+n@47)AQR%*p#aQ{fTibPFp_YgIR2r%>$nw zC7L`L9i*E6_a7BD$wx;>wL0vtGH@0RJ(GORgAmyT?A(>_wEwf{-#D+Pq1z9gNQ1w+ zktmNff?k%`eP8V<0@7LhvO8#GzpDT%Ln+;oH==^}?DDPjj?d-7E!qbZ}; z(@I^sGF1u1y%Ex@7%K%m^V#f8(G*odxJiq6uZXJ)I_Q~B)GG#7ZN6y=|*A_Tyj-w+T9VIbxSL;O%^V263gXL)l6D>I;9r(J^lbzSq<_`8kt+$Szn%P7T$zMk}F7yub=q4?fr~KLQ5uM49VhttF<2 zVxqzRiM*#!W+ISqt4rTEtV}lGRTL5K`^_OUg-CnFkQ)Hr!28X3M#lQ6 z4Uuq5ELF8*!)b)2DmKtvL^)f-E~p=AqX}^NXt#(&;dKxf?$_IeMc$3xkq|%03DKh< zvLr^W5yI!bNJQ@l_f6W1V|(C=tHq2^`+YM}XPI8>)R^&Tm=VV^y2c=Zq%cL;f$!1m z?ie#imZm>% zPQq`Pl~|NJd%=&{b&f5O?X_;_->vib#ItKF6w8c6dlo@{*&7CIS>-bb2kNU!269Qw z;cHxXU5SIQec{A$|0C(D!=m`Q@J|F`DPaleT)J!N?(PohTp9@}>F#c&8^J zv_Q1`e>n!1{tC4&A8*4|CXqDsCQUQSa$D0q_@k!KQB8~3YBDE9JQ=7rd8uIP`C8j3 zGk5S@eHfE-xQMK3=E>e;7de8N`{P-rdhV>caD*w3cfSvLh35{xESx#-tw2Ljj|M0C zuJlxgfjwYezbXthJ6;09_VfIi(N2ifw5VSGD)K1S7(dL+Eq_KPyn4y*`GNm5CX_j1 zQtLjWyE|9=ngebS$Q~4&^w~0OW%*7a2bLMw>qP~FugIA&i|O47BvG?(Mmva5kb?Wn zf!Q9-PhtDzIZXp*u1Ko%nP)ulwHk$act_e6m1>h_k6jO1I`X*S%dS$9pU6^LdN(-P zA$EgPD`8sfunM=BI@AE%LK99jq95l(la%pgV?9ULirw6<895vI*Bn@ol|$LwX4Iy0TyDrX<8 z@`?TwbAr>I5}I>xes@#TL`WNg;x9AAO{8K!F?MQ|8L})paYKi5IV3MPsHX6zA}~pH z(wKbfG`l8%k-<%`g_aP-=*hrO-5atMU5RprDcF;}-!M~nE+eO2q_2h7jL&nB~8!ids^=sGw?p9P?`84ZAqm3wF@(4Lqk`8iXoMrz7!<&xamJv&y8 zI#2~SoQy%+z2x?p>L%1DSc8i_v;|f9$2_}{vGdH#4Z7@dOr&ZLsF+%JQU}Yvt3u>jv>8b8awHxZuOd%~{{l8lfJ8YM{=?Zc-n6HGNykMAN*i0|^ zu!(N*<%FA~9eDSo+5hTG_c>O4boZ!vz(zRRJSs7;!OZ)pWzQvd)ST?q21&GUU1UfW zDRq8TwLa+9k=GsUj+o!b4a6a*k_4^5e=lnG$7s}-8Xa4c8H0i-tELT?VDDA>4I9cn zkU+fPz;+ECE1ttJ4&aR{~O$wu`pxRN}-M(fA_`VnmU?6$}YTgYGOCZq%+sTcmlgf7Uu=A)9tV#kAF(HrcKv0s~)cp4eYm%Kk_6*90dasTqj ztL5Ce;V)%YRERN&cFmSjFWL8cmGi6GFY_?;Ue)s!BU;ci1 zu*^}sCN5s>c0S9p)Um4UIiVr>ED?X{N-2^lK+;#>_%os;y?fJJm5GGqBR-*F21gUc zw2+M4R=H!QI;Zas`f_%zX_z$=(T$vf;g7k&%oGW0c6v{@B#8v19*ByeUN6_Ue(lq7 z8Z`s^srCjk#ezFMy?ZPzfi7pqmxsr5LR3nw7G0InpiQN%`6Ybt2bu1vG$iIvX|vDY z$B7i^ow#(v!q>ItKYxOKLKeQ>R?XggT@HSEK2%fPessQZH+O&D`{V_9fko!eep)>4 z3@w9bzO-W-ZvfVw&6&aE=muX#Qm-!~r7xzcRJB%DMp|8k$DaK$ic+-TS+bH=wQS8m z$;?w@O31g77ESPv|JJv(4rF`W!9QfS@Iit6be=3fqva?-fqD#bXr%}&@*HMmIi)I4 zZo007GSMZQ=FjL5HGM_;vL(OvENuz9)0{tZSzB6Cm;Jq2DZo5^YFse%+}=*^jTV$C z`OscpE(0)%$pNEWR*I>S!n2IO5Rd-J?q(om4xu+LnAb5?6;0_2Tt%isd`rpEnkEh% zce6eH%}A&bnD;@OpP!3PgFkbXj`=n}m+xOA_pi}szMTZ5+d}q)N>riM4(}A9)m2rB z(97e`3@^Smm?v>+e6mZD&`^f~%_qMkiOQJ4D}P=dl4eabD-R3L=RY&|xgp;i<}Foa zIBM=sZLC5z<$d-z>VM6#$e!Igx2V1Sx{s(Q<8t$?=&zfO3;ivnKL|oH(XsS= zDxb)z;TzGZ+!Fqf$-ZQCjpSugwj!v0h~J{qx+O%Ddz<-DCHi6J9cwhd^p0t~Lm2uP z!OTN^BYol2qNsv^G($+dQ&=>JaN!|dh@t$JuvJ0jC*yI#+XLl8V%^!a-yA!Iw6YfH zP-%vdcvx6;)|F}^ahB4LpNy?3hu@X`sJ69JTNxj239FPJvKSxZ!+l!mF6PtT$fLcb z>QMRb7vql%tV88r5c-8Q0*%PGwBE%EveJ>Vpr|l~Y%nHJzQ9CqM~GB|{9=^JH91ty z<{DW@)3GdnM|&|CEidYr9~d8%3W^Jh;Tn2;AOu9G(m&Qvvgu)6BWKHOHYz9K-=c^` z-xhB?5awuk{K0Z(5f2}o$f^*XEIkDF?vmPGO!EpytE6>$7M~tQB|)cf=lwc@>}Y}z ztGqHKIghOR(2Q_Hoe}RcU!d1(ZmI@Z56GPMSN};RW`lFo%ecd;L$_| zHCi?8_ptc7ou#!f$HX-nrwI3Kx2;N^#PDy8_ zO@Ku!+*&v=LZ;&cDCu>jw%4!{F8LM#>U`lps#PaI6=+GxXDummA62+Txid0PV+K<4 z>58Fm#3w9??kK5b&ochfdEUErfQpnbB z5N0U%+ZiAv*w_D=@`jamz!P2J_92XsPEGVbiVGbz#s#c6ANEKkk?f!3f*;-$6Tc(G z-=D!w@nWrUat3~<33*1E=>!IVItN|EiuajL1fu@55@aYZmZ$)7#iargsJU>CRuFxm z0we%Zq@dTGQ(MAH@Nvxoa{XB$C|6!=Uj3Jmt#>GBTV5PnLrA=DA(ir$l?C0^ll?Bkuy&yCP}#Noy1G1jvv#0>^}DjGC*&@} zu5KXXP`SMNTB)3Nt#q9-A*?bPa7??@tQW=m9MpGB&1FBSp5hE^3`C_`%b+>+AH|`b*vZlgrtn z&D58IRb_5*5>>eV|9BuP@jd71N|qQZ&7F?MK%Zd3p_sz5AVyh}s^rl*~Z*|$pbC;CWm?re_)#~<4)a==EsMObZt1|7_ zs(@0Ajc?ru(TVL8Pp+KjBbW z2^K`OnAWkrw5ByT%T@BYhyHbGO?wW^U4pl7u*;9onKR~Qq`cF5&CN)zRUc!dUaN z{+XKCfcc^eF!j7~|CxyNu~eTNe0qlIDyisr)>RxrhH?u(h4^@%eJ}v-Ye%QEuIdol zmRrOt#wYj)y`X2V2ZU1yi1y#}%V}i0E@RQ_HhVHgEA*T{v4}j`qAOMVl0APM50r7sKhMZ3JC$U$N>FPJvtt1rs*mP-q z&Ti;?RbFku&e;^^tPJjw-0#3~dIt6+$fC0T+ zIRaE}GbGTkl$^brYHz5Z6xlfaP} z03?*%4gdthVSGBob6jNmYCX zH>tTRjoDh>IhwP zi6PSTJ@qEU8GOOGc-Lz?e1@(mQnpu(gr8_FK-zYM*P+sr`Uxt=18(Ajs-@S615!xB zy&!?OJr-AR@#hzW#WQdL33W@hoQBZM0{a7cv$<8xH0>AIdfy! z?Yr@I@6o(kkU+OcaEx~#fsjp_fL4Odi7$cBQO&onL~Z_mlUfvi;E?RRE8bbX1_r4E1-`PC~r$D)?RhpJv|%*$%8?T;hp} z$K&p6a&-0faiyHDA2CV&n;imm&1BRsf{>!Y1Ub|RRMF}t!twXTzmu`@ykhOfYHGEG z0PC18%ns?<|7akt&Ps3~5T8e&b40B@QoccFB{y?a_2xUe2G;AJG;MT(0R+4@SyHKX zw%RTPylnbihlT8VIO!D=rhTRxw{%v8lWl~F5%(eD?@@F)91AgEo+bu?QuK;XbXFh0 zrV=2uLCg#AZ_#Fk-z)KX9d3Q4iY#hnvi<1EO{<>Y*yR$jKEeDS+z5D&8mnGvC-FqQ zQFWs74(a@F4)aOY94WUncBUHOoFt*X8fhM;vzB{Fb81p^P$oNCIiUY*Kli&W0biZ% zOKgjncvbOMspd2zYDQbRA65^~Y-us+m*3kmFA9B=np`GSR?om97?LgBf$SQ?yXjHqhQ5wcX#8hsWM9h8IpvXhl|H7!4lX2>uJmp$OW z-!$>;Z3iOjqA8|ivJ*Nl2>5_vItN&YN3_PI-@YnDY>om?u6n>W(O8Vkoq*rnT=2do z-Gf|yE9}g&1)a~Nfp%0wEazHFXXtfkv+PXt@T+-cT?|G}zU7hYt%~%DQ`54T{G-nt zaXG2KbowPFz0oyl80ivaXZ%L($hm^RXS=6$VbmGDlX4-Le9K2*P=0BG5Z`h_D3PJY zjB0#ADs8pFlUjcQ{v+x8K~j6}{GLVmM3o@_bd`PJ+ojH#NrVz=?E(U^%j93jI4VDX zsF4}1+|{z*2Nx%iD`+SOJKZ#pGZ64m7kA&W>#*(##OKe6d^-P_`5Kp@IttgJnV^H0 zftmVmsx)kI?*5ST~TsR?L3mvE>Kl*#8 z!N9D*dv8c8V-Mz|e)ww~5qL8Xu^01(sgcD0Dvs8;3(4^Ecw18vt>M)7J(%u4YcQXU z(w=lg-GdXD?%k^b4SsqeS~y*uUNup~G?(wNqa){(QdcG^dhpXjplyPw7aY<@ zD!&R+IGdzI>d=%OoEWt;bT>)j8%yO%S9YfkKZ|d~85mK8=f<2avF*A1jPV>%z%r)S z&e7b|=j;lvPOoWtb@NnLxD0S+9o$6Ng_N;nk!fpCG*xVmtY09G3ZmZ|1>}$)e2#VeH1srW6iuj&0|ME353WU!0j+ppn zzT0H{oj!?!W)sof+#6B+4|tyA7M{SiS#0mDa9NA?&e~VYDC9-hXeL?6fkopF+*T8^ z5zq9BO<(-O5Mrl<1AFCA3>tHZ!(mc$u@9*UXW`ske$iFS%r;3}DX7YI?jMre*M7Iw zsS0vE&#(Q0<<1V?_61T~FO12)QPcGF~gnU^}6&&2c6?{@~HAsx=_MhS^G zEnb|#0Zro%^4|)%r}Gt8+9qKNxFVT+x%`ewo%rW0B!wrBJuyFapUGrZEj?v{Ky&0 zPVlJ>TlDpQUo2nG;z#TYWmA<(xT}q#M5Dt?(;Oi4^GusF6r&YQm3f!x)l_ ztpy#UQ_desV(IUW+SyGLM?0DiF}nFo39A`z51sfVW}!Q}#}U-9y6@yhiUmFTU7xiT zvCD_9*fq5E%OmjiTun8!C+T*ugGsAv)dQP)3G_4gK7m#4D+oYNS*H2xb{X^j_Hd zEin@ryBxkCE8!GhG>!=E7&{aUy?=_00FIK=7(qqpJb8;#%d>LUY`$xYCW>U(Qh~%> zezD(wH%cchx-_>MOfb7kCTr!7^rDSG{~@O>G`st@K8fx!i}c1$7D-bsXU>v%!7E8q zAQxSW2=}a=mHG15MxQ!Xx^D8GYk4c%g^%0}lW@~$4#hImFc}>X4 zb8MJjocg=n+5uCvcWOx`t0ggUWn|AU_W8R+t*1A9CfiSin7o!k+j@wV zt{eVMB@#~RRZZC19CkUL2e8G(WAl&ZHh}Y+h8N`>WJc2@Apm30Wt^D<`$BsGRycw1 z>lcY+PoMPVgO;3TsmdxPjD6n2^w8bIpv9J7<{IAmYwNhA9zt?zO*)ugFn2B4oc3r<$R30gLe_?UX1$LQTEY0P*`c(>uls*8y-H0r;KZsECWDS-TS9AHd503U? zHe^c6^NzNcPdC|R7LFXP#lF0RN#-(?;ax=y=?Uw#cs{Ehpw2&5`0taW?Yb3({1Dhc zdR;>!Lrj-M<)Wh?pFs`hKG~nv{xB6}rs(|JNr2B?I|f3+f-Y5*;1X7oQsmZz~bi?aYfmo$`a>~hUhsaj}2m?!?>4hC3*H4O&3 zr&>X^v{v>t)@?&TPc$0Xa|&TlqUB2j#L&5=pFH2F@!*&>4h+K8P%q}qs8y77vNp3) zlqE3^eHEi=N= zkz&xJ7Q0gR>x{-n{heAgqHnV^#BO4m_v{;(0$KR*9qh(qPsi%*2_H%|oy;FDnlPeg|516@O^NBj{oSz{PxOoy|kRis&Qr(U=05p|R;>F(aJgXUw6xV^{5aNNqxmHhewWo!HRMgzn~;z2v~P_T}0E( zLba0@)|cT7*ZtQ)6U%IH>@_y(X5(C6`d*9VP1)>%!F{(=tG=`=nY=!py0HHf>7sY4 zlz)9N4oZvO)phkGenTQPG{0m4LqXSFQ0RK3h64~hCwculeH=22bUHiV-rf`PGG zP`F^%hsMWO1P4JVupfP(c3J+_-Z_bdg7$T7MK3FD+nx2ZX^hGNof0WO{rs&9Ap@)* zM$F22GdzA3XOA)soQ9W&G-W$J&A5i6E_pg|+PniiGsh^QF zm$(>%BB`JCW28Tg1`W<;yheV)mR*5a!*s`wgdZ@GvU6A^(tik)KGQ>`cLhwgy?-euG=>(i`Fpm;Sk zPX7nlz+jaJ>ZJ6ds2|tyN)leNHonAL*EsD1Zu)zZRQBMp1L(;jmHQr$*YroJ(-e?)> z)DO$gKWi$-fBF~aN&c){{d3H1_m?N`B03eT-{|_|F<>I%l{ZtefyiC1w+l2sE9v^d zO*m{bQY;wTw@;oLI^T=2J!`R&j=*>*uW(=LXz3ItG?Q_zM^&_^sImh#DXK3VueQua>n@Tw*m zrs1)7wtG0IgY=ikQ`|1(uuzw4iq3pSU zZXMrqVg()=$MUvaeL~2FSVBVX5b^1ct8*&Iu|t~So6?wwT2)+kJUSn;`0VyaXwl|V zqCH~IcO>`KaV|B%d$Q)Sbu5ajy+yg4KX^})yB@1-U6e@rlaA0Y3N~7dlKFcGe?-IgaEUtG*9LIHq zv-_~!hFZ(sE?hr9uDxsE!TKiN@u=^hK%_*K!@iaJPOC1w86zR~x%+=E6BJoAkF{Pk zal}Y!`S+%V3(pn|#gN3uVyEYy-OfBMU!0zOrso^sP+9kP4yZ{DvwW(|jqiSLEgBMo zHm2_qu8diJFT4=u996@P8Aplpk;VPTXHyRh3NiN|hXGQ%Ho7z?3;5b4FUlDr^+?XO zyT;evI~M(og9f@9EWwuX_i5I@)z!Sjrb0<}j8qWemc5;QN(z|E^|TIS0iHJ(T^&Nt zndJn=4^=27CJDcleHizDe1}H^JC3$3p!QU!%$<>s$V`gy5A?HQSqwJmoRtffm`!>n z8_|{=0}Wc(op_})+_b>0O{8s)jxnu8vVP!#FvJI;@6{Iv3WyM(PuWi;e6oI)rZC{C zdb3HSf{uOJ8H|t5s`D-PI>_t>#dZkOA%rg|X(!cB%o;6Zq~;LsJWXduUwv0nx?-0b zGx)Vxe%wE;`c0*&>P!BvGQy$A9ur`(<3~56w_igc_NTw1^bY$qW#`IaHmo|gtqWn# z&3qmvmvNmxVtx&Kz`h$YbN+;BLQ@e9nq1#SsxPkc%83qJ(!JiRNx?|Kh6!Vcqbz&T3!^^jHjm3YSp zqomnYhu%;OH1PA?;nqP2WDjOQ8i^kB7RUWU#lXu=Boo`%IStdM;^w;zQ~%UD;lR}g zTi#6CA(Hm&-`M*Z>74?kL%MN<&lNNneJux!M`ppxDv+A?EwWYm+Qr41G29+xT6(80 z`qDGjtn|)?m&=DPrzw-;sr_SbEl-0c*E^eW+)u7S;r2~r@BVC_I*uj7eawRy|BQ%^ zkd1Yv`Ika`J9)>rL5MtAhnapeg17l0S7FLV_rql(+qT7Jz;n+6{@*;N>FcTQF+=Y*b zG^K!M!-i#i{^-ZPQqEMB$JxmLm@9eTL^pO)3+9=WX-z`1F;yGoW)T=BtAIJO*kwDvw5_d^)FhqlmW(4L00tsGTHFR=?R4W)i zn!RJ6h}AFWXqenqXU1@dz}Y4_C>(UuD^rqr^Cb`c1#*s)e=5sTv{b{KGVcH z7M|X}`K1dI(Jk_=C9}J4m6APhmz{t#x_hi1X}rUkGfGpS6^C-fRJT3tvc|8hywy8nY=U%;Y(e_i}x7{aJPBxTltcr9Cm1AkGuWdYpBUlbw-5lIn zIvt~1Wj-9FTJnIlY#e-Bs&*pZ1n0E&{tTlN{A;$h5!8vUp<(dhVD3I4ag)19{G5f| zzYliSUy2nGSvxtLd6GbJ)&=i9c}-XGc^$D4i~%aZNQV|Ntv-z~08VQ@iAZ0-WZ^gQ z=@cc#7<^8CRd0c1s~P5dS?0Z?{Cv z4p`BzV6i_T$Je!**FVj8uD-5_e$9by&zSj*e0Z1yjX%c%DLeW5ZRU99Z?WqUG*c!9 z%3USLk4Kk^Fh;x3EJ0I#<@(o55s|Y4W*b&%zOb_| zdz{#sm>ST47Pl(o>zc8Hv!sF=JpH6u$yb(ORs@pBt!{l}*W*QSaB~bgXZV+~mD1s7 z+3(|`8%vnHK7BYgXfyHQ3f7>MjGr`EYWeQcS{9G1TyT!}-v&4RnoMeB`hQdG^lSMy z(EZlAxOA8aL617jQB5Ou_*anX+r!456~U zk$rBf2%{J5FIEzPN)}NNlRZ!_A~>(!>-G--i2A@;M?ra3`P3ET7f2M-_e|oxELCu? zh0v0_f}M&rLM~ExBYu2=#3Iw7i#k)`O_dt4OU^TF^*OFOpeAB<6x*_x%NB|O>px}XAcb=VlUxt*JWU3=~;b(Gbl z1rNd*n}i@~mNgGkaRi5fO#a>a3ALFWJ=wBVmn9DtnvE>%!sIsSQQN0WNrwjF&$0)j zh94)H^`@t{FCiaG$~D1>o78z@j7Q;_a4`Ko`{{d(}r)ZpX{ddtlmwv@LWZ){5-O9Mx=#_L z)0_3t8`OlwP(l`5P_KC9%NHN0`Uq2J2`?%UkwDCViJtXHyiX}_aC7o2_)KV(C7=^# ziwCg6#rTPXo=Aq=Sx1Ts1l9LDGxwFxSd$E)3GOqyVxxAxE54_j<^cT~mHHliUS0&s zc#cb)Un2AY9>m4pFd9qtf$x;VmZ1%m#6lq-nW=;N4&2Ftm7fOH)6=`%^eE-K<3$AG z7FE!T9$;UPU7p8rS#wXb3e3|#tz@fBt_Pt_$*XF?9$US0l(4L4nXMqSRiy|%qXW0Vqq-0VQ_ zd-Gi>*qKQl`p^H^U}rrP)TtA-(X-Tfy^|}c(I`m4ex~)0oDT=%Q&u?|eFR0p(gZ87 zD_gsN@_Vc~Adqav3Oik&TD#l7g8T4;#qHk139YKY*Zx)|e}|!O46gmR2y+j;s@l4L zT4fB^_k7+v?R*^r`w85`SA}-A^Rv|u$fWaY85ZW8S<3(ew)d=csI&L$*R6dHzUK}a zR3f^dqF`mRwa9b>=uP4S+%#YC+TSwC9#Lon3hq10VSAGZ0Gk|9W89-#XRo>LZ3MH% zx6a&fN4VpF&>YQ9>nxvvOZcm373Q}+dvW>k|Qg7e3ckJb@ zVF=VZ8q*aZHa(0iUJ9gM)$X^>63Cl+-!)g(9M`L1 zq;WqV78X-7FkaVwYMlQG63d43NK(z8aC{F~qEProgyIN55$GP#j zZ|sLQ1|R0Olp{+WXHA%`Tuxu|#_{G1gpI}7H?DYaw#g?sMP{9yoMZra9mz z4)O-o=hj2|f=m3`g9LpGHXAiefVL?*>tqOLs<{v#X-K)Q4{+y-iURZ}|&V^Qj~Ka`(**LG2$?7d}Rb zsduvQB8{&rg)}~#y3t@j#Oc z1mHaPzZ+b}6cAtz)15R!!LGVuMLxp$1}=~%)bcY0)so-uf6&%t{D7^6Y=HFkHNTAL zopHdPyI@>l3;-Ov!VA*13_+A^+t_MrbZl$^l|r(g;poobD}yag%oCUSucTY?p9A{k0xO1<>fK9)g3&MEm1XQj^XO%8x! zx00&SQ!|xtC&8kOJhbI!9WP?r$@#GE3)*>$XH7`fb1z>q*`GCXK{m(50ndE~J7FJk zB$N92=bA_C0?HUuy0vSLBqm8KLH^0Q=$Wz155||tbGS|uZu~-=Hrrz8PSb^dBqkWX ziNVSFWy;bKSaPt~XzsA~2{}V?0D9k;{>>X}Hr+7T3cG_uPT66fj z4gm2qnI7{GU8F9St}3uQ1?Mie-muXv{tWUtxmfSF&05xu1uwZVXbOg5t0w0L37TeD zGdiqyK4XK9XziAaldfg%Wb)Qs4Xbi*&@Y-xhGC3vB_SqRC+Lopi(~X~S{t{MUi51T z6pSMeU7%4HvA*&_1c^o-GvLk#k~fo8K_E?L3_#;sx(H+!&+dQUDq6d}uRW#R)B$6O zd$-j}06Ul?qk4sXsUdnG=eHEOQ_3)DgcCC2dk@gv(d`&&gI2qkiK z;Og*st||igOqQHEff2IFNpEK!V#_X@$A>k|n}KV0n!iHFsz23ZA-l}NtgpCNzzwZT z9Ut5pcfTjRhD!0}y!}L$GT_byAwA`%2&T0A9E6Wb8Q*%1@mgh=v;q#uiagp%e^?op z)o1-N`C&y3=LVx8N*-a>TiA&~hHvA!@4C{ZBn^DOhBa*ePqD4N1EZ8}Dhc#3kD;*9 z(zvEmpWbi{;QaA1HquIw=q5ZL89u6D`)epTr~&$v=;I;FH z0-$b*GrMZ7YcVnb#@dd2Ef8 z^oO}Kj-dfXSn|kRsZUTOqJ$5zrsC|)nNp;NSaUmU)t>|Ic=I{W!cEFFHM*30!_eE~ zWmRI)mu#roeCjLP>|?xF=m|31IrL7jHOw&y3)CRVl-Hd^8QXqpWO2S|BClpMYhOk* zdIJmjchw#}Yw8SvqUj3d5=Sb+wXW1U7b>Z zw5YFHDS->SawAwZTcvyo4`-ZG!CovPU_!(^%c#XY*`|lLCr7oS>Tx0{z|Mc;1}u$TD3PLtF#{=& zLjqjzFPx~;RM{{F-A3Rk%GYpBHcsrT0cl!WS@_?!E7te(zi+ zrlUCLg=^UUT77}sxWDxGXP|4M>Q?x`#O~Y50yP9;@I!ENvP#NqCQ;X#LNGR^xT{4i z5|amoN)Iegb$-K!ayLkwSfMq`0!qLOsloRKF`LvZ-1b>}Y^bwekwNV6*oC!C)?Aa) z!2Gqf&7jRst>XX?WGalp@becw{^eQ|-#d;1o`eUVC(Qw`CQ=FmYuoTJiuzg(Yzwd* zoqXq$yPaJKomhO8$!(G}M;uvv<63ap5oQOKqLUlmq4ASDE!daPx!yW7n-@UIOWb!X zp!SnyMs`an^GOb8v3W^Uf5{-I1q}OGAKhO9Z1$(dF+Y&rD?)n;7VnhAEJ#9#=1G)I z()s4CF3I(@-~~p_SnA3S2v#PqOi35?<-Q1>_y7(m*gF$j^rMY zY)U2)CE83E?{Sk>4Q5(3d29RJ1lB^)^4&v9_s+yT-JMw7TPS5xLDVA>tu1z!(;|7&%AlpG2c z5_fWhDNNb7gJY&bRtbjsD=vOz}BL{xA;OrGedaHp2_82B$cfY(A?6$5JXHLXQY;-@hkQwwF7WmS? z<8Y_3?L8ot!B=iB+NCjsHQOqJ3g}ZX6h)~M;M9Qm1Jtsqpbv(i z0`Y+*dS~?6aKggvEK%+ZKszHK=$3ONmdA?lo#~h^IFBKscYf|+!V>!?CZ5-#AJWEWpgT#JBV}h|dl*vwn1D#<4jlkJf%5#A^_7;YFCcjc&6sv77_W>pbM| z&=tBTY`Y57kgoP!F3k%W(_1vwOQ|LOr?A8O77PpJDd#6}eTaAPFpYNjkINXhJ-Qax zVY)~?_Z7Q>gPufHgc_%mH(`$Ig}e2lXVZEUZf2kqNsfuqyrMy@Xuq^idu2~jtp<~p z+;3YOD%}F&`J#qQmo0Ek-Cp0*^2`{~Uxl!<%l)*KACmRavsV@UOEg~QS({hodF*h9 zNujmP-(iJ_!K*^l*#)qu$BfnITN|XX3PD(`Xu$kMyQv->iNkKnd2y?9qBNIOJ2P&V zXEKKbd>7#0ZV(P%fx%bHOvf(544d@iuOcf?!5OMo7DC5vo#srCk0Qt2Vq&%-FX+j~ zbFh6V4*tk*yAZFI3tjxTq};AOk4Bq)dSFAj=|>2mzZd#0Q5M_2<)t=mxlPCg;7Jh z^IG?B=uHUeEP~1bW#$@4$P9e>1D6j0dW(l zd7{ti5s;u@Ro8Z9tt~N-YNtU`AH*{heO0S+?UdQG#Fy7Hj!Ot2jF#3q0(vUEs?gQ1 z>6>|KZGCy@ux>Y>Rq1qB)$xGYgYh)F7IJ?0-=WGimLWH&FoZR=ITO|#emM($3Ao=Y z=!|Md@D`oCSjzYUS=;zc60LrgJMikBE8VRd=FaN%UY)6T`+cSPYMc4>3UXOM>^yhi z>fq~X?U~1#xhHZHKNAJer&`-h`W!svBmWv@Pi@7H`X6`Od&=9M>$G+Mk-4r@>smqX zb(r*Xt(l#h=qTc5!rWB|wpsw;mX}y@s`OJU?>{8xz+eyK%i3zt6{dBbZLQZ(D1=o0 z{r%sYuE{zqMd)>dh(Cez4XTlYQ_SxM*2sa{7(QED4Nv?-ERB$SZa2Y{+wFBDksLZS zY{kX*=PZbfl3TW)k<|ZBfXeCY<)mk9+r2Rb)WRV1JJwfHHT3}9x} zv%B&DTHu|ML99RwcU?Y{XEi_79-wUEC9F?A*U?+KmIm<+6>5GCuo?!QWU3O>(K~~y^gm3XiR$Fv6)elK7$q5`!}|L8>R$ z9558FQw{YP5;P_8l>%a7yaAeeSg2CtJPcuhm@hVIBtf=7Q~iF;YA}#BfJj1ezlXCx ze(LS9nqfQkVa)s&zsAMjZFcaYgc;%ByP#*!58#s}@4y^)Jp~iI8h^5Iqgz9dZD>xu1O-Y+ zgyc`W?^Z2&X_puPSTpG+8A%^yR)W*a;a|=^%J^dl;=$tNH$}cvUlAc zB4%=(=H(jjbV2>B?d+(~(3WsGE}}7b8e0WrOg4G9#ZNDL4A1 zx{_1jKV1{~Lzv>Y_U8H1wZ=gQvBDT8D(1GMI75TTJz{|O{b~VW9231^`6aH8zDx9# zjD2SqQ|-#1{v%h}x{7eMyWpQg$UBL@j~8G5q`KGT{uz8c|L=urc-hyXipK4lnjxOQ zeSaQ)@6M}qOD&(?mt~NJKmOfB2F0MXu7((iw;%ovd5*C^{rh*fyJiLPSNe^dK<4Zp zZ;&`=F;%iK0;bAi8u$Rln!5dLzjIO4VVyV4Jw6n^!m~VdW8Q(QQJ3*Z>CrG&hvv05 z@n!RS%zxA4Qhzv@tRw6GZXzb~WWGzJg!W^^hr6P7hej3BWQKl@OAVs0?+RaEFfF|J znwhp4YQpNjikQli32$Oz@rzL-;t3gLTX<{ z=??3M0$r;89v4|zM#wL2gTKuJJsBbW=!PuYz-rck4(!i0z()+R3yLX~^Qs01QHrTa z^*0Xt8OcCxhr1_)wq@ufeBQp|xnU<=(_anfzf=lBe!Ux9-`@Ryo&vXlqZsixML;HR zwMDbiYU2J&13c&|)spRLd4Qcl1teTP2Ce~% z@wX?I>2^mBX}Ih!yL~kw*s=EggSWh#5gXHL{Qb+2)SjVSF5RIm{d3&Q8c^Db^^QJ|2Sj`uvsFTzmxj)QHnV9rJ>EYazM7 zV>5k{zgLvgIh_7j2h`jC=YB@@hQ9|!o!_ipy_^WDM$`%~G!@*_j5yLUpJfslMp&>& zR~0P;--zcdbZd&Bqe{^%+6e%8$9{nVWzN(I*R9VI)rbSG= zinY-U&5e;1aIJ9e0D!dSgN(>4&B4Q3p2v0vkvVCaE-qAF#0oN*nwNo6ycKQnF;bHF zDJdfA@#SKcTVKcUKY5cY5vFSn?W}U)8bnl)6gX(h$XYnqAC1MgR#_Enlm8psaxqLZ zSjpM){g_(uG0(x7+327$H%wkycf}&SpdC>nkLC?!*;I|H=p4QeC;F56{!rUsrnFj% z;`tc0rcTaBB1CT@oQ=bXJ(YT?ntZAe2mfEtm(evA-(xEt>UL~zA}F+nt8%PB8{^54GG@Yoc1 zL^D4Ncs_q5YOV0oC#M>sahX<0w6<>U`4GApOZM`bm+(7k zuZ3;XEF3Oj^syzs4QpHEm6;1+y=K0$AHRJFG+IvSQoho7`ZSZD!-G=vwbxlel!Fht z^uP>9iOPwdmt%$V1WRwF5L+_co-42Vq3fyEWF4y>O+gqg*J)&OW9z%8xs+@MKmVN? zlQltoW>!VHXBfZfJcmJN$1D*SG2y!&WVO}r>!VoCRNE$$B)EevjF}1-RxK>l5nU53 z52(I0uvq?-f(s+?MqvUc+ut>sRME%YHTSlRm zm#7gj8U&E|ftm{pEDJ-A=+m4;b2^44(GGZN-`# z2Q@Vt6@xh4y4c81+>n9PVHa9*T|-z3bE~eEC8t0Jc=7NU*3%IWQ_%sET45doLGYxcKBDxIh9rS(U-&%KZmAp|dT z#^%dD1#OmuZ4_sYr=_WqR0df`#YSRI^-R7URA9crvR&02)6)7|NlM?#8ovgXl$eNp zUbMBcj^ZGm{7rMD6WJGK%i%(cryIF+`ziNd!VBtOjGspZJ_8L6UTAl>2Up}A(a1P| zZST@s7YWMFPZ1JMejA%>pyEQJ&SDv;7iho~R(Qx)eWYzZ*)giOetY1(rHYk3r_h4L z3_Y_hYMZKfBSvkp6Z9XR+tUXck;o1$fh&D+!4Wr(P53leLdJZe3=}(}VZgBkgf85) z+bU_i{DR5Uc-A-1L=_7Ktpcj?*{*r-F^C_00rT@nF5tOxT+hL@Ho8iH|I^AJh#DcV zzVt`ld4u&=DclfdbX`DdKu09{K?oawBH*7079+&z2sD|160Ml zr*oJO4Fqk|i+4Z(($!(p>3-kBjI)vxg5x6mh-P#lp~mpKb@tD)Yibbk*F@c&}SV5J^^9g70XLCfxsDA9K;Vz%rdBtZbmueOJh9?12HHb*EvV>=ht_%ER+PV}m{JKyMI<%hD~ z&Xtcf!nM^W_0TbCRVzeBNI~B=1CQa%3a3Ho9hHqTs~)tPhW)BjwLs4$<7GiVu~E z45CxW;kZe%*1*X6*B;e9j-7&~uSKg*x0#fl8{5g1bR`<}jwa^)BBy;nlbExsv!uI) z8QmVD_nEZbJXwD)=_q*YeWYx`mqo`jmC+j_Lnv%-!VW9oba$b_waS#?ye*^=^|3*`5;i zp~k|V2J`K_utY;e3cQ>DwfOvFP(12Bk9*F!IU99U(gdS4Z5EP(w+&*6$|9ID?!-+U zY>xsz;!u>ZgPkuX$Z4@QG9t>iwlsYv z!`C=H`mRRnzBIBsY>)Dpj5fq@XY=b!#c4Rb;f{UCa51j4S7pJ)iysebS-9RlE-zVf z>r%~*+vZH8D_h%&@O+QwhfXY#oEiCc#x4H#Czogtw=Z*y;b6X#pKUaQUI9weU>gkH}&-O z=1-BgH1oIyr{j)?!eP^4x{tE*9C~HTIo;X<@*JKQTD>Y1652HgdD`9e0gm|5NMt4@ zH*S`CV?P@Osd-`=7!#X?a_Gh795Y_7R(x5luq~aRH#}I)yaW0D<(P9&OOS3im5dp7cJ?#dKg9?aB<05ALRAPjn|;Cx>A2p1wy%f!Jhp1YR_KgU zr+bugeU=p{6B6EiYbvia0Gi$S?#ZAaZ)ZDm^i!gOY!rn$ZGKGZ^w-0-Q1Z(#-j`ya zD)ph#lQ;G1rC_Ofq&XsMa77}89Hl-lGBPgL*cv^4=egepEW zhQG6WF;j_TW#_rvQd9RaYatFh0UR6On;1^(g+zDXp75q|zUcwIED$)(W>s?FMkk1= zKcOltq>DDOw0R(|J04f@=>FZQXnv7hgAUoBA}vte@$9Kme2{4W|8jIoNM!fz zkZQbf1wb6<+Co}^VMKUx9%3f&M>O?J$|N(uZXxi0rD|RC@6px?uN6d7<9>(6VT-&-h#@AqOy!&zX z{8(N?z<_zavxw~_O{GB)1{5< zEU7&nu}>B$FD0q1E1&Vnayypw1&zR*F}I@T1jR3VH#t{f?eXGY>)Ks;Y%ge7%p{Qw z$MWi{f70%trpZORO&M-iv)TR>NHR8&)*PJBthZo(V_E3;PL*hICldAN4SnCv9J$bm zLGK+kthZ(11Lm6Pd+E6xxEt~1h)dOp39I{x{Q=pzpL#2p*vaO}MNNn9sf%X1~dKh7nF;!wh z{ioiQhcNTl5Oqn`DLPE5Qb*w?auH)G~P<{Z<}@e`Wj<6^A$CRNOT+NyhsSCW84|p8xEAWIB@` zSwr9PceWSE^O@IRM=7&KBL1vrNHypdW<`4#{7$ihHFE#kmbj z`dE1oUTHAp&g}`eM{YZg>GQ*cR7$@y@{QTSZIRmr=Dn8JP}{BE7Qm9#wuaDzG>*0f z)xEe7HJIc;TUfVpA6y`w)X5+Q5XiR!QEPx8bBaeC#4M256|{8CF$n+mgt71?+!~oi zycu5j9vQEak4I~;PYXf8ZHfEtA5%k6Yp~o@wJtczeFcrk1W->t;qi+)GOhg9W?<9V82z+R+M7DRo3%2B$5%=bDOWxTmkp2>H@M(9G@<0d1U1LBtTGSS6 zi{K-m(F`i!YP7vU3RI;^tO`^pN-+P_jb5)<7HJ<7@v=;O3)hFX%g-$+7f)E?K1`VM zg6lU@_H_xvxN4+p9WZZz!4zqGtOhUq45V%4Zs+17_!O=NyHxqwgP5pkUST;1I6RPf zb@{7e$Pab0^p30sjg-;1NL41c7s92p=);$sYdWW)>s`d@Vc81Jt-j!4hele?+D%K>sQl zF*i^?Cb{N+`@;N#J*$B?aWw%ra*RHP@w8C>p}uyggGa1J-dx9J*Ekg{-3H+r>cIoX zChFD`G;k+%&E{jB`){~^jg@JM0jYmx2gLr5qOccg&Byx1&m+^!pac7-sUP}Q_{jxY zDEd={$i|KHve~S?IrwjbBiF=Hh#DV3SYekMUub%E{1#8ZiYDUj;ryio$K&GbD#D8p zHP=v@lz8B+=c(~qWzs7xt=!$;ico1_?!wgvkHJMhB;713Z1H?RCo?)DGTjVH7jBzh zc&*(8a6S=kchF}cn+vc^m(>OIdEBMW*WKm%ck7dsYDZz6^ex0Rn8nDK0LN|pRaBs) zK9T{t-a7^pJ|lX>I*!OI+)ytx&g(oJ1QkG0F_cSaT%(Yy;5@YDTXXL*jDh&a^Asl2 ziT~3m!} z%kK+)lUkm%a9{td8ejLH(&nMpT%3NZd`e{E&4*IsW%hmie^NiRT_v||72NsT^E}0f zX1w*jA%O?sC~ycxRKs`7*Z< z5rD83lglNqJ_$zg20lK)Lzx7;irBs>V2XRPbyFZzn$2$eCV%LSv-9xx4-plDJij$| zrF){2Ew)&+N`$z9eW)@L1eoG1i!J@lK_?rq9wzqG5OughRce4vlkKpjZ*H3cj!UBd zUQ-BFfyGNy6!iflqX6eI2_hBuVvMVE47iLPh%rxZ4OATKY>wL+HBzRf+kjOxajb@T zK%5KJ?iJ*jkGeX4JOv-pri*Z~UkQ8L`A^Ivd5aUl-gGe{uc;UIMA>R|nel+xe{l~# zkh-vff|6*6B84~f|Dy!ic`1YhvO=K+`=t~VwrOKJdJMw4Mj+J@W<*0lG?bJQgxi`I z7uL#*)Y6ZN3JN*Ih)MUNFsUpa>s6jw|;adDeQ9WCk`MM0y7YC$_O zSEux+YbOi=vOKqoG%>Cn|>EmQ{Yo&15Md0a%Y6GqVpEc3^>b1fKhXjVv=|ciw7^G#skFoz}bA_ub7oQ{#yW5)>FdOyn z6eG8E(zc@1-k;_7J#9!vU60}f1trymQ+FZ4qL52Wa^Z^M`rs1I2M#m;urx#>FonYrin6rVm62`4#};}lEvu2*G1E`>@-X(VR_0Y)yzbF0{>k4<+1$T` zIS+^`tHb&qNXT-Z)I`K~t|)={FnARgpnLSmJbUAbtcKsp4dC8nAi|tREA@Q}rEgC* z!`1M!Z>Km-c^@!d*4Az5UD`fN%KQ1TcRJ=beWYjH1KUP?t<6e8|C0#B`vUCtypDrBpj~DQdpYZEKzH}=ld?-SxdR5&V(Ly}IaoK$2 zoyGy(skyZ9$!;~+9#?0K^Y&^s{N74W5qRVMXfV@1OYM&zE{-HV&KVFqX?()R$sGs4 zNN0QO(*w|wgpQTyM*xANwvsDJ-kt?qKtbk@L&FN1LXyihx$nnU)wBaGBMO#BY#VZQ zC6`-_s6#Sc0w3|Nt!>!$C~__`Un9Bvy`hot7_uSqQ|b@nH-!4-?rTg@3=9&o%ny$Y=`ivuz%_!LJb`&H>ylVgnp+6_^hhu z+0lOH8tvQ)WF_^joLb#{3nH{qaRION&7xkE#A~U(Q1|19lXuKCLJ6IU)-AX8= zG$VeHm!gQn3ww%qA|rFRNGbikh#TI%ztVeQb$_nDJfCCbagx>`ULd2O&@I%ZeUnCa z+Gm_ueSDEtFzLSJn6QslnuYgAOx`8~<@`4x^;F!EJBXrIu{D`T-i4TUsMfnM@o0jiK}A?hSTX+$_yOXt|#yU-z|E=WpaM3{hpYT$jg z1C1m*@T76!zh&kP%@W?99nKvqscfMlHTVSO&EKum^NQ>7^fg5k+Cya3cK=z%=Msw5 zL|fm4xP18jN`3zME!Sn=V{_dcnU11}N=XXOLynalmiFC7@7hn{Q=N`}C$Du$_irkE z8&{WX&K|l5>z;kEK#Is;AjFxQT2+^tdhNA1{b3`67qX`~iKij;tW9{0mCSJ#3X@VO zb^aGblaZM1{(@QAJoCZJLqo{AsaT|!kBXOWmDlWF0ui&SQl+f>9}+@uz}`$P-m&#l z%in9nbyc?iJZXOQQNB4AW)l6INb~!hc!ndzxA?^`?p?$$b;4JiHzV91F9LYJB5?Nw z?3bu8LajetzWG+IdHyVXrA#w9bGQUeP{IrGky-xyGm#}c>wUmeUqW#i1+E-ofu;7W zdCRhq(EDaZXY3}wO?qWkI1(djq_7sSX1Q*zoPmeRw*F_`@T};=sz?F6I1c1%{0GU3O&JMUk3e7)Wi(`*(G2NWC=3)fxqvbe8xQZ zRO7&}(eNwP4Yb(7>bt z^1UaI_=&@lrMDhpo;_f!gz?iU{Z?aIC)9X;KaHSAb?Klx(9oWnBwSd-ycGWqAcpqv zLU>jE@OxDQPP%WXd6nW*J-?44=u-`FpL+?u_b590km?vg#>9owTbb{AT^3C_R%cou zoa;&ds?s&vHaoo*&?E}}RVF3Zd`lu$;ay@x9Zs=(!e>y45M(5C#e2PKX$XjcjqZ0^ zDJ;;iT<>75D1JkTUhKLZ^p~i)2$ScvOo;&Tb$@Om<-bHC8)+$SLIuRkg))xZHAL#lzw7F15Sp(9AzLH7Ju&%&$TY_9!eTk&=ZW=rs`U!u6n$7_{uIzJA#6p z3n#-J6Rt8bm)j;z!O4XawW@-t*`}cU#z)}f8j0fxM0jd|H_0_Z?0~8SCH7& z;hV~?qiZIiP}Qz!L`Qx^l_Mb}nrT=OGGn}s{wS?2n2NFn|Bgd8aI0t~4cZLw#Ry=e zb+23Dj`pUdIhC$UXy;Zjpb`-ib~-Jj9z=%NXup!HvhhsE0ErYe+q#f>2gRxq4!7KX zYY#3hB#G_77yoelEJfBT;|?vRMC>8059^m+!Rtr`toI08K?@C8dmyGOtp$x8!%_x% zvi`Rvl^`@A`9Z#wl8TS3*{XD)i<=~sM?-By5wH}6d1qb=*L3W)E^q1xug5|$GP%H3 zw25U?%h9%~M(T5V^X4G@r%MvI@`UL#VWNpy26LIN8PXvJ3jh!uK@m0d>FR^Z>9UzE z6x^i`FWPD0?5eC)aBO@;m3BvXL?v^5&_cB>KjJ#t(&gEZQYp0Df*lu%Bw~%2t9LNy z;yP_XMqvFwHg|M7EqqhmQL*ZJ!qRzK_lUuSeK15WUG&-4P7?9>RB;1MausZZiJ40{ z3@Ajz*bQL6YQqcP0dR_}0MVAznKsX1$okq{SgR006h*+PFQA~AL_=f+_lh$(iJ&9u zv~>)q1nS;1w6o>gdtBQ-MSFH3{=!91I^W4dMT^3wY9R<;L0dE$R_&)yWfmNypzZo0 z1RN0YOIA;XqhZaqELB=or-e{k?e?IEs$F~tG~a}Z#Vf)RTOh4MF<@jry`x4szh&J8kVgn}GhP2!N_+_Vk(qpp_BjP<}|(r+hdNf;Q$4Lp#@W@@(#jp3A>g8 zEA(gV#qJu#5HDnaXZ&tt@wr#1+{2KGDn2{^rl*dC6>h@KB6ua ztN>5B)kW^;Mp}6If4Gv}Owu6GJ%2B(Fm||_Bb@-^`A^419n5KIeyWL3H?BtTreR>L)i4$dBh{7+yoAHT&xSMW$7Bg$6> z_y^zh_((0(yYo-{7xI8poW?N7=xT{J!O!KBhVrc|fZvhAk`%Bz$XxDzTTS~b0};o5 zTkW9pYK{!aJa;RW&m0-o6Fh(G)+;n%j^_q=EFjF?!WAY^sA7k_457z`+>; z;u%^IcTy_gmb)w#+Fi!aJwzc=q|0Fb z%Ju38`BcDf9*seLE_a8GpL(OpIUO`@h~HD-;T!~|{#@;oa;-JYU2L%96~dZaFvB9< z^N6@KMfC$ z^M~J|w5{e?H<}ljm-awCky9u94OSD;=5`UCXXL!J%o3r@F#*^E)cz^q%OG37%abma zzW9g+so?1a9QjeFb0uPG)Ptl-iBZ34lvaoPdCezV>|Ey#rU?cc_eUM( zBCZk7f3Zz9jRA&dWvsE{sg3c5XEy2=T84Q}<fFNL;TZzij^`xanEu!Pe!4uZ1I=58M4hMKgNd%Frb@vHI>Ys^&ldUMFq7)nYIn4 zI{h)$cb`oC^w>iwT|V=K`Zx}+g_X>%MRQY`_i3X!s%W;Fj4C6n31A{se90$d+()#f zb~l+b;6B@*N^Ez0&n`cF}C64`lhOE54$EkMS%IsewryUKLL<{syDx<-4+3syO$ZA zO@w?~l{3dZ2g>|gYwsgUzAjYN3(to3{)KOuWcu!RSo!(A;?+2QjU)J^)gf@-*g{~k zs0sxItG<+rKhFUwFgd$D<0}dH(%j2CZ5bq};Iv}uX<&SAhhCJaGp&0Yrb%_dE%%;* zZ=XizFZj8-;Tfs}IL}?RnkL_T`QYUPHSgEUm7_YhH4adK@ZxXw*lv)OU%nrcDHGhX zjFekq@mpw*1Y4)XoYyQs7<;|l&O%kXu$LU1y;gyhu4T8Te6<+g?S!*#t%7ss3r6$>@ze8iy#R}Yid?Jn=TG!m#VGHe9zl$@o-9@2(mnLJe zl<{!*!&sx%`rZ}3SECmAo)LOF;xdXi>0NsjfyF+ljpx~2Aw%|XQg@=*-Ca$KY4Kwx z>{)yy(KJQ+;E@jzYyu^#GwkP{dU6pE$jCO}i?(0JCxZVD8NNw(K^6FSJ~j3?rQNdk|TAv!JT^X)-dsr@VSSrH)jPSvo@mT-85dw z>Qt>cVjn(_5lhqF$@to}>?-xau3e03&og+ckHOYd`V~a>I6-FTT!jZ-Z~jg{g(Q{V zoQ}FHE(fl_Nh8JLv#nH?AOJyq zqwsFLwX5X7j7uiw0g$)r6@m~Bf|k8%`{&(iidE2cyTmtk96Lw(1*5XoPifcC!&u{y z#%N=l13BvMIJpTwt|XNmFPiQ7hHNpxNV?V~8{-=qVCO!Who{rLn2Knfk2c|Ieu8!k z7zn!`&=SuTlL6xKfb1z}Og@-e5LXj7ay7GREcA z9Eqx-3~M_~_<~Ps)}gvh{CPzEBj~&qVW|Ho?f5n;hoNAzRIuFNTQ>1dS9Ilv?`iuP zelhw9W%btQIpy`6F*>;mVftY2?}?Gq#DF?f`4U@eRbBRhl@OvF_(k@eUikE&fXC_N z{1eif&5121A&#ue;a*^XzuB@Z+j{^+w_?#D)*bNDQ9XKRF^5_3F8py>*hS0cGy8Qb zSuUe_j)+K4!jJG{ocExUv`l9RRFS0$KV*ny0 z^nUFt9&*Py$IPZ0^V`}{&bOO~gcUP+d!OU2ZoUe-rc*pctg6EoU1Q}P8ZU^2V`nOZ z@yS*Jp|xyFLSj)g$El1jU8vI)#Uj2f*L?CR8M(8QDTB`I>lyx{kR3kIS!HeaE(4`< zkT;CXGR>>#!vQW)n&Zw7F?UD-NR}+x99;4=Wu%50R@jM25OC=tI!XHeUbVg|xUg6{ zZ>H6?9hmg?yOo%Gl+I(^)Tw)icsQl(WQ$4H{IgxH)9x>5P57e0z+^UGzWj>T^ccEk zLp%cd*w!JjL8N*{kvnbIIS;X$1=TRiL`b(`TN0j4_)2(e(*ase$YM**|jqj;h z3Hh9Q>@dh=Db-H0>BV-CDGnx-Wi2ug^z}?hpc}qI@j`kWA7D+M_(n?v79aJk|A8=k za60W{&S;wRpS!pTqDOn+vrB2@(ERg>&K=cw>sM#Jj|ls+CT(7wDQzmY8fc~oHTitq z?PP7gz51xCskb%$cbuM3(*+Nk`4QCY+u-^iUyjK^4$S%kPnWL(Lu%`PL?{i?Y~I=G zdzECm3JX3 zd$})mH39b`CeVSmdJgPNJqx~H7@dylFFiVgx_5Znayt6WQx~TV+q|0f*r-2#ER@py zhyQ*5px%1>((>5&pxWf$!@kafc2q*?Z=SDdW#MPk7$eVp(!DG}712r0+ZM_pm4I@k zVPI953UXYJiQNBmP++I`=me|$hGVSDeCl5It#dIO9?e^vOK`$1cU~^KspP73&3N{e zUimy&*D?g+{zayrmfB~N6G|_4SU%#@M}dX+MBoNXTA7$it;e}skMWP4osltamdptQ zR9{TmHXGJe8(sHAcO!}4b*x`&vgetN^?oRm@kNzbGHjTxoWy3=E}V-+Vx&?q$rR=y zVM^UULu#zmu~yC{hf*14Ye#qYt9eLKwO=kgj;`BJg{u6xf7!1_6;slu zV5F~z7XA5bxB=yPiQ^&TUj&q#`ya!knR9l@mT z{mc3BuYyT6IrH^N3}MFCl`D@aWB2*x47bNwOkNsL_ZzuB*Y;lB?_>9nT2!dB)D4j`yJ z**|rhFp>s?I9;%PnoDiO$M~@hF*-EITS1Un|1OY~Q+M`=ky)Zr5M6s-8MJXw3XhtH z3O_%e1;A^W0B#=5fc@Rb$9|APfN_C1k-?kbSx?~WX0UTUWe=4KKQumOcAP+VqK9G-C`ZB1D+H9Qv1V;z zA$bA5K%?su-_}54Qy210u@DUx*zhO>qZRot>r-j=Opvu~{yHHhK(-|osyZ>p{wTh< zVPR3wat_sxw3Y)^kP0U|ZUpRc(Ym)?zi^Sb)qi#mxIlhpR&sMZhQDypI$->{`^ECW zcWYP+q`;O*Y*O<0?fui zmonnI)SpIKinV({iTCZbqQa>$HYdJ%CYyeMBLI`Y!vzd~s_z<%UpbzVjMg=Z&#mlJ z1?h$3YMZj1l(vV4Tp-C4Wn>(T`xT>u<&m+{=8cAhXc{-{WVi+34|Aj{V9bHHum`^C zx6#IJoCZgnRf>V2pw|3hLLAOgEhG zEtu`US$_l8XqYgi4H^-o%9SL!ATHprsL$R6FRqp~Oftvo(r5RvjbCEnOj1y>D)>In z9{2`oqul}h)WFQXlzH9gtLsJ^r=d2Qq_BrYdQ>dA(Pai6DodF!v8!~%q&Ut$Kqf%7 zq(P=*>~ozq&a32sY#O+mMgzC1^j%;IGIzM&=Dq*7SefXTz;?%L8+$bSZDejNCiMpe zP*Pc$wo!djro!hp3)tCM8|JcDf=hcK%uzRhkLeS|?^Xrlr$FLOVfUD%J-FC0g?m*& z_~q9G%g`HGp1@bQ))!u$ZIt^wKx(joS=vJ-fKn;IDTfS>2NMxfy$$>%K7cFk4pvH1 zSrq=W9N}fYMU*fP(>Izw`OvZASBcZ31yQ5Rl|Due`yp#d^mBOAg$hr={7J|HV>U+y_ zBbr&+1EFOh0~U3~!ys#)FAOrV_!wSD+X@kK9(IpSx%P|%=+Jx%#+{Cm2Z^9+N(*$$ zrLm2Ke+Y;lrR1zlaD)1X9oA1a0?g?lAKCA(YD|-6 zut@{4X#&WTJ2L1NS;HH5mEyTo+3#4T`diOE_6`AY!s%LxzY_TUr|B}dt0DcokiLt} z^=^@%5D^{V*7olID&7jHj_SQWS~+|lLsu@1Ez6Z3a+V2tObkvienYr6b@;IgxlVjn zLGem=Xa)aqfa$V%;`8`9#9s+4`+IC^9*9vR{C?S1so(F25>27!tje`T{twiQUgCp6 z_{pq%T4k6CvrZFEy?P1ht76)C{yjWBTH4zjjbve50(b zg>0Wr0&4qpH&}f5{Y%3$=&#|!4tmQdfZkir2{YRg&E1K~bP@`9!v9Z@wR6q5%7-~r$lH+2)okbyPu z+~;CJopJKdN^chyVIsx`|yC8CP18~7jt#}$cm!ycy5 zQ}Z!m7RnE@EBDD$4{Rcpg2@kJtL&KEUBziz0fVxoB|@P5Z~#f|XRpn)H+&!$4s)MU z3Wrb$M%Ih}4JR}%cf*d9d3>>!W-_U5*3DMtvVghQh0{MiHwqo_gDGeSO@nkw`B^gg zr&M*OJyBPZS6WPaVJ{CM5fo>9z}4U*`)N1_*a3- zG51HY;GuH8+|uw#kLh1T$tR;!(^f=DCrfh{op;%{7p%D9YoFh18J@-RZ&y?Y+02WD zIo&4}05i%s(4%;FZk_|V1yh893b~ZaiUBI_?FFjS&I6VDQXjplN?|LQfEfEde4>;S zHoXYwuT#j$R>kw;(hJzCAD(U=`d`SO17;ZKwkG>-$fc}JdoAQB6f*<)Zo$v4H>puf z8ZMBB_K1fWT`IWagDw>A zBr#HiE!vbbhiUi^$&a3bY>+m5R;&^HyTjT!6e{2bV|C=xiwF(|(pavKSHbd`rg@5+ zt>AH`ah{^+LYn+wR-R&_CYh&Q@wXsZP>?0ckLEYb$dk2)D&&^_&TVpc4g${+sX1St zQwVs5K8)_9z?nICyhM2M?G|9ymx~?!UwAu*X+7*KeK;>j3I55% zKmD5_J9u)>ocYHG5j6p^t{d?nO?(oIq-9YeBu=Qms(%J@hAt5}6}LJ7giy+A8iZgv zXgg}03-7pg-gu7}4IV<|Xfb+(1X=UM9K_P)F&GjvOZ-u{x+x$QbC9XQp8j(c|4N&2 z=!7iHD_@3;wZtOAb%!Vv^AmDf#Z19UY@8z2aTnbG%Wd!P+$6ws9G)vqhY#c^po2ns zY+N1hb%=|zTcG(L)r*VE2BK-g4u<2j>bZSOvYs$cmKO(H;M$6iEM;Bh1{mJLd${p3 zD!|aR{hdb$AfNa6Y1Nm8{c5VBr{FCSk_f%bbu-Lre|H+(VS|${rqO);$Q;kh?g_gI zRb%_Gi&jPH<8l`#BK}A(T0*-)tROXqu;GTZg|Ch)xa(gZETm>vNhFbkqF!+n&|C0z=z9Xm00 zJ*v(d+!L#{?g{u;cXL2|c^n)SZEpBC9a~16+)Wk!lma~V@tcdtD&sfg?(=0Qv+Jyn zeq9ATwX7Hl8NH>5twlR5z#ie2HspAC-ZQS}1%vElNw^n&| zR6zi~Wi5$e$F==_Ho>&a1Jb|bssnk3ZQMjlky?h3;JvkaZasYs{+twRD@4FLrjSWX zA6IyFMv1G*8jpB1rb*mA$2TQ}@ePjFXK6!-9A`OJ1Zjt(~Xr z(x)+6MoS1v1|LN)=;rb5DN zVDeFA=QX~rmUYK+5@51nQ3LuM%BM+MCbgYk9}k^<%q(lPRDyx z(2v29#&fCn+;vk_1QU}DD13)`n6a)ee$it>;6Ah-HOrP9;L?CAMqoNHnt&+b+(!|< z02oE&-VuWE|JZGr2B$P7f=Tb~p=Z|&*<6wH$_~-GRe!$9+NT!a z^#6Xg{~ZDsqmpU<=M?UuF)}gVGvHm%EhsEY7vPNlpdUsp22w)KxqQRtDkkA9e$!jw z+e-{&ujnxG?Tc`K<;cZge9do3=fl>0GQ^Sfv_E4aYgws(J-b(|u_4zr$FC)j>57E; zhXzJ1`z9uxTZ}W3W3PAV;C;ujt|hUt2qJh}n|H?*Q9&P%1r}5^kf{`bjX1AdkK z=X-A3O=JSeTQk$3!mP;BQvUemmz>;5(eU~!u*>3-Np~lv7ci8Tx@MDK@4L8Me<>AxGYxW~(`cdO1d^tC(~fKC1HOBz`7Oh= zS>V@_(twtJzPr;Db$`@;c+VqZ3Ah~f-aZjbf0)>N-0a{qjmXnduoiTeWH92iS@%~7=@Q2 zAzH9xD$GK0U0|VN7h1hNX!S6f9Bx02>UA4RK+ph<6+!U(O696n@7xDpal98;aUQ!{ z%me|S@E2o58_I@C%a}$uf%<9hfR-vI2pEp$ce0_Qq*DTQmY zD7em$%Bv68#H`2hdsqdN*78PoIfk?-vI2q-@Gvqa+TPaNA7|PX9TTd-%&jq%t%@)S zd&;0IHRDN?*I3JTd7zRn@4{&^!-vQmhp&+dMDAetrj>Q}?D{N6sG_!W2Ds zu1ZbQoI?H&Rc{>^)fer7(lF#u0|-(M-3%epAt4|rNH`!lfYKmc(k|utOFagvmF(S z^p)*U`2ci=&UJ4Xz)Mlw3akx`^Of~aUcsm4X1--uaR>x{Rxnr+(=2aTaReBt=zjr@ z`93O+xvgH?TL}j@KdLiuGs_#2_|6fImI#oPw1WnU;Uo!qg`%mgaSAGHKgLx(s<68ZYSK?#-EHKkO@Um)3ZI0t8#;1zZ37!r3p;Udx4+sMVM zo+c6Sn71d<-8|VTLLblg8L;_QFBwptT=m)(|132rgW78!P#g(hAHN_-QDfj_=3r%F zQHU5;hb&op)RUzoLeWKjz`awb2N@itVtY$XGF&x*9mCrLvs4D+=Qv=RTbzp=RX6NL zhKhF;6VUhy7)q%)Mfgn$>q`}`2&syKmnkXlnKCbOir9*6Cv4ZpkQm$wx5@=q+?j4$ z2L^s-s}ozdMp}Mm4kG3qiHK1ZD>3tLkt6;jpNLfDL(!~Z5^75Cv4?a0``~P0m22mw zPtUu}@kLd~9+>2WdYEYd6&=k}*REume!zsmuyT%$)FYFtqGLSJ%yhi%T)ty?DEV^3^&FtqML%tfecQ)j=&el&c6Ko{DR`s6Nh>=cZ)-N%5+lR} z)PlwJ>>W44fHrO5k&mj)Pr;fN<`zqD;5lQyDx|_;ID-$fj2nUbx4XMhx@s0x)vklUSQC3Tk?C%4w^2JKlPs*=aJxSiU#XjOBOLW7 zb9Kj#?QFSzei?D*7yDLR({e0K2}ohUg}Q1TIf7bry|M*T^CNe6iMfR6V+5ds}X zu(OaQOEB(FGAB*H=1eju-EO;zo^}x{mdElowlZ0Q11R)qgBT?J7<$(e(G3NoBW_S) z0?NJ3p|WB3f?BlLolAz&?ikbf$7g@%8sDa1&9+cma z!}9y!BIq5JilM}I)e6Bs0@3nlCDZJrm3v1}?xXqp?q>zSMDt2x@To^5eWYVy8W=RT%WL_=Nz-f{gR*>n1SWtZ ze(+wI_*8)wx^>`|Xw6g(JdYXnZ&da-X=bDh#0d?C+4$-2^%w?zxZy^?vb!BAJq`bn zLO4ThE+~w-Qb3vmSfVd|8O>V}&q>ndS$`d_M1oS2YEumD?tqckN%;P^StXV-AGXU1 zjQK$VT;3}-!*qgVc?{VBLvW1QvUrT{+5wjh#v*SrmJSh|X}s_tneaJt z&RF3(?fUplQSnUqp<9C&l%iNlWkUwF(ja=1z_FFu&Q=ioRf&QVtg(DN0hNSs?}OkQ zq*ooNV_JNk+wlinsY7;f=96vh<+*}?5dPvGne@Ln97X&?sAAS*iv&crq_mo5XmM|6Co2vVCE{^MRZdzxa7j~+W$&QM@b-TFzY`R z>Zi#qK-&t(X|)@C?}EndR!0sU+i#$UPP430$#3XAgBX7;h>Ie!#Dp(1iqgOzM{iSE zjmGEH?_ic(mS^+d?i?BxnrIfORVC6?%J6KSPj+CCLwVDbuA?b-xvozj&3=XT3^gW$eODs#!>a^9pV%4UwfNl-Bto=G3`rq zgOyOv5AsaAYxF3=_-YRcsf7eo{Zr=TdKK%M5O(}&FehehGJD{J*w= zr4|3D?v76EHMA^0+lGZUYVH00QF&x=e7Tk zI4)vnKQhDPbtnB@2#o?9iBv)M5aCO>f&EA5WiQGq&d0sSk2EXt>F&U5QG*siogX4K z+kv=L*y4`+Lig@{R6hg+L>d$ z>O+$C)ixs@rgICe*+@HVVzRA8y~Vb*Dxx!*>;;x5^ZUS?cCIm$6}O#Lx$q~$cyQXZ zY_fenZs4Y5rE3N9K&Fb6^38jim19kq+a7YiGDSb$;PFj1y&0MDLwrgb4+7LgH7m?* z$6VW}LL5SxMgmijqtfTxn7}xb_AvLY*{rH8`d6%Gw@a+>U--6%dohk>3TFE~$!^Oo zwHpb`vCj%65K@MX$7#UHqP;|)!~$n=pvCZiG4|*cwn$-O(ndm4Q2{Hp@64&k;uftY zI@S%gt>d1TN@1DoW2hel%TTnSsqhh8CAo-e*uzw&N7^azG1_bc69;yI;3(d^ejf+k z&7kE*81Dk!S*|%%%b`m!E0SMC;T974%96!lOpCp2to!^WO%HTtRfjCkbOr>!lr z#5k!@C)jiekD(;*485XubaabcY!l4#`zEbv37Y!#MuJT_#~7;6oRVfv1YAogr88If zup{O%tjt7ADIPx8>tfylR5c&4{)))zc+%g^=W6Snrrn5#v6MFqv*%JcD*!RaDWxItvEDN{-i? zYrkl)*CD~b*lWpV;MRXv#eIW*WYU&&(xGsVcvllXIGc48)L3>28*XGpe>wGz;4*~4 z9G2&)Gxc-qZ6m?y2DP+Grtk@a3h|2k~bnx=~wd*(X~j!5(L zPF6$0CL1!Iex~$GuX)1+aKeQQH(LETWRkaN{M`tTV92_v8vWvd@|b3UmavyWWSNog zFD9Isk#*x~H`XadKtmRLV1IVNeFM+@(R71r5BgWn*#=RhklDu5T=yT@9|k~bq;NZc zsShT8gnTTJmgag@^vcJYeG~<#y3msJ z4){PB>4~Di32PTpR%Ejb*zd3qM!JWgiWRnInw3lg!UW={lFcRcd`T~kh4g0%SN^+h zK)!N-C?Ni$#+{7AT^R4t!XB^gD8kJ5`a-cMM%@3wm2{sF0eD>#b85K zQ-_>2PheegF*~uQIrJt#itc~#Vpu2XwDb?!79OMy>>2E?ZfBJ5Tx>Zb&Z3BipM^nA z{zeBS^f1hrj#DUt@$MrQb9Mz&N%9%M3GH@TUTtqO_M z&Gn*$BAVsIJ0iy(leqfX`jTZuQl;=jHCtkzH@7?Wpkcl9FMl2jnarphj0y-hS(+ky zV7>z2)^}bO=v<->VD=ClaGj$cx&f|4%Z#5%K&^{wb6JWBZv2ZG9qnjW&`L0p4&Gx6LheJqU(TcngDu=6Edr`R_^@ zAqmQk9Da^tl*B45&_geAU0OcY!K?WIMJkW;gG%|OeA-s1LC8eM#k#|25_CuG!TQH^ zw$8EBgVC;|=ybLkrKG^?I1k%31#)U08Y=MJ4zuTp%|zOqi7)`# zzCjJEZqL>cMfBy&%QDM9mSn zxv^pdWjas1BxpoeKxxZ(?-#`R_ zP)}K2DY{&}IbB>-uG$fV*r2K%5xYT4*<_+1Rc>8J#- zO;{W6OiIRbJ5rK%fIq-svYzefefDl+Wi;#(55>Xr2lp=Ui>Ig;C;=5Tb9mo1a{;D^ z{u+{OA?}ZAb3Ga1^j-tx;9 z$iUYxXB?e(f7yY(Hb-YcRb$y;l8FCra4wH={m6wqtse?q@TXl#N-C@&I@EL%ElLS= zl?~RT&tPm^lUH%rJqFTq{W9#EvZ*8@IOYYzrI&6ZTkrfpusPs5@UFL*{cf^3;m#|6GW6{XJZEhG++yiiH)}(@y5HALf^i7szmtL!A z(JEo@?J$&pT6a9@2ccWo?MH6*(R2(|B*IpalB6;z0xjhV*dXx)~Ixa@7uY zADPvxfi~b4W}P_j1e@oKfnn=EWyZc8*iSSC_9SBOUHzDVMs`a`WN+a=aKQg_-+5hBf`at!nL|U=c z)6F8=De{8FdmAv1<8%>H+n3_F>L)uzvR~90GjJn0Vid?{B1ep}anv)((qbAoCKYzR zg8P^TssJx5H2s>>LiSS{cd35NU^0jq1CO16f^ID9+LtT|C{HT7wPBzTIt>Ibs32ku z_Cy(cdGuMDU4UXvz6Ys#7mUFy;JL6A_2Synjo078lw_+qAPfvpT=~5|6;|35g3EK+ zXU+#F)0sQYfdo3hhs0u-im@iy*720#w`v#zT(hYwnXQ97(v0Br?v;Kwtup$ru7fI` zUc|@=G2siBYklvN%oTdlwB00T_R#mewwL(PgJhw=At!hR2NEZs(-hw9E3abPu13fO zBCeE!u7aW>#o(|ICWk2YCM?K_OQRz$Na$0x)%lfG52g{UzWT?*H$ZQ`P|&dKSjt&LwJG z0fM^JRe%rA25H7Fje;K~70|(9ch0*s{*ePW-nJJTuN<(OH1#p`epIe-nmtg)zX_sv zYgYNN(c_K|3q@<3hE0`S1HDTYU=Mkugw^|hE$HL+15?e57_wCIB8O-5sT{zmWSr5ABU;c@YfB)vc%MX&)>GEo=MNAda?nQPhTnk}JE ztUL7iSyHiPUS?qtiCV5HCxy{k!jC^9U;kwm$Vkaw;@lMqEoap!nSB*tdZS0lHRU?< zv#8o3pGW+kp^3!i*V$?=DJR(}%}zOLdnwmNWCgfw1RpdmZY2-q+yp)(@fyc0d^)L% zLaaOtEH9lU(@ceyJgVs^Qg>>IBRcGtuJb02SxPA&W*j3MFxPAP@+IBWiQ_bKimWPC zkod4baT=il5B^hpd{9L-)W1`wlqOx$WjANT0?7O5x z=aLo3X~a&E%C4I61V9P2h(}|g#JQmkDXGiFRQa>!M%{7)F?84TR`v*BiB7j%sbDI zOE#pk*vTuHCWRpxm>i2O%WufE(6kDK@Tos>C$mW#+6*jTPJbOp!+iX~Cn zabO~~@rC2<(su5bS}jv6Ty~v85-UxBY<5iDZu9{2oR6M!7mD6SzRYz~@^1d|#q*Nj z30tFhs=S*6;;{*KLtMAh?P-`MykPG50U;Q#%@(3mvUZM&ozXWcq2`Ve4cw7xHY$i!5yx&N^57-Vaep+q(N{ zj4NBJx-Ir2pPk3){ja(S3IftRmq8zzoX!ne_#YpC6E}9dTYkO%P25p{C5&L~qPgvh z(ds6L$;?d$;~z!uG{!nl6W-g?ZpCW;W7?5H#Ih=#Mw17#S`x|3NWebL^c^E7!a89+F4LA4_wxD?-_P!b+@G zOt0}UIV{pdM!IsjXr|=>bG=#SS;BOWGaCr$xhNj-CejesO4?`8G?AOi6#986FQd2K ztDo`btgvI0E_?iKD%fTP2`Q~)HkB8i+^hVSvUcWQa0M}|{x6ry=^On>{DPB%9KCpF2&!1u+e#+6(GGWM{cwNtgxQ@-l!PW67!p4sp- zN#ypez>v9y#@y=TQl^60im-NcKVjP4iOCDDg#v*))pj--y!<|9O}UB3l?k%@fAJSl zBh62yTjM@GKKWY9vfI%$3C9?Bozxzw-%hi&x%_h>s^cIeyPcJp6XE58F;#r@G__d} zT*K$tqOPh@u6_F}0-|?MW@rhrZ4W@SS7CZDd`|4M|49@5Y37k_wLSPOK*}GcPE@aY zDhg^(ZOk0BdNx%bJJr}%A|TyuAhD^x^TUd5wxPAqvhm292e>Q+32y7GIfFG#4?jD? zJY;I`4*hA^gRzCly$ekgKuT;(Sq$nh7niKlLbrnu=o9%{?c0cf0zMJl)WzeYkeeff zx+&v}3df5%p%ZX~cIIqOC8=|j;GtCvBtZw=(0}PwR^3CVnR!-ZdYkIAow(2Cv1EZC zht5NE|0UZKVY}2^uEmfT0~O~cLCEB)zgimHvw2zeTdR3L9AmE6IOo0!hhrAk7EP*M zO;g*)*UkibVmWlK;IXb@+bygVso(d0<1`u9gg0g^m@6#Lh`Gim=6QaQq2- z%67;9US=T9oGkT+g73QrAHum;ExjLSp3Tj-p0^ehauqA=H0Ytd$A8wI*Ky_Na4Fn5 zZGQU7Y4;$>O%B9%I(GSeP-u}`x_>gWuegg}-X?T^cH-R=tf%V0>2< z%urf}gn+^F2?NjW`K5PqlIOUGZ||>@@*#-j0g-n)m@V7Vv427#8fkFX%R_Eyx3}@2 z$!GdewA(;%C%YEY{a&;%OL@~ZReVJxU~1p*dOw&{+iEYktq?j_-RGDLXhZU8!~_9l z=sgW5+1h3?Fw=%Yh;w#zQ#*E-zgGoC#e;B3-Y0Y~mE;XfPupuRJQ&hzzf(hcm`OTo|ODnS)frF{q-ju|b4IKnjFZ(GKJixZ6*e&B#~Q9m_lE z=ZyJ##cW2x!8+xCe|J(q0eq|}QY1tsM76wU5#LWGF&>CqujMpXn&R9qFx;Bc6Fp5& zycA8>`uNKkfh=LxFDPIj0(E9;_D|p2^1`=OLgiaG2p-NO1RzfZBj$%}o!fHq}qEs{eaY-4)#TndJ^j4ZVmbS!*d zVPlO0!ZptpZcg=~Z3WlYR%;{F<`J3i4=G~TSN%PvOlH&%%Ox&S>#NRpKKY8agQzbT z#cNeLiDg8lAC30*42Hsv|A?#NP|*7?2pFmmYdXl|M9xPu*dqKvaHqmr+QD=6DZ{Ay z`^FDhby$uQP=##I-S)7x2haYsyb}c0J$Vg`qFIiTc(`xy>iP8 zTU&8}QDNq0aJ0PKI8bymHKfse;T>6QTYv)O@#(PbjGX6p@V{$f;yIF2ERZMP_cfbW z%so&|(J3Q>!POZf)avRD8zWpADE`otkAmK0Efo|ryRZcr)<jQDRquzysT73eoeyw$*TdyH#t zN?be-wNuYxbKbX|+T? zvjxyDzN){~4Pu(dxHQ#JG!%55Q&m>W(5A;)YQH5Tqn^5vJ%^JYl}-EJ^RyLLJrCh3 z$m3AwM(CmBcM^{{FyDSwvw0!zde0zA5??w$<4%E|UNVXA4}*TS&q~4X{hk8GWt&i% zjz1@EP1kzQ%OG1pM+9A@Y`w-HdgHus(BXsGQ;yBcpgU zKrxtMvOzo2@RQ-);ndlX9o^9&eU%b)EGNO;W7Z4h_l#k}A42P-1|`x6_LGN-TPQSt zGrSiu+L@CiMyC6K|7gX#Q$*wK_$_z4ekM}XQabnSRJ2k~`5RG3>IA^Qd}N>6JPG#U z52Sn}0S*(=wZZd<0s>q*C~fdi55m;4f##(&Ldc?M{Kz6|uQB#- zQb{s`=J|&F3Gd{ra9KSp<BlVte8*wZY3Sc8NDCv)HJNl1ne+dP95xm9Bt z6^P}MUZ*G)x8P*zkgf!YWl==S18b6WzlXJl!K}b*=9O&&jGN3{sG*HqS$sJiA;p{u z&Dktwp7U`cEhs&-uA8T3Jz{xLX6e8!2a76~C*l$hJ7;LYq!J8u%1F>AGeiu^F_dSd zvPDhLIbGA)oiYJ#m0l2{8@-?bw1 zc0kTJ1(A7}_1<>R1KOSANm3D#_SbyuQf*lTInQ)CIkG}Fv;88~>^kM#`tFg!#gC44 zzvPq}4)moLjd$N(j1< z#hgRnc89PxmmxwctxPoAf<@AOg^{Md$v;tMN3U4a;;q!kcVPAt5ig{YR?sOHU1pSK zqRdsXmJFYO0fIOeg1e`8@0 z@acQJ?Bqi8XA87SWLOeXvc$)@Ax(x1dH6KnHX*_s>SIJY=V^OK0@w^6qT5;=GO}y5~XK&$ei86ua6nR;~uPjJru863#E`C3&RA z$Za^~A6DLUc}{qwznvY)||2i4R!yEZ{0=y!fgFb2+}eI<~)&~h8_ zetra-e8XV2OR;9HBq>NShRkKFnK9iqAAzN-&lcGj}cH<>e(E}5E7u8apU%{r= zd>_MrZqMV)+cw^sIF6#p8R$(+fblo{8z`WXj`z;!71PuQNDrfXdQtAI1A&JZ1s+69n{hvhN;V3&VX$h z9tQhX<0$kH!dAEpBZ_riiLV)LkQV^?M+~b&@ENLfKmoz#8PS=>W#}RMRU?%3YzA=A zcsmp+z`;JUbn$)e*@Aq%c!qr8nor6jY+2=~bq%_%SH+3z@ibzS%q%|eJhg<& zwMoeoPL|IjroSno9ldbAj*cIn5skdho(e{7_{()2Keqjvk+Yq;GYb>jYw#8;_%%9w zd?vB*ZTw@4xJP8P6U(V&q{Qd}lwkZnpOp{0qvQ06>$Ok7pEbXR^c0nA%(*WlqLXHQK%3c?DEwg>cEM$<3Q1qp}*LxT%>}b4$dc9!P_D&eochJDVVbwrFApmUTu3 z)NNIO_`-A)zc{W4#qRM4z%l16~$F;@p*qit}(YJXU2fZDP++?;3}@ne%`R zx4vQp38RcBH8kNBB&8H6?F=T}dxYt=d~i5pRqpVj=ddOn63}XBjgW^s2wh0!6sofltQe@rm8WFi5mtpRCH#5go0^-7dpN67&YD1hcs~ zE#X-m@|he-0Rv8ooJSeEk~IIN5BacQRzTn{l!&>_Yg{a-4BR6`vjp&jBAm*^ZWsG+!i%Q=kRHWqYuh zpFpa!c1P4}7>09MwxjpMV-I-9y+TJWa0yNhritA z^+SX;BrlHFX_n|)3A+RFC))i!!7=mK&h!e2krR_Au6)pF^#?RPwbv6Q>qUYIv^Yne zR<~~^^pb{og(!Z&EooNDE?2zcn#{v^a_m-a6P^M<2Reb>Hu{*_zUxDH%R&}Nu!)RV zJJQ^^%vD|{{}3_w&0UW&fOWP0x~RVxF9)1Cz^=h;Rcr-~$*G|BWee zU*pA29=>9DIkXukXJ6-zjpZ*-vgG%!lvfn_1Vg!9tWNY%aOUlH`(#0USADl}pDA{C z;-#V$6M13fjo^H$NgAJ!-;Z;IZX0E3`x*?uSSmkrWhXAH*nE{O#4v6`UOsZ2gZ=7* z^@!=USgTFcBi*WuvUE<|09uvsGJNNhaOjz#Nk-E2+?(6_G4E?k&f3pzT)*5uBZ)PS zBaNK9K~F*IF3C?sXZ`N9FD=?S$#PD}%DMlGw#r!l7hdH^!af&5uC-rWp?qOh%>*?h zQH!aRu_oJ#%eFW2Zn0y7oQvj!LWO#*n+bHaEJ6J9kmN9RK~Nw-Uk982(E*>$siD(R|SAIOhQ2T z_kS_mSilG&<#-x+r5*(h%oyN`0;mJ^|3QxMjbX=BzYxOOd74OoJx7AqO1z5*7i0u> zot1o1Y~VZgA;mpl&6$>wl-4|ehN6+UNNg0i6E!bcr8v5hXp09im+thTx+@q}z-bq*)NKTPpzy-pBB_61m03yAW zSXW{~^EIngy>>UR86Zv5A4}a^iL09vQby3gK<4{czS~yy9!AOC^r7|IAObvo5x_cp z<5dp8GPq$Gue$AHW9@E7t$Xa3lTyBsqySn_VoG=J~khA3Pm&l zTb~LN0BwLF$A}%0mLS?+-%)-C_tQ#hatP>7Fh4+Vv17Xv+vwY1K&cpindh~7^A(vC zFlUlJCQQin%sIVV20%XGZGu~f(*pohT6Y%<^B~Wc6^SbEb1)m(N$r;5YTyM(C-W6A z)PPQ+8-_(MC3<6tg^dWqxF`dh69C)k^Z;lk>h6vR-ZSAimg1i-!V&?RL1}+*1#h-F zw(F7MSdt~JD z=Yoq zHaU&R?tJV^r`@YX2x^^hGJJ(SO7oreG`B{R_jRF*`dWA|Daz4azmALr^Uc+@UaL;- z>lmG5$k9zDygPyh9e)>ZIi?PZIK(wiY=_(GfDQ9}c{g*u_0IlBQxZmN9Wih$m35RG zFcBg4md;g45W1EPpYV^X@4A%G5tmWYbw4>Qv7}WO(a!h6Wq)6_v)vpy1J9r37r4Ch za+a9EFdKe(r-=c^)T5JHebd7A#hp^7DyQ8u)`!V5Lv$ESeWtdkhQ3kL$$U(Fdmg5= z7Ce(rvP?BH*>u|VLEeC8AM?s0Wj8j>ekQza%9i~wiGSOuDSKNN&7M}T)o4STBIi6p#vE5}IjStJB7;Iq?ozNwndn&(|LXv7dfb``%lm4eya zn$`cqG(*TiYlum*o*T<$hardl-xo=)=Dk{NY~8Q(AGWXsXF zFtBo4hG4OZYbu(r-><>eo>ck+RFgLVyd})Y^y@r(&+KC?<>1XbplG~f4C>?sl|*!v zZrsm$QSSH%ROnp%4qHbqzyvCq{sz{R(0{t1IffN)0Rqe=B4t3VlQOu92`rO}0HH_= z4=D~HcXCltXk>4aq^fCZcP2Ri+WCh_->vWT4Tr?LZSE*?9dN0F6;~){Bo%MsVG(w* zEk4P{h1u1Xoy|HHLy@D&u*!C3s}Y~VT4u6_m{^d$G{7YHe7TK{4RIj0Dgp_bN@i`3_{`Q0;_}oqzb5gpjV!ik7D7 z-I@RWhWtSDuKK()+qg1j=rX`AkpD8qN(j<^JpzkkHv{%*);qH%jpQP8{w#BvGb|SL zP6|C(@t(W@n8Vn`>iz{iG91{y)d2}@nqH(5XP1T7#G4DaYn_^cl+3QoG63eX(*poH zELfAfol4PLIclfi^*~zD7zNhbr(nHRYA^)2$9CEV67vfEmdtkz#M|j^Vg&@CZ=91Q zu9HE};sgIGeo8uPXZ;nVGk~WSD~8&KhYdh2^dO{HaSuUbj_(0YBGr4o40C{J{C3=a zlwX=p61+lB^f&CPZ$>RlmKHvvY4Wtd`YL}NWq$ikX#RKZk0bN7+sf9fO9W%R zEMm2Ch}iMP3tNg>F;S54j~U+vaZsMkdx|yi7a;s3OlV>CYs6URSdvZ{EimS82){~n zSrD0z7|WdcD_Juwy^8)pTRC^1L`&*)Z0{LO|C(-{Ai*;nMoo0vn`zI~u1);uaUT9bONw|r-nAQ`%fWY4Bb;d}o-}f*+ zt&M-fnKN>$(lf3HZv8zF8oOtDXhnMdS0{yG^j$N)SuCm66W+?@I6bDFm#BiO^R|H% zGfz<3hUY}S3yAuh^yw?%yEFe#9sPOnu`q9=j&1p+)25|jPUy}bS~`=iXH3KQW3-9a zKV^pQ1WHdRg*m)IvCCR~x{=n=XTps>TGSwbgk5LG?gY-p=R`0IKITR@TyaeiuZN6j zdV8A+uOcUPyn8uk1*sh2WF5 z&TdG1Ild)4`p&IIwq6=^=s$oagg^bkYtuJXB^(i&mM=aT*r(Y5!(~UEbeY)D{RztkSgbU z4;hxDL?e=7;nu)Bw}+4OVY=iKl9cvR_4qCp*hfHlz9@cl)TWm=yeJUezI|8@Z54fs z%e3gn;r&1x&i?uDmg-L7)*VY|Qkp}#s4aJ|(COQRxpS3y<6#aDJ@a!}*5d`r6tZ<1 z=(0bRdw~C^B(&TMXfw~7>4S1X`h4N=~W*2e)%r?W888E&?$7e`?nr zOFo7(;u%~06=)%cR{N9QD`!#a>^D->@#mm@$;WPC?da_%@HzPNtKVF(<}e=iEndfD zFSfK?`~L4w-lF1iLMD#W`|lpHNvVI*d5Sh`g6?*Py3^5MEmJplU@`?th>s4fw|| z9G-s`QTH&L-(ui>TRHMy=+2=^=fcU&ukF*btVA^-^q?yegmtD5%L7|)>Ltk!2U=~O zRJM?NtlcUxfIPXVY?CA|uvpS^Jmk?t9@6ss!+Qbm)#~cTUr>_61Gf>U#mm9Oio32kFMJhr zT-BT#V>PM>*O~)v*#+EdZ+Or;!>%^3Ss7wBdn3XjPclcoko$}5;Rc3?QVc(O@q^FW zt{T(mg~XKCPYap%(<07S?CU*N&;R}63y*+SljQ$qyP%PFG;FS%3bfb~sF)dN{Z4Nc zd99kpuIRG8{+^TKuppJTeggOH{eQ*}m|jmX3cX9j^&k=d%>k8=%8iB|QV2XwS@r1* zd&bnDxC}cKcL^YJm;L5?hlxNEMXs{&(m@JI{fS@=5GQH%4h!)`O)Q23?WJ4 z=V?06QpM#Py;o{`u>*14lGA#rV{~f?%VOFQKHMXOKVS%-B9{u~&hNP%@$}ZLTvsjk za~$geI>&)_ZmCBfSl85k9nsI-L;Z5X4sKhD$UL!&_i&8wCA* zly@LCknn&Lzqsc8;;Vn+rUo5?aq&?s* zI?X@EpA-tuXt^mj6WS^!Qxp(nDSO=6^$AK|5wHHTM!V-DnYJSCDpz$P0Mw25lTG%9 zYRT@52<7cqHP$~}V}dtNQ~Z*qIr`_GjL~u_{`#3@SS|S}&xiNlj$}X9%&~S0{S>-% zQPgWUViChrS+GMa#mNv8JecQvapv`Ym4AU~W?ub?daAqNe9ZIZe-;(;Kx3_p_9s&GMAQ6zH4EUtXqmc&W*O`FZoh zpk~x|zH;1HnMo)5{D*?cnz!07V&A-L)&AYUXG-HC?nY9oy`;Hn-4~3nTW1m?7{iOA zKR6%t?^ewkSn?1*t)Ul2z1N~VSJ3nj(&N5TZlcPBGw7h)wtWi1>1H2Wt^QI^gR4Rt z1m4vdErf-dJ~TU1!mX!p#8DZq39^O{xqcwls)^iq6vDpi$F2s_?@Lnz!d@IIyl$C3 z+9tr$C@12u(>2O^LH;l7b<512qN!!F&pXhy0fP~aB)5qoS{4PG&>OpN$W^}Y-F_-X zV)iT?+RCH*RG`8x0!BsfRw&9bbM}?W_d?crm&yuSCn%@1{EL%8NAibe+x_)K;vy#UD2oK;FVBw-XDIZN-oWAVdvv#{jF(FcAC8>^27~tm^Tij#r^qM z^>g{|>C3Ur`Fq2MY~Ki;iZvil`PswOm>%CBFc}ZFdXvhlN?V-W-1m8s=&g9eda>d+ zrN#*57&C|8FN(_ED94x|N#KiHAn(0%ntus({jQ;No0&(`EMm2vThEJSP&_xQ43xXN~*S;)p= z@|dA%3K|si-$J~#6j2-TW?B0%=Q`h84^hm71;J278&JZu1TXcR4!O#gla|A(iq#*` z5~+8C#_4)JJaFA6n>>{fq7Y_lZ?!_Vy9#&8;C$lY5e+YG?E-Bc+6Jf*@5hlF2CAgm z&Tr@ZV`n{=(M;W?E_f#(t|MxeYCEhMS8jM(LaZg;FaGt#ikw3-=ll`G7^*CW8$+KI>c$nvB0psP)Fn7QjbO?ooXH zQ1XNgJ}Tw^{fd3-IX?LQPQ|Wv(&ni#)g%PRtR_ZLfI+eKf=mW+#RpcIqle)|`NC}? z)bN*++ZBa3fs{Bx{fS>+JmY>4RgpxUyu^Zrq}mIXUNh}+X#RG6LiA^c-89PjOi``x zQ8)X8s9<8T)Qv1>FamOQ2?QoKg`SGDL=y82>*Bmdz%!@2DVW&9*&TlvOn;bW*Kw|5 zpYJv6W3ltLpgR1Gw=UXnNznf~lPWS>NUue(slhTY_w@*+bvE8woMS3?!<(#EZoUQY zwMur_!v)x%5ZR~lBQz&(?^oMGIhz|zCW8?a3~|uII}Ks!TB$oLGqUV|&y-@miPx4F z*|sake53iBPO;VuIkI|otG0u3!Su#!(f2(04v3T_)$DJy>U}!!mw&ip#XUFI)#i7W zf0skwyHtcgMQA0pG74H4q>JMQY)MLH5a+?k;z7J>!rTv(zO8quejnswfM#RT)G%)a zcox*Fal@g_W&ORseYGknQnqAEKnvKT3Z=qFRS5ht(9Ex)kb|X6#66_TJIQb zgs!ILw@>Aiy^J;R^yF+(jrn#{9p#_@34Yg(Q_xQG2J0Cj#?)5=_vtWO4t#_%tSmU0 z;@=l`wdzCI^5)s`8eGdx)SHF|Z$&Uz%H<$BZXl{E_|F?kn_4|b6I)=&s%|Gz!NI$4*K}&nXkA8&shXcvKn%H$RW;QHi8}0T5r>SiV_`;f0M~>T?lpC+JKh1 zMD+}E(3kC6{<6zwul?IXov&kF$gs+BRy^`$cTTkU@IW=~?YxZX^cNcFlS%vURDKg5 zW&9q_m>kZ@AL5n?ZrYv};pZy*J~^S?DfBri?8w)+KfeBkbmRLExmPUl#Q$_s4R~tT zofg+#poE_sP-7@#j=Fq%X3wV85qPGX#=1=KUxQ!F<8R&IC>c!Txbk=w{FRPyyT-b!Ey8wtE)(Uw#=5vPClDU0 zv*OxZan0>>y6+r}uSIP)O)^7DbiZe&8!&tQ!Qt@M{v#lv=`-cv6V>0iaBv$JP@ZVa zle%kI(Y6ciYfP?ZeA{?J4splrV%c%Ao|XGfo%~C{pDn5?FpkY);)ukSyPrCe`qH0- z(r%N(14;FGlKGh&Rm!}2tdA7zuvpJtbE@2TaOwXc>Z`+|dfvBbR&rPAUb?%RT^f{b zB$n<{8mR>qknXMpDM3m=N=j)Zr5gkVL`oFI_wf1tuIv40_U4}ZdFGipGjrw)CuEg9 z$Lmb|a>L%yu}cJZEg1YIq~90+`~2K)qfD^<5ct@_(-`>b)xleR(&hJgIT~~OMQbir z-~Hv3Jw91f?5HUKD_lt9O?2*AZiHy;Q0GjZWTeUyT%};`F_EW#U&mF`vvBNP99>+3 z+w5XFPG1@R2}Z4?^u z`!g3&P?OBg5w?l^XH=bN>@ATRTu1sJ6cNy7otnS}?JiQngTMGHh$rRkYm!IwTciD| zl=C6e(8)K2jh3Kub-26g1eNBmZ&Y>QPM$GvtCLM+{U5Mf|F^jT^2~3~+J^`MY0u9RDJsYk4r$TG#{*l8s#=liOpZzDKkU5TriLeT#YPC;M-te0z*P6< zt&-SQrHnz>IoFCGzp;~NfPQ;o!M~~96Vsp1D`Hxea@XXxlcsK_v^@>WMU;Qs$!MPx z{|36OS@bCJcIqJZAAG+MX!goV|J+H5z?ymg(k9J>d+(c{1dT6<9mE;*znpH^I_|hO zihX)(5gq@O=f$g@@i4)r@04t-GzoTC0~4{Q!k|z%>)}Y@%U3VIt*qHQvOvl^Q!O*% z=~?jHGAL#}fR#afNgl(u9(IG3!6*6F2kdsjZO10DBrj}#HKCM1<0%%Z6Pcr{+4c?s z0a)yQ7Us99PA)Xa&afAE25w#@v0_Wh#U&pXKlkH!wGJ(Rpe_5xQ*Gx$al&+#y@qe& z@Rr4~vn)Ly9>|ku!J7hl@8swb+j$y&{a_y9@U}vMhdCn*w@KI{&FZr^>1N}E6^s?H z^`uD=&sG9>-!LTTG&y?>d5Oa}&Ou1nLSBPSOkDDl*ev&TF}$e=XCpOc$yH=)GY3^%5-uUk$iZ%5NlHaQHqtO`-gZT6uNFl(FJZtAm;N4&E2=i>YSQ z>>z_;M*DZxW6cv57Jr$nG!RX^=TH2@nO<1GG0mQ|tf_Jim2ToKa7k0U86?n?VIdx8 zs;*y<_}(nl$atpDQxIb&9b6Ig2{rj{qBspWKYc5BzEn6$kgth3usUU}OwDxa35cnU ztW*09|8h-u7`}2>C;UZ;@coFA2sUsN2OOoZ%CEl?>|~K0o9+pmaeOKsa+We{4i{{f zD(Vj2rqH8!^?=xxAkYrt5QcLzNPOm~CthJ22KUs$Mbuq>U97ax{rw z!}P~EG_*KXcw<5Xw8@X{aKI!awlLc5rNeqPQSK z;n*H0Rm7n03U+MIq0;(!{&Vsj@3_GGx{v*9KpQiYV5yay0_`un@gyAFXxV0#3ndp~ z`WhqH1$jF?fx6B_xQLgh1x9lXc$?)XJ#?Zye=XGkpb><`>#Zj4g0`hVUtZ!ScRZ;7 z&4WINw|fQ&ZDuLpEAeSI2v~k_4Z7p>%VKeIxn9kOAv!Q=eDYiU$H*$Bb)!$;h?f81 z3t1dj zmXTKf1GiYDQ+h4716drSYYf)CX2n5Ow~Qk{JDMiXc!jNLqU3I&p*P+GB?VrO-T*PE z+-6L^7|ObuJR?$S!>sYFFfHDT7t#N!NKn^85io$4C(mP6)J9%>dR8l0Fl?5%K+51m ztZ!P_{|$fF!P;}OyxL!c>sPPv#?ecL`Z>z}xvS8<_NuFFtrmuNu14*?!A=xaYwK%r z4xOUozhk?3;so2+O3o-YmV8|(RFu5y;$`xHD2fHg>O%6rIzBfxk#`{aMQc_xPcDB% zV>S4^MYaxN;|A{2+Ti`+2r}Im|W_w zH?nnNP6M#~IwjO9O;RUK`d@J-?UM|xk?Mko)8{`%Ns}%Z6|aTmB$$=|Q34<=?h2Nj z_77}brKVq@WGP9X?X+jEOdGs1_sF{HSH}}29G(vYf9hK$39LRARSy!p-gOz5wzBy8 z1_tLykBDH2_O2d?$CjV4`v)a4LSZcL(@=mBzv89sh-+kkE8&xj^?a_;<3TommiJ5G z7s0`%rM3&WLox{r6wW%l%t$MnH*^Vs%p`M6#TtQ<{^vJa zrb92CMNwZr(~+pk%2`(j-JA_hqh7_xIL(yam`V<@BdCdVV-;fH?!}PqXJ50UA;01? zhSqw(t7$evy43q?e=O5Xhaiqqt16bt^zo|oBC6ZXYfMAO(o#F?{K0V*kS_#3-JT5j zf_%DEQ5(mdUU*PX6Xg7o>M`aykUPowk(54QpG1vis(Wggp(B`6;b?;9YHlJ~z$w|J zFlff}`(hH?UR;q+RJg^*4k}5yM#(GSGOH?7(#l7<+DXPsD`pTgu_88KZL>@vW@Hjx zJuBOA<2{1NRu@KXurrXDUD(_xfLn`kksZfr`}Sbc139`DszbwA#pAdalJ*eXM_9r> zzk&c%d4siG8B7|-do?oaUd@PQ%qcf9GRAo3Cmrp*8<6to^gK3~G1-;(J0vQXIa!)5DWtH+MC$GDvb$TMjzXrlb~5mckP<%tjsNf_l$8J`_L&YU_8ynTQ(cX@BfAcsmc=_Zl zr)ZPLT%GFe`WggXhRvGm*_WPfgrH9WT{%)mb!2jv_kDzNXWQ zEO(ZUfAIlTN~auf$MWlYrgz$ZM8%#QtnE6cCi=hh&YbQVS(2A+6JnqxC#IV(?)+5|WJ7T5Pw;*Ljg5I7~U{F6i zZ0#-m8-QBv;0YMiizt2PEW_XP+0yg+hpuhw%N%&(yTd*>+t)MSM_`N&Uvq&6rfVfi zS+nM3Osa!&1&cdCB&(m;(a|AeVA?I!3W?2V$|@Wo&Vf<}CE?gj^A~A)#6$k-9O}q7 zWsl2_WAUhQ;sePP@zl$3gy_1h!D_s)p?ULgp1({A`AlLG(4jz@sBf&x7VK^##_bO@ zpFcFoM1PmB12Zb3iu}V#zZQ?P4uIH)7-19T;e%o9cLT+dX^)D9GEwmzyUmA8eG;4Y1t6esJ~MRA`3P9bNsZqTPkqj=5%^P!yj z8zSAq31n=F9lcJ4?xi_KxSi-RbHi_Np1K`UDWs{(mE6+}+1xm$ijVAKs)IjLx!t}z z6t?~wM&Of(u3@wnFX2OvEps2L)qjyp<8~{CUgJl700GPPAh+1DPo7O0*=1h`e>@Vq z*u_MSs61YXaR2w2bsQ`}q0?`H4Lf=}(ys`QO(uQ(uD%T55~6_WZhRl#NH;M)@X(q^;G+AyyKsCkYXMR z2n>0~uQ(aDhllbGdFHW%g96&B#N*e5&yuP#w3;WtI;RTO<4Vo-Srl*A*f+0$|CLlC z4N?lYnNTYa+lus%vkJ^L!lZKll#G9lKCZ2ogB(Y5pL~pit;*|pL7q>qF+$CZKRrPf z_@_IFC74xsw$G|IPq1q~c#>=YYpyRw>B8ro0!-Dj=jov<)#?_YKO@M#4T@SKKV;?{ z2oW@Y=s-UI0!x^%3Nya>o%tQfpAVR@)do?Ubl>rZ(*(woyV7;G+K+^Ot! zB0BQAw;-M@QM1Vo%DKQID*kxki83b2KU6A}4v7AOGnlhv8iOY+BXUJV+bne z8YUa`4_~QIFxuN9Lbh2E;{XHs2kYo>Cej8$2~_CmtaOl7n-T&1+ht}XBeRh)e;ab~ zaX3Mn<)YpSA=oqt)=YG%&jYXiIw1YU`}iM5Suvs8zV_)1gEu?q;TUaNR)(9TLsv_=JcIQDT7Z2D zOIj~kTw+m7sIN!yuy@lmwl7t>K+=F19@IAWivA~g+D|-nTlH%sw$M3HuwE+>xkx(; zYLx?#tV=R`Ov)v)a;*l-Z&cBD+?PjwNq#qeN5=f?&^DV`@0KeO`a7OoSP0mhm4}>1 z{YjJL;zv(3I$kIAWLd&tIm2_=k^}VP)=olgY+qkH6<54r zga^a^{7e&UgB+vG6MAF3?V32`nwoB>jr;& z3h^mJ+?}OmfoZ~n3=t#lt&l5|xTf&TTf`M?`-{{@KBZ+KY`Ec<(BW71UfL)j4 zxcm4QdXhN4ia<6 zecuyjar_W=?ZPhRU233FK+WoOtXNf~RGqS; zu_cB-WjbjVNOi`_$B*S^>&rq%l_ueB=4$I(fg;m%+!E}gV3!b}@cz7|Ftq#jlsfGK z-=DB$<*brpAc1(`NVSvIaNhh5KH+HgNujK=YczQ9p;VlW$cwx8ypn=?A1ujyDZ()90!B%HPAbfj4~(B6eh zwpo^(!EBo#g*U@NHGRgB*{n_^h4&Z7xjF}4hRZHIZdCNFf7qVp_oi0J>+>9%XPwOPiHSA z^wYJK?HFPx3Q-Y*M)*<->Q+MWex(#8qlP2UUUrW z!J&DkJ~&3`!jkk@fHhh4 zeNU?N4CimynL7NK@Q@|xivVk?=s{m9Y-S=7Mx!xHN)XRFY0HUD7lo9kYR|YQz@#+{ zgb5z7O@8M@XNpRpQw?W`GGOK!vxWrmY?IHq(AlikRjKAPg#|EQjYk0lhisE~T&%gG zG1aMvbWt=cUBe)cz=eG>j+-@Kw5B%Qb!MVovsPoai@Sz>5>kiG7oDyLNcT2?G+4hU z3jmV!S3|n<%x#ZmxrQ$W4`9wxkIrMIY)ZGE3H_j%ui?wgqrpB|SI?R!D%ApzM4vR% zHJX)qxY#vBU$f@2nzW@`&4hl@4A#hSUxdN7Jc5HZa#Cnq3NpOi{b%-hc~ch zi{^EvTcslpG<`KPYI#Q4CMWiJ>>mGUV9gO#>jr!%p94Nx=6N2lKMraHs^!(2Zj-)x zqZy?!^9yjyhpDR};6fh>h!P=&Gg{`;rGQa0DC;79!MGZo)2aw(>qi?ga6e$@BiGXq zbm@r)OcRX_A8MQbE>E|aIit=Z%UIZ~M(4Jss<92D=M}lR*!j5hHH2NBrUP|KEHF&h zHiuTG+XJtC$YRYPbFE>`>zLDG8%56}xHZ^oBn>o#T;}sN6Ew6o4E43m6RxJNq#>dKsRsfebTI+~NU8(8rdt7CGjZ#HSX)hjST9HKWA!x#0;p^S0vLa71O(9X zIvtVz&g;Hf$@fx>r-0Px=w2FQ0!Zb0?yGfn?{%pWsMfcLB(e-+DtB})TGN)LADu-) z`+%JuY~i-N!4&@djKkSGaw+=bu}!J*iao0u*zsa$%x*B2M7C_N*+~m8(B^7?=A0q5*9-O# zezj^@{&+uA#;|p?(6=D*ZLYyd)-;9R7_r1M;z_5Qn#E8}>z#9=WY!(jgIHDfQTkhy z!J+-k0s^NE?&2mA6q=Wy*R91#C}O2TPPyjF)b1Fav}`Kt(k?-vvn-rEpK(=WcZr_-ZYt-3 zFKeu`teOH)+WAon>?Vz6o7hM4X9Y$02NUIpqg{sLwj`I2jK-!OoaeN?X-t} zJqA#HegGv}1W;B^d#sZy@zZ@5;ZCY;@vogdtqXEpXdY3FQ0kA zKZKs7eLBS()OPB1S@W7zs`87&t?vq?eY)p|kF^w1#Fi1(s>ti*-k*M09~%2xk&5e4 zKBdw;dv&`*#)E4Myyxy-r|&c0N@Be}#h(7FNn!D_qL=u2?u=~R>&u_=5)&>Duey5_ z?I%BSdKV@q3%ZMM4`%PYj?Dc$vf<(2MJX{y+AB(J}ev#mdH zH~BsH40!Q|vSDNOm_- zZ!8S;a%+SoZxH5?0(;Qhd)xljGtn>poqM=PR1ku57Z99=H&^z{b)2bHNMg_QquvOQ zrpZJ{A!ag(2FdOxSqFy-efp%#lDZ`*qZ@$`OrKwqZkHp6Fy;>y>9IOeFJv;H=9kS} zFR#Nw;O{XR>hsRDU**3(^EuTRy?*LSjrHnJwAd`_$O0~!Ie?h7|AxRX4~(Rg=XOyr znFgPJz;k22=)Q%X_+*3({11EjN0lnTgBi|VSLj}L6s`8Dm~M46^kaSF>gf&N`uw=)n?kz z^-gnenYKP(GqX_$2f1o9Z>agnXH(nbjVm__!r0c@uxd_Iw&DQeT!?lw9QuUjs+P}YcwAJEPz29NFlvA!y8Grky@Mp5BfNdgjjL@vnX|{8Y zcRUDL_ri=ptq)!tcv$3G9C10yN{n#D{glxywymo9y{#-e-#`sID;g{_?P%sp_piTW z=oZ?1oF&=ukB%12via~>+#`mae8fBzH`V=SnYma3w9=2~>DVN;0&QU69BMcA+79WmPr7Nny?25jpOV$7?Jy8Qt@RcFLU)bA#Ztl~f~$9-jz zweb3B=EGnMvYOM2odB!*($K{Olq+IAv};=Pds*btxG~Nw%m6C3`ub|Bc9JL)wM*4S zZ=1*&g7eA{am4)S-8PJlW$*A(#bOXV=a=`T?AZnPGS2XqZ^T852`0Gjw42lN}T%fdP>Y2_$+nU7f*lq-sxUSO~)E zw=I=IZL4@mzHfMCE$b0X4hNT+X;zY;mNd9DUdh>t;yv7DcVMNru>OhkxG9FBrmA&u zFdv9>u^i5IwBl|zd8q=rt271RB$}b0-aJ=?S*i&bH8TbI3;9 z^{E|&y;UgiFUZV(*1*B^(M@0fq2(Smi0pYen$^!QAT%Q$-V9jAqK@ag^YEi>U=;*7 z+X_PexF4DO1{1S#OngA9u0PyQ^Dxh7PCv@gk;m?*0T2;{`B^FM8z>FSc3n{JH5!f1 zfR>&zdyMI5YD#}1!nNuB*DuJl!(mv?_ETlpiqlQ2_(ltua69*c`eED)kNj(XLKRj zV_iK}5Bx%ui(?dhp*bX@4`g!re_kJ%pxk3UrpIR^KaL|-{LLWK5aY3-c(R7hL?IV%0w4M z;>W%Z`UsfIm3*h}kSfkhZ&h3MTd)#R<6;@58>g!-*wZj~+KIZ0<2+4ccPTpNT2;Uq zsAG?(?_l8pSKCQT@wh10LuaamNTxJlBA*qRD1Cg*^;0-xIbDRwyFf(kegao zKmSBmj(xFQY&#G$8TlhD%o;&T5gJ^69xO^CC7APzK1C`6))&_xT0$p2QbG1oRkFtVQ~!3EW|#vj*8~y%o1!-orJ!jlGy`E7Cc5&ch zgEMjar`eo6pa2m?;mPqks14nmu!ZJl7&LfhvZHY|{>L|aIaFcO7D*QgB(BS%UMx6Qs=FV2_ z%3;At+P51_0F?a70&WI^<0B3J2W5Dfu{%Q(bcpx^F^C2bPnC%Z!~*Uu9$<;2!BX`F z%3?{Wn;ahuU>k1b?@vDdgrV%b%5S_Ff`R(XAVo!P4`kx-#96s@^}&ei^l z)IzwI=1Xv~&zkyBc$_aWC`FAvTfGEXH?ihwYs*IpQhrwF=(lGVzNp}u!|#J-PQBU{ z*?i6SP^+=G!BtgXYN!SlX3h3qMao&~U&!zVBplQgX3pEcS(R&M7rBLiYsWb29=KaSmg;cc@nnwY{S(53pBXf;;)x$!d0kkl8YIwRCWgrL_pzI2Z8E(D z$Bejr1GteP+S7+y_60b4MLyq6CTz+N$+*g;?V=`=k3Kz)s-8McAjX6?cY->CAPTA+ zyx;a|Hd(NWi6YFc3fv-`*ytvlFskXiy%Y$YF_1nU{Fv(v5<)jIw%{_c^lecYk5rsk-)B(o z`e$fC4ZJH6#D8m@aUSE#UC_m(H1`keClkf_R418Czz%d$nfm>`IQ4g%{-PV603wKy zb)?{E>wPZ6j^arVl!B!=5btFTR2=Ha8@kB3bHNi3!TFn~V|lTFnzo68D3b52E`Xia z;f?^S?U&<|n{IF*j=}Pv{s0Db1od%K#^4>5bpdq}4;t$`0)xG^EffcKe&q=J9)F14 z=jXNG@-{aT8<**qqrBD&6Z?td!Dp)La!A4YOIzs*mcqY@UgdFzF9+>KrIrHm&VRF( ztnq!cg_Up|zgI`+2% z=dv%Wh1zWsZ6B^Jl6^T`tZy4xoKMww=ip zQ@IJJp>+&fnSr3(&z;*2X+2&bc2TT1gaa#a(A*hkH5}yYXoK9DkwIQ7^*yqxXK$|? zh^LOZv=4bTs4Po>oS^B*C!DVYEA>3<$u9N;aSZ;bY?|cGkPh$;0V?$w9^M<^mok#X zQTq_zR{WP?)kWf)_SX}Kl9cc7re3axf6QsrKERK#)|G!`bN<_~%x?ok1UtB8`ds(; zpc`VUWdvaNGsCj;=<6S%wj!1k*roIqkHi%~b`=jw7?GIBPqa1`$1X7Ez-~pG(lKXd z-DlUL4i~CaUmsKVK&Wbjr5|k(pJh(d+7<>>0<V!)Jf>@Z1CKGruF`i4CYxQhnoTK9?9P`DF!B_)QNFL$NHzzcis%J}zX zBYB!GonY0EhMHbDsiiZg1W(w9~k z6@}!pls1o2acA)(E=Ph72?g0r`NJ?(CF&~WX3So5ItH2YL9i5<-Cuz5z`|Y_6fB8s zFe*yk>|T^>Xp8Bx-BW?Y;Stw(uqys|rN98|x^1U2D8ynx3n>^O9E{1ZQ+W9h>VDN^ zwah}loPgXPi~^+6gzPY?L*Cq85U(nQHd1iLOb~)aYzeed=O zJQ=Q*2N3Q&%ZJ!CPZHBZ;OzOQ+?^6Qw}~$jJSii{L~qD3FJm?u1riYmHaf-R&ME1W$G8Y54gu@uBKjaH> z_(z|V2_Cj zlUoY?`ti$_b*r)LaAN4zhXiO){pF_SAXBg_`GnkxN7Tq`oWPuAwcUkKRoLo9;`+Pf zpA41~iwc*#g5k~`Pa(ruv!lI@q%nC#VsU5YN@>x#1uungA$^CqS1%moprf!3wR|txy9*omgA|AWv z9giWG zRiFp4ZR(UQF7X6)K6HMd)MyYb*e3I26cYIrynm&fN;sB=)!%NcXGoNYRXEaSl+-~i zvUfs0+B1)GlW!&P0c}{S$a04+e?yWGVQm^9N) zNh%)5?{3$3+WxwBwR>dcBLAT$vn3X^43Q#(MI}^K>?zbxf>vikqfk#@(FL= zjpK4ffa(}QVA!{$_pJgdWe0pO1NVF_kuO0YtYbKt$wJ9j${;XEN7-pwdI2 z-rV~6z@U3?Y2fPxa(2=5>|^?G6be?{)2FIWM|+?)4LpEX(B2j7{crB4ojuUfN}kT0 zC>z*wHo#$L!|pm*@JrSz?RG0?r^Jx8Ibf2j3kOm~&UO*V>D9O{;5(Q0en9cv)1?l8 z4(E~UeL6w>pO3QL1|&x596Lsv4Vdam2V}Yxp%`T&@1xA5q(~Qs^AKmqN1I9MJ%Jjo zyu%MdEMrODRHj@$I!bjgQ|2<^Pgv$EEtYvU5V7L8{(VKvG*3F2a&@%sqxa;xFNg2H zaw>PNz|}R)fIN!24QPYyxd&YmSQ8%5U#byVe5>oA)aICFTFfVJ1w&XAM=20qI2f?+ zxTnwG#mr}t`*kt)#>|rrJ%AW1fVC@M*RO+9{WBM6u@;igg5io|dI&ENSo0{xTkV#! z4y}6%ie#I`j*Vc~8r(&VfSNK8TvE!Hs(ME27z(w6p zfJ{J3{bzy3F5g{G#RF6VD3MnH{O8SnCxFxav$pmK8`F<_Tpld=>fL{ELLSW6a9{sK z?iLY#kh7l4Riq89M)XE8Lf7*@lQBTYR}1JqmjkdLsqf&wkBjMd&vd78w~*qrRtx`6 zt?XkC^x3Efl>`5CJ+=4iJ#8cRf2Q{o-#zSZy|#?I);4oLQyxB}LnRj4BDX0M5iRsDnC48)H!<;atsRUo3GbVg% zCdE0|K$`Vnls>j!!i^|-s*<^eA;_G5gcm^yv6sMbN6~d*<|;FP@WdE-$YosOt5nFl ze@=*L#{8id%!Hhx#r)TmKyd@R<$@jJV`(ZUhC!KBS{Es?%#{=83#2+WjetM?>#sO1 z>O+f^XN=pT|6(F&tyBQz9ocLUho8Gn!Pos`jG};NI4C_K*-~(7>u&rs==03Qp$T;>Ryq1uaM*e`bN{1I6$al zD+9fo9LyH*^u8m7C~Z1O0Olij%1b>^La@WHSZzh1hYMrq`Fo)ZULBnfVp%C?0w+-w z8DV~GrfmX^0gXZd$1L1|c{MQ%|2kZYj>ifKad$|oP zp8daOX%6nhIvp$k>W76!G2bw~kLPSA9k4b<^?QjcOs97SW;l@GgiLa(W( z5p7ikf<>gjFWXSGPl0pOvY*LuZ!d6I(n56ox>yjr%cC?yUz(ho^0)9>o`pMYUB2^& z6?ZEkmt{YnHtbi%Q@C~7kzCX%Hgm!?yU)GV-T{ml^hMf&a$*~6z_ohr^1=>n@)jYy z)fCT{XqTEcDlbXD!YX4tJg&w=r2e3~7c-tLk=+tcDL@Z6!|*LJay~K7S8vMZNVFQ+ z03H^W#Xb(jv_XXA_nkDC##4Q#W|{pP*c?yy{ddrx2v2wBkv}n~&v_H-+vzj|a70!@ zLQB?Hw-Y%NH91l3XFpiV>r-y&MbCI1E1O;L1nCN4{9viA|H^?JoK?=a?A~)ZfdS1f z*VY=43P1FZdf7TiEEYyQFrljSBs%cD(WA!{dyPa+h zU>;R{QTi=IgLR4Or^xd^{ygdKyARJ7wyNC*QmI>N!~-^!(^Z)Gu`JqCR5~W#m;LFe zUf?mt65v#0Z_S*(BJO>p1!VRDth9GIJ$n;$;M$yaaAW8k`Si$<{)bZTvF6N1?T z99e0$n=oyCojl{#9}fYKf=pcAbws%{k2)qDrZhCh+IXJR%Kw<4P@)iO{^Yi~@Vb zf%sosh1{*GBu`r3_A2S_4l@lL10TJk&;yni61((?sUd&C-Mz)@_iJ_3G-yODI|ujHk5`3`fE`NQ3!p(Q|i=>2DK|`27IbXr0RlvE_vFz z7;}nB=hr_{y=Fl!5JnvLC6aVOz`9TE7hkF?wQ5|nlN)=D+`NN*Cyr$O!93`RN=)FF z24+>-M4}s}H+%kRwv3>+$^61cu|(lc)3zdr2gKVycwtA_KfQQ0k76%=6=kgwZj}Nvr?2NF_Op=?)A`Io&_%SFe!wDLWcS zfs@()grp|Brf5~!F8Rq4=H1vGS8(+lzwZ%t&}|%Y6-k!z=?C+OJh)yNV$X<^yxs+B zCLMwd947@@n|1U+@5ahM8e`qYs3ftyhp)Y>ZX%4ZDC9Rk!rCM$OMKXcA=!UYKFA;O zqkXB;@_oQYkf{347=vhRTTy*&8flppH| z+b2#DXAvK3epV%74ZJa?5*`Kyt3JK>f*(~-#Q*0zw$C~Dy(&eK3{XMVR3gKeIuw4C zzxM>;C{kixDQ#}PO2n}6X?d$R{|IP+sB4)-j2p%E0ApVPRJfC#4dZ{Q1o~!<0{1HN zI_^GIq<(&%q2u}g(|=QaBgehDDp70TLJ8_!>s@gP3+8@#u=VHrRO0D>sZ}|E)&KC* z00GJH(HzAZw$CvS$fxcN{-x}if8B<9Lpo%AV7(h`6+{UE3Y%hO29EgjS1^PEcaKrj z!yH)ZD-20X!bOnYH(Etuy0L*6%&T|bcxmhS7Iio%-yFgNR@jwP$egkq}>G=!_9t?i}8Tbdv?3RKI1PuT_df>!S*e0T2ggW!P zPhr!hdY>1Vh+iqdMUAEV9rJx23n|+T?)s%h|^3trjCwT;K3wm zrjT|iQvG$}{yEk268ys!5~!=zGlsxaf^Urz4(%cwSfdF~R&CX_K;q5)59?%qe<0nx zwn;MbJz?<~1s*QRuKxb7`%og^CEX1T|}-= zIDvr$%TB<_w-tPz2K|ioq#!%shGCi%Srk*uw9UK`f(8Db5MVdzboHrh01byy*W5b>{LkgU*A|Fith;$*CC?$=ZTk`F>l%y)=8HeMm76|y_~NJtCYLgO-d0GE!l6t=@suyR^XpF9LbQs>_%&zr zrnb+U16Q#VGUDwgR6tR1DjQ=2FHpLN#>q3CNEMU1&V=H_t2mXM_(*%t>u&q$d47ZI zZh@iIefywW-q1JuuFu(c=$#Ux#Z2mCO17oLoA+HWS$RySGT2PU52(*Mx4u7Ku;i<7 zBErg(GV^OQK2Q3~@VxoJm5;A1?uZImGr4zrt#@rd>3|AVSCRZ|KRIvy$wXDcL3GPr zohIhv$FYgGZkXhjueGx+U+*p41it)uFOeqD`N@cwbZE>d`J8kJFw3L;e{6pPAo!0e z=`HI8w(W>l>&~^Bc`Ae2%lP5EFQDh3IE_ajSeI^wl`W3j`+Z|*vC=VEM81^a!NW>TdzXcEliG+HUK|**S-^yr0{oeDg;~P&*v-CFSdQ6bK z5KnVO30^!*DrAMyT{#bnEzqT~^SGpox31JNw&UYF z=;Ad^Y?GxEL4IOgKroKDWthGUgARwQ=RO{Pd%GS41jIwR86047xSI9KyQ*ZB9wFqf z;J{F_$O~LLNP1-qFXC^rF|j8$2@HL~Pi(rW+?c2wMbz(2vminRz06*&9Zek`-B;_| zDw9>ofBt^i3sgu0IW4}h3Sj9md1=d#Ai}Zxp>rgcK}bsFtd4kw1i5c zN(xJKp!^ycAH^Ou=1&1ub6)u^#myYdp_yXB?f=KrSBFL2eDMMT(kx4Zk_!?`r_w11 z2uQAk)DqI5G)T8f!;%69N(f7*bS|Y5OP8S1AV?_Q+4py!=iYzzOq`iHbEZFMzcbqg z8MDa8alTxtZhIjl_r2bJuVyap?%7HhtfpXOUH>b~)l;?WVqA0%sDg(v$t=G(eEp=K zZw$w9ArXHm1|o{(uOzx%E!CO0Asa^Ov&q*cR^QaVi?4=!F1_UcO-JK*+vBhIIuDmy zw5;luWAHGPM%w3=oAWa#$f1+(-BT%d$cfW>XufJxyiy*Ch(nut^a-l8Pr6Y(>f}%V zUOB7!Yt{IT)wu}{0eZWRKr^AJc;i8#M$QK_93}LRUoconk3gBHeLZD{WbwfwmC?!0>!qA(T4%JNTw}VX3V}Io~7DKagGj@sKRMpQfmzAEo1X&nSmOI z9w|(EzlZ4TmtWNUdBAzPAs5WB_<%D@6J3&9trER(%cZMCGCn0w@O8;mhl=q!uSn^o z`Nr7_OhmC+BVNPv>4sq2C+>mkE;Y;DhdR;bazi&1%TzY-RwGxKUcn9R-fyS&MDd6` z*g2;A$C+v;(<>+|GHLglI&gB4=kd<92-`;`{c}UqyE15SW$^XLQ0D}e)@Ot3rP*aQ ztVeEJ8rXJ@S&8hynmk4`DxF`mCvQf&NavvnrxCjUIFY|2&+gHx_>H!#5#eR?p;^~5 zhqAyasrOx@Gy2u1fZ<-Ko?oS@($Bm8r~f<|yhI%#hDg+;l$RTI{M>^1L|}K+!$OPW)9}6WVl3**W6@Rx( z#1QE%e*2GgE1U+|9Hmr9+p3=ahJ9~HHWp&1Pu1(IoE?u&RvixdbTn z$viSaO2l0@IhB5Zhm3c$=N>>!8tiZTf240#&)99^X!y40QCf!nPHU6$gSD^>4R2aX zHE2{%-)-{VotL{UH0i*6HsSBjdg}u`$o1NqTuB)^-QHc%nD=z!?$XJ(=mdqW$g^mB ztLY+F*q(MRTq5bF1X3y9#tJ@iA#u$4Jh}<^HGQ0l}j=@YVk$PVsP}nh1+BLtFOvE_(Mn_xAvLkZ3*7CK= zim`{Oqr&mg<A>V@gQ5ue~(Zy9uZ}Sthx;w7K)Fbx2R~DBq{^Zpo z*g2OkEbct1mEQKgKfN4ekoN{r#3_35D<}rI#-C_J$zQ30lUiy3-fu^jX$fWMx#^W0 zW_MUbQsI_!t>m^`V{t+v)uyLJWALqB`;Rg@wP9=B^}(;Opd$KvuoT z^t%jjQWBK76j|qfaiL147CC2_FU#O)|0^VZBxkhS4YOdF* z0*S(Y4zKfnTnpE6s#a-Y34j0G&*i13bgYe|)79YIRk!ZWkjd;)Hm9|j_Op)W^aex* zO7q3?`+ch34{nvP>AV7#=yKx=75aRwUQK!g9U8d9_nnlaCb%%^&hiQ(v-tEr7u&O~zLeBbA&)vl zrWMWU%}N@)+Z01@es*;uYvF2co~j}5(QvspTSjc^BjA1z5^}xx%K|v&zI4|2Z&Ivq z%-A>W(p>?70wDF+mD1)x%yV+ysHWP1&pTfwZ?F}5KpbV(*C^H`65lX$-Ju1vo|#~X zVD090Y1dl8mA9qQ1RNNT#51RLKAT!exxime5dWWu+=8~Q?=Q{L7vZD#KZOmBUzc%} zIQB}n)OTL27`;Cpcph-GWpr+59Y&j!6pDC=(xnAW^>C{SmzT$Yo z2-%d5Lfx6p=ggVUzb?E;wT({CbiP%99&9o`*SI-mMn)r$Dl`2s-Y*h=$xmrWsJvYH3-N}g?+p?BQ2lBzIOc?Lpf%aRk_vtXMVudrQjp{+G}e_5!oXIu@+|D@zW0x zNoFWR7e~tZR{02Cf)+CB?B3s>;>va?=3d4@vux(qz1~i8Vf_IeEfgtP`iBiSaD2^d zkrH2yr;y6M){c;zlP@_7JB9Wwc)7nNd1aE!SYFFT4~;q>6OUgrVQ2EL5&ugrwJAOI zSneaz1>;_bmds}$=w5hxZT^K5Yth)ki`3E+JI8ktjm$3-BuxJpIQdA(*0@vm%M}cb zZj3klr8s=T>mS27-C(AzSTLmb#^rP8a~&r?{jiefBNSjhbAt8Oe$l_>Cy`pdOO_gN zVXGdZPS+;d%i%HkR*b{?{Z4Xs%8j9;`*BW?MjlsK-fS1;0P}Os1h1GdO{dxLFbR3C&qe}(lsXequvSB^LT$atV;T!f8E*p+zJUrRl1<9OGNV>HA!q;TkKEXK_< ze-VY*F(*CTNs~ZfF;$O!rQ<|vn|rvsN!#?^p5|WCQn5bngTT0lDF|Z1c#ZO|$Gevd zU9T!&PSbfBl(>e{-Mg1AufVyY%j@6QUcY+)5qA=6Mj_^7{k?7{VMpr_zE8Zpycud) z-4`g=h+@X={QRkAb|7UIn7#WJw-@9d>h5XJE6x3y6mPyIzAzuK9M`O%aCZ#}AGh<+ z5AfLywRJp=9>>bZUb^v~zE;?~jDgf(iMk*icl?m*3n7h-%Cp$#oZc6TjRey>W^+`hGu(^Y#*_6dAMqG&noyGO3h`z7f+rKIWgA&Q?kE3Rb%l zYmqUJg5=Q=8~PmcdzDa@0QlBqLQ8dI~7{@rQ`1} zq(dyyiO&amD+e{iM#>~)9Cy~VGtGqOpj$!JGi0m$Q#JB(dCTD zoPsufq%=zoAN0t&Y~6}wRrjm?zK>0@z>D8;Y(C-AriWlhO*?2Fcp8Qq_|@7_5gp;B zF2c=d9^bjf)rxqiQsG|8onHyNqX_fmRxc!@Zsj-5U6%*KVm;4=~AX*Yq`;;Te~5%P%Gd`{(F@)6W! z+cx3}w!@la^1D|oQ=0Y2`s;YG3J9*7c5$pNfzR~s%b2?nAV~4UQHw4cZL7s9IoP%u zd_odlzjuj;_a}&L9sP4vNbr<%um#p{P*3DF9I!7q{FDz=OGZwmG7^w{Xt>%i-eDX` zEx!^QA*JmPG6yRNhBp=2YSsJnS)$i`)vDkBycnrxi7E5Ko4HCFgYJyJOqCzd+fE^$ zKk$_^dUqUu;A>?t;6IBM82+h|AB+!(6#rs2tx3anU!}ggA+mLowb(R8C3^m$Nc?=~kZHI1zwzx53@e6Ed6{@ExaQ2^ZXpwu2H)r~80G6@ORruo-}hclQu z5_Xw6AT=SnjKFy)1NHGu3S^V{TY({oq#hkk=EyMwPGH^06QnV_1A3Y#$siK2r;LC& zMV#p^+;ZBH#MB;VXzNr0UpO)Si|J!l5y08p_`~^HXIPk(`I;HK^L?X$$XJmTqgy5r)`@t%r{D|Bbsmi8P zg7EiN-VT%`iV$*%=QjBv@%*2;8R zLkDz|W?-G}-ffBZje>)XPLeF>_?dO?Mc288r++f`-sQqny>|C}mbi51D|uuy*N2^; zn}nFJTqSO!zA6V08l)?)`@C6P89LbpkMLO87{OrFhssydC@Wy}okb;mM`J#U#p{Pn zB3-TP&KO|baMw=+I0HH*gKAwo++v-pYLzt+Z08w?-=D>Dji5IPIj-nd{VXfauDDv@ z<(K%8FZsY_W)F;C(uu`;$qBD{P@@aID!J0BnNkQBk?#FP92D7zjAcDi!c(B6rZxhvxXTW%dOg)YxzFFU9rX@ z-AG7rgVpO5OkKRdNH`w2Uyl*dFehc%_>*hoXz445>@FNg*CBhtBgW&W4dkZH5Cw-iq<+;ytdfv}_4 z8I5TzW5Y8qcSfIBw#^2K$ERFcwslsUO*(8h2e>GcH0$4ghxp5>y5~Dd&}yxbUb$65 zYLy=LadfP-2dvfQcQq=o`Y^9eDuO4cokB$`IRWbn9vtys{1E=q_ zHU_>;lol$K>=C6^a8P48py%GrYa}vaB))PYW;X^Xf#vhtDFNf8U)-l{J#MOIC{Tsx!J@JdF)d%X@)cOoQ#QXo2X;k)G1n0 zSk#-^7w)U?GpP@@-+;d^2Y^9)KBc6dRD4SCZuuaj&WyoF#P=s&u^$ir-x!kmZPgXp zMBcoaC#qNJU_ra&KbQte32YwZQ*r?2D4*2Jqjx0r651-MK2ap~*+~nF{RzBz*-6Pu z9MUGybFOuOUke?0K6ulyJ&|{W#VkBLj0Smpl)8A3gyUOS{E#G&m;EovSOXZ708Qou zblt9R6tw&0wJwW~T*I5+{`ocjEWuH(%2`mMZ~sa*UV;g*(B8#kMk8}{^C%9F`7uv_a)9h zrL=3iVI!#|3XE~LJ4bh0h{de;gvj18;nX$1`87dc1YI#DIK`qq+b+KnYm_5^cC)7P z@a+-)kM_vnD)DTdtOFdZ1G$wy#e*t0veg>ng!^oA*tS3OKdqmC z$1zN`vrGbAc$NoCm5gkDYcu`*SUr1`H6kt^o|he*q?L$Ew&ecwqoy&oEoA0r?IUAs zh_GqMX%a5pVzHbCAuY08f3d^(A8j$?M(?mxjEANo*Y8xF=DYwxT6VZTeW&SwZ34-C zAN`Ns2c>(TS4|aw($|lyLjOS!(!w)SrHmg2(XM71D<3#>uppkqRT^|R;pO@!Gwu%{ zqw)a}V?*S+kFNKsD^-qH@J+mR<$jK0N1%o z5ShU7q2is%a+Ns96#kwVOjKp)oyYbm;Lb{)%jccy5W}pgMLc}fvQS}DfQ4RKR&=t- z&>#_(pE(wbxE3}Odi^r7>KVvyT2abKX%KN63jU2!0vEi?LdejZ03W$wsr(0Ax|u~F zNdotwwaR+>2CwTbomG|z9p%xP5lmw%_exkI?>pL0*cuPmd;|RbnKpP!Dt1{TJ^4y1 z-XNAiY&^PxmXM?a%xPI$l=DbOT9fI|6|96iNdm4e)9s8|IJ3RyEMOc&`?XITfGx&$ zNgpK|uuBKGV%$oG=USY-li+Ch$VanJ(vQJhXdJQ#*uflm2B2op3wIpsAcH_B6;}I3 zhm;x_^?@|kQCP_uH*0-<+`MmHq3kG~io_=3gQYeViIQ~b*f@Mg()i<-XEsXD0ikm} z02IVE@?hTG45ep`(2>EfNsNU&{unw8#5K%vSHJ_atzlMu1tkFr=&K%gJmT-(Z8Gl? zL(&>f_xF)G;P{?88>}&j2z`K;n-hQ)<^%i-gmUdGP@@1ce1$s^?mEM^FJ24$7q6z| z+F8_%xJ0ZE#y|y6B=#Z`_*}EQp8l8p$eASJS4kS=?!$yZ4oMo&P@!{G9$1a6de^2+ z^fg?=^=KCHgN_xrR$bmI#Wk`Q$6@y=LBKV;)@-plNnoVkkk}iHB*e&<=;xY)phs?N zB-L^)OefJt|CCBaR$7Z}kQ6sqzs7arzR5d!E+_U51X|PZ}S6X7P>q& zvv5GP%76KJ|2b%!z_(h%67dEW!lsEz8cpr;MjK-@!sv>{c&ccxye&%B&~LaId8|Bc z$Uhaidt{viR#CuZyam=+5I^6u(OQP~yZw=kcNtV=B=AJ^To}C*L~Iuh!h@Qt*m#yT zGN(qKp74S23hS!!ySR-0t#4bVJKTNB(N>(mx=DGtJZs|k)It^U?q5O9n70Z`lp_lekMym~ivRH7;2&GQ_DvrfgyN*0-3ew_}9l|I~UZA-;?%?DaZ`G{RL7q7UX=S+J zo!+w{V^D8Z#-)-O@{d5XsG7mZMH9?fRn0uo_EnLE>Qx@V5s8#iZ(M*a6OB9u>9sq^ zv(DS2QV#g-JV-oo*@04)aHYV@IRm~4p9`CthTgaHJr2SJbi8R+MM_0CE<`NG3_mWT zwy7u2z_CmCF!GgiwuRm);#!J5$YM>v3BS4<71mCk4dNBUr-Us{)dG@se%JpOV8>Xc z-8*mau=+8`mAmei-!_n{w{hXoBGJf)fySgue~`fmQT@}|kL1}V)DbgVoNJYLJCg>y zT5V8Hj8TLlhSe3ZY1I}WIUAmc$H8tSa_XARroD7Rz&3uh_5@UN`7SlmcA$z>Dab44 zW~&!sw~N7VryFps^zn=XX@b38fMC=kE=tlOuFZxO@_T~jrWVyP3ZP{7Vudc*F7feE zBh>aBY|Wj#zs8AC_}J}fCibQt*HXg@S+L)#I5q!_;{Z-QO%|6%oHf(-8}-;4piQFY zlunB-G|QZ{k57(z+X)927Lu65>e~Cls=6X5qb)@Pnq3qBnvj%k&7HKN+MW@9oHxoD zq1txBywtaT0GP6DZpit8@`G)lggaK%82y)Ok=cCan_D${-5?W;FP|#GCDBNUXR>F^ zuzVmr%geC~R!1kZdjgU?FDsL)_%{ z7gNF!`64nPm-#jUM#ZmK{L^rY__E+I+#d3~Kcm|mzF;;O*(XjTgKnNsv6-mDzFtzbYPj2L8PHyPL2Vt??HJfv)xb7+YX^8l4i~^W0$bZ7 zn=XFDuIgC)g_wb26v>E39v4;%pGJE_K!0ec%CW ziVoYbP?D?|D*33k2AsMdsf*FoRqZ({_9U4xCQE&b&1z*8(eZ5AjydWW zN*V?6Nz{MTe~W4r4OI=Q%A;BpA6L6Znj7hbAT?Ba+YZWMT5fi`6Fq#7Me$^WB)Ev)G_>?07kU<7%S#~s2E9OM6`-c#v7{{JP&im8qpc?;A4*Qhc)gNVu4 z;0rHEf_IA}H>iYJbK}IKHML^hpkgx7_|9IcNFH^#xsy0}LXnhP{&yv1G5Quoe7+UC zDv}fLntm>L;UK!I&IeA*7p%J66AD**wL|*Iw~J*B0s_8lE_{Iq1gl!CzW>8%d`h^v zHuR0t{8YB@;0T!>-+V=NOZy2)R-odIbT2$cc<{pF5N)(t_0HidpCj4DLCe#CqwjaL z%ej5A>zv%4UeX$4qd%L&^txa1*rO18Z~i7PGmo%+dj27dcnSCN`OT`frGD6LsqI5C zN2-QhUF7)Ti%_51zk}wE&}V83w4&l!3Inax373t!TuXO0$NN|KK+ZpM^o{}(bGGa5 zC@mv4(OYLZvVw72800u<<(r8`43f&Y)@uS|eCqbHf2&9Mwmr61q4O_AIciw-qgcRZ zfCFD}!*!RaxQBPaYwBN+J))+va;*w%%t@iD{b;A)d{ISw`+o8CGb7&1cge`{(?X=^ z=Mm%6fCtSFhW$E!hH|FTcNz z8)Vh&9B#nNTi^-~@xD}mB@bjL4j{lkhc4 z(wj-vZy}!TJWo813Q!|RfSH1Pog9p{ z(wTmIZ%262qY&<#Q3}VJW#_?%uF%8jV0P=lgs}7FQ1c!s6W2Vx_h&{c8$o_Jp;z9cU!ZAG6Wd%9N0c8?XZs*~Sx zb12LEXBsl~Y}yHtqcQ~}qxLLynA62^2wv%myFl@48~Fy2R54zt0kfuuL?Exm7#I9q zlb_Mu^L6B`Np`65-UHa}Ikl{_0nD`^-bvDh#$kB{TPTXf|4MK{Q2uE%X5i)L+sYhA6pF^6{k~DcMkDa*5$X_hz#@~bd?ypwKiZWw5;BCy-#|yIQ2mEOJ zq1G%X-&dcllJEQTb{RP|!*>)KYohg2Z=O@-Qu|8lBZQA#=LP-UbrC}3c|UF+;mc^C zBMHeCQ`tXYsI9X`g372KCc?8~xJjrPzrBTLo%ijh#PjC!#w0>)=_-VL!b=4W(=4Y4eTYME1L z>mEVEZ~6$KJAgFi!#6D+zL$if4gQap?= z6Pp10N`k+hMpVm&aSw@S3n%leIpVb2BccBQ(#dsG(<^FVZWgP?=+pS#4M_T)mJQF* zSfAh)L5OTCIx4Ei@oXa(C%eG97qBfTJ-&dWlf?TUB@%MB@s2p~iR@J(P6Xi$AR!6Wh?ErU2Iumm=}YWU@hrJ7Qit^MVOrTPdF zJNn#%%qFb*zKXtS=>ktO57wUGJ@LX)^-D3BwJjD`8Z($-+F@sk4(%2DD@w5`K&H^R z{>Y)1z|lxp@UkNN-xqD6y6Hk zkt7_sAbn0`&;}l%G@QJCeMpaM+8Rn{R$UR711_h^vOy>n^C)wS5#!$q$W}&Bd-J~t zx98v1C(TVXK=_XZ=CHbuW}ZM6k8HM~0w6y=qLCnyaPJUSoe*v42TY;yE4OG8=EkRX zeGgz)S>ARmI;C7LDv^zF18G&rsNA(I11yQ55qD(el^4DAe4`_OSVEt$QEr_Lbybh@f_OLx3LQqVCA%C-?@kx)+Cv2UTKRS)^sEL z&l~^aZmH0kJIT+XEuAvAj1?hz;JLibQ%5+gy3>oS;?b2RU|Lz8RpCiHpgV0KK$a4F zVzbFIpLbFPSj>j-rmYAC-&;i+Mm6pfMUQB9CUtojbi!|pI++QWZ_;*B`4*$?BGs+_xD zc2dfVw@I!JkkH%B&qT8?Rf+XPs1cWt(09+aOD)Ikrw0Vs1-NzqzMSapLyFXZ7%y?Q zEoF6|$Y_0#OO<&@{N`nuN_X6!)Prkg_NpbMN{%T{v?SN>@{#*J-4S6{QjHNMbJshk zX$to(urLaXe#QPS$h$%ZR;>Bw=Am>iR+ca+-HnTsKGM6}fjjz>JY^fvu&U|}%SxgI zUi>2wwgFM^&jzq!MUwDF1B4dAuLJo51~@CH)_UNIkU0Yssvu9@CPi%@-xa_X;L6qy zvGKmuH?5w`8MIc9w4fD>{Z9IB&}-8SCZ}~qr)T|*tigS>$Gn74RKF=D!G;%~lAO9D2Vd+#ekCV?fJgXWKW8oisv$R zKH$t<6E`L?uLj&qtAVrrt}Okcvd8x-K(;*?8{6!P1g0Lf*(#uovv9Kc`b}F3cjzsK zpVRWmleghQpx#WH0YIo&F^Yp2A)E|OEBisIbu+QSiVd4K6g`(c!_LDN{?E(d7Dl46 zlLfztJ@mSmG=E=L`87?KZ2ROHMBQ_=i-~t#_HMJ(b^9F6^@XWvSh8)vR;Hdsx@hbV zp{0mADN_3~gDo{l6W{8h-G{!Sv8#V=Tc>X%+oHEMn zwEpRu|M^09t&~(4mMs*(r%;4%c=#e>?JaH2!D%x@{yDXBR}OH zlg(xRM081B1cIZ3!c2S~ z{)@N$)Sqh4A?=%ef0xYz?{qta)sw-lLEATZ{y^I3VHQ2};HjP)D&n8K--_z5)4+~{ z-Mv1_^8`iQFQa~@xj%lm*~{H>HwCkQ6!BWQgz-D!F>BG$_(;O=7C`~GdV^^hwMB!q z8G~Z+XjbJ+W?v&FtMYedAG#X39f}k*ju*1n7fd3Pbi)^4e)LeW5qSe2-M)Br_`cS1 zfVD$yV^-~G4Nm$IH4zd$ICl8%>r3^jJ7EjYZj5M7JEhivW71w~Sld|i%hKOkam5sy z+d`jhIqVCIOB=P~A}S7CXDnX#XqlchgYZ6bi~lH+h}61viOWjWI|6gpGnMKZeeD!m zyT%;~V+{8b@r;P{sAzo}&mQF%4IkJxqc)!~@rU$c3SVAfmypD!c~$Ro1t-38edjAJ z|Hw-^b!g%Uab_{aafC2FO%Vk$r=0@I*SjD}MswqStT=|Em5`vOAsC@eb&DHuB2>m} zTaM9+HHqa=!BKkZ>}p_2T#q>tR?!*M?SJ#ptlJcYqR@#{iG1h4dwk>CTjgI*#M8aS z7I>WRqlUUO_jzqgG3Nb?^uoBMU$kHThP`<9)I^QETRQW|Puj@gICpV-=2J={QaZa` zvKVAK#yD0Fp+XQN%Sz~$iW;h3-pyRuw<~{nGtb|@kiuc%?%BOr4vAC~rQd|$OxVC? zX7l-v14oa?B!`O5uySl(v{m0iz!)EaaZ_j6Ncd zkSRz!-9+h}nlv#FMZq}Lk)~f!Sd5VgRKsl^q_mhgoOsJSe&2+)^^?sL)T#{+iIlW zq!%2X3mT=-rYQBcr%Q9QPh6qiGZC)bos@d~9lcI(`yc?-`rhs#Ku{!(*F)=~C5!S)t=~K{H z^#xlea8jj6$SIWqC)n8DggfNvnW9{4g;MDbzOIm!eJo;}0GXH|!>6_RIaRd_^n`tsr>Cb*%p-MR`0g zjgyY(Cmg<2VW=Vp;X1DKdav3iMzbbRHuy1R9c{tLctugkTmNknM3d)gmOtHDd-+LJQJEKHUT zZkO?kP3m@#qIh4hS|W;|R1|DRpozP=l^*xpW31s`mHP-ZPVO5$lvUirEnm~ z*FDxJ;9AoJJ(8!9kq$!1t|9I4 z5BkbQ69QN-RO*00jjI+Kg3DVp)yin}8)pE0h)ywr|afb%isP153YpmcnG zvqqX?6gN2PZQV3e9xrE@v-UgojsOK-JK*w@0Rn|HQrLp$k|;x9`sFd?Fla*?(qUO(g?G5-QFRR)|U1UsIDRIxwR^;)M1qKhu)ChWh#qOXLy?x`g`#dvdN$s1k<;pRO-r!bVGL7_Mn^hG)HQSu zlck-9In!3-yiFI?tK{i=?DLFSetKzE1szUKY#p|?=pRW1#}rp7vp)}HYHUYg*xANR z-Gu_1$eo>|`#x(>@zQuV_t^#6)Sw<1cE0+Dm(RU6lC+qcF~0Qo(2XOd=eMy)k~AUe z+o-m$IEF8tYwjpmaI5@qVZN|Xw}b7=rTiUq%uv?#LEn%>!M&j0Qr>^wo7kei)$H~% zOM}Up`;S4X`ohVnlJ|!x0D#JjB-k&5xx8H|NA{aDnB+_QhMMyQ3>r+&iZv6OTEUt4 zUa!@ozd7-qIG7Y7bGyKRXgDj*xt0=yQ+M%JiAyFlZ5#2?%6o#;j6A%74P=>t+M=9} zkvqGmgU)NwDcb19`JTdr%N4z}_Qj4)#UUe;GPhu3i5gz}G{BRBw8K`I6r*om5Z(AA zs(tDgJSfh5WLAtm$n_3<&X2BdB~CCIgV|M5e@M4|LeF>nwN#$Yk?c^qjdMdt3?TL* z@!l@)x++iu|^~0bX zm*QK*&MPpefK`LT=soC@kD}QZ0u6{p8pgF(v_A0nTD;^X_GlLakU;+-Jz2Utbz#39 zu2Pm_oG zxeWYt43NcW3$XFo)&UnG0#nJHP9zI?hWkrczrzYQh&Oktk-m5KhAbnH!B=a-lyKOC zlI1`Lg zGfI%JcRS%^#69;pYj-DCI=utZ%NkW8NAH;of{cb$9p&g^?4!c5Zhpm~oxC#IA= zioMng`(|Z})3QNflRYXT+6&9WNlu!*2o=tu*dgrH51TxYgJ9;^m1GUVDPwGYsYvnPBuQ9|&JF7Z0 z5{{9k-CQ=3YZ@;hO80$X;R)q(U=Me_uMK$Va`kJHPLz%;wgw#XO`tk9~_*EtSObCUG)H~wJ51jYf{9C;#r z57Q*9>!*#!V3o6Tra2U0-^6R^SxQJ&FU?2H^VlPUPV1}Xe!dskEDU&zwLl}`TRc%g zTY&bm-2c?{@(p>~Vj4bf5F3EOg5U6+0Eus#^0gBf^bEq`Euy&5LNsJK+1$AlGx)+Y z`iHC@Fv^^!DRXvc5>=Owrz}E^&Et2z8a%YoVB~7$Ik1igwHIuGV1**QRUHPhk1p{| zpKziry6=?(6~_dz(sd>ZD{uaGR|WbW6ej&&9#{axQ5Sg%}BtlfBX~1B0oBxBr5H)0*S)j!?ffXi6lQiO2*qh)9K{ z#S>(MwGEIl;>(m1f&DFlpyiAo7Q^o`2v#m>lCngqqS&KXzIvGPCXvA{pMZn}TNy(o z(Ov1{sQvKovs_51l#G@#qZMi=t@Ab#&4+LGB<4S+;Sd>Or?yr*gt757V)Qvw_MIgI z&#NSy#Gn$W{nGD6e0t58nTrbF+D$Wn*w!X|e3b=-~9Al|Jz0^gHzWk_g8g`H+ktwC5cbMFl!()UvZp9PY z*t(wfzT(`(QPE zl!>!>xf=}&-ewHqF!YZA4w)cP!$y8|V;JLr(^dyqw8N76d!7ezOCXN7MHHQ|yZ(I- z{TUL%Y$N#E`sKaE#M7mg?~m1mq{P?n7a}&9!qqE6Wzxm!i}`!82Pp|Pwho}a&xx~i zWxzos#cO$M(0uyOXX@>Iwk*@O(C2~+!!o#f6%l6B8F4LLScEZYCnqCoq!b40+@S6a z9|;~Cft=y!rz{ovZ_#JWin~U1KWP30KbzPuiT&(yMy&v^_(f85ll|K>;!*QfP43Jq z*H6$`1#Tn6WwXUTw{3w$CTDmEPlyWhFeKm;neKxZ7zZ0lGmd0d)X>~`K}yCieeKt^ zmBOpd*i;k+E#zbL7Yw%=NKAdgUCeK{d->a`x*Iz?ew`S+zpx@Lo?;6<&9teeXPubbK@>dS`4=e zHF?pe>37z-SDkVaD*nEfE{%JE|4BfVjT-c_-pD+ln;g`d4okaf z*d0O-n(b2it_G=D+g(Sr=^urU;|AHNiof87iD)M#5Of;QC?3e<=D@3#RcMBu^^r*b zyTS1)l9f1)3mfQ;T6ifjQP@y z`Z~L%j#HY$Zf*ai8g<(JoYzj;7Q*Lr{tnt;Z$p5gK#m_t5Bb*dYA-2%3GRyHp28TM zVt(kak`L>i{#ii$GF)NJV;(MKzISP0i0=OP;zuSbVTXDD5=)}TIW+cy8TMQ$z2IcC zfx2Nj7bYjSv*fFjg$>`NQvl}6&SK@I|H^I<$WNs{8DOINQK|7lH_i)UzDD!rd}j74t+#u3}wRrj?O zyA>XPj^l?^Ve*DvnOheP?;kj#&r$ErGFOO+(=qVF7s%{5`iF?MqMwen?Zp&{*|w*u z{hsOhFzga#)u?}f8Z&JFy9bmCBa@*b{)fNGVE`tj)3%RDUtGZ~-I3kti%Riz^nLb1 z->6$E>vnCk;Me%TolVSU6+Ns7?cn=Nc)B>+H_q45e}=VX?s_+|RBdbd1+_wsDoU{R z()Tt!j5gZ{1#6XS|D4WByg9r#wwpFC^9kL`){h++)@($#Dlv|R!pwbg1Od;F`@FRU zIER>S;YO}t>$X~(HcRNx-hDH#%4b#J)9FuVL!?E`wT)jsrD(%*EZR==!^Dw;3v0XU z{j0o0=?i6W?u-&l|c476}(h9+lJa)nGFaM2$`=`@A;fFucUz%^PL#9P_Ima$8 zyX+GA(KVbMEYziS>CbofExY%ch=%}Nu(xQmHTgaeUAq`nIwk=*c5uqeCOS`_3qse7 zF57IG&YU>d&nKO=dJlXFLf4L7?~UDI;2oNlrlw;ty7%!`utjLR=fYe=p_|7ES$z=; z(zkqukWwuS!A(-l3(AMKdHzf(g2Iyp^e%{C9n2 zT4ktw+qXB4AWo4@jH@{5?@VrcXklZejT1d=XnuE+wl<{VVQuI2_YaiDoe^u?YO9Gq zJ`7=Z3t&G+=!Rw$mN{&1^0GgYVs35K@pE$hFf_NgL|R=>rIVJw`0Y2n1xDh>Lvg?J zCzIl)s3Aw%YFHMmr1&*mAiXkt8*(qv2%NG4?d`;JGjob z7FGuwH-H{7wzXaR(&xm@J?~1*93>5=0m^@zycSm6U}j*+f)>Kfxh>|c$7-%_AkL=; zr!R&JDOuy)&EBG?mb_>BZ-pf=k4($u*+kvQk~V#do;XV-g(U7&j?;PdQ-C%I-AYzm z1n^Dci4@hcG?*(cE~J^q>obep+M|j!blrdn!8G(*_&I9Y9Jrnm^^|0$x}6i0_QmJ$ znKKWWp1gD86+$o%h0yeO+fgl|22DekYxeQhjP%#^n0p+qr+0+ z9Kfub9qEJpSv%(9DSiFjk7%5hZ(&9YrN7KxG?PcB>9?W0-KH2RD2EWM$@U|+=5)KV z^S#}gT{Tk!rqOm!R#Xa(QTVt|}AFpEMSYz^NGjH8K z;aBbcGD`Oe5@NBtC?Y;UXjnmESNtsBzPg3EH*3Err-=4cFn#vuq$7dS>21pWhRb~$ zRWBseqT{AR*qB;nvR!$5xoP@!MN{6Isimfo&Z0wq-kNU|QkQEJ>Dq25GLC5HOd;v? z*`G_rPj&(x%asDxp4=@$f6bb2a&eAZV(SHlZs(Xkb<6OSKN5FDiHbP8x(+Txr?H@X z?h0+u$2$~uyoYI`o%77SAxk54&!re#o%n{|)!FT`5{$*_J2^otI^-zQWisMxubxZ0 znMx;UT8;9M`1ujnOf8@Lq}%Y?J3%Cd=JQ#gtGNG3YGPBgMqz8aIyZS;DpCV3$3Bnt zTkZnEj#58)FZw${$-BP0Ep&g9NT>{32(uh{u{Ganm1)VJra1WCJ-jy_Ck3_Pv;NWbnM}m^ql8SM zM{&(+D1l6IE-Y{R*t+(RrC3N(I@c_IX3wp-X7UgJjUNpYv)`>PVQMk}7h->64^0IO zA(kF(W)9@fDfh9gjc~U&H9w?2uy6M+G8aaETdkCA_`}~OZAA6~i zgKA#td4eHR)_{lQv0Z26$YZ-{VjW(OJ5;h{fb*#>qIdDtiXfDul@6W|5FFWtN)-BWCeunc^yJn*E4!I1!c z07rFz1e2e1nZ5wH0})@GDE5dX*?&7n{jHk_X>J67A8gBnj2jABU}pf=^B^5bD>!M7 zPM3k9U~<9Q#K3C>j6EGi+~$5dEI=U=k-+%7M@;G`ezOZ4y3u8=5?baAPi5}mek6$r zGw*ipWKn2+U7V<+8NA6^s}s4pN0Pe;&Cr;@$O!2!9Xmn+vjKxjE&Uu4)ODfE20)Nd z9-M}>&7p@4(fxm``PO!MRE!|vtvq~5DOiExzt7dNiR5{n3uB1%bjhct-P(nup9-AH$bgn;zYA@z|)y1SMVq#LA>l#qI6{XW0f z^LqXMx%cduI&)_3+_~qx&uq(Qpta$b;z$w1QCbBh%$T|hWF+j6)gOgpz>!K_{Bu6) z^H+KdnCF-ql{YagfEdJsteAtjSf)LsZW+vx!18pd2N?$arusdV?Jw4h9oP~0sV3lY z#hD>q<#}8-MW?CJU>dqpwBc;Np&O$X5zz~GjtW_4OisC@N@&s|@B+^~e=h2!hSWxG zQBBTqH6J0c@hJn+sG19D+boLppM`F}br3;X{aT9skuzg*e(ZcU1Nu+OVQlQQ#p2k1 z6K3Hkbv6x}q7U^h>NFkFly;wQ0_u?ujDgjIPSAInwvsrUgn9w*j!Km_5pb#Sn~O#} z{41(!a9G@|bc6v6XRZYjX1?yIptnfNeT*5VPvBZ8VU~;ndCILT{N0wD4vg!# z7VgK|u!7+wbz~d!xSYDe+_pp-QJ``Xou&2cg&QLC}zgXwqs(`#$csv)VvSDu)KgXef z6<~8Ms)hl~m=xttbt0#>cu8-XTd2Xr5p1@6RUU=ZQ)d;hYjX<2}zCH1lKOA z5J*+aBW+5lu3hN4Ca$tt1joE7A6kkTy61}g#iFqHp4315LH%K$#z?$-mK`hR74NMI zb7WlLp=9NLTsCurDcARX6l+3Y^%(*{>ai;FB&ky5yluf|Nh{ax*pI7V-u-(__VhB{|fHYP)1?A?@51Li7R0#fW%u-3`cnNoHmDP!W>6p*08H3_f}NI8`~fto(mng~Ij^8vPZh=N zK8$e4J4|u*u>w*p&(j@||G7~`pDcwbs0OJP4nhtZm?e>oO@YY%1_fsz|rj0rYLYbEl*p0 z5Y|e$6T5LL@GE2f89zU&tVqs!F&l+t><0ep!yjE@gOBJ>9%h^5zz^zn{bg^K zdiMYmARl06O~IYKB5#n+?Op(jc;y~hSz^LIZ;EO^ej|@oeSrQ8Ejme@YAIS-&)bY9 zewG#F;P~3#QNYW*g)LptJta^{@#rVY3Jzs&HCT{Do1nLolk4de2 zlxr|hl4?*l0UR*xM9&|my=h+T@NLe1Y*v8ryP6zfSUdFOw`OzKV>4RJ|NRyYSTj^b zi7%?@xCMZZI`IxWcBffZI;}|l1{2^akkK8J->lMC#KvpxLYVT)?&C$fksxZxYeIoD z)+y#%l<55jpk0!W$UqEXyZaTd5koV)I&$sru-v6gAY@fT2LXlsfR-!f5eiq*!C%zF zj{j(!X3Lb^p|t06o18DsTroEfc6?XTOT54`Xk(5kh@gbGos&7Q-G9_^Jm|YY4c^;AAM^2&O1r?y3jTp_7=C1<)z*8b<=2L-wy3${|`1C+yDxY{p=cDA$JMwnp{K* zJjl&&I;zGPE5D5re6U+FLw)rwy9AB)?QcQwf(r;2Z4Iw^7|vwo^&fyy$mnq-s)Yb= z%K|a&w}8}2pJAYrcC&p2E`R{4Ai3ua2Rk0rL**i7@1sQWPhzwA{$lCadY_+kgM2Gi zu-_V#RnIG)zX}JwEdQU+o-0s|pp|U?H6Wyl#CszHkS~usoPIPxzx)(i`V_2?JMx>t zDF3I9pJ+%+Hv23~2Rb#3==DaYqH=5r+8L`l5U&}_OB__$-&8?N02~wU;Swf1?H<6X z|3>3cI&q1dI!Ih;KC&jF{`PPLYd)O*^N!gG)A6AsFigW9{{QYBBKsJGL%FD@-0vz8bolNMKLKaQI^KD3 zgsBR^Z#9T_L@QS!w(xjo$`})`)H_A?(ZTx8XheP<6k!UMqNmnM?qNutqI=kj_J23z zFILQ@oDZ{@hq2??|86gHKp=t4{xGO)KThn+u*))UlQ;;q+C;&luc}TC1PBW4Q`yL2 zmlj6#&Eiz_c)T=f!9Ce@Sm%3Xq%+wYjjgG0dX zA$8B5WIRt6?0Ar`dA5cl$`5w#lG~6p<=Omg5OpOYSoKjjOtxDm>*Z`e_<*XMGMDBh z3ibl7J({8_G4Nkn37<1a0H(#q)x;Pp8tJkXr<*|?6neC|LMRP_)Bd(9oQISv8MVa_pV6A<2G_HLR(;CKee@F=kcK7T zR>E{R$9=t`3OL-iI9){)>i1P`z^lQ4n;rPns8e3p0E~-wgp7?L z9!Fis>r*cx_S5ek#yGx|C}zW3P~IO`00!zJvE2U|3Hh1zeB}c`ZTf{v!wm>p!_5X1 zg-%xnlc^VYe9@oM-YmGdSNo|^{eJB>{tBQc6k+>a3gA~i`w=hLeFb+gK)+rShPU+& zj>?Q7)NY@!zJddEg?sw91@Nj7PFUh698b&E#`^NW?H8YTHPPC1*l%cB4X%gdo+Cc* zsdm}zQ64d`H5oVS+j@nW5|8FOhb^$X)x+5&l5NFIxeoI3k>OR-WHoRt$e#{gLJ{pQHWdbp#XW$vC{SNxa(LQrd}R9c#jZ(FbN^= zPpNx&dcualBMAJI6~Ox(>l#q4+gVuDAi}@0Yy8hqeGeI7#q|G<1!UKva|U0lU-h>U zhm@hHP#!*d@M8tIU}c&L`rCl-AJ7yhhd@)oBgGBkfy#|o87|USAu%IB39;aq3kmaN zRFs-2$R}~+vX5EttFZz&n{kTk#H+x@!WDwKPYibY@x8@$Dk#?>U?>IE!BGB>3~XVw z`M;X3d{4}M%Crv)e0RcX;VOSA32OKM+yuT_0`3EWS}JCdUvW3Y+>v;{A8!YmAqOt? z>XwK0p^-DPzK~Q>f}1y}pnh7DMr0``P|Bo&8H`lB0N6J_EO0~a^|um+b5W~1^k=~; zM7i|0VxZ-r{s1#01+IC}1BY?Kp+FQaY5ihmMlI#apM(oUV~GQ~C^YY=v7p@^`r*us zD~)*C%IqU*6p>JjJBRP=gKE!D8%Ewe`^E5CA8)3Nn}{arB%zlTGfMM$B;H#OIpbv1 zDW=a-n0Z3^mGUU3gnhZeDw%6DGy!W$PL=y(iM@frBs7gbW@uK`?l>=u5(>FDxuzoN zDjR=jB_JyY(>nqz15qageXIc1Vyn#FulC}9KI#q%B@bq~XB!N$`gqp$lHmZ)VfR?7 zN?w)nYC_k0R1)}t31sfhh&G>?R#9{;-#a8}a;9$JSkuJ};Xtb7e^6d^%?(sbNCp6( zu@8jPKngDORfGUI)E6|t8xt1tga&w!FB1E6zn(Je+9inPY@$i!A)zPT?#MDL3{Vr5 zz-RXKpn3j6F(R5z$Dt^*|N4gX5 zRmK=8Oo`?kU@0+7#{)k2Zc;gr4_gy-19uG@*>q3V5>+oLXhd~;A#0Ngfg}S3HgbTU zUs(?Vy42H1do{EWqHIt9MGA8>HVki;Cg5JqM5XF7p>08*`M_Y4Dl$^`j+5BZHOc|p z!ee14$qh2@rg5UfQq6n>{kf>eOUe{Y)JF!$RuBKRdJj5`WK#hp-&nG)wd9BqQ48S< zAQh7ih8^$x1F}OSK9p4yMv;F5flP9K0aX;@&3xcg=LBS*LGD&V?O4O<4OW=t4DXTjTkrozBX&QtIS|oQ++$@xFkiKTivCB-7P|N~V(H!mRtr2kzJ-`K zXCtWn9t6v-ED-a1(Z!h?Uw}t{iKKVn;A9;uqeZF1ho@jnpu#PVES!~c+@)Vr_2VPF zG6wm$3gH~~;XmH9Ubr39 zh6Tmwh`?Y1&OiRz35M)xo}J8c%I3m`LD2NqkZ)0r9$TnZBzZDy#I;n~Z~ZI{`f3hr|L z@6<+9uIgTYah;;JDwD70vE3!%x_`OZmG0TS|L!r_Ns4FZ{-X3W+Uc2su+iq?#1`u2 zWlVpS+uO&LfODzdzsOc7Si)ME${v|ocQk@EkkNCDN>j+X1h

4zPI0NVbT+WIR*r+iNNJoSbW2?0w*C3S<8|Jln zGTIW9=3D00tEtzM04IkJ{xz8p{RHRMNArgZ%}oaLN>T<>DDIZPom zpkxC4Cs4j@4h4?IdB}@@%OxP>CG8eyw|kgUze5ys=8E3~qez08VdZPHmWkYzVVd0j z*O=iI5=>dN7KOAgAt=2ZNBPh5072#G4J-NY-LHd&ZwMi0AKSfwkpDt{o&WNMq!Cp) zL)UFV$KMA=QFpmCT6BkWGN!YlHzV4yE_1`5O=ryiu|< z_=o&AoJEMt97Zb){0E)Wd(?5x_H6eIHXTP9Y__>q1@b^95sDanh*pOgv%3&Po7r$a zs>uQbW)gBI#0`KjDm1=R6~K|+lP|j=$<`TMgQxN<*AiX%`F*`aXAm@G zsq`85;~K-A1Q&d-UH_W5#g{n>c^gum5a-IuBwvn705ACmZ#=r)s}3{aQo}QMHJ<+% zpU?CTz*@)59s zn3>u$O%9=9YK#ttv8{gzc3dL5NUAJln!HY}BKtM<#Ac{xwa_Bl69_y+X$5f;MQX9e zUcuGIrum1mZu@fae4QHL)!h`@ROEB(mkAsyO@h{$b~cEHTqnP$Fh+m}N_eWOdSe{* zcUKjn^+mK#Ltyc%@bPbzsPbQH4w<@;zsI{N{grs9J4UEEhOhp;s>4O|DKsz+dI1w1%$8dF>^xJfdjG8)WJ2lE7lj@KJ4|TO4NA>fH zp&Mms^$8XKl>UFNIUwcj&*(+TWI5NVK$!=IhK%=<%A0gkajB|<@*Wv+qyyA>KOzV4 z5baTx(^F5rl~gtVZD%E&JZ}Ri>Cqk~5apE(?aj&fWoxe-aMhiR8JK2&CLwyIX2k`^ zUy9MwW@ps*zwt=6nYad#6_N^pgrKTU9s{}sn<}h*5V~b!NTA@=f+5K&ZYgjDlLsQ7 zMtb6PxCLbwumsBIrXue8V4^y?Qgzt9cAKJoow!;LXxI^03#+V}r30N5%%ReegxxCy zE-gYK3zZ$tdI{0rcZ_J0;;R3po;u*vr4A^uiPtw5nM=dQD#_(A)fQ>cjgfApuybfP zgTgM!WovlKYBR$oCGsrP{Q9uY_jD;uMTJm9gHZ*W8^B9!r6dtZk0P|yMS%l~PS%gO zBqJ1NBP!o_Oi*&5)wbCtLS>D1DUjx%ood}g3SG)pv>lel7sZz?2Ok}ox z4TpA4JE-i$d123ubb>uM+=9x^u;`L-8wu!?ys{bmXG23+AgKKBQKKz^eYKgb;|5X5 zEP=Izeaeq?34sok;e<4(*5DS{QzQj@LF9X9D^N@Twy(7MsC-_r@?lDLA`Be{P~U9L zb!rk;R6e7}TP=`zzo4yl6l?|I79rbVjp^()0_`r5ib=lQ$}%jxD5I zNh+kIdJ};cU*Di!Wn|RYz_lp0D+Sm! z#EXgu(~|66{&Dt73!AAl#~7rFVaB#gTBVUM-0q`Hk61U zpP=-Pisvs{>VemEq2Aua?*>oUQ1zULk_lu+jCjdGv`0ZEdmid^QVLq@F%%BX4b zGaAs=fSn`Sz@h}G4c(Dv4|7%rIoQAwk_iqIzY21MLKHc{t40`64ehlHA#KAg4cLVO zZQ9tNo(f_m6??WGCldogUjvM$aSR6zXm~-f=Y~kSqBGv)5jKN(lY$Do7tpuC)do<~ zaQf~k)ThM4Sxj8W%f*@>@qs=R1I@p&( zMF_aSn?op=sdr@bp+-#Oe^rEP{B`g8tDYjE--64cXB8~9=b+=lRuy&#<=(&SPX4kF zL|u*LePA_ec70FkncXHmM0K&7rti9-P7UsL9rSaRL~X8;Q<7-Uy%^L~L^8&l4j~}y zvHcrfFoQszVG#u?scAGHY=>yJbCfz6bx9ASCPEK=4{xg74&%0KL#U_?-Oh%ZZe`~P zS9DR(L}D7i?XbM0-%+z>X2+01672idRu*jkh#GtnP{n#=syDQwQjXI9#@>RVK;Hm+rkdp>I9Oi$r19+Un5;KcRd)K)Q1@ISuk=NUiY+E{djnE7Xf&;DS~Lhqk#OKVtXNU`dF0%U@Y^;` zhv9#Gk=x%t_lUPSM53RYw}`<7e?95aO)WtweERsZb4F#c3#6yeH&%N9!5?ET4x{qw4Jc+nooiJ#eNe@v!mNL_@HDP&vT_Ig@DyNsi(*Vl^g zvcD!JW03Mr_w1|pb*#_U3);QE4Ni{=Z{qSl>=W;_tG45DM_8JF%I;lRjEZf%_JT~Z zbh_M*vz8)EvJ5hD@Z!XK>bSKGaw+w4bXnzf4ZCXJFWi5Z=$o?lq|>00g7bz;^@7bw z`)78siJNY3578DA3@s+#a6O!{=)i^>@Tl)ry5J5QwZUPn$~_zTLl?x7jCMgnez)$^ z)(BxXCz&%Yo|#=A)-sW7bY+ms59io&RIU&oAB1yEj@lplquF9k{gYRGgGREMjiN|- zS-363zASd8kbD;gxyo=3$xn4&uez1J4vM6945>SEFx6btPqVX6+c>MhYah!b+st}r zy-=fJ*%f%m$wcD%Fic&}4lmwYQ|Uy_FcE{?AN6tziQyEsStE@%j6OUl-})+A;l;=D zG!IH+54tj^v!fm(2{cb<$Xb8;*b_jS!$7c%uq(DkkCXqI{3Ui1F2eagI{ zX^vU0?_I}E={)VI+SU2B+~VK?tBW$PLKR+Ur8;*(hgZ^wuM((f_Amuy5CFEUF%`2~}@WN0}(stNpMSMES z%J58A{s8E~N=VJi8( z;e`i~)*`!8oir7`2sdpKAR~#gdqof5rr_A!ON%spp_Arhmu_|djJoA0OjN^r`MaiV z=IHiE#b~een53``ikn}*NI&Jxtpp>Gp_y< zDOda#2jePdpIa|Z3=9M;P0wFZrM8`4w`65{RIjZud(p~Gcq6TK>rl6k`0M*J9#j>r zzv3!%T%1ZgwNC|T?Z>wpSnIf z&RE@bedNX1ML8Zew=2nnv(V#S6LTl#vuG&qSN7YTbLBx=i6i zd#ge^$<_}2+M3DD^rBUCQlDaC`K!&2hwM9jT`2rGS4SRG#T%h;&pNO0Zm&j=8K|P8 zVP!zCQRH5X9?5l6U^q6~H=5@cBRY{$o!u@J-k>~c{Te?QPIdKE45l2YT6g)yrC5He z{wk9v?H{)EZ&tn%Wu{7@LM|Eo?m><9A?AL^Z{4}FG&NgYeH7+5i`X_#D9vwnv`y9D zUi$uHYiejIr{DBZ3#rySTNHbaHc_5+t!+(ZitP-~l@%C1vA*I-^yp0u-x$vB+|X$* z?!K_=Sr@k-Iej4F45J529Y`*+M&!y0eHp)-eC##Wz~o6^XJqi%xuh$4JjR+sb}1>d-2tdB)B3hx#KeTkl(9nTg9=>Tk3U` zccL}&Uz+BUuf8HQwppas?;tY7qBa;ph@*{pD2CH^tATF}mD?4;vCWR5mMY&iI|2-l z$^m$Jv0Ltd4UUa~4QdT_gSduc(t=K9cXy_8e!LgvbI5TTEw^@)WI7u>>Hc_omW#VT zmhALR($~j%%lyY+wp23fGT{zxV0@?#SO70peA-95WUENl4%f{c+^SRiHJ%TGSJ}eq zO1ASXawW>Jp8>Br+A6W0tpAn(mPF<8?|6W}#D=+ig#tv@oe)>3825gcst2mh(?@lyJEB$QH~Uk|pVjrH+z87k@d`t(R0r=mhN|wyYrT(LIiJJ`3zNwk z?MP*s`~s>l7TY5n&u<(nm6f0WwjVT|Cg51n#QIQ8*%#zoj)B|YTzQF9P+FIS1vyaq zGT^(gIR1538!EiefO)lWlwEAxQ1K+p+ooc>$iS1@ zlP?CCu5foHH$G3N9^KRHlQ_soUstxZl)n4kC!fT|@WO2)dvkWS`J-F52Sz*l4)(>? zRQY%*d}Lhotwc9{COZ2RLn{ZX`{a7Jz5?k(3YqJ5a}KqMvx}sptvyue^jkM;$sq>c zI0lM!HB!+srA@5S8%K0;lZ%&Lu6Kuhb)RBC`4fxuOHCo<>)JNmZ&3ct zOLYsKiyQ;j7Pao~3 z%~LHjy_N)YrG%VMgfE;cbLJuLi=#`Cdz#c2krNUY|BREX`esvGI{ z1s)r6w?5w}2s)PCW)?|d>O4S|r~{S;3sz6=1ichO#dM`mmaLwq53-GeW4Gl+SV7+I zOkFA2=3U~6jkFiV9Ncl*w6IHOzo9E7|3w4cHhi~3x>bguWSy5jEKTx-Ow$O5mqE(! zjlxe{Jdzq5`pHGUx<`yT?oG9Dw8?7yz0skc7~KK9FrO~1-Q-njo7yGorD~hHApdxV?Sd9(&>Sp8Exjlx+r&Tf(^oFDmPKga=OpMuCmOYlHY3OAe+>^bV+V=5&- z>fURLH2gq5!3h@q%Gl|^3|6KW8FwFP{Lp#6aU7Az;lMXUn<+y~uYRCcw8GpNy4AZ2 z-IB=@OCDs@y&#s{7#{Tj6D>|F)#7Ph1hYu1tn;2HmMK z%{e4fB$GVI@sU6JgB*e?-0>=u(9N-}NN%KNS?gUI>*hFA-r$hssv_kk*($qMx9@{o zVUK4u=uT#J3@n3CdZ={D-COdR0~5mMclQIIcz{o~KinT59HfzNarSZEZa5y7v|eEq z$6;*Dz1WNcU|refC@zbIN1hQibL3!(WPK97`*{|_;0_ucJ(!>H24XCRSI^aNH? zXJhmSMe;*5Fn!fHfU_xE=87#CtPH7`x<+~{JbsdfSbrP%0uM&rYruwdl`K!(Hx}}T z7S{q`X9rvWrmY{wZ#{EJ5dJDzeQ|@?ju(PuH_r)wr3HjaUY&{$5;WT}BG9|5Ahfe4 zG$L3X~mCJ4iq@uDDCuL-LIt?2Xo!Gj!CuJu(G8Jtr(0jX+ zJktKs{jSvNj-y}0iw|rkS(dU9+m_8{wCEK#x7|cZa<|H~IeJ^dQQIOVd z-6Ie7g^NbD7GyLIbBnYyz^k059kBn1ktmd@xoBxA>LNp9SE|F)cq&@{Xo21Z+gWDI z9zgPjRGGQx)V0XOY2v`NXOgvsXkk-tlN+yDuo=-tF?Jf>wlfVKy{x^qrT~KK9S$}D zXH`R4$sG!MnUS_;nKm}IGjy6ZZ8hWE(E!-BHG-kr19 zdNkla$q)(9^W=jlv_4^LHMJpcl+VdmYoeaugp07Pj|zGc9H`4W5a$lRhI!FW8WXV8 zErtyC(ZI2Tj&&8MFa2rCpB2#g8p3P7#U0SsjC+UIfg8`>GFJO+Fnaw&u(|(M!Isdx z1$_q>%5H^(4Mz)w2u^b#O%wsUROkB|y$)lX47WGqZd70qZhXrKX*T(R0K{0c*H7e{ zqs}?FgxgA^TjY0Q4?*CM-U#uPWF2A-fY($?_E@0=yhb9uHTDB@@AC^Aj_#!G+{e@C zKyG!6r*U&rFyw%T$y3c#wb=J4; zy7&7t&vbWnb>*t==?CPq(y@%^7YxGRhgqew;CBuu`|7M09lk9w_olScrK3yxbDFU-=hHJrt3>PQ}=|%*@c~L-@S(R%}3aECwFbh}- z)C7$(94UP+z#^&%fJalvZU^>QX+gLJt<%}Xfe8ds6V#C)Gn+2Lq63q+5zVzh$WR0_ z!DKlECPBidb$Ij;f8b?66PWBjiFF>SMs~18Gfy5${Y2sFyVabayQ5_wGYFENHN6VV zDnr5r0wL~#1qyX~DubZGI*>RkXR1!nk*jCE;)?*P7h1GqK4`iY*IX&$=^5IXhaLTQ z+0B;C(N~*;HXDnzn@d+W;V;3jFXX&&^Z(d1Gb%olSA1-P!NvJ`+;EFszF3dEPnbi< z^SBABx&4Cukwz6sgVA^Pv28Gq=J;8+%GP$R`nhdJyQI*hY&aw~9d6thmGbag-r?CO zNkx_9-{Q8oIBUKfUpq0lJYVZszAQnvELMsx{nKee7xxl+033R=B%eX zGoH6fp1oSc@+3k+O{1K&ja6P4{kwbwc z3Rir+42CuGgc;ie;$Bdl&;pk`a@_UCEJNei}jLrc3FJ^OQ> zx?=V8mDpMoDO#maq)2un>(|5`Z!)^?=X=YEox#_DYC>M1Vw48aKe=AlDivi1Ap+=B zdC1drhTKCVTO`~hUhFVpfb|~gy3EoMB%tiu?MsSiw|qKylBV$ud6g6U9Xlpx=!#~F z&Q#qYNJs%EXKjn)~@+9n0hhT`Mkej{0Tx;0@! zz^v|{NT60>KqHNrEaF7GJsCLpjb7?4>2f?k(xoLz7o!>iTdbLxF1Q~OaxfY%&)zlo z9?sbY5S^YiK&O@g32I~(a4OAfra%@35ujkvWE69EiMAt1AA*|>mtS;XZxD=ZFUhNG z$WyM?Wv0p&X~{Z99CSgRdtHsTXfPeBNc4$so3$1cZq`P| z1pY_hHHF&j3P>NHG;5t@1ofCpFt7StQN~=4q3H^9HQ0esOeuXWW)CVwJTF9A!tjk?28WyDDuR7NgDz$d^dNF9CW}xB zUT^|g!Tv0RwjB0n9YF_E`V&CP(FQgu$b9c=S|@x6!U6{H?iJLom_TL_#nxEE72c?fWky>7pD}2GQD0RVe&>Rn>Wza9XrGv3~BK!8U*&=H9?U2UVA#-r~ zp(;Xk$00sGevloQ!gUab82>nGyF1LpdtD{n(nPEod%J~YHj9*5<=9a$Od+_w%8R=;#pTU5Do*#Rx38O?i9#Q z-8lYy)W|VT<$(PM+Jp3=hDf6=C#xMcF=q{ioi6~_#2nqCI+1y6n5y9-3zZ2Wy3AM1UjNKqnb@?fJ^YU&-$6N`_-MrcG3Z7jf^^}8S& zp!)28M%@1cv(-`P^n?He1@!z~mpcxU0wnJzNS6i|h`5+#%*GbIF(8dC`-q=Rdm$CwOIcV3;2k{m_ zd7!vKTa2bykO<>ep$?l1p{t%5uaF(cD4+*p!1{pZ^i=)_sC^CWDZXa^JXvZW>xMo1 zF4$I}lO@%|aBxGcB$Z>aT?HRM^i0zHI#a^VLMGj|Ho`xSZ2s4xcd`9p{fWl?S)V7K z)@&1QPJ^>7s1iC9p=3OcOw_a*)n(B|{<}(a45=66s_fa6wBh&*s%jJo(n3)rX!wLh zg7%bphy)gGC=%oeo`;;E$%le051imMG(g!26)!GN5GvkcsKz_x$FoIMzyP$E(9RN< zmxXr1F!1P$c$7sn_h(9}v24*O2(qE`*1NC|II5-^~{z4Cm zu{^lw-6hpoh}2-CqKWL>Z-*?K3u3N$FOSM@&ed+N&7~&!eCw87|K8#=A6K@vFt6vc z_R(hJYm%(&=4$OI0F($6IIW24TU=w1l;}-Ke|jrRVnx*b1b5@>OL`-j$+r&O&6oCM zdJE&XOG<7_uI?PHMdvnOT4Yd>-*(XQp|1?yo88JDM#d&5PhR=5H zg&w-{CzP<)83;RVazrqAmh{pFFSvuW zV5iAvQAu8tFYmjv_Ki^}%+INqzuWLhppwZLf$bbY03q{-0cYhK#h8bA1|%f>d=_U= z@or)j?MoQQ77Z*qwB{BZ-qN0(`$_)7Zru#Nf(ctu&RGlD>I4H{u2n%w|E@&}4^4~C zertuUl8@nbq8zhovXy#^DYNHgYr5k6s8$4p@QpnC5C91T~^2mzv(CO!rXpeU3I&LQe#Q@I>o;RIC z_fybk5c+KN%}{|E>M zmE-u{++Yk_8s8QY1xezjKH3*&kkpXJoxJ|ndm-0UyAuM2Iyx8W^>fJ^Z|G2jFw9WA z1z7mIi+cpFk#GuSkeEg@IZG5M0^AC7iOLIKnl^vxg5~z|Pf_ic1-{rywUc?+`B2sx zX~9$~^oaYra!zyah|42|!7TWF3hMSmME`-+_DaK01o@9lV>mFXl@bw2NeiqpyR>JgB zu2;D&2-gCvveMivXq@Mbo4Pq|?$^Z@sOYlaKVidL5p}*DpGQXLput6C65PVpVFq+q;Y3T>@&tklMZK)ZrBT z?&V!zc7hU*qXuT;UVY?EIEB0=FISZ5gH7%%Yw#d zA!hCmCN;b$#M@9fRH{-@6*m@|P;-Y^5QMh3aA&wFg#;@CcvO+^u9vn*|w*Kg+fFOF?JR^?L ztj6s@c8P^djJ z|2mIybiZtw_Jpl&9W>~Cx>%ZL+E2#+t%iooNKL&9+WVQ)V|&w`#3iMmFGMn8zP#sJ ztx-;4rrGynGx%t#>GqR}>rv>z`bE1-Z|$KA=d}Opys+9C`o&$c!`us3yY{R_n+Y$Y zd4uaVRBWHqO-lRKbEQ3P3);WBq`6ALm&H$s_}v-xf#P)iv{$>RYuP>fjx!A~R2ALK zKGb|g_j3e=UB*{JqU9Zj6CI1LOI%C?ngP5|E$={4cQ^Y`*jaJsQ~1u@5d-}UG1n@`QF2L$Ik978WyOub0IOriyZ~@ zBr{yo&2Pw+Ek7re3%E?$Pe)9XS1m}mCpntInPuM0T+gS}rc%-UAzu=|%xrw^NP=@8 zbgg4!Rc~X3&1YUscH`S99RAd9&Ljlj>bzYvs1*CI>n~lUQq;F-yWY;RcP+zy{u*@7 zUUt!e2NQ3Pi}c|?+KsZNCl2jmrkh}$q}kQ{r94RbupgClUo9)m#o6n9+!?p!`+AXpVMu&D+7Wf85 z&@+xw3=}v8IneKN30~>Wt$H3D;vh27omLeS95QdXA$rVifW6<^p)e*w-0r^H;!{y= zdwoJtQ@`nXN@b}6iO*?uzA>YQHgUVYyMF1(`dD6RDl3!d!Ec9ea`lw@K3(V@aNDLq zP%-|9clnGBRj;l-zNn0l7^PG_WHM3lvgmHZWAU!yckGi*_3gLl`gM)$YXQ9Jme+64X+WL*Tg9F6u%R)-0L(^QT~~RtxRp@hTx#+BS39H zSM_y3=33)rfqp?sL2hp@m;^_`t|4cXeZNz@8p6@Jx+htMX>DU8^Ujyx+{HJ$&@~c$ zBs1^3r97cwf{^P2xLnHh0 zy}rsO!QeM8zBV;%d&>z!x8scVzJH`ry4~mej+Fg+o|`&V!P4ut4xhKRe64J<+Y{eF zC02X5g0UAKJlmX|JOG=Hb%WKF@9S*wp2E#cRaB~_jAp{~<@I8NmPRt--#I_W+(vz) zu5F}Ty&(?GUFb1Cz33ldQ}eiZsvn*e31?S^hT>f5H#XtnmVY)0aD~q;?NLWaTN3-Y zxOxN21~0%dBpQhH>5_h5UoSQ{T{sXCVKdS*vJ6NDdfwLpT;my^mvcI@i9RmNbgs)o zL#uPS_%96VqV@S?dUW?yvUw{A&nzEUQJb}xL`=phJKR&rwh?NO6Rj7$Ge2S+VHuq1 z8;PLKT|rZl9(>{K3beC+Z2iEBkzVZ~27E@WUNl5Fy;w2dWZC$NSbaWu+=ZUQ9s>OI zs*Vws4YRxcLeQe&0ALWW_yMrNRjx!&+Ci~**)WHi`i}7ZXy4!iX6)D*2mO^>mOs;2C1jv-9_;lgZQ2VsxUi-(ZBKVAm7+q zzuCud+b+!1%QmD^yuEX{&4{^zp4|KUGQ({*(LG7bg970ug`S=vXV&ia>dDH~7$h3m zKri)-bTS}_9u+5W)ry16h6jz92e`eF87691Ou5zKmAEuQ{fJ;(?V~XrNgNpc*q3FR#bnIYuT~ z6Z#~^j_?pxRIXkPVmd4A-(>swXe>HHgvR!g>d+??^QY3j2|jn+{2bq4afZ#+oy5oP zs?6*H!|1z4(nQulW}S^UZAS4W2~<+jH4=R5zNrd@dnP>l#%ZSWMFBj-C^X&U0{`Ly zLv7zx+O}Y6>5(oT&w+6oJ$UaWXq+Y5b!VBLvhvKEMhB)WH7V(lgwzGVf$#p{H83tn zwVu}#q9P?-DSPg;G#|59-*{wd>a%Sk6>UZV{M=+};WVW$4@ya&j(^D^C4H{biYAej z!_w(is@XkyC2rJHyc=NhpCR}`*|%*RHE!qLwi(q}@!d+TCAT>mnW_*XoCZCCMyMYo z_I2UTF$V3=Nr~{|;ydcY{+?xWrPZiSCEk9#w#Ik+sOb3V=8>SwxUpHTvY}L)SyJ<9gT4Mua*&H4!5;nV2oF$h!FEit!Zg5169S4jMYL3oRsAn7{Niw6AC5{ zeCl4SPBJ%%IFIhYP?~%;)W@7iXVppmb?;pmVkm#QD~>|A;t%fkFur)_GLxube9$GU z&S*G%`jjbHnTWetc@9p=V8FC97!APw(zFuoFReuz!3M;bOi5-$tVNaKG#UZ*4c9aU zqU$X{1k_vz2!6-znngEoNYokJiW)7|9i*O#T2@(V&(1wkw$_@4^Zx9z)t)L_SM?*S zueITdOLp1ov!3^K7F^02PTilJ@y6M)clGwLGfU&vdwBg6X{+rS?P`4yR3}F(bipy? zl1l;n#0~#jnh%b7%U+{n-qqJk=uEQgH5~60&L4q;)6d@@Am8>>L_b(b>Iqd6Vy9;) zM;e6E1uhzlDw)P>MXdKwQWGpc&T)cdX(55q90Bq+mD;et{Kb z%7;UL;~axvxH=%pBnnptvb$!{O^2L*bko7Kl0@!eCHziK&9I2B=D>NBX(c+35@Uk& zcColC6P-tyR#L$UUsE6j*h7zHrx2X*nO4Fk9&X_nl;Q*%RmR}wZFFhiBpmvqb3ObC z1h#c`ARDmgr=Tl4too<<;U-!y9KbhT9)p`G#A`2~C$gp5yE7#^FIrM%&nL^pH{DO* z=Y=zA=h9rb1}!LzK&s6p=0NqYw!q5`yES@P8Poi>ts*8-bD zr#;OXk3O1;*@3miZWqrViN4DJr%R~_I!ql^LO%|`ob5BFg2(=N&$GOeD!|h153{28 zvl4D*Y$(k4eUu+>#swaGC)f3AxoisXEzZKk%FEHuiuHzly1%yyy_5S)dxybnw-ewy zWrLPY2J=Q# z6E9Ci*lkN|gL^w^AIUx&p`){NK6L`XDrHBQz(5388r>H*@58oVF&($+23M z?4oLYI@cA~cQDIgr$Z=QV%p=KT5AG?RW20rusS;IX^?s&#UA0I&5!H(OMhL5JwmC@ z&%8t)E=}c=yf79zgwMQEZycP)(0L#Rej>;`Ap`-ZP9ah`|9m<7(fbf(dMK|5#ZWJv zG_E%`)NlXu2E>rKV~-&`R1CGhgNt%mAKwk_Z4=me4Mq$cIFvYPD@V>nDHCFn*$To) zwS;0AM8&d=vrXvMn)k-SEt6=NgTCVn8fV~=pt5zS z^0bu}fAXE=^tW-;6-wHXVp(^z?Oxz_9b(qHIIKcufI9_U=PTdjlU^YNQ0sm~L-=P` z+-Ohh+pLJ{J{0S^|H|BN*licO=}BJqm9r`{mAxU3TD^R->*mbcIL(?EN-aVF-0kwN zdk$;(oA|4%UL;frtxo{iFc6(LB^M=@rCcuA!)okCYd9+jKOftiCU=B)UA}TgJ%3le zco6XUhq{*Rgd1TbEMV~XUFh{4nAR1u-;f(P2W>} z)~@Xl97roKi1dyaRUp&$x8YvrYzn8=ecG$Bq7s#Kb~lN}q7tRegEjb-^eusYSS5_@ z5JTtUtW`4CpU#1O-G7$VFB}e-dV>&*DUvDgJE`as6DAgvSk)K)d?c!JXkD@`D`vcY z#|_sLO`q+f8~-BVe=#NfC*jE2-pUbE;kJHe&3>vr1Nd@Sv!6OGf8opP0KEps@PARo z#Zab}2h-=pjN7uNbDm_Kvt!$d|Y4Zw`1Pq?wrv-eb|;ZjiFe zF8DIMC?X*6xyjoig|Qqg&>)2(dqAIx3cW=VcnStPDG_w(WJ&vNfwn51Wg}&U5f{aZ zs?6~V@S+HhZ(={ZWgUoYv@66ZwYJ5N( zd-3Er;ko(kH9o~?hPAM9ANbNKL~&ApBE0v=F52xZ(Ui3_4+4V9@uIw;&@-c4&R*>= z)CRwBNwEv7D9dKoK&xt@=P0}`L^ z?!dgvr{WJ9lctPs^xwwPIryet>e)MAthqbwkpI=4*^j;>HZ|b zp9**SrJl_$aI+i=oPQM7@ZB+hq1pfBarZVFLvgC(?P|#!Rko}j-mk=jQCGTM$YM%& z3|Jm1{eErHHDCB-g);3xqpv_)NahJJF&f>c&;hp>#>n2oeIdtyF=}$2lc$ZV7`GXu zFd9f6gjcr&F80$rF`BInP`DEY3AIr~km41m)M-a@saLaP?*%QbAI69(vwVvh;E=D? ze|}5hPNO64Q4In{zGOPB|Aa#xF1|M(U^#S>4~$ z)39?$H-|w~VC=Vf4RHDtJmoY`&di$Vp@ri~fns6&(@5}{1D)svHj=~+AVo-BAb&4V z6j*{h=edB`(djt?yhv8YwU`o9E*V;4Zg{|v#iOmU&&AN@lK=(5G0wHEQKvkBAR@S@ z@!F&cdJ_h|!|@VKfEz_HVTEf4)lAdEdMn?O()Y@;jp^HaP8x>XPBmGrZ2NO_Mt0+s z?B;Tt&)SzVc)N^kbEPd@Z7r2;bM2+x=J>14B{+(l*OrJQsjYNylKQy$IDYupsPCMV zKw@qGoTfya9TU~<8g;&WrGiC`X+ zFaUaV1*xv0Z|U*dq`4+xsh#tgr&@NcS`J?(0$>EIb^>*$p?mFvL$y@eJBJEbg~DC6 zb2K%PX~e)kk#;d=2u)l2FrWDn8#rVp7nC+xk3gPOvVUQ@1ZaiX?gd5K8`pC}=D9yOdZUMgU&qvWbGilba(pT3hiR8bEzKt=W zq3TNn@pIFBG#cQQZ-1N!K#v5@HHk?#zzala$f`kPkA%n`Cdpl^%JyiQ_ZPR$kJtd8 znXi{FNYP70zV6t)h*`0ac%XVCPx8tl|3u#v>wC5NUQid z3uCX+@lu1~Q!O#eHR_Zj{kx^j2AIwPjly%GCdZJ*o2vjL)~zP-&vwHlr2qIJkOTz+ zOKjMNlpV7i7`YvxLGai>AoJxB1}f6tj!#p^FHP50mV@1mFS@BzizV8#GPGnADkm0m2qe)f3zmVM%MJpXolefMzVob9aFlzp)mU(xcpap(PA>Vy4# zKoS_#aWsWxDFNmh0O`+{&uL+nxxcq_c;A@B{@$aQVLo53;R8d2VN9&5)nd*P``zL3 zCuWw%OS2lxF=PjjRjSWXaFC+Wx0q2@iyb6=&V!bG(UUPy`hAPM4j|9@^)-Ve3qF`X zzF0Iez?9ehardf@!(8Zf@b^^IzVbPciAtfjgAc>_<*r*Hq65gmsKw;>i=3O~Qap%` ztOjs=;Ej94;1-Y*dR-M!75YhlifJMKF6bN^y?xyZJ^zrPSS%VIU`mIz^qG;W`BJ+_ zqk?U6t*e*B+meG@;YmoG+2(jKh3L>+ca{hMmZY~gx%nuEK@VF=EPtb$z zFx6rZUyB^J)RDoBd<3NW8lnz7I{e)3e7T5W#AKSM&q{}M-id#F6MPS|Av8KV*i|w5By?@0sP0sVu_ZB+6cK7!x{XvfnpiS@oNF3@Mc3kxY zIa>E*5cHGK5FZ2^P>6QHOR>&TkSNjkYOJcIbdY3?aO#iXF{mLxwFfC|5(S#OHlKHo z82A$PNu2&q3EEj8p9KMjK$HdgDPDgS49!HR;7D zQBru}xKCyp$mZ;)zE~t1NCvdMnjVDSySe2jB4y8G(aUJc&4;^{qyr-Z#dg&3>Uv0c z>8nE!Y+{ycU=`ih57@l)&V)^z^j`3~(pdQ;#`~ifpEtUm4hmKO?hhFoVt9K)=F5yD z6YrNk4ucYE} zuuOm$o1*tv@S>2&V4HlubOp0XlsNWdi-N35b6?A>lm;?)G_Q=UzKFh^5!~~B>BNx8 z7S|RW4L{LM2u;4Q81-84a+2WZEu|`bh2}3zOQVt^Lu>{kQMcQI#ZwQtHSnfq?)=Zx z2U40)vKjb^5eU@Qi@nQ=#AI>69hE<4I+jMqRu3SX46*@5%;bIRKok|W>nikKt(J0B zQAbj)J#k5TrS!na;|XWi1>hf3!u8wNohTF``}2{S`y;daf~kp^`Rt@`hlQnfhvHQK zrBIvG;C4#2X5*pzl)Vl|$lrsE_MU$SpBXHYN*ySO#~t`kTVphocGL(j&CH@!4G?Y6}lAa@0(7iI-l9ljU4tW5ZP@AtEKiTkjAg*&z5wsMaiFYc8` zsuwC*)Y%s8=`UoSqR{l^jCYSDV&<|} z-F2u2)PDmQ!qV8vJ-sE>Ro5R75K8_D0gj46yawaH!q!^%PZWoT`aXR@CI!)KfcOpN zhg3uvq!1l#maP=>Vg2K<{e<)W4h{b2t1nRnYqpq*gP2#C@2=RRcLYeTk;Q-3XsiTF znp6vgkb(hVNzBye<`9J89N7|L)DvD+=RjVuK0d606$M>M2TfPq$#|)33}o!M8LaXP(R(O!#<#G} z9U<+|XjuTT-9uaJVY(cIc;5V9r~ghAM#sV&hTKaEk#iQDGj&zZ@;5}NLimX_HD2#2 z0I473iqOma#rD@d&i~HRKc*2IokQ_6K0kPgi=SWTUjbVIPJeJu6x46B`9aJNLE)Hn zw58iXgiz7en|&TexN7I$xC(s_d5w?}C2SbsP`bqhapHeq2@~xPhS{i)s(z>HAF3iY z29%ol-xHUfgR%v*w}AFkTdMyrlw&IXT?Dn5kbemNzt~DC@{1Ty{{IzI8^w)*Z1&Y* z)i|axRtB{iC}sXllm9_xziWbMPlK2z{80Mu^7!4ne`rF_{uBSZ|63AY+Y9CtzfFSC z#|VeaiEV2?^8dM+QMQu&-*Rn!i8A@kVAuetZs`~Eu8kT#6rui$x-fVC-2;EP717dv z4|PODKVlrQvHZY4w_N!DlE{yZ(vK5O{KM^EL1pho;Qe2LTNjiI)Dm zJcx1i9}&s39%|elo{e_oTLJC=pS8aqS%DO^(_ZL@f@R>iCE`bL`!D+Xe;dc2QY8Km zIojicPyLAbRCV>8Nv}%xW5$H_dDbH`G4cOc6k;_>fXm<|&@I^>*3n5*uG( ztt@L8<3a9cSB;f>#`yJIZry%FA{VpQ`P_w9_0h#`y`}ixLpOXMDbq-Dlf6~F(quJZ zMdp{SZsHL7Xq!pumy!HSfd`LDUJ>sgdvU?}g}pVm(P=tppCq=OtzR0Q)_}HD2iZ*+ zcel1MIxPxqnU39n<(q>x6K%F!*Yj<~XQ=rW^)896l}WCW%kA{r%a^y`u~^|9!~5+V zZ}+`F;iu}d>ww>*Nz1r{WG8#LZU7BAp?+xT?qgUZ-S6O-t+>tQNByw)y%^e}a z9U8}Chj-LFOw6=N<=%}G%)dK#O+4lu9DFqlvb+bXef{~hNT{n%lV9s9}Nx^kR*H?DiyNS-R>=wK)VOxT4@ zNI*!ruoVL_Y*Rv!hvj6>V4MM9sw4%* ztHv3OfDYmW3V^XD!iZtmyon9*1>P~-m^jArQ%6R*R1ZpQVdYCWko5#{0ZN!$N`#lf zCN~os(y*Eb6-LT;Bo-M=*45R7|-X?3VX3NqW}X4or^eOFTi9F zVU#dzrNoAE?yi-cCuy;TF_RK|k6AXR@@XQB8X#Q-5WpsfVUyg4HO|riZ5J_YoM(YH zrT0j`R_Yk%t3;TK2<9@8{_9MR@ZWSA9WxReOvi5OaHL9rK>^T%j8cOE#SZN;qX z7Dj-_A#4I8f(gW6oH1-ux3Nax`M5`Vk2qR?h7b?--$KY;3??5T<;PZJ;=r&8V-0N% z5CwrqTNu#xngio?@jNDRS>lAlNC%1-{r53!ti;BH++C+v27~w%e+fI{H({$2S2zN* zb%_(Au-Vqc1`d{6J%Z6Q**2NA&}KyLS**lT8+ULVE7u?76f@Z7h-4 z3YX9dm$BLE*lg!3SVJ2vcNL>0G!l}aSu&OuuVDl%YG5!92qrJO`6znG^Y(%sHMG*3w*Rqj<}i&;1JV2;Kxe+FiRm>_t2Q* z1ftr*oOWT4h#pG&I3Q#-`Ouit<&*7kW2G zH^0Ga2!p)YC4{-zrOmIsrQRpcWt=Ky+nDRvoYnI_DwtieI+vNY`L&zv%MQ7zE6neu z?u*I^WKQ)bdCyPuZrUasdQoLrvA)yID!KwCj?ur|jDF9yev` z4`w~A8{gSB!}T^ZbqrHZH*{C_uH$^xdZ-$P)~}LVc`UE{tT)*ty&8>l^ZYozVXFJZ zYqR^6&!*Z;N>>|37D{pQ>#ATUi;5ooeZ> zj94ykX=`N9Q2y4tvC_M_(IdjKV}DS=ZP~RIm(6ACSegpqc@N&Q7*jgCA729W>mPdY z!u%+^wn+~`_)^u^^ug;yZQCTF?ABnvhXj8FMU@Jl#>kx|9MNg5O4Dq$fL3{DyQ`wlm1X*dQ@rNE z;@s&`B8N0vPw2GL*MT1P>bpGi)Y zQY=G=We_gcM1$Na6el9c(0zd4+)yC@gl=opG1sIcx~*0ET@PIqY5k8}A2?zabN9&g z!6Pc?D1d|S(M78D{vm^s;He}cmo!z(znbSY=WD?l7yhh zr+6Q9&l9Iu6~~~Qr-MkjvyhlF)o3x=Oak>x$*9XKHH9JoV7X;(S1UIJwj6`OYi zeYalv8>~)B;^Tw-^OeOxEH=O=XPB5EgBJ6gXrte$C7~c!WX@nxAmAfbj2$5geCCEk zATfct2leit$^LK%?0( zxk_JAjoGH*GS7I>AW;@RCk$yDP9-W)?HHFDl!L{FC=bivM1RWo;b<#~-Sx7eqTI8Y zPY3Ua0l&jVxpKSPRhZ{EcQlypo#Q0oh?B?jste{KX@*##9wXE<=ze|N!5scPfHFHQ z{6dADAemD!SK}{(%U9khEP>Od8;t;H?a{?|WZY-XBee*Q@UGX8EPeN1z5f0{+j_E^28nNdYJyFxqK1hu%qNdp z7t1>)$cfIke0UupC4kO`3yt_3kQJ>JO^8!QT81SCH3O7irH;y z&sUQ6*8d*TX!-?^FQ(CgvpQ54kd#FekT^2ro*@Z{5q|O9nrS@%iq{m4xC6%Bb$Fme zc;|+5%vpR;kzY)_lJ>|SfY~Z~+7+h-27uA`L*N=b<6ER{d~wHk7Cm2W7z{jN9vfV! znzNi%C$TAC?<) z7-z9tPx}VTW0sC3TuDkZb^nZ=U|bDp44<~bCJHL-1e6$&=!xc_ z*&1=Dz~p$;0uiX_=GUIH8<@0BTu63vj)6lSbU7FSglNrjgMXjybo_z@qVj<#kinOn zVBQ}<8<)V!paJ4Wo^v0ZNfVM|#7j=3>aEWxyYxzbkiieA=c4dRtT08G7^@$2UIy`u zzmOtBa^Rf&ig#8m0@MEIH_p2Y=^7jBpj!5?h+Ep;%xcV!!py~Q65eeqA z1>KV`cKy4^tx){n!Qia*O@ZnhFf^juvlALFMUp;73WNv_*$9Qpg0ly#^{%>FYFJyBBl;oIFpJ7*kFR3%1RC3npAA0++&n#sP5Sy{2< z_MNnj;EDGn@6{JAVE=d#79(=aopK7rYo(JY!eF{SUtWx|=#S`ZjHxjY0tQjbn12YAkTt;^Ltey;ZrFGaTEQu3s+}=2;<8D?w=#>Bw20RPc{AJ%n`j}_BTOa?F`j&Yuv8k$?!Lm&mA6nkpRHYq5t0YEiW5l2>Jm+ow zcdXn8uW>=H?WAHi+HiD32sc;u+^aM$LZIt-+G;CpSJBQ`FZ_O(Dnq!t+@!UX%w^vf zXUb-CHdCb_yg(=e%{7iZQqh@K9WB{4Afd8y!zvSP>ymV65+)=wtI~S#ed%RbL8yW2qJ}XIi;&s zo5r$jYA~Ig1J>FQa2e@da%QtJu@<s zB}`-1)jMk3*GFQt(&{I5Dz^Fk{@qXg zz7@zv%aD|rA@Toi*z?A}88+Kb!v@m)^ntXQAm%l0)F3*PMA1&bxwE_#brBs(XcoEB z!OoPPT5X1N?bCx4^J|R-tDx+naz`(7gG8`yccGaem!;u9%`#Pl=R`*+!H2kVjK;zE z4H&dvhc6L5IVe6F*#$L{h4g*UF))GYnBuGPz`Etq9G&rBB4B((hw7Sbav!iF%Qr_M zBios6re2_T8ejsm^1y{FJ3o2hxg2Ia@}c2x1&v$fty40D2#GAm*PxY<^q$()i%kc= zlr9*ll2BT|OX26OLr$Y+JgBEtHYZx8g!xW0?>nG^47mbbl?`U-+k3&D_xNEr4U7>^ zgVLUvL0$-1Tg$7N{=RU(1`K!$!`pjn%seZt74aaEm7-MNKtOC5PyLuB^{`Qq*3$QCx)8Eli*}YW(NMn z{JdnQlDH@hWBu{0+}uNJ+(zc(R9`!IYn(Q;u6?4m_DxDo-rsdJzq_?H5Yzh3axwUR zce2w|Px}+Yx%^kWR^(^8JEAm3^et;YwL%OxvNAVXmJZKL4lNgjp11LzZ;rx%{O~%pM zD6Ea}thV0miyN-ez84y_)UJ$AkT7VE0wd1J&lO@a(pJMK7+69g6Bdicn1!!`l2N+Q zUEYeiL#(9d2(=sPO$QOH(N+?3q9A5i02oC`U$ts2Chza3(^SP%xT)a_zQIVDL<7Js zn%9GC^)n`b>T;8Q;iCPhbRqka;lSI~oUs8=tF+JM{b zFCO0XI*);p6Tt5B)plB&vp=Pq=*n%cKMvEWx7L)N(-^7}MQQgOFdJ%c1Et;aPRP%K zA?YYp&vc<|3=kedlKEY)p?|w6=rmElLP^#~rDG4%Lof5Q0^RNd>7cic|G1=3<^uRG zWCw1mS&k4wNPd(7^EDpbKe#Nn zoDd`2CFd#e9<(Cce7h35BwzZcSr<+tKpXzsp7#BvDi0fiD-+~I=-bho*6YfhZ4@^W z3T&mh4TAYv5TF(i$j5rpQBRrKd<`l-@N*g?LmDv5-E3V+GWmgz!DKMSTC7Rvyp8y0)?r|CLgo@Z_L7>#2SOy1w^OaV}{+u_TG=?R|LAPcxbXRu9 ziV@0EzF=u#k6%~24SqkPru%wLH#JH}GeMq_5E6OG3f`=Q7O;a1-s2&1qAN%J>$93Kep6KXRPluT>i<&=USu zAknu_(jfzjvId(183!$S{or*+4tO;(;zU;_&?`8`&`YQ&&E*PrBDW>b&g6g#Vs#xI z*f79O{;Hn^$-zI732)WUHM1U~;8Q5+VipXU5{lIy#dXX2x;cI(Rjvm|BGK!k{#^ob zEp@8h2NGcovA*gC<5L?lMET(keK-2-_=HM5e-EsdElZ$>WU}V03uU*1eG_3-;RCdn z%~1kcFeb^H$1+f*i3bbHdb9qeSn?@+tOw&F>QNvoq2vdQg@G&@UDhi{2Z`kF_%!i=}h>6K2t~r&z!l{K(*?aJdV*@l9^AnFe z?a8w3w&N+^a@}SZyVw1-{!ra;GfxupUHup@_wO!hlxH&=WlGk%xo!)Pa!97c_^b|H z2z<=a)-=%_lF%Sa`Yqy2bVOD{Q^XlaQaNt+NBymuN9rGw7kuiHk_Z6j5>cZxfg~m) z5jJ&|@u=km&e$DY0^Rx9xg2UJO?G|1x`=XNs}>AM6z+s%$Wq=~`?fVu$GTbQ{Zq11 ziK|i0JG$eQl$eAWx==C<6pf04d=Z+Qe5*+KdH*9-kz+ZPJiJqle?fAOA#H4Jt-%vC zyePlOM+k*jX=|@67fnH8mXA0{YAP8OCAve2O-YH{{_87*b09fKS!5RkaKKE>BOMi$ zSl7`6wCs~O1)xKe*(fV=bVG&=d~9W*1m>NemmJx}*j<{RE2!y?$_2|DR4zmjP`U7? zb*o$$pL2L=N^SB)@t!Zr%L69YuD~4 z_?Sp+EjrYPEn41+LN_iBxIgA+gAs`8&qn>9VVszfD`ux6TI3RLTr$0_^(r}(QD7o3 zSCA_D+qG-hegnz}Y+diQAL|ai!PeDuhg?52q^AY^5p@fu05t4C*~8DXCmsZIf4i}{ z`I+g17E0VXg8$B|GsdewO1Ng3K-m;IVlq?$W!fonRHiJ=IlS5uFvL&_C{M%)v~B`( z-murCATHVkaRwUylrRc(nJV)&+A;p{hYhik00mcw;My?}S*WLf$r5Ep6x(WSUqVkL zBQe6EU3^*M1mq7enrs7<=I(LTp~`WM8`9q)-IEQ-%nIJ1QXUb!zvwe!6)Osu{H)Na z04xeRW4H!sKQQYA?M(e|64w3{{Rh-xAYd>usGgE~pl+#wx+O_A9NV&T*+U?Q>IOwr zWWvAB20*@$dUv35iS~Q31WMwh=g~0&Wh>g92xg-|`SU;`y|Ec9fi#OD{sN;=^`NiZ z4qE}0;A*qAq>w;2q{BlfPkOMcaS~9H15PKS>UiE0hj)0>0xMZP)87bs6^nCg!c^I1?Vvuq2IRgkaKhpKyAz(rzRGpY(0 zz*|uW2NPCMAqRy&3u6DdDt0D7d*Y(0KsM)1Ltym@CzLjvH@_N)mU?}~HsD>b0nZg6 z=ElDuH%7Sm&`qZXV}-%9`eio_;X5cFY|lymn=?><9F-msJ|-+D1b^(F-k=ph5G*IY zAM8bk5p+ZUf(L$`{ z^x;GR85AlGka#sljFGHiyMn4aeXJ(@d=p8ZWGh2M6=aI6cv5K(vSLtSB3AZEqCyih zV`M)64{d)Qh-Lo$kK;4dq$Djg5!$9KH6g7MQmIMVk~9;kq^1p7LopIXS`o<-l9q`n zGubIAg(#Ylu|<|-3khTUJ`-S*C3+$RZ`ty$82K{g^1WFHTm`Vk| z9C~nX&M(1C1ysU(A`*5*^t-dFWgja~{z1P@tkYP1vi;6}j$H{T2rnDC zj^d?SY^=_M^b4Lraeh{b@WQ>*w!NR+(lLO2aAR_nkQCkQ4nCvnZLG7v zRT_LtAo~uY6gJiM_NWipFAU~Qm);2Mv!f$xO65C|ZA| zz%@bOW(G1EkJ=28a&I9;l-X}Y=I`clqwC=vg8}U#^f~^!FZ{(yO#zM@flRPI;kzY! z!YI44vKN4Ys6lL^$Dz&}tddspvE@{p#M~#p9ZoJ>{bfG7c7Wm2vTR?a2m2g;>6hw4 z#RDhjc>dyS4+V{tqhcFnHXOP?83Bi>@a21yAb0bT1hp`M!-R7JtRN$|ogFieacWL} zVTj18vP^Mq#wk5GrD)41D8alp94Ok~H9SdGVbK{e29{OSqCG-1{(s1Hr3X(pT-`aA z8lPeeohnDNhIDH*YE257ekA7N*6dHA2~hmuX|=0&nB8Z6XX=@`iLy>HAGcF%OLziL z?@Ezi6`D541Z7pqY#WJ8XfudF*PBytvkXfvOd%_HhFV?{m+RJQ?=<-^ayDNGLZU_C zz)J0+`Tf5VTcZeL`L&4CJOdE#r5^9vMO^@5Z;$pu^f3ojj12oB3=cHH=KvvCd-6x> zf^yoENOsf({kTbqz9mm9dKatDgbL+1Q@>Lir$F!7If9zQazU)?9J`F3Y)ZhbGk{^# zZ1)zic04t=w+*M}`$qS{4!COe*1y-WGCGHx7I{&B<`K;P1HI=F=grU&gpR4hJ&^5~ z6pRC!#(Y-E6pHRF=dqr9y%%ePb6UT5YN-pp)()*ja0=?ei)@ZP-EP~TVqFFs{!@0y zH&2GrJo{%HoUr|nQUuV{gZj&9;$A;fuic$#mio(IBioV6-#Xf&NAAjc@ zGrzBQ9s`p|6F>`oPBu{s?P}%b6!dI4-SOCdzxp54%$wb*51zy39IR*k!GMF0$E*4~ z&;o3ER8?phmx85kGq-c&S;qJeJ`a;5@Z+MJoVkiP?_A|E?_33JfIB;zQd!3kQXIVXhcimw(WL4jZ-Pf8yCLn- zaDg+e;vAQ?&4C(BGnha$HEiSp^Imcl4r*{`R{ube|7U-;Md{Oz%wWR*C;mJ+d;emZ zQhiF2wUZL%43>bsU48t5GtD7n=7NK{n%3aNuGT|}Tf$V745T zWX{VOo4H0{p5&&;#&!w>DvpWBy2Q+RVWaXM=3O74^QQ@LU^I&61#{j(n8fUk*_U@v z8239F9nn{ZG6&EZUdU!z=4U{X zQF&RiX;R}dex)%|L=nxDwth0V0%~xFM8nHneM4Bo zj5C}B&G&YrgQB8|x6eup-sK$HgtSN*AIVK#al zVlBf6k=;OeI4;^4KXS9iD3E6=6$ZamF$b+USSwS0AOp7_E5G;?1>?D&^K5!pplYZ8%V10skxxuNUX#E_kAe&ACuaqoXfgiufuXM823$SIFFMUXdddGxWG5yFc2hRLr} z?at7bVm?$ewJnecsgc13bG+YK;c@2PHj62#&bJE=?>=K}C$Pd%;yL^%X)K-7(CiLM zgGVDRqrTlgLiGuK#{qf2fIeV0HP#n4UffdvEd2UtzB8K>ZT+p?SQTc+7e{grrATQZ(05}SFA6m?$W*;qWIQC^r8LPWx7x7b?)5uQrwo~ z8xvGM3G z*M=I*?GS&iJjXSzBHA_Kn(5OoCapcKl&+X8_~y1PBbplOcEztSjHip^Yh$BBs)kD#CYrEE66{ekE*8DM92`hjf%)-H=atgm z=Cs9h_s-j~|89(VysSa>-uD~gi$&l3VQJKe6Z>Wb9WdA+yQsRtGqWluPG**$(yQMR zE#v3<rbplLW)5D$4H_rHvY&e({59 zmPEqPn>c@RK~=k%QRCQ}=lJaQgLO8Y@jb;js?o28eo9sslqblh$}hH!m$k$B(yAsx z!*OJDem`T_xx8f*&T0{_U>uQX(TQ(yeG4PL9{DTG@@|V?*+rYqua>wl(|EI0N(9<;xzk=#2p_;8(5yu5Ktq(Z??0h_%ga%574!ku1;VP1D+tPZGO z*gx-~rmlODSM#4nSBiY%eHs-?cM6p5{i{l_Jo$>T+0)0q^!tI@04F$EBHg*`sF_+k zE$U#toZWGUymzohARaCoGuhlq-H6JWJ+95h? z$`f)kEmAx)vQuAt)}KEm{z!&ZaGYglI(K-6Y(Z5HoT@QvJC0oJuTG&I^gV^s<`@y! zx%i?f8M-*|fxlSxU*9)A75b5aC0W*aKD^g@2;50f`KLc}EnN}LVy4;lwpi4!WcJ3FNw1qba>B2b(XGLZ`ROLt37{I3C>NrVvOmn zDFtwe+(ZgsDMQ~-vzuKkitzgZgXeg;EJw5qiQbY}cl3rlVi-^FF)Jn&yn~>4OR}h} z5m|IE-I;VSy4FuAeS$emud7X_57f(-n|QqGl#}4sBk>=$f`x5TDSm&a{fj3y0w73G z8g$JSZCWgBlcw*#wjl~>89+)ly}B77 zId>x2)U=p{_8*+w>AF_|6Bj}5WcqdsoMc235Ykm^(48bI{-is@OZUQg4m`MM6TjlQ z_s)KhlzLa-@UltwFc~+JCw#Ty_y@ul4kXVbo@sdk!WS;3uqA;U>uHl)pL<80`|V4+X9>n;*`~~$@7)4?{o|qeTa8Gz z!Xkh8W2n?q{Owam#oqij0I@^S3rI{U(ONi(Xw}9=->TrT=g()(JZ^2BL^^62!$>!?!QF zQWSCh96(9gcRrpjjwI3d;R&y5WEeV&ckM{orA=PT&As$H`O^d-HJSDz6a5?9nBPjK zgq|i9z)ulGr%En)&!*oi@)p2n<80}U%Yaukj%~Y4j;Dx+E**P1Yc6TUnW^Hafqy94 z?mF~-gQTHv;eq-k8I6F(bqEq<1B(SnhLVOSfsgPc(2aF*lR!7{blzKv5x5mevLrv1 zHS#u@csw`QSrD$FHWavN_gWA^C5ALGo`5t()Liv&l(RoJrWcssyK0}^wPi|7y*^$EtD<7y#vP1AlyBBn!Y z;?TYbU6SsH^*V(ve1(I9gDH?g7kGrwrF7U#=@6ky2~vwN%^ha%`?J$V#rPmeO*`}? zm`?TlCY%%TO1~r76tPy!U&&7sLvxUoc?VCSJ<|9VgBUktQ7BA#Ux6t1}y1if#M6{}3(|d?k!L>oFpl{GB zcpArJ7)l{>iz2``bAq^}1*8n^@b(vuT*>(`G<0Lm&n;Y+@;M5fZ26>f|q7D6p<-P=crQDhqJSi7$&6D%~A?0$hky0*KD5YEo;)EA+52(O%Zwj)S=anzMT4L`NRFcpg zFToJG&XN!r`~ipziVs94yg$Hrg%P7vde4c?Q688DM8;Df@}R?768_1;gD1YOd~9h8 zj~dQE((cmEF$Qj>buBTQ0Vaxsm6Y?!?yUG@IKmd$6=92fhOqq?;#haAa~7qa3u5aC zeYH6+&T5XS17#aRP6RRq;s!FsJgpR-1~S#3^%$9Q>bym#{d}tZYg@`ca!33p0|t>d zp7N+408-4_g0Qh4sEM!+4;oDHDE>509o~2B=h47pt9ruYH;4$}@xKs95F17-7d6eV+}gfkkOM(!n1Cb#p)3rTN*iS8!#*<)&3X5a@~nH)&BT) zhbX`|^h*{*5**k}IAo1Ia48f4qt^Fg788LS4-I!RULl^i1<^a`Np5JeaLG*OK_cf< z5V1rYKZ@AZv8XF4?=E;vb(h$$uab!`8YF%1r?gUeH_=Efc-)c1x?ZuLS-_&8$y5;W61TtOjDu1I4^%N8k#NOW_UTEf zteXg>`9A?LM<)zLK+U3l11}yczP~k^S(h>_*Sa~k{-3N%3@wy$@9&rRp8oZ{*;7uf zLgffAadbs>Tlu1?=*(`F+!RDjf5`{qGIH^JWWku$s(+)%1HhZf0>&BvLlk)srw@9Y zFpA7vnKp1ARZ8xD)O)+i9E@bM3+k`IWmlA-uX3{AeR-=*f=1IMs;IZgvU<<8opScU z=rkKI_-WYY;*1jz6%R)pNS`19zv z6U=j|2)QmmV*l~5(7b+s?PCUv4_;O>Ici9<->Y^`rfn#{{$Da};^X-tyUIAUk&Zi* z_#1e|o1*}H%YVv0Xh4#w7GuB(km=3JE(ywp zUu{<`g{&p?`viwCs7=+t=^9NihZmy-*B?KNUT-~!u4lY5>nfA?vgUz5_%RqH^@ZB* zABt!B{cl{ku=#VWanb9S)E+{W07{8Q7_}d-NfD>axdzWi5%1J_N106v{>Nq`PKw?P zD?zAZfo2Ih7X8IEC7ym~Y8lzHK&sPm1N8;lWClwgO5PIvr4PXs4?F)1vG{E1gIC*V zPbn@2`Q>6VYa8uV>{Z+}!*Bv?XLAa^hwpu;Z?PnUry=#(l3+^`HKCjwk%Ii2Sbr7& z`r9#%-#T7<;z$FxZWSZEc>6(&^3dg!dTLdRezI(7-PW^cY7ATANBz2FG~Wzd1Z6wf z-Q(rBE{v_|?2bF4)MTTxtL$ieakC7W#Ch|$aJAo z(SD3+Yz6Y8O1`y0_X(_*@e)>Ao{t zZ?1Qt{~9ZN+N-%M5}$n)t9`*!w_14-{uS5*kxD!5AkVqSpY6}}#T-Ypw2p3)R%~WH z1SvA9LT~@gV|I6?vc1IfYa*_V%i8hTC7jRJtG?1LE;pUEQ^9!VZF;-iUc7zht1`|8 zI<^_}na)~uK_VkXFmr>~T&DuxYa*tlb@J?LQ$>&9WexfxY(n?-4vQL;3N9ig@nFzw zKB+gCe>BZ0_2v$5zJCfMlp&)p-GV;IEK4TL3k~WV$u|1s{PVJ@O+_ga?GEDf57+LX z6I#ZehYC9EPMHFFy;~Ivx>7`4O>)~(%rgxe6BH(x2Aw9b7z0<4grm56^1^gFn1T(b)<}UItN~_9AGM2N_vCZ>pPK>I*Eb1zCr)|qz z{A=AVFH3u}ZhK#OdhhkAozWt$A}|ZxeoI$wT23!w3THpOH1zeRc+7r^ zF!R+4P!qv_9>6Y7c#;4D#V+st;VzlRx7dO}fO`eb9;t-6S z*UTGEcEQ$Qm!SEkTG#);KpLBxxrNVE>WN6IBi_8+_1?|06r-uDVx78dU5KYme0{0H z3PJ?XU2b}TdE)}d2h#8EJbOkB=br+Sw+o=;OzrO6tL*%>-NC{&7;sI`A|bKoA!K-8 zPD?|PTlt~$=4G;AcqxbVs_UX=EA7TKc_VIR3+-%s|Kvhei5?`hBtjoDzgk&&i$pKk zJ$!CbTT$$&^xrn89!^|lh*&IHqxV`_RWTseeByD3+qA*B%DPx{!5{(ip7#yi3m%7@ z=**JXVwb0qZQ$`U^w3fAgI(Uy!VWuH%lTm2}{*~yqhk~#b-PV`m99X!!{UD@-vFbUg3Gx&W& zM_XTV-2FPO0ENK3zZx#i^-KH>=eeI6WndS~l-8mg}I@q~P-A1b6e&7zz+?^9g?wTy0mAZ5RV)Of^>E#0LB@Pou zn%J5pw#kmqu_(A{ch=&e^Q8oD$?SU#F{SEhqGpZwqhlh5F(;0yNrgN1?KXk7kCz9b zpSGguy2JaR`EGG8fJwN>!RUs|GU*K^ws|T~FHJg~0@tOXQfx@q;=TswPVK(GwiVpY zy?3Isz`3)mx2LanOy7IszMj6noO@6AH92?0_OR%>u5VkGr7PHnX5Kq#nH_ci>(ZF|acznni4|tenvQR0XO(@szuCyK zWnRcU$M^@a4UhNFcD{G(L3_i?Hz|U>PPfgPPj+dPIpNnU=H)?MmuFbKZQr+jl)2Hu zp`GGM)pPq^mnu6=F3RxSaDL9WK9Wbiyz`6B^+!2!zaDeTTe&z$8TZm8V(H?Sr<1?d zn4oC!bR7MV@!7RWTJvavr8^$QY3KZ^b3OAxrdsIy30gm1cFWt);k)_==}3O(p?!)Y z_#>_zRKt3GpK#=yIX+h`SInH_bN%+b^UAnLvnkA9uIMy5gSPMu+rlZ@!s8xidiCO< zdAL_`y4rb*wC1o;LEAz%oK*`I-xeA;-MObuL>dnj#*Q2n;jwh=no$wg4SyCF*3c_I zdgoW2fJ`A@+LcylasKk%v$l*qvi#aCK7;n}MnzoHkv>h29&f_F1NaN7){Uy_shL~z zXv2By+WpgoGp%ct)l;eq`;v459(jC+gQib;LKh>x*6=Hxb-VnMb?wdRN^S~94i`t* z%%s&#VT;2Jt_GyHI9%jUwM=@q(bqdf5f`2lk6wbjmv0`fHP;K(dAPRebf)phFoT^f zQpOG!GbON573W~9Q|nCE_?5y4c@eRmh8+1LBE4mkKa_9zlF66imEXRGq@D0}#Zd!w zlHM(x;8b<6wnw+{#f;ZhW1*AgBUtqdrL~&#iZ+;Cz3jGL-}d2nxnxbZx^uG0ryFV% z>G&444bw%Y8aiBjWE$LadQ`*-izVfGMe5zpy6IwTs0GyHx?EH?Ir7oNUisFSAzy1u z6g|GfEl+*&@uM)i?8e?2w>L9uV-3Dx6gfPC6n!+HE}i{_P`O|omq>6YB^QYPB_MQy>U@C*~o4A1=(bEwU_dj_7CqUR4gBUwd?|`NZbct#on3jiib%W?NVjD&SWtyWD6b*0jS= zpk~$fs!Q-CDIXGHtZ3wLBJ*LM;|aggsEV(qNyFWB17X!0#X97s^KM?5b);WK#Ukbc@fU7F zp7hdaal2&ws>)rbaDjtA#@68VM$GGtny#hgn1r~_Sbc4l9rjpt=ZT_%?@XL?#PIL% zZp#b1_!KoJlv~rgJHizo>0U>`(Sw(lNC9wo`m7E6i}p3eZRGIY;t|&^mLDi60_41k zYTr=!y+e~du9BwCP}ILgeSn5E*W_{U>T9!W7(?&jkRm%-vXuYbfwTCNl)qLbk zrL?+f_+ok#9|~#ern|l+1H{Q5c-j5pz2fRCw=cU`3Elf*_e(={SlN88(;3l}~sbawITcmu1p8LMTcZ;~Dzcmwc~ zcQ@V>;WOXD38)c)`uS4 zRoh*#SzU|Yls@}LgVsiG3fTeCQL!YNBB}aGh^zE!8Ms=Qc-4jhN3T@OyoS_D(QfD$ zS9f?{26J>?cp-UQctQ9a{{UVd!_mjWiaVx~`JnUQ@W2nmFV%f)g~LYbP*fwUR7po> z-uwL|;(d9jS7PaPON9a9%6Ck-&L-`%yTH-&(32Pv7cdP9t!EEly) zj^q!q>kwZF6UB$0tC7xS&bHPZ&Q_bZnVgMG9H>Q6=RBVynjQ#>1G4oiFH=e)(OAn> zJp6EwTb3~31g(K=9z4r6qWuwq7{HdPtL1yLLKyk8%3Xt0q-27>Ay7qlHw1FSTi^`6 zd%)g!_kg|eV26%$K2n|N&vB}J2)vcUrysyN`8e5-S2~!?($$Lkw*01?0M|Wbc^!!U z8Ei=`Ewatwf-1q~2Yzn9k?ZFT4$RM^V#&`bv^fFJ$qyepn>q7yc;29&d&s9b#UB4BF2$`@Oh0nP~wkVJ>paFS>d6F%M1sgjtxXO$ z0O@^E?W?P_*2MMQqg;Cm?w2%}NROx1&7o|@3G!Z87?~7lc*NJ9^n_24{Xi+-- zc;$%pHN8gWy4+Cn!9nypd?CzvL*uu}P0nu-hy8)R>HuYV5cpj9f6%SUvbK%Lr@cl^ z4GW1r(y5MoiFckDFg`nC?~ep{N8H$xOF9qB+Io){e}BMP@CeXAU?u3v@ug!4R*=>R z(*kvnsC}gXotD?(pwQ3t{+I)YqwPuiblN4WSMB%~KlH(1oeLZjnGV>gq+en0D?#9q z30G?2^;I{yU<320@3^ljHSIV3G`M&Bc(OugGkZChT@f9Q)(r}M+%W=aGmf0JsHZ!et0hzrAJq1V^{h| zyJo=HS5d)KNvpdFWx_RPb!S4D>Z~#6OHZqI40-|*ET!{Mpj>2T&ZZV}i-MVO3)>I^ z9y-;%FX*HL$it4vMLnhigfA&+@;?SbBZ`xOK!AR@ZgB7SC$znO&s8{221>JeoLZM$ zLOS@HP-xlmgMpITrdqYLOx2lWTC4TM&u8tDfb{azDB}_pXhSZ zp2ZS~nDV=!G7Co-SeckkU(}-Mxw)Q7CmdsPDfJza0i^sdvroOy^E`b<(+(GZNUR+ zpN4mDwW;vJKbRJV~1y-QWkm=AsHsVrz%Dm;)WSSfC z4jk%(==DTl1q%spd(Z-Nu!xbmG9;;C$gIVl0+3QRN1`xS6WoYx&m2w-xvM z@iBZcv(`X9rawJ_;vTl^S^BVP%Fl!b-SGjH(vgqiUxg{pMqj~+aEYyYJ9L7#@X<%E zc15E^>HNCWNtW$3r!U?VorvJ$OYftDAZA)+L#KOi6LA`ux%XVJBFJBuWf*lrHcY;{ zGz}lh=-p^9K};lcak3!MGeyy?x~W8D>`LgTN=8SI>jJpJaa99v@v6<0(Oq6QT7#yKR%SQPs+s-}*$gEW>&Qr7yZl321Ztg04)?ruVqu zceB(URduURJhn`A6KdS!hQT+e_K52@4$2QLv_$eOW>E9pSE0PmeNUu zer1rcIIS762fN`}H^pc%&4s+)qL zLn5jvL!|qhJV$NxLM&I$_||7uzkEeq^g6CtET={ck}dUm2G@nYt`)3T!b)3Ya{AP8 zJEMemr31q0wHIBfGFHijFV3=DfwvQtoC)UteoOUVkHSqi+y345&onuKHD%7=fThB)GjO`XL*A?-@n3|I8iVdLYxYAZ&EIC^ZV{wavsLjWCq+aFS<7Ne7ewp-drG*EOEzjp{h$5Veexa4~D3s|n^wBnZP zutIhh^E^TFOqGXnq3rnRL1+n_K{YWTovIp+bSm2J#-sO@f45VZUXOOe%0{YaW#IR# z5ygsBA#ri7rU9@P~5rs9hY06*#IBZwB;94K(?n)&Y5bn1tJlte5- zE^4{Tp|ECnyT&+HIR?V-(Grt9;)*62Z}El})-Fe?SNTn;$)zRDN@Z%tX~OS8@JBw&4_SqCQ8@QVG#>aI$?tS4U@IW2*Bnz`SwsOAV;+XwjES4Bay>gdub{^`(xs zydw~vZfUtB8lfpO-Sw>3d_-vXlBUrod!c0Y#Tee63-7)va6XYhOttkTW@1&*R-$jG104D==*2h+r)wJ#;fT{fqh=aRqxqc6=m9(x`mtDASC;C3oRy5ft&V1&zegmzR*9AC3len@~l%<0_hV#)gFsymV z375`&t)Xvkb3}#RY~-}=qq2g(&TO_)@?5?Bt)Jp_%UkqgR>ekvUWL`L!2H#_r#Gg* zl(Sk%?uy;eS|!|hdmDZ1_BZ1}H4e;{a=s8g%dDyrA2WT$nVF1cGZV~zB9Xq(QKL^7 zmS9kG-I5JW0{GopKq#&U8u}rbwTBc*PxiQHTzzO|%FiO*rl&mPCb*_%>y9%kra=m4 z2bIb1QZmr_5Iug``NB8TCyEWnOJNfLK9um-xcaK+X2s4dab3m5q14hSHgC=&++Y`I zq~d)OqRa@R)Iz-gnrJimmFl_488{2#S!cZYgKy?`WQmW9-$Xwq9GK1C&uR47akPWe z10mV!X}FJV^lcq=dOp+v!8Ll7Wfc+a8|>wuy_||&q2xJ(gI!N}9F^qYb7 z`Z!;2e9ZTx5C%+9sIQ4OwNeY%pkKa)4URzBndTX^?w=*a#AoCl7Cud;R+(J=#5_s z?55<6(zo9aNGfZHbEha=BiQfz8zb;+0T4@9&_Ul(UBtrQ?LdJuz$rFIk-i~#-TcMh z;Q-t{jA4pzifEs=#UYUm&aP+X&?KWM_Z6(L=9HzU;m`iyXz4d;Me_+@S)sKK{aK0x zHnhk`oqS^7j z9MxQ zgvMAlq)DY72!f{%iH}w7$^Pr&YTu-G@Q*&!EVBlPMAfXV@8z_X8 zGe!XlA?^u)tppi)U>~=5NbaIFFG-S zvL&F8awf;D9>kF0@x+s8-FWuIXnG9TL2nzdgN(;kUC}|SE9)Np_z#Hnjazwk`5aY`MQPg>eTOf)w?1;w@=g>aY@Pjjk zD`uX4nhG>%)G;Rk8Fz;PetShZ6v-?W%Mpjya` zt;KN+KA4{Y+?9b0AU#h&dLFn_>D*yMQ&d1ZiV?P6{XD!-weShNU}TuldKUkWhN6@S zpLH{2Q_i7TLgIX7;^v^Osb=D_H4vSf*ij7;qGN!_ro-1Whx!76X93V5P7^I7a`>P+ zd@&)U$m#P6ACL_fr}~K2e^xj$*`UJFIUR6u8l*zuxYb_^F@+bs=is>tKQEj<)Evkp zU^FU#N)PH>^I!_c&xnH$U<$GCt~rc~%h3pH0MCeu15>zDGrg+H=0nvkttLu$D$cT$ z@xT+7Sy{|h@Hxa&BY;j(9hkz6YXe{kp&l@Wf{#rnX+@WMaVP~(u{i?vdLR^j91+Jl z=$=qGbr<0>^TuGzA;FoXM6DHHJf-LW15-tHS>8NW%U;gFQ~?PD#SX29JP#ETl!C;- zVhFFnV~a@%V_?H=!=>tQ2PiC!VaS6M@YFwb<56}th=4vk)}0*w3(^?g##Xap8Ygga zH)))vAFM;FHE?d$*5;MXC&i{0ab@T)qH{0DuBcOqx$T>3G!~(B^o;vk*He7&y|PCF zHBpNFtk!wtv!Mk|3D8_0H;_fB)Bcg(Su4EfJWU4_{C~7ytwI`A3j6Z0aRwOVD|94p z0$f^XZW}L)rf_WQXSoZTop8*K+|N7O*hl0B1rQR#@y#?L1%+lKN%k!a?tI1=E!F<< zQCJ$a?B`IEf$r%@Lz_?#>)8hZ(UB{yKBO;VG@2-8j0$`H27l%Gk-vX@dO$kPq7R3s zCO$Bot_D1w#w!4Ip1ys2^TCuVB=Lox_xb-(`<5m^cJfLZ-mOPxmP-E?(3Qb~4MtdO zbR_n9{=&}3b$m9Wp?OUgaF`J|B{r?)C2;#Vb1N~<-0Cyu2M=Q!Vk3tQT5NK*-;4QP zhBPm26z{oComdnp##Y*~GvdiX^C z%cD5X4gU7c%(|{A+Ov#vttDvkjDIDYbkE`wUy{m|b``fZ_Ik zTXpv7yU4t-!ExvS>;=7|QR>?tWN5^(bpA7rsNp9Oh4TG@k+%{X!@up5u z?H5~g+Pfd6;_G{dFIszeoNhTP5bEC{JvvQi-65ah&l)Og9K?>E#U1TOQ7bQm)a@TT zGqc;UNl4-7wuVShPp1-Yxp=@M8)d*NcX)oZ1T3eib1DglVb=w ze5n-g;ZIzo^RB{9mQIVl*8Yml_1(l7?{$OEOcM~A5~23a8oGt5PT_3Zq*tqCdf%>d z$Lq|)UmxzmnY2CK|8lm3F#oaBr$p#JPk`k1HMJk?U}9B`Qr`{5(urqJx7^Z9U`KUl z%HJj}siNWf+Lp=qj_HP3ZsBoexsjr2^3A%FR%$aHk@9y7lN&6RFyA1fV3a4|b_m-X8Ove+Zzh}w}o_uz>f7tpQi z#f6lqL0C5wbTVvZC$<{mv!bB|KUM!?K5$xPB##elDISz>pA%O$kv3&?<^^2BU7k5< z{$qIc2VM{*_7{75djn3TB6BSGDKxQPSn?<->9M%4vCO;l`)NE&F(uK6{L#TMii2HX}*N#=@Jokap z^h7eTKWW|=G|#-^;x0QL&-};I+8CCuq7j~SrIgIm;B!&>7XCo9;ao^vi7bLZ0C6X} zeC}J_B}YR=I;17SiaZdlBfi0%Ntb7nSf!O73MvI%qY(sn?E%4eXFU zdArt~V%~DX%=fyJp#BGX<~khF&l3m}3VkVtg2GrlL-kDZ;k=o{RWcVsS>2WfFU zNG~9v=Db)=d&tGT-6{Wg@ErzWUeE$O{}3?X@1*D1tibuSo^_oMsq48ifAoCovmMgF zzB5=YCz3X*7POgB`0JT5Bh3G8ty7d;(fc)i(DZ19quXDSCQ0`zV}$^s2>zGh^S|Q} z-Y(WM&YgTh^UQNt-rcey+#0xxI55p0T?N}z&C&oSY2*uOM3!EbpEg59Bl4;Z)P3^y zY~5-+SNBk0s`+!KYC+=u|Dg5FP~nP%sn_83D@2B?XdHW8n~wJ!B~u5b>1Y9a3VF`G z-)DukG$hEZ8v#d@*;4|egYvtsCIqm|Hy{4`pNS^9&ot57nEEM7_3FyfjK zc3PC8{ka^&(fc~&-%lXs@A7OWRFtXD$sWNv-M@=*;nfTds|%PM@Hnw5okZ}6*kmR; zzI_c#Z1OC}ysIEU>t`5y-yFv0lrE1#M$vvW)}4^vVac^}@&AF})C#aX1(?w&fx}g^G8e4#(b~PIq(&W2 zH6}(J^BC`QO!~CE?h;dj#HDiMr&LA6W~FxDFYD_{HGDp~>BQadMrKaS92b%4VkuPh zb(O~Y;1fr1ckReb+04nNSuU0nswPEbxwMR}8dtS;{i)tCab*qvq3XG>?Z=7C3d(Ey zLs@S8>Y4Hm4Z#yVBF2l9dha9U~6QA;M2e!O@q=wMUwoOOUy3hE3 z>$1)1JHzv*3azy}ZGG<=x_KNve`>dbR;Rn~UFq5?rHw!q1|3jb~cz z_)|)2wt0?N=ciTpE^S|VoHuf6d7O~XG}Rd?Yq$DO8UJdgd_v^;Q%Wv{Z50i*ao!@z zk;)Qw_X$sc!duFJC%cIjs8GOU#&Cl}#rZN7=Br2L+W;8RMYt@3qNPW1Rj$UjlO zHcrUDKz{i|52;qm{Sd!3lf=g_>P*ANDuxN?Y2r zw)?!RXw%vnu-+o6bYynafn5ou@#cPYDq>#_Q_<8BDRI%wms!y_W$^;FZS7MwEl?|; z@y**kOCn~U&e*9*Cy%XO)9HV8n60Kxfb=zEA>ry{>A#E=XkAh9!aDnkW9FQTWO(I^ zWcc{qLLxJhi+>yG6|mT-X~s^2ysX0)VfDzmpmcGEp1ii4!>NW{`-Y9(w21t!R8!|g zA7Awo@0Mphe2tnqiyx1uzj#SHFs&`eB~5so$l@S{bsL88Nl7o#y?sDRW`)ytO9Ssh z@se6wt+i_%q~<1WNwQA1y8Bu&qAqCYDml%B(lyzkl3E2yd|v0a;9{M@hAVYJr{edD zgr0bmZJ(IE>I;vH`7&OO={dW7Li@D41GTobhlnf*N?Wt4;`MwDy~S>Lk(-)a48Z907#=frNHT+VdxL7Z&Rqjx;Y@26? ziro>MJS8{iG82DUpmW-zpmcfFVe#G~C3*>^wf>2cOWO}ZqVzBMZ8=}Ij+KET@vW&qdnNt9@Z~@Oak+KGqCyf z{?qZPRWSIW5T!M{!;Zefttm%}*TCRs5MtKa&Ct8Sw!8yU@|w9aTbgD-8m*e*(RM>( zX4gJ|sinWdHtuo^ZjxC1*S=wIZ5qX;y&9vfaJ;@gn?~!*)$RBpE*-ezP>>RzXPS&* zleqM{amLt=yGLe0ywxS-wTD4Cd3jf{BH|3Y8oH|8C(3SoOx=UEr1D@WLUHT6yvyWV zdn!o((zUQmy0y9~spy#dxP;?-6q@`BM@+r;*qT3F>DuG@nNRzq7F-~!Ki{#$&D-m; zWWKa!xzAsOG2iV`I0?0%3s?uMNA%Lc!vfZ2$g8egaKT;CTKiV2w@C8ATd6|+Q8mI{ zN*>SF3Sre1%IMX)zjVH^ePyrK_Qtf!y;@th+&{j$v8hyNFy8fD(wO{xwZnfpyUi}0SH^rc7F^aR~a`a@|- zJa#oR`9;~tb$a}#aE}b$R^MY;w(Fapa5^|UopA-pUgB2byxzMO*oHHP1Km5s1rXuq z0PqO?LZ|zjeh1?^==SV^<~f#h*D8tp80Bj^_l^w0``_^byC+8;g(S8DVkVbSDN3_*n zmRHB`t3)Mdt>XuyE=?E)U{Q|kFj!r)%?W=mai3IAQBJP>`oW0;C6`(UEpmH@-2l?< z&(oV7YCJkGT(&j6(eYAXD@=~evL*c3T9{n+^#7g7gZ)FAlVpC;bC#ORU;mb$#qY#w0 za?R=Azi?x|hA`jYz1v^2og6%#UueqBa{nvL+ja9!!0)blFlpzu)Fb9)7GnceuiH-4 z6?RoU2Lh#g7qML17jeb%h^>S@Ft|833a}$+dtt!(74asEKk> zyQEg-%FrsONw-2JPt_HAU>9|TF14PSals|>7h!R{IX24_G-3}js5KvKDf(C6B}q!- zYMPS3mtnZIn?Lf`;2! z;b6P{U0yoZ(eLk@ySrDTz#J_Vq2iZS;u=M zK%H)JbNC0Mjhcp;J{q$|Q&K*gtqG9_lM!l?OE;&KuJIb81VpW##+^J$W;qu=N01=4BgAi|w(?!cUG)VW&#RG? z{T|8&O?cgljZhkgF%aPdq0?y}0z$s{6o^}`BJ9($V!byJ2&@d+h5*sYw_Yng0D?m0 zKv2O22gt)XElV9rr~fvSxH_~W-$Z5xOq{#Qui1A2UBxhy!G;-bsNYh14!S*R2N(z+O*&T zF?GBf`MbhOK_dSh^KsBac>42n5}J5!i(#&o*Ln*UaRtcihxTm-bNJB1vl(H7+AYqF zgGV8-3m8X7kRyN6XU)PcLvNqBI9yk5OA^rTJJ0+!-*?eiJ$9{Gsyw8hFz%} z^fDZp94pa%x?>{(PIlcPtt%z~a^-fr#?4@B18z3Vjbf|TCv?!whH+E($C9N%Q}-xP zWrX#aGF@`b{-cTgmR45 zIS6AfYySX z6OEH+nJa7;j(9F@kSon65Jv%oxF&A`hy$^JC_qrKl{PI%^AztyDz{@1&^k$39LfCF zen+4H=Uw>@_8DL3!Unc|Ns7|s%u_F9X?zGGrv^blnPqT#d}FnABB zJFz8mGB}iiDqLBLim?H$Z(GRRym}UMpVx2aQFg zpW6Eqp2kr8-|S7S7_m)ZmJ7Cu>RXby0iz4>I!2nWdi4ak*K*9gRQ5lZn-cU#=Kdqb z`(tzeQ_i*?t?sZTS1a4dzawy6M$)P=%FzJR)&FkoPpAqig0}iksfw9@)JW}IA|_jj zkuE*zkqaiVH#w$efS)31vv7Nrb1=E}mITV*GepIo1+iiPJP8ih1&5;|l)NiyX)u?b zrpQd;#F(bfyFj!dc$-*qYjh_;Rr3h}P> zFL+%ZKyVt%Ly4{VyQm0}UcpoU$#BUbJpegGa1CVbFmC;K3P|2o?;gJ=&Faag)^3F$ zA!INjj+{cs_N3A)hxbK)wDSo6Z1D*`&UN!cdMcbsC_oyJt;Kuto%tj_-O%lE?qiS> zDOL$!xsJxIgW^5Lhrk7=QOSFX zvjNs$qTU)ShZNZw;enz^kqRLx1UFWBBVEyG)TmdUxyptGWY>X^|1+WubXs`9Tp0Q4 z14KjOW;K3A1eNv5=!T)$6S0P>e`+vUXKSuFyEzFwwK1)-1~uCPhqx*N)c?C03`ZK7 zJBO$2CIcY8ThCMJPbCqNaza!z*6N#zW2vt;p!HR#MJPqq4{Ci98f%gLLB>;dL+!O6 z9DM-HowTlO`*$#p+SOF_8z?!ITOuX2zMw`&!N&$C{NycsKW}i9SY&W`95^|x5gxb& zgAZ^BGI&9ZSalS~;6?wz;1M4e{O`ONm*4?4G+9W3Lo*mD1vqihEdfLy7923lHS-5k zYflOeLKn4D@q)Yk$;pkK4mBo;zwrdw-$Xd4kNQD`CzOr^FaIFI>w?_l_oi9pfF$WamD=%*sWO`L1)l>C3GxOwm#`6!ZgmN;QHI;J)lOc2bx#Y$l`m^oc= z5;hei!_Z2l5(%9(DvqcGvnn&=S4m)(73CV~v~=A7zK&x&nr4XsZ#L zQn^36^{GzfSzK97lvI_-e>c7 zzrBg5vPS)9^0(KjF^r|Q^J0tEHfrgWcKtla9cqQ$^x{etSxb+Ho`WHr{D_|a_vX2K zXIo~^hEzGAdd7bV@{jq0!>Vhk-X9>_KUQO{>D}wrC*IZv>Y4O|x}w8DW2B@1+s0aX zjLa{ssDUvr8R(`i^;0aHNMzpU9ccV^3`{8S3a9{YDl$P8JWwHo3-qu?s_kHfaFY7x z+|o!_MT)gF|Mx^57u<74`LDSiS(HXH=jWcdEUD*{dkU>=YyMXXSn(B~&F4XrEP(c_ z3g1;I|Nn@4^FXTh?{7S}kTD|DMNt_u6v~(+LddM>5NRIo7qVDwle4p>{d7j@t?{oHh@Ai7V*LtnjTJL@C-NX>ehZEQi z=9nAWQQg8sZj46(f~Z+bTYCp>85)dQpfUTlVY8rj-AXzfXE38-guJ#O99{9io-$UA zpwG(vaYhWVC*M)@!cCRA{=E{8Q6TI^68^cM7G2+?J{g?|IerBhnz)vWY z7$#c4w3T_dd3R$%lFPYR?h$Q0ws0j}tNJU+xo#C5<1N&3U()bfqvOPFr?)^?b zeiHqftBzxYUoS^h^o+NJnU$l@_t88L!wJ0b!v^Uxm%8dX-fZEYRMlrZ!2b!+8HNg& zhx!6agG%RGpLFr{>%X~48Bl>n6|q71A^Vx!x$d&1@~!2wN#cn=U6L2@{1txiJ~SX0 zG%z@-f-bIRzVZ%!Oaw#&SD>wOg`Z=8-bTCo!0fZlsa~FDdj>u+wM#~Ksl2^OIl|W{ z(wOl31q})8B2Ctepi=k=^FXQS_Dk{*88kZ+ZG1l$J_nz?TfRk(e%<~3=a~rn=@ta$ zdddS57J=jPVhBxG=I=lI>K$JG?(M6G-GyREdhDRmtSC#e>04pRwm;!E(l8% z6Ff9jSa6peerf8Co=CFYy-~#kzq~W9$Z@qVyj_5v4<2LPh6Zpfod{1vpH6aUPVmmc zrk54AzOvTs`L8T}Gp)?7;zhUnj&5t4MSdA7PWd&3-2a0qqhOF(FjsOf{lM*Dm4UoZehpC9wUzDwn5s|ku~Gw0gz55TBX1OAeV*K(ds z(%3Lf)ebu53a_DJUcJC8)T;?e5AV~$IChh&YRBCJTlC+6?^*;_jU;tI8!xG#z|KG2vLF|lx>YRA;=7w5_<1}zmn0NZt)G}`BU%(igv)#>;h5+`*_2*rE!`91ooEj+A{N5kj2OU5X7})1;MYk!!bzSK7HilPG_2v;9sp7AoEvG7nU>@6Ue;S*`n%YE?!$ToF}g#%ey=Jmpj>zxbuw zcj=^WsSV|wrMmEsQB_~JeO>bX8t1#P^J}l~*SwL_yH+Ycxo*yDJCiQcEw^}dE^KzG zF2CN*fTRDU;$XCyEq7ad=v;TCb-R(tOsM%~r(DTHw#_Z2OGoEi6wAeY&IKJ$C$^nU zy;+n++*xs0;Y)ATTF`ylEC%AOMAom%gRA{t7bS?yDP_m1M87mut!4UtiLvZT!70vo z%vOEr_5ECbpDLX}>PDiM_o^8Dmo>{VhpAh6TIb#_r#koKbPpAo-xGa3t8Y-TEuYTC zWmfU)yQ(jl_ivmP@f7VX7yB}O>Fex_PvBLawx4IqKdn@))$bI1?|$j&6Wr?+8TF2? z>}shppMD?w@h{U?uc>|=_1LdPhnJa`l^4s%CgV`(dDq7|>@K!h($n~M;j;<%89#f% z0AaLFQtC+R5_WOs2(BkH)Yk6W$Q_D78=6+-lzLgIlCdJDEYXo{GFX<1-8D5>%>I*f zqMjG&b3#4ruGvJ{m}gQ16OLU?;Mk?7EH5^aU9D1P7QFUIn~s5a^NLxczH+bV+8ROV zmlav5cn-s+3Jgxgv8aWtYN@!y$tT_Np?lr_F!mb?a|63xmZw&tU$! z1V%+=nN^Pi9-F8vgtSKvU2PW(uuHNg0>AQ1j5XoPa%J>M z0nNEp-I>tO{xwp|ifd;wKTqvyliFt6yf3|qU+4w}Zn)7$$)sRC{UM&{J{*QnS{ zkJecp9pYUpN1qlcqnpOA7DVfm1Pj~3RiX4ZZlSYMPv7^BizQz=+&r#|YXQLtWI_AtH6u{_*5V3B<1;zY-rSX{OR(9mYychS}ETA&4D zK6?#{rp00+P%6y)q%~lqXai!^=JU6AeCYD_QHTNJK4$R})Vm%Uc!_7OW7Az;OJTju z$FRGfk@R*&7esKF+=bvy76I(1GjixEJ!N6^Px3|QIvmU7t#>Ee+Bs?&+X?V7zjRcS z(4u=OzAqztDDN{Z%B8?pUa=f!3W>s6`;)H){wfKl$F$&5ysh1JyQ5@`ih|AhaI+b0 zbm!QzSe+wj_l>m#!2H`-%zO?N`LR{;gD2@2vU$*VPZ$G22|i|0$yY8!y;8G*+e`S^ z-%qEokxq-rps-lwh-^nuJBxnh?VL}PVP0m(uKMW_Eyygt@+Of$>+cf7yKi*g#0G^W3p`$4vpog5wEVvj#MXs_M4ZxWNrI?whgDV%L zF08o4-1N@?(Sz74Bqs$b$0Bc|&}WY9{$cTBrP1ITHnsMSO>_#Z$Tii@pkNHGz7n8g z9GuJ~$4Bevfa5gB~T#gvC;xmrIyDw`5uG*;v4fF|AGzCi1q&~J~ zW1&f%&j!|8L>#0mTteq5db?``E&}b*C_BQUYHRygWp2gtT*N`ftLDO~^MbG6&u3Qx z<$|-&c;EPHvSaHTR#WzLt+WoEs}OF{!bsiMAYJ1$208me)|IVeG1O9Jm z0_>m6!hH;Py~cl zx+yQ_HK`tTHotCru;|Kcz)u|Xwj-4k-QxRTY^Sx@za&)xSvPGj4hp2>jb$C0q8KX8 zua>;t={{ud$cQVWF~5F;Fr9^N99xD)AG3vkx6D(qI%Ae_g>HD(M<_XWu#r!9*xnK9 zVMF)X6fKk?eehE{Rnt|vj8~ByKu$@{w#KV;c#BgoW#_R{|-t1yHo_E>s;^|2sqJaj_?X1KRUg3ZUo zQNw$gS9{OzP7cT)>X_7^3mB#X7GPGvcA&n5e;rz_^$a@ik{(#|aSPpM+aY`Lo3E1| z$|uAs@6JqpEU)Ael{GIFpVBQoCLnT>L6BGI1rw-$iW{PjawA;pI7Si|z@YJt1bQpyNKhQ*@UCl2k6B*3g+goJQxNddqSmlfd9T^BNH$w*q}kC z^;YmjwPfP}jd^YMH>sdaLsNFwp~Z?-{vi)t`*l(MZNjbVK;{?}dQl??fE31kdTeyS zf}wzdk!jP8^x}42i|WxI4E(^uKLa3Y(A8}SL6Oq_WN9hmoALNz|DT|@PtPA>1}*A# zf^6r-vK?zfI!G!&QlvWQIj*VEA*BMJfKs*HOqo7|DYm*+d+=MBXIkx6qKr#rA2LIB2*)k z@6N_BGe0NvqFs<+do(#T5>!YFXnHn985`p;McM}Os1<~oEc8Nu5dX~u-wgaK0C17T zh7?@XIhCm0q_sQK)YijqOjMU>FRhY)`DGp`6IA34HIPE0vjsX55E1A|5JF|79uXWu zFTz9Jke}+Fta|{&Zk&_4`v*P#TRG@!a27c8O^^>I%Ldg*LOn=S9^$Pt^Pms70YQL| z(2l-g8HjL;a0~P$$sO1Q>46_PsaXXV(GAi91zND(0ZcKuu9bh}44HzWh?S)Oh9o5@ z4}If7k&-KYfGvHI60*wd1y=dS38~0XcIgXb=)Q5H@+?cgRPxI@RDk)wPvWgS*hq>B zst`lWuq)_f4XNyhh8zhHRE16%X8gbEB6RFo&B=}%q9hyiHm}tFy-Pl<>bod;T}7Z6 zzq>+L-ws>1p7I77z6$gymrYSeev4$II^zi9EEHs<|7+HBrLX=P0}NOXMrg8QZ~U$T^074~?R@BXlKoGaF* z*F0skE}vl)T|iKG@v46_b6ZxjweY-$`6_kI;7->uv?F|L%#8ja)70|GmQZuuPb;bP zxHNd+x*GEBlFxH**#m}!FONlDF_W)-=*@3-US<3(d(iNS%UNXRlnq~~pMK1f?qF*P zJ-_E(eqC}GT&7&$Ek>B0@R$v&`&f4+F?y}?(~hw4@ag5QK{33^JC6cs>$YQBw{v%7 z>MN%Wc6af2D?gr4oSI0T5NuPJI*KBICdT%9e?Pv(I0tBm8ZC*i8*lw=buE!!WJZd97pXhuco2>^f^M3TEg==%gdXFOA{kz7n?;SK0((9c8;IX z(4Sg1_S#QK(x@(BO{K}{7>9#vc2_!$&+8L3woM4AvKk2b-?ns7icFl<4HM1Mv*)ZE z`zZ&{Mywn26RP;Rf(<7x+Wq2{J>BjW+dp8-6ZXQ>g{(EB>~&WvO$eLGTn;5GU9yyz z*R|lazVc}@UbjnQvQ*QSE2u^0^yp?)?AgT%?8WH$H8`vFdK#^Zy>al+|av| zbzz$h(P9B1IEok)>*O&6+~YzpUK^MC*in~4{nn|7*e3ol{+?2aO|(eU z#I9X>@AJDw1t3~xRAh|G_GZ*%j4P!cBL1O)^!8@7WQ<;k9#a0H1YX|?xBbCvMxma; z`g^TDtXS$YkacS1ER0#|D5RO(7R}-?TTs#3VnCV_^baixRZ3?N?GmMkG)X;GmXsqX zwZoqEdd`U=Zrp{HVLPV#7Vl+5Ze9y6O!v6xA4;$C#dFG%VgKWiPDW7#h8u%%b<45I zha=?MGsjX#V~o{~q-B+}r}*sC!Lrs8xOH0ghIP+^WEBf7YiQvGF}gdPjwk z-J>O7*RES)0|c9ah8FrKK?64}rz|I;awN{c{8b@lnexd?HFyYO22=u+gdK%OGl6mTd7s(S- zTMRr;)xEAu(%Z@k%hqTCENljLVAl_rA&B5%Aj)tK2WQz;6AN9G7|)+*$9Q3qkE7)c z?2^PCk@SaA2Qb8n3Rnh7X|XZ@D_~~B4IV(tuQ|xQ!ruW<>@Z(IC${Axo76*=Q#FR} zuFgw#fRjsa2XGjV#w@elEncN=v!6=&8!x<Ot*zxy=(qp!}QD844ROh7Oi%2 z$PfU)Tre7wK@&{itpcOL7E&i!GxX?cRxA@mvk2;jzU^K2}|N!1U}LXl7onN+{_3Kd>PAX zdlUsBor+QJfk(`U;PRNqh-)kZk;qa>6CC)^#K7)oy0779(Ih+SXASO3k$wdqaXNLz zS9SBKBDb3imnCWNoPo2i7s7-G$qnspqHFZ@hs&;wzbw=JEV}rjDdR-#bLZ*ta;@_x zXpA#MnLh{2)EoniaC9XElg72anoe6Ffk}xh4oEOGE~(qzF5=C~y#Ij_UKut}Jg>9oITFG&7?*Ts9U}rsWMJ?CbQn-+Ye+ zug!FkkXz%f1epj{S{Y0bqMjC!GYO)WL4YXMX3PVwV%=DNsVt7P{BhFqO)5ysA0Bs! z}ihTTegU2NGrnsEtB#0$1V<1Tl3 ztFqXCwb_~%klPM{K;z^n!bZo{Rntl2U6cn@H-H0K?8(6vMNY1jhT9f)5l#UX*YF${ zSHd`tA#q%?^FvPlnFPEYW)k8b%b3GlW7xD6f{rw zvZlZ?t;a+>HI@b7{J(nuxHx(mxR{n{`bl&bM~xkd_7_7SHRu3F^97(e$iSaTC~c@% z@6>WLSTudcFGo42t&^COYmlDXZji6VCzd{w@>Sl^>fp!tGTEK&ui8+cReXlZp!WPn z9((|idT3HAo4&m8acn^tmw*Q`Ajskt?f@HNQLO~hNo8w5IGAg`r$ip)Uy_}M7R=;` z0%N>;o}R~ecpKxPWNY$sF*{WOOGE(FPXINcHeqyp2z0!0*eekocoI~awjNVHl?P;N ztIDxcMN*Js?GFiQC~@?g7c^`T^Qeaq^KdOCIqb--#zB_pfgN04>>d3G!f*}f+uABXG%3sD~11*-GTvdb|5VALVnrX9DV5_Jio z;fFM#9K0Q)V=cIE8ecI4Z9-@*%+1S(wjVmP!35VYfw}YfLWAT6xaw1%p&Q9HuB6HN zl#}25#_>l%D9VoO)c`>ZGKkD&0vaHu`?kygfqCS}=M#{<%d8{(1i5xIajOZ1}sp*B#f#EL{bH zavk&x@!DD3Xf&qFUmib04_pv3=7-Ycl&_t|Tr+Hzppz}t046#kV&Nls9IkD?@AmT5 zb7vp<`g4#H@hui9%6HADI;lVzFCPuM5%|)Cvvr7f(K%E?_RLdy`^#$ert__H`?QIU zcM~0oKpmhqB(x6+t>uCdV<~d0Qe^QNvWUDf?g-TZn?fLrr#zo+ph(uU6IvQ~=}dM{ z?aOs8>%qkF_GF3X`4gcMa%a-eBIU6rvZxj{y9HcW6Ion~HIXH?sEOovkA^1F$jkL8 zBt}BN1)@b?E)!4^*|b|gHjyt@>5YjNk#HIGN=xJZP$ke+MXIHcDg?_kbWrg#AX6L_ zFT6Y20n0ys!WkBKu-~xAnEx1UUMVDivP}~%fuZ1jT$wBnv2k^$z)d0ytgh%PK$;tnJ7*w;%0vm= zSRLISdIh+>6l~!Ei&jnpfINz=Kfn?5(oHOK=@}?y1Yqb~W|ckY6G4{e?FHNEaTm}B zjBm-4ziuW_{p8BRB?&0|A=G0PyBN(n0J@f{tMFMi^W=@#oJ>0~!+` z02Ayh5DwT?fSZa*-NHB+7dJ)g)_ZRZZbLoef8_*F2)lf3XrG~Mp_UP?h&c>nT;~hm z{Wl+Yo$mPj?z*8d^rgjv3&^XxD|e5?C05r4a%gl!`8BF@yiv3lJ=Y zmcCA^?@0mfZ>vHCH#8Q%hnQz12KNrhUq&S1Hd@b#X8oevxZVjsBAb>2{=vbYzD>YG z&#(ks$a{s7LK~ML%3%$BW~>ywRAL81cIBatc=*0Zn{pssB&D zT)umO)G{F7W}MxJd|Q1ABEC^9HsXMTeI_Kv(ey3>bX9hC9|$FEAe`2n0BzGntY+iB z3&Vft9P>yb=>(;*8%5%q@ko;jbo1KWN+(y?S>__{kuu>2al&=(*p0vu`vFg%+qcHa z`G2!{1Ytr6fkc1V7;cA$?h$daV;tRk*3cZeC&&=x0LsWY1FrFrTb^(MJyyi=sHM}d zP)$5uVLbX|y?b17dGZ)m4*Q3xPGSnrVk&vA&4E*K}iWb zM4c2=Ev#pC#%xRM1Y|E+gHMFR0dp#|+JGBQn3O{;|tPO=m4i+TLJ zLTBcZobW?hf?R)xK*$kRMcUm~1}ag?h3zDvhulxtY{;@iUolsP1QguMjt!!QckP42&V0rnD|W(n|Q z0d;D0-}iGwXrO3-d#Gvn7dMa;T96He{F@u7^47BwbX?&bmNOhEX9T67d=Su{5{tw? zcmWj&Yy(LeeH^eDAVP{-B0i$0PTv}aQdxMsWscP&k~rm=A*K!eH7}W*c#gyLavhwLO@oab9y$hstQK?(W>r18h3m?2bzLB#(yVW4 z|FxTjh5;ItP&ObWD1+g)lu#s7hcMyc#)JnNCnOepuw`(;p7Nll901lhTWTEeLlX%f z@R$++_46IZHusa*j>Z=Ckl@LG%L(t$!=B%^6cAIsjg3`-n-V;i?1BJ8nV13JtZyxm zopc5~tO~UZ=uD9+auh( zf)V$dMuxxxg;R>QF#}ses1)F%&hX?Y6SbreoTE}Ljmdo%`~cIu~>DY#`fC*j-=;42r@iHm{F zpyw`vY*+@KeE=Tc=ps=N^tN_;p+@hASQuiA%#g*Ddf);oUphMJcf@IW4Q5!geIV_GMR!C5M_sE0Es`(lViJq$CLiy0(6bRk+`MB%}-=?6+` zK^uut1U&;4?Jes6x;*oR*nPZdrI5~ywdz=}`IlkE)$XtJhkVz3zfAb9wtOA^x}vzc zP_^p&b?WPK+*e}Nmz^-Ij(%Ma)MGc1*lcZOc6I`EaHvS{bdhd!WSpI-ZT&#rY2UL5jTQTApX{xWS>{Go2` zljdS>)L_k_C988Ex;I(TnD^ga2s5{e6Y+NU4ek^Le6=}?onzpS6pbeR^5ejWJ7vwOT;do+SMgA*PV$j_^{G|%_SXHD^7BZ z*qPQF{;}Zq9ciXIZn}%&{eCs)3U~F(o@?KAvA4#BhWh+RB?5Us=qV+FYlC?Ewl)!g z_?wi(F2UL`8MWf6`sh>YD>`k5Yg+S^18*M~%rrex;hU@LBtOxL@{{_NfcRF&22a@EwPQ1N#=pi7d=gCr%tM)@C=_ z=?4H<2VjIZAiB6rL|F9m0O+irW!qZw^a(yM=6K<7EYHhq3(;+Gx;NQ9YVaaj;}|36 z40?8!ZgeP+be|Q2X1N5=-XEHHYhF=w?V;5dWPj_(OuNV~&+|)j%1@2+_w71kL)(_8 zA6WPMu71B1_eBrJG}BDl7e2@fFRr1H3NY{wCv%&z!qG5agNSCawnyIN1cs$pUd#`G zcB`4P{$W`&+&bgC*6aJ_#qg}jS5NaTp;UhG-b&Adp`w0uTY@VU^jpF8D!YVV6h`rSp-EE%Rjl2Q&dXz`{d z(GFy-o9!$haPXgmI>76=UCM$WM#Gs3n&7|wX>`V8FMEID9pz%@G>DNx;%+U7(e|T~ z1gVhN^i1Mn+N!rAbAv7Tgki5%aZ;a(dV8M!!SoUlbZ`tE#QK~#(JmsOQ4CXSlw(|~ z8pU4G4qQ%SoeDs;t#YF%5*o$F0gQz_Fomlv(g7|Y)Abn#`^Sh+5TudFnf58FSCYD& zDt8=|hNO3^VKCr(X~=IGrkRrIz4tOqbs`*0W5Gj&i?%>qd{=+F3r$GdR3co!xh!%V zPPLl4rx6zyH_y}WPVaTuKjiOC+l>*hb56#F_ReIwVKPSh`W3I5o4@xurkFv3G_b;rAiXG_O@;dAbD4}CDWzIp*(t1 zNQ#V^rnI2etL$MoFArXsvg&P~C72w-X*xa^Kl2q#*ED~IYz^ktAw+N5`VPKN+`$nK z;xsoM`z(k??7P3mFgl_9&j7SF+AaM=QQq)qz*6DO=&Cna5~%0_ zfI4HoQjZM3N;w7iDuL>ncw#v3025?`{`sGFHJr?rAY(}H?TFd5bxuR?sqk}roxg%R z#zTXrUV4CF$1DeWQ*@b>hCbnD0uZtPSu-M{)pDuLa1ets0{I*PiCm#&C@oh?3jgV#9MD%-F zX?k5ee@|sEQ=5xifdmJZ%^i!9#7_=%HO(@!{T#=(3Amy})b=js{{?dp z%ERU@M5B=p2=qONDZJF}QZ3$ID;=TK?dF2|(ZkX*L~cdXJhl{4i% zjOf+H#r)^=Ltwq=1&?r$PAqRP)q2Du8kS1X#9IFrqM|BEU(! zgGZb~9zY}Zy{7!PPY&cVkE0#mLry!!Gk$hB*>Hz}cQ8*E4_14_+b{(n>A*KNn6&G^ zvY(7Oz4yxgYQZ3%6G$2c2v_zG6-;4?fctwCfGNAeMxVNS9fd?V4*wwdqdvew?~w+e z9(xhEj9yTmBcO>NLGyWd(89A$g9c@CY80c~r%w8A6~cfNz~&Y!leUJj+xTZVzfIR^ zaO(1{?oL9s#9w^CtuwB<35vrftT@C${B?aafSLmR84!$U2-aWG=-GZDE=E4Fta2spgO(21o?a&v<%v>BHp5&!R4iWdGwozA%;Z+8Og*@0j>qqKSpiMK z_h3K;gRDR!s=C_0b|8MCQMbj!>p=!o|7-7~@eaK-RDl(d(u_ z;se$!As;}Ig5($@LxN25L^%tS14I`(N2pJK6(^wFUp#>dLQva9#5gi$R2ERV(1tP% z+<>fb(0B+=+EnLr^D z?0<&D;rpols*?Z;17-7a%yoVGwR+|8@hVaoi=(9G90bI}|NZ`foO!(S*WwWf-Kj&o_>JHNIktiv9R4+i}ng+4kj z6UAHv+vZc%*v|C@Ax-|6$1k?5PzhJcx3XMTY7K`M=I55&l9|7g$oj_tQE@wXVZ)3nR{9I2Q{>I2|T#u_pclh*dgQz%qwZ$*COHZF4{Zgsc<@PJQB3Nu% z*1AkU@o8?^?*fV!Wxzp9IgQ@E<#(esUk0ud$5os;dAz##MbRMN5ZAud#E9F!(;E3G4W;p) zS|$&;e)sGYD>L7Xv$NfTaj*B9t&s<`=b!z|%1j^Xw_vW*L74sNp?++lOXW|rVcdGi zRv4h5V`G!js@ysmt(^MCdj}hv!n8L#OkJ*|pNz)yHM9DGGa8NkPKGt&XF`7wWQz_G zpXH{F>UDA5+v88hOkw?yX0t&2{FYvE>2qvhTUNxc)$C!s@N`qi>%BM{K7>G+LPvTZ z^`a%J5nA1T>aF3oy8V6yvb-%iYZ<7d`*7!VD?%)v84aIm>YFPf(E#C!NRN1qqlft( zuLGN$nfSQKHx~-JKeMlf*P1-FuZCBM`_0Und|2Eab<)__?~ z)gB(&jAAlEt1Z(ZI%M+U$k4oSPa+xe#2Wh%-HAZ^>NFPxqKn9t-^X~;?W=En=42-K zlH3IEei%rr-k)Fe#;cq~{%pj~wxTy=@NSHO%xZ{LJf3_T+iq0pnmCSFoTVs}x>lop@*(cJejvaw$6y4~C1v4iir)NMLo|MPZnFST!|Nc_ zsC)6#M8F2G+=P`~;79==o2}WH$yW|aXJp$V)9nFN$E!E1e&zvJS{pOLb(>WY3?^p* z_Y8D^VDM$+MxE&PD<;99dEcTF}z~F$5eI`dLuCTpy(2DykSW&nweq*joO}-C##cauiu@~ z7Z=prKA3q{Z#^UvE6keIK`Jn%0~^c|7I~fdw07VGy+Rs_4CDy>8*!XgIP&{Vzjkm) zYJCEpTuzldWF~YMcwwCy-+eJs#IpLl+iz#oii_*;3Nvj3Lo_leP5gYAJiMm*IfBG@ z1MVvjv`$b2jFaW+eMsF2xKR=i17l$tV&Q!Z1ia!5la%LgOmX!b3M25~m^oOkgxd+r+4%cxBK zw(O@4%*?*!3~o?&?Th>P^0)Tgi{#-cziW5<-FPK=Ec7a-6=|}~;#yivD}FbKHSaqw z-_rF7zKTgj5344;RdA7fGZ5lgbi zGabc$1LFbeya-Q*oBGx2G{@ygG*z1dc0fBjSZOud!*7AB2t;nV6)Z z7O>oJY4&%2n5X7uQBNU;fZj6 zf!ecs(1OE{AxK3h#xapVhB;aGc3xJ6{v}&1npuI^!%Ewj9K;M}*eSp{@fM&F$d0b# zBq+sgxkd8((f41SGmiZu_Z=rNA^(;{V+H~NYIy^^Q@$8q_TK9lyxksi!}$mv@o=>K zws3^y`dwsFb35Evm%iu(;rJH8cptE@J@ilj;Gv}0{Zj;qqZFamm896#-G#G!Q{7z~lT4Eol!0Tb=?ac(EVQ3nZcr zxCVIvw?KTzCueF8K+8{EWylAO%)Rjr(g$8~X@V-pc_3++k5)Mgt){UU>x!T6wTxV0 zU|Fq^3R^U?D^0}6SgnR4!FL00Dgbrka2Xr3b`{2yQxV`QmAs#KnonZ-V2%n0kieRg z8C-QsxBdNbHaWxy#adK2RvE0|w#aw!TZ;ZzQ0f7fh2!)v*BASA6j+$qS0lS1*I*E;2L6^ak_Sl3 z2suC}UgiTCgcZ5l!M$!Vc>I>m%Sqzm*TDs8kQ7B{YbCY3 zOh7{6&q9>QPS+1!aNV1AYjeP_?8v10Ngb}y2#fFbe@$__(E_L2^Wpxecte`NQ?O5y z(oiYp3L)*sc!tl{lK~arRK1IomX8}J6y;HvgGzCq%iS_IPzC>B_Z)y)1V??XSejsZG|w}tOeAA|f=3l5)7lN6CDf_3OH_n@rMQhp2v zXQ812FGgo%`~jZJKvv08SMS`1DV$b3G8ZeYDA&9n=zx5+ECsf25yyuCJF6pBmuF&Lq( zJS;~Kd5Ka4+Dgb8(1v&5npOnC8A?V3tg=xYg@O~r0oM1s4KucKZg(HIz8kNk_Pzd! z0^?Bg6h!P5#!D!i-~rA#6Pf`ySCWSsIv^YDJl2UxNlm#M@j`PuWSc&e3~x8cn7@0X z`@1KmpL)|HVxGlgmOy-Fd>z5*4cic?zJME`-0dKka3J750*w13JzQEhQr2 zdIRwsV;~{v3RmId(RkEhUw6*f*4r*$;H0&7NOQ@%a(23=&t>n*V{oTG zz~{>_pXoXG`eb&xS!_0$2m2?wlk&dqeXWi7?!>k0a8;vnQ@nfq#R7uUGp^A*o#^v@ zmj|fLJ>h;2A_duH-rbrrV%vl6zT@T#B3w^xUWFZHsKAw74H*a!r%9{WRSZZs21rt3 zFLt)^<;(agIr=668WP$TxTqvG24~y)rMmyJ%}9oIbJ@EGF)9Uf7eubusaDANy2l@y z!_f>VmdH80vUD8|lWR7K@%KQ;KBSMrk*UQ245zqS{N?)7!e!5f=ix>H`Q{kx_7vnQ zl~k`~_&7KAgO9-KQrZ~YAg(kgW^P*Aex!`0+}9CYsOv3_JXxBHO33~~ioOFjkDbJlKk%@~q}k=6do1C1|* z+`*OSLOUZRrc0V~c$r5KJ8Gif&jkZl{&wMlmj;O*>zgK^X+dzow5;$(4 zA;)|f?q{Q_*oJXLMo<<4ZY?QEvWYEQSz@{LMtn{Re87@8H{wG3NSH@Kn(w`!m|$ zYokxgu83;lBFGCyl7vj~yiuSJ5%c_-8r)Vc*5f3WYcaX8GBdXdt-o1$8TPjJmC1*W zMjXnf(-OHqI%%LWW8ixp^gXH{Z$R-mYLSSD<4puo6&hvy@*d;&9cj$ADA-mO_Flt~ zM6)DZuCFj*7m~S&pzNSp!AOw8JO#V%CQWrD4N`@6HqD4Ife6@~HS zO$2aQ)NH<`n#ALVgeu51E0k#t#W`C8{yo(~pAAozp8dZjM~3+d-P@*bDNAgVavtD1 zM9O{BWbqcHJeBiUw(I|p`yRp^QLr#@-w!T2EgYx1gj^HiLxh_ZTDz^O?d$_Dd~=b6 zq$GAIK=b7dG z6-(tDo?>AuIzTmt0yT7^1xvCY!d7VHcJl?wqaBq#5HJs6%}|nXPQ&Db$Yk?n;A1&E zhH1%eM1hoScr5MLtzh|YN|X45Ac0K9A4=`{43gOeOajWo;l59BFVzQlOnZLJe*1G8 zs+d)Z8<-frb3(%Wqe6lp8ArzBktg!hdf?-H(>{$>ZQtE~>wW?%p_RvNDAFfzmpv3t z5)>ZfJ$q~8$yLmY6`!#~o5+a^ zYsQ?}5r-WH8kNFl*)x=JeVzX!GKCkNeFAOLw?uElkU8tQfW9}`VC;^475QW2pnF2j zIW3{Ayozzlaug0h943lJzx;t}F4idZ|G!lxVG)AwpOp#8z(AO%Gl3-x6`!S(;HSF( z+sZTu`iChKs;_;WjUHIN3C40!7n1>dYkf>D<8)b&`-=ZttrC*zKlLT~M$Ww=7WhOrj=4q<*quq z_laPAfP>MFABzfT^bcxjf9@X`4zwRjRA3D=GFoecv>9z0JHCtw-bT}n(WdOt8;rs8 z&VLd~n;)E1Vl1((Vo@Gb2ahc1SxUVAM*3nO^a9>DO&tUE0c=Q)S*9CK>f zf2(0aP2Zn7IIp6QI)4yn(S1VvNq&Nk3Dcr)elnsX2_Tl)NGwURh-ngZ!B4TnNlYzA zp{>LYY4e=m)BQHqd6yZg{J0qypbq~*E=^8LivA50YAesTkf%l(VpYXMV)-uCWW^ig?JwKD3vHuJT8@XHI|l^I0|Aq*QEdgBy=|XyegV zYiwXXp8ar4#dMoFO@m%bfJ*Ki?#ix5>4~(VEU=cDb1vEEYg43Dx0wrn(mNSs{&kUO zY*vH&p6B6x4Y3O!B8&Ge_@9c{Juwt>U`=xOyk<`17`o>Z;4x3e)vQ4IkJjP?|vbY{}eA8z)p;wJ)_M$rtj7u9F#I_yg& zCWhlq#}#qjp|H5v<#fbdsmWyeWybWyCKIr3VGmOrAM#GW0PDrpB9!wC9S?@-r(%`8 z3h!-h+cFqeA1| z$vM|RdxU31H@lDRB25+dZw-IY?oZW|i3y56p6(>x;@;h@tS7_1F!h-MtaR>@-^KT@lg%h{-(s=X4)CZMBBFQ zjg%|2eNb$I_>!kvs$524amj#Qt!jKAi}Mbl_>d4~ng;=mWP?9P%1NeuY=wD)!yJ*D z+Xf{$BCC?bs;{mIdOjE08CB0=`3n%Py)jgXGWGMrP9e(Ok(Y`Zs>g!*?IPtQv_51x ziO1&vAu`mX45k}jp1O@$ZOvLG7kT}`LjkC#mpEYUp8LDi-I0gb<=C>z|LCb!Q`dU-#d3iE3h+U@ z6D+r-gtFlzSrG+2wY*=D+A6Q)k3IqgUHJaBh-bOO6Mq zp7v4Hit;HJ@qts2n^v%>mG8c&6Tx?Nzd4rHeK4h=+U7+%SaVzw$?1!+^*bJUeaPUZ z5T%3`A431R)oM8~5>GP~Atz(jEGCAhid$+9pmhT_3T4P3ELU?iIr zI0jcZRGo|5`Lfp{IO}7U;|U@D+N*z zy#!pFUDYy>Nir0=bYQVSBue4w^DL+H88NJe^nvH;V0~)CuFtepHF6}}tOof0;>;nX z0&wiOX?VeA6RE9mM>V*M+fn(7NRW;Y8~EDC*DU026W$O9FboEMne;Cyi$_93)p~C7 zMSc&1=VLtsubTy#2(J@r^cF2*4a zSaD1aGM!zXTd-Mg37Hg3fv;`X+@;)t4r0@9PrpW&B&Lf1l-E)`gs`duI56lNBppe!^4p4an06N4S+CMh2aT6HTIg+IabsIth}Qc5(H10Xgm zxTN%aO%amt_<9WAm4a|=tu3>)dIS3E1**pcsax0+Fru#NL-Q|e^U}qR=>K$ zm=HFi?{Y~qVOP=VGIV;^nilREk^^_CMf1~NKn@DR-5_U#-P*!Pam_BzF1UX8w^)vq z>T<0Q2#$6qjG%tQfaLUi>Om=F*BCURbC-)0T8=TSc-j?xJ%+xL&Dc%SU&N+CFkpH) zjsO8E<2x2!aQosb(Z8%!)(V8beF~zt^GS10wI)+}kGe#lei4k;u`V9`Q)fXM5(>g~ z2V=vypzNYlSvEvTLgFMcPDMlAlnN6m#*Jhi* z2%=ODfe7{K$aqGtdUXOkn8)Fd2ZsE`bi2$@5Ka7u;zy zTKhJ3&Uv2q^Zwue^ZPt^+rIa`ueI0pUEj6Vb*;5eOO&X0D)Gq zX(n*PC%Zmj=Szo^^D5qQRODY`*8QBg0k{d+%QW%p?5JClU%jR8wmHuP{BiOg7*+|; z!w_5<&yflK=dK-^z&-ii8CwK;@ZS)?05pr*h3|L>KZm1Gl1BuOmRgd(qjSxCdwK0{ zVfESA)+7f;|5gNMVV?BQ8I35(0eb?sh892-hYXa~0(wn$Llt*h&KeAW7rqQ4R18LX zb>eJl`P=UY1`@9V1)G2sB!%e!P!fLCN@{?kxjMoCgjC|%_wVRk-?J4c*ddy|_7HVd zrc?BymL;e^I{AV9;qp4fCp~<&P<~Pkcrd~+i{2g?CyQI$0i ztF$$3Fsf`_A^@!M{vfcYKX8A$%3q<1a#n;BZ%O-$HN$EZ{3FeME6eD#qjH;d!C)`w z*t6|5C!Nc<(_U}j;E9|B1ua+jYOrz7H0Ah=R!KeCvf*1#VSL_HA*?Vaqf6(IJ zp1J0MTLXP=fuTLEgaijUc&72nY+$) zsn(YR!2=emiKGDoDUHO9=Ol<|x#H7*D{=-Q!2|qhLL#D#pphue4tU`F*oZaZdQe-)gB@~@z{A+2g~>a9DRn6K{~plx4(j@6;AX*kKckGExXsj!zy>-T?5N= zL5-U>I~kpJ2dDQBc4VBmJ3ILQN2T``47%iAm~Ty@95{uuygZT9`5(j#lV{qHHQ9e{ z8F0rK{04jd6vP(InQ(6t&t-*1h8Q&{UVv)c2{eEU!rc zIU(tqpJ#psyB6uc)K~AO8(B3X7}iwz*i)qDl3kcYC4Whp!imsa=7N=vNQb`<=xJ$$ zN%)r>H8L6taxJ=RSF`L@o=^UeL7n&UcF3wFLR=RoyG@^5R*xrHPp&%>BW$;^%BP3; z^D^;Icn5Kj4^Q!kJbdXi^y^F3RE5#xq=vQZCda|{#^4L$jSZnFED6C(+k;CvR>Iz zbBQ^7IXngaqxn$Ces$eh>+PHO@b5n8>aS1rELld0e|Ji3Cj2J#e5%>UMHzR=*FuIU zY`;2(I_Fj(O>K_50^Wc<H&-Tk9O;?-BXxUCMJJni*=a8>K4&edSMRgs}4nT!9`tL?N_(FdG@R_ zn@3q8KLmbf`R#{r50|LuCZ#Zks{_I%e0M=brtJqKVCwRxlFAg75>6)D^9$DtYxtp)&QQ1 zfwfN>l>;-QW&EHYbyHHNsGJ^`>L8UU@+pmgSJ%wE0f?NUtj~+fJ5?+8^4zV?rtlD` z$x5mgvkggkR;iVVpe8iarEeBxZPEa;D3XrO$|8J)*DPwGV)~>*6r>}k&Rh=kt6g=l zNB_dzX8`E7rEjE|VpR~lvZWSU0B~61$N^Qde$A;?w5acW;{?;~ zZ_%fJz2Z%~YH|xI9)t<}1N4O&tNyHmE51Oc@) z6K7R@QuR`cLl$r+vLIU@iaYV|W;j8Z9$XQtzW8+m7~GsjQry&|`~rOPk{wpp^@=(1 z?@|znPjxT|sd!^ZN^V(|9qFoVb6J0Fx?i$=s_$#wn@$K7#P!Krh+@%m<WMvM1kQ_x-X>6StJj=r)n@p(&R z4;O$%d-Vm;_2DD>RIgrrKXdsG7wBgA;1bSzGSOEk04`Oo{JSG!$7Gn|h|k!eu3zm@ zJuUnaZeh3Q7^ghDZv@!{t_!sc12BU;1mQG*oNCfQowR1imERvo)tCkBupwYcC;uI8 z-e3rGGk6|;f0l=U?r@@1jiU16*C^h6yrAfGt`5;OwMnn7t6{eM0aD{n-+g{|^Ip{; zhxy%>&y%N=q;zz9xI_@RXhiAbrC`#{6C);rXH4^(oKk(`s_|U8OsV; zuzR%!V8Sl9Y=0-W5~nJaI^LLO^t!^8EyQ$A!%)FD6WDQwIsm2EaY3>j1w% zo`{=)tcohc&F9JNX6vnoBHHAtKbruH`76cUcL)p%ZUz~BQcgHeNhVMqY|JiGldb{L zc}GxmkCcA>3*uGvDAWo%w9~uKUQBcbEV4#Jr+#vBN;0t-rBq(5iL!1Jcqd1tj-L<} z;^m?rouVq@F14TKP=~*eV){>Ix4MzVIFQ*q(K*X8V0JzgCXokVO5Z%rp*u-5FVYxq}{E5TyzCeZbl`FsBeZk zQjcxPT288SMajd=;dGx~r`{6qPU88ByI0U%wF@NcM$6d7-4Be45kM=tp0$IElWtF^ z9yeJE9+c%$!?ThsWl+seM*7~Y?r5XkxsGrT9dHj$xiSk3tD7^alAxRcJ??qhDLE0o zE7LA;zHWeV;2z<3uG+1xJz!zs?gvavhA|F4E6oQ=55Ax}!zc%cDA3Rjy1t(jvd|m_ zIpfq1!08<7dk*}&I#RygPH~Xsr~eZx!HHk+#;lD;qFZR9oQSRs zzk`sR_D~>BtpSWN2OPxuD!5|RA;YI$wBZNglTB+BT?(D6HW@eQUVyQk3_unE0>v!% z5GCNFz>vX5-FHw2GM+K;%fI-RsK+7=HO$VGkl)y0wBM18REO0^*G;=_vRR?kGn_yS z4JX345Y`Y30A$li77D@|yart+e8RD!g@qD8%u{|EH$`)}iSFS_(7apYaPAMriRr=U zZ^p<8I$ml~FJl9Cqh(Qd4v=Ps*09{i6<@99Jj!m3ac0=EB5Hw2>Uk*kQr`VF_3BQMgZUKyQf%Uphz#bnG~FR&!6X zho3P|)QmXK@7i6pdmXFjlzMxFZw)JODKJh*o7`D&RgpBc+;2ZmQYO08{bZ!J+=!-i z546e_eh?@E*0*MiiqeLaI2MT?=4%i3dlsUFD)G5@b&x}_Jlm>+D>N=2rO_uPj4uP1 z**hr}s%TfK3WlOED@i2yDZJrG#xUO^01b|2B>>8R*ai%Z0a%;@s1HnCeA>RvL1~0M z0GSz4s4c(24`bt4ApBvkc!@p=N0WF1GZJc`<^i*Cb+-Y)W1dc4FS*yRzfdRYQQ&@E zTfiM`6$SPO*2B?_ZTirAlVIE`AMg+0y@?lMl-{FYvQM6Asogma49sqaH>)V zk-1sFB4!erelk*89vXD_%QBlbpq1U~2w!kZ9TQNLeqfxjP^|)O=lwQRj_5fA^a8c~ z!!4l!F&tSBKva2%vy~nPGJ(g2VTqZ6yb^$&>;^szLh@lg){IYdF8$y_h+yW?eax3| z3J89YDQ-ADC7FH?1MYcBiJT2)kb}CBYrrJ|{Llr^pi3f>14QL>1hVF|B*_je-*Q!_ zJrc|qWRz`kEBX2iwITJRwi=wIbX*Gw&e?pOF33F$yq+Cvsn~(AtyY^h=yGWc|H}h!!u2zsY_Ad6&Eb#P^sX5X=&e9I_+=m820YjJd!b z%m@~6%Zv}0mo`;9Oq@q&jkH2w23!&{JrR%3vm>_3B#Cac8y{(~A;Fujj-WowS0HrM z03ttocqTz0vkMv(p&Sg!!B0#_Rxoj71pjL!Q{|d!qdQ<03^94kn>mj}h73x@mkwA1 zvcr&f?OhE07KAcHfbWXQN#e8*SO>iDifwZWTsdJ@yR%WJDUb}pQwGB* zpWO-=z6_9s0R(fdAP3}NV02^8m{r{%r6xl1mE_n{vCSHAD=t6HG2k;s@8zA&f7$xD z8DIl*T%ty#Eerk8j5MbJHb3~zhdIu~oSm?L10%H@>S_54A|y_Xt%-;{E;qIt*@OtJ z7&LG-VA>;LC_d|~rww~KB-=!Iwpu-^|IrDwCk!l!FgPJHCDq{^OP?O38}mXu0aDZY zk@g0pUzE-aBr4g_rw3M%h>m~);$IA$h;W?n1Q&SUV!Zc3g$SHBVi2`3{KZr|Kmr}Y zGfdZsC??n}9o=-Um$(}r5nP4MI3-L47KXoM`kfJ``mP;DD-qq6NU*yV;B!_O2(HPw zLEQCWA328xaM|`{gtN@@MG=?B(0DZP%-_B!kzp znuuXYxH0{GT8(G8fzPSY@k z;WGvW2p0W_fjp$#c)ST*lTGu$9~n1>QVm%xS<3TLO2f+9^&=4c&sPoOJbMDF0XH*~ zZ<7NVa_|dL4g|lL#mv)7ZVvb(YRfV-ysL)bS71R8B74*3X{MQ%Gm)7FmW{wUfu@KA zH*zMF!F1^+#sCTE5)d?q<3IA-r50<5Ks-GTk-7`xoP|7t*}_NlsEXK>_OodQkXPU; zjX{Xn+dpz`4;U?F>~Lc?&pgdwvW=c%o;iI+n}cN`Nc|Ndn!(wa3B-|{$@WB2fOVnX zQH@Olw{SUhF5OPd>OxNvfjj`K<7Tk5=XNOHc`h@!E_4Gy%+jR~p$vq+e_GT`K(o-I zyi@Nv8011s?SsodoK|TN^*b^(yn{f2d)mN>h{hxt;~mW5j zV4m`1p?b*4KL)@MWSyLaaE)Jf-_>78{|Zqvlf?{93duhF4jOvOpC-O~W~GLpf$0bf zS49XNZG6Uup1=|_l?$S0@ETaD8RolY_Lj`Ub7@mOJR2<3Anz_*sKH2Q{z47d!dY0T z!8j=nBbb@VLQ5g*hJ@ZC@f?x;66-~Yn8?8{^fZd&V|7WWr*7wlDaFM3MK65?~lOZgq z;E_^8EC@sJb@CPj8cf8BFcH~6XicoA3E`43PWgh!20=Wt^oi>t3>t6^CQj%z7V-^7 zAe?sSH4wuTbXBmB)AU4tns8E{1#1y2YWzvrU|Lsq|7ld1`z!um2yJru>3+frESKXt z2u4k_oByC3NH~Oma>f~7&Dqp-A=pEJb({GXl>+H)AnYL^l`@cX%4ywT!tf9sRuwR` zMeWe)8$U4lou?ddQgD&Y`5XvWXIE*SjX;XqdWNpw0daG?1j!`i8?J^BVGaXl3_Y%# zB@(Cnh!xO$54k-cuJ(yCoHO8*k@M}x4}vdjSvp|$kxO*RRoLkY;)d8k1)*UNT%%|Qsg zs)@x=LOdbT(MaURXSa58JcfT2%X$=?r}9Fx6ba`)yOQJ|b{I)-6;lP;@$Mqx_M(<# zosGHkBHa*LdwHWr0Ug=mVLUm#K1*h6*nb2Wp*ZVf%_Y2x68Udf{_99~bDnxvB6b%B zHLQ#v*!p&K$Tryb;UQ}h!etP3dj0QqId~K#BvT&p0$?lbis8~RY(~MR!?EqKdp5&K z#9q}>SRaLP*TYsABo)QDPqlCvP!o5qI62I!yJ{80PRgQcePV@sT0*+=cgQ2oX^FM$ zcP|f0F@=5lE&;3n`!Unz$6+e29{XTMrfKWb=^zHHswej1;*JSGQhH;0bqRLDcxKj+ z{mCNnI`Cg=CzjpF%YpQSAIn{mikc$)>*~5|x1QWhAMdgKlbpc6DtT;&{Usu5g_$C9 z3(ULJiG|4$wdqCV9DF>d)VTRenT<((h`=CN>!jKcM`62YodOz%k zD5c>B90{&zDYah^C;~xY{&%1QJ^Y*n*7(6{pl^KTpF#AD*C#Ld`_(4SF8GIUBBZTPu^WmV5rBy2Ozuyr-_JPSyCa-0pr7E zWgp8^7#4jOhIRiiZk$7l`__;Op@V1s7R4ZkUdi%NB|v~!$?E05n>$__Yp@triKm%? zSZ7N7)douhbn2JU4HC%?hKj>yJ!|@VVhzZyttIQ{5q9Z)!%4)z1Q9%y2DdI$FO38D zN?^tn%1E3!6NZL4mt#Osgg(?2);B1t{LDb+f6u1qPQbiJPy3&PBW(UU7>VI34c7sSz(SAY~?F=Kv6bh;N7$yb3t8LC|i z&Q0VSd0`t?c}`>OGre6@b-EXp1g$zgkVCZDJ&0c`WnnFOWW$+V03ZCL3KaL0@A|}= zo0#MAu7J9F;2#p0`wUB31C>e^-IhWZyz-Yju%ytCH8N+qUvmoV8k0AGh%GD2a*qNL zP@_U$`=~fFaK^UfvK+XU%X-qoi95%Ot?sgy+;Lpn-A#~BtwL=AZrVQ(By&;Ds7Dfa z?mx(htmIL)k@u?fO)FeQY&<^=Hg5bK{wgGYh;Da8j}Dt_ zgYp;MCR({uU)C{&d>!=~@$`7vaXXS?sQPoqDbKCu&m&65tYvk}w|+?#h z6Jk%o#){}yT!<-A4x-v!V;q#R>Begb71rhEN8#)J8Qhi|9k*{*`AeAPdVFG`#mxea zpv1yj7lzFPmB)v!22mw&zPo8+-(|5$6~5#UfiI)lRO*&} z?cU)VL?xnI7?c>s?Ag3>Bq-7mu_ z=@lHCpH1%6NSLCp`LBB|<;lw+#|y9?&r8D{c+@ z^tTZTG7d5&SdZv9#e|GG8fp)&$WBiF4i@? z_*m!DXLFUc%@^6sBig<8bLnt4?XKH1xceGwlie*~L;g_YK#Qx9` zPEq zD^p_}<(wxx14M~dip|gXLb}B0MX6M`T5cS34c4mL#lCwV%&htonB;nWnjk7Jm9*Q5 z=X#s;Rq=E+oF1FVg6kx!;uE8q#me3IECpG+p9PCDSA{(b&Ja;Z5VunCe(lJfnOLYS zo&e{se!mME^Xt3k#}WM-l;#t9w9Yj9#Mb*uRTjUlNxy|-*xd##^XLVM23V%T>A=KM z7~-4tAaU9pP^WU{CZNEQ5$|Ll*Q5|63hVGNw1iM*V$`o{n{TT&(~;I{tFrnZPj3jK za$wR{Z3#ORWN57d6_|}j>$19pwT=*v)8s3$N+oB19{op~#{h(|?xTRUlmKG9M&GPW z^&i!OsNy>c_UD~D8Lp`ESGv{Bh~oABKG`aHh01m69|kIuG_BXz2?jadaPba25f0y# z*j$n(PqGunggFQ>t$|(E$#4d z|2mlU++Y^fqP|(6&qnTcW6>y-;dnhBHmZN{5KLJ!l&P9mNGo(ZvDA+HVrI-{br>zo zWo=+$Me%wV?R=uIn!^Af3Qt5__N)>frh@s~Xvr1i2$#JVRSgm+d33d^e^7@t>V4tR z`Lfkgup_&wg}eQ@INbJ?vbfU+^=|$oK(p(I@Zt#nvBO*1P4$8UUx3Cyg=%@e%5`QR zY|2`Xs|HaCS7mOr+^EAzfMv&8yNxq?gv$s$Cb;LSg2_-st1;$!Z>Y{*<_h&bY-c7xB>L<&aB@a)HzDu2zE&bV^0+Cj;i4o^NR>FDZI;WTG zm*3HS9(U^nHZFO^B_lq;hh0luYx5W(E>K#6$^jdme^b@yr+O|_`3*Mbr-bdaO=ursOtuGsW3)uv`qIArZl0@$X;WWa@x=M|4XIib43 z`V`(s22}o$69$kDuzcL5zC$4?n-NCFSy>ODk)A)~2f7RO_um#vhZ-ahO084}{W^dm zqgYrod4{PHKH+KZ2t0gbyJpC{`IgHN%mL~0C-Fy5_%3EX^)4uzC{eAl&I? zYoMilH$45fah%{t(o4J(YgZBhAebmH{Fkj^>Ise0$7qXi2mrd01s(r&c%eti+M!2$ z<3TTQ(#Xc>y4(_EJsy4nCJmVFC5*uRNqCqi-Y8uy@0{8V5j((IM*(3X>mrUUOSis> z*5H%9A1tLBT}zy1%33?&@MyfJ$d;=y7U4C*-r7WKmN@ClO*{`tRJHv`=fmWlHPy>&;?O(iaH9|RWw z{}J?~n>xn@FV|nrf1sz6hrW2$On~ZPmS3mo66ej~{GI|YgP9>KQ#Eb35ktaEb*lmY zC~;i;jAx=yA{0Zi7!Ib8Tk!8Zy^*572bJEJvl} zp(^W#!}+Yu1S2h%3y*iZ+4(^rW2n zxVPXMJgnumxEPUV{ENvAu+rs2Vy}Q}A;z5}th^K4NX7p>^ypa}`U_Wp2}BlEVK9}; zL`1FdNnbnK%5AZrwPV`~DtLVaq+%F3@L{5tI7tsD2lZxrEr5pZy{wq)Kd1{41aKRK zyONn?xm^*+=tGc~340o}X1R!$S&4XLnA?0**%JYX7FeslHdKm=21kR4(Gx~G4A=-da zP|e(*a7blwLI?QRGKh5irXdR+B;-*?KL73f-r9lw_x)OZ90H2M-L9(N`rkv-Pbu-X zfZzF~@lw^m$ZWq~XMX0?;IX0q%7|y|`6dn5L%5_wiv4v0#)h@H%?C{e(|Esk-c~M= zSvPU!=L&5(!z&*vc}Q+AlgA|9yeerb8e#m!VOMJ3%hUT~#UVMP>sh3~y-&R$KI)rNe|}ltmsAAZ^o;2_zaCm| zbf!7lcHym9VGMoE-8WU24Ma@jUNpAAf!v^H@q^5W)d~tbuRYIn7BRW&)_GaKUt>L2 zNdI)xgJ)8Y!WF0knypl)T^b6F1uFZ9%6OvH;5Gd>cWie^y1ue!f-U>`i7_{#S)gu@ z%g*95*cZn-c%S#bR&AQD3eQ=Q6CCKbYUz475jQByVe%1ck)d#X?Qgg!M806^JyRmq zu1~i+tdQoXnc8Ykc@$gY{q;NT4F)aW&<4^}-ixv1bKn;~Xan60ccE_lx!NK%5QOI^ z(WueadbnHP*!jU;Y)L5N75@yIV?Nul8a(mxGuT0wT_Rf!73sVWui@4R2w)blvM!%} zMjCR7Jh{3*r{`tN8KvIP&ZRvYysi)>%dlZ`kJG3>5=#8D8l{>B?AxAXSoy%hO!6tv zS5l=URO}Oi@jgevSZs-o3s`DBj;rCQy$ft+)n=8gD?kl#hj4S2yz-ssdIoBEcKPLG zHApdWZl+MhZChjLyU4ibW<7n(l}M-!s)m$KKRMX(j*Py|F6;mn7{ZN z%U;T}pWtFKd7D&clqhw0-jD4M525XYm0Ui*LR>MGT5A-#J)|6U7*k>1c`Vi*gyn76 zFo8vKSQt!&J`j69f%Jds1wefkrsbFqwt`?mXJGR>&I(+~ou`(MSn{ofjS0)?mNueX zIIz8P{w<=jgHTM{l3P+tape97LQXnZ&K8qDjf>hAZ-}CCEj-U0qUTPO%WK2@JXZ?? z+z==(I{G0S~AkLx;t7t5p$7f@TQ(tU|W$Hw4OpXqyYub?DkVhGd z<<$d{T(Gnq>(;H$H$XAS_oweVEAY;ttk~31?vMQ)1n&S|X)15NPlA3Otdj=|>W6}L z{zs^$!vtJEVns=L_FNl?4!M5U1$Ow+w3k{CW$6Uxa5|-7xz_=}2f;D0!3BE^P!J6K zLX`N9{)8S6rm(GID*>UQEZCZK5ZXYF^HBQ0VdY6g@p9j0eKY8 zW#V(<QRP9dTBn494lQ9vqCU;+)XP_{B% zEU7t)yP`KKh>}UXCiW0fI7Qw>$Gv+naTe^pNMvrF*#o77N>bSUbj~it0%sx8zN;_> zyAiP+g;g@JXGMX#Xr!mGVD66%OI+y|vQP*a_nJuRCl5 zPxpZMEP}lY(KU_meSF2agng1ir-?M&BPU~wdRAS{u&5(s5rL?hKSTI^km z=b>fw99oL=Xar$yfMx-@;C`S-Xvfj(M1goQs!9JCK5jWH0u}rZ7A6u^;h+_wrsOP? z_(5eN?opG^kjr(*^?&|Ep%(*e+*7i$e#Me2ye5+`1S}d#Fhg1Q!U)5>qwU>Vm^yc&%OiVyv*;`X7?2?$~w`X0+^)nl>T1pE(ySyYk!F|4e9j3wv2_=FS>h&B%>vw^xq z8`dvGs}kyiz$tXI62-*`Q_>!+``QX+jTDeM;xW3){>MAO4dM)l!IYRo1P8}N8-vdx ze>_1e0VftZWRSx=X&v5q>s(|o-QqFRvhfAoEuj)1m6>Ch8El?Rd3;j#_$YlW`U7?H zMmOPvGN;ZHlk*Ryl!mFCMJ-KF4w#3r^BiOsLwJj`UWUljs(u4aPohNeEJ3v)^8G1! zZ=iZ)4;Jgkl3V)7rPze^z_upX9JDx~&0(`0gth?mmf+3#MC(Qp%ui?KO&08uAoqBH zC~$(9AEG!?Y4u|)kz5EAtbuhVIy1;6T;wXp#3ivn>;zyhIR=`Zh6`GBY`{9j>0)SyEkIpdJiONerFtVD`o z8sT}4RS+z;z)*k_D6#z?LKP_cKV|WT+#rR46O@X%5YiA0hr}6jfqVFmIOXw6 z3=d90w(LB0Vi@~Sr;AlqKy=^_$q#(FFxbaNUO)?01Z~3t_<{4#F+l@tQc4Gz8w=uTu&D>jU!P;N zP1ef+A_E{Mi(q$Q3yf4S_*;wJHtl7Oj4=(cp{y*RX0aK{K4D(~r$~vOV zJ)i@Fq{tMaN$VlBpu>s*#8BQo<$4LwDn*X1XV``K1(C4+s~7soSo;4`GXPo!PONC+ zfApV#fq>r+=bi`H1`JutEb!QEnqp*(b23)BVZxEo1qqT6IipnrjEf5!#)%R z99K9Lz_S4NV=QeI*(SH?KbZwE2FwBx{~ydU!~Ls)S-?PmSIoR>1FAFg3J6ktW%S2x~xgA>w^36igv9fcZJA z`hP_%oAR-HTL{)x7R*szs060)B9$jG6C`(aLv)L!yO@!gKYfAu@J%s}fw3dDWSj?oSbCu`h5_1B#;UDlcY>7-0bsk^SJgr#&FDdg3YMDidr3B8oBz3wd zZve@k8*-p&>NlN`y5f2Py*_=Izy z!;A&W88K17UQ|V8r0FKJ?26CBRXOg z30mrBLnvBUw1vTD2?mbJJta3jwzfshps>g-Ifs*9^P-&r2*wxe9+QR93`ZnvHV- zg7Dr7BuQ9fafr6o8=Mc8dD8p=7VJNTKzAmNsy;chJU9bgOp9>c2k#04mg`*~gb<%7 z&W6Hba69wui2O>M)5PlsQm_LGCUcuw+`Y_uX=Ll$~}19+XvbS(}E>pT_WGAEpB`4{0RRZVE#q>OsJlr#=JL56W&)3 ze~BDa+^HbY?Xj)`&c*S3WjSN+EabKtnj*9Ze;tU7JAGMrTXpSvQNG}pEN3o>(&Bxe zDIDB$%BMY6Wi@f`L%$*Hn&r^s=)h~dk~V&G=dG^&E3v>bMMH5FMC165DCk*%pULM= zCOylQFXD8vOVfzI_T64(g}-@k;Q{qwT3P0MTchJnlj;;@6zG*a^vYo))K67LVW{$6 z{sU;|+4KrtW5*@(i8m^4SweyN19*E{Rek0)NXxFiIJ_6OE`1(cgA^sf-KRayU|UB& z>@01{Z-RYet|r61(6qOW!wI&r3M@shxtnHD`TdyJ;uR z=-yb=$#O>Upf*V1V|4{o66UzNrWUFM;k|kACO7!RxA8sm*FVnPjGsIkZi{>~{xav)!9Gtm_oXTP~xBmVsRMQb(Ur9~bH40CT|##g?>In9oI zn$S<&eT8`s*5c{vA(u7lDi5i7tXW698Ga-!vXv`l&(b}LA8*_}E4o6}Va+<+Ll1y& zU}d%DlKYxRZ}zDHrF?+UPk(TWow zfim3ULYWwUMVF`U%$UnkG{ zv`T!PS2%U2y^&#ptZ2*k&y)$8NBWvOD@E;3{g4#7x;s+VSaLZWruQ?o^fxb7N;gk4 zhZnOH8rHCz`;3YA!KqI1@aDzJizULF9dh4gHCb5yI6>U!?Z4cI&0kF@?UJP1^Uo&m zQ>c23mc=>Ls$mz52qpcXfLA}0YC7$`mfnwRuD`EuEV?%QR=;t(5`Q(jd6MSk^kuZm zIN|W{#O|u{rAp~{bhDpZ(1zct2vhg4KFlx3nq}LyDDhnX;nc3LN9LZ$;xNB?QlMWt zEOs({_)J+Oe;E!93J#>(pL_k;5`X7S>avHIncz)38z1PO6+PKjo@v0Ib~e-Uhv;@D zcoR*!y_CpR{Xj8}O3@XHjVEZA#g&B@gmpPp4%9t>j6les@dVVtg~0dlDz5EJq3UW8 zV(Ve$f>d=~=Q(S*EZ$lqPB+`Ia=dJv?nt3p!hz#eB_~&1x&O=fe}9}-dtKsj)EhrT zKd^3gWSA1ac~yy8LO?pnTynXTwK%CspaB);7JiGyza$nef%hih13q=y&a{zEKg(V= zxMI7p4Oxj00nv(mpxvL4`O1@l4{))a+>*ID6YzstEr6F`-! zmtFEWZ&70s86+#J%DVuus0G%Gkz#)p$(QMl_~{xo#seJE@iCyhDg1sjGv5&qfwZ>X ztZJpf(oEfvuPcrykU_*Fy1l6|@6QRQC%ySYtCLeNs+m=s^{CJ5yk5;t9rRa!d9Cd6 zzU5OtI^L={#+hq&UA?i|e-HfurKPd7%MMFtGg8@o`gn9>@$DP^IuQkE~;Ca7IE@N z>6f0Wrt`~D!CTV2|EQG|CX$>TN-9hw9Xm|TBUVJaKQ(ZF>cwnhEi1+aM0rMzRl}g+OK*f<)ZHA~pkMiF#cdxSVKoga z3tn&UhT2u6FE+_>akT9w3JRRXFMUdcuFXm>CVIiLTg{3;NmnpgH}ueXm8zHe_Yc4i zi@qhoJtURs=#JUu^NESxEYv$;!F%E9+gizzcMqxRZ?LkK8^h>-{#=l0OVhgbtmXQG zTg7Nb&a1$adE2Sd(V0=HsZ}kK-6bS;J^NT2AzKl5qEE^3ot0q?`fBOMTIqr*OKybw zN$JKl)E@BpqyoP=8NkZ=g7%N$Q%zg)A4S3CM^LzMDfo>gqnZ^X>6S65ifQ{zkW5e( zA9y9GC^Ui?{R`uI))k!m;-c5AZI5^pH-CsXUpRH7(1;uT*I;1&(3{~GT?-RInT1)X z%*8xOq;u@Cq2X!&C(^&GeL4pmanURPqWTjS%Zt7h`}hb-UZ%Th0;(4my-=uUYq9*o zQ}}g{k638*+uBuBG>_sm)kMtUfvaH7Uff@!X6555nH~MEc2zehT{o0TZC}+(Hg}k` zev~;I5S(ZkCKqL z9sc}F@OoCTISJAHYnBl`7fj*X9r$fkBp_XshDVGheV-s;)NkDtxSFinWvE*>+IFzV zxu19ckD5Pa|I8k+_C4+l7$JchH8WLo*vP#gRH>Gfc~8eSf?f7K0MAm@U)CvgA;Sp7ry?m+qo7H z>LE_gpUA(?L{bMM@;FlmR~UlzGSx@tivW@+*J`KlqCcQk1<=ndrG|_CeFj>zO%{&t z=~g;43gWhdYBm;k`IL&?{EXOVAoy$ z2M}%$KxkYB2E|B4MnXm@+-w>N;-b}~Z>YEEm`DCvNoO`qA>$1@)Y2bnRF&Uf^8?2E zvE=nmvF+)GCko)y-y4x5nOcXm(}i6Ckr!7nt!M3t0acpE!^JE+fMx}&-t82lso(J# zu1UGzVHI8ZI&HM$4#}*se_}LQH;hS5x#~qm$`~F~-q1?lO@Clt6>*O{XYd}=Ucfdc z{GufzoCrJg0`F(bMBKye1^kYM4aXU<8uqX{Ol9{dE^gHa6NOH9qo|XfG9iya&S@@3 z`ANrO!f)wr25AKU!KTitWAJR=S#N6(EI;)q?i4#rh{WIcaAY>l_ON11(tQiEAEq8} zkYP!!mK@oD-hfkV`^b4}xciJZ*jKdzy#WVd*2wrO%5+y>{R<}{lN0WSP$h5@y0uvW z^3(Q9W=nTgo`#bb!PouaOqm=W?Ou3;>|ak{v^gEf5S)sqXp-YN|K!y7eg3}?OaJW< zYW`0g0(jF7oh_02QK!290vEg6%3}Gy@ajT;2)w#L1J{H+jX2Tw>y%;a5gNmR(Fm~f z4%(El5fncESiXXlPqxe8b@+;A1FFlGZ#A4Wp;TH-adz1TiUX>PmoQK)hyS%2x~o%6|DVBX3JPk9jT8=!b|)S9 z9bFs`9i*dJW>_|<7rVyp82-PL3k?P3k{^^56!3Ejii;sO8l94>&KATtPI1Z^UmN3d zcJHX;H&|sUZ=+l6A)i$B`f@<5s6oTTcM0dn#K^-+DR1JthDEB6%S5xaj7=oJEi9bW z3(LwIw+!KU(x!3Sa)RDzX+hsMj$Tvk-MV(_dJS*1-Okj!ZxeNWvvb4IQLhbe811qR zs5r)D1hU+;MKx+NEu9s=f9>J8X_I>Pu`|bmR=?iHyx^o3#q@YtMUjJDbQC&F)bIE4 zd|@*Ce17XTlSQ`g^KCk&qSbcp+{quUUT>J!Ah%`b&toRO)T=0BnfJCx)|KiqT=H&< zb=uCd_LX#2>`^Pql`C_*vi0<`CV$gVe&`c+dXl+(;(gLt19yB0QGABv!>_wsrr|Hf5T8$H2JGR)~oW% zo9{zc-^%>OwPTOTs{QpRR%mORyxem-$#sA|>b0crL;}slA$m5ZhNB@cu1Catv5-Yci3+r34AXJ6TVwSyLVP9AvcTb%81EXuBp$Bc`6bjuOtSCY&2x^LExqz(zanA6gze){Q`n8 z{v9fCvtl^A;CAyScag4FVv&P|zQuh*qsc0}3pM4FH!FQ|bXwfecm3YWg4d#Uh9mh5 zYZQ4`EBqB3x@7OAE@l6(jSNJ-HQE1MNdER_Dzo5f4o;D&aP9W7 zoPDBq_WiiZl)Hp-@6y}Bk~iA*ZqFyDqWrC!w@AESH1hEG;~RW6 ze>Jc9>ygBG(oWsKuf&&o9lP-H{Hpy&IzCyKqXQ;E(H3$4PRz`PEepP$=Sjq3gRb~IEuV>QF z{dNCi(a0m$u($GEsr>phFDh&IXusDBxNPZPRrmEh$+!4!jLqQ$PqvH)>wAp)1XkaV zK61{x$d+4a^op)SN2x=L(K{E~i*^DG zLq|BDpE}03CV??+!;z-FCj??_1^J&|84uES33?uQOlVC4bCJvN9qwN=T7p9Ep~imP zfB$^Q?kn#l*DCYE<6;4`c^&f*+hD=B<%fQ(tvV*3vYD#Ch}Lbyduxj6`l@glukYGA zY$8Ty_k0*lT(pepSupdnPgdvJyZMe5m`=GBUOE-_OH^pJ(k;8K;hknRg?u!7JyO&f)jo_n?{{`~!&e~p;h?VpO54QYIyDbu!3 zf%{mtJS(j=_vcay3UnHC?!>^tZPuNnf`_03cj9P&%+`TqWU<{t%cZDpFR!^O z!2e3%a*f*St*`b9`mGtcQm^IrL3JCAmXp26$cNQjVcE%f`aZ#&?9!`tUxNR3O|y{l zGPScV`>*ZXFKzsSqSDW?o^tI*!-v`dF)KqUk{y4HQa4hrRgzZXp~};@mCxstHeSTN zVUX?PgT*|%WDNO<=TUg;@WTltMycnXXS>CmdyHGTh;WaN_LlS9ICy&N6M`4I}5(ulF+d2DUGcT^_$xpEaWDHLr1?JP3`+| zT_$;13T@Zsu)?Ctl3QN=JyI_qFzV&{^HXQjc+b@Mm*mM`Uw?mb7$5B!&yC67`TP5D zrv`n^XpA9yH2Z+u#Luoz^&NdRJ3A)yy}lX63=I{ykGYS2`5lt05i?le>G5;8Vxp9T zW72u*yTECqe3^-+35}_*9q&q~esg&Jl;bGs3l9A;HcEf`I>+Qs_tBr9-uK_RLq9e7 z(|+o=i+S5Sk6&NfibiTQY6Lh!u5&PrhNQuNhPQX8v~|~|p~=zGhZoHZ7{^D(F9{fV zy65M5brw$j)E=-=&7F{X(`;6DOhxs;rO4oeN{f8h%AuC4?6$-fJy*TMmwY0F+g>TO z2QwVs-eX2*QP%o+nQNJ=ie{$$x!&@#fguJ|J_66Oi^@CXWb=Q2|1>mOQ{egYeV0*z zSk8H+3+Jl%bW3(w9>1k?RKC+k@Ian&u+ZOr+rN&VmCx!6eBN6Z)b&Yx@78h3#q349 z!sJ*M@8q*)+hy^|!Mr}IjZ5Qn*sVjOT5=UkQ4hQ*utlWd-=uKQbv_IQsI?MJivHqS7|?q!i9(o{S6 zlZs+H1_zh#HKR*E`SCZ)u*XR0o~ieP6=@uvQ(wOJOKj5A&EK&mdt=S;;Euom{7Z1n zQjcqVfk(>NA~rX(nKONp;(wbf`K_I8ebQTLXZ}71eX68or3Ztz(8_6QbOb%(BUL=O zufcpgUgnh>ODH=N&AYeXn9P&RZ7*s|`BW^*dd?%2n>dmIFJfD5v`DY>B~#?;ho}2W zY2Jl$(A&PTJR>L^+hV_6mB}ai&}iq{QgaQi`3Mn z#p}nabbEmI7EjE^4ykP&QU>`cCRg@fVY@XH$=`8ENHJuHe8}=zRzBb1V7^0+yZMR= zmOy{2Gu1aKZoOl2w!&7ChZy z{**+L9n<_?BOt%QUib6<(TdHRFZIP31<8CgPJY5ZahJO{ik5cmr~Ls-Hr!hBtp1$l z)3|G0pYilcTJ6Day*m!Ph*C*1@YbX-SS1e*uGeesYj2`gw@EPKUMSE}3)vKU1xmc0bKGBhf9*a0QU6V2G6tkX?)4_$sXs~Cl+ zah>L@Id!-5gT(Ec2pz?@Hr{KW=)|Wv=dB+NzGSBrZ9DOygnM&=>lT$F+h|2Cy9o2Z zC2zWQxv!?XGO9SAu;se+V5B+Y*ZIi56D}6~^`qCYuB*2;VRL-;?NOGg6|6)2{b}QO zhtr=MrUcObJQ^%j*z`D&H9vTB`j^Zdtxqe1c^Hhey@%K6+XFclC{g|Ie4ahSB*`nOoI;#Da-WnL~)CCYimn?wi-ux&-uu!N8D`m4*n(E zn&0l|-uZhZWIQ+R^mPG_NjKe^g7Jy^&We$?OC53^9$Tl}j3#R)vPzEswX!Uy z6UgMlziypxyc-LlMbn~m{{U2I2z76&#-L+}^tefEP`|{FxaZ!33JyH#;+0h&I zhxZk79^GowVFrJyRocDXEAUYwu-CSg|R2Tk7CvhR5CGr+&EPKgX>3n{Dv{vp1|vt5p?6a znxvgc8<*+k4zPSSR}8CikYD-Oxs~B<%`RcR8Xg-Fj^NO}0*$2p=7YA~MTePnRfCpz z9N80{I8fNH^k~iUSbD#*7MmLe5++YCPXBg$(^clp)ZROHaGuEGvunG2EMj#iy=vKw zj4122%bLH1z8wpw`hRNs5_l-rw|^;=Ldh0M9c{9geQh~W2_XuJv1S<~jGZCjBvC0# z3`wbok!_M?EFop7EHey7g(+qjhOx}{-lICFbN=u7z306D|9|SapX-t{b>$>mz z@uBCI&TIFg1jDks3F+x5L)IJvOJL%MQ8FyXJY{JPir~#GcIoKn-jx+#m7QrA+fgNv zQw&W7;@gq@FBaUp<~?o>yxeZlEw(<-FqYA$yz%S62f=%(M_7~Mu28(US-`>9mdq>9 zpc~{n+jp8QB-_?Jlvdjpd!bj?9cpqO_yEb3JlW-B#=GtD;_Jb*poZ*@W36BH%c>qp ztL&rRJQvpe%;AGzMr!D5`ADaJB~QMNcD=E?20OjOO?1mzXEcSjJr?T7$7j6La|NRE zMv1Ri_K2R!Uq04&&cn1twdW=-;O)EkF9sX>zH!}$=`>*c4_(V<2(N)XpBuYBc0 z*A?9G6o~uTQsMS50r;J&sw-k`HT+Ant{WD;JGN&k+6AkRo6z&f=}SF5)VIWgj5c%g zZbRlYy@r^c&_-u3CaiQ7$8XYQ8R$U&r543m_d~u? zCqGblCz%o{o_3 z$@F7AbzR<-avwtLZ8lE@?MNGYpJm^f8W(tRN9`r;;V!FCnfZyl{`0>e6Q(Av`V#TC zaoK!D##eRBvCIv_UDUEBZL~E1r7pZ!+(qL1`ZtFv@OycBL-CK^O$}Aam1-|mveta9 zg&Kbe+F`dPo25g@9Ka_7obgD?s@xyzOSCWI?{Or8f8xbQz9Z|Q{r528Ihm=R_)XE_ z!xWwFj`j*9Z$sU52+|`yD0tYCYiP<0TWv2f*bG0n#t)_IK=g%NGRKnF0-Hc2{;6$M zm;Rw}HRDa#P}`$^NWL$wL$|-`1SjVWJ1lPDTD8_+SJ6A`tVBA!vH4(?YJlod^}SD5Ev1Q1tUv**wCxsmkp#JA=ITU8Dm4QfKEQ> z4BG5|(Y3V$j|vx9nXY^x__Ax4iK$>&uk2D2yz6`+VY};7^`#FSQ7+(JyB2;k4_4dt zX_rNs-4*4F#H2@Qe9Cx%=x~7$8=atU0jQaaINn1UZL-f z?Q_uLdEL$dCmbQ-9Y$^@it>ADMYyPlP_UmND zz|c`gL$B*EP&Bn(N5k_p)xrAnG{~p)%>((BSGR=sis@Gz-ft2Z*Lo1XNwTPRTX^v$ z8KsZj+qeh3t(~4TWTVW#eM=NVMAR!A8YDZnY4#5+ zL@&HESI*6<5+Qfr%GE%s#Hne!)^xl2zuw#0EjXYAyH{B@EOAiVwMa-$*Oh)&-+R1M z(4$MBdUK)p-+2BxOm`3yBqx1ifuyVoi);8@jKi}GV; zZ#AyF(Kl&qNOIcu)IGt+@dA-KXE3Gu?BOKo=38N52B&knC3m}jg^JotI`vtb3K`Mg zZd-Xf6qD@{V_~jk#z%6XybTqT2=-KIe(Kz{;e(g@Dd#O^f^Ov`r9&~xhy8}el9f>L-Np%kF?DY#I;y-(#KV#PLaZ21vUhkQQkr{?(9x4 zICc#wm%I2DV%O&9DJhX=35>LgU2sne2cn^L;R2?AcE9S7j`XQb;?yW@vndH*q~IRw zS`fg~uB|lnZ21~|a*HW_T&>vU1yoX^O5Sxux%)tF6&4~XF~HNvksIr4JNQUDm&QHE zdrIJ$LbxN8w>@tjBTr=g5T$Pb(Sq`}{LN#ux5J0PhU6=bIo1YH%!zx_W#fa?s8GA* z3U7zK2^Rv$p%}fQ2wu8aHK^(CrE%Nwo}h|{1H16ad0?bC!j72-IwO;-9p0IO7)VX( z1aq!pF0(d^$7lntDcI1kC?YL)#n!2}Xnz5-EZ5qgE^;DQx(s?ABo$@1tnkWNOk$xm z$rF_BZnx9%bM6Tj=B{!IdAED6M@3*$%_EmVWjaX4L4Nc!kI%=N+*})jW`AuRUD49G znW(6v%RUz8mO_a8#@>qQ#3}nl9gX#wHD5}a*mw1n*sH5BNYqgQpIaxE?px35VNuBI z5(vTs&ZAyPT|7(K`k=d>al3c{A_A|cRlXS?)U@p^S|I+=J0E*NkBMsN(d)2IAByycg4prtp{r-}_3ZC{t3+mo6*E#xPJf*qrn6na9sqv4>zoB5RP!Nf}A z#q&j1>K0wES^L_iUFVwZN=;`qTb`GkA{>w@Wfcz^kT5p~OM$esW(qSc>BdYSQ*2s9 zONpLmTRS>2J&O99SJ2_2VN97~F*?f6hCEPWty~=M78fv1>R*d1>??_Jsgx3MDzeX1 zx^zx=w}5@6bar4RPF1~k(m6}vtK|NVo2n$!1Y~PoN#eZhf|AdfE0!wiwt3$dd;PX+ zR&qaYhk#2h?@RuUYUf&`m;65Lovq#v#Yz>UBv7~Pc8V`1&feS4n|=IPBKFsFM=ry* zqpp=)alM}Q3I0f|bVXGHC0`jW8s8Y=Ajex+z5k*nK#1MeI%Rb3h=1~Dc)nPv@b%&6 z7tbYKQzZ?wC}~_FE7`=xReA5Rx>OUiRtJhL+3`526xv+sfdBLx*}DaPsV3`sNNu&& zaRb-}JhsYZHgs)haF*sxS#;>yWgj3sstR3O?EwS}AcQ&tfms>4R@(sxPb)9XhHbAs zrh!GTcQsIol3DrMxZivDX8q=u^$TIe8K0MKD!wxlthUk4+3xSPXJ=c2k41$4>w25X z45_((e&b}J>WAyg8`k>~y6lg26%UOq-85mT3MQpB!pftcHM&<3-X)$+Ui8+*xzkOH z>&HScaeij_Q13^3_C;8?pL3^o78i~sHP4m@m2eN&V%L`kT|8Q}N4tE!`5Dsm=GJ~# zMisUHz=^JB1T83WDAAZB-0;9o8gT@&=27`=+T~F_6@(MTc4HkYraeI;c6qNe7qSY- z+4eVpP-FIapmOf!p{ZabO)AeiKHC;E#j)NGtcMH8wUuF!X|9qllXE(Z1C5X>?GQIVzmKzx^c|btDs>i;py@o1Q|Fp3uYM zc2AC+()pD}t0Kh0VH*^~=^}$jFbb_PM$t|3pI&Q18bQ|{$39FqLhr@XBr7|2l)fLj z+NU%R-MY~$5N^6c;vV$nF}`7hebugX+Qcv6Mtp%@P$N+sl8HJ-u7cGA)}t=Jg1L=*eHXX(zV?bGDMPl;UBVo&@Ur zQB3=l&5kTaw}MlH7dsH8K(WwkdiwEDINvGq|@ZH?j(- z?f_M$Gf=UxK-E`yt#0J@roFVVkE$8Cso@qUXH>d)?6u*K=dVM8wfbLFN7a7-c5xNJ zRSfDJK-w~%#`kc^NPCj0Xoj>ZH6(;mnmiR>Faz2dOhu~l)SS@(NmB>i!40qPo$@+C zG-8eHpJ5sis}AwNeJ?|gAk$AoRvT1ftI4WIkm1b-Cx>3|4iCU9e)wAZ{2KI#vg-!Z z?t3>*`}o9L7F%5%RCp*PIWieprlDEWCDw#%4pzRCGH__{oJjA47gn?U@}RhIZ-E^2 zT~5`ZyOo~1Z+F(XsXpthaQ)5PB=~%m(XLE=a;tH_JLKlOoS<8>4q4ki^-OzQ_3lyJ zoNFVwuEZ2yvHtm;(Hhu9&8{??rRf3Rw7Bzj12w3-R30=)S5YWAmdt6M#`;(4dcrQebDVjpdj6Nc)|6bW4H7 zHOYP1`?nV@8)@03Zu)rMbp1mYMZv_u%IPKNl)=49r7D~a;fBfvgJf*d4cy2{J&t|8@ z^6^6TwMJ3T`>a^9nAZyzSg(~Y=$|y4pIErVc$bCUN;w9*XpR=D3)QbSpiTJtoz2V9 zk_itm)G92_5s?X}s%RZJXC9JL7bmyzuJ+F6%L*;@e+T^CicG_w|DL zky1us@CM6S}JDy+j z1fy;$V`y-<_KAI z!$`Tk)6BFTrjTu9qY_(p%d>-WV>_u%?0z(C3^G%-99?g5|4F$QopOWwI8rnr8YUt) zdEwO)6AAc+d?iS4u>@S+XgJXV4e5nSz;Re@($sbuT#&Al5Q%KA?}9iIpQETaY$$0; zj|SgC*GY*)w$*pJ{7}>N+DG>9PTP=i!4l6v*Ekw7W~V>)#db;0rRqy6tLq`k_jOeh z^nB(p`t*K-HbmI`hmyXho50u75L%8U3(FLP$eAur_PU zrrHq=A4j}m7Zz_s_J#|UNY|cwd5Kht?`hwjNG8l!A>z>NK!Ayq4cD*r;4%W)627 zVYBR2qOKdagglAMQtA5TqIeMZ8v3Cc{Phw|yjR?1vFFlBLjj1iQt8wDqbvo89&Q8Q zbhq$D@hbFHdU;I5htQndYH$lW=$tJumg)xeM`+wMPpfQa4jeY{gy*Yt@y?yZD_*vRwjiBe#g2uq{QF0?M&Q)HKS6JhNsW2qUKOcC{) z=FTklrn>}6STmzIDnFV$nCyCqaACaMj#bm{F{7Y)lrSjyOZKxhS_z^CJ~_fd%|myZ zg|cSyTuKDp@|7BbqE^Bul2aVk(wk^;z{R$)8dqkHg4(l_vQ3AB7O5C)5&pM zd3!%@7r?BHk=~>*vbZf?DkxA=_i8a3M8Z=x8hkowY)dhCzhE((OoeJdI7DF**b1K>#hq2O%vtI z%2U36Ub-A^TTX76dUvTIE}pnAOIAiM4wWtH8Ty7S;TIQXpEa>44P7S#9OJ} znvLQYB)U<<>m2y6?7f%Y+PhX5UDX9^9 zc~j`nSQKt^m&TZR2Ez^;c*3bptc%Rvzul0ZCOSG-{3W5Ljq`nfIQX&VLQhWu0ufP- zh^h?_4GF8I&wZ>(Am--gOdaP@Viat{lVHW9lAO&M-VblM*_&-M(Q;ED;`Sg_)0iSP zV^$NF3|o|z?`Xdv;>JPifotKuYamt=znJtqTh+M#%{CWpm*kGxeMv3IgT~V$H7~3t zUgx@`b6RI{#GCDLR-{knnfZRn4rZD8A&Cx6aseXd&q0vn586T6pp^@{_PFDHSfaxx zxq!e^^;6$&-o}T!B;LveodjcY%Oxh?ZuS7lYR_8JhMl^qeR>ypgSUY;)K zu<-s>b-HrPwE?tISBUgA6Gy4)J1v`~Hh8Gro;NCiYrHZt3qg9r1&g-?w@$q{bg>ON ztHVmd&U(})8-9j2Z9{GC3PIjX+qygJoH$?RX-Pu=EW<*69B)2ywHZ-AI)Sv2h22e2 zSANlV4y#b#a{%2?B)^5=dDnilujL5Ar?14Tq4?ADmpt~P3zg9`6^V+J6@Cm1Jz2_g z(Gk69Ry4$oDrpUWZ0#F!PJAS?#=2XPGTBrUY&R|z0{=LyTkOBF_9bN+U(CB|P`4Uj ztsHUwhoYU|Pg*Vjk4bscrjikrX>PJHz!>Bnaq8NN^dW~rmdw%@m6jK?2Qcz$| z*y58b38z%*7hS&wVnm4Yh!LkF z(CzA7=rzbe0vX3LaHi8wkP$;0qDeA4Sw8Ho&5q9z(M#o0r+2XunQNPK$+a;{^0buA z2%87dfC-M1hwae^N0mYdmqFk`D6fn?YALD5RZFc7!wocpf zOxH53fPvNSLIigq4vk}M-U#7ysf5$ZGYf*|u836~?Z}32I@nj4HUg3}ii%unq!AgK zRxEq( zr!2!77})KbD=}>hO=o~`V~_v_RoT$+12Ihf3CtF%V*n^=2HWU&Pt_luuNc^D$tpvR z5uu8nh6AyG`w?6AN9-FJ;033AnotINF9*Sb&ewt1I)Q=aYLp@x7@CkDtgrs?1cTpL zn{qsxX@D4ng%6tF3Rq|T&bsMG>}ym)!!qm@BY2#l>BnL2pv@M@=FmOYg;-1EaJ&1z zGr5t?p{&`3kR@U^Pzg7vgxAZkCPwgQhNeG<$*>mte`Dep!IISuW65eou<~`(y%1rn zW+iINtavV4t4zlov8v_F-t2M;XV0m!TZo37ZPvyBCQnwUcoiao_4)PWeWo^N{7OF2 zFuVCrb~grdbJ!;YIWDVW=^>OB>l0PR*rtt?(v3K~I{C^5YzyS-6fQB0Ia?=#4NUr0 zSIG5l#L$7=n?coq^xvBIvgR2Hkyef0xw|3&?(T$c#CBWhGECbyHjf$D&)*QOSX19q zl4CD?4|j(h2C4xn@kc7zAeGyHakilMP-TL!BFUH~M%yj0wE!gepT!$E@kJ9}eUH!C zT_6JI@j>L&3Y7EjRij^)g@}LQ^Ny7RO15LQ%7X+D(k66$8>lrPm%;ptEsnLy9n7E) z;tfLodmYCy!25-Df-LPn<-J-r^4ABmoYH{#7|G4WP*yW*WHod4Bnsr8X3_^9d>sgV zeYVBQq9EAVj^^EnJ1p4&zNE#<@}#pvKqGP9y~Tr()c2YV~IQ1{%Q-?WRy zHkGb7R&0=GoF951_61dp4_`Rf0j*%d{iKe`3_~|L<-)ziNhj;|!>iv&>3N?S?G9_p zy#c-{(+nS)U>MG@Xyenfk6{;hJ{d&KzqQ{OW-Fv6#Ae*yMmBcWTA|A8`83C?M96Dq z5))v3iDfE`Fc?~eQSnTvQ^-((ep#_|jIuFADfoeTERkZ@GphYzC6sdYeoPxoID}CU zvv0o%I(pxFPP5+xD`+Qgf;|A|H^K7j%W-t84e0vvTc-K0F<)(j(|liyOdAN~z%9~z zmm@)UbwEH@I7DO5WL0=cL1XzT3v6OG@bhv72=USz^se5p>lG-~mn#}o`oE`AuKQ>= zg=E>Q|GJQShuma;scg(X6MpJ~0l||(deD@;C$4&vqe(`v1*GeC6Dj+N%FL>A;LT;v z(Ji7A3UVE8OLk4Q*ZBj|MBItL#)XRiz0&67VQzD=-^a7n;|3IOH$~JPi;~o{waSpU z6>a!@JWku2KVVN0cS!9M6%SKL(DjEZy2=MSR>wgO|4_=8>+pRqGI7dAlyZr`JPu5f znN>z?fr>ruR>NntIBlS`SBtB@ZM{bCh*gFnm=<7tU#>$8OyMQ|fOjJ98^HvAGmHZT zzSp!}EpDJGnRkaNMDSjd55#zFN`VTWDWs~hD8~2?X!SU4uW#`#T4m&akM~2l3Imen zPT;^K{|;jeU`q2jFsmN#8vuKY-&sBRomDS~m2k+F0+oFn4=_`J>W4ooS7A~)fmdNp z{tol-cbHX=)#N-mR5`4&e~r`rjg$7o?gJf1&T@F59yDzSS}%U;f&V8eFtOjTEY4;C zelP$JRG$3Qy$3q}?^T=xzG2xx3uB-0kG}V^>H`%rE;M!1l)JQ{+$L*sCs}O;#c231A65P=h&Uk^%hqz$(t~ zD=7T7LcA-VdVQM-z>D*I01I!`hg0=u|Fanfx^Qfj&WX`mURDFes&*Vdo!66kA+_I; zz{dPe_?Cz}>O1H|mGJK=?v5MyPD56%#X0qh2H+j68*JD?V_Qrbm4%b z{7fK4zlCKNSrA()N5k?fNGF+;VG06P+WAjv(!Gy9%2l`=SdJ-BTWOa(4iuJoxe5n4 ziiPF&14n?u8JVj<4FHO&<#rwYPL^FBQkH5%LRr7jw)ALWIX{&$wDlQxN2o4Vj#-ys zLg49cxwn}uQp$>vcSy-;Pa7KPWE7*o#l?Ah`HaaLZjrU$4*tA1@iW!Ca6iTiyzd+jE`IQ) z_Ky!ah9f~d+8a^ZN=#fT{;B z!Jm*H#+>8k;&S!$cXJ8!g!%r#`j@RR$o}uwUrtB;%$kEe!;((h4){fbYulf&M*+5@ zqv?@j=2qvGl^p*=m~as4KZIGG)IBFoumPajCN8e6KZPO3xUA#ia)1v<%oD?*49<;b6Uc{(Ii1>^02JRKX<9 zuHoW3wDzw%DIb9Q!~RrjgTv P{y2i8G4DQt{lN8KXN1ji literal 0 HcmV?d00001 From 08860827e377a11dbc7984854054125c3b3858b6 Mon Sep 17 00:00:00 2001 From: binh-dam-ibigroup <56846598+binh-dam-ibigroup@users.noreply.github.com> Date: Tue, 19 Oct 2021 16:22:01 -0400 Subject: [PATCH 037/122] perf(GtfsValidationPlus): Close CSV reader after use. --- .../conveyal/datatools/manager/gtfsplus/GtfsPlusValidation.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/com/conveyal/datatools/manager/gtfsplus/GtfsPlusValidation.java b/src/main/java/com/conveyal/datatools/manager/gtfsplus/GtfsPlusValidation.java index 74408e5eb..c62b056b5 100644 --- a/src/main/java/com/conveyal/datatools/manager/gtfsplus/GtfsPlusValidation.java +++ b/src/main/java/com/conveyal/datatools/manager/gtfsplus/GtfsPlusValidation.java @@ -167,6 +167,7 @@ private static void validateTable( } rowIndex++; } + csvReader.close(); // Add issue for wrong number of columns after processing all rows. // Note: We considered adding an issue for each row, but opted for the single error approach because there's no From 05b8032d779ef7f488340f6e49753244ee0e614f Mon Sep 17 00:00:00 2001 From: binh-dam-ibigroup <56846598+binh-dam-ibigroup@users.noreply.github.com> Date: Thu, 21 Oct 2021 16:31:08 -0400 Subject: [PATCH 038/122] fix(MtcFeedResource): Add syncing Mongo with RTD carrier attributes after sending modified attribute --- .../controllers/api/FeedSourceController.java | 11 +-- .../extensions/mtc/MtcFeedResource.java | 88 ++++++++++++++++++- .../manager/extensions/mtc/RtdCarrier.java | 7 +- 3 files changed, 96 insertions(+), 10 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/controllers/api/FeedSourceController.java b/src/main/java/com/conveyal/datatools/manager/controllers/api/FeedSourceController.java index c8420cbb6..ce162c215 100644 --- a/src/main/java/com/conveyal/datatools/manager/controllers/api/FeedSourceController.java +++ b/src/main/java/com/conveyal/datatools/manager/controllers/api/FeedSourceController.java @@ -252,13 +252,14 @@ private static FeedSource updateExternalFeedResource(Request req, Response res) } // Hold previous value for use when updating third-party resource String previousValue = prop.value; - // Update the property in our database. - ExternalFeedSourceProperty updatedProp = Persistence.externalFeedSourceProperties.updateField( - propertyId, "value", entry.getValue().asText()); - // Trigger an event on the external resource + // Update the property with the value to be submitted. + prop.value = entry.getValue().asText(); + + // Trigger an event on the external resource. + // After updating the external resource, we will update Mongo with values sent by the external resource. try { - externalFeedResource.propertyUpdated(updatedProp, previousValue, req.headers("Authorization")); + externalFeedResource.propertyUpdated(prop, previousValue, req.headers("Authorization")); } catch (IOException e) { logMessageAndHalt(req, 500, "Could not update external feed source", e); } diff --git a/src/main/java/com/conveyal/datatools/manager/extensions/mtc/MtcFeedResource.java b/src/main/java/com/conveyal/datatools/manager/extensions/mtc/MtcFeedResource.java index 246c15ba7..a072a29ad 100644 --- a/src/main/java/com/conveyal/datatools/manager/extensions/mtc/MtcFeedResource.java +++ b/src/main/java/com/conveyal/datatools/manager/extensions/mtc/MtcFeedResource.java @@ -11,19 +11,27 @@ import com.conveyal.datatools.manager.models.FeedVersion; import com.conveyal.datatools.manager.models.Project; import com.conveyal.datatools.manager.persistence.Persistence; +import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.BufferedReader; import java.io.File; import java.io.IOException; +import java.io.InputStreamReader; import java.io.OutputStreamWriter; import java.net.HttpURLConnection; import java.net.MalformedURLException; import java.net.URL; +import java.util.ArrayList; import java.util.Collection; +import java.util.Iterator; +import java.util.List; +import java.util.Map; import static com.conveyal.datatools.manager.models.ExternalFeedSourceProperty.constructId; +import static com.mongodb.client.model.Filters.eq; /** * This class implements the {@link ExternalFeedResource} interface for the MTC RTD database list of carriers (transit @@ -139,7 +147,8 @@ public void feedSourceCreated(FeedSource source, String authHeader) throws Illeg } /** - * Sync an updated property with the RTD database. Note: if the property is AgencyId and the value was previously + * Sync a property with the RTD database, and syncs Mongo with data returned from RTD. + * Note: if the property is AgencyId and the value was previously * null create/register a new carrier with RTD. */ @Override @@ -161,6 +170,9 @@ public void propertyUpdated( // Otherwise, this is just a standard prop update. writeCarrierToRtd(carrier, false, authHeader); } + + // Fetch the agency properties from RTD and update the Mongo records from that instead of what was sent to RTD. + fetchCarrierFromRtdAndUpdateMongo(source, carrier, authHeader); } /** @@ -202,7 +214,6 @@ public void feedVersionCreated( * Update or create a carrier and its properties with an HTTP request to the RTD. */ private void writeCarrierToRtd(RtdCarrier carrier, boolean createNew, String authHeader) throws IOException { - try { ObjectMapper mapper = new ObjectMapper(); @@ -228,4 +239,77 @@ private void writeCarrierToRtd(RtdCarrier carrier, boolean createNew, String aut throw e; } } + + /** + * Fetch agency properties from RTD and update the ExternalFeedSourceProperty collection in Mongo. + */ + private void fetchCarrierFromRtdAndUpdateMongo(FeedSource source, RtdCarrier carrier, String authHeader) throws IOException { + try { + URL rtdUrl = new URL(rtdApi + "/Carrier/" + carrier.AgencyId); + LOG.info("Fetching to RTD URL: {}", rtdUrl); + HttpURLConnection connection = (HttpURLConnection) rtdUrl.openConnection(); + + connection.setRequestMethod("GET"); + connection.setRequestProperty("Content-Type", "application/json"); + connection.setRequestProperty("Accept", "application/json"); + connection.setRequestProperty("Authorization", authHeader); + + BufferedReader in = new BufferedReader(new InputStreamReader(connection.getInputStream())); + String inputLine; + StringBuilder response = new StringBuilder(); + + while ((inputLine = in.readLine()) != null) { + response.append(inputLine); + } + in.close(); + + LOG.info("RTD API response: {}/{}", connection.getResponseCode(), connection.getResponseMessage()); + + // Parse the response + ObjectMapper responseMapper = new ObjectMapper(); + JsonNode node = responseMapper.readTree(response.toString()); + + String resourceType = this.getResourceType(); + Iterator> fieldsIterator = node.fields(); + List rtdKeys = new ArrayList<>(); + + // Iterate over fields found in body and update external properties accordingly. + while (fieldsIterator.hasNext()) { + Map.Entry entry = fieldsIterator.next(); + ExternalFeedSourceProperty property = new ExternalFeedSourceProperty( + source, + resourceType, + entry.getKey(), + entry.getValue().asText() + ); + + // Update the attributes in Mongo. + ExternalFeedSourceProperty existingProperty = Persistence.externalFeedSourceProperties.getById( + property.id + ); + if (existingProperty != null) { + // TODO: convert "null" into an empty string ""? + Persistence.externalFeedSourceProperties.updateField( + property.id, + "value", + property.value + ); + } else { + Persistence.externalFeedSourceProperties.create(property); + } + + // Hold the received attribute keys to delete the extra ones from Mongo that are assumed not used. + rtdKeys.add(property.name); + } + + // Get the attributes stored in Mongo, remove those not in the RTD response. + Persistence.externalFeedSourceProperties.getFiltered(eq("feedSourceId", source.id)) + .stream() + .filter(property -> !rtdKeys.contains(property.name)) + .forEach(property -> Persistence.externalFeedSourceProperties.removeById(property.id)); + } catch (Exception e) { + LOG.error("Error writing to RTD", e); + throw e; + } + } } diff --git a/src/main/java/com/conveyal/datatools/manager/extensions/mtc/RtdCarrier.java b/src/main/java/com/conveyal/datatools/manager/extensions/mtc/RtdCarrier.java index 920d7ee32..b9521a7fa 100644 --- a/src/main/java/com/conveyal/datatools/manager/extensions/mtc/RtdCarrier.java +++ b/src/main/java/com/conveyal/datatools/manager/extensions/mtc/RtdCarrier.java @@ -100,11 +100,12 @@ private String getPropId(FeedSource source, String fieldName) { /** * Get the value stored in the database for a particular field. - * - * TODO: Are there cases where this might throw NPEs? */ private String getValueForField (FeedSource source, String fieldName) { - return Persistence.externalFeedSourceProperties.getById(getPropId(source, fieldName)).value; + ExternalFeedSourceProperty property = Persistence.externalFeedSourceProperties.getById( + getPropId(source, fieldName) + ); + return property != null ? property.value : null; } /** From ea6d0f80d0b8b97b3b6ec13879049fb5630071a0 Mon Sep 17 00:00:00 2001 From: miles-grant-ibi Date: Mon, 25 Oct 2021 15:46:22 +0100 Subject: [PATCH 039/122] feat(pelias): move webhook URL to project --- .../conveyal/datatools/manager/jobs/PeliasUpdateJob.java | 4 ++-- .../com/conveyal/datatools/manager/models/Deployment.java | 1 - .../java/com/conveyal/datatools/manager/models/Project.java | 6 ++++++ 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/PeliasUpdateJob.java b/src/main/java/com/conveyal/datatools/manager/jobs/PeliasUpdateJob.java index 3e8750dff..f4d358c5f 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/PeliasUpdateJob.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/PeliasUpdateJob.java @@ -76,7 +76,7 @@ public void jobLogic() throws Exception { } private void getWebhookStatus() { - URI url = getWebhookURI(deployment.peliasWebhookUrl + "/status/" + workerId); + URI url = getWebhookURI(deployment.parentProject().peliasWebhookUrl + "/status/" + workerId); // Convert raw body to JSON PeliasWebhookStatusMessage statusResponse; @@ -118,7 +118,7 @@ private void getWebhookStatus() { * @return The workerID of the run created on the Pelias server */ private String makeWebhookRequest() { - URI url = getWebhookURI(deployment.peliasWebhookUrl); + URI url = getWebhookURI(deployment.parentProject().peliasWebhookUrl); // Convert from feedVersionIds to Pelias Config objects List gtfsFeeds = Persistence.feedVersions.getFiltered(in("_id", deployment.feedVersionIds)) diff --git a/src/main/java/com/conveyal/datatools/manager/models/Deployment.java b/src/main/java/com/conveyal/datatools/manager/models/Deployment.java index 9c264a76b..5acd09000 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/Deployment.java +++ b/src/main/java/com/conveyal/datatools/manager/models/Deployment.java @@ -74,7 +74,6 @@ public class Deployment extends Model implements Serializable { private ObjectMapper otpConfigMapper = new ObjectMapper().setSerializationInclusion(Include.NON_NULL); /* Pelias fields, used to determine where/if to send data to the Pelias webhook */ - public String peliasWebhookUrl; public boolean peliasUpdate; public boolean peliasResetDb; public List peliasCsvFiles = new ArrayList<>(); diff --git a/src/main/java/com/conveyal/datatools/manager/models/Project.java b/src/main/java/com/conveyal/datatools/manager/models/Project.java index 9b7cbde17..28b3d677e 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/Project.java +++ b/src/main/java/com/conveyal/datatools/manager/models/Project.java @@ -90,6 +90,12 @@ public List availableOtpServers() { */ public String regionalFeedSourceId; + /** + * Webhook URL for the Pelias webhook endpoint, used during Pelias deployment. + */ + public String peliasWebhookUrl; + + public Project() { this.buildConfig = new OtpBuildConfig(); this.routerConfig = new OtpRouterConfig(); From b40040ddbb60bd27ef7d54357470a58f7129c0ea Mon Sep 17 00:00:00 2001 From: binh-dam-ibigroup <56846598+binh-dam-ibigroup@users.noreply.github.com> Date: Thu, 28 Oct 2021 17:34:12 -0400 Subject: [PATCH 040/122] refactor(MtcFeedResource): Convert "null" from RTD API to "" for UI display. --- .../manager/extensions/mtc/MtcFeedResource.java | 12 +++++++++++- .../extensions/mtc/MtcFeedResourceTest.java | 14 ++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) create mode 100644 src/test/java/com/conveyal/datatools/manager/extensions/mtc/MtcFeedResourceTest.java diff --git a/src/main/java/com/conveyal/datatools/manager/extensions/mtc/MtcFeedResource.java b/src/main/java/com/conveyal/datatools/manager/extensions/mtc/MtcFeedResource.java index a072a29ad..d0c9bf202 100644 --- a/src/main/java/com/conveyal/datatools/manager/extensions/mtc/MtcFeedResource.java +++ b/src/main/java/com/conveyal/datatools/manager/extensions/mtc/MtcFeedResource.java @@ -280,7 +280,7 @@ private void fetchCarrierFromRtdAndUpdateMongo(FeedSource source, RtdCarrier car source, resourceType, entry.getKey(), - entry.getValue().asText() + convertRtdString(entry.getValue().asText()) ); // Update the attributes in Mongo. @@ -312,4 +312,14 @@ private void fetchCarrierFromRtdAndUpdateMongo(FeedSource source, RtdCarrier car throw e; } } + + /** + * This method converts the RTD attribute value "null" to "" by MTC request, + * so that it is displayed in the UI under Mtc Properties as "(none)". + * @return An empty string if the provided string is the string "null", else the passed string itself. + */ + static String convertRtdString(String s) { + if ("null".equals(s)) return ""; + return s; + } } diff --git a/src/test/java/com/conveyal/datatools/manager/extensions/mtc/MtcFeedResourceTest.java b/src/test/java/com/conveyal/datatools/manager/extensions/mtc/MtcFeedResourceTest.java new file mode 100644 index 000000000..f790404f4 --- /dev/null +++ b/src/test/java/com/conveyal/datatools/manager/extensions/mtc/MtcFeedResourceTest.java @@ -0,0 +1,14 @@ +package com.conveyal.datatools.manager.extensions.mtc; + +import org.junit.jupiter.api.Test; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; + +class MtcFeedResourceTest { + @Test + void shouldConvertRtdNullToEmptyString() { + assertThat(MtcFeedResource.convertRtdString("null"), equalTo("")); + assertThat(MtcFeedResource.convertRtdString("Other text"), equalTo("Other text")); + } +} From be92d52c9a0eda524b485e6b7d39302b3e550781 Mon Sep 17 00:00:00 2001 From: binh-dam-ibigroup <56846598+binh-dam-ibigroup@users.noreply.github.com> Date: Fri, 29 Oct 2021 15:12:04 -0400 Subject: [PATCH 041/122] test(MtcFeedResource): Add tests --- .../extensions/mtc/MtcFeedResource.java | 83 +++++----- .../extensions/mtc/MtcFeedResourceTest.java | 143 +++++++++++++++++- .../__files/package-info.java | 7 + .../__files/rtdGetResponse.json | 1 + .../mtc-rtd-mock-responses.iml | 11 ++ 5 files changed, 205 insertions(+), 40 deletions(-) create mode 100644 src/test/resources/com/conveyal/datatools/mtc-rtd-mock-responses/__files/package-info.java create mode 100644 src/test/resources/com/conveyal/datatools/mtc-rtd-mock-responses/__files/rtdGetResponse.json create mode 100644 src/test/resources/com/conveyal/datatools/mtc-rtd-mock-responses/mtc-rtd-mock-responses.iml diff --git a/src/main/java/com/conveyal/datatools/manager/extensions/mtc/MtcFeedResource.java b/src/main/java/com/conveyal/datatools/manager/extensions/mtc/MtcFeedResource.java index d0c9bf202..a9ad5c9a1 100644 --- a/src/main/java/com/conveyal/datatools/manager/extensions/mtc/MtcFeedResource.java +++ b/src/main/java/com/conveyal/datatools/manager/extensions/mtc/MtcFeedResource.java @@ -265,52 +265,57 @@ private void fetchCarrierFromRtdAndUpdateMongo(FeedSource source, RtdCarrier car LOG.info("RTD API response: {}/{}", connection.getResponseCode(), connection.getResponseMessage()); - // Parse the response + // Parse the response and update Mongo. ObjectMapper responseMapper = new ObjectMapper(); JsonNode node = responseMapper.readTree(response.toString()); + updateMongoExternalFeedProperties(source, node); + } catch (Exception e) { + LOG.error("Error writing to RTD", e); + throw e; + } + } - String resourceType = this.getResourceType(); - Iterator> fieldsIterator = node.fields(); - List rtdKeys = new ArrayList<>(); - - // Iterate over fields found in body and update external properties accordingly. - while (fieldsIterator.hasNext()) { - Map.Entry entry = fieldsIterator.next(); - ExternalFeedSourceProperty property = new ExternalFeedSourceProperty( - source, - resourceType, - entry.getKey(), - convertRtdString(entry.getValue().asText()) - ); - - // Update the attributes in Mongo. - ExternalFeedSourceProperty existingProperty = Persistence.externalFeedSourceProperties.getById( - property.id + /** + * Updates Mongo using the provided JSON object from RTD. + */ + void updateMongoExternalFeedProperties(FeedSource source, JsonNode rtdResponse) { + String resourceType = this.getResourceType(); + Iterator> fieldsIterator = rtdResponse.fields(); + List rtdKeys = new ArrayList<>(); + + // Iterate over fields found in body and update external properties accordingly. + while (fieldsIterator.hasNext()) { + Map.Entry entry = fieldsIterator.next(); + ExternalFeedSourceProperty property = new ExternalFeedSourceProperty( + source, + resourceType, + entry.getKey(), + convertRtdString(entry.getValue().asText()) + ); + + // Update the attributes in Mongo. + ExternalFeedSourceProperty existingProperty = Persistence.externalFeedSourceProperties.getById( + property.id + ); + if (existingProperty != null) { + Persistence.externalFeedSourceProperties.updateField( + property.id, + "value", + property.value ); - if (existingProperty != null) { - // TODO: convert "null" into an empty string ""? - Persistence.externalFeedSourceProperties.updateField( - property.id, - "value", - property.value - ); - } else { - Persistence.externalFeedSourceProperties.create(property); - } - - // Hold the received attribute keys to delete the extra ones from Mongo that are assumed not used. - rtdKeys.add(property.name); + } else { + Persistence.externalFeedSourceProperties.create(property); } - // Get the attributes stored in Mongo, remove those not in the RTD response. - Persistence.externalFeedSourceProperties.getFiltered(eq("feedSourceId", source.id)) - .stream() - .filter(property -> !rtdKeys.contains(property.name)) - .forEach(property -> Persistence.externalFeedSourceProperties.removeById(property.id)); - } catch (Exception e) { - LOG.error("Error writing to RTD", e); - throw e; + // Hold the received attribute keys to delete the extra ones from Mongo that are assumed not used. + rtdKeys.add(property.name); } + + // Get the attributes stored in Mongo, remove those not in the RTD response. + Persistence.externalFeedSourceProperties.getFiltered(eq("feedSourceId", source.id)) + .stream() + .filter(property -> !rtdKeys.contains(property.name)) + .forEach(property -> Persistence.externalFeedSourceProperties.removeById(property.id)); } /** diff --git a/src/test/java/com/conveyal/datatools/manager/extensions/mtc/MtcFeedResourceTest.java b/src/test/java/com/conveyal/datatools/manager/extensions/mtc/MtcFeedResourceTest.java index f790404f4..8358a03c5 100644 --- a/src/test/java/com/conveyal/datatools/manager/extensions/mtc/MtcFeedResourceTest.java +++ b/src/test/java/com/conveyal/datatools/manager/extensions/mtc/MtcFeedResourceTest.java @@ -1,14 +1,155 @@ package com.conveyal.datatools.manager.extensions.mtc; +import com.conveyal.datatools.DatatoolsTest; +import com.conveyal.datatools.UnitTest; +import com.conveyal.datatools.manager.models.ExternalFeedSourceProperty; +import com.conveyal.datatools.manager.models.FeedSource; +import com.conveyal.datatools.manager.models.Project; +import com.conveyal.datatools.manager.persistence.Persistence; +import com.fasterxml.jackson.databind.JsonNode; +import com.github.tomakehurst.wiremock.WireMockServer; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; +import java.io.IOException; +import java.util.Date; + +import static com.conveyal.datatools.TestUtils.parseJson; +import static com.github.tomakehurst.wiremock.client.WireMock.aResponse; +import static com.github.tomakehurst.wiremock.client.WireMock.get; +import static com.github.tomakehurst.wiremock.client.WireMock.urlPathEqualTo; +import static com.github.tomakehurst.wiremock.core.WireMockConfiguration.options; +import static com.mongodb.client.model.Filters.and; +import static com.mongodb.client.model.Filters.eq; +import static io.restassured.RestAssured.given; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.nullValue; + +class MtcFeedResourceTest extends UnitTest { + private static Project project; + private static FeedSource feedSource; + private static WireMockServer wireMockServer; + + private static final String AGENCY_CODE = "DE"; + + /** + * Add project, server, and deployment to prepare for tests. + */ + @BeforeAll + static void setUp() throws IOException { + // start server if it isn't already running + DatatoolsTest.setUp(); + // Create a project, feed sources. + project = new Project(); + project.name = String.format("Test %s", new Date()); + Persistence.projects.create(project); + + feedSource = new FeedSource("Test feed source"); + feedSource.projectId = project.id; + Persistence.feedSources.create(feedSource); + + // This sets up a mock server that accepts requests and sends predefined responses to mock an Auth0 server. + wireMockServer = new WireMockServer( + options() + .usingFilesUnderDirectory("src/test/resources/com/conveyal/datatools/mtc-rtd-mock-responses/") + ); + wireMockServer.start(); + } + + @AfterAll + static void tearDown() { + wireMockServer.stop(); + if (project != null) { + project.delete(); + } + } -class MtcFeedResourceTest { @Test void shouldConvertRtdNullToEmptyString() { assertThat(MtcFeedResource.convertRtdString("null"), equalTo("")); assertThat(MtcFeedResource.convertRtdString("Other text"), equalTo("Other text")); } + + @Test + void canUpdateFeedExternalPropertiesToMongo() throws IOException { + final String rtdCarrierApiPath = "/api/Carrier/" + AGENCY_CODE; + + // create wiremock stub for get users endpoint + wireMockServer.stubFor( + get(urlPathEqualTo(rtdCarrierApiPath)) + .willReturn( + aResponse() + .withBodyFile("rtdGetResponse.json") + ) + ); + + // Set up some entries in the ExternalFeedSourceProperties collection. + // This one (AgencyId) should not change. + ExternalFeedSourceProperty agencyIdProp = new ExternalFeedSourceProperty( + feedSource, + "MTC", + "AgencyId", + AGENCY_CODE + ); + Persistence.externalFeedSourceProperties.create(agencyIdProp); + + // This one (AgencyPublicId) should be deleted after this test (not in RTD response). + ExternalFeedSourceProperty agencyPublicIdProp = new ExternalFeedSourceProperty( + feedSource, + "MTC", + "AgencyPublicId", + AGENCY_CODE + ); + Persistence.externalFeedSourceProperties.create(agencyPublicIdProp); + + // This one (AgencyEmail) should be updated with this test. + ExternalFeedSourceProperty agencyEmailProp = new ExternalFeedSourceProperty( + feedSource, + "MTC", + "AgencyEmail", + "old@email.example.com" + ); + Persistence.externalFeedSourceProperties.create(agencyEmailProp); + + // make RTD request and parse the json response + JsonNode rtdResponse = parseJson( + given() + .get(rtdCarrierApiPath) + .then() + .extract() + .response() + .asString() + ); + // Also extract desired values from response + String responseEmail = rtdResponse.get("AgencyEmail").asText(); + String responseAgencyName = rtdResponse.get("AgencyName").asText(); + + // Update MTC Feed properties in Mongo based response. + new MtcFeedResource().updateMongoExternalFeedProperties(feedSource, rtdResponse); + + // Existing field AgencyId should retain the same value. + ExternalFeedSourceProperty updatedAgencyIdProp = Persistence.externalFeedSourceProperties.getById(agencyIdProp.id); + assertThat(updatedAgencyIdProp.value, equalTo(agencyIdProp.value)); + + // Existing field AgencyEmail should be updated from RTD response. + ExternalFeedSourceProperty updatedEmailProp = Persistence.externalFeedSourceProperties.getById(agencyEmailProp.id); + assertThat(updatedEmailProp.value, equalTo(responseEmail)); + + // New field AgencyName (not set up above) from RTD response should be added to Mongo. + ExternalFeedSourceProperty newAgencyNameProp = Persistence.externalFeedSourceProperties.getOneFiltered( + and( + eq("feedSourceId", feedSource.id), + eq("resourceType", "MTC"), + eq("name", "AgencyName") ) + ); + assertThat(newAgencyNameProp, notNullValue()); + assertThat(newAgencyNameProp.value, equalTo(responseAgencyName)); + + // Removed field AgencyPublicId from RTD should be deleted from Mongo. + ExternalFeedSourceProperty removedPublicIdProp = Persistence.externalFeedSourceProperties.getById(agencyPublicIdProp.id); + assertThat(removedPublicIdProp, nullValue()); + } } diff --git a/src/test/resources/com/conveyal/datatools/mtc-rtd-mock-responses/__files/package-info.java b/src/test/resources/com/conveyal/datatools/mtc-rtd-mock-responses/__files/package-info.java new file mode 100644 index 000000000..df13e5fc2 --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/mtc-rtd-mock-responses/__files/package-info.java @@ -0,0 +1,7 @@ +/** + * All of the files in this directory (aside from this one) are mock responses used by a Wiremock server. The Wiremock + * server is used to emulate a 3rd party API (in this case Auth0) and it makes things less cluttered to store the json + * files in here instead of in the test class code. + */ + +package com.conveyal.datatools.mtc-rtd-mock-respones.__files; diff --git a/src/test/resources/com/conveyal/datatools/mtc-rtd-mock-responses/__files/rtdGetResponse.json b/src/test/resources/com/conveyal/datatools/mtc-rtd-mock-responses/__files/rtdGetResponse.json new file mode 100644 index 000000000..018834197 --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/mtc-rtd-mock-responses/__files/rtdGetResponse.json @@ -0,0 +1 @@ +{"AgencyId":"DE","AgencyName":"Dumbarton Express Consortium","AgencyPhone":null,"RttAgencyName":"Dumbarton Express","RttEnabled":"Y","AgencyShortName":"Dumbarton","AddressLat":null,"AddressLon":null,"DefaultRouteType":null,"CarrierStatus":null,"AgencyAddress":"AC Transit (administrator of the Dumbarton Express)","AgencyEmail":"new@email.example.com","AgencyUrl":"https://dumbartonexpress.com","AgencyFareUrl":"","EditedBy":"binh.dam@ibigroup.com","EditedDate":"2021-10-29T16:06:07.914796"} \ No newline at end of file diff --git a/src/test/resources/com/conveyal/datatools/mtc-rtd-mock-responses/mtc-rtd-mock-responses.iml b/src/test/resources/com/conveyal/datatools/mtc-rtd-mock-responses/mtc-rtd-mock-responses.iml new file mode 100644 index 000000000..de947d812 --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/mtc-rtd-mock-responses/mtc-rtd-mock-responses.iml @@ -0,0 +1,11 @@ + + + + + + + + + + + \ No newline at end of file From 785398dc5dfa73b5ce3eace7cb0f14e0189c5aea Mon Sep 17 00:00:00 2001 From: binh-dam-ibigroup <56846598+binh-dam-ibigroup@users.noreply.github.com> Date: Fri, 29 Oct 2021 15:17:26 -0400 Subject: [PATCH 042/122] refactor(MtcFeedResource): Refactor code. --- .../manager/extensions/mtc/MtcFeedResource.java | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/extensions/mtc/MtcFeedResource.java b/src/main/java/com/conveyal/datatools/manager/extensions/mtc/MtcFeedResource.java index a9ad5c9a1..0a5a18b6f 100644 --- a/src/main/java/com/conveyal/datatools/manager/extensions/mtc/MtcFeedResource.java +++ b/src/main/java/com/conveyal/datatools/manager/extensions/mtc/MtcFeedResource.java @@ -233,7 +233,12 @@ private void writeCarrierToRtd(RtdCarrier carrier, boolean createNew, String aut osw.write(carrierJson); osw.flush(); osw.close(); - LOG.info("RTD API response: {}/{}", connection.getResponseCode(), connection.getResponseMessage()); + LOG.info( + "RTD API {} response: {}/{}", + connection.getRequestMethod(), + connection.getResponseCode(), + connection.getResponseMessage() + ); } catch (Exception e) { LOG.error("Error writing to RTD", e); throw e; @@ -263,7 +268,7 @@ private void fetchCarrierFromRtdAndUpdateMongo(FeedSource source, RtdCarrier car } in.close(); - LOG.info("RTD API response: {}/{}", connection.getResponseCode(), connection.getResponseMessage()); + LOG.info("RTD API GET response: {}/{}", connection.getResponseCode(), connection.getResponseMessage()); // Parse the response and update Mongo. ObjectMapper responseMapper = new ObjectMapper(); From 3fc8d79507c45e0debea70254c05cc34c4686535 Mon Sep 17 00:00:00 2001 From: binh-dam-ibigroup <56846598+binh-dam-ibigroup@users.noreply.github.com> Date: Mon, 1 Nov 2021 14:43:24 -0400 Subject: [PATCH 043/122] chore(mtc-rtd-mock-responses.iml): Remove unneeded file. --- .../mtc-rtd-mock-responses/mtc-rtd-mock-responses.iml | 11 ----------- 1 file changed, 11 deletions(-) delete mode 100644 src/test/resources/com/conveyal/datatools/mtc-rtd-mock-responses/mtc-rtd-mock-responses.iml diff --git a/src/test/resources/com/conveyal/datatools/mtc-rtd-mock-responses/mtc-rtd-mock-responses.iml b/src/test/resources/com/conveyal/datatools/mtc-rtd-mock-responses/mtc-rtd-mock-responses.iml deleted file mode 100644 index de947d812..000000000 --- a/src/test/resources/com/conveyal/datatools/mtc-rtd-mock-responses/mtc-rtd-mock-responses.iml +++ /dev/null @@ -1,11 +0,0 @@ - - - - - - - - - - - \ No newline at end of file From b75eb29dee5e1313962276cb7603794c50369070 Mon Sep 17 00:00:00 2001 From: binh-dam-ibigroup <56846598+binh-dam-ibigroup@users.noreply.github.com> Date: Wed, 3 Nov 2021 14:58:34 -0400 Subject: [PATCH 044/122] test(MergeFeedsJobTest): Make tests pass (copied test to use merged test data) --- .../manager/jobs/MergeFeedsJobTest.java | 64 +++++++++++++++++-- 1 file changed, 58 insertions(+), 6 deletions(-) diff --git a/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java b/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java index c5a3fa40b..2a4d13798 100644 --- a/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java +++ b/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java @@ -450,8 +450,6 @@ public void mergeMTCShouldHandleDefaultStrategy() throws SQLException { @Test public void canMergeBARTFeeds() throws SQLException { Set versions = new HashSet<>(); - versions.add(bartVersion1); - versions.add(bartVersion2SameTrips); versions.add(bartVersionOldLite); versions.add(bartVersionNewLite); MergeFeedsJob mergeFeedsJob = new MergeFeedsJob(user, versions, "merged_output", MergeFeedsType.SERVICE_PERIOD); @@ -471,7 +469,6 @@ public void canMergeBARTFeeds() throws SQLException { ); // Check GTFS file line numbers. assertEquals( - // 4629, // Magic number represents the number of trips in the merged BART feed. 3, // Magic number represents the number of trips in the merged BART feed. mergeFeedsJob.mergedVersion.feedLoadResult.trips.rowCount, "Merged feed trip count should equal expected value." @@ -483,14 +480,12 @@ public void canMergeBARTFeeds() throws SQLException { ); assertEquals( // During merge, if identical shape_id is found in both feeds, active feed shape_id should be feed-scoped. - // bartVersion1.feedLoadResult.shapes.rowCount + bartVersion2SameTrips.feedLoadResult.shapes.rowCount, bartVersionOldLite.feedLoadResult.shapes.rowCount + bartVersionNewLite.feedLoadResult.shapes.rowCount, mergeFeedsJob.mergedVersion.feedLoadResult.shapes.rowCount, "Merged feed shapes count should equal expected value." ); // Expect that two calendar dates are excluded from the active feed (because they occur after the first date of - // the future feed) . - // int expectedCalendarDatesCount = bartVersion1.feedLoadResult.calendarDates.rowCount + bartVersion2SameTrips.feedLoadResult.calendarDates.rowCount - 2; + // the future feed). int expectedCalendarDatesCount = bartVersionOldLite.feedLoadResult.calendarDates.rowCount + bartVersionNewLite.feedLoadResult.calendarDates.rowCount - 2; assertEquals( // During merge, if identical shape_id is found in both feeds, active feed shape_id should be feed-scoped. @@ -506,6 +501,63 @@ public void canMergeBARTFeeds() throws SQLException { ); } + /** + * Tests that the MTC merge strategy will successfully merge BART feeds. + */ + @Test + public void canMergeBARTFeedsSameTrips() throws SQLException { + Set versions = new HashSet<>(); + versions.add(bartVersion1); + versions.add(bartVersion2SameTrips); + MergeFeedsJob mergeFeedsJob = new MergeFeedsJob(user, versions, "merged_output", MergeFeedsType.SERVICE_PERIOD); + // Result should succeed this time. + mergeFeedsJob.run(); + assertFeedMergeSucceeded(mergeFeedsJob); + // Check GTFS+ line numbers. + assertEquals( + 2, // Magic number represents expected number of lines after merge. + mergeFeedsJob.mergeFeedsResult.linesPerTable.get("directions").intValue(), + "Merged directions count should equal expected value." + ); + assertEquals( + 2, // Magic number represents the number of stop_attributes in the merged BART feed. + mergeFeedsJob.mergeFeedsResult.linesPerTable.get("stop_attributes").intValue(), + "Merged feed stop_attributes count should equal expected value." + ); + // Check GTFS file line numbers. + assertEquals( + 4629, // Magic number represents the number of trips in the merged BART feed. + mergeFeedsJob.mergedVersion.feedLoadResult.trips.rowCount, + "Merged feed trip count should equal expected value." + ); + assertEquals( + 9, // Magic number represents the number of routes in the merged BART feed. + mergeFeedsJob.mergedVersion.feedLoadResult.routes.rowCount, + "Merged feed route count should equal expected value." + ); + assertEquals( + // During merge, if identical shape_id is found in both feeds, active feed shape_id should be feed-scoped. + bartVersion1.feedLoadResult.shapes.rowCount + bartVersion2SameTrips.feedLoadResult.shapes.rowCount, + mergeFeedsJob.mergedVersion.feedLoadResult.shapes.rowCount, + "Merged feed shapes count should equal expected value." + ); + // Expect that two calendar dates are excluded from the active feed (because they occur after the first date of + // the future feed). + int expectedCalendarDatesCount = bartVersion1.feedLoadResult.calendarDates.rowCount + bartVersion2SameTrips.feedLoadResult.calendarDates.rowCount - 2; + assertEquals( + // During merge, if identical shape_id is found in both feeds, active feed shape_id should be feed-scoped. + expectedCalendarDatesCount, + mergeFeedsJob.mergedVersion.feedLoadResult.calendarDates.rowCount, + "Merged feed calendar_dates count should equal expected value." + ); + // Ensure there are no referential integrity errors or duplicate ID errors. + assertThatFeedHasNoErrorsOfType( + mergeFeedsJob.mergedVersion.namespace, + NewGTFSErrorType.REFERENTIAL_INTEGRITY.toString(), + NewGTFSErrorType.DUPLICATE_ID.toString() + ); + } + /** * Tests whether a MTC feed merge of two feed versions correctly feed scopes the service_id's of the feed that is * chronologically before the other one. This tests two feeds where one of them has both calendar files, and the From 9948c42ff6f01217fe5df084afad09aa20bc1d42 Mon Sep 17 00:00:00 2001 From: binh-dam-ibigroup <56846598+binh-dam-ibigroup@users.noreply.github.com> Date: Wed, 3 Nov 2021 16:15:54 -0400 Subject: [PATCH 045/122] test(MergeFeedsJobTest): Make other failing tests pass --- .../conveyal/datatools/manager/jobs/MergeFeedsJobTest.java | 5 ----- .../com/conveyal/datatools/gtfs/mini-bart-new/trips.txt | 2 +- .../com/conveyal/datatools/gtfs/mini-bart-old/trips.txt | 2 +- 3 files changed, 2 insertions(+), 7 deletions(-) diff --git a/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java b/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java index 2a4d13798..9ba974580 100644 --- a/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java +++ b/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java @@ -149,11 +149,6 @@ public static void tearDown() { public void canMergeRegional() throws SQLException { // Set up list of feed versions to merge. Set versions = new HashSet<>(); - napaVersion = createFeedVersionFromGtfsZip(napa, "napa-no-agency-id.zip"); - calTrainVersion = createFeedVersionFromGtfsZip(caltrain, "caltrain_gtfs.zip"); - versions.add(bartVersion1); - versions.add(calTrainVersion); - versions.add(napaVersion); versions.add(bartVersionOldLite); versions.add(calTrainVersionLite); versions.add(napaVersionLite); diff --git a/src/test/resources/com/conveyal/datatools/gtfs/mini-bart-new/trips.txt b/src/test/resources/com/conveyal/datatools/gtfs/mini-bart-new/trips.txt index aa89b063e..056042142 100755 --- a/src/test/resources/com/conveyal/datatools/gtfs/mini-bart-new/trips.txt +++ b/src/test/resources/com/conveyal/datatools/gtfs/mini-bart-new/trips.txt @@ -1,2 +1,2 @@ route_id,service_id,trip_id,trip_headsign,direction_id,block_id,wheelchair_accessible,bikes_allowed -1,WKDY,3610458WKDY,San Francisco International Airport,0,1,1 +1,WKDY,3610458WKDY,San Francisco International Airport,0,1,1,1 diff --git a/src/test/resources/com/conveyal/datatools/gtfs/mini-bart-old/trips.txt b/src/test/resources/com/conveyal/datatools/gtfs/mini-bart-old/trips.txt index 1e7e82d6c..c3d8d527c 100755 --- a/src/test/resources/com/conveyal/datatools/gtfs/mini-bart-old/trips.txt +++ b/src/test/resources/com/conveyal/datatools/gtfs/mini-bart-old/trips.txt @@ -1,2 +1,2 @@ route_id,service_id,trip_id,trip_headsign,direction_id,block_id,wheelchair_accessible,bikes_allowed -01,WKDY,3610403WKDY,San Francisco International Airport,0,1,1 +01,WKDY,3610403WKDY,San Francisco International Airport,0,1,1,1 From e90b8c1e126267f0984a139725010523284e9136 Mon Sep 17 00:00:00 2001 From: binh-dam-ibigroup <56846598+binh-dam-ibigroup@users.noreply.github.com> Date: Thu, 4 Nov 2021 15:39:19 -0400 Subject: [PATCH 046/122] refactor: Close resources, address some PR comments. --- .../conveyal/datatools/manager/jobs/FeedToMerge.java | 7 ++++++- .../datatools/manager/jobs/MergeFeedsJob.java | 4 +++- .../datatools/manager/jobs/MergeFeedsResult.java | 2 -- .../datatools/manager/jobs/MergeLineContext.java | 11 ++++++++--- .../datatools/manager/utils/MergeFeedUtils.java | 10 +--------- 5 files changed, 18 insertions(+), 16 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/FeedToMerge.java b/src/main/java/com/conveyal/datatools/manager/jobs/FeedToMerge.java index 56ddcf413..ee9fde233 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/FeedToMerge.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/FeedToMerge.java @@ -6,6 +6,7 @@ import com.google.common.collect.SetMultimap; import com.google.common.collect.Sets; +import java.io.Closeable; import java.io.IOException; import java.util.HashSet; import java.util.Set; @@ -17,7 +18,7 @@ * Helper class that collects the feed version and its zip file. Note: this class helps with sorting versions to * merge in a list collection. */ -public class FeedToMerge { +public class FeedToMerge implements Closeable { public FeedVersion version; public ZipFile zipFile; public SetMultimap idsForTable = HashMultimap.create(); @@ -37,4 +38,8 @@ public void collectTripAndServiceIds() throws IOException { serviceIds.addAll(idsForTable.get(Table.CALENDAR)); serviceIds.addAll(idsForTable.get(Table.CALENDAR_DATES)); } + + public void close() throws IOException { + this.zipFile.close(); + } } \ No newline at end of file diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java b/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java index fa385a0b5..eb8c3e4bc 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java @@ -271,6 +271,9 @@ public void jobLogic() throws IOException, CheckedAWSException { } } } + for (FeedToMerge feed : feedsToMerge) { + feed.close(); + } if (!mergeFeedsResult.failed) { // Store feed locally and (if applicable) upload regional feed to S3. storeMergedFeed(); @@ -388,7 +391,6 @@ private int constructMergedTable(Table table, List feedsToMerge, // Iterate over each zip file. For service period merge, the first feed is the future GTFS. for (int feedIndex = 0; feedIndex < feedsToMerge.size(); feedIndex++) { ctx.startNewFeed(feedIndex); - mergeFeedsResult.feedCount++; if (ctx.skipFile) continue; LOG.info("Adding {} table for {}{}", table.name, ctx.feedSource.name, ctx.version.version); // Iterate over the rows of the table and write them to the merged output table. If an error was diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsResult.java b/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsResult.java index e29666323..d39355d2c 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsResult.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsResult.java @@ -13,8 +13,6 @@ public class MergeFeedsResult implements Serializable { private static final long serialVersionUID = 1L; - /** Number of feeds merged */ - public int feedCount; /** Type of merge operation performed */ public MergeFeedsType type; public MergeStrategy mergeStrategy = MergeStrategy.DEFAULT; diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/MergeLineContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/MergeLineContext.java index aae975a5b..051b8c053 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/MergeLineContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/MergeLineContext.java @@ -487,10 +487,14 @@ private void checkRoutesAndStopsIds(Set idErrors) throws IOExcepti // future routes/stops file. if (useAltKey()) { if (hasPrimaryKeyErrors(primaryKeyErrors)) { - // If alt key is empty (which is permitted), skip + // If alt key is empty (which is permitted) and primary key is duplicate, skip // checking of alt key dupe errors/re-mapping values and // simply use the primary key (route_id/stop_id). - skipRecord = true; + // + // Otherwise, allow the record to be written in output. + if (hasDuplicateError(primaryKeyErrors)) { + skipRecord = true; + } } else if (hasDuplicateError(idErrors)) { // If we encounter a route/stop that shares its alt. // ID with a previous route/stop, we need to @@ -552,7 +556,7 @@ private void checkRoutesAndStopsIds(Set idErrors) throws IOExcepti } private boolean hasPrimaryKeyErrors(Set primaryKeyErrors) { - return "".equals(keyValue) && field.name.equals(table.getKeyFieldName()) && hasDuplicateError(primaryKeyErrors); + return "".equals(keyValue) && field.name.equals(table.getKeyFieldName()); } private boolean useAltKey() { @@ -605,6 +609,7 @@ public void checkStopCodeStuff() throws IOException { if (isSpecialStop(locationType)) specialStopsCount++; else if (stopCodeIsMissing) stopsMissingStopCodeCount++; } + stopsReader.close(); LOG.info("total stops: {}", stopsCount); LOG.info("stops missing stop_code: {}", stopsMissingStopCodeCount); if (stopsMissingStopCodeCount + specialStopsCount == stopsCount) { diff --git a/src/main/java/com/conveyal/datatools/manager/utils/MergeFeedUtils.java b/src/main/java/com/conveyal/datatools/manager/utils/MergeFeedUtils.java index 4f846d46a..b9de7ccaf 100644 --- a/src/main/java/com/conveyal/datatools/manager/utils/MergeFeedUtils.java +++ b/src/main/java/com/conveyal/datatools/manager/utils/MergeFeedUtils.java @@ -4,8 +4,6 @@ import com.conveyal.datatools.manager.jobs.FeedToMerge; import com.conveyal.datatools.manager.jobs.MergeFeedsJob; import com.conveyal.datatools.manager.jobs.MergeFeedsType; -import com.conveyal.datatools.manager.jobs.MergeLineContext; -import com.conveyal.datatools.manager.jobs.MergeStrategy; import com.conveyal.datatools.manager.models.FeedRetrievalMethod; import com.conveyal.datatools.manager.models.FeedSource; import com.conveyal.datatools.manager.models.FeedVersion; @@ -17,13 +15,10 @@ import com.conveyal.gtfs.loader.Table; import com.conveyal.gtfs.model.StopTime; import com.csvreader.CsvReader; -import com.google.common.collect.Sets; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; -import java.time.LocalDate; -import java.time.temporal.ChronoUnit; import java.util.Arrays; import java.util.Collection; import java.util.Comparator; @@ -32,16 +27,12 @@ import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; -import java.util.zip.ZipEntry; import java.util.zip.ZipFile; import static com.conveyal.datatools.manager.jobs.MergeFeedsType.REGIONAL; import static com.conveyal.datatools.manager.jobs.MergeFeedsType.SERVICE_PERIOD; -import static com.conveyal.datatools.manager.jobs.MergeStrategy.CHECK_STOP_TIMES; -import static com.conveyal.datatools.manager.jobs.MergeStrategy.EXTEND_FUTURE; import static com.conveyal.datatools.manager.models.FeedRetrievalMethod.REGIONAL_MERGE; import static com.conveyal.datatools.manager.models.FeedRetrievalMethod.SERVICE_PERIOD_MERGE; -import static com.conveyal.gtfs.loader.DateField.GTFS_DATE_FORMATTER; import static com.conveyal.gtfs.loader.Field.getFieldIndex; public class MergeFeedUtils { @@ -127,6 +118,7 @@ public static Set getAllFields(List feedsToMerge, Table tabl // Get fields found from headers and add them to the shared fields set. Field[] fieldsFoundInZip = table.getFieldsFromFieldHeaders(csvReader.getHeaders(), null); sharedFields.addAll(Arrays.asList(fieldsFoundInZip)); + csvReader.close(); } return sharedFields; } From 8179f873f950f83394f7c628131f9b2e73fc72e1 Mon Sep 17 00:00:00 2001 From: binh-dam-ibigroup <56846598+binh-dam-ibigroup@users.noreply.github.com> Date: Thu, 4 Nov 2021 17:21:01 -0400 Subject: [PATCH 047/122] refactor(MergeFeedsJob): Improve exception handling. --- .../datatools/manager/jobs/MergeFeedsJob.java | 81 +++++++++++++------ .../manager/jobs/MonitorServerStatusJob.java | 7 +- .../datatools/manager/utils/ErrorUtils.java | 16 ++++ .../manager/utils/MergeFeedUtils.java | 4 +- 4 files changed, 78 insertions(+), 30 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java b/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java index eb8c3e4bc..b7f15e222 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java @@ -219,12 +219,11 @@ public void jobFinished() { try { Files.delete(mergedTempFile.toPath()); } catch (IOException e) { - LOG.error( + logAndReportToBugsnag( + e, "Merged feed file {} not deleted. This may contribute to storage space shortages.", - mergedTempFile.getAbsolutePath(), - e + mergedTempFile.getAbsolutePath() ); - ErrorUtils.reportToBugsnag(e, owner); } } @@ -233,14 +232,21 @@ public void jobFinished() { * the resulting zip file to storage. */ @Override - public void jobLogic() throws IOException, CheckedAWSException { + public void jobLogic() { // Create temp zip file to add merged feed content to. - mergedTempFile = File.createTempFile(filename, null); + try { + mergedTempFile = File.createTempFile(filename, null); + } catch (IOException e) { + + String message = "Error creating temp file for feed merge."; + logAndReportToBugsnag(e, message); + status.fail(message, e); + } mergedTempFile.deleteOnExit(); // Create the zipfile with try with resources so that it is always closed. try (ZipOutputStream out = new ZipOutputStream(new FileOutputStream(mergedTempFile))) { LOG.info("Created merge file: {}", mergedTempFile.getAbsolutePath()); - feedsToMerge = collectAndSortFeeds(feedVersions); + feedsToMerge = collectAndSortFeeds(feedVersions, owner); // Determine which tables to merge (only merge GTFS+ tables for MTC extension). final List

(QG0Vrc{`rmqY#ea0Q6)31u-R6wdox(oQV@HQ^S@x(h0e_+hE8F!X9>}w zAv=Tx?C`8V9GC_%>~Qi84$KHDnN2zeSjVw+Y>9`F=z1|OJS&p}vyGgRQ}p6PT<70y zalCRRBO97Go8}60beT^nPSvOiR-^%599;<3X)pN+5o1O%*N3w+TeUlN(!4V{VQMsH zy1519Sz6wKDG2>>o*g2jH5!_pFtumJh-Y6y61nK{LM5^8*LLsieYnKbsP^6yq2)tO z79#^o=zSPsUD%+;fAYmPam1-C0d%8FzH^`}Ux>F;4VRG>QgtyYWz3nDZ$7jnNnGl5 zqy@SxJh1c{-;hOEpSNW&-B8=Jk(r)$@6>+EHiefjoQ(B!yPnH*9E3xX!CdVwbUo`4 zdCem%t{|M!_JW8D4Sa6-T=t1*%A92f-Q#jKF$?Rt-h_?)h2^5Z?8Kp@*J*rzeu+bA z$34(i7g;xg%_l!SM0QWvy2W-)d`n1~m?(SKf?Yz%LP&kTa5Wwq^aPsm^_W%@iuEm? zd+Gb^rArO7GxMm4k!f5(1A~&WQI8HAW_1snK8us=I_Ke!pH&wr@49GI0sQ5zgRkI; zcB9UD!JftE?MmkwW_nyjR(-t;?HBHFroZE@v^hi(LI^y+p0B^Xb2=0}>63DgXXNc$t?1F_N+VoU$*{jhCc zyM9VWdiXGOiodQtxG%Kt`R55Td3VI21fQ&_Q8mEtcwQs6SXL`l773$W<=t)E9~=6J zHWQMOzBi|(&lgTl@6ez_h=k4)mTRwUdc|vbl7hf%Z{Qa>hJ(e)1_* zBK=Rpr``m9P;Ib%6x=H50eratIKhWeOu7bgj}^0M#CTu6aJOv&zB(>%(=0pvPd0zG zrpa9EBupmy8heH4_09{fF{g7?xkd#V8$ts<1m$9Y6Y6u%TcLSa1qj`k(H6I z9lKREOo;v*#1;W@n1A1hn?|>S$jginM!I$oSb8?vaAL8E$C$=42{#HfgvUrEHn0Hx z{(JMy^Y?+K6QY-cup{{j8`tvFUep;p<}{HLjW?SB+AB?O_X{Vyp=%EzOe$Y*<7r78 zWYST!bNlv}Vg!vA@4DKv39KXq^0qzmZ9?X>p*@6b8;u_Ca`#(WaM%IIbl>HmJ_MBe z`_$Z3w-S2s&UebZ z)Aq4En2jMU7Mpx!si?L_V50}grxmZOfD8+jmB3J&LJ&8Bu=s4<8DIv__$o{5Cyn^i zc|NlI)L!!95eyk3(MXOmLBy_`bG>5&1=eb3Q6b0gK$e1<-uZ2S&i~6ls=LA=@P)=G zy<>O}sLA}2k{2S*)%NJWQVOj2L)lXAcyRdD{uVF4UXnN0K%$3A3g!^3J4n z7L(eR>53wEYt+@P+;SF+TF`fxti5=( za2F2YT`UqNFMLD!aauBNbP=YzV&heQ+<;?=l^(*DZNrNeP<_zycC~eb#i7;>lzV2A zQ|;4u0C8i}Kg_nz{93H*V?Wi|USGeqIv zS042uJCZU6_Ag5{ zX;)L=fWTHhSbcI1SnjAd4%3$-hORfBl<^gVa7460pD~_>K>X~-F+W-mvS`+K=*fZk zExlv^vWG5ZK`fCg}8pt7AJ6sjEoCs%F9tS`*ErQ4JEYd^??bF94^X%O;6g7@rEh~ChUTl zUh~vNX9I?=LIciTvA4u~pQ%3X@i(|}resu%?-hTSLSQQdsJYQO&0@~8kiE4+5<^!w z_<~soQ@WCBQADCQXY$KYY)d?6WxAkQOK}ygg#FUO(QjdHU6}~eKBsj!YLqW$3gzSL zMCP)GnWTuRQz3in;`rO%h7JU^@5fNezkLny)R~w){zNh^cI;hOU3K&sf*T{lrs@RA zhTMXD$trl6C~ri;ol{TZ+MYGfqye>Kr=cM#4xd^XQL>%${RYqae5&vlBWqz0wU^*B zq;d{Ue=`Yi5DV}0K3T-kF13;~^^=_Qak!yE_U+dWNNJ}>=82W??Zc9jY#dVE1tlz> z8fDM|o4F3}0_|pKSbH7ql%iW!(+eehYspA2#w@C-x0+(hg^@KA^H5?QC1UGkvJ&3+ z?|mBGuhI5n+wWMHT2BRNA68@n;?9QTGB4cA!9;KdLJbEkwP6&!GD0twQ0WZ~$X=L=tu*F8emWEokCJD#e$q4hp$zBSXM;W>dmnXM zp-=9$?(;2`Fc5Lxe30`!inrf`ndZK}W+vfb4@Pfp5QsQ8v>xg#^1-EZH_Wdh>!*^l^S@usaxrsq%AQa0 zai#`~CghZyt)9ExDTi{#*WYG{CKQ)AAIfLlU4b)qg8U18d}S8?EL%FWN^0-iwTK>^ z3VtZ_3&klL;6x%qE{?rZ{d%i7Zk}G>}yuxymv;c z=1jWB<$VIohCX4jou@C>2MjfO=bsIB=#+q1+d7UEAb-bx#vyb6l!@Qy)ZeLUM_l$) zYF$BcHVl67RLouNQ>@~cS{G_Q^R|iD-)B}qk>Z_ltA~F$F|VpT&3_i}P%)&X;f=3% z#T^7QHAS0e7cjv)u0d6M?k8Qs(m_DLs|IIi9#@PGg%8SE#9qp(eX8V{(HKkMOdwp2 zHjfKydo`uOXIi0cPR~>wP0@j4%E;7Oj5AbynFFfR;CsSA2aZ*mdPxnAFsf~I3S#3> zgR}Xkd^mFJp4SLMHSj8^T64;v-MM+mQ3u(2E&|Z+J%y53_TwO?aX!$A5IG!|QN^-Q zUwUJ0I19h#ktE4bgwlUR@4wjMak#mEcChK7{_y*($^n5;*D%{&b-2#~exCKsHBVO% z>Z`e(3GM1Yn3{%+KE^5qL4U0FENMP^y;UbC(zEp!g9NiWhZ7#Ai40ZMY}o{MtXxvO z=TiBdrP&4jt!2MxUhboE#Sw-Dc2=X(4)m6-hnN;+ekpPH*K^MX;dz>Ve$9`j2xg}a zd{#n5^Gv9jT8(>s@urfaOiaU)gM=bMn6TY__f!WHD|{g0(1jx?s`6tjyZn+sB+6rM zsan^z)29XQj9`@baO1kOD+|Ot=;2$!|S>t14;T-IKL1sKL+1_ zvj9=zpjlyMiFbT%TNAl#I3T9sEv;arr~`KFDyqOAIVH`xPk@Z_73PhPN`2G(8ISY= zj6kbk4VG)01QiH>_OS4m9kOB-`B2vRHn19ic{;N)2 zI(1jsOO)WjT-2c#;9#hrugAb}6>I3vAYlqDRizjS`VpaZ^dLZ{on0!MS&m;o6%}Ab zUUyQd6V0PMxlZ|K-13wHeuX170!FM-FAd*FV8#z#&IKqMacY&|!Opr*xRXdT>ya)2 z2o<>@0*8jw83sSNhk%OOqyjh(b2iPracP};_;2ta0}_Qo4)oK=V88Hl8cFb(Tdg@& zh@-9z;(OF5rDH*She5S6{IG>3y`Lxi4O9i?0T4AA72i~<2vN9iQ)IkMrR6Fsl9+_c zEQh=zB;(90esltn5`h~)w*SPt5dyx?J3&yHl&pDv2qG!0U;$zR5Hd1amf~Dl8a4jE zvW=jr`f*D6KNr{jU4hpN?4m2Y8TXh?hzLun{NYNxX~FM^*dyaQ8L?7I4-VQ4r*bg> zPC-MvpEZ5|BjrXg zlPNlWg@OIFfvXqqXMFZ1f)q~>47Yc{LjJzM280O9>=urIm*TEjoKi;50=-d2B4UX| zuSq88P=SyEL6Oge*uW{2GDwAaU>s3U13X3);@$coquzvzI#^nj(5$Po-ifC@DQ`Bdmr5CxB>6xn=mp`*i#kk` z+Re}WnZ0b4(9Ej>I6A2f<@iqO^n8T=qzq{Sjs2m05MnFLgmwQQ!8YMxjmB#A)%o4j z>W7DU(5xPEPcj0{OT-0f4*r%}x2_+wkX3=o$*AmEN%e|$k+S$-%=m4+6nPQRS4*vV zQP^haEQ;IA^(TfyDYBFEb>BztZh)zxmvrXQCq8OtqNUTIKc4aJ^QUrAS5u==+P7IX zT6z|j%XxA5gumL(V$E0%CRoH&9nx<6{v@s7eHoo(py6e`o3?{&c5Pk9nmeeu#m!Wl z&VG}UWlMP~ALr>?aKljCJ_V<)-x#F9DRa@5<@~GUN)<1JUL7pR;PB$&-F|pTtdl&g z;?~XGlfM(S_e=x2dfzzwY&~eFh2$LFv_%*F48B|imT=R|Mr!9Bi=cDe-6tl|)T8Zd z2CgR-$fB{?XqJ_uS>w`>G&`(D$}^v*-zRPyb-p<+|H{H#l(bRt@aL$Ak`wx07rJ5J-fNdMG;{Ag+0<&^DHGA>Ha_-seKRsu(m203@>C6iGVGcRQg039$w+lG z@OMMLN$8%`i*A``uH<#xxXROByuN*>&1d+&W`mQJW%rP&kF$Z0ZZ-;4Wvj4X>vQiL zTX|mNatYgVGs-XT<#}UX`~J4hH>04_mgmLU5L+%Q=6@7t^AC=Jmv_U7E>W+rK+p_<=AWKt`=iiCpquO=lfCfze2;pQr9td ziev-#F_+j&hN_?K@Xj)-jGk>#D6MbHR&DJcHqD_o^V-FewwAtrKfH!=3VUvnEL66- z&>@%F<97Ltq(0Acf%etInX^6z&->bE#Rs*T(8m>bX>~nQE4?`Gs$G;Gv@sv8KVEEU z#Wvoj4Qzx{wr3d!#S7EHg}&$&HKO$qn>l)%o1ag2gh$-+#SeW;AmeV*Su}U_7(Pc_ z?4>nVy3r<23$R#K9JqI#G> zwv;HPv3Fi?tYRO=xvuqlPS*JTmf0_B+hFRTsv+!R{dtMXA)RF^q*pvh#ydWme=IFz^#Hcl$mF15JGNi*-}2TZRm-eQh+90$ki^+F&up<7z~?r}p{ z+G+a6*+!$XtmR#4wFcO!?Xs-)dI`IxWixqxal`YBs?B^B{2E?y!zubKtG2Vao^eCG zC4`4c9uG%>QRKJLSanw3fo=mNi^)x-i>z;Am9uHs1wl?>}sSTcYH_{l){Zx!#8$P8Tm^SmZZ1rW$ zr~d5gl204%+mufee*WNB?S-|p>f0aQJF$bdN7K=zC10<8h$qdjm%p?XkUh=sDjV3a zf9a<{>qjDL#d$T)t{y+M99o@fbH~d(WpbK4UPGJQhW&%5;l{S2eBhV8xTFHBAC`!Y zms@|CX^RlUI z0nUDLZ#$2Dy>57l4tXqF+r^d0{DPt~h3?RpggtJsJv~n@NrnDw9P?r9fu`7}fhgGy z)X8nxrjA!k*+7s$J3z40CEr7qk~gv1=;GTHA5q7*4bv`=`DMiO%bu888H>oCnnsu` z(fVar!9gRiS;YKCxteG--RBXCVbE`U6(`!^WXw{Jv>TTXHlR|4ieZz}y70zC(C5{< zxZ!2mCz@4P^Qaqgynbo@dIOtmr#VcsH?}%1G-+)Lqh%noQ&muw6S{a$8#GbDUBQ)Pn)Kw z#cd8bjx-40)iOHk#CdD$y%x4~{=6GSeN{E-Vm8$uF{=%_wR!-$$+D7Ra&i&UzHw3q zk#fAC`Qak#c#(fq)hzqF^^HN2?7(93_K+h}!!@3db}fyY^my`514p+>Z{0Vv?$YCR zHQQT7v}~4swF?&pUw+ZsteG8?!N@KhRjLG-DFnZdlp zbQXe1>6#>DKW!X02rr=CX(K(SCkb=nke$#WFX!lIYZ75vRxv)z|LgRG^I?LGL#5*H z8>J|w2PyIZexaIri9amx^HEPUIlo0MS7#zxiAhG>*?&qVWu(#-cr#X>x3iSr*Tc z#q1{As=Pd}Uu57IgK%q9P8tUJ)pZL_8PyR3F5Az4bYVJ*0fLuQ{XHjQlj|49T2BGC!BE z8PB~psE+rD^d+R5;JyD(aY-7I`r3n>&tiAO$3(m_%6mVjIR0Whyg+fOj}4i!YO%KA z1N|EQp&65Z{5%<=Nu&YJP-)QO%x=jOu5n4M3`#N>x+Z}n}P zaTs0x9EQ~rzIlpdD!gzx8Px2sy!3;L zRIge6%HR*;I=a^=$+ES-SEmR;-&(pa-4-QXx+}fRaJer;A(7?Zy0e#B&ceL#V>%z@ zt@CSjvvFn!%5quZy;dt^N)mS{2Az@Btg48#RKN~PTsmiU9&36P|s&6ZD7vg_J5JIv9d9+nd z8miM0*|txi_1#6=PFq?`5h8A>wYl5zsx5@3SqW@q7&jYVouszu%*&VpMH#O3pWY3S zoJBlPR_xLY_IG4$qrynQM|dx!QttHT9n2_i1QN}jQVm}5M^dOL?rH~zt9}@xf+fU9 z5(VTYWPisX_(fePRYY$t#}X*TFmLsgd5}am(m_S>S_d^sb$5o!FrjLc=wV)ZM=b`) zFYbD&G5W0^ES+yPd6%e65>Bs(#PiY#>M=-vAv&d2=nLssC8P!VK0Xy043CS%RUN6- zL;d17xJeb2a4JF^@HJhi5rgK}<4GxghQdl#6gdH6*Qct3m^G2vsw0E?!HJGmhg1~_ zr$NLy`3c6&7%abpen~kn&}chEki8p*PLDS{_$&_<+M%9d3lNyR8YvQ8mM*Jvd7vWO&kBQrMj zqbw_FF+6#rH!jtKVcAoai*;a_foDBVuMJ7QWJQvwYFuo;hD@w_^S#LxKtpbGpdm*N zML^c=O(NtSrId!-sV{COVK7H+<>7v6z9&BsFh^UJVFa}sIUuj@zNUU*Fz8U{;uzpj zvA9Bv%zlN=zZSE=ba92FXnZ znJ3N)c{DW zL|wE3Zw9BKW@=)=I-q<(=b>(DO;{ZeRmo+jJK?a=4)nftKMCk$5lL?Tc6~IFyw^65 z4Kw(SYtP5&)9|TzvHxo_ye7}Gc*}NXKO#4E zhod)#c9B=oLvWqC$g^Ac`NPWfyKA|PYg=7X|Nh@Z}>(Lt9u%!}DVVL*f5Tbx-seU?sKd4rL~-x1rElv3Tip#>`kO>(qV^ zb`yo?vgzt{HubsQOjYF{63_By*CxIElNj8q_3h8T1Bu%-SXXT(HjD60 zf93R4bxyvA!VHHq$0T_`&{k<=77zVVu`>sE^Ho=4M9%6k`P93|*!E{%sc5>Eh*4Gm*~X^R^0E9PLkfEm=Ftk^F__55|zE73uH!{v?sR#pDnAA4q2> zw)9(hAv(wXnIeb$*3hWxS6+X)*}__Kd8_h&TNFP_4dvfmcJ6oz=j|DHQyqVs5J3>X zw!QwV8(lCyr%f771m*kq?3-h#rxb%HQE*)-W*M)v1~VJo^W$_X_j?hCc1v2mi*w36 z!#uwc7NSj&VE&dyivT?S=%{s^NV%&h%8*3$B7&G z&XU>6#f!bQqClD1$|O`EM!K-JLu#LjvbsT}q}*jK7(@-KhfLb4)m5-)Ug&7zWk5CQ=km>tn-(`QTvoP zK`Joy@$akX=p!clHmW3Wo4UH7$^>`*MeT02L!$Po0$e4ZPo?~c6I|T3S^YeaC(leQ z<~p32N6rwic{-ynbLj7$%IjV}ZN7P|;01ZO4h>n4i%Qj1lN2vSv0qp6XCEF z3qdAbBk~v+eQCU23opEx;#MOO zhk?q)<~DO&XRImnze&=(jVm#aR*^*pOc!}#l*99evp$aPAQ>ScFrjlMc{~+}>5F59=AK^1B|zpp8P(dj(^b*$M8aA?cxRkGE|U zzPx8n zc^hiJkY~GQh7&TxR?qK*YolY;iU!=$;A{8Z8M<)?^yR^HXMLDXVOL7~^b<8KIVEMu z(c47}?_+zjV+P;LmXVl;SW)iYq&0a?Chxm5Wiq?^iDC z<xxNVi}CcB)0i*Zffl5B0N)uPXux@(TJxL(QTe)?xAfc}n0;7pSlpHWo@StY2Sz zji}85N5v6KHxU!S*+c!~vFh*mg3}tqmP29unM|Rt6JC8TVdz%QdU04^sZ~+zy-bw#XH`F$@(tF6D_xzvR>WiWa3WXM8ZUn4Ki21@+04$ zaCT%jfzux-9ZFkn5*nlxzk>$-yklIaLWJ_fD4$-dAA`#GcTv0jr!18K z3rWKrRrbSAmm~x-3aoa2pLevoph{}(s8ZxU)o3IjoE`2NkvBSoX%+v4Ew@|U;{s;= z3w5m>jTOzw@ocejiW2p@3#dB-iu_M=)Dj~4BAw?17D6cUMsC9o$Q}oPKM0>~|2l!I z^`A(o0Dm)-Iiv$W5+SKI0KvaL56xEn^T0vm({ONNZs5g7ET(tTwF4WkQRI_RR?FH| zZ=Ya&*Z%TI21s~zuV9of?=N)LDQ6@(VS}T(|0zdh{P^)lZbpZkjCNIZ=Yk_m`Jmi& zN8uW7`jQ-2M}Co*5GN#=6{32K!Kx39d{ua)s{V{bu>(|deTJIM8shR!&4GcjLXHcP zy!NeUZk9l=k)U6Xx}Jb=23G`G()wGHr(kc#B?Dl|fSp1zYtNe`+COJ%S{#t1PV|O~ zy^m04{Rqn>Eas3AbCUO|5>(^%hrlR98L-_(cNSQ4S6p(YAmlufsbHFKpOjA;Bu4pV z8$C3eL5-f-=WLem+)N=PpDp@(#fvto?~`qf&$)9AaCLTAkaBcOTe%yNyK7VgbGP+%H#&}dG=Ee^r=08QrH~Dq&#`PyQ&XD3fy7QMI|uNqQRsXKi+7h?UZI zXJABN=j87lHZPp9Nk45-S_SjV7n4IDy6JX3q+;?hMFN)GdFcq{ScIPH#qOvYz92Gb zB;0JEn8;swSMKLℜ60h(dq&ogjG_gL4t8=W|+^|7;%uw+kVm?RREizrzGv>ZR); z)fYagouD#CCWJEetejHwLsUX0Ehl1a4QD0@cj z#V_@cz)#02`VK`=`{8Mx9vf2<~-*9p6aqz86(e*XiSf#5OR zn_5<~+ujB$su&gA2P(3PmHjg4AAl6~`~CK7U;gNo_jd-&(FI6KPk@>4acyycE=@ymYqN`fTT0PWHW z0>YQmCJ~9rod^xB2xufCsq;rZdsAXkF3K194?uSPUs6>)`N4RIeGH)?b-YpQUwaW6A)g0 zxnKH?8bDG2+Qk?KB82=u5|Wt%mR$+1ko?0E0)l#|1`9Sc*h~(rW~0jTK`1%!AF9eL ze3RoxZg21g?m9tS)}XRgf! zc@sWt(y)IcNX$j)Az)*B*!;t`Uc~=H*rI1>WMxFK(d73@mkA*6 z@6*7O?99e7; z*;x(yLH3Sy`v*=l;V8nfv0#O%bqr4TekDrKN3d3G@4woEknRW5=jjJa>_lPrV69#2 zr=(#$G5!;!DFtIKN=uU;Gw(43uEHPDCoxQdiDV!op~!NBQFeysa3YCSP*LZNnr;VS zC7LkCUh;KN`8hrT8@ZQhb~<~h9-p_}Te=~Q!zIT)$V@ZOPHS=U!ag{d*0}OrS;JsC_U!bk;lvPFL~+vV9~d2u z`mEk6jU>d3qOk<721wMhgYX?23Ee|8{|{Af9uL*~#{Z)cV>kBfYj(1iE&CFs$ewk^ zlCdTGk}#GK#!j{rD#XZ^C3}`35}E9T%Gk3@-+Ow0e!s`}`_J6x+~=J8+~+>`Idh-u zzOLtsY3DFxv=d6m6vj$wTvnd|U*0}ZHycTwzIu6XM;oF9@{FA*Ld7(1?#F!4+Aea{ zF^W7lrj^~OsQFqJm#bM5ukE0hBD=BuZS=!W-?EAhy|*88H9Hc%-~Rt|gUdMW*iRHy zKYgc2ynuZGQhSokWJOe=XGQgO+H$fymF-@;l+v|aNzfWu?}<2{ak=h}9`msDjV&9A zsaznezh^v{`>m87H&2$G!=*a%dm}BbOXcgHkv$1JxSH_`#$v5Z4}C(Oi-?YP{!A-l znq!YrQvLj{D^!cSTrbZ&NP_U1A#Kt(llTK6z4wgE{lS+G>ej!?AurrdY&2+QOZD?7~m3k^6W$rQP z=PBc|#{8{JzkNf>M1*TBBYqo=|IlzOT+c0A^d=ozSs61m?zT^?mo}|?Ofiw&1K!EeO#*Z4-I@2 zFKiV2p}<^TZU1b7Kwrk?#yGG*5wS^k}x{>e~!3wS~SSWSLj@ARrv#ZDeV z_1AaSWN}DpMay}t^d4YCe`N~N(lo8VczRiI$uncY8mkxdIyMrLt2oGi{ciVE9E2Nq zv})-kjUR)wnb(KTBi@=7zN=7)N=kbf~&~-3LwBkoW z_G(=G8I$+Dj;yLCCPeL%P!)BDu9@1+DCY+U}zQuDIzjE4#S4K>;<<~ zVGlF(h$>^uufC%5xbWfD(;-cVZ1@ryiO@3Z4j% zO(FJp$=9%tK&?0$cx30YIoeDfxsz(W-H>s>5`Gh(N^>LDe zUMdB(dw$&tI+b)o4dCl9lFpTwSV2varX7Hn=S5*PS|F1Qk{@J=|FTcB35p?S%F(bZ zhly#I-!cJ#lhfWtp-y4bn88ACqjy0uv<7pMTMcPX<7jiOc2SK&!H+KIwzS_K@1mz_ z6V$?06?6tddfYflhk@vWkY4Os2$#J`z~g%fBLyR+z@@m_CH4 z%{pZlU}b>^Hi{O2T6WhfcnRbzo<+rsTIBHrM{2P++YlOp04$fYB@ca*EGiI=J|qu3 zOg&PI^M@R=2rwfX+o*=rW^oZqMi6&T_z^ZiCn2&)KA>xny|giPErQWj05-;ko&2Nx zqf;hl7WNRDJ2Yu6CM-wak=qx3mh2Wh#geK7$uyp~H9bo{T5zx3TlP&Q< zx=-MOF`*Q|hw^lrx-hGIu}*-1yBe**<-17VAkfmNhqGqgh3W_wN}%nOvkh05bvzbL z$5mZ7kjatwaLx&Pcqtd~!L$;LT!y^yxvgA4CwPNEjhbeAGBhHW4oG7k0rIDc&03N@ z@P8!n04iK#b4;7N%fGtJULv#wyP$bi9u7dm zw5oE4Cr>Kw?>?7JuV}fA0c`GjV?2PA9!W~?p)MC40}?4HCEsL(o5B@-cM9KwJ2f>+ zcrx$QV&PmY8m`4Rq6_8b>wCID5i*|FDGtChwE;0%s+I&`(P;jpk&8$2DCY9e^Eust z7~s7t5sCrv{;9=^RcynA8|Kz<_Ff9yup}3Qs4f0doHhSD{4l`D-Hu|;91~BfEOWRG z6bJ}>IqyK&4w5##>mBhz#jwX;343bYycW< zMlA|b6c6xK|54Vqwtxx%aRLFibvt7?gc8Q6gA+F#Jz*LSp{4*}tz;PZkfm#?$u66onfrGEB_W{fiHpQLBq8|+7SkGc34Dzt z5XUlyt=+y65Zl`eS0Ya8soYF4O*}Ek z#saX({?}XnamdVF!LtE&1V}HrH*-x_{#Y;#OA%meXA&hF zfGtO8ibN$F&2C-SZ;V5d1F_)Gs>HEW))O??3qZ~SyXwqRCqJqq@Y6W{Epr#RoBjtI zW6x8#HeLo4nP!f^G7cM3!vKYiKyVxUS9=AqxwGa+H6nlzLOAXTDS;*dz;o-Af=qEc z;6+8KMr(Kg=$D574glD>-(|)FSQbywT~Z*E1Tb}pL8n2>eVv5=>Fj@a)O#j=Wb2K2?klB@pu(}eG`}l;HaRju&b{$I-gh6-3G+NMnV05&s)t{IrKlyAT}Ek#NQ_JWg7IBn4wh~SUb-m zU{9gJP$EVL3`=l0fg_a|aQ|BFEXsTk;DpI}g6&Zl^6X>zpF)d)!VU`EGrbEwm*EQG zS6=er&>%EKn(~uGnsGle)_rauPyo(6DJ9fmKwQV>QbZR-&=yY@nAx3xU|oo|dm~Ib zkQW1-T}5pbF0diLN!coBA>Y@MPeZU>@4Ln5T)QKKggb>4`{1@Oe$~GRL|2lw z8J|j%xOBU=PqX(!^N#nwGGy~;@vWXvIfatZY1A#&AvQL`A;Yns*52K8g2}PZ2~RI< z^SCK+jIl?&dkA)*W(lO#1dN>v4Hmqs*QDiV*1Z6yXXm-E_X8FgcCK2*O}0%sj)qPw{G*wjpf2GlJLp!ByYd`zY|!PJ67DB{6*q2+n6V}Y5PmlWj0%WLkoezG)Jg58qkS? zdTmC!7O{E=teyxv&(q!)aGMN|ylyT6#sTg4nP!<;5MLX*+5gy!VrMe@mNJU~;WQI{ zPiGdv|GAzp#erx5v8+Ajr?*wFm3`T-DNc8o@1*OfGR^Ni?PvRHCH&21Kcf(ZUgNr! z$V;1GX%gFRHmO&G^{y%8OhN)}hw|z?PQ}al0-@8bybIB?It>lMpVYXh+U`|gw`>c4 zV2#q+_J;<8JhOmeQenFrO`-of06mer(i9&4>_QaY>E+{zNxIjBmksc-4`LDB%V6=V ziDnSqn}Oo_QT;0XozH9;HZCyPtMK_A-+9rk@-n*t>xW^)}S32dl`_L3D zTo{>LaqbQWOVRKBBtt-q#&E=gIv=p!{UTkRulgk9)6!=o3)f!y3qRl8Hp-OaD0u4YsykxJM0zZ}-+JurlcyXz_BGSwKi;-XMb5}Mg7#5n z+-w9vLhA^}6FLxJY>x_12dK=`woYoed_u^ne;xLzGl-H2?GKjzIfnYEdE=^5rz^;K zvlD9ANcr?Q;UwW<0*KF7tJx0IW4W~VjB3wk*%K_paIAOEvH-x@VR;2d3Ta<^y@KtY z2I_Y$cU~n~q$BJn{6k66eB^5fa%{FBVphB5%sL1S3AV)t{%#3fdZcZTR2i zJ|uON$-%#LY{fgv`-uR4Kq#7X@yf027K#)~|HazP^AD$KEkrmLj>^G%fA0n1*_X^$ z^n~SBJ1gW>`z{(f+rRLpPBgybUfB{8rwpJriYJz(4i;i^3bPq~ndOpl3i;|U>xvjJ z<=aEt-ZGfV-E%`m)W!BIrx>nl=C=H0Soo=lYpfe|;y)03Z*oBTi8-cCJ!5v(9t$tw zQ0w89YiLwQIc|;CA?CG&_#8~#WIk>2>YKPy7+gK=y*Iga;bQDF@x6iPCO9ObppOe3d{m)ngrbrb^Rq{#LzBvCoJ zK72%eOFxCmsR?zQjx^d%N^<(>s&vTXk&JcHKR>~Vq(4*>v{C-EZ{+p*65Gd-v&k)E8R3D|k)_@hyQnl@e3NfZ8xRy-zt9bLlb1>6 zESj0W7#kGF)=#u22}bgv%5g1GkiX!PM}ESH-T&c z+gMrx{-_K!yiPXMuN@FZ3qvz>XB)vkDqirqaQDna2qV2y&(gS%Yy@{?zlf$R)j;qm zM;OxJP-E6I=z^8Z)i1?B9Kx(psyqO1wh4bO4-d&^$qr$ z^z986#aywD#rSb$7dOfn1RNe+#vBj33Tu5l7okvpPKEekYulxp{fbIzA>D_4s^2m| zCCCYFYic`p38r>FAUoY^5Reu2qwR27WZvvzLP~so=_obAXqo`pHXIJmpKYKnM$~c=_O)jvA_{*W|44&q&r!HVYmh`k>k`*lA9Mi^G`U>z> zuGVwjhONGs#*O@&Q*oyeeFJj}DHmwGRzQsC*W(!h{P`_~Ll0l?xl&(DeZ<%42tT6X z%c0Y4Tfv8!SVD<=`kTr;19JyI@9O41yVao3V=>d^fJSa?7orsf3E`Zr&cU)`*vmG` zWftTj>Bi_PPGx<%Djt1I|&nMg4}TH^4%KYfp0STET~hg%RQ@IdB}?cISqx$ZY5I=LriUZ_O=<4t!93> z?_`ladeYD!V3L-jDA0KjiL?In))Ds54yZc^22nV5;xP4U_uv#p`#>RC)vn}Vo81HV zZBHjdpM%22y5S&;bn{cYVI9Lls(b1afUu|NfW` z-p?bQKg;gCV&i#h(%gRY0>_?fNLJYGo}WzzuyI|3&+0vj5u25_b{nqI7d=(26SA^` z4Pt0KGTJ!j-YJhyVGG+ptDCH)RqaFXmiold`@ucd6!nQQJMt;W>bG4)B|CX~3SVjG z`EK$2QUQYll78WQ)!qJ8$wThLa}D&czt6&5bpVM!WZ9Q)fq}QkAK$0M56b<)0{#0j zsY!-a>}H|&X^lZmchCrab4Lo^4s=>JBZi<;tx*NhF>coC|qypYn)%67JLxc&*v^>GV5n;n$Kzqfn1e+;6>3gKio{i z>{ZFO5uo&>hAowm%?y?fYWgiFn=b!BW!J~u$3?&+B2mg^eKA}YGdX)29a?ULPTnuj zVub-ERm|iEhRK|tE2fw_8qH_z(mDzap>f#CB3XW`^|>kz%<-W9WLO}pS_(x1Ys-wH zEeyo7A8^X7|5c^+tEfzZSA7_-Hrrm<0J}vT!0trR_JYXVBGyP4b}}D4f@i$TRSyCV zT-~!^H}6*(!F93w>G(!r+;q_^Kf@eFcZ9IV@*5eA-(Y(U^uz|mJG+m*QMAD!I5HGR%cXVe$G{(KCU;U z1avjs$k_&$4$n8;?b!yuee$w3r0f2-(YN49Utms2<^E5Y zi1$B1zo}x7H@eC=@Eqc;pMdwknE;9*LDG$o!|_fAO|ZhOGG|NU2aK`=d?aN7datW* z@V{kXy7-(s@XOa*@xx>iypbeXkbwZV&ridfcQhEm_rtm+1GY=La{JMc1_VIcx{d23jlOyx>)3;kPKbCa@O{L~6DrQy2oq>ebYW=DB$p zrX9OpfyNW@f)yf<*OmCgL37{&i}3(XdL`{Rx2)Y4e%*Vwt0VTsP`@$e=aD& zn-_P1Zmr`O-aD1z2xQR*i~?>rfCuwMqr*@58iawi0b1L z0|DS;<5~4UFmf4Q1*|;6_dTg8j)ZQ2+9c}$4NW>V&lQY!7x8~ue$oEsfNctd3U#1A zoWua@Ey+}zlH&ZyHFiivji!}ZhdcuF@l50V&0pOmn~J|9Z@- zXtt`n{{JKt8wj?mf=qxid(IxsyaE&vC!Yg_!la^%p(vtukxWuzG4a0lkrH>(#K|36 ziEy6=3M{%h3?cy$ScVvLL$i{OXWEqvD$W2fj?&tZcykl9@vJK;)RvIZLqQB$5r{+{ z>}*Dg&DnsXa{Tln3G$LpiEwKC`WFctSB`(0mKGemnDCHPr2dzC+1nkT>A_P``R@I? zeW(;j*aaB~G4UbW00P5+o&kZpIUAJ6D1K7lhcz)gLk-2|}X(QCF|y4z`kl zE1m*IwL|8FwMB=e1}0o&)C@v+?imD5ajAQTM9g}rz(m>w1WO?fzL0?fJB_GI1QF24 zUhrlRl)jmY++q``d>6znU@7-cLDfyH3m$sfI2d$J4lvS|m%%Hg#>Y2iAYvr$T;HJp zKoJ~)Ig^>u=O!q)<0{a6{d^24s?>xBhzB>6-2fiKoa!vB}(+hiapRV8M=4f+*{gBN#d-& z4y^Oq!=40zlMDde9VY0MChHUd~>woC8+tV@#21!P}S*1e;+JL6Ox>f&@`w zUv#DxGayMC#vm{~u1+f?aWx9w2-&9xjVXQ;DuV?xS3tWK$PWlEYW3inqewh;;z27Y zYG73VoL+Bdn)$n54)JFeIKBGm5a`pAqc?sUhhE{M2c@wKTA)*SaD{U)X7)rLE7l;K z8R#-T9?fMT;I{ZIvpywH-&mAiZRp`QYl!Oq>&ce6blI(WN!<`zuTr3c%OgZ{2)Hx*vqTIwwZ237`nHbVLySvRZ?|vaSGrYw_;Mv3 z9eclBqVPCEz;-4rnuMz5yATifd{N>~ht zcQPX>)!p;b-)d>Rq&2NyV5#cX6tAN)NIxkXW9z{={X3)TyWm4>6fQ4MNf1N1tnTJO z`am-71MZR_ZhB>m2lhBmy9pVLO-ag5g?n2BeVRHm|?ZFNL zS889fS{x$=sMOyH1Brs85-mWNF71X;0xS&YxmL7hhEZVPUO~$OzDKC z>26xiu&OSEdct#I4di_wzkNEG@CAs>2j(lrUIpc7DJrrEE>(Ne=j};UH?6a)3u%(3 z$G5a+sMksj3oLp4U>PM@=XYobyY0C7Bv!;>RZyb|0c5*>H|Hhviq4ef2-3XU+ZKaK z=5y692dM$U@t?yF9A8Zl7RF-kCaY((70e30?26#6`^i7o`ZE~9*N3gKC*Dw5t)kwL(wZ~+_oL+=NF$qcovq&5rG=e_3>}5&m0fo z`n=6Z?^TRNuQFW<>+XVglT9GniVYxxu~IFPa$+;ixq|If9f>71vZ@B}KfT~!6?5mVn^$fr290B?ktWtf444v&w`55O^2FEuH zUD!em%9=I&Uxvd#IXKbON^TWH;>&p#RK8MFFgSXa#LZC!g75l*kWA zBA~`Ah4GKP)+w&6Ur>AB2~PdoHxVH5Z7S>*PYA=Y)beQJd&LKQ4mZ-<0V7e`y?` zO=-=iXk`OfO88BZ&~szn)ex$hwB4~wWjX9lME%CV%&a?*jOLF<>CZfx5d(a8#U7%m zi;jTr6+=;SS-cCrNWJ(?0T^u^BR@XY0rL*GD(I9oH89M&M}1F$jsc3E>rYv2bPl_2 z(9O$?#{0fee=AZ~Psp`Y#)`?)C!~8617f=1`;b!{Jw-Z7Vu*b)5fGqC_5%;1qu`Fg zVf^^+l}~(D8k)@7Dr2wJ;7Gk6=Bj;IkKR3sa``xGD-;nsZtxDnNh`8g;bnEQ{zHM`WL>no_GcOiH}$9+)WJUx`u$ zJ`b4_(h9T({hA2*q~T}!V*k>q9A{Xq0@X;dzb%O*+QfEJzd~GhIft1rGp+0T{YmfL zB|hju9Xq3Fq9YOWFW>1V{iwg-o&{$Ty?RE>bHth9 zYCyToaYpC873issGX+VdK=Oh$u(rz!=Btp@(!b^e(#ZYqqvB@f-AzM3Kl3>-i}j>g zHC4v?vWUC{FW`%qLO3IF-LKI5Y1_jtJ) z@cJIylwMejIL~*ZH1i$?xLEeh7Bt;byU~^Uo=6xdXvFX`eZ>P=Q=6{TC*w0U)&dgx z3P4(N=?ypg_+(HyACv;92~azWKD}SM5f8sO@EA;>IRENexJNo@$ZA@PAxz@4#^Lay zM&=~jPRTDRfIC2|rS}!!$q>n66}PkUM4;*Kc`9KoAz@4hZa`%BtQIKCfOXf(F_r`Y zIcDudpdCUbC7DG$_A}uH5IT_-U(7HhPNWCD0C&`4@JGt??A zIC0bKpYW`?b_X@%gh|0h z=~{2Bz!H+@E|NxDn+FYOe};}2X^5k<3|~qmIVYt8hChNnK{2yi;T4XhJ3fJ5gFHif zX=E`Y56tOGGlfqHp$48v)ogVuh=41O!vKPipo8!MiqOGWpYRWmU(taXCi>J;q^mk= zYxjBgVECXfdqpjoghIPggF2+|=HQ`Pd^8;VyUYf{l3C-gO{{(g?C|2IFJlF`Bi@i8 zZsi!LE8Y@?IEyyU3vw%dt zB7vvS#!_7+{m51Rv0EogROvuTb?KS#?yw80dszgDj&e@f5>U2_jTh%wxrR!<6EBWTQ6i-HS0zn3HkF$2ThzUzW){4d^4KNJU5YeS zLB{MVuvw&}37HuMzXk0@ykQgwH58V#dC7w_)%3-Qk`5GJChU}ohlW$)S%gBvDwH#; z?&RQiN;Yd$q3Nn>pE+&6#s8eJx$`vTA(^GeqQ%We;hOW~b_RrW@1tRffxw^o6wcO9 zWDI>)usn4|-g+#YIFqInpOr>s6%rLgJMi;u_ST~gPI&6~-(QCg@f_9bU>8B8zVZ~U zJ8VE=^F~sW$dEwpfT1{r+FDi`-v0g$_W;ljXAh`fbQ00WG>Ro%nR=W^N}hHUsw*%j zlb$KB6@HPMZ|<7U;w9qBtav`N;N8HUZ}c?bi%8EG>3yYhnnG_I=nEDyF zz5wK3`{(agu=}AXoRA%LSK-`42f^|gEAmKyT z^`_!p6kO)N&bGfHI|2&oB9(e@+3rB2PnzZeH-EU>;TgXaBn z-(=y!fPSpXTS<-a5p8pG)_$L>Ow5Biu$e~yn;DTN6418{b}*k*`~-y(a`v9FP6Fj0 z1@i4AD)>~yPnglydMY@gaGMcJSeRP-iS8>0CdN2_f-I8Fyv-BilQ6@BfY{F%M^_%L zOCH%ZAngVY0E?#IvywPP<&HCu8#5)ENL=pV;8_+6{cJJ9RaC2e7wph8!*u2fkQrIW$~?0o2X8a%2B`K|w!tS`!_`|( zHbW+{rmT`z!=BjXGu(f8#CWAKIN6fBwjtP$FYH|sT=1O%pTO9STP?RWOR-wf^w|R*%EFS!)1|r}0cpZyFOO+2Efq*K7 zt{)>+F86YfF=GWb!|(0xFzX)n^;KnOpcXkpANxMJE}#Kqg$P3yW_J#AI)k9WRdBPV zO&q7206mUPl+b#LzfCyX0$7A7ok2Kyj%F*TU5VWh-HyW4XG(BUvGna0YC5f)s`}fH z$QeBy7GOrT?5VEtxKbiiTSgUJCKP(maLg*Cn?aRRNxJSql>CTD%zg z=XssHx~a$v2g;ga(EM~qRCq7ECW2pdq3U|>ya@@kl}VJHSD{PAl|`8S)^-+e4!D@H z@r5->69vuRJ+o2wDv&X+(mJJ$K}zJ|AoGqgGg&%RM(G*(8VYi8-3(Lr073C6 z36G?uU->>Ip+BjmeHBMd0Za_7h{qVN)eIpmqD~Cg(&vG~J2iN;G!e5TWw)eHFXMer zlj0gpM7SO_i9vMTF(lD-y2Sw~chqYlRC`8e6u+T>^(^m+i50D{0$Oo`*KD#|?px27 zMLJ+}BLgEl?R9-(@ad~y)MxyAU90Aw7U|5WC7m60C-6a8dfie|`;ggUhT2wI8AyIs zZ!?$BiLMh;Baf$*alNLTJe>isnP$wOXju^UIPI+UDOL(gK&cSgyp>@Y= zPhH#MKvT>}D;DvM#9HM96JFmw7s@`Y(+N|WI^@JDqLkQOF4hVG1yPVc$we5y1{I#V z$4Oh!(#sr<(|{I;y6jcKNrK>gW`Cvf#jL>bkYx&H|aqBm*u`$UNh#C%x*5LVsWgV&&y!) z^kdI*rL&uWdi#lgS?=n$C)abib9;$B^$p)?j? zi`gZbS}>upKW>q!fdV?+?L#dO)cOFkS3S@)cWngBVJ*?KeqA#{4^t;TTHy>P9qM`p zQLqBtY&G=Qw}+AmCc4+Qr6h}@fMy04btYOEgTj(0GV|$xNP4(RQuPemHOL?r?S9a9 z7cd-r{HiZ1%SqiSGaqFgTs=X^@$3ws1p|ooOqlZa6*xr}YM2t+=8Q&Gd$aWj`5vKb z@mJ^S%`p4@wl4M+H-s*rsa$Ra^^?3$#v(rLB>1k*l_F=9ugPYrMwCJVB0cpnb%n4GyCnYU za97noK`Xg`b4&0$+M}5)55ct1IyADj`=utjYX3#`lx){$7jIQE(E!Vbpvr|G7*Cli z<%f+BpAOg-Vks4k(ZjRndF)zwxwZJ6heeidu~KY9T(?#eURt$22bDXR>{>pN^}{Wn z-l&!7|NNv`jmjTDE^>!hv;U+QrxVnnX&na>!O^%mulcXfUg3Al3N7`EwJVchm3d&& zEITGH&$SQpM)=uT4%<+5`8IM5Yk7d&S`MNHxL}*2k87mQ!F+!yBzhHV3p}KA-xz$c zm7;8{8pa}Obz5bI1%7R=4Rtt*0XxtfW8WJ(zcy92jU^_8cw*H9IfWT8E$7ajTB`{N z$UGNDoq^+(b5^GSsjM?@nJt3eMXRJB$|62JTGrVLuPDx_jgragopa@R`vRPMtNY@y zw~KhFO5#-#A~RvxvgiJ4LIv|ws-~&A*}e$L6OX z>pjf2z?0~_kYzFpbci*jNG!@fLB#Sv$!#R?I-_`~rpKc1sR)oXb{2wp@}nwZ3eID1 zy7~&X?|b=h8Lb)tY-_q1OJj&JvFwMaS%ImEE2LOWo;EI;9TJz6PdwFKY8o9_+QHly z)x^kL?)5isFppp|ets;0?#9G*qn?$A$_&z>XF6~D8mb&t{b=6QdWz*-C6-jvs=>}L z{z_8L91448Jwq!E(lblTc=-gMT)*>>k21uvAGc7M4_>r0=|Ev>0yZ|9Hk}tQ$oZtb z<9U1v^q8Ln`gR6WWTV_SdS6JJHi;XpNL35nTEu*g_Df=j8S1+8wQFCp|B|>5C!-q< zW6PXXb(Ks7(JW4Mvc$KQv4W}LcD~|)B`lC4e5nHmL#(C(>d4bfCx^PsOM72VhyW+7 z(21p{_wcsLpl3BrtIhB>Y+O$-eE&w)P?zpU`PwSyfd^JoGtD_RE`)`V_f5J@@aD>x z_BhH&!aiBwdtbH)4j~+cTpY_KKbs}?6m&Z@!0WN5KGz+M9lzqDNf90H2m z42c}NTSO>Ql8oXVsg<;tqxN@p^uCOJv!#(!9lROqcX~Z4i)RS20-2}7h+9eS323Gk z3-&*-y4tCTOt!|@uHv2$#^2whyOwVf33RX+_>d-gfooQ2L>st$5YI-5N~!VN*c3T% z1gwf%iSLPv5;y9U!a^7NNYk1`nqKtw$pS;pw#xGs zcx`2DNmAX~4d$WyKi&mv2M#>QbxmKI<%maW(BxHnPI(p(D^@|H4dg6^}QuRoNKI%h=?(lI}^hyT)|Ly@H_z2EsC zgR&iR?Wc*sU7x#YQDYChAcwSmg(`Q!L@6oXZEcX>sKCl%G{M)6%$5&SIiM4BP0RQ3 z^BYZAhW@-5&K4i5{h(O^H;OKVyb2tAB5hDK)Ht(%d7t)%e2!`R6b*~u)rH`AKR zq5lx7+7uO>C}C-CQ8Bs%gngN8H9oe3KTqpFs}0E#ma7KIzHuy|BAX7BAqQX`m%b7F z1J~$aweJw-7v04yxhFT=^Z+}z1eV~R_S=#ifewsBRpx*}LuBsd%s%cNW>V~ZlMcpC znNMESAV$Q&O0|>mA>O72JMQ-N5EBbdeM_2@^ORZq&E*^!5C?DoF&(Y-?v%O3`829a zsS{(u{czo;xKgz7Ns3VUJrr|I)pfU#SaB6q&)+2ZKielo`QZfx<_0 zIP*P-DNWJy>HBJ=OOH+&;UDiLz*K-ij?_DNzMEN;ayoGCl&irccWwA%3wS1MQc4vA zkB}vpRgGO;6)>EeEd8FWPw`d7C7}_btyD(m?U4am(v!MjffSl~+piZ!7ztAzm>F{Tr!oWtRo$=rmW^^ zodtV;Bo(O9si_Qxv<7npQDz;7vlBwjs^dEH*B*oB6_&`8_JT8^rn`hh%vKeXt`okk zRL^K`(jq%$J`6yLSge6s(0m*T2*TNP!gn$StiMmJC9{~ST9?$N^;VP&y#fTIqa<<; z=iVp!LhPm6+WlFU$=e;A7bfo|am{<#o<$GE>cGM}A*bgEZs%>3KeJ!=ry85D_mX&9 zJ#z?GBoWg%wU!Y2N(W}!`TK-f(u22kB{oOGc2YBFh*G@4$BJ@MB8AF(5yRYN!^Jk& zg;>=3p0FrkN!Y$2X;V{)+HP$YwuYwyti^EG0Z|N0_Bg5yEoZVrwUuU8rZ%x2m8qLhs@&b25^J}ND zafLYkaL!f{+y}?6N2~9Dj&&o{$Xc5%j(#j+=vp7l{{FwyTpDe8J=*MJj_v>U6N#m2NQ1a`nXTe~=>^AOP&Tg&XXHd_+>oIUY4YD6 zKV_sS?wI9j(q2E-N_GAu5VG*aUiR&|Qr<2Q>%}?xjmVON0E7e)DB4jC<}ax7V&} z-qv!=HSjSYNvS@5L}gsVXyo`v+)Sa!f#RT+qXhGFk2+3~ma7w!TK04MS0}$EGg_4s z+nJ}0u?Wr%7Yzl;ZCtH97%$5F~jA+wAIx0FLQxHh&E-_ zLA->Q!~91I)`5&cUuX>La2W_;?@ZduitM@3^j#A4vGU0INh>t9up zB#yu?!K^mhq&dNIw9b_6;=tVP*~#LtI;NdeA}oD#T>4;Pa21HeaM=>Oa2a<`MT>P{ zOWj$+uPxHarj$AU4iAXS362=(Zi<53Iu@iYZP5gEi1USzT1SDc(75rdk#3iCi+q%N z{vFo!-(egj;{S@#1CPsn<~}+VdfAj!N2+eoD|>ESecQ5B_vPrFfA}Qi>M>l29RX*M zV{mrM07c}oF0(t~U+1zq`?5{A3H7ld9nDH#v1&OOg3~^^q7tK#?BYcASR~zobGYXR z1C5Xyp2ZO)-2e!fPcWT(FVbO@?lw5h1U!GrTK!FHaB~7q8#GU-QL79Arq^?Y z#=^Qf8P_+Bo*&0VOH9({YVSeGY_1gou>}|lw@Dc=_RzIN$q)>dQ%o&>#?J@xqDiD~ z7b9b*{A+6RUSDm(EVN&@>W7A3rw88vTKFeF=0NCc{5|??LY!i)7>P@wfgF8+&&q;G z9{XPg9uoP-a8}w{F*J3qwoFpo6jMGZRHV-ay!Y?lKckx)d16uVY6q0_tX61@ak)fS zPDeIrt9U_6pgSnuD~+yPHIGdl@>ZV{2^*f{LjDXJd{pkOHxsi&IC`GXzm8Dj`J3j} zrpq5$RFA=Dc~9dH7@_`0%;7+OPgY)SNw!tnIN81w#Qm5q#~m?q3yz#UAq+8W{FYCn zzG+y=%SOaUvYh%GCv$gA_eR_Udu<2Q+JF6WBmv$^ZDXCleCQ2p8!Nd8wJUMyqaE9D z`XFi9U@eXM^WFzXLA(uCZ5uR+k`k|gInV|N%2WSEjC7@BzY+G1!|tPfWb&s6<3%uSY>uf?zJNvbHu{( zBS*WEdS}b!gQQtQSZ6X}-8-0#U?Y`~+MM~9-DP`evOT$AKaZu6^y5#X=ZZfF{q#DH z;);xYsx*#*ch{v69PJ|g^xr9Yce#LQJiF%i;eYi4wg4LzQ5)XLX~I04d3>lsi+>4* z&fhD7hXf(^aJ^~cGq}w^zW>W+Qak$ZsD?J$qUVb0&3-=iIY`tAN5DGR8B6qml-{+J z$lD|piQ+k_v0xFh;f+_LnnVql=Md>;QG&TT-VrtY1Xz!+qYd?U#4s%(XL3Naib4;r zFBKTW!Di$c@OsET528!~H@KqX1~VvxQBPJFoFHb2*$xDj#hP$5dI6)VYNw4BVVX4? zejQ{+o;&ymhuB+@|ELO$0zHygb+cQOiy;kidaQikbNgU43-1{QF#;YY_WiW1)v0ig zLzZ2wC>8^zTw zOUCJixIW6qz{V7@Km>gvX4DYf7^JERZ_y?dblMQq2#*0?MRuU!B^NmZn9V^bnj4rJ zZUpW)=^&cT9K^^~@mT)LNd~hlSzu%WyoYkcfb*M)ts~}MNEV1$B>9U1g`BYfZ<}qP ztCH362yE%n>v>LyXZ(M*A+xsyF<+U1F;3b7=&~2p3FD0 zBoOlU)uRLyC(zHkOnko;20~grDf?;}7$o;pG?J(3K>S{V1IGt*pr&UyM5_%>MJ<05 z|BlbHA=HN^#tt$RKs(}SSV5j0>{8PwR-0OAEYNv1OtTBo5~rKMR>Ei7&=NZIE52^F zpo-&l%;3TAqF3I@y|_^!c5c2AuD=B%=3*uRUD99J`k|Qx+r-rE+$EUhJ}1sm;LR?3 zM*ZOAHF-vX2?DYT^mSrS=opw+dXCP@k zX7l0?yI5x0vpK#EiO1z!M$f+%g<4(eyzZTLo;5D5y(97<@%P8!<;VMJ8@?!aYfVagtlcTFFFjr*qG(^Ea-G<_ zTpydXbWm_M_JG~q&!ukaj6LfY!3Rq3i;qDXkR?+@m=uS19fr%{mAhR3&WCF*i<%F^ znM0YM=}Mzm*Jn+qL1dd{%FpRwV|v#%wSo$J+1!bxZWp~;Bfe6cFO+5_S6f$>);eUl zJa2&N&@>E&jMlZkj9x&RSDjVjXhCO=L`Ds}>eH>pM{!PQgz4 zgIU=ScPC@M#i~Q^3!$#k*fYh5qCZONy-7wy6+H>iy#ZYV0-XhW4H;B2k zk+>~Aiy@9pm&G4dLkKnctG-dA0wHO?A3)2D-*`Dae3CoUX6nyS`e|@@Aj1Bwgv~i) z^%L{91}u~9*sDjjHZGnhtyBlAzvkKBM9)+pL+>wayyh_5cf%bP)*c{9ROx@JN2|ID zja~S0p4z#k#SjtZ9qd>aE%N=WIWWq@f>~!Acf8d~9J%WFyW^oB`VB?)vDt3l%!Q@A z%A>iR)xPMwSJ;r{w*u_tNF%;iR^dXyvG|Zw&t?W{^gS5eg3Q=SP}iTrj2d^ z+&y(U+{l8y(9fMeB+$e)97a&J9lTU*Szn{M=F=^5Nz$+0-CYhF6G49yaUnXw7Ik8Ic-x zeIkwblYnBD#;#6uLHKi^6BqVIuRz(}=ng1dit=&8tmVal-{m|~Q3fFJW9RPL{&yiZ zt2^DM%!0BvxTGjgvdn>{d37dOupXRAEN95AH4_+`^M3yLX>_NqxqV_tEl2^CH@u(x zSjf@pzQWhQ_<-uT`n#Gy^*QZM{ZNnhJmL%^E$g2Pi6Tm;I=^`^nSLh+ zqU>V*X|7>u?m=PYKGKU?V$lqQjL3&=#vy*sz>cpm$xg+*5txn26nIc-7QJZ6jYH(1JFxK^zfEzsPe61kDN z&f_T=+G|9USN_WI zUfDcz7IxcBhrftViDeKyaj^EV&NYMQ;a_ec3n}d{EJQ7F%uIvAhpw%rfmhfS{nOz~ zDb%!$UNJsLcr~)|_)_t*CEq%PsI2-;fqR7paEC0ki05JlhNdDGxs~^(7Se#z@MnG; z(~=8hlA4mHuuQU*r(yuyRE7Dg_ndgLz`CeJY}o< z3lgTTNSMu~fS^U%(sw#-;o!%9^h(2O6W2SNqQk79T{eXN?APDCF<*v>jEC>7oywcc zl}8`H)dw3_M2`~(Ew4cn&$hR4yMxh>)>oyB60<1YxG%#jmhSizYHMYBl*`VvL1+#QxPg`N+Xz=^sOut2Mv4;HfKXbd%_XVfbLp$D9o8_q$u??*@RAfVEC* z8IUETCj6To?^U`@71>PZf=TI425E}S_$)lgV&G@~Jc-VF`9p_HB42?yCryoKKa3vX zMrG>Qfdi14`Q-Rq99A2x<-wrc7w@%Zs2*8=e#;1^8@tJhPOSSJP;^Nja~II1-xGtc zoeE&W9X`Ld{g-dXV$sihV_R1QG9HX&yjH;Cu=3Z}VqqZSite#BTo6eXQ zuBP>7Y!<1Xzf&8Zasj`@^7tu)>DPYEiI*v%er%dcf64k2&Dj6NKkA$@-_y{5$q-MQTA6nP;lx?!B55ZVu)zr z*rVB;f7w&32N}f{6neIuk1sGKr5FImmjhCMR|Ce=ujYW@ zVwmKOd;Y)-ckZ0bl*2-_EgrL9RKy;|ez)sr0EEBTLk#LnybM`htfB{zRwR5<%tgMU zj-_I_#qXDEw-IuGMg2jTyo_qyiZY-h)fTupqkU8MP2TiWwM34G2b}fi#!-`h-n>xJ znNArcHqBxE+waw!NE&95t1*v7s-T^k>B^IYfkd+~OwI$xEbTpxTArEq9%ItxiM~~A z`b@3E)u>FKn3kRor1e-jK{a_P2ov-N zw#oBQWa{5^ zK@{SSHz2jg?!d#GH!|h*tw{5&&yq1MUvp-AfMF%up&Jgb@%w1ZhfD7yb?{m>u)3c? z_LT|rA#u}iq2U5S)jRmBgXez!=C-(>VR(l*BTjPm#ynEzw{Rf=DWHwiK9?zqbty0* zwZ;CK25%Lby`rF;8JDM{-8tnoo{uaobROcMf8g6o|6c3xvo>SG(JKrc&+f03mpz&j zrc4k=qwsGggNGuaqi2+mIOj*%+)O=9Eqf5F5^j<8Y$-I zSMdXm(Wr1@QToe&&L8A`YJI^zGy(7ZFca`tC1=nKB*QRiab0IzBi8|!f=A_37UR=7>4%Bv-xhh2QdTxR_9IE*C?Ri zw92PlnBZv4ATIF9;x&$TlX${<^OKOIK49wL)VG}9K&wQ`(paJ}2lK3{XPF}b3|kF= zA?uR?0n=rUU6)ZHjr`iBC>zw!lU=E*PRw{u7=1|Q*vIfFAl(dew+og~!XksAN+zv5 zX;Zi(l@)f3Gbs(y;Exf=h|*ECzZk-aVW*yX%#UYSfy4){TmmZ(GkEmr$QlF+KxTo9 zw?^3rJK9;Lc;ADn%{U{?xSW&`SC_rK4o`w-A)1$;B8nhkJ%eJ((Grr9xo=c5Lr8V_ z)GNo}1LI3l|6Gy#3_L#Pk84m-nB7f+d_)nCeY;*h&W(wp_8*zDydgF{C6j!g8vC1t z2ZUS-p>TyTEEjeUB{@fe;>p)TUkCf7G`D8Nxoa5OZ^S4zT@wZ#-q-ap+SzPXCfYh4 zf^cHjXED?8*TrEbd{m&Ljq_ZOclLW_+Vs-4T@C`{IXqSvPsmj3>4^I}C1L#qDf?n` z@MbS?LXF^%>WC(fJE)_4;YYU<(pCYBn>q@=T2UU0`$5*%@@_^#^}U^N!dcSV*UerD8hPW3O%$Izpo+ikc- zv6ZX2*Lb<>2@>1?d~C-pTvM^NgukKYjLe%WEs^goMU{BGzO^y5WrG;oLXUt^A+gQC zW$hXGoy)4_TyvsU0LJ zX74U{=L9kIu$_z1Fo2D1Z5^WEW0KFzwsyH*^p&lGA!*amF)`~`tflpNUClYwn>38~ z9=_HGE~+EP?D-jp557$ahK2sSwcPEf0j{;p?uq0nHr{W3TORgYGm`^cq&yODc3$)J z4nS>i54Yt($*aoOpRZCu=BU?x`N9O5%(h?r!ER+4(7w^odJ%BP()0=I`wM&pucZMl zb^lXe`_VeQ_u0TtwS8Z>U2{KyJ4BS4RG@yddV1jPLYc8+qnUosj&-o*PyGymNeVe* zQd|L^cX~hvr=}Syvmwn7tSuN4sXt)a>#_A2=N0Dr#M3b>JZD4yOK-XoDYPUUDsR@b5z%Q+AnQ%#9MPM~ zxhvfpyaCxxRe9uso=McC~40>*QOuGGRH)X?B{QBt~fu3)a5qZ{&#G7;O+YhZNR~4cCRb^0{^BCLdPw+w*}YmuM-v z-MQCKx}^xXz~|{u=us8xdo8{&fkq>`8+@G8!r}pw4@GHUpVesl!F`Jr`BZqv^zQZE zM*?6LxIq5bqlViAbjD56Hf}QJ8~BPwkLuHOKujg`#Lhv-pS?#F-7)<+0PxbqJ{|Bn zU{cQ$rwe72(MkZ`r3N2NIO-X;T6yB;peG_|MBP$j4LH(%8H;J%ilvd&CId0H6myI5 zKu{?B^M0LG!j_HOW>`x0b-#|?4Z1>7#ARG9_S_7}!)v~}qU(FQ`(J&c==vr^)WLfM zkCx7t>KflUMf5wx*pJ$ui$d%FNfX+DCY=xj?<0mOy#JS73K;@5E3E@NA0G|TLeMMU zUBUN&WCe{+)u0JIMiTm)2%LEHowdpOzk59tmeL}y^ynk&{eEXbW3BYe(Lpz%>$?}F zhG{92fqn{{4KihJ@+bu3_9x)t2*xrz{w&0cMWY5=0xe@(VUO zr`Ps40fXWC3K&N^ZH94}*(7-N#Ebh3Pe>WI-9gAp2Jmd1WpdsuT=NjQb~XKk6QBzn zcg79Sjl%MCp&-sXBqa!ezVWxYTZ-@zvboFMtcj#Hhb-k+40@SM(YI|i;`hR3Db+hZxwM&m@6iXK#{?mRk5I1w$GG;i_W}vS}@Sa@$)Y`Bly`_muH86j{fm`P+ zGz!~C8i#1wQi7}bhppmT0?NnWmu|u=b)#P^2hQN*`ypZ-=HJ)})=qhMY8RG+s~6P9 z_3+++C&upNOi}`UHOO4d1neq383<=xj(=oHh<%82?p{;sjO*K(YcMzizTi3arAszE zgfQ};+6Rs8u*!E*(?W8$#6zd8p04^%yiK~Z zfFg*>GSghtmRnkaLfx=qEsroEqEfaOzHvS3YkP5*OYE`#uU4i)8qL$wUlcShxW3L8 znTPkCeGK*S9fkOx(5C3CkJH?JZb{1xDEr;Ds6UUHF3S)%d>YMtK01q59+x15EAx#1!}xwqitJQcZrKD~ePx|-Spd<~ zL;G_^!)DGG>X|-oNo_LQ$WI8L;0F6OhZHaEo`r=+Ol9XO1r{lfrW0sn8i~{9xQM-U zW;((42TRCn`9>H8l>50H@J>Hg5vO|sT=attrr5n%eGIF!r06DGP6ZOW*02N7uFOoJ zFr;LPT|yQ51bqX!P>_6nGZ+eeDQ!^b@yjftg*2m78$%nj8y#)oDR3lK@_Ap4IfN62 zW_|&}E9^dP zWp+8VU$&4$nYN_<_#7$hNO~?5dPXM;Y6FoWK*&hn9&4;*BH;Y}KM!tFKs*-PK0#{8$pAU8I%Tj6EJ=0QYjCMf(e_BPL!y!Ir8cE0eujv0qDxD2Iyj9pyXx)P01g$6JhWKJWl#55l zZ}?}W)*=YT>wh!lEA;=d)%^c#F+{|+(t&I&22FcY3|`NC-p65t!ar=OwD18{Nequq zq+7HFV$|MbrFvVugwy9lYUg5D4esZ4cr- zpd(V~LZ`li*9n6Vbc_dA6%O3~HvywHQD_P|Q{^_Hf@)=nAkbooH25r$F2NkH3qo2L zRB1=F2x65ZFtpKuK`oU*q)l~M5CMdb2@S5J!<>4`f}6ZNn<;*q9CoWXRcnGO+7zmg zIxm@5xfOQ6nP85@D6_wfQhuVx&xS~Li68>g2tf*_Q7j1WFw2IHBLW}8MPIAQj?b}y zeWZE@xDf20b!^QSkhBOfl*@o97lTwdZpwW9(AnT9RHhe@S(w=%*+)ygh4@cd;U0yu z8M2LC`8~3YQdu8~VFk}%V>L+N(b{^N_{1TE4sg&08;4`E!y~sMTll2QWH2U)d#P~b zm0_;1o!}SsQS8602rbGnn=@reXC5jkKtZr^;+rDe$n2D-FFfz{ZWc*1wUW$ei`+%M zCM?j$I{6{NBU|>~`rlcxJZT}p=i667-ST5nK1G;}D7K&z2ahJaW$rAii-p1Ev+4!5 z{dl;?<3@#LdR~;J z5#`E!KipCY7GSH}O#uqH4m!2Qr7+_5dN)ZPRkzbOC7KW%t@>RQHqvVv!c9iZPMn@E zDQ_C(zA%ikhH#@!+1#)*zkt@z#2jxDYIQ5X_dpkrp^YHsuj0m7cc&z zCd$Z~uh&Pu3wT;&ia3vfKNX)5Fe_bjOg&j2H9SmY?|Ny7#R= z7nEzLi0yu>7ILMoxpB)|ryio~C1Lp+xa{$?Qb_6P^OLHa=>{{Qk6SoFGOuKkbw5?V zm@|S}RcS42jG89FtZzZeU>y>FDLaxF7N5j$_Gt6jTGul*Wk?%rd52ywBCuY|C$G{2 z>!*p z&Q4ZdCEN?A?A`b@n>VX;qXJZ3@b~w5Q=Bc%e^#+ItpBXsit;2a!Q4eG%DuHI$1E!Q zQ4RMgu2e@Oh_{YN%we71vxT-niAfAM!XJ0=aGdJY!R-P7bg?u{{ zz4vXOiPqTMW1z3a5Ie2dv$!8RRWecOx*&uO^z1tzpQmSy7lAh1L9bZ)AV%;JYt{Xv zS95~^GAU=rIAp_Pr5 zk$Bls<;*IuI2Y!38+>*lJ8+Ss7Rzv^l%xW*)W{UmDX(>`Ag55vhS!|<^79=6)U-BisfVxIOV_8oc$g#xW*WXR`wT`e#k zM+(}qIEv>%19SejOz=B6NnjyuYA?Wvn_HG)Yz@LkmejJEW+v^%S+$w4uH7USj|eQ4 zkq{M3v0~#Sxt7-_VwO4)v4J=@e>-44Ci@u|tExmy!pBX2!d3y$f`8iBtEu3QNYOUG4}HZN>b*6a z^_=mXtSi1}s7MAA$LFT&m4mf2H}dR$2*{~7tezWQ*?coLiz9)0_7508W49YTI4A1G z*wuZ?*K3PY?`q*HwVZ;q)SF%5&;9e{7i{x7c$YFCA^-hLhNo(E>^m++Yv?;6>g}#g z5({X=$n?2THMgeb5q*X`PVVeXQTGbTe8UIbm11_%ey7d#!I0c%@I(Aa0nXGf>-g-V zc{ftwwYaWU9{S1_Vj}(hCb%drj~NV}tBZRn8g@k2kBwKwi}&qNlK(dJaTrH+RP=iD zPl-0%oxZx%TE=jare9oboa=LFKS}mjC=ZRFC)VTO?XCSv5ZrnB7Q`a^bN{Z`7cm7M zIFF5m1fe_=Df7g`?=q);%O^$&@Es1wkz(4y_NGtj8;Y|cBKGok& z9^@=JiNi{hYJ(Mz+F$zX($e`rHY++*B#hgwm`4k$SdyNjxCIi+vw_t4ra>h23SZPrrq zJWlD4IU8~9nfcZ3k@d&hw?V{Pvfa_WN@^lC=|@?5%VY9>T8rUei>p^T0V$!@6vvHM zj(TRB?2!f#HowZpz4Eq*W8$ng+=z>Q;|a<^3K0d&oQQ_5iM#{#oa-F@KhH@EeAeZZ zlqX=JFoDr9c-ZZb1sf@=$3mFj$qd6+SiRrsy0hN<&4y>8eH|P3Wl7=7pSXtaV37&G zUvJ%dp=1s%bC_f4H+dAzRkavhI9DwxDM1y1(`&&l=YD%pC3L`&uv$s_+;Ar5siy>I z47)LIv&Mc{bF)%a1n2Up_2hog;f*A!jd1Fg`6M7}Hpi)GTUq1>4V-D#Ghu+rP$&7K zSLs8PLX^EetqU_>%;P@w)0K%^q$U2z1+EFH(A<`8o<;bhpUq09QAw{K>yGRfG|Yz; zFL3u58J?c_@gS2gq00G~_}7vs@(;t!P(EVW`|uMh`yMr{TYV<&Ob-G166?vC6I>HU z^J)#TCtC`o@K2Z3ymbQGalP`Ncvq4zbyYnh2Z%3!Vx%q23|``zSX3n0_FmMbJ+lT! zO!eyor3Gw+7B-qJmri0_R=d)hxu(yddUPzHb`GzKJ(DdIB!`B`hp5CD$0-Cib06v$ zQOu!7OtGrK&}JG;U0}eY3sp!otxVoLw{KFt#1)P^22%dvQP;ykC*B0kgKw(%sOf+2M>6z zA5SD-hgy~DcU`8wnGZXeNtX9X=5Z)r-LC00Wv$|Ct&x}yJDe+U8oY~)s6J%SzH1^x?!$Pk3VkOoR zwVD+u0+%&t?I*Fd6Kz`lAqR9FXRNTkFIt1cG_1vKqBgF(VD_hjWS?YI}NJiyf}idrCydAt&ne}t!`@_-h>^$x*0Iu5!|er^=`;ubRK+eSFax^ z9UZtLq>Q@wSQl)ww(D!c3dNg9!}m?Mba(GsXw-hjh}biO;+-&>10x82g|P-1Mn>(t zM0KxY~xjp<}=27)w;bmUJp~^2VkSgC&Q-_@}`C3JFK%}`n=o;(ks=IH{P7ch}vV1 zo$feXA;N^`Co|^nM~*BaD!*Kh8CITA6D6bs;$^3H#SH9(gP^3lj<4&DoV==ijZfx1 z=(pc-cVgo>Q*=79PCq3!wyLkMVYPnjh>)#{m_CzBlriu%#@1VpxvJ1M;_=-)wJyjV z0$sqR$o{!{_CQjpN?f!#YHef|r;p90G5nicoKi&f^_xeDm_&+S(+3i@&n8M&2nAK( zW1Ef0rDR_0zQUkRdTpI1T3VZKk&C+vIkyDP!p$xWq|KjY?<1@^KLmR1z9)z+a>nn7 z$9%q}AUQvj_|wP34nPzt>K3cr)Cq;h?Y`i+W&wnn|IY)LO^cMT>6S$^?Y~3m;5jsA zlY?#VN+I{nf_nFSesc;*lMBSm)fW$aA1e!_Du*D~*%l_-wV5&l$;T1P>NB>ShN|%W5k9VejS^$AVQ=(IqxtV6PGt2dpVdFtE3f zJDWOj-VWL{l1>~@3(Ue8|tWf+48yHCmP-+aii;F$_epVF(pRWnQ7L-&;vj#uKu zrdzCWz>F|BV3l}-z@6yT4dE*`zB$T%Wis>H_wtn}@E*6Tp7-M#h+k6LreT?RZf&x> z>dV|SE99cHn%hYjvt4JM7W(8uTykxqardoKB_Wij+Wm3A5-(&88oSNpqO0uP!(eP( zu)-hbNB-1b(Ae5y|9iFSJ`LN<(#^^5WC|Roa+ev7cPM>Cq;B+YC+q z)?!HlS3jzrIURZEQxzCA;5w*B+t}hF=Im)d4U^Ppx{?7G_e+?? zrS5*A?8qNz*@9-PM8Q_U7SBouq{60fS( z9i=3;P_0D-z}-t@k#%BlKMJSsWNU*Zb=Zg?dOKmK>eBKr4b#k&iamqaJx&Rfo`4PG zqXs;%jIch*aeXZUbe6lU_pYh%D0a7NZ3+y z=*=QZLet;)tU1^}+e<>p-{rySn?%m4hdw;SNi{$DY^pnq;~~vCi;H(*ehjBX%xSDCyhfjLiy)a{I^Hcki7<1K#YDcb$uqQe4 zl4W1t?y_!l+xQl;^Xg(T&k*KEu1=`(2Ng38gcmig_ieT$GQcdiYb7dU%^! z8dEXs88FZoqN9{#|C*5-1^h zg0mLsa*sXyBk(MKpVIwSbpqn@+AKYJD*g$J%yoEk5_OGz1*5ryOCPefItwBriVNm@ z9Kx2npb;s*Z9h*u4Gz6m)zN{}_BYkBV7g2fX!1C*qS%Hq{Yst(H z<-I$TG%(@74?nY(u6kfF;VHgjR#c#(g5*S^&FS@1eZ6E~WB%p&VKe6&D0teLExfk4 zYq>weV1qw)btUIEybgn1Zf^g)y>&Rf^{ijNZPb-4L%u$Jt3-e&99m}Vsb4G7#uB!B zC=K5HEFXM9efX+?8mt8aUjj3deo0uaIwg(595Vb_#KxoxHZXEl^&2ZmsjbV zM6S+q8lOuPpF>fvsrHo4@TS~?EM?sBn8fD~v3OLYsT_ISj-D77J&Zf0R8M1y&n$z@ z8y;WXwU4NLUGjW(;YMoJO2n~2=XZCU#6(q+U?xX#wa z7(m-a9xsko@3qsI?eg1NFgC%>-`J5}&i~AEy?x$`N?OdJi=-l7g}pzuuHNLv_I-x@ zCT-Hmy07RQRs3wO_hjy={iE2z&ymYeo|6hHO3D36-rlKSyOUuR%O5H@BdoGEYEKW! zAKNyGn?{jJjU4dW9WnrY&{L8y@97@6Xv@8-n)(+)?Gj|Ua%bKOhnlfU;UH1_9o~=4 zL;FEMN*vi-{MW2^0QMDj9YTFTR&owafIIwS8XhKMk4laevRb^HWp@sk-JGqs@98!~B(iv+&F} zvy=8(z>~Ujt+A{{0*WT(6jvJ;87)rdEKR8kZ2l%E4M^W@VBOyeJ&dR|TTmi!p4to6 zgO!JKVwtb6>*>MTk5&Q|M;$4Lz}xc}r>$Y%ovH@;s?tTms6Xh8#ifhe2-B)@6g(>a zU0P~?FDwKm;;|m^6>u7xcNmB*bR&$X9%!%7-O&Dw>yfu3LcUcjX%w}qN^ugar~j*? zZ_*o-wy#68NpE(kfTixqgDejJCjGBiiI}Np(gRVKfiL+>f58|Y+`Ly>9B$dUHm$^W zOL;(|I%#gXw?O-TNkW#(qfoNzWJ=uI=RQ|^(9z8Pzp z`}!89aL0vWNPf8Q*Pv_uvLi=SwrA!G?~ZCl^{k3~UtS__o7B&uV;2s(s+1&X0~6_c_LecwI1 z`#{wy!s}iczm~X!E{7#(PI8B1vxOkG=xIv2BxL2Cn6)s-` z1{VH-@{*sM+8vh^wLo%eoF^TxY45hr=CRSxJPpO{h+cWFQu?{!H3Ss~_cVJ!RUF`|2-B`tk+zo#+M(<(j6S{v+sY%zb9ReDt3ed6%L`}2GgO>oL# zQtCrGj4+gP#%dzwkB-|%m`FKuisu{WSu!zImiag^L{Dq(?^rU$Tt&|jW=Ca~b$%c3 zg-quvvuO3XipVj@m^91guQFK)GKXG8>#}B-rZpUA^PM11hoieZ4j7XU%7*kE4x%4B z+>}w~HB=nNiAewHOlh{1XW`uayOwhR9oFZUjd|>RQ`Y~ZVc;;X=-nT`)aI>I7S7`< zlV6$}#`#+>SyR|zwk@)e*<}xYelCkX9gTSt)~DAn)~kXV(YUY0QjgJ3gD|l$Bm=~X zY?-A#{DQF(v16jWJUvSD=34j9nd_hT(}Qt9hu+O!-pH9l;)>#V>8*z3>*AX=lJlSx#kTlM6J;2R~wBF+EcOy&Fw(HCfLv zS(l=ycnf_IF6yOFjt%c_0Om5X0?Nih!0u5F*vUrZT43F>te_$3q7H;hY9*9&BV$Ug zA?uk18XPXdly~W>hH~`^E&6X1;Xcu<^fPF1Ih(|XwD&li!X@EyuwE*A`hcV=f__F`<^y*7EKuii z6sQxW2(*-=l>bUT%0C-`EO7ulHvkPW#-J$^ascR(1cVo~W1&SdPUri|Cl=*GdCdFL zeBAZ`+!_L)>MJzJI|o3mFZ2dczLNk% znxer5H0bUKD4#9@kSIt1gK_ZnJaz&s)0y6AtnhDWAY8SeIxhYuDy^}x<0uZcyDgp5 zY#eOE$zxu14h~V#=b8Ag_@oJ4JbS*)kkbrXwBh8wW(e_g4cg$$zB7%G_soAbqGx=8 zQ1&^Y`=!r9QSmt8MujS)vF9j`92Z?9@y8FGJlEONQm&C3oDB5SA%I1Al+m@XlXcoi zG5^kMZA<3gBOgL2{|G&`D$P{u&t&J1-t?~M^jA@cteR5%i5MfjgjXs|8wuBn%4u6Z zd#q)iw7n|FZdIP9vy*MbA8pZ9q2MpKlu#wiI=}qf3ZGQzZgN$x@{f#PC(F&MWT{mG ztn(_OR>(A!M|mmXBxa%e^m?FXdv?2(24w>5+Z*BofzzcPVGw7mHQ(XdPs+T{O4t=sY95ndpgktKd|x03IEwRd==cM!2G zdwQ&?U&2t!5zAG4=DS1^Q=xlA%mes$b7Z`WmA%Xh6NuLW2!0iu{lbc;x+^Lpk)v@} zczi5Nd>Y#@N5jwoo3|126z?tKeZ=u$QJZ{m+jKhDf7A(7cY`6=JdKb;orn$PMI^DY z3yEnR&94HkUw#Q)!I5rL&jfuF^svIleq?JY19ym=!V()hCSH-?6Eh}Wlg_k^j_8{( zBse{GLwQ_GD#%>A!i%{tqvMFlmipfZ^>~Utc`ZZzRizIu#A~%GzcSBwqZ7*vmc7c! zK_6;WRMn>*3q~hW`Jpn#D?X~kZv-o7jShb9%#ks4VB!~nBu}U#kZq#98M8-B1NeV( z;+NWT4#cEtnRAFs1CEE4tb@(pS-!gBjZQN9nMHaPs42Aay8nT2#~9iEZ!^LxC2I&) z71b44p8_xDnK4O{q}LUVR6aANczMQ1w6)O^3E2}W7|*`8hc@ywWi3vYeo_e-do zs+wtjHFgE8NJvF8GvT}q#MfWh@*86FbS{*>-`F(-FD0t#fOYGVVn``d`x`C8gDD}RzlovBOji_`+HXrl3p+yUUPGTv> zUS-%zf@RZzLyzBH4%i%bIeubXhwZ6+7N(vE_uo@V^tvut7Q~D(i)XA((Ii18$HoZ%hB6LS7EzZ#1A$!)(%K5b-WTu@xbHOz62`&FX5GfZIoPJfmO z`wKSD-01RQ`6`f;F_~Lh27`b$$K9-=RQnOc;b``niMk38PfNcJ9>2xv{kW`2ZTEgi zv8ycoCSrx7+r9EacMs#&9Dnct>+gXGtem8R4&oO|yvBa!w;UTN3z2)_ZLIQ_)en;} zcxj~6}Wobqk|LN?0bH-wP-hdvDIQ^f>qrEQ6i zTcNC5l~A@)47lmksD>fupS(=1eY2;Z_-ei9w*xb@?(lYuuz5XRHX|JnJ0ub=zk{;t z7-akFx%SM|RN|UfY~L;}iwRumi3Wpm*77^fo1(`&tP^SH`3YKgImZVJTio+fv<$6@ zkINaQFL^`IQ(RM7ro0W0^`p6prkH35xN$Fs+5VZU$G?{r8HCsu=H;?8MUXaE`&S{C*bkRP5>L~&2xNMq;am1W2yY>{`3xP@Dl4A(p_G4*6_(sr18GZnUbfcV z_VKa8Gu|YjL`+it!JIHtKh)^>SS0a;M6;cb50fOMvzevh09+Z7G`^f$$tRgx*k z5F8mFE#zu;lA>ZB;GB<;f!Kc)SWPf`=2zP)d?0IB^wn`Sa(Q_5p>uHy$wzbbmPepf zg8)!%4|#5EYF5b_FX#nVdk(sxDK!L8R1#WI@qD>x0lact5=qvkh!x9Q@RE$oNn0L% z;=>*I#^7)2?OfV1%*MqUJC%Utxd!F^wXo04vCm_48)PMD9~2jk(Tma9FJs7VmLI2O zsK@@Cm(?J)E`h|fOI^5pd3Y8q(3%4;t1q;W7zy~lK@W!gV)d|gq|lm{|FoeTPZ^8| zZzB&s7|ZlB{Ai8{7E}GijQb6lyF8XD&L|5yG4QA}TgLlR35Yx$4p6T4=(snEfZKaR z`6>Y-ujSRE1s23DF+}8sa-kR^VMBR943V*+Ow)+&?DnwAM4^&kq*q}kI@8UIJ(ty7L3 z8ZLCN&~)Cum0%2}7JiRB8H~GjW#$!u=txlxLJrZLh}Z9v4{w`QGIb%gMS~Wi7z)p? z=VqbCa-IvayZzQ{%}GQ)i1XdGuPrJ!^|ds4(YMxCuf>^f=UzhPf9S&@hqB@uZ{uY! zxq-ewA9D;&jUYJ?Ew zWx~BGM$sfcX_Jpw2Dp~8E-JIQ+I^9Feq@YjPFCtCckMXW0&MKDuI^;|Hq$o9ZZGL-}7 zno*)8`(Uskq>>MdjTC%+&DI!|2U7(rRUN=a0A`g>4`cKenE-qw{ls8c;|<#o&qLR# z`yK^mY8jYfvx{IHN`BZNoI*m_OuVnP8or6Ww{pj262W)^4rnhdSs-{*N0OcHh2I9R z^uut4SKF#=bBofkE|R|aLElB3HGsm9`{Mt7S*Bw^(=#nQ6VJCMVZzcj&lUIC?|3!w z3p~}5O!oWD-oX&<7x^)lCHItfN#A^YPORxU-3Di8t28jgeOBqAKO=5e%$`hqNp}BD zyLD?;68 z(x8bS&otW%oUis$34chJhct)qS9G&(wA)(avK_-(s#z+=wP#-kcwKK@HzDUoLU6g} zh$*MEeI4mq#$~Q#&7Ghphp{7Pwh@$ybRJ#W7EC7KhX*dzQ12J#l5W-R-^kv;_ubV> zmKOll!Lq99wLsfR;l0F~15<5?%(>x~CWOvc%24ZUqgGp7j!-9NS@vqIxUQOJ=o{L=43(#~E(Eb#8C z6T%)E?;_<-t6Gx5%T;YabV;g{NGL^m^jX{1yenO`B)|xlV{ppqZG^HN$UK>DNbUp_ z`*6=Y_3Sl#n^u3BtSkItmn+@p1F>RvIEHs~hvzX6ye?R2W#CuKdrM8*fSLFtE07N} zc4sP(-T91=`^Z6mMq{tKWNhnGs5y3-UrZTBk&Mzt>K}o-LXqr+(VX5Qx7wP_`uDze zld9L%8!342Dl2W|;JkXy|IOV;PC%pA%#?naR1Fk0Z}Zk`D{ZKgKgUuDwv@;g+_jK; zPNxcHo%bbyT3@pd@jq=P^BXGtGmnE_D~asFJ8&C`Y~hjjlP|5M*dE@!m}O7rn>=<~ zjPE2>s!Q@!@J^1_7H1y#8|V_|r9%3Im})5t){+xUA0LfX;}xH9WBvY&@HIF&B8fhb zZ}-s)iUwtGIf4!4GyC8KE#>{EMy4t>2V!lq;cZlJ3BY`xa>k6Ll9p)Ys^lep(d*{O(NF^c||1yK0Pc#|9yZic0!m%AfB3EjvO*5MSc;gxez;PN`Un~ zfJChivUMQpBNvVlrzC^CgU_1BXQLKo#Bx&NBd4W&W%zq6YcBvix7&8r*c$sKx>5#z z(&l~U#gSB2wtK@i2$8y`FNr3`qUZ^48Oto$RWR5f)bw)4jHSZDT>e0MRvw(GwKUup z_VcV>jQNtxvs>>ceOB~M(-yZMa%|z*kJD1*#_n8V4wuSBeQ2Nz0rT}7mEs-dYm`wB zy%Vu#w+o8!g`B6pX?Xf*T`B9wf)*NXF4j6NnC%f0+l4TOOnBSPsLD^r=jf0Y-1LnuWcxN7K%#Dz&g>4$DfnQ#oT z15SjnsAg+sVav;5!j~?@f03BsR9K?#S~F$5J>YwPmDA0jqAtY#u`*{Me@Y_S2gy1W z#sK38jyR(cfgH7hJO;^|TNCCXI_kSwGTz>=?`$Z$oxH=0u(saEKXx?;(7a&Pd{C|~ zbN1mnwe2b*e1b|EWY=bRJH5p58Zn!zWUPE-v_==>7Dq1>#BMS$$cUFesJA>^$`p28 zsVtGF@y|iH{nhaeUxqU6e#F>6eB+S8Mf9%{LSurX{#Jo5c0R#G?R+ERs(t)-lD@P~ z~7BU;_K8N5UObV&-ov%k03_TYeZhWruYV5td6Eu z9tB2_lk2m4ktGz6(vu7I64ha1qwOFI8a06+WCN=bwcBu{uP1TxO z0Srk$Q6*>aV`;mFG`qbh*npN*%?=z=zoA)fF->QC}tN_Q1!xgw2! zu$1;wYG+)tSXwm($glq-+2PfRA#bv(pcp79Up>*EdJ**ID`CN`A)fZlmE+Wmw$)LR zo>m($G#(85X2eZ0JP`kOr~2DNUMJ(O!ZE}wn|gR8*;fX`Tb7#)o)0nF8js7f#H!oo z=ePbbYFR%mSFhdB=hDBl=iIU?be4b5?n!sT&)B_gDeA&&P4=%7|84QVp@z4nZUyCM zN?k2~)Gk;hJj{?K^{jQF=NYT(-2-pcKU1tY;dhOE!F`luRBw-`+eCgM1{W4<=GC<8 zUPG0f&cTw0MRH7X<5GEW5@zJwX#X6_`m`3zNybEcuE1g}R&OKymd9v#^zqMYHlp79 zG|X%`>m|02;3mDgE<8bw9N+YdX79Lqi0Kj^YN&1dVPcht-7g_kSZa^;R4u734R`jK zGP8v83!IQ97}u~{S~s$~Fz(ZQBP;yffA1Hfea6+u-U8P?R|X<)Sk$F)q_FGxq#f^G zTkXCM8J+DM{b3E;SDJL+V+wwn-J9Q(4~2=WG3PDL3J zW_P;`7_pyZ#bahSi{+F~e0*7|F?{```&~%t$Z0DLk^r&n*bsQ#Ep8fIBP*S95FS)m zNfoXml4Az5$X7kP(nFzQMQJwqX${S^@f!8_#gj4$E3t{24i2S^tX%x&6G*$tCf>}l z<Da7VmKvv#L1^0-}@cH zOgzY7X|?-_^@V~@v57Oz>RUGK76hukS_@)2TNx}y5ZxTGJepXX;;8Q?~xOG7tDO3{lqN1 zxROeG>s2Yi(>A4vWl@hrHhQh3xJ@Nb9Gig1!od%bOd5A_D1>dqgkBPnw|$JAedB5Q z^jzYHZ0zj+N7P#fMA>|Qpolc6uyn(+^wOzFNF%8rup&}RC`%(C-JOecDJrP+B8!A{ zr=+ZuAc9JRba-dq@9*CG&oh0d=Q(p`=6p7yVs+~k9f8!!H8eo#!COMt^GUrCbaZhc2W{&Dxz zVE+WQJD)KQ$WPlsT;cgR)q)$D>f15srPS&g;TA7CtEk9=&lc=Gd(mCu)9y@5W)i^w zo+)LGc_B(Nv2N;zLG@M=R1_YUaO~I)pSJZJqFX>`cXRX`7Ildq-leW&h_1mFJE}bw z)$YO_6J(`eqio8%;dAC3*Xg5c0yCaB+vr}DYK4EV5KmNuAVpOkybjShKeJ{Uys;VM zZxp5jJEGT5@u%!G_whCG&9W(r|wn&QlN#XU>PR$!?d-Ep4ic!+kuGcK1DrI>&dUKhy`dS4;OKQG|+w+tdWP zCN39}MRzmYPWNNuVajwRmJ-w8+9>&QK14(CoQ5pVm(bgL+ljbCR6Av>c20u}PV9QI zKcdTpdL*y<^Kol@*CPpt>>O*As9JGP+N#|q43q3lWIgt-=s924*Yk(9|-GYW{ z-8ZuX*7g<3G`)Qut|KmGO5ExHSCHN!8n5%g_+1~79i+RO1riu@AjZWIKkBA2lr)1& zm;hLTNj+9Ho_OM;XGCp(#-EwH;M`C@q;z_!ELRo0o=_y(igM z;`F#y%FfSI^|)ZcD+RRwVuW^(KSX3n4w9O)+_lHQ^?MYWNo-i`$n?76PH(=}z_3zLLSNre||!cNkHb zJ%r;SN14RW2&v}lMw0@bEUZyv9AsZni4WF(Gn%w0cqujhLz6A$_Y*Ln$z0RQnIOA9 zkwj;`_wL-$Tr~PrM`$k*ZugvtB8je&uuQd&bu^w+B9+EeB;^kba1oo`WKU zBeyH!F7R=jyZ3g!s$+FJ7nZI2PS4>pR@~R`MVr&K-S08j!jV+ofxnp|+>f~5#G&@F z@43+twhc8s%P2;0HCbn7e)@HTiJSM3!{<%ED?hb5wn&J4Scm2qed-s>vwD= zC%DZ^Srs2E=x$Z`J%zpa&D-4^I>vACDLZmFsWyV|il2;TtrA%NlxOKSTu`Z9H3I+G z4}H2>bInO*$pf0~m~#74LPa{^IPP{5=k)6OHPKH#i%!$oL#J^b&yEdBdXTbEdKFIj zLmPAlT0wCtCge*}KisrWZ}((7wT{QL*BtJ5?t4pD^Zfcq$m@64b!^`@rX-J5O5`M& zf7|S|2(gpg$85HuPkC_ghUw%zJGSQvnkfnLJ<=q#5_Tq=Ex6Yb*-6a43EUizL)mF< z|9>a!hq5KPn*j>JG>5X?1oNf2tX{a-r|id1FJfk96_>-@ZGjrG&1Al>9+Eg<(YX;h z-DM(ckmCGu)#{0!eareUv*lOnc0zepfp_NRl%z{A%z~#CVrLB$&lPxr;&jfP9fa~^ zEZ@RgE%$t|VvoJB)fo7tvs+BdHzON6Ug>D>g zWTG)sJmyu(Jk-Xxhc@MmOJ>m zNyuLW+&AHIJt;NwutfVa_3|)nHi(a$#)h??cHjIy?|NV;&<$mqXJp+{E0OW`5es|S z^;D|Na#+r#N;>cE`%?!Sc|A1b)stOZN{_kE-ITJx8If70!Dn)4Kk5MVyQUfNA zinYIz=vTa&6FjblvHu#Ejqlav)};5LJx)Hhe@0>8+Tfgd`!D%nBJ#Z1i5n;9)^@VB zWO7563&K))1st1u77JqHc?H>)e}WTdVW_Rxv5oyL^cdAWa{(2-q_K^}Um?}c7{MTt zl{y&Z%;|^MUDc}ZdE^77)$4_D!ZuJFpSOytisV)?w@riQ#X5R=O7btC%k`P~*tSx6 zN}UX2s%iok#@y6ENK4A4%yvNbz(}sno*gS|laF*zAjw;)jDm}|Wq|Yg#(r@cLz@I1 z;P{@--Gfemn~w%XO)=I5H5(f$bcXd$A+kq-9`tl}ov%zXiMm3R=BJql4OpU;P%Lub zSr57)+e-qNP-%_(gsdT)ZeNey9v`Z039&g@>*zzIqb_ZkVfFLhC^f$>T8T!i9Fp-} zSv|@cuhdA=TY-bU-hg<-i1e8*AB%>^8)*?u;)B$v?4252L+lhH?(G248-% z%hCB#aWt^1`Ihm_zpeQ*rXh`B95&C3a@~-1i@mMazA(Kq@K??RH~aFz`t`X_0}DNH z-A)uYY{P}9dfNASdfCP%lSe(%pnk_fPq5xl&FPJrhMJpWjkeyUpKBUCjwf#YU4qE^ z&I=4h6aQOf&0~G@#4f>CG-+OOH(r)O%H61dmj`BkBuU=DO4@+BO$&WUZ8!%98CNNJ;>fZdm%CUtTa>F6xxfGrC0`CMCD-=QFR z|9UH#0&8g*yr;+Kr_v0LX{n}@yyY5Jw(cS>bQp$m*$|K-C;NN@E6YcnWRSc_na3J% zS)K50J8IZ+jMq?^`U`1Z(^g|$(#{lUw)}D#SlOl8hAUfMor>=0N?g|A3!5eBh`F6|7JJ2-Bk*Tw= z3|<<-8B%aB6pG{CZSi@QIrn7SK~X8uB7)a^>mS|OQ57t1&W|Id`^>>v zH-(j1`DkFer=er|TpFVFBfyHYNv8eKCqaI`y%TEhu5Z53yg8s8Yu+8N*3-{%i= z*@`5p<@s@@)mahDU?2F+^T(^8lV^d;u7T?GyFfM0d^Pb|*v3|!?x!HnZuLf;%Q1cv zP8AY6#FqJo?JkZ)`-XrI*FQ$ol=s{X^lAWFwEfijo}nmq4@KYAE!Vn4ydP>@p+}Ey z1KreRRTVG&nD(r67DHTiYnP_|kvzGp_LZ!orqd4#ri)F+D`jOSJ*)%v#@s5sN$K%DE+zA=474gBf8%;;?t zMO=IF$&Vpw7$iz#@FKXyaS2nvLDn>;58ZW97gQ^4^ zm&G=S-28ejWb3Lsaxx!Z|J$*7Oz6cFGC!r6!v6PFA+nhmRfK-ebGnyCrpB+;;c}4! zKKcv(6Aa*~LLpg0I1n!ZjtL-%ptSVi6)ws$Vnp-%k58uIo;E9 zfQ6|JCy9+^d6Psi>sX7X|KG=53?I}m`OG8uem_0EHFQRA#Yt z3c5QaIfHjM*vnfkpMX6B|Jm*W%F z6?ghq7@2f0rkqlDM1GY-oLuk{aEr`W0hDz#3~2lQ?Qo^oxEhs&WOG25S|e6k#mTW; z2JM)>n^)qj01#(CJ*QRVAhd}S<`>stpO#sCYCr>2{O~Zx;PT%O+It8W&N=>hiU104 z83BO$%<|1YC6$>688AP!4*RkguR&7ME4A7PDnM&!Q$$CP`w_!)S-r5>V2Z4Y$7mvo z8rk_+-*Z*_GTudMxWOw-pS^Ji+Fc)5h{(xV0PZx3Og!!gz%!M-ojrJLSq-1o*AK^y z0Nk^a3hwp?M1%J^*K`o2V0UvR9DuyMgG*ZHt>9;r3~m`96d;hWy&zKKY&h0`N-D)l zt-L}+XXiSA47j#f;=A5~6c#>*E~CheZ@(X-Iduzt^pos5Gn+{y~vO$NxLLH&$#72*gK zm2{L>1TYmtiQMkty%~nc^=&YHzlnkFdvV+;T4bS$GW~6@2Ol*wPq6T)Qw8K)5*m!p zN!D0Jx!Hn|FHi90QQ)C2@Hb*xOWh5HzSmhcvvG5z=$4qTChEUG^MmzX7Bx=KLw4ZB z%hLpHv@n1c;{tAM(M!v4q*OQVolm%?5V{aX4?uK?$68jgkpm7C#rLF%7hhtmlkh43Y?nWeS`gB)#<-skwrQsaD|ioL?*EL$mm_yh_Z zA9B(41tOH-ZIKGvf7V+#b~;N1r?*?RR*O)zm+Ec}F83qP7^eFN{dohbo!0gFn4y8! zMArB`Ay6{bUIT*l^FPA72{r=evWyT6l~U5rW4QBMvJ)?7$M%+S19K^ z&9cVn(YSZQham7g!8VtL8p_OJy6>_f4ia+R_16Myvy5;<6f zf(&~ha8^N|3XVy#%p|%df1P9y{r<)ugmIQTPP?YetrL`ir@K-q7$4qvx)sH1o~ii& z^{AUpSQx~c%>^~lu5sSQXa8nq>PQS^pGIhGXbi@cW-fs!#}ruxAZ&Y|VB$NJJc?8X zAled_bVsm&XndhdP=^BzH0pn;+a{aj!!T*fQ0XN5Ym%-mBSr^H)BJg{tHKVRtV zLz$tGESdgNc|p}a?H2emkO$1ZHd{2vv3Li+|KZL!w~g294@o*DR#C&72%Bp*lrJ=V zJGZV3q^Or)fC-dRuY!PON$VVy#nwyk)D%bpH!@k8T)+!NUeyNJbN_SuuZWm4 zIJ$H1mQ~lMQG9{@N~?EAtYm}}iQW4I#|Y$tRj7}qVDg+@3ivDZGm_2BW-rXMv+n`m zGVtHHPCW3?dwo^{PqmOs8!So#4~OJXy+a}<)5R8Y3ryKxWo(LA>2r0u_eI>=6XsMhy0fTU0tt_%@wtRyWYNMCRnUdx z6^ar;kQr+~z4a=CK*DWlZI6u`A`6E|N^vjBjBW|_^mB8d(Zsvs@Xaa!3VSQDfX`JU z_{$%e$NI%u&~!0B2TyO-_{SWRbj#Qtls?;IrqCats{Kr`?!5#6wSqHcU|wcuED5ChnedoQ+sfL!-l+l0 zcPtmV2PZO1@0__H(x5BUEubCmIA16k~d5{x5>oC-^g#?vUt$3PB$b zU+Z0Y$qfBiAB{l5slPml95;=#@vlF5D=8`n{a0mQX%q?VWuhuubS}vrHXB|ayUp(-=cd-mN^AVE_%XaZfa*>d$xK2EMT1j zxG__73#ZpNcuY(u_6bb!87)<~X)56}s<*;VFTv)6q%J)9okO+O;7CvbH|4fJG@@l7 zVB<*Z69|~D^8g{6`%qvS9bQUwSX)&3C`0~PMlg&$U^nQGhd!4 z`8P6pZ)nj5|6QTDb;$F`Gyv1B;Is5!0^U))+j9i(7Jy#Ew}4AOW2=QkJ+DQxBO|;_ zq>f+CpxLEgej67x2pxm|k5#Iu^!L%qccIhq><$1a$ppV1CWC?6<6hqk*#nJB!y_gp z;Y~DHtRf2AF$JxF-Ruv>*M$oQ3#ONlQ%>{U097nriGrhn;vr`ph(OmYSUmv|xKN%N zVU!=MRP$g;C#cp#v>}JA2SBZ~-oepk zcL6q!UcO6hQ;RZ!v#E;CE4pNPC+GAdzp@3MTuY( z!1vy(ESu3dtA*a|<1t})TVA#npi#`27U?pGM|pw>2TUWG}#(w;|2jHOKBid?s zZe>uX=Q+TPGBxmb+6JglP_-5vz}vuei(ZvR+ybkWQH6i-A&-$Mbn@|F03g&S{xD>Y zQ9xMXWc72S3W80cQrZ4$GO$$308Wt5-UCesX=SXD4d4Bl;gf8QxD`|jc)4)<-&Fm3 zEIpu{?C;I{P*%$A7;isoZjW#<#j~AdL9fjv`32bo{mvgpd9u8JmeVe)5gLO zju(v};O~$qwVR`4Ws`by!Eb1AmF=cp3!=)kAl6(V-A#5nPrbaLzy1$s1joJMXiczP zmpXAsaf8(O@GZt$X_V{V`1^_*@a4}oqo`N_uo+LW)l7jiMBh;^9gjJhK{v)jTkvez z);0cDO{E*ynnnFJmjgT&Yx*}z;hzGeeCYq<$v=0KXJpX*wezh6n>k?4)AcK=qnp`- z>36aGZcU37etDk=)=Olg80}r=xZk6t^Gr$26T^5P@kR-Qp zp0?`PST*rL%VY*rRGQr4>}^Xa;82YY-&Qq9OS^<&#AIBC6oI0xi~ANQpj(B`AYMx= z7@e6pF@c^0v9E^Ja0171q6-=~Y`8OSn3EJZyfbA@ku>YDY37D9z`S|8v%a;hQq+aS{&K z-DdUd7vqwHfF;qMFHSB}@d{tsWcgT>{^X)+4{s*Xs^Nycc$^<=nQraHBaNYJ4-1qR znZ0nnwtD$fF#u?J`_1(IIm=ZH!3RX(7o^ackNS`OeNg~vJxlZblB!J4{{2@bd9TDr zrsc6*F5ivHbY^hcS(cR}Ov+byfCO<_XyS~&2jiwSDtUX{ zPUsB!{vNvk8f7gs-NPkqbRB|dGl%@>by5nJ$_h#&>VsWg2O4twc8s+SeES|9YMd6H zO1+TC3`)!IrBenBdt>|V^ZYhYeNPTnwq)g0Kh}Lk_9BYC(hD^V;{tUAPWa@@D$`J5 z0OFEeGUq41Dvg+S*WAuq!)-+NP+=LB+nk<5%A0`tb|lbY72zluu+{w|n50(Q;pt}E z4~tdI9ORb6+EW|R!OizPEplc9pLTCM)koF9L%6QtduqBz+M^Q=gH*&L zV~DQW+Q0{`0=>kAsT>*>o%$X+gM#{ViuFAK!}+v2TJjFVWh{8v{y20V_H8Lz?6NQc z*_U!ELvJCLdgI=$!((IjYeZC+sz0 zBfhj(aFztlYg#l8%=zg=jA`e`{0rRJwmVP{Rn&g>WRW0DFV^|HNtGb%F4;B>N~M^_ zT$>G=;ErRQctL-{ftT#)@teGOARX6LFQd~eL!K4X{}6QuRJGSj%uW5-qRi63*r(IY zCAcGy)#ZhEreG8|>hJk~(|bcle>aZ*bdHg@Hb6L=K1THud|9}BTn+k&0qgfrWkvhK zO;v2coNmyTug4J0<07fMYuw`O@g*BOskd?Igr2bM`u2e;?2H z51nj_6mw3Sj!7asRKs#Yi<_A2Fem#2^}j@WVqdrZSd7`SO>FQGBMf`d3k9RkER9`v ztS)etRA|1N$=eaq8o$w(@{E*K_go0WqpY&8Hsavi(#YG&Pfpj!8xl#F{zxNpaJ(|e zWJ5Y?tIrT9NNa{?9sHGd62zEiCUXUvzmSS6eCDJ>7{&2SdvtwRe|liNCSki*TJd*k zAtSXh0C&#%OQ`rJtq${eVxQ|F9Ai4|>>oB&1s$3GtJ3|SL6QI%qKe;f=V}>KAeQhs zUfS_#``8Rw1OHQGI=bu(k*I%L=I4z5iQ#y)9uxTx{OjyqU3M1dfrXn%NeO-iPOkOY zDH-O)N2+0Uvp>_YppYAbJ7AMf0}C0v3PFgjD=OXt3`LVf_Zjyrfcl#+EbP0gi4b4! zf5ce|aKKakGO(*4+uF3)oPHuQC5K0h!TbQ$wZkq zjqktIs`UQ^cL+=+oTgjexSJvcR^H9KDL*JYWQ)^A87Dt*h|K=>pguHFw7^aXt0z7> za0iOug#ACE_|3l5nkhay)Mf#f#teF7S}k{dO0oAZe7fhfFx)g=zI&W%+8GC!7Gt~i z6)j*fvn?>hyC*EXqjK};j&sXEs}wwzj{X3YKPd|15waC2K8cx68^2pik9bhMk@j-p zUAXim? zUNH=j22v!B{QV`ePdbyc1+Z$XcN=MbT7rN$LE?{DzO>F}ozOgb0aig(%{qhhKEdB~ z+&{WL>a4u*)~g9@>9ZgP8@u9rQf)9#y;~Y%^2*MTn|v3%XP;Xyv~ZGON0lj}n|8R9 z(0B8<7$U(P=-C6}sOiiP;y3{Wp&Ll7vtYZ2r2sq2svFyqw_7YdlUo}ZZ$>)71 zq~!$kVc52vcyiUWL|E7f$W>qaI53}{tY<(kq4xOk^DwaQDzXV7K7 z`SkJCnj4|L;O=#ydv}U;LdW;%S_hQT1h42Iuv{W1e!y|16LizsbTAES!E~Odq;`-( z04T8Gih^J+pRL{)ZE{(TIxW1Lc8M-W2iAU{BFX*JAS}08h|XzNe_uQk-kvahJ5lh% z`_3vTuIzqac&!>9=y&X8V4mrONunS}`xCZwr|CK)>DE|2%RvbKt9!%bLMYc;MA7J= z%xS#tuZp`BtwaeE!5V;BWZpy*5)M0Najfk-J1I3CE4g##MoH*%Ft~lqjh5&pnCWwc zNPsi+85>u~Fr7KNs|wmvLet{cG%cO@WUtz<=~9=Y=db!iRq=BsU-a<<>0WyQupKvk zR6?p+Jtlq4j$)8tclbTHbVoo@#S^9p0;^_wXO5s8{G;NzXn`&1sp`MjG8;o*f3*``k+W;zzq-me7Jt3};!k}sJv;h>!_mp% zJaa=Vn7-myDhHY0jI>O!;pLA$-HFL#MMddV#oUsf?WKQ*ByZpEp6NjXxj;_6v?mUs zCGU49wo$FIna|1p9qlDJL(P z5%qwn?P7^G`osxmxt{Se;LhOI!z$EUQdPVO7iOV*_*L4K_*J&f8|u01XF=GHl&J-O z*`d80*jfzivqQ|PsJf5AqqmpZZzgvcFMpV0bv{md=t7L0*nf)^;+9@Jv;UTU(H>X< zEg+F(`zjM>l4fuf(f)OzF5a7^-(7MFc3-2hYk0Zsoz9zhZ-h?x*YXE%bQ>+XyuJ#J zKY2;mW@A^YJ-+F0XCrWxpP#(4!zfeeAx|CEH(E7jK5jITSm-M%FFtTPjNYRj2;R4I zUX=&GKjE1t#|9+8<}2j{q9un&qlwh#mVg8b9M- za~-fqGybY!I{PCk(euSR;h9}3(Q1O+yN7)$(X{+JgzL5-Fx70MMQhC*I!QBz{gaZ2AXF1AzqcJlp*N8x$lI|<@7(@2#O&ROUo(O zBoSFr_=`Wym>1!yl}evN&XFK}955_`yxhA9vGpSJ6#yy40Et_It(W+rN;ovU`qK;ZH3|FH}_N3$dn#PB1gY3oAr?f-E|6%}3 zQKdyoX|? zgE^C{1(o5b|6h~O`N^Dy1SL@aD>5(QBSLit$e!m{)oMX}svj|zF~I!cy8jB1UH;ai z{<)Zd-t_pVb+2VGAl`a>l|x?xa$_jZc4}1rL+As;d*VR!`iHdaC_>dVE63mjsfm~)EJ1ZA=?G+|dbL|g1kmc&W_Bu!gr;Hj`rq1pF zvVIIaRt1FiBG*Qkq5!8>($0J_K?jP9(mb^VT;&tgA-gp=eLOn^)53)%H|#e4^lK)O zYYN=AX33)w0|F)ewsni<5*37Q8)dYL4{s*y`vGDX$e70owmjD8gq+F!f)c*|XV&O} zy^`D&x7{XsuZiQ$oHrnqiJfX40HzPaFz6<-DooK@xBa;x+Rz&4i%d^aR=qNngUNNn1Ej7CX?R9c;sZL%!3;=7f zJF8FZ@#pFWo||iHxux#7SZh0**3UqICEsEkjBYi!QcLrx(Y<&Cw&9x3d57Br+>A!# z_o>);A49DfI{$b|f@cTh z0&7Aa9}WJS_aec?BbRS;@`?vNbTHk#>!pTqhIcR@Uw?r|sHdVdq}{ej4k^(UWH&?5 zSq3(kj}vaEd~XExu+V`*CCgh;{o@>;e>={CeB%k8Cl}4q`@TNQmA?d6Q{-%Ce`|S_ ziX*-;%LPGQC|*=J{qyqsC}5Q7Q#ZV2`CbJVe`$Rx*eht1Df>phXfOxdMtIBlRFuZo z?3DXPE75>yd9YrW93PD@Pi&r2fjIGiX`~ZF+$SJjrAEo75JnD|>b=t2X%3QEc7@;7 zTf_01F!(OxUVje@Se}iXccgCMc}6ecnRnqsReOcv+&CF#aK%KglJ6Rl? zchJnMn+F2{JEu^)hAek8H2Gc7L1EagYdS<)7|}mAlTs zmnn{b?RTB1F!AeQU)TQCc)_d9@LS2lWQ=#&Wy6u9CC!r@y;;aXCak2J3WuC8BY6Rk z$ZX9caU*ZCAIaLwVX5qNIujZY!$`ficv%Kb@bO3luQ}GvfKcncL`$4)3?WfN#ssts;2>Yu)ZLM@W z`Y%zv$k83?`?@bTGQOup0ct?2JN$%-sEN$$Qm~zt^2KH>tcUGV&32QgU5fIBBIMtF z;sI#FO_-A~A;^v5K$fM|G45dpvWmz>BYPu_fICYZm;CF=F~u2#F8~&+51fzR*ZEO% z869c|n$v>qq4nTktxa`!uZ$%{PHu0}i|&-o0wjY-8B)Gymb zgyLSeU8T&jCr*7zo~Z$K(*9yX1oZe9nzx~}tfliQGedF0cZ_Mo!ApSWWZ9c!Pf0uz zUS57{g|#*DS|8rs`2sfUzj}945T7>g9j|de!t$(g7793$16}&nUm&8Ob;O>!1|mDv z#FsE;;nPr{UcZIMnmlWBvL+xikM|LtM2CF}QLQ*QbT2KLRpg)n2m;@y@FCU(?zrTJ zk7F0Cz>l}u0A!)UwZM1fiz{fSc)}=o)*ZRq*G5bMAOStN>7^@PRGl$kTj{_xFZlH8 zKi%+#)d~O*aGukRGZ$r2_ATq-BmN#B44PQJ)PzCr_U#A%CG~T7i-GN$+_b2HGsuyk zC9dlK<86oZn}TAv8#KT(wWEs^W&vaX=KR&vJiMLKNHn1#eCfA07LO(H3DA9>f&!7Ke`qZ`VF;pKa>blxj{wv=f5 zE9cZ)3N=*Xx2HxJ0E&qUP6PlUe9nH%$>1H;}u0yy+)Xzcgv+8PtFdQ(&xf89Bf9+S_lkNZrUdelYnBcEucJgAwp2B~PRU z6GLTNL=Oz1d~qDle*v_tyrjuX;nJ6a5>Z^Z*G!^#tcwwHbb%T7L!i+&LLT9fjnyRVrCBz zhJ_~Zr~uH61Oj|^p9@~8koph-LWPJ>xNfuNB0g^?5eg&%Gk;UYmzO->SRepq^sZj# zN~8P$=shqryD8j`I!Cn{0GiItpBOlEa1($m`OkXAlIcKA(@;^o z6^}%;TjO6+{MtN|erNOF>8-D9^AC`52JfEPFNpt(n@@CScSPT73ie)i`a15&){`A| zr7VM(@TrH%05oiJvOh5X%h{E&*9Jb^?}Yv!89Qh*t*>_6U}ew1P__$}y*NRIj>t@F zz1>iC9~ZKx346!}0`6!S3GPQxvwc(igG_5nNWiDoHYk8YBl@3wQsny2TK)g~0_%Nz zgo+_YJNK@$cbOW5J|2^UqD1}gvowhQvqtvod{Rm&uFdk996T?0)H&?-FSWzvlKgNl zSostw=G?77JG!))`?>rC<3URe5wx&M`^)BY%WxXAI_O+fUpH(z9r$dmwjtz8Kd_yA zv-?VFcGCi}KSeiczEO)5cD@u*S>M|y0iox{4%`4Fw8tT6UvH(qEkCK3 zU&uL&UO`<5v)thSQ?P*?Ru0*%w3kyaK0r1^?f2hxh%Ay|e3TCYO4Ge8TK?7XE97un z3C!3L#Hyd^^Zl5P*3`xURK&L@sN<$BBE$8WsM1ia-v{;! zG3sZ}7Xp@kP`Shxa0Gqx(#n z5^;vm*Fi4JBl^CL5{_qCS0nl!-hS1t0H;Ja7b+exio;0HYWH%q?GEuV!gm;Lj`0>h zYMN#h3H3&@ZM7_I8`kWo@~NMFofaU~8RFViB>UrH6mrvXJ&|$AB|EA70_CDgv=F+L z?VaC(Dew1@6h@R?H@Vat!ES6b{1AfT>z>C>(;iF@Ixz(M8ah5h;=u3RUA5oQz>KcF zz;!2+O^dUjYlE5*Ie8Db6__i}W8l+s8%X5<>eRN#J3L)1BaDMxfzoPSEYrRtwY7Zj z&^N~x(C2nW`LPECq31t{#nS5U|C!Thjiq&?r8?>DGJ-P2$*Px1h1fHL8!R1}H_3lC zsb1wFkM1Lc7`no-p2bLIO6}P1QWSWfDkHn2M$~ug{kG!`wwx9ti<;)OD^VCfo+3jn4;}<$n#m_=MScD z|4V*0AN?zn{BdRn>%*f{b1P5w*(B5vP>SK*rzBB2)l8s%GuZ4WKIdR{{ zw^O6z=%QCPt5W1XU8_=`<*``)m)J(z?of*N3@PDj*6wER&pcK(o(^qe4(+@Ax*+qR zydJ}I8QkI=8|S5RSyOkMK=)Cp6&3gILFTFKW+QJ)5w$ESA=IjWN4C60zX=tmeD#Wo zY**1{0)xVzt7$i$uq*%UIa5=_TEFIDA~UtL6|XSv$CL}FUWE*dU!kR!8%v{!z@Qa_6gn2P z`|lq9Wv0H=zCNwkcY53CEjnY!KR8(R_XDnSt0#1*(r`! z0cYI$cyyIy|JOT*Hz#4$%z*stpWgQ9Uz#gUgD1(gHIUI%wo2cR*Y4~;z+^nHwY5}Q zD8&sF?#Ylvx5esWF{Ll6KF&!_rsi+%gGc7_Pq|xveRk)9fq4`z$Vzyreu3R*5w&R02 zNj5%@`A$g{4Vtc%Z<32nl|Q4T%3HFOE7uVlQU89_}LmV2D$28Y8HKS=6p22QhE7WySQ!+NKCYF z@Orc`ZacYXQ&Dfk=agOYJ+b{=Z1A8VirF?6dFJPSj8+vEc2ww~7b-Urc0Aknm}XqE z{OF0b3-I7PTrnLvTnqRprV7242)Z80;%`$WaRkT(n3|>*g z?3+FxMRL5m!hiKLRVbtNUSZ;K`m66rf|{Rbmy=+dgRyiQg;xAF>?kEyZ3-V>E!l-` zR&??FCM$u(dNoWCXL;GwHNAGnsydTyV{fV0zOt*MGBGf>_ST`5ER47*GwV&QmhoVJ z-`DBNWa8@;)BHToLddL?rgDod7Z?#qvKG$6eP69UR$6Pp>+Dz7Z`qWJby)>ywgitY zpMf)5?Y@~yVRsre&&D6=%iZ0R(JS6s9z>j_GO*Iw)_PS+P6|Tw#6@8iWs`3GH&5*1 zV1D;DBXip8&BEZNruJ{1ALrGUN+H+!$FSQ$9m`quz}8qKHu72;rZro_N#<694_YGa za45t7wXxB>;TgohHoJCnng;7ImC9DpinF+JSoKuK0{3*$%U3bN*LE6nki=*1OHw8K z6pJxBsiEbH&96wIbMKE6NX}WpE)cY}?gm-?v1_eDi$9LuhNVdi--AQEPpiAe5CfOBRVm7 z{v%x_S{q{(3N`gd<9BmT zZED5C&jLxCNCwOU<*LGFx@hfBi(Gq;+{B=ih;sPtc-YxR#~kWOZBj*%+Z}3~kXi?J ziv!xun`;3bkd3dj65)(D*Jt)7hoBzMFi9TE3nAK_&MCPrs1w%Ho5E#xur&`_J@RjI zF7)GiOfNW8Hl*ztEe^a|ZZheZ*8B|VU1z)cCgWl?VMszD1zJ)5^{d3fM26|K8mn83 zOy11iQ4lTd$irbiY@kr@3enqKebk9}YX5;z^oq&s(ezUvB^pdz#o z;-7D~k)3NYNL)P+Txc<1sQF3ZOjg8#=%U&M_vD)4wn|&}L;+tBU5R8G&vk{zf-T!{ ze(OjY3nc}r2sw2aqE%~l>FUh{rWgWcqJn}<4b*{K^Bi>|@*~_fI&kW}4G$ZhVSI6o z^*u8b7d&l4_VKely~PFVTPdRHF-*Kj$evuwkqfR%)p0Vln-=Y!(wj2s$y#jraFjFe zBQv^_J*7xWQ_x~i6dS#0e{DAHKd-eE{WhxprQ}}2Tvr?>QG}CNlE|%bBBE3zWIprNb=22By4V#KXP>IBcH5n?mvnmZ+#TX zTTE0|O-{YzfVJ_}t_(2OJUe|gHbNkk^&3gX47uNC{z`pFG;l#hTfM)X`ljjJYzIfA z2Aluc+JTb?=}}c#*%Pj%yuV$mD34t$xY3o$Ehm*=^O*EQhLKWFg3{{tVVP(~K^dRY z>OkoAh$}b0{Vp(dZLrf$_VMrj$tD68-V{ZXzHoqlzrIZ|2-V$=df}esW*- zjjd5%+q<;P`@L7OWhC4YWD7K1nHMTBM3s9+#1hE=)Hr&-3b5>1S()rbe79C3G-x@A ziJ>_E!Q^e|)iigpNPdwe|AMa4m)B^xo2X5LX)wP(^8{Y9#J&mOerIHhl%Z zg>W~KWvXKoxo3OO+E~U8@S%ARk4LVj=m+jBP}5*o?;zCsO617tUt8%1Za}~2yDr=9 z{|&7cU~y45ryKSmB8qfiC2Rilv>4DVcEU<=-F6(7WiL_tcr;f}dp zw`ZTdzwexV?_c-q-&yOObIrBZm}5TU88MgonBBmrW1YtbooVhln<%AB%4tvsEfIK` zru4{p88`I!%xm)a6Mx=)XEc;WYv%53@5xSsrqC(x$FzQrbcl*SD`i@W3o#d6mTj_z z;SForRSo>?;DpLTEAze@sXJZ2IdE}5ZwW;oL$dM2p&j~~g8g@{A67wK4`pjCowp;C z`cBAn1Q9i|8Wbd6I@JB*z;PJOY+y36M>UQ??0EjZp%LYeNGv6hj~)Ya$tEHO#^cE* z+#eN7Q^yTVXmX|G3piLA{ES>#c>ImD&W?I$9B5cNdl=~EtOj*Mf!7+D0*s@mJU0vL z;{NsCpwX-I&wAttj41@P_%1xMxmpt4?Z?R8#*t&W?9pDH7RGC9}wA3M6{>Cr!l1SRUrHhOm6dO z9E5cfW@Q%qd?}Y3I^?`KQ+j*?THal)nLOJa8xY-(wDgU?^3G!(0~3*Zd44ggO~TUM z@c}XYL`|$iK26P?FqGByX+RXO=c%fkGQ-znNkrBsW&tdhYnt3neECWn4pr2}PuA$Ksk^cQkrUBdf_b z3wjU$)&T^F&HDDJx=KvR)#; zoc~F>U+y-gKnanP2aSN1_*JWdR5TA0>1NBtORqIG_6Sp2D#A){OrEPJ>Kw+ z>AgfmBqHPqvTe3(ReEi<(*4ss0X}pTY#*EDo_`*TCl$hEZ$5j(OvyJulO0d~9}49YkXm>yH1!WaKW@M;Td)w`>oCeOB7K960s_>XnFokmH8mkRk788s6Z} zDkq$;v+=BDzO=3(P(GmIh;#Q0Q%|$Q>h7l9)Ra484z0?`zZ;@$3v01lPqtwu;)^@< zUDcE5uv)OCNpE2}-o0THf!@IkUprAvCnJI0QMxO5_cZcir{;oa9ypqMhkfEv_P&c>>zuib)dZwXq=4%BM+UDZSej@Ej}jerY_>EhS=b*+ zDpN(`ZM8ws;NZ1F=P8rNz@l}!$%HyzN3Ndw6a1;Y-ttS<^vX{0JtU7=s#&y;s^u--Ga(C zTALMpqJc_mAQgGMJuE?wLH+~HQt-NYvRy`x@cZ5I=JptjjdeUP;$k(OM$deT@!VUj4(Q6R8q=~<`LPj1a- z;@J&C;kU7xpjSVEqLPVbT@$Z5NvDMT=tIM^w`j6rYAvH~?X1#k81lLFxKzNRXV;gV zCcM3idwN8+a$e^SVHurRKt$-({XQg?;}tQZV2Bus_-*IitgHy*`X zx}ROKErfO#rx%&jO%lB2s5ng+zwyJCt|jKRWfvvUmYiN~;$;DutTW3M^(9 zP9-(>G&+zlT<4=nL_ZP)vp3^<8Y!#v!px-A?+dt_7UbzvkA!wOnHHR|l=wpAbIt^M zq&uq96P{Uik<%wNX*NC}njL|E&}NF((~ImIkFiJ*@zZ%S4SWPyU*G|i zgJLSiFgqBXZ|fi7{as(5?TE74{RBzNU9}C?1YSmT8=LyvN6xF=uejCz4!h$G(*h+& z3Ayn7VI!d<65p@z*yYh4NZQX4DXK{lu>Dx12sCiXZF-=*1|Cqss;pLh1kdr|wPB+t zb#vFFO^_~E;zeGftMGWQA6!5^!Vh--wA#ADweKmOn`*_?Ci+U2v9I za~Bl@?ZMxTK{SfToqw?CVu_Y8Jo`jT$UsV!Y_Ki!z-p2hQP z4pl-%HkV#cdRf$?hn!;rh!GubKm!pI&C> zn*%l;h7SzHpT;QiUpL;K8%8ew^7Gpx&41oyeKS zXJaS8H2Rs$#>FoGTf)0q#%XF1i-AF{m2V|1l-Z8osFged&DJ$V+_eV0iV^fb_L^&6l+;)&w%|msq>y{9|v=m}D)h%RrCS!y<_5f+!)5J9A zcrvtmXO3NAcaQfMp=s(ybca*^2(Et1{?@T=qK`&tzLX*oM%UI zri6k`rj>8!y_d4BfI&$3)Wl{5W-jtfKQEH~z<1C1kr967_k{W$*_Y45Etrp8EtPF? z@h2iu5TcCM&#Q%q@wF0UF_%nbAqsk})R$SJ#;Z)<&1yO^)!7o9_Z^y6+#TQ}KbA5nS9>2rc^Sm9WQ5)`TgfIAL(O;CAo;7;``BEeXGlam;ui;L8-)6$vNEu?_$SVR@0c22tV4$8W%0^Qka3#nK#lR7LVh^z&*R9|k0cKmGbx$6BSF zxEGdZI6X|(>jQ}@Bq_F@CQa*_36PAvp2mSRTRt0)cy(8n&jswrqz_89ad*6At(1V z(W0o>t0}WLTm>pJUvthcs2!-r(UA<`KrL`xT$7)8HtPz`-5D@Iv4I>K8jYk#j zIy_T)Lk2W7(p!6JdM6LgsTXweoFl+2(9oI|YVcx&G zZD35XV^+ODW50R5-q5f`7Va$-ewtFW5$ghdJ{thvJ6GA41LI@~V6^EDu8l{M)x7oF zG~)Z}?~i%3<=??u;nvE$BJefGk}#>fqFz&me4GC!b*T4>XIQ=;^OoLMcv0mht0~2q zjf!qfsqT^W+V`yYRy=d_@7T*M2PVi8)aaG%KYrjA_5y9-bN1H(oX78}a=9;O-^z zlW$fln9SWKuKOuBNixm7oPWXH%-Vj5>$Z1~2-Oy^-}{E`EeSF$5*=OYuy^~nmg6^; zIx@OOjR)zOd6-{)zH09bQCc50TqbXB5tLCsO+e5;l|0c@?oFhT|M{-jtKinN0er^- zi5N{})uUy-zGn`dJTB}hQ%yp2RJ(1L2?bZAC!BVSj~zLnbDf&vh#b~guSXNTE9J=) ze{mHLtJ57C9YbmgK|0y4XzzVN$6chi%Hj#=jPA7B;@_>i(I#Z_bPQEqWna~x;(TM#+o6JvFjB30 zabwoH0kQJwP#?O`Dtz;z9hLXf$KYOP;sKM-`PZJ_zO(m=y}zc+<1$%zfS0;F9Pibw zem|M8R4ffMy?fobebG4VZOfCUreJk<%QOSO?2bJ|%}>VW;7gOcNV~^muGl;G%(@cC zJv*2@Vb6Lm7SzkSlZLnW?c_>AK-MMVHcPPdd=dg5?|^LnK%#fTa5^Q<%q-X|_0q0= z)R^LoZ#g_~#A>%|n`~4RLq(LGyS-h{EFH0c4sqCvLQ;EbRrPTvp|@X2rR^UDZ|^No zKPIC-PJGVExu(I5g-keh*X)e!$?kdTG`91dr7vgZuRds2|8SlAdVI*im>k#2f*Mh# z<0tfT@9~D)Nf;QFcczqW9-=s*$8k^TiL*p;ccr41lZ{{4YCG}5k~6c;R2tQFyf#~1N4r2=>%l?DU)M$|45{w26t`BAc_ru&uP&b@bW47| zv(1}yNNVhWs=MCO%H38Q!LNfI@Gq#pAdxW@T)%6ZFu^}hvq`DX&AndXc@e*wru5Qh zJ>4j=yHS3NQH933qkGqguph54uV$3#c;(cmDEN#5){H*_n4 znf+RYYVN^mgjtq- z_evmun)-bBm-{Wd;d7r`<113U*&!?4E7ZrnC>9YZyIDHvJ+!Hw&si9HI+_Ejor43;J(uJuSR#!u-5ygtV+f zE*W?C?t+5LYo-qurP;N~Nc(G#N(lSVQ0ttNhQ*|swz90Pf46Y;CB*|4$}GtfTE^A< z=Y=D?2nvV{jH;7-vwj|8m-{Au$H!eJDAdTECz)#C1*;KTpP_f9f9Pu0N0Q2?ZvsL| zL;`(m)((c}C-9MOSeC$!{-No-LQNXBxUEbJvN)~ddE+07ecUg>?#ugOp=tfCWTSj1 zd$Odn$wnPXzEnvo$Jo=>trUz1=v3_T#`|xpoMz^b$9sGGaUx$V_uK$Cx zAzo%cq>|^H+Cnqr0q2fbkKSeP8~WZXMyFb8>0;hBWXG;$BXZRA%GI{KL!QQSqTyMR z6%*@YRJ?IbA7bu%z$gzox$1T1y-XHMYMGXezzm<4e15U3 zgy+$~8{ak?13YwW;0bpxNVoC`9v9e-PhoL7D|kGA)O#wSF3eJT9$)F0Odd%KnfD9- z=&aU_>y^)0!mZ3auHW~OFBq^I>KcRE`li^2oVRpPnfgv#UTS@UYv{c#^HuM|H@*$t zxv^1I1gXig*3p9N$7R|{XzD=SX#eDOXM|tD=iui>weQab1Y@n@@8gvlJf|SAW6fU>DH_vDdYY8y-&36^UE$B)}krjD$YmsJ+t}`+nvt^#yclHv+q1_mp@{yM*FnI+JVvL+DdkXvCyfIiA$9w zJEYa*+*+Spm6*zlWw80$*lS=aa4Lk1=74a4X7l@zkZ<=?t$iLoSgDug%G?b-`+rCb zzJ6tC@_i50x>Z$POakY}Ya4|u@V7^JxplQeyN;^#r;cKi$-B*~o}s?{TGxd0RZjQ$ za?8Sx5vI%&1cu4H89iJa^h9t?EnlV7+*8yp3&i0d&+O@G4)$DZB$p7#k73x}`S5m} zh7J6h*&!J(G>{j~uw5F;GQ;)Va`V<{pVBm*p0$t*5D9G*QQwgbdS-f( zeASP;_RN;CketE)1b0yrl|K8?(-drzWIF8I7L-^vxiaA7^R~^ETEM<~OPYFc$C6fa z=lp3ss}~Vdauv1BD@poi_Yz zlpebR2H<>xUVo!7ib6B^@vYC$yt#8{vT?F+`>^sPhv`K1O)3F;`0(WW^4UxS>m%@y zXGovwg$&q=^$_81J~T_S$B|&?EKS4G#9{RdVInE#3#t_rMaER~7@oWu(_cK=6MX5K zTHnzjs4!crCMP6Hn2IeOZS$dBzPc44L#J}refemcat^oKi=GUb_1D*__9@?$FVOR4 zAR4v_pAtz_$*nmA9|JAga8vu@!xx;Xc?O!|uq8RMWJtVgRzyLB5=!ei`_<@t>s%u& z9gpyw+*_tlaOwbc!qxzLug%*h25I2srbuT;YR}GuPwkmCGHg=#_uocPvND>wJHG_Q$}Pwz?$&!e`s zUw4{OZ&}`jCH6?8UTO252n}!?s#mwf3|p8f@%?fAq5)icaQ4N3PpANm%po?rktL&KW_d$(O9e>?+a}(r)f!&t;D4P_;zZu7QejD zL`vB~l2O9pjxaliR_x#m**Z6yqUP*W*+jK1lYKn)w@PVhS?C*hAvBq^=_5zeDT4F z>|y@Y7Q5?i7diBB{|7#$gpEB*L$k|h$aK(B#~}WwWl7<)i{J0&J)%m|*I<$mI9G6D zJGPkO0-vJhxJQNLlz6b$MB&i=4@T%q5D(AOHj5vwslm4)7v0GoRx=Ep9hoqOlG(YU z)_7x~yvRetgjF8bpu;zWs#PqjH>P^=jlIuf{X?2f#H%@SkEFNhG@BY%2KW!%tLv}7 zruJ^R@r~`BdUA#3`Mi7Thos;O55lS%7WmR*hT=+5Vl`aIARe_n^)Re9^FhvugL8~W z(oQkFg+HvN=JrEVZ1PyJrLnPuE;(N=Lr-I)n}d|oWBRFu{XR_a#b^QtEA!2l64fn! z4b@+-e>XkuL%emKw3D;)Q6Vk-6{-R??}E7H3i5_Kz7pT1egMyjB;UI?E~}X+=au&I z$-GC8F0s7ftcAreA;~>6B^L=JXNjGi==}>^j!Z@c^B5aqIx25Ct)n^sH(-tnDmJZxH;F$zpYb6&SNgB7HaFtnUZ)E=wp02#8@J}_$zg{#vqwFR*tpB9-kun$W~0-bjp0+9?RGI{ z6Q~KSqNooG%3Lvuu4;*gV9Ph{iZiDv8~5PPoPPNtLR0FzVSyWams5mfl7r+&VLNh* zePp;ICLz<(CihJPiq>R~1q;_UxTQK&PO39QvvvBS9n@!@#Sa@v1lE5(x!{~C+T(rQ z(kLK8hiEr@ssUG0x%kRDHDPei~1|m*3UIkcIs_^PFO88{MX@hNQAt+Ex6B2#nCH zb~mPK#E|to@^!s;p)GIUkV0UT4$;MHr9AcuiL;5XQt3iLgZl3UTOkn20Qq-P zqkHDE$4~7u3y493KQ{ZSHr<~ULthj8Op$GI+vp@udP(q%V7p*zAThcp?$Ap^V{r*@3lcu;}-w#JA@&lbd zXeMvU1thiTuW=bTup|(M+g2zKRm+aM2S7oCNm5XzYO`-@sJv1IK6N>^ASm11*>z1~ zBRUUOSE^`F!`0*K1I34`ZmAs)tap7+YrPsemzWQwX^8`$T26~9ZzugQx%-7HBhFp= z!3Phg#vf4SMXl<$2Pf;OD8=^GpFC|0Md=}~z=YK%Rq26SY7->&uI=E|XZP4Vb!rFD zIJFxlQyPR+Z7=*Y324Hx>0BMC(^AKcoT@9!y) zvM>79RB)4eky3KrTxu!QnT*{~U+<~q#X}ksl#+3J6E-A#y4QJT^PL`l*S9m~nRGpz zOo4Xcx47wxClxm=Pab!ApKLnhwYM-yBiWR~nT)N+gjdyP>5ATjuDcp%TMxXfA^i4pjnvQ-f>BhkVtl_kMWZz0iF>zCEg`Pe1ZdbMDU-# znX3}sum9a4WWXcEW5&B>Z)2-xZ*FCLi_hN4UiB&o9wDpmrq*Lh8-4Wu+fMuNb`UJ> zz{7)Q;Nhi|^#%*}ge5pjzAeRPdd!@XG~DWFS>Ph$=et`$Nm@N9*82QWV%4t)6nCyo zeK(cMURcz8IJNn`u~`2=!=iDKZ%K?*Z*Q~5#!yMcMw8?AQcI!a#?Sukt)+|Gb0d!1 z^C5#9y-eG4Q)H4`IZRtCD_s>^E8n}eKTLRTErx6lZm*3jZ?7#4ZiPsCtlM~QPFHwt zx9F9xcWp0wZjS1CY%Eu}g>C;F^<16zT(=4FT%XupS>pHHi1J*Y&XL?|3XybMuh?Em zlicbR@vPn&{V?dhJ+i&(X&E&->eA=2t>-b{(zG}e+&frS8qeghIPbZUGsrWzvDG`{ zxrA=stngUw^4y&N;IXY7+ep%PepCyA^!;+r)wXb9cr)Gz`Dppo!%6o2omVMPl*0f2sJs!uF)-U6_ z`7%?_EGoL-$9X2Et<^%w?GJ<6w_-iK>wLFYCMvf3MGB(}cx_(Q`RZ*iiFmGjOYE?J z7DBeY(CfMNNu)_JxYrwA6{xp0Sg~T#Fz7RNv~R+9bZ%5#XRb@XgKT!?<+Nw@=uCCC z-tg?u_Sz@UjTRBbt+SKGjv-{8+w*#xo6wY-bYWv;+8X*^v1Sl8r#`GPXd6OSu`-q8 zvDp${vA>Q5o;kR^^<8gTXX^@mad#cGHfPGev=ta0&*nz$5G>B=VjjNFS7lc_QYuFQuYEgysOSAL=-Mf|N#1+t9D1g8SM9VZ z^XwU#9q}Fkn-ADQ9(u>IAkp`iuyHz*`nfV&%JeSPq=6iBQNx#b>@;}7DyV!t0!AKi z7|_h_%B>%X&#j3qr_wu8Vr$Bin`%=_)2LJ=5UJ5oIK1T;*f(+W9i_#Zn%filsswDH zV~>HmQxHezLRQgse|n$}Z{$k1mR)IIq+`-^(MgUV4yloy$Nf0s3&yCl+BK@oYXf)X z5}3{Hx1d!N&El!Lr?FEoI~E;_UouD9UYosshGz#Yg9fFdXtq;4dnBInhkA-S%C~lx z!)quxr?b1SF@#V3#XIj*Rp2N`Lk&_*HQYyYvcIkXG7Z}y`2s<{ z9rQFDs7IBi1PQUm)hQZj+jp3-;P-p)L`e1_X=^F9K)$pjgFwSQbyI6xOQJCEW$O@_ z+k)C8nyK{kq93$iR`A?-z9m0Hf5*PLxkyQ;?a&&^<*446*sDAgGj@00-8Mi1r)=HR z7)}v)#>ve(Fk*X6(;Eq?Erf}xe}-r1)o>-!`^lH=k+#~iq6+Zf#4xuId{Msh;TlzT zC49nzBY4V^vgLK~ejAzcI(RYGWut+98~CHKn+H}I>rPGE1e=7_p|x;8>Mx4JEzh(N zJYkxg57gaJ`^}X;SnGD~EMwhm=Ql|&6c{U3oOsUAcKI`fa7b~#s0gW}d`!i^G^=q> zaMj;F1pigu-gPuUT?q!rKivn79RLrkfxpjS&zFF@JqCM*7ar8O(%0BsrbO_tDR7LS zP&V8%JYDq+R31pLQCfEfq#hVyR^B=W$LXWmw2I_uNuUutKnG}yJk6B`j-1Q1Gj@1| z4Q21(XD(59Cu2HX#PMmP8)=FYh1(Pq+f)^UOI)` z7uB&u5^w0Eqk%u_**q}INN8K)5r7)~CFod2g}QwEEdDhL-KSq9>q@U!US}t+J8gO+ zypA6&Z}$HCV09deG&PhnuT1#?ua6rprnu5S{u_>!xfyxhzrd^3>)CiUT;A3 z7wf88usiBET_AgrbP)nK4IR?<4}*E};EB1LsfV36SmD(pClOPqUGc=wS&PJ91sznI z8&e^_q|s!67>YZo>;A6@@FC(;_^F_x14%GAO@**1bc6?wI9kY=5 zj`e?)mD5_4Rew{4p)Rj99@!bu4^gDr=Rg7qo4an)17QWi32;zzn_iha^qb`?f z^!tF~*V`anTK$Bma$mZ5@Zx@GPF2;yvEM6j0@Mp4LxTb98aJ4S$7B_Em#Tj%;K6Vk zULK_k0*||zzY1SdT+^im0#2EE*722dSXUJZj2cBZ&adg74C3QK!-gx-Jx}oiVK7Mi zeCFB6!h^46=}5&tTvvm&sI+pD##DC!AWCPR4N?DLYy%m4MOUj!)v3U5%#}w4cU%lx zPQTfgp2Rw%x+{KK2VH;DRjU37xeYW77DHF{Ey%2^is>OI-B9yAVg`11{sbW3_T_dn zj>V{jGQn%J1`30UX`tjX_&1u=^ijj^)5C=_dmxAQ<++5VuYVVsuWHntpP^1x;p1|< zZ3dq(lENbZd%xIA^nRo!DgcIa&QYXinBgyxv;Qn|85LZwW4M{~r(w0<%F>nlQe~=^ z{K4q*Fl5Lh4Nla6I@^HH#$8vHUFZIg+7*MwKh!j!5h!@EOt3E-voa8B#!hs)l@3n7 z46lO@GeQm4KfH?Fu<(?fWl}dVF%930>o|A~=Rw74-kUiH^}eW42Z|QQz~L4$Ql9zW zz#&eGyfZ9;xwa4otcM&;B(@OOk?z~>%DVHDfcb%H?fxJO|25t65#id-Ud|;Tg?&-W zFoXYzE#QJZEQDJ~wOs}};R6$+EqZR4)vz{3&c zQF1^RNZA22fa;S<9<14zV@9$24V==SahqF=4EXUr0 zus-1Iez=Y!)&QIq9RW4dLTm~M`~$8}L@*1(A$$OX_Ak>)t9MmRDAIAIU*~jD-BF0v z#J*1nXSRR15U}A9U>v|WjyG~G{~I*^A9ouY(ux{;u%>IR^Mn6~t8;q*4I(kX1}&u-GxqgX|7S|SO~Uh+sCpTiyAmXrHPxdi!YM)_Zr#nO22G;3iE!8 zWqrjmrbwhKo*>ecSt%yc-2PccUjz*RE%*~`X6i7RM4)GA$UVhxf(F1j4-nEJ5Dsqv zrt4gutHCsa3nqO>NXsY?0U#yeigtlf&;@dXP(*m8cd8J&(`m~E(1$ZSbujETH^*NO8QEMj@ec2 z$>43kb%WQ`0@sbM51>&HUm!>NRTq5UAN~^agP;lI2PZK+gP5)uHm3iIjcL_Uh%*pS zf-+LkoJK#v;PpAwHZCMMoNt@M)#Mc2OCTNfsPUk9Wxj@Hh-hTF6Zd~g`&ZPfb0;+ zAv*(iivkL;G2~N_P`G*6U=zR@7pNI|IEaCxE-s|+w5FNf4@>5_ADu4tu`b8LZPVmf zCxn8SxuVFfYgF`?*n*IZ0GuNr0L4!K8>;1=?)wL*zETuuFwunbE6e_{9@b%hGesQ) z9mVSZhd5LW3p!-n`9t{H4go-mKk9v4o=c(P2j~j^4n+S2kTwh;b^iz;$oi8oDmeVN zz*&U{;|ttnh&hlmU*I zv$9hQa+sANWBRh-gKQ0=_z7UZi(y^BWEo4MNEiq?{)&z+J3L1sv`{`kmdIWq%FRi) zdV>W@ervkA80F2i|CK#(p5?dHy2Yi}u$3S~D*=@Z#BY#IK?Blj6ue<9?Ok%Tpxh%F znvQ?)*Y)2X;>%y@qbwcEQe0Jc08=CMPYcbcjut7Q(=O!fe?^9mMjVha|Ipz3gFm7$ zc$C_|r~sBw;MT>s`L_@MP1I2s=apIp{ax47tbLrIa*Xsw{V}Z^Mao7pHQXIVE**31 zkXRx4G%&x8i&;_RfxO-f7BgpI<{Rn?ruWA%15(EO)~a76>L$!ty)Af`kQrLm-XPDU=2(_bIv37har%>~tRnhLSzMj29(_ zcTRcX4EE1qsWw&_08`A-NfrRWPW>19==zWT&wc(Ys{i5>b8xoUPy2UN z_sJGrgfQ_BV}xY(w=tq*78Ngq2HlW%fyhrEmZX5p4XRwIfdx!+pE1q@NB?)H_&Yg7oPycRUKC8D zqzT~hM~3%yoTTYrU`3ezVDGP30lBc^bE)f%0^F2I#TCj+h%tBc_Bx7$30SU?Sg zhl}$D{z}z-8I7pGf!IP_2^;pGf+n#3U-5@yZRav_bklsuKog-F%7fSd7gBHNnVvm1BA*I2Fn&o(&YsfP~1j-4@z6iAgECRV{fs{GE=;;Qgn)oVQ6Z3@ruQ6}a z2w7F)tGuJwX+&0Sp5pqM2c?;Z0=J$s7QVeH3gxF zJj*fxp>nYwiW0G*oRGlGF0p`D5Ou*~NBbE@q*kAjeU4;M$Yad#TPDM;Xo*)YfgRc` z$pr8NzlK16)B4i%eyHSOUSl9NTy#gw+>AS z_WsQS=TMnnWEhq5?Q?OB$=`K-yWy<@3Kkl)+mQtoQvxV~z+C?*VJ;UVscEp?4V_6! zPzJbS*w)bQiW=SOU=&7GFj*}NoAs?TsMrz60W_#~(*ou32tV?GiY%kfaHx_Q8t|gQ zd5sFF#E=n=3KFE^HX&qywYoN0aAXYX+A>g~Vp#sl3Ud#BsuTiuPfva7@�ln03I? z26xm-NT8wTXebt;ETJOLAyt!=7jl@BtVs8oMt0YjYk3$qO0^Y|NpIPJ^mZPpSB!j$9f{gly=MWHc%D)b!h+b-i(@4o5l~eXRBQZ%u(cY!Y=W5-%&W3a!9MIu};7#2QagsfRX86gjDp zF|5?nA&`f*FN{$|%|4#b0k)mKkOF$p)`iHhJnU}(da%YMYQV0LIgfsksyEFH6Ffe+ z*Gmz4`EXkxeG}pDC~F$Z_{@3E^r;pJM#8>FZ8kLExj}~hRyyx5qz%8s_FI%MAVB=3 zLe)R=&!wiEu{$_zj!E6e0@f=EDNi7514Lj)Y(J%@f$1G=6ZE5vfjaO=Xy*XiQm7dp zsD&WR%7q(sebq(zGq$CG+kwIDjM#~FjyBsLjg(&&mEy+gmL1XmULbB4CvE+6acbrj zCn^Hk(~2XUnAM^pAj7$k{h(MVD#7M}w&p_&s)0ce-T}X$O^k-SAWF@jGx=YV{9Rs# zzsc*&K!tCyUQ!%!oI8r0ddmc^Vlowib z-kaA^Ne6et6?s;xq15whPgfP^^8Zz2h4L!${J2^xKA7Jsqc2KxaPsDiXml<%+e2Q7 zme&|nEbVDkJparZGTPXt%x?vco|1-T~dH^;&e;+dQgQM(e$T#POY_rbU#=qgYI`JJ4o$+4GAr8v zwEo!rw83^l(T-xZ?fQDBTzTD@zQf4DVmnc2M+TWCZu?UQXPju`^OxC)cTW8m9pi%7 ze~1YbN4s9b?9LJGhXNwsqR{m3>7*kKz?MEOD+mR;5Hq3)pn+odkb?!{1yWxW$MaO(VPii6uN`0dUMv9wGOhyqLBQQ1!rTnQ?#{wgs(LdD8< ztS||n#35AdC4wdnjT`!ZM5fhtpFgkE0LmfcM!Dt;93ivdMh%?VVJRo)bSQ5pL{QI$ z9MJ9)KR&K&S}C_9#~rGty>9zd?RQZKxBNUs-sQcNj>%e1z1v3dd2>MQq3+6FEY30~@vd z@QH?9pmxoOH0Z@H!xp~MfLhQm=bd5O1@Rk{m)+5$fq>SLcejsce2BU6Az__K+66?EJzL5=VFDG8)vUoLpJ>=B< z#1&o>6xkqcJ66&oT4_9YfWEj(Sits>#BTMn#I>I{PDc*ivUS%oD)n^I@PnV9_N9|m zY<-I2wBPjq{>4cwxHscl>&#O!o2JkwUn#DZc{*uXJm3&faCB!n2-of(yx~lW3#4D<+b>-qerPb4E9dFs8~85P;N*1BxUJH0 zhP_YfXvKHLYl(cMFD5+wQ=qez_{cDW#X`TRiFM|a@k^5@JI8cyUGI~cH?lt99V`?l zD6bVH_@X8JF)6XfvIqV2 z#rXJ`JQnI`9V_8Agnp%S7O79(CMg5GK&>z&ia1tI-e<8(tEo|rBij2mBXGIkZQ%2SzWy-;El(bnT*LQA>(dP0nY zg8#FV0wn%hjS?^)uCS}7sVw@JK7w3xMk#P%kMtunwdm)a7o?v`=0AYv*4o>fd={#; ze`P{TD+lsADEVJSGiZ?lN!uKI-$8$DFdFY#7S!^}BH)cV+*M4t-VI%*rc5BHQR*@# z+KdBwU~yMn_l2a`xPdF~X3#7VL{lnUUhqk&o4;kNsmu4hFo1T+Fg`qn zXVLshA<+Q3RFX|yq8IL@$Xp4EN%cB&*)WxbOD_F*$HQY+%|Yk2GPg})X*Xl++V2Bc zXaz_BD+w{FK4IrgQ=f35IBSVrn11P(;t)b<4OR1y)@ruVIY%zlgi`@bsYWZmj;AyX8OW*2`P zhUCB*mFO5I;D=w`NWlefb_w1droofD;KW1f+U+U4e;QA z@iAqWA^`GI3ZdYGtDhqnv%%uf_1L?pYt1?WRveVfDgL;vEHQooh+Jd}ZnP&mjJom# z#Kw1kZGBDgwIFpbM31!3;p7w_5XxcsJ^{`jJ)}f%1fJ_$nmEIBO`xo-@BJf=2w-7{ z1{MKvKAXFL0SAyrC`kVwi8-Zo*M9{^qt@dmr?_XHRnk>b|ja-UsSlJ6fZ1#o$Hx z-Zu6%?vXiar0$%vrH&KOH9NqYwsuq|;@H*k4Dy+NgBzE;%@CfQ_W>y{sQTH&nsyw2 zepV0=eTHiI1y@+(1<`L`ls` z59r-Mcj_#OfEFaDe#(jGXHNr59|8sYm|REG`NJ^LY(SX`A>iz-(cqqx7*$w?P!lz8 zv)B^Be3&!>3p`UBT}g~pUA78exFctR=8HUw*my1vKc@2n00w|Xzx_V=Z!6F#cosF5 zVhB6jz-l?dZ)A|`R6Ty4byiT}_!YRWcrg7H%nQgm=d3;G{3}=u_-<9d8RT8mS&G3K z>VXuLFJNi1!5hN;PiKU|_8|1o^B#+h6N~^Y0%OBix#jCO!GL4Nku@5nf;D2CqU<8| z)D>hLLM{dk5GMrN#z>`6t6+3gTk`}L2pJ23$Dyb~sVcloB%c4X&qrmVfCQVgw6Czf zfQfMs2YfzS zz>Bkj5T*)q+1&lc+L8k19l&hzZ@d!D*kvrH9(mzBNOc%ukRTvEh?p|Sm4H@|l7Uum z)(NnIkV(tiEZiUkEd|6(Xw(HT*s+P(3S?bJ?nQF|$KN^zt`f&7eNq1g6P&9=8m+-lyr2%D4dTMAv-aS63ii(G zK!Exmqz1_T2Y7>@{;g{qrC^XD!#v_7GH#sE2qnLS4pGS;=l$f1kXM44L}^)C?_^3H zO_N79Ps~FAgc8K3As1b6_;!mrU5k?(^&v*VjPU?k6$nw$G^8IPW3H6}lF4pMih*DT zkPTfpGS3D_54-~@0TRt@H>TARC9t*MlHbHdCP*aAO1tRWFQmglL}3ddJBVyCHV_jw zv)Bf?TbLEl8wySy7FssB{2_QA1Sf~Zk3ewpZyF!vMbU|t_Za1FSwuAf|IseXEg(Qa z(2+wDo1aJG7eg+C?615R`IuPDZbcB3Q~%8{|GD$wvJs11{QsX%vJ0;=p#_^n3!*f? zlsKKAQg>VP0}4-&CP`z)AtL`9oj?$z;EzM~5d^3!4>0Zf9L2Qn{{wNeqyT_d<-$3z zTW~$-CC9EnIG_Yx69GXGAHf9&TQlr37Vf2eYv@Bf1D@w9OEGw!D?lwc9P@y;2d6-T zEei@$aNG`7Y6;;$N}YQT7F8h;sLO8RoIMy`1O%~wV#pjt$A|s_34%Yv;61AY8Sg(3 zyAMeUTpx#KRZNIPMB?C%z7~+#pr{vx=hZ?b(u^e(pVvE-gee>1tt`yXYZbBA)PHO_lX#RnpNB+-G83uHwqzxF0EM?nbXD|neQI35&)L{Xtw5OPUyPeh7kh+iz8{pTPB+{x!( zK?+QT5dC>h$>A{c(|2hO%Xc7Bl0#UT+Iy1nHV_sARvch?3amKJrfA*Ez#>}nje((k zdI&Y5=%>Sk83M}x5Ew!}75o1vpbcJGm$;&=0WPgXOh9l2HW-2{oMwWgaY3Q|MzCbB zg9VGNK+XPH-k=F?IJR);Eons%h_(l3h_Lc2@QWHp2_=M9rvNHmP#!@WrXLJZqs?^) zQ7R!H*7_yzb(;d1|I2u`9fL~l2B?hvf(n)s1AphvDPU0WMi_|!?}KpsaVJJ0MvgR= zV)%^BG7p(Uj8cto69yryO#MC-X)A*FLGKfOm4%-<1YMgQbctp4L11DCwG!a3j`Jmg@VeK`BRm7)?BC}; z2(=$A5fc!lAoA{KqB#A37tj78d_A5;DI7}>vLDa#`QY!Hf)xi@Mq`HM#0IR0ts53p zZ^Dwcf5yu1b#pHidN$;2e;?0Y_>*11;@PtGd~-dX1!0NAc)-$IcOb}QS<=;`nd4-=0j|6F1BG| z{Eu7|^Fg;VpP;f2!Y7JV{=%B0@&hBvkw0N5m){`t980fPhziIdz)WL=zp&-|2g1RZ z{4j_@7B|m`x~BSOM9%+y_-G#G{ex+(35uAXIxG5+9D+r3|?b zX2*XPfWXuYH9&^k{bGZ1YOZ4idAs@v~e|GM{lhiPcz#uR^aJ;jIG4MzE+{_=mK z9Fzg)`k_jXocOvqha3$u_zf-+rxIZFZM`tGK(1;>3|Z)pOxd>&3&_EWD3`NF^W*AQ zzw<)4bQ5f5qlq{;Ce7>(Ch?t`0ud6QBtN_tOOc_%^A9Ewm8RCqDreifA#uY@syNhn zkQ@3%xBD-;D_>;}LlT6v!3tB2{!#JVT1YuJkPf9c2znsNhq6^fB~vNR^Z%^O0zab| z)ZcA{`v08^#p<(&CP75g%Okkz3Kv%7T90lqJM+a6Rga)7ILzG zL=31YoHhJ3GTrD95b5mLEDFaVBSINB6yl(syU$J*ijTt^2#3Jsr%B(C5v?0^T%@=$ z$j!0-lMnr8=l{!x;=JQ}z=guw+8UIuRn0I%$xyljD|MpMuJGYM2f2DdGa20T>siId zAa_Nbju#hm8Nf*p%A6rSu*Z~ie4~=K|4B*z#TRV^yFcB(#4Vt>P~k_0AC3cmv8AYQ zXYol#RqJ=T^Iw$Wa3?_jR%5|A(S&0TjbfpHU;M;np(wZhWAgm7^n(zC>a>6rl#;d$ z!<5DmTx|#{{2)biFnG2-CE-`K8BT(JmOeLBoBvSx%#)A!%UT7_7yVmN3v)$S<=Gwp zZ4L)PsE7dvL8wC89QqUK=GBl9A;9Abq!)8vEPjL<6J$iF@R&Yt86KFvZ@3Z*a-now zzJ3U#c5i{p_CBuO{4;lkdWJmZKa}I(rT?MI0t9|UM=yzx_diO1@|X>)lY?DB^%ZHX zbUTLCfPNhYp$aZUElPH$G4(Y^^?f)~Y5Z0HgnS6+L4Q{!|8Gda?73mwu8zaXDC)9= zvmhv-2(X|Uk%lxsk{6`>Sg42|S7YVnYxgtJV_XV~9bo=E#;nJVUw}DgDD@9xSsdU5 z8aD^2!r_2(*kZkg_%p!$r&2R)10Svs`ukG}(3}5IAqQ29!Ya{NN{pP)uiB8E)vvl0 zsn-M3P*-0w7n(% z^E3rzLg=vj_B%K}^N+#RKO>-$`(F;;{x=Z+7kh#`FGO_=FF#d)7-T;@^8eU5uFqcj zRM1k$fWQZ#5QR-V=$7buYy}}oB=&j7?=m2qRzet_j>T3c8~nWa@OtJ3))Qy z>S~MR%E&;+%0#f_+C-1?C*KRKMu&Gq)J4R%gqizYTkNBwZy#||==nBU&GP&$^%L=& z;|kgLtqz~?|I~1y!QwDIdHoEBm_$xyFJ0Z<&QDy>GAXGxlFrz0YwbL|wx> zAkfj_0Y#OW)~>jn4d00QO6yk3txMr=$P~3*_lAE(aLbRTXRNTTjVdOi8^1S32fo`c z!_S(wR~o-*_> zW%TJunyf8Hu65-*P4^vle6G?inIlr-C(iCB+D_&$POq8UT}HPXKB*5Mj)UI!;+l!` zBuP$_IwXYV?q&m7BXDCDm)B;O*eo&kbXWS0^nCl$1*s$LpTc3Xcu!dso0*?O!IHa5 z?E>%gDIl|2tfy>*&1Pw_w59s6Q+ICDtma5s;&QRqn_)Zao4%1Y{8qV!*$F2kC$EbY z#>|tD%rAIa_k7GYf3iC1@@AW@-4WkDe?+kLnQ08|$1824!bhH^8|UeE8E16giVjTg(G8)efA%81 z>wcJbL%VN4sGY-yy&X01@x{V}gghUWR^7B^E4nt_v{krXPWw$r)10Q6u+`R12>wwK z%Yx{5Ig#ECvjNU-&hLz^wqZ7Ed4$#xt!`=R{(2eHCf#jQw=yR8w&j<(8|@zx%PKcw z8OzDGnRhr~l5=8pYDGG2uXRM?{N#Q$Z&BsOrG*FarGCLWLqWAtA)8tm$L~lB97(ik z_j9LuAGq!MF8y6)==vLc%>6zpeBX&$8ds|KqlAX#%6Qs%W_s^}T&a4^v1^?o ztG3%@M5`{D{)4hFm&x>RtF+3(JzIKZk-`x_6Ltdr57{2>ZL?3U08JP zh;lwY$;Cdgw*132^DgN_cIo_KlLb63V!8BcPbS%vo4q-ezPqFk+u45L7#Om%{cVIP zXSnjsE%`x)s^1dhZC!Y_WPG!gqyt(HBU)90Vx>Z)&_6|~hK86aFcG6QcQTdA&3-BX zJ*oOE|D>8hxo#PLeoM+IXRc!KGvNc;daiR4@W1CUVS@XB7L6VwJev|C7HvFR!nicy zFsX=$V9px-2QdSBUa3^QL-S4DFd%!ZgU*2eL;dWSxga9h9@ZQ`E!g9MhS>R`X^HdD zm=D$&5Bdl~R#%+{xtW(tJBcCk&$#fo2y?-;$V}Pu6GIh)Z-mfx{@Wbc&OqwE6$BgKHSe4nbn{1|GXgpolxr)2w)dhlIs4H36}l6sbABAi zDb2pVCz|9`h*Ue5CUlM;gzImnb+cS(pvJhrK7T`BUy5aAyh=x!!}PGE^I8x5WxzGC zV{5Ca9cv5p)GV|maWrcAa;khbNpv@sL|D?FYKGUZUZ}VlthMrS@|N7?P8GkK-$hvX zb&~S>7H`RI3-BXJ((IkPP~ksnH%l>}be3*vXS-xq-|%zqYI>!o_V*LDo?EfszSCP) zq~0x8)7PV&*b~8&PvV}rt!!U8>a56Kn&e6E42fN{36WR#T$Z5g-@8w1#m_ync#r)A zdeqC>lPY!Z#FkyInZ1ELoZD~i@YBA$tA8(D>Ncqmc})VfyFBR=GbY6bE>Grn<89M6 z>Ei5u(fqcsrC2`o+C6KJVEuZ+aaPU8icF?;D_*sDdm<8rBxugtSWkX3qP8_Pa7p~$ z{Isxsw$#9dP`QyNlIU`JaOGFd=`wAuMO3}Gzo(|&tXdWWZGCCz(^}>Iz z&A$F)s?YMF=f&MGW$x%nYC;cB<0BQlju~yT{<9VDx|pk07~e@B*vag|Ly_S>y`vCh z&^teg8quh+FovhiWu_O+zWt`BPA`RP=KDYuNw|*|-(INRSwYQ9aI-vm)5${hDaw~D z+KH19g>RXwSp1q_Ggoa(a^*Y~A|Fwhw_tR+i+86S4ARZ%0fu9E_$0z)+g}f`NQ7Oq zSrqR^eM=OPz&_%!o=|!Ggo9whMDH6I-8-7qjusfQgU0ZrMRT^^Dq_#~Dq_!yr&OCO zu=Oszd*(}rl*%5SU_b!kG#cZ3FF&`|!|Hn}x=Tn@abiV_hnv;qG6kQwW};oce@Ay$ z|7H~nC#jGf>ON>=$;$R=psw~dhAI|cnn3ixbNMb$uyEd@RLG4CHhb9PVsoDm*y+08 zn6 z$>x(&R&DtUM!Fbt5AVgyGfB{#YTxgNjn*)A5KNh{%!N6#i!8@fI~>FKd;!e9IYc*K zJy&s-bZDUCi3v^rg@8#i{B2nfRMO~v{rOzK!Am@bqjCYP_O^25$^p-R!@r5_Ma2Q@%!{`Z6x> z3MMam&*xEjhr`sKEI5G8YY#mum)O^P%v|_M#et{PmFVhkoKh#&^i1{7cp{cdGpXHO zsM8u$+}As#=S4zJ0k&^}hu78}{g`(R&3mA}F)fFXCR-uSrP=+SmE|oZUiG!T8tB@| z2}nWX?0SldTc#|Ij49?Uf{mWZ5FRVPkZ0FUBF?$3j9)q_VqThLhj|IEi(}gfmG5X& z>@F3nXPV;U?h%jf_I!^}sil!*S1z;Hq#bu3VK)zJ$ZxUT+$XL7wsVk`-eewbCM@5D zy(p5H)Z8sJW0oCh!+W2`E~8C3hl(Y4615K#-=lIz159?O&!LVZa#_7~TC?P)Wl_p;KjT7)kAMxQ!o^qGE9Aq3|N%>r_gzuF58M!2$Q|@hAiQH}N zh7X9hBpT`C4e=C-8+b@W5HtA_v@y`)sW;p2{G?sXJ4^Gt=r#w{GX`4OeMRl)LHWSe zA)fAqG`1n0DashuA)bl*8Ye$#_wt4ZJTLm7^W^dKqW2s_KFqYTbUp8xY4vkrgyC%u zV}v=lMHo%Vx!Yf|Dx0fmTFTJ%d}gM_)AT$CgxPxky4slHSDHk!i!drThEdQyj2ZUv z@=Y7<%i59!PZY8(6J zr4dICOBgG4#B;|=c0~b}mDQ6pK8IPZTre?Y{1hx|)Ss7E zz@$(QF0y9L4-409Ql3`icpqmOPME2Xc~0}@i5i6@FH@R_;n6_$P!B_vKq4xyi;lTp zWZ%?8cw9(#c&8el%67HBt;MMJ%AAam9b>zlW{%QDM<+(`cUciTG-)4b*52f*6k3}Y zja*$Z$X^&Mb6&f4=FI`qff@rwR>Kb8C$%!(CLKy)$}%jy4x)w<Q115E^(D5fbb=PG$p5bik zdgR9@>%pO>ORwy`TfKabgD~$g^;4scV!UU~#Fi%~i?^Gz5G+g(ex zWpJPL-W{>|U53KDiQR%_darx;1e2k$HLUP}Zc7^j?=el!JJ<{{Vwe&0o`LHei)kbL zKbNoX0U#>Rr}y+_wsw?B;kEkbj&iLxqXBTQrO0ETZ zdhC6x=zY3)*&vKE>ZH)sk}m%@v6R@b1YOjnWm=^A;EiD;XNP&G4&$e`7W3EeLq6HS z@=D&@h`TY>OsDaalcciP@@K{n#;=di3#{LME#5A}l4VTJ#A-))q@{TKaTd{L7FqVI z#8L1vnW9guvJ3~c^};Cc#KQM2S6vU9+?(+veLW+a5EL6Owp&^MnX>ok7cHv`AK7GM zyZ6dPQr@{<-)7@5NX~SAC`+)iK`GMv>XpUf4-{V-XwPjYV|pKWbDPxFtr?Fw7GR%* z_2pcOx104*9)(TuZT`Z*yBYR+t0!r<&0c|%-jrkIRd~?IxE8=H`yo11k&LNx&yFW> z-%}`Y%KJgtZJtx!w^jCLox18OhG%{1>ed{xc@|I7sok+9K177dweqb^PbUV?`WJ7v zwG_l253<5z$}Mc@3V(Bzbb@#rp1F<-3U<(p#xNnlg_N9R{)Df`#7os;S?#7sN9FD? zzO-ox+7;sax4+pL+0OSF1FQ14qw$%jchcdOx$k>BZ4N zr&q=)_XsbP)68*fdu`b=U%vO?g{lKsLud{f=iL*BLyV%%vJ6-#q@B)>oipj+x^&<- z6~V7}12>zq*sLzdZOZT$*?y6**j($-r2~{>2ikVU@AlA{*cZS1HpjpoG)y410F5e9 zxJcMrSH;S)ZOZUYK;UK@mQRY%!}~one)oHhhgvCT8I;FmTdzk=P@Y1AW>QKL_F^)+ zekZ5J?)GEbAapK8a~i{HjkEJw zgZNddZ{9%SS@f`aag_8X9VVVJ5RXg=UOLCN`yrbRRS0+U#+sn(OStaZBifP(Vk>>2 zb>MVKi3NyCF8?!c^SP(@cMKtknJi%quMwvgS6*|k)n35UPKi1CW&amlpv1QVU#h@`Mrlc!c*rPL&(HV^;5g?Q6p>z5816g;+2RVG1^QNtZfx3;AumEdvN@dwC5Wqb?bza#w>Xh@LOR*4W=?Mo%Dm zwkN!o?ioCGU^ktH9A8(cm`E5*wDB6#QoncECaXPz@ZH218r7RXxU>7?U=TNk7ZZ z*e{j^aw=)H=GaCd-rQ;O;3!o$%~_Bc)3R9qGB^H z0lbdCz`uDm=Ja{>lj$7yo_#Wo{Cr^d9*Smw=xar59rPqC@l^P+Gv%2I-*$7ART>Wi zkdX4&5?>^I{{?Ua0trF-_C8R?90_E|4M_@M1d7)Be#b>ZH+w$Nw>!eesm;TWbz%B8 zaetc<(zL%4QQ|p|jKk$Wf_Z^S@^L|enEdf;~Y@M_5|LgYq((pw!Ls?cZeQg;ZE>4 z7w$ruZP$(}2D)n~`y@vw=o-Z#6NZutRz z6QP;5@Ze_GkgX@lkpSyK^g|KhKZcD(azBBldB9Jqe^+e*dL{SUPDQkxhSE#A2L)6oT8dG{g0{N^3IrzTIspZu zWmjg~tafy@oApX(vLMU@ z>E>ZdeA#`$vC!8k_e=ma#7%jKcV?vAq4(ax?BQ^MZ}t+3ubT_f80f zdT$Z6tu^7J!k48}xlt|*0cp$0(mM~?(rM_E4vJ+(>F%SnrGQLZwyWHytSs(>+H6j7 zSsbBq{t1Vi*Atd&uuQKpq!u<8kjD6_?1@OBnw&g1Wd(vo19B%PUk}e&bY!tuqC!wZ z4yw1z=FpSyd-PBrMpBgjj7lu1(DECUxzIf-w3v5Cpi0XwHvT=4)$CAc;l3v5lGDu^ z0Hu^HtUeU?j=2hIMLtj~QZ^Bi2$Rw@$M8`2dI6{fJk;v%MmZ8{%zM;=-~qb^{8)_u zR?MZJ1kn@g@pq?EAv~keP*?~19U1B6~g2A~)9A!?tb8~#I4 z2$J9)I}P}Ie!qlk-tAO~>R1fEH$bkxB_6`6S*&uXB-agvITzuvRiv0Ds8ndqcW$!T zyRbfhEV7#6@5*lV*`$%xX6*qqa)mpm_99$Dg%fI0Inp^$N^4hy>!$W+P@w>!qO?33 zd<(*JdyW2~?`U{Dc}M(|G6XJ$lBpc;1znDLe6ne3gDubzY()0APzic0Wwv<=g{MHkjfU_fGdZ ziA{+y^ErE{XjTu{-&JWn&G*h-?B0d%4$4zAIla^3<@MA4we!rNWD*_ZIU4iak>kh) zpliFILmr!rdkPgAY?+I@!X~z)<&3nEiBK_(Wu){gmU?V~GAoa4tGS-fV{cM?U-{3v z-V2`ga*tSbitj`ZzyQ(2kyRGdh*W?EnB5BZE&efv67<9;0k{IPt+pkKvO2{DP=Tdp z8Uy)07`&R%NNCnBmCte^tR%I~4vnmRzFziSH=jrm-f%=J7U zS25s$jCJwe^xPj%L{HlCu|}i@r#Th%;Z7q6W z;>1M|GMaCX=7W&S$nL~2cd(U`J%r*Iy}S&VUNj6$uSAdsri(o(oRtlnQ2|5lz8p8& zbr++0UJnqxq~DAryz&uASV3|tk}y0038N<<;l(absLPknNO$VKoC;;_#mw0QI0X?{ z*#*YeaB;tldcyH~l5jXnKwX(_4lfCyE)QJ==Pm`-3hakuMxx#oVMBo&(aW#+zU9^$ zT5HH|8eLaD7(Bjh%^vm;4TcE)$Mr~%9+c$E2L;Lk51aCd5FwLQcMWL&tZPYarv8jI zx?ZXE+%o~Ha6E8sd?MB6V0}?zMHe;VKvJUB3@}TlwHy^7#=&@_Td8JEfcIs)*#xS4K>OY`uG)*RjU=)1O^yUWElXYPGns6l8re6?hPYS2#74Tw~$6p{?!BnbH5^Q;IUCXSJI7@DsGn&D6hgvd5_;0$163!ExpEGz}1h7%uZ zT+ly=Ui<7jdMyTofjvT006%0PEFU8SAscfCOd5=$Ydeh?$u+|0$-(FW?}6Ynj;R&6 zjcZqPG~y^7(IiNXtv9j2sZ#5<-4Oh4P`T0nuK~RDI{+hvGX?;IA{2p%5D+oGkW)abPehUfuAEGSY9@h=K`0Kp367=qFMBb$a03}yo%_@_-n2u4@}1mkQP z3VK?=1mkFIH?;iPU~)zl%mtWWAj?e8liEk==00-xQ|AIjdunNXlv=*F<-;)1v2tZJ zj*UMM=|Y_YQjBiaR8X?$uqv<(012RU@_x$)Xg~xk8MX-`;ASv$0AZv{11yr*2`LJ5 z0K(wr5QI?xl#3~$6DT1DVKf-o1{{zNRW_Jx&J&sP^L@{&uDT7~DZMw}M{y^~cR6~C znP3a4&cUfK51lPneR>94G-Ahp)%*@Wb!o`mug1;d zNui$T@jm|s`&Bht%G`$U$^a5pK{_%#y6<7H8)Byi(k`Lf(3ETZc0iRkiuF~>s6+zkbq z+f6pfLbL4f6@kZ9W_kpaKCG}W59|`O0li6CO3TYbH>>Lm zQ`+WE2O8Nrc*q8fj_$?L3pfILgJTiBfik+lgZWGj49#uR@B(_lz)&mdXGrs2aF~ml zn0a1oKu-u^G>zh1jlD$xP&@!14EV6}qc0nvQ9cSUgaPa{iv3V8ncRAQs<*T!m83Vp zPP03enzKgs&W23fb7 zNbn;oo>He8T{RlR2+MVW$42QZA{fn>oq>%2JA;HCfOa4O91pw`IUZ>U z^{QYcaPLL}p_rLL4Ik8)F#(!BMAKA&G*CSKlUOXC{z)@tX51*ppIz{l&;^_!OWw?k zEV&*zX2}uN*~QJ?I!BDo1!YcvOnV+xpRCmv9%}}dh#(E*0;F*e0aJq@4W@=w6q%a; z0_lxn{sW{@CWvxCsrLQYXA7*j&lY|lS_4_mAH@s2W%eq1dyh+72oxx}ay zt(xeP{H2C|LW2sv!T>b*_8@5JXZW-GAh;lCSiQ}52KwPs7ublM*6c4Ukmj(lz><)* z2tFPOhqN#i5CC}#jiNr}EeH+)EhxW7<{Z>;-JDnL!p!;a<%~1uKd^zBa{v`G=NLAS zId?a%;G>`eZF}OIR&2n3I}RK<;5QLb>;~lwB>xaZF%_f-U|8zLR*y%Qx0^#|h;SbI z3K``4{SftZ?!lQ1M0l00dV*ebxi`ers9}+f;-&jIWN2SN;na`VDF=(4#&HQN3ZZ@) z?|CM00qs#Jgn|!7OmSBUeqGg{Le|evZ1$cp!h;wJV6juB0U3)ja|8=fqkKpM5aolm zz{ZK}B?@FtR*)SrLt!-_6T)f`)gY2?Da2A>86fBb=|Y1EHF9aN=a~N0R|fsmzpXja zkPX266*}}BvhUFSxNlvMp@HO3Ll7m0*3Bq6{Auj?6I(3aD5~}sA{3kxWD*bS$}m7G zcynJP&cq=VtTI=4gkS)PAK-E2^9$2G*+^8iED(qG9GM#fYC=j4PBR-cscn&OI_b`K=|0 z+j^pGZ4&z|$7ybOe08yr`%z>=kV01+lUn{+wZb)({A9<3Ai>Em_#;}cIGbOq*gHEU zT$GnkJiGEN*C8RuYU+!xR=(VlE|aG5+~|#W6xX=N=f1DLyX@9VqP9Hs?y7jK9|=j( z(k8hJdc1}wyvv?!DyOrj??5xH}l?z?^Ogx$SVKPP8S*pZd zqi1IveRV2WslOOxq?qsW`F3+3Q=Q@Yju&l2ezLPie9d(PUe}trE8LWjb{Bh`XLLSt z+K@Q)jN^DtOfoOcmjN+-?vyi*r5t#isPQ!e4WJCti(Y)fmY!PNEIzfnp`YLKJroLi z`7N7REt-*-(Pw_yu5%Q$1*$pb3Wc z=!#N|EMH8|q@XEKfQoF}GX4?$FhL#4$;uCo+LlKmCQUAX8J32*sF-f$hl=S|B*+V) zaZo;KQ%kD6)!b1nUc0%b(BwLmNNu5sVWox5UaNERVZuAIU+J+QL04wDMsZqM?6s0s zGb%nb^77*R_epISRmn0aF!B-v(o+P#Ar}p(tSUk!@OO@lM*Pm4nDFbytj$ zwb+&#v)YG)W$vlF%J%8=JlU4vU$_$WodnfK6QE*FrP?pR~Rdb9k)fVfLl7}u3$4&#@_H&nlUA~v09XVnzj z;tLf|8@zqCLX+K8W~gNJeWx&}|Dh&oCJ1D#aJiq(RbB`B@1zq)xgv@ zsmU~7$^IyeD9)%}U_Iy|c16Mnn=6;C0=aR2a=MTfc9}~mMq8-F?sWdtA&%mbZwf^w z_w}CTj~yB|ULs44q*9BZ^^oP(8dZ2gmu|z#Xoggrn{~oQJUNlq~bFvNX6?aaVidd`qL?Om6(b{15_LaAQcY| zXGR9#~+-=pS>MDbjO09G+DhKm$tbDeb;} z*WH^kRH%W9C;sK~kK49DT~&7`r6(lS6OsPh=oygS1BkY3hPtY3ba2w>wO2qoE*%P{ zE&Fm@(5~55T|hTF7=4(F8kPaggL(tJ_78nD1yTG84HUnsmUBS-3Jl!4fc6M|S_Y)U zcV?(DP8z8PA4R|ig(-Vc-M{j(jh$q7YJ4v(QZ>~b2+J@U)Suj3H4!Ny6kqf}5%r7t zN$((+k7O(BBg-uc#Z@5iOU+HQ9tP^%5vcrOqNXK7q6QBNoFETM{tzon$zUE7gbN-N zXhR+pHJDHXJg5RZ2WM1SgFL7VGb*H!lVTne=7R@?=g5P4lxE+6g8{@Uc+hiOKY#~S zCPkl%uC6`Lt*kjDYVqRx!P(ykEQL?`$=9Ae`>O7A!#0iAWCB&2uQMDyz3CR!;ONo_ z`Q(wt?skDxZxJTqnt>J<6MbgK=KSR)Q^v*Fh1?Ej@;CI#*UnB@cWF7=sdAUKcqiq1 zvIj5O_O0~SSK1wsjCPtG(#veMkXiYL+RkSdzr+OhSyWam!EdD6;pI4G?#jNPTT}2m z+{bwrPa24MH!hKTs8KD*2Y%xlnS}DzSf8nn z$LI}a!|Yk&#Q9m3t;=@d!;UL^Hs!aM&*rV(zpXWn`_&k~vWmuQ`E@L<2$8PXabv?r z?3N4q*9uU_a{G9`=U`*cywjRu@wCl-!;$gA{Kc86a{`*0c#bPTQES$e(=^79ERG+^ zXE^?*p3PtkIP{_Z+JwbaQ9fMBi0rE8(fV;~lB$plAW zG_yU$$N@1-S2;G{Au}`loo}WMma(%2N?Ug3(^W1^!f&=FlO~Rlfk}*!sty?acKDXl zI5z7cJtjoh3wW7fBhByFDx2JWrlI4WGwe$4z68#WIsJvchQRqSB>iRYADcrui!rl& zd`KlyGpLf2V;p{+OupnNN#r_cjK%zZx5`Xbfw`})Dc>9Gl0hz|lLN--uL(sa!0 zFU&enEn@rVKf2To`!qQPb2Y}9HS@-+Cx$1*upeG$7+N*FvRc|dA6edsAr`N3Y0=nC zYrJ4K&nZm1Q4}-^Er%)Aq_8Gq#bWjZ$UHu{$GEhyL*`NCN)86V+$WJ|0GTQ3CPy#; z=03@1N5Hel)WZOngaaVxnaFnxh>u=j(q%e}fTLlm%9_74lcI^^?k~A=CO6j{ANZwM z?`~viCEd(T(x_jtV{2UcdbllVzSmAG`(?{4asT))b5NHw-;L{T=cH=Hafz{0msB>s zk|oqr=NIfzy2^$0K3}G%Pw!)6$L-Y01deeb3~C0SF@C3+zNuhsZaKuKW{pZa^e=yX z?Tp!(f#F{4arnrqyfwThT7D%2~(?01~|3aG=NzPqKs*q*|L`^!>QE^)@w7hYs*?| zWsz$wZmN6N<~14hgCc@!-S+DTF?AZKJm9Hpshs~r&tFMBtJ*M{gIFB2u71CZ!vsZ^JwViq)5p`?3hkS;#LNQ#W*y_wmSz7m-jC(^@ z@kgp)Gk5kb12mSgCc|}y<%EA=L7Jhntd|uq(NpnK~q3l&B7~ll|moYIgMozW2M=`SIg@?JFuncX42>Wj#j*$gk0!Verq3}Eke zXSY59)Y|=40y9{<5lcRvhCrW3$LDpDW6v=m_Mf@VXSj+MkIrVe(gr&Ric)2jFtR)zO?N>S z_0Y(gi>RSHVxN4VtLD(Gu1X=r2l=8zAkucpUfnrvewZxpXr5$q;Ka7>xpo)WuULN4 z@q$EG?gs|n;kqk5hJI^JxAp-d5XHx*$=~pAr+QE-xH>G%Q?NMP0qnqrQhf0G2uoU$ z;bwlM9i)Txu5vC3jGM}39P`Q1OXld>VTqr0f~7U8_E`-~89yOvGH`+QK-qY~OIL31 z@hL>Di!T`-VkL$CD=?;NC793EwWG)Wks^V${wrz3$ksND5zGEg@g^8DX_vy$Ma=m^ z8))!o?yog))_|L93xj)MnUCe8j;r4+KqMZXHY8R+tyrZsi+WV4JU(7FaIHy=`jnpX zAMDyO^P8OvG&GMU#R4A_VMr2M3>EM;Sl=;?WG5Oba@tRBVY9hC5|8NT%v_9hkAWFF zBl(49-`be_PF%JKBo>8V1I-Lhde^Gt34%80NxlV09TE*wBoNYXjvj#x)N}_Jg#?HX zzw$XjP_*yKSB>4gl3xOh39Mmr5Q4U*=s%?t**;bz;zZ`rVow4{ds-?ZFx+;Ep>_&} zd9>VUxEBB{AP5o;m$h71DJ1%)P;7wcVK{fu^7xp(6bb! zK|s+z5MKS#RTOpu9>Hz^9K7iRfDJ0XIq{}I&>XuI&MsoxYfV9BE+VOI0|V^N7mzY9 zE)(}9$$=thrc3_j8C4+T{k^YTE6~hctTG+b>6<)lfo3j2pORtvnM=w`m0nw~GAb=i z--n-epo?NKhVd;tBYFx7>K<{lHrbVC=13DiyvRKxx3)uI46WXNVhdVEBDC@fhXGx} zd{s}7XXU}HT+W0a}m^~T}xdVTDr5JAqaZxQUdvj zF3S~Ntg-ZHR_ax74{?*J*&WMdFsD#sIk;_iWhfq}YF;2eZTB46k^24IL~2Lsqwr53 zG7u@2VYYY)s(lhBEhu_E8QRsy^^l)v(TLuT)j&kXM*;dXa%E}+49)D()7qiwTjLr8 z;|bLigYxAqNQ&eQiLNyK>;X|vGI{do0Z2LxzXWexIt+@oqh6KE{*VFJPAZy;P^8pg zI;W8M@D9(Zzj=~*R6%-2Ulr&xupCt*Q*6<{**3@&*&{?4#l7OFBnxg8p8r zQD*) zB<%!guhrAJL}2fiGAmnunGPd*nONmK%-@wxQ5m|U7suim5O@9l^Hnmx7dH_j620MX zh^I45XowL5IjoS8d#UF9;EBOyV&3hmZ(|~}RRpyb78d%J^=c?2B&;&^XlC#E`Kg;s z9J{p;Y47BGM3Td?t4AkT^aKAAQ{6cCiJi?Ie$XvfW2g&?aOAAf2BKh;;3#DacNhh2f1fuhs$_8?1(+z0Wh9uXPBwH6=b{)0}u9sY{URzCC zTUK71a$YHOUNv=IjdWfOUYqS$>uVg@GG`uGb5Tz#IVnzIj&UidDX}1O6{4%T^y*^| z#kHO3mq$C+f+bhd_}NohZ4^Jycwty_H70+}_;6CP!}1*Ua#6?fK>p$zONy>d%Op*G6`v?G3ER)5_Fz5jf~0u6^UowL|j`pP!v= zmOY<#`8;){SURlI;u){$C8vi!!Y=!wU~2=rY+ujZ^{@v@(Vs3wCcwNt0tyu~J<&6& z>IFV52fa8l-n8B{-=<=!tq8jJq86WU3{5mamc=`jjL^7l=pFdG&mrNzxmF zA(%efEC(w^n8B1eW^B5Qp7}PRAbYOFP6~VVm$T{L2u((|AWG?EF2bm^f{Q9s4Z)FMx_`FaLhrb6&pO2%E9N->PkhZocf z0%?=K>}kESIWFt&d{2Ym{_sevN3+cp%AyrhuE0S`cQT{6XmV&; z9p;LA#&;7+s4mYZN_D-Bp7<>>!pgU$$<@;7>zM`v4#`pWO8B3wIx8Y@d|)eC)RydV zND+Rf;j;WT8kn*9N;=1oav$DAl$LduM&bNLm5DObKr8Z3K)GsU0vfdg80Z5WvVDF(_7e>9Wk`pj%kE&)k^+SU!yC@VG{>Z zjev%4GKP^f(q9+>n4uTlPDt#;Bfs=z&$8HMfNIxUCO|w~4=HEBHniXDw;2Uqg6w`*7+7xK@%UTBQ&GFW1?^Qe^br1t_YHWqvJZ9bq^a z7j9I+SNlatP+OTX?rXW#o8^+vhhv{uZKZqPSFc1L_JHe&mFsNtLAdr2sE9VZf(`in zDlnz(!(nz9;Q0QcvS@&~M?4HjCQ559f5~%3bxc>Z)Oh3mayNauW{?uO$KjD^^CNNX zI1ohU0GiLy*|@_<2B)woESMxL zKoV(Q;;IF;79c4Sj8goE7tp&3f<&8lvmP7{HyZd$c9!F~#qwS7uy8FuL>*T>)U%EZAs9T=pzaQdp)Ch~2J?319k89Fbg{ zIPni0`TaFV(f`ho)q6}V$=||_{5EPK_zScjV5VU|Hpn)B7ROva##{iBN`ZuGL8P-3 zTiE+884h10j!+ z&QYWIFOz_Y0ND@nAX}Vz0RGd;iJR;|v|BIc28@3BvknD~(=~z|Sq_4nf9N6c20a`( zr5h;PEMZi1KVfBCgb^XK*#n1_oOqS5J$Vr;Fh{U-==h720iWiqz-JI6OxT|W>uK+v zOQ9@EKR2I*&s&#%_B{14$t$oyXmErT>(cicGVo*uDMWMAqbXppJINQI|LdsL07b|5 zl6e@vbl33-~Z``cS=3v;n}Z2ek6JZAeDNPlL5uKjR- zwmIN~$c-~cRK2rs;R9_3WgwJV6i*Js(CRs+p_j2KFT@{z@kXXk)k_Y1F2r~N;!Os; z@cxR3WOy6w4+LZ+jKi@Y9ip^7(Vqa8d!nc6d^d(eO)hNUmwafk8Jo}y8sw;c#AsD0 zMoS}yxJk{9=e#R%G2!KZ4DH(0vNz~h z=tu#!2r+no6F=-`srWD2u&@k#6tHCV9z}-)@em!RL14G;bQRL3QDpb#S5O9f1Cbq# zJQmr64q#D^?jDti@~fY35gm-7z4N`o8r=U|!HuiT_5ML5Y+l;!n78dCAA)G{M|c26 z9l`*?CJ0@@=V}yaC3O65l}>CGcxM9yz|>3wk2> zN7;hQMd9uiEZn_v0m9u;v^@)1u_(?=@>2RA?hm|tfmmI9wAEy#_W-Y2Zd3Wo5%RPq z@R9!RupHsAa>m51GukTz6nR`C&9ejpFd@juhJ;Xy;$BJYIQ8y4u7e{4_6^7oB#%q= z4L{o~c_*@@zGhUI+H8wSBR5y-^CJ6+C2nHGP1Ahmyw*O%T|r(VJ=~bgckB(4TwS4e zdOkY7)_QnBJjH1#*Liioc}2*1MR{#l%egw|iav9?tF38IV!?Q5J%2yJ^in7NY7za~ zSlKvX=)7Ae^>T~zyvkaK-yyG1{Hk#m=Xv_om5#L!i6Pp=?Jn>Ka#p+M;|W8ZmUzYs zwAS+1`jVV=&s7P8`0Xh%{qil4cT0b&$7<>FhX~56WH5d}d2Oy^E#mNgfraBRVP#S| zzwg>!LYLDvB&+>?&WmM|`||JKTcxpN>R$+Mcd;}ed$sPF z`NxxS!Wa5;hjQ`qRr<6_Uh(!`56fJP6FywpGc&Kga^cCr`cM~%o3kqB*LU9hXpp#i zP$`hA_ghHjVx{n54`EtCXf5W`+VCvRkL2aMO zD{ZXtd@ioE=7rSKg}%bja~k!q-83sDxUmymMMKLB*sO`>2yT4b!hY(Ki{{Z&g$FHn zwVv=##yk9!S(}I&I;e|f_CfdO1w~F6aU1 zk_O{KB|MkZ8_Dosj0fJ^6P2ZSNspNBBPm-vuX2d6cHlK;BYCn0hVHj41UGi2e)M;} z5KG=Fjx8L)952Oeqzq44pRp0dee4wS0TLMLWc`zq=*P93a)V6aC0Z5f1aVbl@hz^0 zOI>b-C07WL2{x0_NEfoIUDXjg4-XX#i2XU=@K#Bsuw2xCklY;9#Lz9!LPk@3NLTIZ zJ*8t@a4%D+uXYufF_JghMgGv(lWm6ChjN6x;VJQMwh{AIy6krj!O1lXXY=GQ=>%PA zV~OXdi_dmF49iUhHBpfXwv&khZ}3p5p8@WR_7$sL)s8UojD)vAq47yFaqex{k3oiv z0M0kkMC?kJ-0@H^FKKyD)6-H@xwK;9hsotZ3xnxpCY&Hb-n>VqIoxd5yS2t^}6ySyHXz*c@llwYMsJj<>9LMW9g>- ztc)NSfdxf)<^JydEoAiR1Mn^rPoK#b4%r|Zf`-wcCfK#npi8*rbWS$u${TsRf3uR0 z|KCZG5u})2jCTvbvoxBiR=GuWAXBYki>d)Y1TV#xhc5n3Jb$QEA*-pU&wlNI^x{&R zSiF=PT@a(KywT2^4{}%{s=PQSIUMDU0F!~Yw%Toaky7JId{@+7-stoeIX&7tc&85z zZ!eb7HTB$cefi}G=NmxHT+l*}xb+QNhG8*ldAKS7b4MFUOI;2>xOGL-$WyBs&@sjn z8qehIejOh7cWz<<5^M`~;pNl>VYw`V9>Ue_7)W4UkmVe}hD;D%x(oNz^Fuos2M7}u za`A9Cl3!k^Bbgn3C-&f$X7&rXvZijm!7#kNr-1r{NC2Qbg+DgLBhnff9^7)xEYd|A`Wlhrg8wLo zMX;YBpd8~J$*x-f$u9LlF3UyGa3r;pO+WXl3F_JQ^6{YIdZwPB;r^VHb$%^?%}S99 z(7E7SuBvx$aSyZi6lB3Y2s4Mp%FWd$y|~6Ec1({}8!FVMp5fZW-N3gC26)mK1akfT z0C*54x<^()R0dO$5D#7XLm#E6dAIA*-V-^^I=nBv?J~UO5$FN;@OChlnhj^bn7}n)8Qk`$Tx2utR_int!PrfD z(7lB{;MVS|j47ONj`?&vG6v4HQ`3v{^}hcNXIywoT{n_}j?5zF6kk5N*Y>I>Le@q3 zL8K5&8xo@VV+2oC1i;S(W50xmdrJNgL~J+_mI8W5N&798CUujKkyS5z_l% zt(Qf7|7PpdcRIB)q=(w zrj9mT`;!IZ3FkDg1>@DK&4=vF7i4D&wt&|v6@wlFKA%6DFaRd{KldQCy8!&-e371eO9B2 z2g#uXVW4rycm*8!pwRS48{U}aRp7Bkjg}yJdcJ`aG`vED9L{(@yDCQEAv;Ngx;;Ek zDmHBe*8|;PW)5qM{>{s_6j87xN`X_j>0S;16g!Xn;7>@F>xP$uejnC8=OL`9*tGlV z-m~O_cSH)pA15D>!8pe8ryzqk?i4~Clajn5XyLU&Ztnl^_T_<8u3g_Jg^*OpREWx0 zNK%<44Jwf#l`&C~l9`ZjDiIaZfHFjeh)U+{BFU6WW=V#!%`#;A*19)Rr}I4LdEWQ^ z{<+S+ulwHDy4LhtzjfW&$G2XS8w!oyIRJdGUL!`zYLdqL{`4vVZF=|O3`8W(pnhH) z6hm3xg!NyDMy=k&rk9mXl_Nr+Y z?V~u2lj5{bE5K=`^#{O$ZB{}MC$B`YY?B;;qY!l9MzYWr62=LIoj~!1ndi$)S6d>c z{o*rVlsMysUgR@Ixgi(cd3Bx}E-#U~V7eUsQG#&C?M!3}rGIrAEcc7kpx~I1?g16L zk*r@Fx~5l7+IU|(QKhpXi_Mh*Bf()m6>rD`MW{H_etbzGUUk1Fnn!x-s+<_m>X)nr zKHOS|((=qM2*qkr`p)UIE=2Apy_`q#GR1rJ}j zl`J&y2Qo+l;8T)!JwNUNj~$24{N=|ao1ZoiS*7=Qp2Zc``) zzvV!fTCEX0M7)^r#tN;vwz|yut19Stn{R=)qCs541`kLKppev}zr}e0aUg7Sl z)lg=BquLWXMz?)vgx-K#$$!@d;A-GscFbR93T8oDjx(*iQpUG{Mxz=R-s1XZD?kG% zGt(N}QluD}8DeNo@7-G#i-7vsK{0UZ!usjd6%6AI=t@1H!JvgcU>9_Xkd@*Hx~r#J z4%3Bz@Wvq^lCd>EcvxH9nE98y22=c$*I+s?DA&_&3?o1!?61XI?8%EH*mj7_uGP?i z+{F%LlcOTA;1B_r*>w+{a7*#UF*H#sm}oaJa-5+kMRyA$M+GAX@R2Y$sPEEAXx8}` z&0w`(G`lBBi!&G|8=$vq1-CDb!S#!ktwc#`8)N_gz%)=EmDR? zv3NOz;8Y4Mlf46h@?Tq1%Gly>I}V7-S(Oxf5Qs{gLG)s#gpu-#V+bdykb!j2-hi>E zXb(}lU#I{Ku)R2Mg+)^EoyR)8n0kAQ5$tKM_W{L+utW;WREeK?f1+AA^9svakvQ!Q zy4w7nHyIX-3OCcsodw!=mknB_`EVSB@)(h%b0RS(xE3882TgCI(h5#Xuv=3DnTw(E z&Q6GgBMgJebawdzaJmGojYEhE>}CsUXVY^Qsa=@(MQA?wiE?@2Ef^P?H^@r}%;{yP zm1PnkYMP76gsz zrXP;WBvhvYBhrEq1qZ00nel)_45$G=^@_;(Aj2ug4F5tmw&&YA9^`>bTNlF15TGU4 z1~25^3p)U)yh1d||6#?^WPVclcVY4iBd7F<=(himk?$3lZWDv-YKZdoix#+4`5&BcJo}- z6Ja`AI3_Ni+YU#!%At;qOniDV;r@mnt|?Y}jNHFH4=VY>5dK~J%|kM9l;xy?=H|5x z*7mW7NTWB~^l-xwf%uqAH{cTdnA7}jrIj7?Rf2O-<|4fbZfvy1qK5E<>r?t=^N?hk z&|MonA=nnZ=^<|Jdf|eFwio}gFDb|BW#@*1q?oa5$?o5Xmj~wN&X8bZ?0U?P?p57X~((SxhtB_OgwKKFHx9iDqzZ`8P#oI{=_f)!DzDd;N7|H za#gtl8`>AzGC3^3NI4sAJq-Fo%`c})Y)@mnW7@$i2<{-k%q9B0OJ z4vnf3ZQV*eeYJvCJ#zCK+_>^1ote&kXxtgcXr5V^X|D>U($UCR~-FXvoPnA@< zT$0@vTP?aa!A)-T5?NnZFl00-T*u2aIG%aRd(T8PT&}C*UGf=r@TaBieazN7jQR!wHr(=z|oVrKvzZrFCrn%y0o<+|vurIm0@iNzo zx7qpe$x5_zr&q=&?^{`TC4scZX_)$Cp`WwTs`z9LzemSk36oily}znmC1e%}Q~Lrh z1t*YH?f4=E;(MCyFQ>0ul+JJ9y}a*5O|Fpc(dsK?!yxtc7*^xS2X(w+ouk)VXq@~S za@Xj#8w|L5$f52sMw8NOsmV%epzR&57rBAAy%{-IZns}*@3X0_l%ne!zfT%R%#9=W zC=1Cj$mBDZ!n_1mwO*A_(N_DVOTXq#kgtS#`#0M+TGb4Ry=6pAcT+Tt5qXp_e{|>z zVd1^fg%$%7ei_Q1mcC^qhR=C9?lBL_U*)odd$V1bx1*G_Q%X7nG;6V+2uw=Nsj{Lpk?C*n$W3yhELpF_39fAyPuW64P^q zo%mhrxNCR2l!vY=zq((^Az1PC<(2KFnuWSvcVyI7h;`zWn%}~VR2DuiX8rwl!q3UU z?F%tn;oi$PZx7hTX74lam924%>{vvXXtcrJV-CT4{T>y;%=UP^SzQPp1=((_wbT+= zw@&|QBSl(tfPfClOyQ@?NL>rLxB`W$!9Jn`KD2}AJD!`=ZNa})o?W6IoRgXn3^FR` zOfxVM%VxwmYTZ~1nIH58(@;{sDmKp&9)a{n$%TcsP5pyaRyCeo3Fpbxau3hJO2Nx_ zpopaTpqH`%+=402<+kQRFK!EDU=GsO>(Td$t%A>mT77PA1g#dQ^}MsLPS9^TB|972F``v()DH z(j`X&{C4<10_UN91tv8s_&XQTUH!J5A^$_;1EqJN(SS5CzB{`!Gc_<{OLV>7PPw}k zU7~jvn(=pBv5%w7+^OhFXtRF{%hEks`NN(peLu_zf8rGhT6z~2d=?dj)Wa>jc)Pol zM6;ZNG4oy#7_${5WW#!*cR7y*L4fcjrlcnCD^^7qIt>xy_fmSCn2WXSQ$0QJh0@}14qDRvQ4|# zHXIk{LVvdKy#@J1u!Iqz`7KZ`b5^+gPQ8$R6}AmW=};V`9)1o0aby;oFo+GOVbrcS zb{#~>wY?adjwHO93XTKFUZ?MMv}vTS|SwwgN-qRvsFd`cxNuk%HI%K?=q+fTbxRnj7eBPPcx0 zKpn(MtL#vz=1dN{n9~7%raJ^{$~NI$Gk&-WG3sKj{|@S4VRq`}S||?PrR#v~G=@fF zb#O2j!YbG$>Y4SZJ2Q0wl!xw8O-RBU4#7&?WVnGGrS?~;|@N^ibX@{m8>EgC61 z2ABawurl-rKxjwly03IWptPMh*L{az4a&?Mm|30%c#Wd2?#HEqm%qw^rJ&N+>tWeN z>F?TIL6qs?)=;Ut;pgDtBa76f-*__uhQ5<|Er?>*fr{-QgBoQ({lobRVVL)+-kzQ6 z8*wPAL2suWAOE+~%~rL$M^S`&0<4P6q7j$X@j@8}HA5)YuK~o`insmH5Ky!g#L``L zhUIVc1BcFLsc4{DJu`15A@S*TR5I$wmo8{^kAEV)bLZKB| zjX@7c3fzPm3gmck+q^-KeoY>l|AMC?tyTr20m%FC-$)1YSZv6G3yC2vUfXB|p-F19%z+60l$lTuh5~7dDk; zfyF67iZK%nJO+Ig@=MG_zuO^*3if*8K0^FZQqk70p;GrtClcBk^tROk8^u`sZ(aD6 zr9JFly08$y#|}*WI+hla)b7!Lmxd6f?Gz0u$w*`c(9lY-F@GXzPDw^HNWFD{G?+gZ zec#D+93_zZpKuv}f^B1=em8u5UAL$wN>aODdK(~5b-F(NkQ81@jgnr_eQURRphdh=(Bu{kEE+TePs{J)g`*qMwWT&o{5-bOFMqmT}bX*Z> zOg++==rDpP0vEmQ%{Gh7ij$BIAjKh^9$}Na5I;j^NF=PP029@}xBIX`So; zfXx&G+^jrI4Unc=N-`b zLG6_dkQ{o@REL00pV6LlwtZqp0B#V>Dx>z$58R+6Ae1C2q=(U6=s8VwWHqs#1iRl% zaGSZnZ8-lalbJ$L3X?%yr`tf(fRYX~WOT-=5;K6Nc7Km4Jo%C!_dgLCJ0<7>aan@f zduOQv7pGbdKYq6qx_Zc+C9%CT>IG|tZc+!|^z~fINDcg?@dK-6n=m=^X6z{%c>&X% zVofR2vaWvz?>PM*%5^b2u+-@33UgA*3wY@uG2cPH=27c&gq3JMaK6K8Z z@*g)7u&y%qbTa{Y4OJfbP$-qFn4`H6g{%F0|7}jYMG*E?KbYa~dL@+&_Zsxr;CO~; zmTUq1Jr*umiKJcWbhl{q>{f7@2Lq6nD>~_*aj|MwXf%+Dq0%K#0n>k{UB@{nfWW!D zvnWjzHlFh~hx7y4{n1MR{HmM-a8ic!`D(xqhj6xN~7xz`32n?FQI@FC;%J=8Z+Q9poS0s zsXSnyVW&{BAL7>Hd5lI+X$Fz+!OCGj8tWxT^?t~scOvxTG(8L+3!o*Ma9}Xg`muex zqo_a+`uu$(f#&{^epB)k)&7lsmbaSQy5GKXC(k@WvL2K zQ`6^B5!ldV1435lIk15w!mL-Ch!t+d_Jfm6zwYY2g$SS`MZM(6Qo5R!4&ljA;0FqaT=~` z17u<<7ocy+PW9SM7mcg`)qk%h$a+1c&cFx}OJ>RY&-@pqsDR=BPyII~@Bf$o{)3a4 z{|*09w5#0m|HFUz6J%}hpVj}+e-Vng|4IH$$@}m8hX*acJBvXKBT!A^M?S>vF$Mf9 zBiP_i=M-!MZ-3XvLv^8dLbRrmnuvz7vtr3g1YyZG%&_Tlgqc7|cY)i`W5X#Z&J=CM z%(((kDn}HD4j~tN^#tNjYuge$F_?8Qawi(gEj>}?6sM2UpsG?aqC8DF&ITBnuC@Tw z%5au8o$s1(QV4e=4uN8OI|r+S^*v){Iv)Si!?fSlKc(>t;hu_kf<_U_Nl+ID;y^|K zNXx2sc}Q}1+7UCfS}F9omvwVb4z5(PIIrvlFH;`b&wsSE&eh_)qCoX9_q@dA^jG-1 zmC9@84A{)4fvZqO^AHle%rP*g=d$e)%cg}dD*E-fFhx`@jGmu6#CyDRuMgK5;&OX| zx<1nZnLrbo5sM;Na&Kt$@Pc#j9KI^S=NjL4Tf>V4E1s`%7Y@i6)UqlZ?R0!~uG?<& z(auFjh0JkNySDNPZ+gLeS-N7ORHf$-@6pb^7gBC;cF%w0rCDBc7r6fMEdFxdu?Ejr z_}>g)?ZuS>KhfAUFOS~O(TJ0=*Eov(91zT5ZbozvSNEZJdFxI`MO0KCu7+zVGs<4HX=WPg5 zy2HR_r@Tst{BExBTW@$r1IK%s;qN~_ouF}zXW)8&7f<12wC*u*ee;5&xM)tHIye@t zS8ZhZ3St&r3`(~bw52-oU^(;V{oVfW9zW;x&qMi3b{*8;f7f$U*&&$)eRrxlNc3ye z9g-NhUYy*iFBAARjp-ZrJUAY%Gx}O%%(8X0U8lGY$1Z&tU%o`m4-2Dd7KdcmvjZ20 znTQ4V9xU&CDz_ouCW%4BwkM#`Gka~q-nq_9gK+F?+U%12!JDzKupUI%th`6dzY{X@ZZ*RVG-|JD%z5_uj^L~eFU?7NKY562KUnSc9b{lKD zs7-gCf!4Z0i#7wTn|qtJ)?9t;R>))tZBf@3V0TOKWQ-H zs&c6Nmwi%c@WHcj{Z;9TJy&$#<*P79{`5Yia8kmvnIk4yLttyGWYwjes*63$`&4Tn zHf5Vx;ApuQwyJ6IvV-0s7EJ*MWu+*N7Mu}r70%1tz2Hs9l5`i?$m(%9Sb)Y!v=fZ8 zXZ0uFgEGrL8GhF)Nd)tht`0!acIS5QEdkrCc!lZS-5B_c z!qq%tSC$*`!G;#XONpe^1)E296N~*KaMe5iOy{BrAm7Q-k3$&M}X9^^xE-kb$2Ql#bXdm<#RNy^#>aqiT(u;yvB7*aiQdzzW1ygD4 zxI2d>r}T9DM?)V&^EHb+0<%gF#0qGQZGH`bwi?#(c?2d{1FGT;8Mt9_*I;iisCx8| z*Lb6a&neW-rQRxSlvRD~!NDq{bj7g*_|KQ}IksMm!3)@{j)S=Sgfhmc^@uOHXkn8W}4UDd}F1G_T^mdZ-z=KdqKs_hJ&lif@aDLy3E~`1z1SQl5ajZfly}#kZV+N9j+QE&ZUtUgVM7gs(S~BJ)c8bx>aOvd( zroB7PS@gK@^1Ltqx=S~FXQ_r?;vCs~_O@D{9V=titMAgjv0Iil+PTBFS1gyd`?QuR zQ&``d2UP64uBR4%lTfn~KUgYnWd3db>etn5Bo23-g?hpsAB%=#fJE80T)X( znMC5rEUd>gzwC;_uTVDW`reESU zs?$5?+^6}lWAE#5`XI?(Q_>zG-p4(ptK*zm4BOY_^u*=RbW<&%-k)1v%W70-AJ4kz zh>U;w2l@*S4HDi`nTS$H;b)cr0}~wAd_VW1*qvqqFR_#;7#&6R_qt%bg4tQs6J1CE)Ag0;AOl&R!ahf9U#)(}%yX`!>HnL(YLv&!s2i(l++P+52N) zRa!75JN&sb#1y=ThOMcsTk0dl8$lZe7S+<46Z(}C{C!&WpB@RB1yn4^?OA??+ zzViT#YetIO7D9LU>=_8Fh@Q6j{lf9$@Xoe&l*P2}l|8qOhPOoJB|D`8R`8UTv2XNQ z7H#$;wA4|tgg|X}Pr9Z~eN1lo~%i=G-Ef*Ce7NCSfegI)Rpe&WnI;Om#=xxQw`Mp{KzYamb_3Z zO$eY}=$~a~B$)T4J=Ht^aL zR%gI`u&F5c7nBCeI&WY_lXRtn*C+xV7wLgH^Ts}ZU98%4jY<<-yK65jPa|mzK|;;r zc*)Sy$5*DwpP@N1fizHSkv1e$`1D$*Y-#18e`zs4d{uP!|lv!83xW3Saz8TAa~T#HgkFI-M3bdXzYMS|EfW^EKp2 zS5U7|9==s3qn2rVeuRcUy67v8xut7-tZ}SgB;wu$o8_CZ-jlo57 z_!#Fa0x4h<2rguTi+Hdoq~g;#n2HYc+qJznO$!WT3846pB5*1*LSFT;UbF~!{S<0m z1gK7TFwi>an`IG%D+FpBX8zs)YicW&fH?ZY;tDW>nUAK##U(=M;vq3WGJPb(DH|ll zxNQux-@WuC;P={Q`r7oi zuka0uNcgpiZ)GEltpd7}w+%5?$8FruuJAyym-ggschb{5Lr(}u(En}Gwvfi6GKwqz zXO>-T^OI#^Wz0kE0__may$E|}0idv0#gWEzT7*-8Wl%x!+j!c=oPwU8sXMLjGl!UEy^*1rIYWgVZBn%Xc>(<_#(`f-{AR=ygtI}`{iqnK)k;Jq zS;c_>Pnn-dexhRRy<+Vlvn|9xp2U8JcJ1q@%oEFuoksxB{(uXreRCmCp!R>8kC3Y6 zFP;?HIY{6eVEqif&B~0`uV4B0;xJ4moS5<@EKy}c$_Ht&RMP9K|!lH;_np|fs0_Iz#HU0N+~<< z6F!VhiGUJ7P~FVIs)N#i5? zj(E_XB9wm$YV#bX)C@VtV(ooLj%4r0xZz3GhQwzB%Z6GWjzHb6`r=)H5*R6k5y%vX zn5k2+toO((TwERX&@(=lVHuYIY-)pc&c|!(g;*ZudNMHbzp;(d_lRXJJ!KMk*u_V$_ zt%X)OJtK2%PbhzJHX3p{@TlpPS-F7<=o%eY#Gs-15mGOwNF0=^-%0GtZ`eetgZ=*R z#p)0R;|4#@iX8Y!Kfo75h**adzl#r0xpg`aSCojNZGs9G8Fb4p824Sj{h(!IXOu6)#d}JHWTimK4 zt*tY|Rb{_|8>cyyWt`}#xe(b5P^!VP(-QyItnwR4vj1xk@ZgrP0@Ow1#t^Ik7rzZw z*dhcg+=XDpXQqSI4)T2_Sf#a>0^&$(hxGt>fjYbPLReCpkwDjH>J`HyY$3wx5UfrU z)c1<~XQ~wh2+4UWBS9<3OBk;K27pVFR~`nxfw^&<=>TA93I>EPGY)h2fGl?ab}G|U zwzw5#U?M`Les?zGexDcDsd_CZB&*WeZseJQ1?tRTIjTk-Wf`|%^z;1w1X#P)I36-b z0LuzR&%uvj9gx{G1*=|5&&cic%(_bf1Wp80%w+dfY521LY1)8%0L0f6q`KzD^H+$lYb@ibjMgy1vKRoI^WUbHUhsxL%6umc<^Xs-T4CbgTj zm(w`-h-wB~KNuQ8Nt_}AupeftXP_S)R4jn=A1Htxo5mgdj6PgPzQkV~5BgZ1NAe$r zm1WA6o-Po46jut=2@B7pub#)B#)HL7=XVMK4Pl!w#70hNn=#E&LGH zZr3TY6FMtJD5*eS}gkC(kmd;GervtGW^UNaw&C;KOn6TZS>6V z5Mys7r`+$O?l-Qnzzzk_^>W7j)0tpbpx851AQU>AS%sMd9V;U3)rZrJD}mB_%+xHP zgVa34S}8x;EAS@NEC@8?R3u1;XA;&AzD!-3g02tO%TV`DST%s{O~E(mWv%J_fN}A zH67C>3pJF-4j@GgEGzO`dUYDe*ED6F>R54Yg>=9wz*b0PkzIR11gjYd?8{bs{uset z09%hi@Y(~;{f})beOWw_!UNzu1X_$o6v`^xEIfU<0+B~yS5_}6joeJVvf6S1`oI0)g?>WFz8?P^r zeBP9E70SXQ)1?XzK4wvX%BhMB?a2&%q|iiS%&96}d=LB|)JgDkW<0Z+;&Rg_Mg9oq ze=}hNO&jvZd&EeD4Ib%$3n`iJ`hVX95H`ha0aphuWHluc&Z>S#9#j4qU4>={$`Sm= z=2R1aBnz_d$RUsWbFd+w#a0u;Z?zo?ur_vgF2+Nu*6%ua6gP`Jszqa4FsrXxb?Y)w zrsdM$>?`@aEtKO^0x8Z2*5P$1ke1ps5Jb^^)d;o*o};P<^_uQX@R37CIq#eqQ>gkf zmZvozNNqn7Ge0Fk7!MDkqrr$GY#-;wi|B_QxHB?JJ0EW=Caif8$BOIZ7jZ>O?zyI z#M6DnRUWHXGOQv7nguwmh(a%yOr?aXZSF?Y^)b?*$P|8ieh(NSKqB< zpn1Es5q>uID&K&A?8?VhkFS0jILnPA!*^{FAys#sLo`l1Br=rT2g{D_u>QW#H7O|Y zWhdExcR3Kp3x=Uv~c zsrfwDBBef2TRc3mDa{Zzu(4Cb%s|suemKql)7tzm%o$JBtZzn~VsQ{sm8RDe>C(Jy zR(Zhsx{2-iu6%u;H=g=Mf?b*hY5kVBc-3LCm4moGStj#Q)ciA(%Z?|ZruV!WkApb6 zii5C|p1Dt-s99I(W`pi$k`M7b*7tc{H>gn;&7PeQ;Ic&0Z3TXSDMtXfigzXvQV5t6svs}gg53<$t zjCMM9!t#E+iFgrQSCbh?J*X{qJ&*C;lFEkYjdkgrpE3iz2U%0^c!|gjr@hvE`euxb zQ}+Z}8lyM5_GJ|vfBL5QfZoFJ^^G4gX{hB6eNi`7#G}A-j|SHQefWq|72NpQa(51iJD$Jn=M#sz&zg-{vWj)>+f9Z)?E61p?+QT?X=8g8M1{oeutV`ii0kF zrWq|=tNV)aUda1~Xi@R}gu%7>P1ouYPh|zBK3D0J*QW;EvJ#}?2On;ooL&vpR3PObjQz(4A-=Y z1HRR@TkkHaIokEjUL)2^$GOSv1e>?!(MY|=aGxP@fIeHF#pkY{o1((;t9{PR{<`yR zC4xIDcoA=QQpKLhpq}2S;q9xlEj`bB~lMAZm1f6W32&qD!FvGa*=b$ zPel(j-8go=Bf@V`+U&0QfiK<N^VMXR&yymmOQ-qGRkn z&XQuzI+&)87mX{t$$B2@Cm$#eCXlKQU5dVKtEN7Z7m%=yKsIKy3m&CSWV-{`ATsl)`0af|o#6MH4T4-_PrmI;?sz31e) z+xX#PqIPMRwzJ1t^D^P_b1>3p=UN-KaPmZQ*~0F=$k=Xj85BCd*1ZtSMI{C@4Sb$= zRTK6s_Hhzg$Zv*D*BvyI5JRCl%{J%86LrD%?5h&q4-1Y(6trn4_HKGJfnw?9B*0j0 zDoN!nWWDIc-X@?Xqmppt@a>~IiLl>+1^Qqs=R7r`%Hg`!+I^RIPXw@~n1?srKuadO z*n5+!7v^4w&Wc0HO(6Na({}L3LWQ&O;LXv4Nda;qJuf;kma1*a+m*?#W<8^B{(hgg zl4fj&g#mS7EHqwWZ?`SLH^p9=?Y0RITgSDh&Q1hv`G)e&mo4f$#faO;7y)TTID2DW`(4)rgo?SfEMsg14}|%d%{G3i=8X1b zR=4FW#8q`&$~b+!U=*%%+#a88&e$4suv5+gQm3Fgl_t{beM2gJ*{;kwm&&8=hXFc* z4A_Z_;f+kUlMLL9j|m9E{upUgjI$!)lPQ_ZwX7GYDm^2NUw#S~<50Y)KAwD$&-l>~EKfe15m7H>HK_64FAGHqpg^{fY% z0KfjCuC{Xl`E<8lq5(tnyQVcAC>j?AH{aA2LEHan zUiB4pZ5XnP9}07wwCE$mN>!B|0z-S&r*c7b$?}^KRwc%MLagsuj4?80l z7M!UK-~?hhOR7K#+VP3)sgL8TQ&q*vp-e>@;ut+7Z3UvVGnB5w3Q!Tv%Jy9QeH@$U z;IcTz_Y^fGpz3qPuq+Z{^B`x%wtiFcfgcXh_}0ZIk%^9HKybiLbSlZ#ak{EFp}HRC{Y^Lm4wQo(&UZ~k7@-=-`Do-oo8 zo=kcGCV(r9`{_X`ayBL(7tIGOE||t9-7Z}~tM&0Ze$n&gnmI~a0G&p2!d^QGb;sOW zDvOEZIbJMP8YWdGmx*qZ{9)f|Mdv#rTFvAu7RFtf2uEK~sMJlgu5zyb%gQ)N)NDoF zRC7N!B^ZMm8lT#10zjtG2Z{m2yEN>?>Grme*OVHi^a zce%T@qw$G@DK$KE;}W=$RFGy1RPF zzAA5vL19?v^Vymnw#CTV?g%5vkAj8<=@eLkiXKnEQard6kOrAC!O0%t#VmM;2A`s2 zga@*cnB$*n*#HxxIR=ojF?v%>qOGS`8+oBzCKR$~zjZ3^gi>&_?jZO>Pk^eZ6YQ|U z7qG#ly!+sd-|?pFvqAXuAX#*ZH`&at>BcV-5QK2lX$s2L!_1+={GpIFdZOacW-U_F zgN;vXJDsW&+JyUo>WU~85BV1lvpcz$3fO?Fr zD-r=9hUXkA0b4XW6ExP~1JbD(2nHJ;!;16IAowlcl&We!sroMs>r;WA2>XU;V~N92 zI4eaO&8+$iM+#RnMxldebxnj|;|V|LFoiTh@uEvK8EBnT1k)x?g|76o@dzS5X_X-@spiCM%Z+{0UHDNCl>!v<}82= zW1f0rnbUUmMol@zqJg~xQM#OJsaZ@dUR)K^-Pa37df-f8Iac6I5P>rh4C_+?XKH{L zazCt-V(tx`2``Et!1|mTu|t4VEpEt?O5$L&|9;urF}EB#Emy-vv4JpsIQ&3XYR@>! z!1PdAVO>B)9Qy-;aSvy4CsV)#l@;~?wFXw7q5a<)QbXO`g{u!>=CHM&9!#hZ!?Apn zYsl^HP(nFl25gLIbN)tI6U;CutX z6h_IYsAUX~M{BcICNYA+?=xRc&w|usQ z2;`?E=x7Jb(;M+@I& zBQoc=x-NHHqUKbIc!ZcJ3=IxI1~fP~ry-FF8XWYvSq&aTKtD%RR|G0(a8MADJJIQe z;9X@P!Uzu{2oe>JLNKMi$#w)ulk1g0QfMG*{ z-}+Vm7rz!w`Bm~i_bVO^{hY7=)UPqqF+3X)m%A-ibHXb1@BIn~5Rw0Jz8a>P_s|Og z>pA1>iKnHlc&d!08$nW-EEbL=l~JO0Ep`K=U3f0!8$x-c{`wj`62bFoI6ZuHG7*n1 zV-mH2%W$TqUBWAobC2}3;JgA?g`xz(jL$NRbGMA1{k|1B^a+_Ec#<5y-2djNE(rE8 zUwP!g4(KoF1yNK~vD6Ohs_CygE2HZKHNOPWo=Mz+N7t1%qXgw1CZ-EUWe8=v zm>*)%tglI2?$0TCQfi09@Mm7Xef10M7xTnqa8L}-zT<|)Rg<1`QKI05knD8O&v@mL z`{16Ow1WFq^hvv1*6j3oo3n}jz7?(9V)wR)fUjn7{uW4__wA5yb~ir_o08L1N_XfBY?mML%&qDhrZyM&XwTrveeM+?^P@9K;QxNfrk@JTlBES6~V;)b>K zaB9H6y>{ygvHYX+?$^EF{e@ad*a3{=z1G1_5=wH3+OfAR7n?SUZEY%Gw1B3=^I-;S zzvhC1cAa81iK_4l5I|}>Zwng{-?@%d_}0r@d>-s>Zh;bLLN;{C0-tAV*0a5nCIE)34bBZr>Lv3dI!o;YyNEw>ta?AJZ~ zDS36tN7jHrFI3x0Ok7ClDdKX;Wp`8A0DI`&X6mWV^aUSUJK5}CNRuj+b_pk86AFAh zr%lzf}L}ZP>;a)7oRlC`qgV@o1o8hYG z%ZlZ0Dmud}|GE_s0s}zw(O~e&85JEk?C6ZZ>*5A~CfK&gCje{49BDi5 zSZVCOlY!?7tGrwkXhxqo#wO3lAp212Qj^|!R|il<1de^0^dh-l>%;fq9Y;?Bo-{pA z-NR20pWj+`z#8r2@1?FkPlZ=nIOWi8V&nO^M8(AH(?%N!*^X1qf4(QB3b4mpG*vVE zzIoh3rW=fT)U#OcWB(R#MMCmxcQ7W&vgCl{;Jnc;u=r9()@+~f`8nySCNZ;hr_S0ffAM_1K7 zGjgwhXreZ#uCjQT?vb~)31W;soEyW@dvVWWd}3+B(lBkurhz?&4|9A3|!!Jk#NBQC%fZap$m9 z>k~J3tEBO@Pqh(yVd}a+ zM(jm@Ff;j-`t|U-eZO7YRu86UZOty1L1f5w{FNN6@|(p!{JeN}w?orgp1KKCblBn& zY&X%;+!K!XjjKbKr#PnB5BjH!a8bI65 z0f#d=(%`MJll^B-T8ZR7SqEm?HzGA@!O?iLmV0ER?(7)q+pln}HygcmKfGtC)qd>e z(EN#pIvuwYM=0iMc-()`#$|WM%Ouz1e~zGt(3^+dEUYARH%O+9(Avnjm2fS0MG5xD zuYAi+U!L2l?~x;!W?$Q-%>VJNSdOIhxKxo)uDg0I#j4{jag-RSJwf;kuUn7A!loWI zQd7xOyb!tU6YY@g&5_6ZvMY&Q=}Nm1GU9Yc8tN_^%CPFWnp>U7Wtey7dwyE$>fF|2 z17Miqq|hbqxu`Y<-7zrtKE5I1`)&NN=YBUf`Ic+&q0b`B@H?Tkkae32>hQJ>to7~5 zWX;a2&GujrP~+I5#_whW_Qywuhn<_r4x_JDl)G8AwB;ydzC1CxX~N7IqLDn%rsLK{ zn|f=stfaV$)*?pXCE2ado9*xj*Rkujam%z&Hyb?#l#-?vx6IK1N#-R4Z@NM21MMW!wx^TYDP-3j&M0%!o~t2_zIX+0XdnlW+YB@3y-C)+EAlso!tRKB|77gt|qeyd)}I%qTD*X**GTwhgS9Wnm-kg>uxm9bHa z4;$y7>0%iz`8e5YGqI-0C8l#j_ek|Y)~&h2!;imQhrfx>9eX|bt$T7HH%FD5*WXHg zbcvqgQdK?G$Yt{*MLjhP(wxfF8j4XFGgaai;owJF}GBMC;bT4at4RCciS3Z9cyRE=C8PylnxULdVg@4_87egWLn%LoBRf&7D ziGPz1UEd!Zeie$GGdnpvOJG>$h`J7W(;*2ZZr(@@&h3Y%HLqPhT;~2>mZL=x#b?e{ zaQ$xc{I7O+BGVH-x!!19kp~*Krt6vegoSgHpij?vIvSBmd)l0)uW5J7k}gbc3N{V& zwoH9%IJjMP)1t|Jn|cM``i;fB<*5LZyqI)WU5Dyc5DtIzMv$TKzKad9>e2ifKXSxA z=_*GSr7T)Y;xV*JR*dPoqFGick?!-^w_w)q{$ccw4Iu3O?V+?-Ewrc~P$Hu3S+- z-TIWabXzmy+gId@#7hgq*le2pi-#_)^b%S?w~gP$K+Sw<0Y`P@g60Ker&>A0+nhFKi~b?1(?^RRDGe(|@ru9E)t)~9Db-SUw0a~N;SH4mG|@3rErL~=~;gUPY; zTB&zl<~iQ^++iKYb8I|qeC}9fV2a_`c#`|j`7~pRzbWN%5@nuzwDC3EVwp34zhU{K}NTUh+D+&9z zakidhd*$<(dR~9pCDZ)P>kdyeq^&b}c;n!v!xOg~T(^k3@9Ca!v8G8oT`;yOd(=lX zBsa>z-Ay6;?68M`-owiYA-VS)CQHV=52hPUTy1dOs#9$EHgmE;{mQ{~lZl%R<8z%l z-PhkSd?7MeBi(l?yKHL~K!{RBW^T?3hX9%F&O@x#LlYLsLkgvVylxBpbjgaGt!*x_ zqAP!H)o4;)gKm_uTh3796^AJFj5MasR}y#s*e`N!WN!E*W0OuP&Qj~)T<*32 zCE51d5DYr_g_HBsPUnqzl!+Dk;IneKY4JO`{C?Sus`8UA!L)^prgz4`hcrcvgMM{Y zqbEN1Go*BnZ|ECoZ2rzWA;fF&aPBceXaI)UvooPWL0Q+ASohW+^ZIG!bNPrT z7g@F7tYH8(%uSAdSh{FZ@!Icq!78!l|)v?8GxSO7O#Vy0r*bEBtrv4GH(p z{bhu=u9S(*$2q4~G*sURlWE@Al`;jfx}vu)iy=)?OePB3!r81^&W(T{6v1vIV>TF5 z{%70*yt8u`xK11aF}?TQ7AJ-9st0)ViH)inRE@qaZ&Lv0iERmYua@qMQ7`WGS-$-_ zES*c{ax%vGNeF)s9R1U##D3b;k)Jk|D+-%BKRLZA!~e(Jd&l*>zyIT zV5u92Y9&1Q=_YooEIoAXBfN&(IX=_+V{X_i>_YD#rKl$cYt05iYw3N-e2;of(o5Y6&=qUIKFI#8m0!2koVjg&v!aHw@5B!kt+DGxjRD{bV*{=YFuDQ6YFP7TCf14@^KI?8UgGx^Km$m zB1^7Cmb~uThyJTtpIE-cGdu5du5*{7b!9k|MWXd;GgIr!BTiNT$l4Ir&$mt`bW!(+ z=YL0)=%>Az6ML_hsc2wSZQtW6c*&LP!#8n=^YsH=iC;xt&SV)_T@gTv5C(HP-+Hqi z!8%~y%enMh%VA-LFmu!jK5#2Pu`F(88aZ*mPrOTOH6PmZi zjL)({co)lcy*#Q`H$HI_QhMwv|r9_ z)fza8TsB#*`fSi=Ny3T_2TFkZPqRVXfZ@Qbns^}O(#Hd?KIg22F6DyVX;G|Q^|$~< z;vbJ$&150;FWW-X+b(AQS{Yu-{KePk)boqpd!2ge{YYvRH z^D=+Pqsp5vHZu>s73(lANlC{Ua^SPxZH%j1->57<f!A(ya+)8q$q|h%Rq7E)qUdeNGOOMW z9-W!qG8^GUMajw6yC^ILP zYW<0!FfMo|X;DTAAGzIcgF9^eZ~S}=v-r$VD1@c)7}v1`z!-)i3|J=Qt0F~MsM-vg z^nB$IciM}mb&E2q#>C?g=3Ab8)u320!f30MT&36Ye1ypfN-DV{_;jd|Ka;L!fw~il zIcd=1KLcTv;|T6sYv;%YJXe;&2Fhha@$e$?`nqMs~>{)Z97KKH*pW9G!#|MfZl*Hu55&G8&T*`$&4*&Ek$p`j6Dmnc{ zsM`Lg=PX@4W67M)Y)8v-xWiOOp|VV<@xkhQLJ%=A&AIn~VURE1xisy$G!roIF67y_ zMO)CVpGy7~yM^mibB8TCSxEJlZ`ZCXpZm6*vqYZGQD06>_Yj?COV8lfFTHu>*WKN0 zxsJ-*VYhC7$@$vGP)4e$i7R@;lh)F%S81C$lAJFOha~b-D=JwKl8*u#zb*9 zlDHrHnf9pu$8{~7uQPsErta6C!UB+EO=frFH%M-)(1Qd&me_IRP|mmb&rNiWqgMK0 z4liltw)WPyT2&UGgs^}X3?Yi&_)WOwsVyCKd8Q)Feqj$th+ql5JM!7;SFacz25okV zr%J55fdmd}gMc`GWABYw1(3LlCCqQf2n=PuHn}6eHMPn|Jx#Rfefg-pYoho5qZ`Ri z+$hFECt#R>y8K`Oa<@`o@?ij; zhTI7BFvNP?0)gxJjSGf&-V`jc#fW<+m(IO?Mu(4qL4)P?fj_V27dP17esBlgxQmB0 zgGrAW;YoAw5VJ;j(rl2(#}W>B(ieEhESU76F`o1u2o&QtxJ>Y*16bm<5%-CEo~x&{ zb#%2pY7Kr5zypy;00Ofy;LaFf=;UE!*nvb8mf*gN?5tvppc9h~elNfR4j_<%-`G@$ zfp82<3}tU|EdBGvYPja*6{qNIyl0d1^*(;{2)ub0zj^-wezSrwHb|T-!V)gnL+yCI zrC^a^!q8}lV*CcV2{utTmZ)oaVln8}lEw|x$N*1{TJg7z+FgW&`Q;!Ga#;ekPGV}^ z%fTyc#S1G0iFa7yttGPI*GG7AFHo-+3p9ej_FVkNC#>(uaxCHVKsHnO^{myYesTx^ zViTd9W$g8v%i3lEbR*bW{U+cTYwU-)yBKrr;{$K0jD$EMd3t%ttlzi6dQ&4 zE(&+2?J#cYKw>SH*l3KvYK4J+Z{3DfiBUcK1|`SAjLi zq9$r%0r;RETj3noNyivL=6(Zqii#PA2uPfKh9#2lk|gl-W}x6PEHH$IXvA-XnPH7( zv4q2GaAVqE-PkS(ZY;!ZJPcM!C%CZ?JH-<(X#ynPVTregu^V6D$umH`5G+8m5!`qm zzY&4;J^2Tgxc;rm;>&fIPT&#c#ws!1ESjiKF*oN?)7bg-AYMZqvjf5+J<^d9f8- zV2uZ=0wd9zW1H9!yy%R*_y!~#u*7pb(3N{Q^qGSfU9cD5g1}LN7hSLyi?M`?1)037ons+5OIrn>m#lQ7l;2U`` zK2n1**R?RN8<8gWd8lV(X?{+1Vf-1@lQyZv(ZR08IO{C)$n?)GOJkk^i;t|c=5wSw z5{7#C76(VVE+@_xxiZ*RPQU9~`s7)uJL6|<5+4_(S}t3u`zB_|aOWaJ<cQhAjMEazu$R#vsuZP0 zO5XaR9?#09k;*yr;zS$$yV~(?p@UJEZq!%LN}Y9w zv1rvFChvS$(PkEAv*7+)E4xx7W0N)g*sR@Qara|u#&WFSwVfOeYiFJfFSG|-zdtWB zp7d^h-#eZyaAz^<-Sp??s*Ao4z6@8k#aj5*n>51B^6I*?_?DMs>2!Q<^1FrBVV`eN z3^PZr!1uUUX=-AN_>p5z zZ@V^~z2)y89y4k(icZ2}MlF{F?p)!s6`H@X04G;$gxRd<+w0K7z?kDn(~K?BeT|Hr zqV0A4bp4(Sl9D3@vuAvvuj|eG@Iu#A;-S1xNBn#DRTq3Z;xV8)!+bHvv3M~t&-vkn zcXRvR{kFyO`LV^DPvqdX=hA`76b8$M9Tlu^w%bXmoCJO4NGIB_-nFhz5tM(Z7=>i& zV*Bh~wV>roS->D&3kMFL-)l1QDBwtR!KcpykCsZd4^&LNkv@@iS38|2RLvyj_>#)+ zMnb;q6c6vz-exz-xT{%mg7M=WrjMU5e7eI_@rlH8dwB6uU5I<*$rj3@hK(&oJCD;I zyPd;TEisq;`Hp~;WJ1mc_iZC@g@J-+j^4Su&U?iF{t&Yi4AVz-?(-c%^R}DTyB^Ly+}>I- zVUV}^RZGb9pL?W0)%o{Kpk753iKUU9o$-OJ6@`^8(fM>1!%3rvC%tWQmYb{J#c7$) zsaG7lb2sexZgeck+XXr_)%F{j(Ah=|7@Dlz_ANT#X8T!l4-Qye3PaV(mkgL4YFC9=H$1AlNg0{&K0kXL^~EJ zPqr}iNyLySRIl69OB7p@NBZzc{j=Ja3V}kBZd?=!@h|+uNGwm)o)P}tXm^J+^~sj_ zM3DJRc|FsOXeZah3r~o7s^&M*+tf$*+;Ah>Eu$Jk!cicj%|&rMbgfP=QCK>cpytEb z&h=X;8_2=k!NCrooqnDq{rk|79yOCBg>M;j;AKyGFhF&UWKa%Q_eGW8Tl9QY+CY`i zM|=r6Tm`}4zj~7OU=#~*x{N9_(&t@`9=NAY6nRz!C<54q z(ur4aF2Ufwa?a24$o?6uR#|;N3>HVC>&^4NqSNZ>wl!ppJwy~^r6DH)lX+HVJddfm z>Vk4M$jaVD3M)|{^=y*?!lQfnKHg!z8L}6Fca90LR&RpVtaJ_x;$|Ch(kLM<0vW`# z2N~zK$;nIt6pnNNfCVNQ^_D1jU8fSS4XW zLBU{#WwL*A-G`j|7dw7d_f`B?XTxmqE6$E1;rNBK(e&VKUJ4n~i3vCdXD^>W4%+0g zhr5=T)*C={r`})RCB!$->EXHoN><6|6bj+{0=9|NQ^*8=)1DxX9<}7@BN6-VZFQAE zCSFIXad@zo4<5LInlnVm-ycXgf_*>CU|aeU0V=Qg5@|S!BoB~qR5K6E5JjdhYU-Y6 zunDKpLB}}_1C>C*0o6(7J`fP|RspT7YF*+PY$&VG4S+?jdU>k=T(vGRWWev6l=&Vr zp4|L`Lcj0pQ*A-ZOE*6$?9Q`(`H)ChFCb>(hvXkuZHvKC$t=MfSJ=f>b z$N-Ze;24V}w~`@MPZ3kRY!2|;@RC5S&9w{Hv>>=p=>S}1w5`}Pw04?IJVcJqrO~(7 z92@_BZhO8=>M^6SB;4=&N)az;>9^E-!f&7u>j|#EMmpz8PzKr3&9E-e0}d&(O6cVjsoDTbd3 z9hMk8<8ApksOif&Z+&(fXs_iKGux})Y7S9&d@;@^T}VG~6a8_n)&Khdpz6!@+I;U2 zMyoH8@JH_RKnMr`K$>YOf~NOA{ZiA_4bCPY}b$RAs*RYHS`2 zazJ&+^{wVq1$>4n>Xwh^_@YLX_z+I%dtO;b#(}YH$_eD!{mpoXZqo+9arLDTq%=bz zg6?z{2!%odMYe_n3P@D1KnN`n^0q`_6&Y5Zr4eunFnCC$p%=iS0@lat*bn>y^Tto` zdBQ00BhNA^(95c}F$1_1DvUU`V+?>NL<0s8hERYwc5*Z3*wcoZ(|4Uznwh?&+?P^| z2n)V(idb{BBomGVxdi=aSL%zu?=YsaSOZ&aSJ#CIRI29V0Lvd zC^$ZEO%Bh0Y23jztWF<~pit5efg`d>%! z=Fk5KlE^7mjE{);08F#u4>^681T=vV4^{N=a(c_fy83;3FG*uNGi%WsWx;;`0=PKxnqSXt8^W%`WzV4OWI zOnZP-9nz75_vj&RxdBj6`lPr2pixHOLlnj@iL|AvL!^`>EE1sqe`kV(=a*Bm)GH#608+oVcLu zPh3s_xE@%6Duw&VBYT_-@E~*Q-&AhVnBM%Lup1C+`B=S7bz}tsi29yxrmm+LSqilw zlR<5T6sLn{Dg0q}li7FIV6sSD*5N=^{YaeVlNT_O7PHAhW@A~K}5 z4M4Lr{W4&FJQ-H;%QE=TK3-4ZNUprI-dVc%oI{q@KaPdE~7Q=*J%CB zueaK?@wuq(^IT6Tr&@su&uXEF5|DI5iAm|{(i}fMs$MKyg`s+JD{l!(4@FE1H9(~n zq+x)ljXy6$Sf~l$Jw$m{+Z0+rah@GQG5k_f;qlRfaVJRUKsm8|0{wACxU6W>uN6HK zvzk`fKm!SNUPOk?ipLQcgZ|s-^q2ceDVCtg{3kqMuyTO_D z)N;3=(j$f=EFlMldJ&Oi*l2kN(SE*1p9%kr)ju-}*8|_HR;UMHiB+K2d$r47&J94` zg&fhe&f1-g7}N_>s2c{~>K1UR`Iwwg4yrn;2a7LH?wGt^m?LSs|B3A>KrF zk>Q`>0OX!!@}Z268v{T@d=5Pzq^Sh#GsOz*AX9khVojl6p)HBC7VHiQy^G~~;FoCC zTph7i)dTXsNCrGh6nG!@%OVShft5sM<{b|e6;4`;o@APW=o@Mnzdzb#1Txz>Jzu9Y=lB3G^u4h zn=h!0*>W@d)td*ZhG5%^LR<7qy9O#EjvAn>41cbIY6;2c0;QL*5eUl8ZbQOBx?Xh_ zfV|urBiaF;z_J|~HagH*#YgRQ52(R}Ned6kTW$_EyO);Z(KB%sHnfD2Q8?6x9Ie1V z2ND&%{f>$(XtX-J{~vj)-egP2b4C@F|e~ef%|o@cB8#+l?ZCn zyo*DM!3Pk?VJpo;+j67bohC^euPisG!<5yanY!GK8D>9&w&rMq{!=dlN*ACV&S``` z9SttijEyBB^J(>Fyd`%FG`MwXhQa$E@33T>uu;hL+>g2reSuUXHj3kfwn6xTAHA*{ zp?p8u2Bc{Txtr+ULlI7BV1uFsGWqC^PCC>!>L#ziR-!*zU4#z8XDuQ6-=Vw^+83{D zLYKyvj9!RL!NrjM!y`~6pl)$fBsg&=ThqQ?qPWt%Pp<|poU0>FSF0*e2xL)~GUIt1 zzG6@~C-jX3^lXz67P`I~K>NV|N8gCqUZdhAOxTeCQ{kRmOTv`3VSGFB8iDCkQ8 zQ?de>I5;N@Y(i*h;cnbYgAuF3#G8r?7deCEdHayI{Wsl+)E$c&p*c?bhOZH!;U$#d z^rOEGDX>C67W#t{yC6#CS^Wy368~aQ&7TTH2yXS&-}8bDDXI@6VY`KzMz~#sdvbS# zPKkX7w22g4eJbt!dFb%iU#@ikDQl7jnD1y*Q> zN=Eo&$y3BfFUIX8o)gAI`ra!1o=m^@GZayPT1lpxSM3m?97xFhxr*J)N0GyA?ynDk zUGsfQw$gRMdzt05kHgUWRdwpJuBaUk>RpFgGcr3Hnj^>gB`8`>!M^MM%nGM|6o)^Q zg}Mtq5x+Kn{`6|is`ksaReSGs^`X8WfZ$xPn}gPz1xo9iestp@^+UH}x$B18TETb= zM)B`~-?*p|+J#~Ts>y@=^!Q_l!sENo2Nwoqv{1D*2u*#mu}|5!)%F=3tG2(9f(#v^ z;4%kEGkmSxUikRNYbzCu?un!3A!%f>Wi9xygbmsm!PBU-$z1yZ_czlb2<4aLSZBm0 zsNdpW;ab0xSN>!9l@*jWH2L0E3q6N?{Fo|@?ZQ&$hGzwR@70p@v{}6Q+&MJ? z|F%k-xwY>@Mmw6EuTgRLzSX7B2yPn%~yKS~lbt>hs{W=9wB;u^xvwoLG#bBOo%#K0 zZiR(dwab+c8?SUtaSk5#ZXDRUc%QSl*w=r6UDIq=)VO)&cwY83jhS-mQt7+D8F)1XVLGMrGmw%`X&np**%YE=MF2Kvv z?L>9bhu*Sp8t?aoN5~dM#Z4Hi+)dQ;O)svXoD=VMeIGaRn%zKEfUEGxS?U0$uLr8T z9`kLO*Qhn2LfS71zL%)oBcftB9p#nFd~4MA)~LxJ{u=WeE7z5bKl1l)4Dbq>ak;|O zGsD=Edu<`Igw2H_Xa0$_t}~N(^T_-&u3W*onS0g=Jw}-dl~WQ^{%ggXT^A!~?rOPt zOsP8kah5e;?T|_B_T-_*N$(aY<`SX^C(_5Hibzagk#khHteq)xz6`6jowIP>pN@pi zMD2|%eDuQl^Dz5GR#S5Du`MTKL#8w=KyI{rNuVl*p#B-#J{#Ie}H8VEl15EVbIt?0~pR zyeM(xO>(h{*{qCu>T}xpf-Ki{Z?j6gjPFTCHJz8mZGN*Kl;uyVUm6ZxSF3b;_J$}S)}!z7wEeK7LUD2WMwU2F-`RY+8H;HbC< zskj~qYitP{*?woJS6&jG5s)~VPAA*}pEQ^irk0WpVwK33(`3syUj0B2Jl;V;l{w9?LKX-*R&Pg0uu*dSXHDUI zG&tEJN>JQ5Q#OB-nr(d(7uauid=E0ylnR}`Lhy_esQ)tpGC`z7+9_~HJUelu32(-6 zd$K(MlL!Ok5<2W8{yOi#9yg!PHR{M_DJg1^NvRw{bow0K2a(N&LrC{$*NPLNMGOal z%}5KiNm#Yh?a?CUAFth$FEHG<3*6k$Kn2FlRAA8OK=jHNgi?is6;ihwZ-xx1cr&>0 z_{aoIn|g&pjJ|KRo+D3N`C&!tV2K)Bf@u_D=qs$+)%qqe@4y`iWcb~@dQkI}Pmx+Y zGS~;ONxNh`^UFq{JSu`Ig#sMOu+-(-L}=)odxJgcz9ymH>qcYcymw zzZ1OAa=i~EHf1WjK<)unP0dU`B>MqYSy|14J+`4oH6SFFFhF2szDcBy&Iv(w!^ScF zp7ktX>28V~2(NCUq7b$8^z``a!T?ecX9aSAFcPp$Evwy`khHm3N10YVa)Oy*XUPttWys(9s6c?-#t8F9HbtLH@dA6}K_%U-dh$E1ARFYp z)dRHVy&dKU9Ox)V1OM+1g&yZypIxLBet<|UyU0i5=2ax+n3U`#=5{UuWY@yC!6QoQ z5M1vLkwQkWeQ=3i%MGSNO(&NrMZI9>8Km!aDL3x(UL0~Ew0DL`y#vwFcBvnfNxv_a zQ^W#rD0w@q=9sAX!lyJbI~EL3}jz;Bmw=)7LQ;EuO{tOfZu9Z0Q`Y694(@H zjyxi9(v<`}$`w9Ac*OjxE(cViZ*CY4Zm`DTJQ#ukQbHdw_0AG$ppu%%8rD~cb@0KY zNGU$TV4UcTARVR}jtHCI9Sh>uXz|c4DcjZJu|qdHfgG@WFZ|BXd9d-FA{Gd;k?4W} zE)E2n-5{nGhA3;)4!f-!usV|`Qd9!~Vd!!b;w+HbZ6jEK)C@W-8T$4PF(?*jq2Lhe z0}%mJ-i`mteQ?|XurCY|@29F40z#hgWpELu1<3(0j@Bx#jfMa?!Ba{1%j&Xwi`gjm z!$gn}3|&NPn>YaBOgU=}9$aD`4ng{-uTjGnL2v<^uT3F4ph7_BkZsX~m(pQE^xIHa z0D)5_Y-d_L)}>ME>xR%$YfL6S-2~LMCvUhr#OfW`*WwYG7XYNw&^yR#+%67z0bE%h zO<<$-I6Pq6C5MV}Im`m9sahj3Ue!TQ0ln4WO*Oipeiil_XaP_$AiC)30ScjH0OPX)jUjm< z>nf?Ui6kG^7u#WB7q4y$xPt@-u}mHZhmamMC>DQ!cg{bZ0+Lw=#w%0}C>hS+#E26c zl#7>Pj3~fv(}AO_xd!Fp&s-BlfICq=ygPa3jH7vBHr9qKs6-e9|nf1afNLVyVj#rD5Hv@f4;1? zSB-7H#Q|pvaNfnY%&W=v?5|{l_SWj)JCsUDaLE3u$HAGAkSxH=kSxIF8khOKLGK)- z3V%$*2b5B?u_}nz$iR@A;WIa+y+joABB2_GID(P|ko36qICC6{8Ku4?S|R-85<>Ax z`M8|ZiRunra9wTT|BsS|)PIvKAU5nhK|?AI+yJtnS|LpqDn$roU>?TdkWk2zP)C9t zPMqXI8G9u*z=IR;IT<1klsZrYzzza#1|j7Ud{o{Od>=VX$pa|;Y+ zEYeVH&-tRHE;*y1W}8qkaVwFCOn|I@eS0(iR{lh1r4XBiC&biI3g(%GL$uZO7n_UP z6V@Eyp%A02@`)f}WxFkN4h=Iu;}%#gNLR^BA?A@;W$l7SVSx7^?LVV_?u|#7&4gg# zkjJ&|t<~JDyYlYEqg#Bt9NUxpTx*E=(_BpcJf`K?&bg*hwoKiY>ad7?k);8*)Fqe0 zj2`YMi!Q0BCkny*?i6wjMe>yGf|p#1g(S$?x-#;#f?|8QSwkPQKHUl`J&JSHXAD+V ztx&gRIvj;wY_om!#zX0)jBO7{y)Uob3i=7C{UHL zVLFG~IYTe;Ak7E*>cM`XnK*A~-o}^k+J4zQg}NF+I>p!n%9KIk=m;vK;}yP}USH~? zkQBM?*}aF`hqSEYT+a2V*RKU%9NTTo(V$Qlg3KIwR)gAKSZ^#(>kJ+s=v*&rT+yFK z9P86u4pVNCmTNyIO6@PCJH{XEat5h4$B%AV5P^M-9&jUPJib-U%AlhlFHksi=k4)N zpk!X+bue#FIuDG&nA(G;!oV2DzyOu~2L2F%Ww2sMbOh=a4ABOEk_O*y?We$oK{^d8%UFzbi|${u~x0pAq&jPdso+n;kWxhtb3*WS$N zY-L~cPH)q^CP&6r>6nd_qvo|EH?ZB z?se0P1-DS@x`9TaE(wTGkwJAZ)uxXM^!kj;II{Ff}G8pn{vV@ul%<9E(lIjFOZ60>`&kaNL=Z7b?#J$i7XN(hNE( zZ9|r7ZUD%W16+Yy)kRu|+aLIeh70o{@^yJzhys(u5tGPEiPP-cIY|e%{*`_EZ7{bm z`y}_xk~Z4AhuG~tHfrLU2sf1xunw*z-vb^Eeje3S)E*JmMR`Cu+{PTR zN?WB8-c)pSt}~1Tv2R)HW$ z(~B{9iW%{ejySS?eO*XAIW{^Cre8f68YIA7H@)tEIL)+LX|hVDEHB2Yq8(@*evD}@ z#Vk@p>kEf+j%|8B%6-UF*KL2-AT8Q7VXVaMnIgdaBX=_E|a zECobK?N}1l;tvn0Ly$66{Thz#L%XV6Ly4&c+l(P(?aX69!Md6(5Qu3s6o$f{%pDmu zf|cJH{VSLs5kdyeB_(^;u>Lh

tablesToMerge = getTablesToMerge(); @@ -270,9 +276,18 @@ public void jobLogic() throws IOException, CheckedAWSException { LOG.error("Merge {} table failed!", table.name); } } - } - for (FeedToMerge feed : feedsToMerge) { - feed.close(); + } catch (IOException e) { + String message = "Error creating output stream for feed merge."; + logAndReportToBugsnag(e, message); + status.fail(message, e); + } finally { + for (FeedToMerge feed : feedsToMerge) { + try { + feed.close(); + } catch (IOException e) { + logAndReportToBugsnag(e, "Error closing FeedToMerge object"); + } + } } if (!mergeFeedsResult.failed) { // Store feed locally and (if applicable) upload regional feed to S3. @@ -347,14 +362,13 @@ public void failMergeJob(String failureMessage) { * Handles writing the GTFS zip file to disk. For REGIONAL merges, this will end up in a project subdirectory on s3. * Otherwise, it will write to a new version. */ - private void storeMergedFeed() throws IOException, CheckedAWSException { + private void storeMergedFeed() { if (mergedVersion != null) { // Store the zip file for the merged feed version. try { mergedVersion.newGtfsFile(new FileInputStream(mergedTempFile)); } catch (IOException e) { - LOG.error("Could not store merged feed for new version", e); - throw e; + logAndReportToBugsnag(e, "Could not store merged feed for new version"); } } // Write the new latest regional merge file to s3://$BUCKET/project/$PROJECT_ID.zip @@ -363,14 +377,21 @@ private void storeMergedFeed() throws IOException, CheckedAWSException { // Store the project merged zip locally or on s3 if (DataManager.useS3) { String s3Key = String.join("/", "project", filename); - S3Utils.getDefaultS3Client().putObject(S3Utils.DEFAULT_BUCKET, s3Key, mergedTempFile); + try { + S3Utils.getDefaultS3Client().putObject(S3Utils.DEFAULT_BUCKET, s3Key, mergedTempFile); + } catch (CheckedAWSException e) { + String message = "Could not upload store merged feed for new version"; + logAndReportToBugsnag(e, message); + status.fail(message, e); + } LOG.info("Storing merged project feed at {}", S3Utils.getDefaultBucketUriForKey(s3Key)); } else { try { FeedVersion.feedStore.newFeed(filename, new FileInputStream(mergedTempFile), null); } catch (IOException e) { - LOG.error("Could not store feed for project " + filename, e); - throw e; + String message = "Could not store feed for project " + filename; + logAndReportToBugsnag(e, message); + status.fail(message, e); } } } @@ -384,10 +405,11 @@ private void storeMergedFeed() throws IOException, CheckedAWSException { * @param out output stream to write table into * @return number of lines in merged table */ - private int constructMergedTable(Table table, List feedsToMerge, - ZipOutputStream out) throws IOException { - MergeLineContext ctx = new MergeLineContext(this, table, out); + private int constructMergedTable(Table table, List feedsToMerge, ZipOutputStream out) { + MergeLineContext ctx = null; try { + ctx = new MergeLineContext(this, table, out); + // Iterate over each zip file. For service period merge, the first feed is the future GTFS. for (int feedIndex = 0; feedIndex < feedsToMerge.size(); feedIndex++) { ctx.startNewFeed(feedIndex); @@ -400,16 +422,20 @@ private int constructMergedTable(Table table, List feedsToMerge, } } ctx.flushAndClose(); - } catch (Exception e) { + } catch (IOException e) { List versionNames = feedVersions.stream() .map(version -> version.parentFeedSource().name) .collect(Collectors.toList()); - LOG.error("Error merging feed sources: {}", versionNames, e); - throw e; + String message = "Error merging feed sources: " + versionNames; + logAndReportToBugsnag(e, message); + status.fail(message, e); + } + if (ctx != null) { + // Track the number of lines in the merged table and return final number. + mergeFeedsResult.linesPerTable.put(table.name, ctx.mergedLineNumber); + return ctx.mergedLineNumber; } - // Track the number of lines in the merged table and return final number. - mergeFeedsResult.linesPerTable.put(table.name, ctx.mergedLineNumber); - return ctx.mergedLineNumber; + return 0; } /** @@ -513,4 +539,9 @@ private boolean compareStopTimes(String tripId, Feed futureFeed, Feed activeFeed public String getFeedSourceId() { return feedSource.id; } + + private void logAndReportToBugsnag(Exception e, String message, Object... args) { + LOG.error(message, args, e); + ErrorUtils.reportToBugsnag(e, "datatools", message, owner); + } } diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/MonitorServerStatusJob.java b/src/main/java/com/conveyal/datatools/manager/jobs/MonitorServerStatusJob.java index 41e6f4f29..13a828e84 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/MonitorServerStatusJob.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/MonitorServerStatusJob.java @@ -325,12 +325,11 @@ private boolean checkForOtpRunnerCompletion(String url) { // if the otp-runner status file contains an error message, fail the job if (otpRunnerStatus.error) { // report to bugsnag if configured - Map debuggingMessages = new HashMap<>(); - debuggingMessages.put("otp-runner message", otpRunnerStatus.message); ErrorUtils.reportToBugsnag( new RuntimeException("otp-runner reported an error"), - debuggingMessages, - this.owner + "otp-runner", + otpRunnerStatus.message, + owner ); failJob(otpRunnerStatus.message); return false; diff --git a/src/main/java/com/conveyal/datatools/manager/utils/ErrorUtils.java b/src/main/java/com/conveyal/datatools/manager/utils/ErrorUtils.java index c0324ff48..19eba4ee3 100644 --- a/src/main/java/com/conveyal/datatools/manager/utils/ErrorUtils.java +++ b/src/main/java/com/conveyal/datatools/manager/utils/ErrorUtils.java @@ -7,6 +7,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.HashMap; import java.util.Map; /** @@ -32,6 +33,21 @@ public static void reportToBugsnag(Throwable e, Auth0UserProfile userProfile) { reportToBugsnag(e, null, userProfile); } + /** + * Log an error, and create and send a report to bugsnag if configured. + * + * @param e The throwable object to send to Bugsnag. This MUST be provided or a report will not be generated. + * @param sourceApp The application generating the message (datatools, otp-runner, ...). + * @param message The message to log and to send to Bugsnag. + * @param userProfile An optional user profile. If provided, the email address from this profile will be set in the + * Bugsnag report. + */ + public static void reportToBugsnag(Throwable e, String sourceApp, String message, Auth0UserProfile userProfile) { + Map debuggingMessages = new HashMap<>(); + debuggingMessages.put(sourceApp + " message", message); + reportToBugsnag(e, debuggingMessages, userProfile); + } + /** * Create and send a report to bugsnag if configured. * diff --git a/src/main/java/com/conveyal/datatools/manager/utils/MergeFeedUtils.java b/src/main/java/com/conveyal/datatools/manager/utils/MergeFeedUtils.java index b9de7ccaf..fc36e8494 100644 --- a/src/main/java/com/conveyal/datatools/manager/utils/MergeFeedUtils.java +++ b/src/main/java/com/conveyal/datatools/manager/utils/MergeFeedUtils.java @@ -1,6 +1,7 @@ package com.conveyal.datatools.manager.utils; import com.conveyal.datatools.manager.DataManager; +import com.conveyal.datatools.manager.auth.Auth0UserProfile; import com.conveyal.datatools.manager.jobs.FeedToMerge; import com.conveyal.datatools.manager.jobs.MergeFeedsJob; import com.conveyal.datatools.manager.jobs.MergeFeedsType; @@ -79,13 +80,14 @@ public static String stopCodeFailureMessage(int stopsMissingStopCodeCount, int s * Note: feed versions are sorted by first calendar date so that future dataset is iterated over first. This is * required for the MTC merge strategy which prefers entities from the future dataset over active feed entities. */ - public static List collectAndSortFeeds(Set feedVersions) { + public static List collectAndSortFeeds(Set feedVersions, Auth0UserProfile owner) { return feedVersions.stream() .map(version -> { try { return new FeedToMerge(version); } catch (Exception e) { LOG.error("Could not create zip file for version: {}", version.version); + ErrorUtils.reportToBugsnag(e, owner); return null; } }) From 5364160d72702d1349443320772510a38076ff33 Mon Sep 17 00:00:00 2001 From: binh-dam-ibigroup <56846598+binh-dam-ibigroup@users.noreply.github.com> Date: Thu, 4 Nov 2021 18:41:16 -0400 Subject: [PATCH 048/122] refactor(MergeFeedsJob): Prep for subclass refactor per PR comment. --- .../api/FeedVersionController.java | 4 +- .../controllers/api/ProjectController.java | 3 +- .../datatools/manager/jobs/MergeFeedsJob.java | 13 ++- .../jobs/feedmerge/AgencyLineContext.java | 43 +++++++ .../jobs/{ => feedmerge}/FeedToMerge.java | 2 +- .../{ => feedmerge}/MergeFeedsResult.java | 4 +- .../jobs/{ => feedmerge}/MergeFeedsType.java | 4 +- .../{ => feedmerge}/MergeLineContext.java | 109 +++++++++++------- .../jobs/{ => feedmerge}/MergeStrategy.java | 2 +- .../manager/utils/MergeFeedUtils.java | 8 +- .../manager/jobs/MergeFeedsJobTest.java | 2 + 11 files changed, 136 insertions(+), 58 deletions(-) create mode 100644 src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/AgencyLineContext.java rename src/main/java/com/conveyal/datatools/manager/jobs/{ => feedmerge}/FeedToMerge.java (96%) rename src/main/java/com/conveyal/datatools/manager/jobs/{ => feedmerge}/MergeFeedsResult.java (93%) rename src/main/java/com/conveyal/datatools/manager/jobs/{ => feedmerge}/MergeFeedsType.java (82%) rename src/main/java/com/conveyal/datatools/manager/jobs/{ => feedmerge}/MergeLineContext.java (93%) rename src/main/java/com/conveyal/datatools/manager/jobs/{ => feedmerge}/MergeStrategy.java (98%) diff --git a/src/main/java/com/conveyal/datatools/manager/controllers/api/FeedVersionController.java b/src/main/java/com/conveyal/datatools/manager/controllers/api/FeedVersionController.java index 73d22c009..671caa484 100644 --- a/src/main/java/com/conveyal/datatools/manager/controllers/api/FeedVersionController.java +++ b/src/main/java/com/conveyal/datatools/manager/controllers/api/FeedVersionController.java @@ -8,7 +8,7 @@ import com.conveyal.datatools.manager.jobs.CreateFeedVersionFromSnapshotJob; import com.conveyal.datatools.manager.jobs.GisExportJob; import com.conveyal.datatools.manager.jobs.MergeFeedsJob; -import com.conveyal.datatools.manager.jobs.MergeFeedsType; +import com.conveyal.datatools.manager.jobs.feedmerge.MergeFeedsType; import com.conveyal.datatools.manager.jobs.ProcessSingleFeedJob; import com.conveyal.datatools.manager.models.FeedDownloadToken; import com.conveyal.datatools.manager.models.FeedRetrievalMethod; @@ -44,7 +44,7 @@ import static com.conveyal.datatools.common.utils.SparkUtils.logMessageAndHalt; import static com.conveyal.datatools.manager.controllers.api.FeedSourceController.checkFeedSourcePermissions; import static com.mongodb.client.model.Filters.eq; -import static com.conveyal.datatools.manager.jobs.MergeFeedsType.REGIONAL; +import static com.conveyal.datatools.manager.jobs.feedmerge.MergeFeedsType.REGIONAL; import static com.mongodb.client.model.Filters.in; import static spark.Spark.delete; import static spark.Spark.get; diff --git a/src/main/java/com/conveyal/datatools/manager/controllers/api/ProjectController.java b/src/main/java/com/conveyal/datatools/manager/controllers/api/ProjectController.java index 193720fbb..d237b753e 100644 --- a/src/main/java/com/conveyal/datatools/manager/controllers/api/ProjectController.java +++ b/src/main/java/com/conveyal/datatools/manager/controllers/api/ProjectController.java @@ -25,7 +25,6 @@ import spark.Request; import spark.Response; -import java.util.ArrayList; import java.util.Collection; import java.util.HashSet; import java.util.Set; @@ -35,7 +34,7 @@ import static com.conveyal.datatools.common.utils.SparkUtils.formatJobMessage; import static com.conveyal.datatools.common.utils.SparkUtils.logMessageAndHalt; import static com.conveyal.datatools.manager.DataManager.publicPath; -import static com.conveyal.datatools.manager.jobs.MergeFeedsType.REGIONAL; +import static com.conveyal.datatools.manager.jobs.feedmerge.MergeFeedsType.REGIONAL; import static spark.Spark.delete; import static spark.Spark.get; import static spark.Spark.post; diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java b/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java index b7f15e222..956dd72c1 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java @@ -6,6 +6,11 @@ import com.conveyal.datatools.manager.DataManager; import com.conveyal.datatools.manager.auth.Auth0UserProfile; import com.conveyal.datatools.manager.gtfsplus.tables.GtfsPlusTable; +import com.conveyal.datatools.manager.jobs.feedmerge.FeedToMerge; +import com.conveyal.datatools.manager.jobs.feedmerge.MergeFeedsResult; +import com.conveyal.datatools.manager.jobs.feedmerge.MergeFeedsType; +import com.conveyal.datatools.manager.jobs.feedmerge.MergeLineContext; +import com.conveyal.datatools.manager.jobs.feedmerge.MergeStrategy; import com.conveyal.datatools.manager.models.FeedSource; import com.conveyal.datatools.manager.models.FeedVersion; import com.conveyal.datatools.manager.models.Project; @@ -34,10 +39,10 @@ import java.util.stream.Collectors; import java.util.zip.ZipOutputStream; -import static com.conveyal.datatools.manager.jobs.MergeFeedsType.SERVICE_PERIOD; -import static com.conveyal.datatools.manager.jobs.MergeFeedsType.REGIONAL; -import static com.conveyal.datatools.manager.jobs.MergeStrategy.CHECK_STOP_TIMES; -import static com.conveyal.datatools.manager.jobs.MergeStrategy.EXTEND_FUTURE; +import static com.conveyal.datatools.manager.jobs.feedmerge.MergeFeedsType.SERVICE_PERIOD; +import static com.conveyal.datatools.manager.jobs.feedmerge.MergeFeedsType.REGIONAL; +import static com.conveyal.datatools.manager.jobs.feedmerge.MergeStrategy.CHECK_STOP_TIMES; +import static com.conveyal.datatools.manager.jobs.feedmerge.MergeStrategy.EXTEND_FUTURE; import static com.conveyal.datatools.manager.models.FeedRetrievalMethod.REGIONAL_MERGE; import static com.conveyal.datatools.manager.utils.MergeFeedUtils.*; diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/AgencyLineContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/AgencyLineContext.java new file mode 100644 index 000000000..d8bc2d956 --- /dev/null +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/AgencyLineContext.java @@ -0,0 +1,43 @@ +package com.conveyal.datatools.manager.jobs.feedmerge; + +import com.conveyal.datatools.manager.jobs.MergeFeedsJob; +import com.conveyal.gtfs.loader.Field; +import com.conveyal.gtfs.loader.Table; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.UUID; +import java.util.zip.ZipOutputStream; + +public class AgencyLineContext extends MergeLineContext { + public AgencyLineContext(MergeFeedsJob job, Table table, ZipOutputStream out) throws IOException { + super(job, table, out); + } + + // @Override + public void checkFirstLineConditions() { + checkForMissingAgencyId(); + } + + private void checkForMissingAgencyId() { + /* + if ((keyFieldMissing || keyValue.equals(""))) { + // agency_id is optional if only one agency is present, but that will + // cause issues for the feed merge, so we need to insert an agency_id + // for the single entry. + newAgencyId = UUID.randomUUID().toString(); + if (keyFieldMissing) { + // Only add agency_id field if it is missing in table. + List fieldsList = new ArrayList<>(Arrays.asList(fieldsFoundInZip)); + fieldsList.add(Table.AGENCY.fields[0]); + fieldsFoundInZip = fieldsList.toArray(fieldsFoundInZip); + allFields.add(Table.AGENCY.fields[0]); + } + fieldsFoundList = Arrays.asList(fieldsFoundInZip); + } + + */ + } +} \ No newline at end of file diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/FeedToMerge.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/FeedToMerge.java similarity index 96% rename from src/main/java/com/conveyal/datatools/manager/jobs/FeedToMerge.java rename to src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/FeedToMerge.java index ee9fde233..c703c1c26 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/FeedToMerge.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/FeedToMerge.java @@ -1,4 +1,4 @@ -package com.conveyal.datatools.manager.jobs; +package com.conveyal.datatools.manager.jobs.feedmerge; import com.conveyal.datatools.manager.models.FeedVersion; import com.conveyal.gtfs.loader.Table; diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsResult.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeFeedsResult.java similarity index 93% rename from src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsResult.java rename to src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeFeedsResult.java index d39355d2c..a5a732c69 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsResult.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeFeedsResult.java @@ -1,4 +1,6 @@ -package com.conveyal.datatools.manager.jobs; +package com.conveyal.datatools.manager.jobs.feedmerge; + +import com.conveyal.datatools.manager.jobs.MergeFeedsJob; import java.io.Serializable; import java.util.Date; diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsType.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeFeedsType.java similarity index 82% rename from src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsType.java rename to src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeFeedsType.java index 6c9e61b1c..bbdfc6ccd 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsType.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeFeedsType.java @@ -1,4 +1,6 @@ -package com.conveyal.datatools.manager.jobs; +package com.conveyal.datatools.manager.jobs.feedmerge; + +import com.conveyal.datatools.manager.jobs.MergeFeedsJob; /** * This enum contains the types of merge feeds that {@link MergeFeedsJob} can currently perform. diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/MergeLineContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java similarity index 93% rename from src/main/java/com/conveyal/datatools/manager/jobs/MergeLineContext.java rename to src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java index 051b8c053..3010def86 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/MergeLineContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java @@ -1,5 +1,6 @@ -package com.conveyal.datatools.manager.jobs; +package com.conveyal.datatools.manager.jobs.feedmerge; +import com.conveyal.datatools.manager.jobs.MergeFeedsJob; import com.conveyal.datatools.manager.models.FeedSource; import com.conveyal.datatools.manager.models.FeedVersion; import com.conveyal.gtfs.error.NewGTFSError; @@ -8,6 +9,7 @@ import com.conveyal.gtfs.loader.ReferenceTracker; import com.conveyal.gtfs.loader.Table; import com.csvreader.CsvReader; +import org.apache.commons.lang3.NotImplementedException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.supercsv.io.CsvListWriter; @@ -29,10 +31,10 @@ import java.util.zip.ZipEntry; import java.util.zip.ZipOutputStream; -import static com.conveyal.datatools.manager.jobs.MergeFeedsType.REGIONAL; -import static com.conveyal.datatools.manager.jobs.MergeFeedsType.SERVICE_PERIOD; -import static com.conveyal.datatools.manager.jobs.MergeStrategy.CHECK_STOP_TIMES; -import static com.conveyal.datatools.manager.jobs.MergeStrategy.EXTEND_FUTURE; +import static com.conveyal.datatools.manager.jobs.feedmerge.MergeFeedsType.REGIONAL; +import static com.conveyal.datatools.manager.jobs.feedmerge.MergeFeedsType.SERVICE_PERIOD; +import static com.conveyal.datatools.manager.jobs.feedmerge.MergeStrategy.CHECK_STOP_TIMES; +import static com.conveyal.datatools.manager.jobs.feedmerge.MergeStrategy.EXTEND_FUTURE; import static com.conveyal.datatools.manager.utils.MergeFeedUtils.containsField; import static com.conveyal.datatools.manager.utils.MergeFeedUtils.getAllFields; import static com.conveyal.datatools.manager.utils.MergeFeedUtils.getMergeKeyField; @@ -65,7 +67,7 @@ public class MergeLineContext { private boolean keyFieldMissing; private String[] rowValues; private int lineNumber = 0; - private final Table table; + protected final Table table; private FeedToMerge feed; private String keyValue; private final ReferenceTracker referenceTracker = new ReferenceTracker(); @@ -94,6 +96,30 @@ public class MergeLineContext { public boolean skipFile; public int mergedLineNumber = 0; + public static MergeLineContext create(MergeFeedsJob job, Table table, ZipOutputStream out) throws IOException { + switch (table.name) { + case "agency": + return new AgencyLineContext(job, table, out); + case "calendar": + break; + case "calendar_dates": + break; + case "feed_info": + break; + case "routes": + break; + case "shapes": + break; + case "stops": + break; + case "trips": + break; + default: + throw new IllegalArgumentException(table.name); + } + return null; + } + public MergeLineContext(MergeFeedsJob job, Table table, ZipOutputStream out) throws IOException { this.job = job; this.table = table; @@ -248,16 +274,16 @@ private boolean serviceIdHasOrShouldBeSkipped(String key, boolean isValidService return mergeFeedsResult.skippedIds.contains(key) || serviceIdShouldBeSkipped; } - public boolean checkFieldForMergeConflicts() throws IOException { + public void checkFieldsForMergeConflicts() throws IOException { Set idErrors = getIdErrors(); // Store values for key fields that have been encountered and update any key values that need modification due // to conflicts. switch (table.name) { case "calendar": - if (checkCalendarIds(idErrors)) return true; + checkCalendarIds(idErrors); break; case "calendar_dates": - if (checkCalendarDatesIds()) return true; + checkCalendarDatesIds(); break; case "shapes": checkShapeIds(idErrors); @@ -282,7 +308,6 @@ public boolean checkFieldForMergeConflicts() throws IOException { if (hasDuplicateError(idErrors)) skipRecord = true; break; } - return false; } private Set getIdErrors() { @@ -370,29 +395,27 @@ private void checkShapeIds(Set idErrors) { if (hasDuplicateError(idErrors)) skipRecord = true; } - private boolean checkCalendarDatesIds() throws IOException { + private void checkCalendarDatesIds() throws IOException { // Drop any calendar_dates.txt records from the existing feed for dates that are // not before the first date of the future feed. int dateIndex = getFieldIndex(fieldsFoundInZip, "date"); LocalDate date = LocalDate.parse(csvReader.get(dateIndex), GTFS_DATE_FORMATTER); if (handlingActiveFeed && !date.isBefore(futureFeedFirstDate)) { - LOG.warn( - "Skipping calendar_dates entry {} because it operates in the time span of future feed (i.e., after or on {}).", - keyValue, - futureFeedFirstDate); - String key = getTableScopedValue(table, idScope, keyValue); - mergeFeedsResult.skippedIds.add(key); - skipRecord = true; - return true; + LOG.warn( + "Skipping calendar_dates entry {} because it operates in the time span of future feed (i.e., after or on {}).", + keyValue, + futureFeedFirstDate); + String key = getTableScopedValue(table, idScope, keyValue); + mergeFeedsResult.skippedIds.add(key); + skipRecord = true; } // Track service ID because we want to avoid removing trips that may reference this // service_id when the service_id is used by calendar.txt records that operate in // the valid date range, i.e., before the future feed's first date. - if (field.name.equals(SERVICE_ID)) mergeFeedsResult.serviceIds.add(valueToWrite); - return false; + if (!skipRecord && field.name.equals(SERVICE_ID)) mergeFeedsResult.serviceIds.add(valueToWrite); } - private boolean checkCalendarIds(Set idErrors) throws IOException { + private void checkCalendarIds(Set idErrors) throws IOException { // If any service_id in the active feed matches with the future // feed, it should be modified and all associated trip records // must also be changed with the modified service_id. @@ -438,28 +461,27 @@ private boolean checkCalendarIds(Set idErrors) throws IOException String key = getTableScopedValue(table, idScope, keyValue); mergeFeedsResult.skippedIds.add(key); skipRecord = true; - return true; - } - // If a service_id from the active calendar has only the - // end_date in the future, the end_date shall be set to one - // day prior to the earliest start_date in future dataset - // before appending the calendar record to the merged file. - int endDateIndex = getFieldIndex(fieldsFoundInZip, "end_date"); - if (index == endDateIndex) { - LocalDate endDate = LocalDate - .parse(csvReader.get(endDateIndex), GTFS_DATE_FORMATTER); - if (!endDate.isBefore(futureFeedFirstDate)) { - val = valueToWrite = futureFeedFirstDate - .minus(1, ChronoUnit.DAYS) - .format(GTFS_DATE_FORMATTER); + } else { + // If a service_id from the active calendar has only the + // end_date in the future, the end_date shall be set to one + // day prior to the earliest start_date in future dataset + // before appending the calendar record to the merged file. + int endDateIndex = getFieldIndex(fieldsFoundInZip, "end_date"); + if (index == endDateIndex) { + LocalDate endDate = LocalDate + .parse(csvReader.get(endDateIndex), GTFS_DATE_FORMATTER); + if (!endDate.isBefore(futureFeedFirstDate)) { + val = valueToWrite = futureFeedFirstDate + .minus(1, ChronoUnit.DAYS) + .format(GTFS_DATE_FORMATTER); + } } } } // Track service ID because we want to avoid removing trips that may reference this // service_id when the service_id is used by calendar_dates that operate in the valid // date range, i.e., before the future feed's first date. - if (field.name.equals(SERVICE_ID)) mergeFeedsResult.serviceIds.add(valueToWrite); - return false; + if (!skipRecord && field.name.equals(SERVICE_ID)) mergeFeedsResult.serviceIds.add(valueToWrite); } private boolean shouldUpdateFutureFeedStartDate(int startDateIndex) { @@ -486,7 +508,7 @@ private void checkRoutesAndStopsIds(Set idErrors) throws IOExcepti // route_short_name/stop_code in active data not present in the future will be appended to the // future routes/stops file. if (useAltKey()) { - if (hasPrimaryKeyErrors(primaryKeyErrors)) { + if (hasBlankPrimaryKey()) { // If alt key is empty (which is permitted) and primary key is duplicate, skip // checking of alt key dupe errors/re-mapping values and // simply use the primary key (route_id/stop_id). @@ -555,7 +577,7 @@ private void checkRoutesAndStopsIds(Set idErrors) throws IOExcepti } } - private boolean hasPrimaryKeyErrors(Set primaryKeyErrors) { + private boolean hasBlankPrimaryKey() { return "".equals(keyValue) && field.name.equals(table.getKeyFieldName()); } @@ -847,7 +869,7 @@ public boolean constructRowValues() throws IOException { for (int specFieldIndex = 0; specFieldIndex < sharedSpecFields.size(); specFieldIndex++) { // There is nothing to do in this loop if it has already been determined that the record should // be skipped. - if (skipRecord) continue; + if (skipRecord) break; startNewField(specFieldIndex); // Handle filling in agency_id if missing when merging regional feeds. If false is returned, // the job has encountered a failing condition (the method handles failing the job itself). @@ -861,7 +883,10 @@ public boolean constructRowValues() throws IOException { // reference tracker will get far too large if we attempt to use it to // track references for a large number of feeds (e.g., every feed in New // York State). - if (job.mergeType.equals(SERVICE_PERIOD) && checkFieldForMergeConflicts()) continue; + if (job.mergeType.equals(SERVICE_PERIOD)) { + checkFieldsForMergeConflicts(); + if (skipRecord) continue; + } // If the current field is a foreign reference, check if the reference has been removed in the // merged result. If this is the case (or other conditions are met), we will need to skip this // record. Likewise, if the reference has been modified, ensure that the value written to the diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/MergeStrategy.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeStrategy.java similarity index 98% rename from src/main/java/com/conveyal/datatools/manager/jobs/MergeStrategy.java rename to src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeStrategy.java index c4dbbc6d1..fd26ff2d4 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/MergeStrategy.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeStrategy.java @@ -1,4 +1,4 @@ -package com.conveyal.datatools.manager.jobs; +package com.conveyal.datatools.manager.jobs.feedmerge; /** * This enum defines the different strategies for merging, which is currently dependent on whether trip_ids and/or diff --git a/src/main/java/com/conveyal/datatools/manager/utils/MergeFeedUtils.java b/src/main/java/com/conveyal/datatools/manager/utils/MergeFeedUtils.java index fc36e8494..fa8fe6252 100644 --- a/src/main/java/com/conveyal/datatools/manager/utils/MergeFeedUtils.java +++ b/src/main/java/com/conveyal/datatools/manager/utils/MergeFeedUtils.java @@ -2,9 +2,9 @@ import com.conveyal.datatools.manager.DataManager; import com.conveyal.datatools.manager.auth.Auth0UserProfile; -import com.conveyal.datatools.manager.jobs.FeedToMerge; +import com.conveyal.datatools.manager.jobs.feedmerge.FeedToMerge; import com.conveyal.datatools.manager.jobs.MergeFeedsJob; -import com.conveyal.datatools.manager.jobs.MergeFeedsType; +import com.conveyal.datatools.manager.jobs.feedmerge.MergeFeedsType; import com.conveyal.datatools.manager.models.FeedRetrievalMethod; import com.conveyal.datatools.manager.models.FeedSource; import com.conveyal.datatools.manager.models.FeedVersion; @@ -30,8 +30,8 @@ import java.util.stream.Collectors; import java.util.zip.ZipFile; -import static com.conveyal.datatools.manager.jobs.MergeFeedsType.REGIONAL; -import static com.conveyal.datatools.manager.jobs.MergeFeedsType.SERVICE_PERIOD; +import static com.conveyal.datatools.manager.jobs.feedmerge.MergeFeedsType.REGIONAL; +import static com.conveyal.datatools.manager.jobs.feedmerge.MergeFeedsType.SERVICE_PERIOD; import static com.conveyal.datatools.manager.models.FeedRetrievalMethod.REGIONAL_MERGE; import static com.conveyal.datatools.manager.models.FeedRetrievalMethod.SERVICE_PERIOD_MERGE; import static com.conveyal.gtfs.loader.Field.getFieldIndex; diff --git a/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java b/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java index 9ba974580..deb27564d 100644 --- a/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java +++ b/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java @@ -3,6 +3,8 @@ import com.conveyal.datatools.DatatoolsTest; import com.conveyal.datatools.UnitTest; import com.conveyal.datatools.manager.auth.Auth0UserProfile; +import com.conveyal.datatools.manager.jobs.feedmerge.MergeFeedsType; +import com.conveyal.datatools.manager.jobs.feedmerge.MergeStrategy; import com.conveyal.datatools.manager.models.FeedSource; import com.conveyal.datatools.manager.models.FeedVersion; import com.conveyal.datatools.manager.models.Project; From 8108d6eb06cf94cba045f1803dcbd021ec017060 Mon Sep 17 00:00:00 2001 From: binh-dam-ibigroup <56846598+binh-dam-ibigroup@users.noreply.github.com> Date: Fri, 5 Nov 2021 09:59:18 -0400 Subject: [PATCH 049/122] refactor(MergeLineContext): Create subclasses for checking first line by table. --- .../datatools/manager/jobs/MergeFeedsJob.java | 2 +- ...ntext.java => AgencyMergeLineContext.java} | 9 +- .../jobs/feedmerge/MergeLineContext.java | 168 +++++------------- .../jobs/feedmerge/StopsMergeLineContext.java | 100 +++++++++++ 4 files changed, 148 insertions(+), 131 deletions(-) rename src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/{AgencyLineContext.java => AgencyMergeLineContext.java} (86%) create mode 100644 src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/StopsMergeLineContext.java diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java b/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java index 956dd72c1..53f618715 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java @@ -413,7 +413,7 @@ private void storeMergedFeed() { private int constructMergedTable(Table table, List feedsToMerge, ZipOutputStream out) { MergeLineContext ctx = null; try { - ctx = new MergeLineContext(this, table, out); + ctx = MergeLineContext.create(this, table, out); // Iterate over each zip file. For service period merge, the first feed is the future GTFS. for (int feedIndex = 0; feedIndex < feedsToMerge.size(); feedIndex++) { diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/AgencyLineContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/AgencyMergeLineContext.java similarity index 86% rename from src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/AgencyLineContext.java rename to src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/AgencyMergeLineContext.java index d8bc2d956..20958c624 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/AgencyLineContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/AgencyMergeLineContext.java @@ -11,18 +11,17 @@ import java.util.UUID; import java.util.zip.ZipOutputStream; -public class AgencyLineContext extends MergeLineContext { - public AgencyLineContext(MergeFeedsJob job, Table table, ZipOutputStream out) throws IOException { +public class AgencyMergeLineContext extends MergeLineContext { + public AgencyMergeLineContext(MergeFeedsJob job, Table table, ZipOutputStream out) throws IOException { super(job, table, out); } - // @Override + @Override public void checkFirstLineConditions() { checkForMissingAgencyId(); } private void checkForMissingAgencyId() { - /* if ((keyFieldMissing || keyValue.equals(""))) { // agency_id is optional if only one agency is present, but that will // cause issues for the feed merge, so we need to insert an agency_id @@ -37,7 +36,5 @@ private void checkForMissingAgencyId() { } fieldsFoundList = Arrays.asList(fieldsFoundInZip); } - - */ } } \ No newline at end of file diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java index 3010def86..1d601b720 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java @@ -9,7 +9,6 @@ import com.conveyal.gtfs.loader.ReferenceTracker; import com.conveyal.gtfs.loader.Table; import com.csvreader.CsvReader; -import org.apache.commons.lang3.NotImplementedException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.supercsv.io.CsvListWriter; @@ -19,14 +18,12 @@ import java.io.OutputStreamWriter; import java.time.LocalDate; import java.time.temporal.ChronoUnit; -import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; -import java.util.UUID; import java.util.stream.Collectors; import java.util.zip.ZipEntry; import java.util.zip.ZipOutputStream; @@ -40,19 +37,17 @@ import static com.conveyal.datatools.manager.utils.MergeFeedUtils.getMergeKeyField; import static com.conveyal.datatools.manager.utils.MergeFeedUtils.getTableScopedValue; import static com.conveyal.datatools.manager.utils.MergeFeedUtils.hasDuplicateError; -import static com.conveyal.datatools.manager.utils.MergeFeedUtils.stopCodeFailureMessage; import static com.conveyal.datatools.manager.utils.StringUtils.getCleanName; import static com.conveyal.gtfs.loader.DateField.GTFS_DATE_FORMATTER; -import static com.conveyal.gtfs.loader.Field.getFieldIndex; public class MergeLineContext { private static final String AGENCY_ID = "agency_id"; private static final String SERVICE_ID = "service_id"; private static final String STOPS = "stops"; private static final Logger LOG = LoggerFactory.getLogger(MergeLineContext.class); - private final MergeFeedsJob job; + protected final MergeFeedsJob job; private final ZipOutputStream out; - private final Set allFields; + protected final Set allFields; private LocalDate futureFirstCalendarStartDate; private final LocalDate activeFeedFirstDate; private LocalDate futureFeedFirstDate; @@ -63,33 +58,32 @@ public class MergeLineContext { private final CsvListWriter writer; private CsvReader csvReader; private boolean skipRecord; - private String newAgencyId; - private boolean keyFieldMissing; + protected String newAgencyId; // move + protected boolean keyFieldMissing; private String[] rowValues; private int lineNumber = 0; protected final Table table; - private FeedToMerge feed; - private String keyValue; + protected FeedToMerge feed; + protected String keyValue; private final ReferenceTracker referenceTracker = new ReferenceTracker(); - private String keyField; + protected String keyField; private String orderField; private final MergeFeedsResult mergeFeedsResult; - private final List feedsToMerge; - private int keyFieldIndex; - private Field[] fieldsFoundInZip; - private List fieldsFoundList; + protected final List feedsToMerge; + protected int keyFieldIndex; + protected Field[] fieldsFoundInZip; // try to make private + protected List fieldsFoundList; private Field field; // Set up objects for tracking the rows encountered private final Map rowValuesForStopOrRouteId = new HashMap<>(); private final Set rowStrings = new HashSet<>(); - private boolean stopCodeMissingFromFutureFeed = false; // Track shape_ids found in future feed in order to check for conflicts with active feed (MTC only). private final Set shapeIdsInFutureFeed = new HashSet<>(); private List sharedSpecFields; private int index; private String val; private String valueToWrite; - private int feedIndex; + protected int feedIndex; public FeedVersion version; public FeedSource feedSource; @@ -99,7 +93,10 @@ public class MergeLineContext { public static MergeLineContext create(MergeFeedsJob job, Table table, ZipOutputStream out) throws IOException { switch (table.name) { case "agency": - return new AgencyLineContext(job, table, out); + return new AgencyMergeLineContext(job, table, out); + case STOPS: + return new StopsMergeLineContext(job, table, out); +/* case "calendar": break; case "calendar_dates": @@ -114,13 +111,15 @@ public static MergeLineContext create(MergeFeedsJob job, Table table, ZipOutputS break; case "trips": break; +*/ default: - throw new IllegalArgumentException(table.name); + return new MergeLineContext(job, table, out); + //throw new IllegalArgumentException(table.name); } - return null; + //return null; } - public MergeLineContext(MergeFeedsJob job, Table table, ZipOutputStream out) throws IOException { + protected MergeLineContext(MergeFeedsJob job, Table table, ZipOutputStream out) throws IOException { this.job = job; this.table = table; this.feedsToMerge = job.getFeedsToMerge(); @@ -172,7 +171,7 @@ public void startNewFeed(int feedIndex) throws IOException { fieldsFoundInZip = table.getFieldsFromFieldHeaders(csvReader.getHeaders(), null); fieldsFoundList = Arrays.asList(fieldsFoundInZip); // Determine the index of the key field for this version's table. - keyFieldIndex = getFieldIndex(fieldsFoundInZip, keyField); + keyFieldIndex = getFieldIndex(keyField); if (keyFieldIndex == -1) { LOG.error("No {} field exists for {} table (feed={})", keyField, table.name, version.id); keyFieldMissing = true; @@ -204,10 +203,12 @@ public boolean iterateOverRows() throws IOException { } updateFutureFeedFirstDate(); // If checkMismatchedAgency flagged skipFile, loop back to the while loop. (Note: this is - // intentional because we want to check all of the agency ids in the file). + // intentional because we want to check all agency ids in the file). if (skipFile || lineIsBlank()) continue; // Check certain initial conditions on the first line of the file. - checkFirstLineConditions(); + if (lineNumber == 0) { + checkFirstLineConditions(); + } initializeRowValues(); // Construct row values. If a failure condition was encountered, return. if (!constructRowValues()) { @@ -249,7 +250,7 @@ public boolean areForeignRefsOk() throws IOException { String skippedKey = getTableScopedValue(table, idScope, keyValue); if (orderField != null) { skippedKey = String.join(":", skippedKey, - csvReader.get(getFieldIndex(fieldsFoundInZip, orderField))); + csvReader.get(getFieldIndex(orderField))); } mergeFeedsResult.skippedIds.add(skippedKey); skipRecord = true; @@ -398,7 +399,7 @@ private void checkShapeIds(Set idErrors) { private void checkCalendarDatesIds() throws IOException { // Drop any calendar_dates.txt records from the existing feed for dates that are // not before the first date of the future feed. - int dateIndex = getFieldIndex(fieldsFoundInZip, "date"); + int dateIndex = getFieldIndex("date"); LocalDate date = LocalDate.parse(csvReader.get(dateIndex), GTFS_DATE_FORMATTER); if (handlingActiveFeed && !date.isBefore(futureFeedFirstDate)) { LOG.warn( @@ -430,7 +431,7 @@ private void checkCalendarIds(Set idErrors) throws IOException { valueToWrite = String.join(":", idScope, val); mergeFeedsResult.remappedIds.put(key, valueToWrite); } - int startDateIndex = getFieldIndex(fieldsFoundInZip, "start_date"); + int startDateIndex = getFieldIndex("start_date"); LocalDate startDate = LocalDate.parse(csvReader.get(startDateIndex), GTFS_DATE_FORMATTER); if (handlingFutureFeed) { // For the future feed, check if the calendar's start date is earlier than the @@ -466,7 +467,7 @@ private void checkCalendarIds(Set idErrors) throws IOException { // end_date in the future, the end_date shall be set to one // day prior to the earliest start_date in future dataset // before appending the calendar record to the merged file. - int endDateIndex = getFieldIndex(fieldsFoundInZip, "end_date"); + int endDateIndex = getFieldIndex("end_date"); if (index == endDateIndex) { LocalDate endDate = LocalDate .parse(csvReader.get(endDateIndex), GTFS_DATE_FORMATTER); @@ -585,94 +586,6 @@ private boolean useAltKey() { return keyField.equals("stop_code") || keyField.equals("route_short_name"); } - public void checkMissingAgencyId() { - if (table.name.equals(Table.AGENCY.name) && (keyFieldMissing || keyValue.equals(""))) { - // agency_id is optional if only one agency is present, but that will - // cause issues for the feed merge, so we need to insert an agency_id - // for the single entry. - newAgencyId = UUID.randomUUID().toString(); - if (keyFieldMissing) { - // Only add agency_id field if it is missing in table. - List fieldsList = new ArrayList<>(Arrays.asList(fieldsFoundInZip)); - fieldsList.add(Table.AGENCY.fields[0]); - fieldsFoundInZip = fieldsList.toArray(fieldsFoundInZip); - allFields.add(Table.AGENCY.fields[0]); - } - fieldsFoundList = Arrays.asList(fieldsFoundInZip); - } - } - - public void checkStopCodeStuff() throws IOException { - if (shouldCheckStopCodes()) { - // Before reading any lines in stops.txt, first determine whether all records contain - // properly filled stop_codes. The rules governing this logic are as follows: - // 1. Stops with location_type greater than 0 (i.e., anything but 0 or empty) are permitted - // to have empty stop_codes (even if there are other stops in the feed that have - // stop_code values). This is because these location_types represent special entries - // that are either stations, entrances/exits, or generic nodes (e.g., for - // pathways.txt). - // 2. For regular stops (location_type = 0 or empty), all or none of the stops must - // contain stop_codes. Otherwise, the merge feeds job will be failed. - int stopsMissingStopCodeCount = 0; - int stopsCount = 0; - int specialStopsCount = 0; - int locationTypeIndex = getFieldIndex(fieldsFoundInZip, "location_type"); - int stopCodeIndex = getFieldIndex(fieldsFoundInZip, "stop_code"); - // Get special stops reader to iterate over every stop and determine if stop_code values - // are present. - CsvReader stopsReader = table.getCsvReader(feed.zipFile, null); - while (stopsReader.readRecord()) { - stopsCount++; - // Special stop records (i.e., a station, entrance, or anything with - // location_type > 0) do not need to specify stop_code. Other stops should. - String stopCode = stopsReader.get(stopCodeIndex); - boolean stopCodeIsMissing = "".equals(stopCode); - String locationType = stopsReader.get(locationTypeIndex); - if (isSpecialStop(locationType)) specialStopsCount++; - else if (stopCodeIsMissing) stopsMissingStopCodeCount++; - } - stopsReader.close(); - LOG.info("total stops: {}", stopsCount); - LOG.info("stops missing stop_code: {}", stopsMissingStopCodeCount); - if (stopsMissingStopCodeCount + specialStopsCount == stopsCount) { - // If all stops are missing stop_code (taking into account the special stops that do - // not require stop_code), we simply default to merging on stop_id. - LOG.warn( - "stop_code is not present in file {}/{}. Reverting to stop_id", - feedIndex + 1, feedsToMerge.size()); - // If the key value for stop_code is not present, revert to stop_id. - keyField = table.getKeyFieldName(); - keyFieldIndex = table.getKeyFieldIndex(fieldsFoundInZip); - keyValue = csvReader.get(keyFieldIndex); - // When all stops missing stop_code for the first feed, there's nothing to do (i.e., - // no failure condition has been triggered yet). Just indicate this in the flag and - // proceed with the merge. - if (handlingFutureFeed) stopCodeMissingFromFutureFeed = true; - // However... if the second feed was missing stop_codes and the first feed was not, - // fail the merge job. - else if (!stopCodeMissingFromFutureFeed) { - job.failMergeJob( - stopCodeFailureMessage(stopsMissingStopCodeCount, stopsCount, specialStopsCount) - ); - } - } else if (stopsMissingStopCodeCount > 0) { - // If some, but not all, stops are missing stop_code, the merge feeds job must fail. - job.failMergeJob( - stopCodeFailureMessage(stopsMissingStopCodeCount, stopsCount, specialStopsCount) - ); - } - } - } - - private boolean shouldCheckStopCodes() { - return job.mergeType.equals(SERVICE_PERIOD) && table.name.equals(STOPS); - } - - /** Determine if stop is "special" via its locationType. I.e., a station, entrance, (location_type > 0). */ - private boolean isSpecialStop(String locationType) { - return !"".equals(locationType) && !"0".equals(locationType); - } - public boolean updateAgencyIdIfNeeded() { if (newAgencyId != null && field.name.equals(AGENCY_ID) && job.mergeType.equals(REGIONAL)) { if (val.equals("") && table.name.equals("agency") && lineNumber > 0) { @@ -717,7 +630,7 @@ public boolean storeRowAndStopValues() { // defined above, we will be using the found fields index, which will // cause major issues when trying to put and get values into the // below map. - int fieldIndex = getFieldIndex(sharedSpecFields.toArray(new Field[0]), keyField); + int fieldIndex = Field.getFieldIndex(sharedSpecFields.toArray(new Field[0]), keyField); String key = String.join(":", keyField, rowValues[fieldIndex]); rowValuesForStopOrRouteId.put(key, rowValues); break; @@ -743,12 +656,7 @@ public boolean storeRowAndStopValues() { return false; } - public void checkFirstLineConditions() throws IOException { - if (lineNumber == 0) { - checkMissingAgencyId(); - checkStopCodeStuff(); - } - } + public void checkFirstLineConditions() throws IOException {} /** * Check for some conditions that could occur when handling a service period merge. @@ -927,4 +835,16 @@ public boolean lineIsBlank() throws IOException { } return false; } + + public boolean isHandlingFutureFeed() { + return handlingFutureFeed; + } + + protected CsvReader getCsvReader() { + return csvReader; + } + + protected int getFieldIndex(String fieldName) { + return Field.getFieldIndex(fieldsFoundInZip, fieldName); + } } diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/StopsMergeLineContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/StopsMergeLineContext.java new file mode 100644 index 000000000..5fc39fd0f --- /dev/null +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/StopsMergeLineContext.java @@ -0,0 +1,100 @@ +package com.conveyal.datatools.manager.jobs.feedmerge; + +import com.conveyal.datatools.manager.jobs.MergeFeedsJob; +import com.conveyal.gtfs.loader.Table; +import com.csvreader.CsvReader; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.zip.ZipOutputStream; + +import static com.conveyal.datatools.manager.jobs.feedmerge.MergeFeedsType.SERVICE_PERIOD; +import static com.conveyal.datatools.manager.utils.MergeFeedUtils.stopCodeFailureMessage; + +public class StopsMergeLineContext extends MergeLineContext { + private static final Logger LOG = LoggerFactory.getLogger(StopsMergeLineContext.class); + + private boolean stopCodeMissingFromFutureFeed = false; + + public StopsMergeLineContext(MergeFeedsJob job, Table table, ZipOutputStream out) throws IOException { + super(job, table, out); + } + + @Override + public void checkFirstLineConditions() throws IOException { + checkStopCodeStuff(); + } + + private void checkStopCodeStuff() throws IOException { + if (shouldCheckStopCodes()) { + // Before reading any lines in stops.txt, first determine whether all records contain + // properly filled stop_codes. The rules governing this logic are as follows: + // 1. Stops with location_type greater than 0 (i.e., anything but 0 or empty) are permitted + // to have empty stop_codes (even if there are other stops in the feed that have + // stop_code values). This is because these location_types represent special entries + // that are either stations, entrances/exits, or generic nodes (e.g., for + // pathways.txt). + // 2. For regular stops (location_type = 0 or empty), all or none of the stops must + // contain stop_codes. Otherwise, the merge feeds job will be failed. + int stopsMissingStopCodeCount = 0; + int stopsCount = 0; + int specialStopsCount = 0; + int locationTypeIndex = getFieldIndex("location_type"); + int stopCodeIndex = getFieldIndex("stop_code"); + // Get special stops reader to iterate over every stop and determine if stop_code values + // are present. + CsvReader stopsReader = table.getCsvReader(feed.zipFile, null); + while (stopsReader.readRecord()) { + stopsCount++; + // Special stop records (i.e., a station, entrance, or anything with + // location_type > 0) do not need to specify stop_code. Other stops should. + String stopCode = stopsReader.get(stopCodeIndex); + boolean stopCodeIsMissing = "".equals(stopCode); + String locationType = stopsReader.get(locationTypeIndex); + if (isSpecialStop(locationType)) specialStopsCount++; + else if (stopCodeIsMissing) stopsMissingStopCodeCount++; + } + stopsReader.close(); + LOG.info("total stops: {}", stopsCount); + LOG.info("stops missing stop_code: {}", stopsMissingStopCodeCount); + if (stopsMissingStopCodeCount + specialStopsCount == stopsCount) { + // If all stops are missing stop_code (taking into account the special stops that do + // not require stop_code), we simply default to merging on stop_id. + LOG.warn( + "stop_code is not present in file {}/{}. Reverting to stop_id", + feedIndex + 1, feedsToMerge.size()); + // If the key value for stop_code is not present, revert to stop_id. + keyField = table.getKeyFieldName(); + keyFieldIndex = table.getKeyFieldIndex(fieldsFoundInZip); + keyValue = getCsvReader().get(keyFieldIndex); + // When all stops missing stop_code for the first feed, there's nothing to do (i.e., + // no failure condition has been triggered yet). Just indicate this in the flag and + // proceed with the merge. + if (isHandlingFutureFeed()) { + stopCodeMissingFromFutureFeed = true; + } else if (!stopCodeMissingFromFutureFeed) { + // However... if the second feed was missing stop_codes and the first feed was not, + // fail the merge job. + job.failMergeJob( + stopCodeFailureMessage(stopsMissingStopCodeCount, stopsCount, specialStopsCount) + ); + } + } else if (stopsMissingStopCodeCount > 0) { + // If some, but not all, stops are missing stop_code, the merge feeds job must fail. + job.failMergeJob( + stopCodeFailureMessage(stopsMissingStopCodeCount, stopsCount, specialStopsCount) + ); + } + } + } + + private boolean shouldCheckStopCodes() { + return job.mergeType.equals(SERVICE_PERIOD); + } + + /** Determine if stop is "special" via its locationType. I.e., a station, entrance, (location_type > 0). */ + private boolean isSpecialStop(String locationType) { + return !"".equals(locationType) && !"0".equals(locationType); + } +} \ No newline at end of file From 0ec01886c326fe6881c9c22a4025120ad505580f Mon Sep 17 00:00:00 2001 From: binh-dam-ibigroup <56846598+binh-dam-ibigroup@users.noreply.github.com> Date: Fri, 5 Nov 2021 11:42:57 -0400 Subject: [PATCH 050/122] refactor(MergeLineContext): Move id conflict checks to subclasses. --- .../CalendarDatesMergeLineContext.java | 61 ++++ .../feedmerge/CalendarMergeLineContext.java | 113 +++++++ .../jobs/feedmerge/MergeLineContext.java | 302 +++++------------- .../feedmerge/RoutesMergeLineContext.java | 20 ++ .../feedmerge/ShapesMergeLineContext.java | 59 ++++ .../jobs/feedmerge/StopsMergeLineContext.java | 7 + .../jobs/feedmerge/TripsMergeLineContext.java | 56 ++++ 7 files changed, 388 insertions(+), 230 deletions(-) create mode 100644 src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarDatesMergeLineContext.java create mode 100644 src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarMergeLineContext.java create mode 100644 src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/RoutesMergeLineContext.java create mode 100644 src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/ShapesMergeLineContext.java create mode 100644 src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/TripsMergeLineContext.java diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarDatesMergeLineContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarDatesMergeLineContext.java new file mode 100644 index 000000000..b67381f1c --- /dev/null +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarDatesMergeLineContext.java @@ -0,0 +1,61 @@ +package com.conveyal.datatools.manager.jobs.feedmerge; + +import com.conveyal.datatools.manager.jobs.MergeFeedsJob; +import com.conveyal.gtfs.error.NewGTFSError; +import com.conveyal.gtfs.loader.Table; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.time.LocalDate; +import java.util.Set; +import java.util.zip.ZipOutputStream; + +import static com.conveyal.datatools.manager.jobs.feedmerge.MergeFeedsType.SERVICE_PERIOD; +import static com.conveyal.datatools.manager.utils.MergeFeedUtils.getTableScopedValue; + +public class CalendarDatesMergeLineContext extends MergeLineContext { + private static final Logger LOG = LoggerFactory.getLogger(CalendarDatesMergeLineContext.class); + + public CalendarDatesMergeLineContext(MergeFeedsJob job, Table table, ZipOutputStream out) throws IOException { + super(job, table, out); + } + + @Override + public void checkFieldsForMergeConflicts(Set idErrors) throws IOException { + checkCalendarDatesIds(); + } + + private void checkCalendarDatesIds() throws IOException { + // Drop any calendar_dates.txt records from the existing feed for dates that are + // not before the first date of the future feed. + LocalDate date = getCsvDate("date"); + if (isHandlingActiveFeed() && !date.isBefore(futureFeedFirstDate)) { + LOG.warn( + "Skipping calendar_dates entry {} because it operates in the time span of future feed (i.e., after or on {}).", + keyValue, + futureFeedFirstDate); + String key = getTableScopedValue(table, getIdScope(), keyValue); + mergeFeedsResult.skippedIds.add(key); + skipRecord = true; + } + // Track service ID because we want to avoid removing trips that may reference this + // service_id when the service_id is used by calendar.txt records that operate in + // the valid date range, i.e., before the future feed's first date. + if (!skipRecord && getField().name.equals(SERVICE_ID)) mergeFeedsResult.serviceIds.add(valueToWrite); + } + + private void updateFutureFeedFirstDate_placeHolder() { + if ( + isHandlingActiveFeed() && + job.mergeType.equals(SERVICE_PERIOD) && + futureFirstCalendarStartDate.isBefore(LocalDate.MAX) && + futureFeedFirstDate.isBefore(futureFirstCalendarStartDate) + ) { + // If the future feed's first date is before its first calendar start date, + // override the future feed first date with the calendar start date for use when checking + // MTC calendar_dates and calendar records for modification/exclusion. + futureFeedFirstDate = futureFirstCalendarStartDate; + } + } +} \ No newline at end of file diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarMergeLineContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarMergeLineContext.java new file mode 100644 index 000000000..666574c43 --- /dev/null +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarMergeLineContext.java @@ -0,0 +1,113 @@ +package com.conveyal.datatools.manager.jobs.feedmerge; + +import com.conveyal.datatools.manager.jobs.MergeFeedsJob; +import com.conveyal.gtfs.error.NewGTFSError; +import com.conveyal.gtfs.loader.Table; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.time.LocalDate; +import java.time.temporal.ChronoUnit; +import java.util.Set; +import java.util.zip.ZipOutputStream; + +import static com.conveyal.datatools.manager.jobs.feedmerge.MergeStrategy.CHECK_STOP_TIMES; +import static com.conveyal.datatools.manager.jobs.feedmerge.MergeStrategy.EXTEND_FUTURE; +import static com.conveyal.datatools.manager.utils.MergeFeedUtils.getTableScopedValue; +import static com.conveyal.datatools.manager.utils.MergeFeedUtils.hasDuplicateError; +import static com.conveyal.gtfs.loader.DateField.GTFS_DATE_FORMATTER; + +public class CalendarMergeLineContext extends MergeLineContext { + private static final Logger LOG = LoggerFactory.getLogger(CalendarMergeLineContext.class); + + public CalendarMergeLineContext(MergeFeedsJob job, Table table, ZipOutputStream out) throws IOException { + super(job, table, out); + } + + @Override + public void checkFieldsForMergeConflicts(Set idErrors) throws IOException { + checkCalendarIds(idErrors); + } + + private void checkCalendarIds(Set idErrors) throws IOException { + // If any service_id in the active feed matches with the future + // feed, it should be modified and all associated trip records + // must also be changed with the modified service_id. + // TODO How can we check that calendar_dates entries are + // duplicates? I think we would need to consider the + // service_id:exception_type:date as the unique key and include any + // all entries as long as they are unique on this key. + + String idScope = getIdScope(); + + if (hasDuplicateError(idErrors)) { + String key = getTableScopedValue(table, idScope, val); + // Modify service_id and ensure that referencing trips + // have service_id updated. + valueToWrite = String.join(":", idScope, val); + mergeFeedsResult.remappedIds.put(key, valueToWrite); + } + + LocalDate startDate = getCsvDate("start_date"); + if (isHandlingFutureFeed()) { + // For the future feed, check if the calendar's start date is earlier than the + // previous earliest value and update if so. + if (futureFirstCalendarStartDate.isAfter(startDate)) { + futureFirstCalendarStartDate = startDate; + } + // FIXME: Move this below so that a cloned service doesn't get prematurely + // modified? (do we want the cloned record to have the original values?) + if (shouldUpdateFutureFeedStartDate()) { + // Update start_date to extend service through the active feed's + // start date if the merge strategy dictates. The justification for this logic is that the active feed's + // service_id will be modified to a different unique value and the trips shared between the future/active + // service are exactly matching. + val = valueToWrite = activeFeedFirstDate.format(GTFS_DATE_FORMATTER); + } + } else { + // If a service_id from the active calendar has both the + // start_date and end_date in the future, the service will be + // excluded from the merged file. Records in trips, + // calendar_dates, and calendar_attributes referencing this + // service_id shall also be removed/ignored. Stop_time records + // for the ignored trips shall also be removed. + if (!startDate.isBefore(futureFeedFirstDate)) { + LOG.warn( + "Skipping calendar entry {} because it operates fully within the time span of future feed.", + keyValue); + String key = getTableScopedValue(table, idScope, keyValue); + mergeFeedsResult.skippedIds.add(key); + skipRecord = true; + } else { + // If a service_id from the active calendar has only the + // end_date in the future, the end_date shall be set to one + // day prior to the earliest start_date in future dataset + // before appending the calendar record to the merged file. + if (getField().name.equals("end_date")) { + LocalDate endDate = getCsvDate("end_date"); + if (!endDate.isBefore(futureFeedFirstDate)) { + val = valueToWrite = futureFeedFirstDate + .minus(1, ChronoUnit.DAYS) + .format(GTFS_DATE_FORMATTER); + } + } + } + } + // Track service ID because we want to avoid removing trips that may reference this + // service_id when the service_id is used by calendar_dates that operate in the valid + // date range, i.e., before the future feed's first date. + if (!skipRecord && getField().name.equals(SERVICE_ID)) mergeFeedsResult.serviceIds.add(valueToWrite); + } + + private boolean shouldUpdateFutureFeedStartDate() { + return getField().name.equals("start_date") && + ( + EXTEND_FUTURE == mergeFeedsResult.mergeStrategy || + ( + CHECK_STOP_TIMES == mergeFeedsResult.mergeStrategy && + job.serviceIdsToExtend.contains(keyValue) + ) + ); + } +} \ No newline at end of file diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java index 1d601b720..b6d756500 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java @@ -4,7 +4,6 @@ import com.conveyal.datatools.manager.models.FeedSource; import com.conveyal.datatools.manager.models.FeedVersion; import com.conveyal.gtfs.error.NewGTFSError; -import com.conveyal.gtfs.error.NewGTFSErrorType; import com.conveyal.gtfs.loader.Field; import com.conveyal.gtfs.loader.ReferenceTracker; import com.conveyal.gtfs.loader.Table; @@ -17,7 +16,6 @@ import java.io.IOException; import java.io.OutputStreamWriter; import java.time.LocalDate; -import java.time.temporal.ChronoUnit; import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; @@ -30,7 +28,6 @@ import static com.conveyal.datatools.manager.jobs.feedmerge.MergeFeedsType.REGIONAL; import static com.conveyal.datatools.manager.jobs.feedmerge.MergeFeedsType.SERVICE_PERIOD; -import static com.conveyal.datatools.manager.jobs.feedmerge.MergeStrategy.CHECK_STOP_TIMES; import static com.conveyal.datatools.manager.jobs.feedmerge.MergeStrategy.EXTEND_FUTURE; import static com.conveyal.datatools.manager.utils.MergeFeedUtils.containsField; import static com.conveyal.datatools.manager.utils.MergeFeedUtils.getAllFields; @@ -42,22 +39,22 @@ public class MergeLineContext { private static final String AGENCY_ID = "agency_id"; - private static final String SERVICE_ID = "service_id"; + protected static final String SERVICE_ID = "service_id"; private static final String STOPS = "stops"; private static final Logger LOG = LoggerFactory.getLogger(MergeLineContext.class); protected final MergeFeedsJob job; private final ZipOutputStream out; - protected final Set allFields; - private LocalDate futureFirstCalendarStartDate; - private final LocalDate activeFeedFirstDate; - private LocalDate futureFeedFirstDate; + protected final Set allFields; // try to make private + protected LocalDate futureFirstCalendarStartDate; + protected final LocalDate activeFeedFirstDate; + protected LocalDate futureFeedFirstDate; // try to make private private boolean handlingActiveFeed; private boolean handlingFutureFeed; private String idScope; // CSV writer used to write to zip file. private final CsvListWriter writer; private CsvReader csvReader; - private boolean skipRecord; + protected boolean skipRecord; protected String newAgencyId; // move protected boolean keyFieldMissing; private String[] rowValues; @@ -65,10 +62,10 @@ public class MergeLineContext { protected final Table table; protected FeedToMerge feed; protected String keyValue; - private final ReferenceTracker referenceTracker = new ReferenceTracker(); + protected final ReferenceTracker referenceTracker = new ReferenceTracker(); protected String keyField; private String orderField; - private final MergeFeedsResult mergeFeedsResult; + protected final MergeFeedsResult mergeFeedsResult; protected final List feedsToMerge; protected int keyFieldIndex; protected Field[] fieldsFoundInZip; // try to make private @@ -77,12 +74,10 @@ public class MergeLineContext { // Set up objects for tracking the rows encountered private final Map rowValuesForStopOrRouteId = new HashMap<>(); private final Set rowStrings = new HashSet<>(); - // Track shape_ids found in future feed in order to check for conflicts with active feed (MTC only). - private final Set shapeIdsInFutureFeed = new HashSet<>(); private List sharedSpecFields; private int index; - private String val; - private String valueToWrite; + protected String val; + protected String valueToWrite; protected int feedIndex; public FeedVersion version; @@ -94,27 +89,25 @@ public static MergeLineContext create(MergeFeedsJob job, Table table, ZipOutputS switch (table.name) { case "agency": return new AgencyMergeLineContext(job, table, out); - case STOPS: - return new StopsMergeLineContext(job, table, out); -/* case "calendar": - break; + return new CalendarMergeLineContext(job, table, out); case "calendar_dates": - break; - case "feed_info": - break; + return new CalendarDatesMergeLineContext(job, table, out); case "routes": - break; + return new RoutesMergeLineContext(job, table, out); case "shapes": - break; - case "stops": - break; + return new ShapesMergeLineContext(job, table, out); + case STOPS: + return new StopsMergeLineContext(job, table, out); case "trips": + return new TripsMergeLineContext(job, table, out); +/* + case "feed_info": break; */ default: return new MergeLineContext(job, table, out); - //throw new IllegalArgumentException(table.name); + //throw new IllegalArgumentException(table.name); } //return null; } @@ -191,6 +184,7 @@ public boolean shouldSkipFile() { /** * Iterate over all rows in table and write them to the output zip. + * * @return false, if a failing condition was encountered. true, if everything was ok. */ public boolean iterateOverRows() throws IOException { @@ -250,7 +244,7 @@ public boolean areForeignRefsOk() throws IOException { String skippedKey = getTableScopedValue(table, idScope, keyValue); if (orderField != null) { skippedKey = String.join(":", skippedKey, - csvReader.get(getFieldIndex(orderField))); + getCsvValue(orderField)); } mergeFeedsResult.skippedIds.add(skippedKey); skipRecord = true; @@ -275,40 +269,12 @@ private boolean serviceIdHasOrShouldBeSkipped(String key, boolean isValidService return mergeFeedsResult.skippedIds.contains(key) || serviceIdShouldBeSkipped; } - public void checkFieldsForMergeConflicts() throws IOException { - Set idErrors = getIdErrors(); - // Store values for key fields that have been encountered and update any key values that need modification due - // to conflicts. - switch (table.name) { - case "calendar": - checkCalendarIds(idErrors); - break; - case "calendar_dates": - checkCalendarDatesIds(); - break; - case "shapes": - checkShapeIds(idErrors); - break; - case "trips": - checkTripIds(idErrors); - break; - // When stop_code is included, stop merging will be based on that. If stop_code is not - // included, it will be based on stop_id. All stops in future data will be carried - // forward and any stops found in active data that are not in the future data shall be - // appended. If one of the feed is missing stop_code, merge fails with a notification to - // the user with suggestion that the feed with missing stop_code must be fixed with - // stop_code. - // NOTE: route case is also used by the stops case, so the route - // case must follow this block. - case STOPS: - case "routes": - checkRoutesAndStopsIds(idErrors); - break; - default: - // For any other table, skip any duplicate record. - if (hasDuplicateError(idErrors)) skipRecord = true; - break; - } + + /** + * Overridable method whose default behavior below is to skip a record if it creates a duplicate id. + */ + public void checkFieldsForMergeConflicts(Set idErrors) throws IOException { + if (hasDuplicateError(idErrors)) skipRecord = true; } private Set getIdErrors() { @@ -332,171 +298,7 @@ private Set getIdErrors() { return idErrors; } - private void checkTripIds(Set idErrors) { - // trip_ids between active and future datasets must not match. The tripIdsToSkip and - // tripIdsToModify sets below are determined based on the MergeStrategy used for MTC - // service period merges. - if (handlingActiveFeed) { - // Handling active feed. Skip or modify trip id if found in one of the - // respective sets. - if (job.tripIdsToSkipForActiveFeed.contains(keyValue)) { - skipRecord = true; - } else if (job.tripIdsToModifyForActiveFeed.contains(keyValue)) { - valueToWrite = String.join(":", idScope, val); - // Update key value for subsequent ID conflict checks for this row. - keyValue = valueToWrite; - mergeFeedsResult.remappedIds.put( - getTableScopedValue(table, idScope, val), - valueToWrite - ); - } - } - for (NewGTFSError error : idErrors) { - if (error.errorType.equals(NewGTFSErrorType.DUPLICATE_ID)) { - valueToWrite = String.join(":", idScope, val); - // Update key value for subsequent ID conflict checks for this row. - keyValue = valueToWrite; - mergeFeedsResult.remappedIds.put( - getTableScopedValue(table, idScope, val), - valueToWrite - ); - } - } - } - - private void checkShapeIds(Set idErrors) { - // If a shape_id is found in both future and active datasets, all shape points from - // the active dataset must be feed-scoped. Otherwise, the merged dataset may contain - // shape_id:shape_pt_sequence values from both datasets (e.g., if future dataset contains - // sequences 1,2,3,10 and active contains 1,2,7,9,10; the merged set will contain - // 1,2,3,7,9,10). - if (field.name.equals("shape_id")) { - if (handlingFutureFeed) { - // Track shape_id if working on future feed. - shapeIdsInFutureFeed.add(val); - } else if (shapeIdsInFutureFeed.contains(val)) { - // For the active feed, if the shape_id was already processed from the - // future feed, we need to add the feed-scope to avoid weird, hybrid shapes - // with points from both feeds. - valueToWrite = String.join(":", idScope, val); - // Update key value for subsequent ID conflict checks for this row. - keyValue = valueToWrite; - mergeFeedsResult.remappedIds.put( - getTableScopedValue(table, idScope, val), - valueToWrite - ); - // Re-check refs and uniqueness after changing shape_id value. (Note: this - // probably won't have any impact, but there's not much harm in including it.) - idErrors = referenceTracker - .checkReferencesAndUniqueness(keyValue, lineNumber, field, valueToWrite, - table, keyField, orderField); - } - } - // Skip record if normal duplicate errors are found. - if (hasDuplicateError(idErrors)) skipRecord = true; - } - - private void checkCalendarDatesIds() throws IOException { - // Drop any calendar_dates.txt records from the existing feed for dates that are - // not before the first date of the future feed. - int dateIndex = getFieldIndex("date"); - LocalDate date = LocalDate.parse(csvReader.get(dateIndex), GTFS_DATE_FORMATTER); - if (handlingActiveFeed && !date.isBefore(futureFeedFirstDate)) { - LOG.warn( - "Skipping calendar_dates entry {} because it operates in the time span of future feed (i.e., after or on {}).", - keyValue, - futureFeedFirstDate); - String key = getTableScopedValue(table, idScope, keyValue); - mergeFeedsResult.skippedIds.add(key); - skipRecord = true; - } - // Track service ID because we want to avoid removing trips that may reference this - // service_id when the service_id is used by calendar.txt records that operate in - // the valid date range, i.e., before the future feed's first date. - if (!skipRecord && field.name.equals(SERVICE_ID)) mergeFeedsResult.serviceIds.add(valueToWrite); - } - - private void checkCalendarIds(Set idErrors) throws IOException { - // If any service_id in the active feed matches with the future - // feed, it should be modified and all associated trip records - // must also be changed with the modified service_id. - // TODO How can we check that calendar_dates entries are - // duplicates? I think we would need to consider the - // service_id:exception_type:date as the unique key and include any - // all entries as long as they are unique on this key. - if (hasDuplicateError(idErrors)) { - String key = getTableScopedValue(table, idScope, val); - // Modify service_id and ensure that referencing trips - // have service_id updated. - valueToWrite = String.join(":", idScope, val); - mergeFeedsResult.remappedIds.put(key, valueToWrite); - } - int startDateIndex = getFieldIndex("start_date"); - LocalDate startDate = LocalDate.parse(csvReader.get(startDateIndex), GTFS_DATE_FORMATTER); - if (handlingFutureFeed) { - // For the future feed, check if the calendar's start date is earlier than the - // previous earliest value and update if so. - if (futureFirstCalendarStartDate.isAfter(startDate)) { - futureFirstCalendarStartDate = startDate; - } - // FIXME: Move this below so that a cloned service doesn't get prematurely - // modified? (do we want the cloned record to have the original values?) - if (shouldUpdateFutureFeedStartDate(startDateIndex)) { - // Update start_date to extend service through the active feed's - // start date if the merge strategy dictates. The justification for this logic is that the active feed's - // service_id will be modified to a different unique value and the trips shared between the future/active - // service are exactly matching. - val = valueToWrite = activeFeedFirstDate.format(GTFS_DATE_FORMATTER); - } - } else { - // If a service_id from the active calendar has both the - // start_date and end_date in the future, the service will be - // excluded from the merged file. Records in trips, - // calendar_dates, and calendar_attributes referencing this - // service_id shall also be removed/ignored. Stop_time records - // for the ignored trips shall also be removed. - if (!startDate.isBefore(futureFeedFirstDate)) { - LOG.warn( - "Skipping calendar entry {} because it operates fully within the time span of future feed.", - keyValue); - String key = getTableScopedValue(table, idScope, keyValue); - mergeFeedsResult.skippedIds.add(key); - skipRecord = true; - } else { - // If a service_id from the active calendar has only the - // end_date in the future, the end_date shall be set to one - // day prior to the earliest start_date in future dataset - // before appending the calendar record to the merged file. - int endDateIndex = getFieldIndex("end_date"); - if (index == endDateIndex) { - LocalDate endDate = LocalDate - .parse(csvReader.get(endDateIndex), GTFS_DATE_FORMATTER); - if (!endDate.isBefore(futureFeedFirstDate)) { - val = valueToWrite = futureFeedFirstDate - .minus(1, ChronoUnit.DAYS) - .format(GTFS_DATE_FORMATTER); - } - } - } - } - // Track service ID because we want to avoid removing trips that may reference this - // service_id when the service_id is used by calendar_dates that operate in the valid - // date range, i.e., before the future feed's first date. - if (!skipRecord && field.name.equals(SERVICE_ID)) mergeFeedsResult.serviceIds.add(valueToWrite); - } - - private boolean shouldUpdateFutureFeedStartDate(int startDateIndex) { - return index == startDateIndex && - ( - EXTEND_FUTURE == mergeFeedsResult.mergeStrategy || - ( - CHECK_STOP_TIMES == mergeFeedsResult.mergeStrategy && - job.serviceIdsToExtend.contains(keyValue) - ) - ); - } - - private void checkRoutesAndStopsIds(Set idErrors) throws IOException { + protected void checkRoutesAndStopsIds(Set idErrors) throws IOException { // First, check uniqueness of primary key value (i.e., stop or route ID) // in case the stop_code or route_short_name are being used. This // must occur unconditionally because each record must be tracked @@ -656,10 +458,16 @@ public boolean storeRowAndStopValues() { return false; } - public void checkFirstLineConditions() throws IOException {} + /** + * Overridable placeholder for checking the first line of a file. + */ + public void checkFirstLineConditions() throws IOException { + // Default is to do nothing. + } /** * Check for some conditions that could occur when handling a service period merge. + * * @return true if the merge encountered failing conditions */ public boolean checkMismatchedAgency() { @@ -792,7 +600,10 @@ public boolean constructRowValues() throws IOException { // track references for a large number of feeds (e.g., every feed in New // York State). if (job.mergeType.equals(SERVICE_PERIOD)) { - checkFieldsForMergeConflicts(); + Set idErrors = getIdErrors(); + // Store values for key fields that have been encountered and update any key values that need modification due + // to conflicts. + checkFieldsForMergeConflicts(idErrors); if (skipRecord) continue; } // If the current field is a foreign reference, check if the reference has been removed in the @@ -836,6 +647,10 @@ public boolean lineIsBlank() throws IOException { return false; } + public boolean isHandlingActiveFeed() { + return handlingActiveFeed; + } + public boolean isHandlingFutureFeed() { return handlingFutureFeed; } @@ -847,4 +662,31 @@ protected CsvReader getCsvReader() { protected int getFieldIndex(String fieldName) { return Field.getFieldIndex(fieldsFoundInZip, fieldName); } -} + + protected String getIdScope() { + return idScope; + } + + protected Field getField() { + return field; + } + + protected int getLineNumber() { + return lineNumber; + } + + /** + * Retrieves the value for the specified CSV field. + */ + protected String getCsvValue(String fieldName) throws IOException { + int fieldIndex = getFieldIndex(fieldName); + return csvReader.get(fieldIndex); + } + + /** + * Retrieves the value for the specified CSV field as {@link LocalDate}. + */ + protected LocalDate getCsvDate(String fieldName) throws IOException { + return LocalDate.parse(getCsvValue(fieldName), GTFS_DATE_FORMATTER); + } +} \ No newline at end of file diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/RoutesMergeLineContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/RoutesMergeLineContext.java new file mode 100644 index 000000000..ae976e342 --- /dev/null +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/RoutesMergeLineContext.java @@ -0,0 +1,20 @@ +package com.conveyal.datatools.manager.jobs.feedmerge; + +import com.conveyal.datatools.manager.jobs.MergeFeedsJob; +import com.conveyal.gtfs.error.NewGTFSError; +import com.conveyal.gtfs.loader.Table; + +import java.io.IOException; +import java.util.Set; +import java.util.zip.ZipOutputStream; + +public class RoutesMergeLineContext extends MergeLineContext { + public RoutesMergeLineContext(MergeFeedsJob job, Table table, ZipOutputStream out) throws IOException { + super(job, table, out); + } + + @Override + public void checkFieldsForMergeConflicts(Set idErrors) throws IOException { + checkRoutesAndStopsIds(idErrors); + } +} \ No newline at end of file diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/ShapesMergeLineContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/ShapesMergeLineContext.java new file mode 100644 index 000000000..ce31ff9bc --- /dev/null +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/ShapesMergeLineContext.java @@ -0,0 +1,59 @@ +package com.conveyal.datatools.manager.jobs.feedmerge; + +import com.conveyal.datatools.manager.jobs.MergeFeedsJob; +import com.conveyal.gtfs.error.NewGTFSError; +import com.conveyal.gtfs.loader.Table; + +import java.io.IOException; +import java.util.HashSet; +import java.util.Set; +import java.util.zip.ZipOutputStream; + +import static com.conveyal.datatools.manager.utils.MergeFeedUtils.getTableScopedValue; +import static com.conveyal.datatools.manager.utils.MergeFeedUtils.hasDuplicateError; + +public class ShapesMergeLineContext extends MergeLineContext { + // Track shape_ids found in future feed in order to check for conflicts with active feed (MTC only). + private final Set shapeIdsInFutureFeed = new HashSet<>(); + + public ShapesMergeLineContext(MergeFeedsJob job, Table table, ZipOutputStream out) throws IOException { + super(job, table, out); + } + + @Override + public void checkFieldsForMergeConflicts(Set idErrors) throws IOException { + checkShapeIds(idErrors); + } + + private void checkShapeIds(Set idErrors) { + // If a shape_id is found in both future and active datasets, all shape points from + // the active dataset must be feed-scoped. Otherwise, the merged dataset may contain + // shape_id:shape_pt_sequence values from both datasets (e.g., if future dataset contains + // sequences 1,2,3,10 and active contains 1,2,7,9,10; the merged set will contain + // 1,2,3,7,9,10). + if (getField().name.equals("shape_id")) { + if (isHandlingFutureFeed()) { + // Track shape_id if working on future feed. + shapeIdsInFutureFeed.add(val); + } else if (shapeIdsInFutureFeed.contains(val)) { + // For the active feed, if the shape_id was already processed from the + // future feed, we need to add the feed-scope to avoid weird, hybrid shapes + // with points from both feeds. + valueToWrite = String.join(":", getIdScope(), val); + // Update key value for subsequent ID conflict checks for this row. + keyValue = valueToWrite; + mergeFeedsResult.remappedIds.put( + getTableScopedValue(table, getIdScope(), val), + valueToWrite + ); + // Re-check refs and uniqueness after changing shape_id value. (Note: this + // probably won't have any impact, but there's not much harm in including it.) + idErrors = referenceTracker + .checkReferencesAndUniqueness(keyValue, getLineNumber(), getField(), valueToWrite, + table, keyField, table.getOrderFieldName()); + } + } + // Skip record if normal duplicate errors are found. + if (hasDuplicateError(idErrors)) skipRecord = true; + } +} \ No newline at end of file diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/StopsMergeLineContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/StopsMergeLineContext.java index 5fc39fd0f..2805bd1b7 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/StopsMergeLineContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/StopsMergeLineContext.java @@ -1,12 +1,14 @@ package com.conveyal.datatools.manager.jobs.feedmerge; import com.conveyal.datatools.manager.jobs.MergeFeedsJob; +import com.conveyal.gtfs.error.NewGTFSError; import com.conveyal.gtfs.loader.Table; import com.csvreader.CsvReader; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; +import java.util.Set; import java.util.zip.ZipOutputStream; import static com.conveyal.datatools.manager.jobs.feedmerge.MergeFeedsType.SERVICE_PERIOD; @@ -26,6 +28,11 @@ public void checkFirstLineConditions() throws IOException { checkStopCodeStuff(); } + @Override + public void checkFieldsForMergeConflicts(Set idErrors) throws IOException { + checkRoutesAndStopsIds(idErrors); + } + private void checkStopCodeStuff() throws IOException { if (shouldCheckStopCodes()) { // Before reading any lines in stops.txt, first determine whether all records contain diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/TripsMergeLineContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/TripsMergeLineContext.java new file mode 100644 index 000000000..1f0b6c901 --- /dev/null +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/TripsMergeLineContext.java @@ -0,0 +1,56 @@ +package com.conveyal.datatools.manager.jobs.feedmerge; + +import com.conveyal.datatools.manager.jobs.MergeFeedsJob; +import com.conveyal.gtfs.error.NewGTFSError; +import com.conveyal.gtfs.error.NewGTFSErrorType; +import com.conveyal.gtfs.loader.Table; + +import java.io.IOException; +import java.util.Set; +import java.util.zip.ZipOutputStream; + +import static com.conveyal.datatools.manager.utils.MergeFeedUtils.getTableScopedValue; + +public class TripsMergeLineContext extends MergeLineContext { + public TripsMergeLineContext(MergeFeedsJob job, Table table, ZipOutputStream out) throws IOException { + super(job, table, out); + } + + @Override + public void checkFieldsForMergeConflicts(Set idErrors) throws IOException { + checkTripIds(idErrors); + } + + private void checkTripIds(Set idErrors) { + // trip_ids between active and future datasets must not match. The tripIdsToSkip and + // tripIdsToModify sets below are determined based on the MergeStrategy used for MTC + // service period merges. + String idScope = getIdScope(); + if (isHandlingActiveFeed()) { + // Handling active feed. Skip or modify trip id if found in one of the + // respective sets. + if (job.tripIdsToSkipForActiveFeed.contains(keyValue)) { + skipRecord = true; + } else if (job.tripIdsToModifyForActiveFeed.contains(keyValue)) { + valueToWrite = String.join(":", idScope, val); + // Update key value for subsequent ID conflict checks for this row. + keyValue = valueToWrite; + mergeFeedsResult.remappedIds.put( // TODO: refactor + getTableScopedValue(table, idScope, val), + valueToWrite + ); + } + } + for (NewGTFSError error : idErrors) { + if (error.errorType.equals(NewGTFSErrorType.DUPLICATE_ID)) { + valueToWrite = String.join(":", idScope, val); + // Update key value for subsequent ID conflict checks for this row. + keyValue = valueToWrite; + mergeFeedsResult.remappedIds.put( // TODO: refactor + getTableScopedValue(table, idScope, val), + valueToWrite + ); + } + } + } +} \ No newline at end of file From 147535f652a9e438a7158d89120b2cb74e2319f3 Mon Sep 17 00:00:00 2001 From: binh-dam-ibigroup <56846598+binh-dam-ibigroup@users.noreply.github.com> Date: Fri, 5 Nov 2021 15:22:06 -0400 Subject: [PATCH 051/122] refactor(MergeLineContext): Extract value update methods --- .../feedmerge/CalendarMergeLineContext.java | 8 ++--- .../jobs/feedmerge/MergeLineContext.java | 31 ++++++++++++++----- .../feedmerge/ShapesMergeLineContext.java | 11 ++----- .../jobs/feedmerge/TripsMergeLineContext.java | 22 +++---------- 4 files changed, 31 insertions(+), 41 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarMergeLineContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarMergeLineContext.java index 666574c43..a1a6cf9ac 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarMergeLineContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarMergeLineContext.java @@ -39,14 +39,10 @@ private void checkCalendarIds(Set idErrors) throws IOException { // service_id:exception_type:date as the unique key and include any // all entries as long as they are unique on this key. - String idScope = getIdScope(); - if (hasDuplicateError(idErrors)) { - String key = getTableScopedValue(table, idScope, val); // Modify service_id and ensure that referencing trips // have service_id updated. - valueToWrite = String.join(":", idScope, val); - mergeFeedsResult.remappedIds.put(key, valueToWrite); + updateAndRemapValue(); } LocalDate startDate = getCsvDate("start_date"); @@ -76,7 +72,7 @@ private void checkCalendarIds(Set idErrors) throws IOException { LOG.warn( "Skipping calendar entry {} because it operates fully within the time span of future feed.", keyValue); - String key = getTableScopedValue(table, idScope, keyValue); + String key = getTableScopedValue(table, getIdScope(), keyValue); mergeFeedsResult.skippedIds.add(key); skipRecord = true; } else { diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java index b6d756500..0e1aaa3ef 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java @@ -282,11 +282,7 @@ private Set getIdErrors() { // If analyzing the second feed (active feed), the service_id always gets feed scoped. // See https://github.com/ibi-group/datatools-server/issues/244 if (handlingActiveFeed && field.name.equals(SERVICE_ID)) { - valueToWrite = String.join(":", idScope, val); - mergeFeedsResult.remappedIds.put( - getTableScopedValue(table, idScope, val), - valueToWrite - ); + updateAndRemapValue(); idErrors = referenceTracker .checkReferencesAndUniqueness(keyValue, lineNumber, field, valueToWrite, table, keyField, orderField); @@ -359,11 +355,9 @@ protected void checkRoutesAndStopsIds(Set idErrors) throws IOExcep // both of these routes to end up in the merged feed in this case because we're // matching on short name, so we must modify the route_id. if (!skipRecord && !referenceTracker.transitIds.contains(String.join(":", keyField, keyValue)) && hasDuplicateError(primaryKeyErrors)) { - String key = getTableScopedValue(table, idScope, val); // Modify route_id and ensure that referencing trips // have route_id updated. - valueToWrite = String.join(":", idScope, val); - mergeFeedsResult.remappedIds.put(key, valueToWrite); + updateAndRemapValue(); } } else { // Key field has defaulted to the standard primary key field @@ -689,4 +683,25 @@ protected String getCsvValue(String fieldName) throws IOException { protected LocalDate getCsvDate(String fieldName) throws IOException { return LocalDate.parse(getCsvValue(fieldName), GTFS_DATE_FORMATTER); } + + /** + * Updates output for the current field and remaps the record id. + */ + protected void updateAndRemapValue(boolean updateKeyValue) { + valueToWrite = String.join(":", idScope, val); + if (updateKeyValue) { + keyValue = valueToWrite; + } + mergeFeedsResult.remappedIds.put( + getTableScopedValue(table, idScope, val), + valueToWrite + ); + } + + /** + * Shorthand for the above method. + */ + protected void updateAndRemapValue() { + updateAndRemapValue(false); + } } \ No newline at end of file diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/ShapesMergeLineContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/ShapesMergeLineContext.java index ce31ff9bc..07b363310 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/ShapesMergeLineContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/ShapesMergeLineContext.java @@ -9,7 +9,6 @@ import java.util.Set; import java.util.zip.ZipOutputStream; -import static com.conveyal.datatools.manager.utils.MergeFeedUtils.getTableScopedValue; import static com.conveyal.datatools.manager.utils.MergeFeedUtils.hasDuplicateError; public class ShapesMergeLineContext extends MergeLineContext { @@ -21,7 +20,7 @@ public ShapesMergeLineContext(MergeFeedsJob job, Table table, ZipOutputStream ou } @Override - public void checkFieldsForMergeConflicts(Set idErrors) throws IOException { + public void checkFieldsForMergeConflicts(Set idErrors) { checkShapeIds(idErrors); } @@ -39,13 +38,7 @@ private void checkShapeIds(Set idErrors) { // For the active feed, if the shape_id was already processed from the // future feed, we need to add the feed-scope to avoid weird, hybrid shapes // with points from both feeds. - valueToWrite = String.join(":", getIdScope(), val); - // Update key value for subsequent ID conflict checks for this row. - keyValue = valueToWrite; - mergeFeedsResult.remappedIds.put( - getTableScopedValue(table, getIdScope(), val), - valueToWrite - ); + updateAndRemapValue(true); // Re-check refs and uniqueness after changing shape_id value. (Note: this // probably won't have any impact, but there's not much harm in including it.) idErrors = referenceTracker diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/TripsMergeLineContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/TripsMergeLineContext.java index 1f0b6c901..ac91f8273 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/TripsMergeLineContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/TripsMergeLineContext.java @@ -9,15 +9,13 @@ import java.util.Set; import java.util.zip.ZipOutputStream; -import static com.conveyal.datatools.manager.utils.MergeFeedUtils.getTableScopedValue; - public class TripsMergeLineContext extends MergeLineContext { public TripsMergeLineContext(MergeFeedsJob job, Table table, ZipOutputStream out) throws IOException { super(job, table, out); } @Override - public void checkFieldsForMergeConflicts(Set idErrors) throws IOException { + public void checkFieldsForMergeConflicts(Set idErrors) { checkTripIds(idErrors); } @@ -25,32 +23,20 @@ private void checkTripIds(Set idErrors) { // trip_ids between active and future datasets must not match. The tripIdsToSkip and // tripIdsToModify sets below are determined based on the MergeStrategy used for MTC // service period merges. - String idScope = getIdScope(); if (isHandlingActiveFeed()) { // Handling active feed. Skip or modify trip id if found in one of the // respective sets. if (job.tripIdsToSkipForActiveFeed.contains(keyValue)) { skipRecord = true; } else if (job.tripIdsToModifyForActiveFeed.contains(keyValue)) { - valueToWrite = String.join(":", idScope, val); - // Update key value for subsequent ID conflict checks for this row. - keyValue = valueToWrite; - mergeFeedsResult.remappedIds.put( // TODO: refactor - getTableScopedValue(table, idScope, val), - valueToWrite - ); + updateAndRemapValue(true); } } for (NewGTFSError error : idErrors) { if (error.errorType.equals(NewGTFSErrorType.DUPLICATE_ID)) { - valueToWrite = String.join(":", idScope, val); - // Update key value for subsequent ID conflict checks for this row. - keyValue = valueToWrite; - mergeFeedsResult.remappedIds.put( // TODO: refactor - getTableScopedValue(table, idScope, val), - valueToWrite - ); + updateAndRemapValue(true); } } } + } \ No newline at end of file From 0b937c376fe3cb20f62f9a28a0885ea01b80beb8 Mon Sep 17 00:00:00 2001 From: binh-dam-ibigroup <56846598+binh-dam-ibigroup@users.noreply.github.com> Date: Fri, 5 Nov 2021 16:00:40 -0400 Subject: [PATCH 052/122] refactor(MergeLineContext): Move *LineContext-specific code to subclasses. --- .../feedmerge/AgencyMergeLineContext.java | 47 ++++++++++++ .../CalendarDatesMergeLineContext.java | 16 ++-- .../jobs/feedmerge/MergeLineContext.java | 74 ++++--------------- 3 files changed, 71 insertions(+), 66 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/AgencyMergeLineContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/AgencyMergeLineContext.java index 20958c624..34255a240 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/AgencyMergeLineContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/AgencyMergeLineContext.java @@ -3,6 +3,8 @@ import com.conveyal.datatools.manager.jobs.MergeFeedsJob; import com.conveyal.gtfs.loader.Field; import com.conveyal.gtfs.loader.Table; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.IOException; import java.util.ArrayList; @@ -11,7 +13,11 @@ import java.util.UUID; import java.util.zip.ZipOutputStream; +import static com.conveyal.datatools.manager.jobs.feedmerge.MergeFeedsType.SERVICE_PERIOD; + public class AgencyMergeLineContext extends MergeLineContext { + private static final Logger LOG = LoggerFactory.getLogger(AgencyMergeLineContext.class); + public AgencyMergeLineContext(MergeFeedsJob job, Table table, ZipOutputStream out) throws IOException { super(job, table, out); } @@ -21,6 +27,11 @@ public void checkFirstLineConditions() { checkForMissingAgencyId(); } + @Override + public boolean shouldProcessRows() { + return !checkMismatchedAgency(); + } + private void checkForMissingAgencyId() { if ((keyFieldMissing || keyValue.equals(""))) { // agency_id is optional if only one agency is present, but that will @@ -37,4 +48,40 @@ private void checkForMissingAgencyId() { fieldsFoundList = Arrays.asList(fieldsFoundInZip); } } + + /** + * Check for some conditions that could occur when handling a service period merge. + * + * @return true if the merge encountered failing conditions + */ + private boolean checkMismatchedAgency() { + if (isHandlingActiveFeed() && job.mergeType.equals(SERVICE_PERIOD)) { + // If merging the agency table, we should only skip the following feeds if performing an MTC merge + // because that logic assumes the two feeds share the same agency (or + // agencies). NOTE: feed_info file is skipped by default (outside of this + // method) for a regional merge), which is why this block is exclusively + // for an MTC merge. Note, this statement may print multiple log + // statements, but it is deliberately nested in the csv while block in + // order to detect agency_id mismatches and fail the merge if found. + // The second feed's agency table must contain the same agency_id + // value as the first feed. + String agencyId = String.join(":", keyField, keyValue); + if (!"".equals(keyValue) && !referenceTracker.transitIds.contains(agencyId)) { + String otherAgencyId = referenceTracker.transitIds.stream() + .filter(transitId -> transitId.startsWith(AGENCY_ID)) + .findAny() + .orElse(null); + job.failMergeJob(String.format( + "MTC merge detected mismatching agency_id values between two " + + "feeds (%s and %s). Failing merge operation.", + agencyId, + otherAgencyId + )); + return true; + } + LOG.warn("Skipping {} file for feed {}/{} (future file preferred)", table.name, feedIndex, feedsToMerge.size()); + skipFile = true; + } + return false; + } } \ No newline at end of file diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarDatesMergeLineContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarDatesMergeLineContext.java index b67381f1c..9b6297b38 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarDatesMergeLineContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarDatesMergeLineContext.java @@ -26,6 +26,12 @@ public void checkFieldsForMergeConflicts(Set idErrors) throws IOEx checkCalendarDatesIds(); } + @Override + public void startNewRow() throws IOException { + super.startNewRow(); + updateFutureFeedFirstDate(); + } + private void checkCalendarDatesIds() throws IOException { // Drop any calendar_dates.txt records from the existing feed for dates that are // not before the first date of the future feed. @@ -45,12 +51,12 @@ private void checkCalendarDatesIds() throws IOException { if (!skipRecord && getField().name.equals(SERVICE_ID)) mergeFeedsResult.serviceIds.add(valueToWrite); } - private void updateFutureFeedFirstDate_placeHolder() { + private void updateFutureFeedFirstDate() { if ( - isHandlingActiveFeed() && - job.mergeType.equals(SERVICE_PERIOD) && - futureFirstCalendarStartDate.isBefore(LocalDate.MAX) && - futureFeedFirstDate.isBefore(futureFirstCalendarStartDate) + isHandlingActiveFeed() && + job.mergeType.equals(SERVICE_PERIOD) && + futureFirstCalendarStartDate.isBefore(LocalDate.MAX) && + futureFeedFirstDate.isBefore(futureFirstCalendarStartDate) ) { // If the future feed's first date is before its first calendar start date, // override the future feed first date with the calendar start date for use when checking diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java index 0e1aaa3ef..2f2261e67 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java @@ -38,7 +38,7 @@ import static com.conveyal.gtfs.loader.DateField.GTFS_DATE_FORMATTER; public class MergeLineContext { - private static final String AGENCY_ID = "agency_id"; + protected static final String AGENCY_ID = "agency_id"; protected static final String SERVICE_ID = "service_id"; private static final String STOPS = "stops"; private static final Logger LOG = LoggerFactory.getLogger(MergeLineContext.class); @@ -101,15 +101,9 @@ public static MergeLineContext create(MergeFeedsJob job, Table table, ZipOutputS return new StopsMergeLineContext(job, table, out); case "trips": return new TripsMergeLineContext(job, table, out); -/* - case "feed_info": - break; -*/ default: return new MergeLineContext(job, table, out); - //throw new IllegalArgumentException(table.name); } - //return null; } protected MergeLineContext(MergeFeedsJob job, Table table, ZipOutputStream out) throws IOException { @@ -182,6 +176,14 @@ public boolean shouldSkipFile() { return false; } + /** + * Overridable method that determines whether to process rows of the current feed table. + * @return true by default. + */ + public boolean shouldProcessRows() { + return true; + } + /** * Iterate over all rows in table and write them to the output zip. * @@ -191,11 +193,12 @@ public boolean iterateOverRows() throws IOException { // Iterate over rows in table, writing them to the out file. while (csvReader.readRecord()) { startNewRow(); - if (checkMismatchedAgency()) { - // If there is a mismatched agency, return immediately. + + if (!shouldProcessRows()) { + // e.g. If there is a mismatched agency, return immediately. return false; } - updateFutureFeedFirstDate(); + // If checkMismatchedAgency flagged skipFile, loop back to the while loop. (Note: this is // intentional because we want to check all agency ids in the file). if (skipFile || lineIsBlank()) continue; @@ -459,42 +462,6 @@ public void checkFirstLineConditions() throws IOException { // Default is to do nothing. } - /** - * Check for some conditions that could occur when handling a service period merge. - * - * @return true if the merge encountered failing conditions - */ - public boolean checkMismatchedAgency() { - if (handlingActiveFeed && job.mergeType.equals(SERVICE_PERIOD) && table.name.equals("agency")) { - // If merging the agency table, we should only skip the following feeds if performing an MTC merge - // because that logic assumes the two feeds share the same agency (or - // agencies). NOTE: feed_info file is skipped by default (outside of this - // method) for a regional merge), which is why this block is exclusively - // for an MTC merge. Note, this statement may print multiple log - // statements, but it is deliberately nested in the csv while block in - // order to detect agency_id mismatches and fail the merge if found. - // The second feed's agency table must contain the same agency_id - // value as the first feed. - String agencyId = String.join(":", keyField, keyValue); - if (!"".equals(keyValue) && !referenceTracker.transitIds.contains(agencyId)) { - String otherAgencyId = referenceTracker.transitIds.stream() - .filter(transitId -> transitId.startsWith(AGENCY_ID)) - .findAny() - .orElse(null); - job.failMergeJob(String.format( - "MTC merge detected mismatching agency_id values between two " - + "feeds (%s and %s). Failing merge operation.", - agencyId, - otherAgencyId - )); - return true; - } - LOG.warn("Skipping {} file for feed {}/{} (future file preferred)", table.name, feedIndex, feedsToMerge.size()); - skipFile = true; - } - return false; - } - public void scopeValueIfNeeded() { boolean isKeyField = field.isForeignReference() || keyField.equals(field.name); if (job.mergeType.equals(REGIONAL) && isKeyField && !val.isEmpty()) { @@ -558,21 +525,6 @@ public void writeHeaders() throws IOException { writeValuesToTable(headers, false); } - public void updateFutureFeedFirstDate() { - if ( - table.name.equals("calendar_dates") && - handlingActiveFeed && - job.mergeType.equals(SERVICE_PERIOD) && - futureFirstCalendarStartDate.isBefore(LocalDate.MAX) && - futureFeedFirstDate.isBefore(futureFirstCalendarStartDate) - ) { - // If the future feed's first date is before its first calendar start date, - // override the future feed first date with the calendar start date for use when checking - // MTC calendar_dates and calendar records for modification/exclusion. - futureFeedFirstDate = futureFirstCalendarStartDate; - } - } - public boolean constructRowValues() throws IOException { // Piece together the row to write, which should look practically identical to the original // row except for the identifiers receiving a prefix to avoid ID conflicts. From 1b57a05108dd842f91d8992aa30af969d1f8a719 Mon Sep 17 00:00:00 2001 From: binh-dam-ibigroup <56846598+binh-dam-ibigroup@users.noreply.github.com> Date: Fri, 5 Nov 2021 17:18:18 -0400 Subject: [PATCH 053/122] refactor(MergeLineContext): Introduce FieldContext. --- .../CalendarDatesMergeLineContext.java | 2 +- .../feedmerge/CalendarMergeLineContext.java | 14 ++-- .../manager/jobs/feedmerge/FieldContext.java | 51 ++++++++++++ .../jobs/feedmerge/MergeLineContext.java | 79 +++++++++++-------- .../feedmerge/ShapesMergeLineContext.java | 15 +++- .../jobs/feedmerge/TripsMergeLineContext.java | 4 +- 6 files changed, 116 insertions(+), 49 deletions(-) create mode 100644 src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/FieldContext.java diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarDatesMergeLineContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarDatesMergeLineContext.java index 9b6297b38..4c8f51ac5 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarDatesMergeLineContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarDatesMergeLineContext.java @@ -48,7 +48,7 @@ private void checkCalendarDatesIds() throws IOException { // Track service ID because we want to avoid removing trips that may reference this // service_id when the service_id is used by calendar.txt records that operate in // the valid date range, i.e., before the future feed's first date. - if (!skipRecord && getField().name.equals(SERVICE_ID)) mergeFeedsResult.serviceIds.add(valueToWrite); + if (!skipRecord && fieldNameEquals(SERVICE_ID)) mergeFeedsResult.serviceIds.add(getFieldContext().getValueToWrite()); } private void updateFutureFeedFirstDate() { diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarMergeLineContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarMergeLineContext.java index a1a6cf9ac..75e8a3475 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarMergeLineContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarMergeLineContext.java @@ -42,7 +42,7 @@ private void checkCalendarIds(Set idErrors) throws IOException { if (hasDuplicateError(idErrors)) { // Modify service_id and ensure that referencing trips // have service_id updated. - updateAndRemapValue(); + updateAndRemapOutput(); } LocalDate startDate = getCsvDate("start_date"); @@ -59,7 +59,7 @@ private void checkCalendarIds(Set idErrors) throws IOException { // start date if the merge strategy dictates. The justification for this logic is that the active feed's // service_id will be modified to a different unique value and the trips shared between the future/active // service are exactly matching. - val = valueToWrite = activeFeedFirstDate.format(GTFS_DATE_FORMATTER); + getFieldContext().resetValue(activeFeedFirstDate.format(GTFS_DATE_FORMATTER)); } } else { // If a service_id from the active calendar has both the @@ -80,12 +80,12 @@ private void checkCalendarIds(Set idErrors) throws IOException { // end_date in the future, the end_date shall be set to one // day prior to the earliest start_date in future dataset // before appending the calendar record to the merged file. - if (getField().name.equals("end_date")) { + if (fieldNameEquals("end_date")) { LocalDate endDate = getCsvDate("end_date"); if (!endDate.isBefore(futureFeedFirstDate)) { - val = valueToWrite = futureFeedFirstDate + getFieldContext().resetValue(futureFeedFirstDate .minus(1, ChronoUnit.DAYS) - .format(GTFS_DATE_FORMATTER); + .format(GTFS_DATE_FORMATTER)); } } } @@ -93,11 +93,11 @@ private void checkCalendarIds(Set idErrors) throws IOException { // Track service ID because we want to avoid removing trips that may reference this // service_id when the service_id is used by calendar_dates that operate in the valid // date range, i.e., before the future feed's first date. - if (!skipRecord && getField().name.equals(SERVICE_ID)) mergeFeedsResult.serviceIds.add(valueToWrite); + if (!skipRecord && fieldNameEquals(SERVICE_ID)) mergeFeedsResult.serviceIds.add(getFieldContext().getValueToWrite()); } private boolean shouldUpdateFutureFeedStartDate() { - return getField().name.equals("start_date") && + return fieldNameEquals("start_date") && ( EXTEND_FUTURE == mergeFeedsResult.mergeStrategy || ( diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/FieldContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/FieldContext.java new file mode 100644 index 000000000..1615ef654 --- /dev/null +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/FieldContext.java @@ -0,0 +1,51 @@ +package com.conveyal.datatools.manager.jobs.feedmerge; + +import com.conveyal.gtfs.loader.Field; + + +/** + * Holds data when processing a field in a CSV row during a feed merge. + */ +public class FieldContext { + private final Field field; + private final int index; + private String value; + private String valueToWrite; + + public FieldContext(Field field, int index, String value) { + this.field = field; + // Get index of field from GTFS spec as it appears in feed + this.index = index; + // Default value to write is unchanged from value found in csv (i.e. val). Note: if looking to + // modify the value that is written in the merged file, you must update valueToWrite (e.g., + // updating this feed's end_date or accounting for cases where IDs conflict). + resetValue(value); + } + + public Field getField() { + return field; + } + + public String getValue() { + return value; + } + + public void setValue(String newValue) { + value = newValue; + } + + public String getValueToWrite() { + return valueToWrite; + } + + public void setValueToWrite(String newValue) { + valueToWrite = newValue; + } + + /** + * Resets both value and valueToWrite to a desired new value. + */ + public void resetValue(String newValue) { + value = valueToWrite = newValue; + } +} diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java index 2f2261e67..6f210a8f2 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java @@ -70,14 +70,11 @@ public class MergeLineContext { protected int keyFieldIndex; protected Field[] fieldsFoundInZip; // try to make private protected List fieldsFoundList; - private Field field; // Set up objects for tracking the rows encountered private final Map rowValuesForStopOrRouteId = new HashMap<>(); private final Set rowStrings = new HashSet<>(); private List sharedSpecFields; - private int index; - protected String val; - protected String valueToWrite; + private FieldContext fieldContext; protected int feedIndex; public FeedVersion version; @@ -227,10 +224,12 @@ public void startNewRow() throws IOException { } public boolean areForeignRefsOk() throws IOException { + Field field = fieldContext.getField(); if (field.isForeignReference()) { - String key = getTableScopedValue(field.referenceTable, idScope, val); + String key = getTableScopedValue(field.referenceTable, idScope, fieldContext.getValue()); // Check if we're performing a service period merge, this ref field is a service_id, and it // is not found in the list of service_ids (e.g., it was removed). + String valueToWrite = fieldContext.getValueToWrite(); boolean isValidServiceId = mergeFeedsResult.serviceIds.contains(valueToWrite); // If the current foreign ref points to another record that has @@ -241,7 +240,7 @@ public boolean areForeignRefsOk() throws IOException { // If a calendar#service_id has been skipped (it's listed in skippedIds), but there were // valid service_ids found in calendar_dates, do not skip that record for both the // calendar_date and any related trips. - if (field.name.equals(SERVICE_ID) && isValidServiceId) { + if (fieldNameEquals(SERVICE_ID) && isValidServiceId) { LOG.warn("Not skipping valid service_id {} for {} {}", valueToWrite, table.name, keyValue); } else { String skippedKey = getTableScopedValue(table, idScope, keyValue); @@ -267,7 +266,7 @@ public boolean areForeignRefsOk() throws IOException { private boolean serviceIdHasOrShouldBeSkipped(String key, boolean isValidServiceId) { boolean serviceIdShouldBeSkipped = job.mergeType.equals(SERVICE_PERIOD) && - field.name.equals(SERVICE_ID) && + fieldNameEquals(SERVICE_ID) && !isValidServiceId; return mergeFeedsResult.skippedIds.contains(key) || serviceIdShouldBeSkipped; } @@ -282,16 +281,17 @@ public void checkFieldsForMergeConflicts(Set idErrors) throws IOEx private Set getIdErrors() { Set idErrors; + Field field = fieldContext.getField(); // If analyzing the second feed (active feed), the service_id always gets feed scoped. // See https://github.com/ibi-group/datatools-server/issues/244 - if (handlingActiveFeed && field.name.equals(SERVICE_ID)) { - updateAndRemapValue(); + if (handlingActiveFeed && fieldNameEquals(SERVICE_ID)) { + updateAndRemapOutput(); idErrors = referenceTracker - .checkReferencesAndUniqueness(keyValue, lineNumber, field, valueToWrite, + .checkReferencesAndUniqueness(keyValue, lineNumber, field, fieldContext.getValueToWrite(), table, keyField, orderField); } else { idErrors = referenceTracker - .checkReferencesAndUniqueness(keyValue, lineNumber, field, val, + .checkReferencesAndUniqueness(keyValue, lineNumber, field, fieldContext.getValue(), table, keyField, orderField); } return idErrors; @@ -304,7 +304,7 @@ protected void checkRoutesAndStopsIds(Set idErrors) throws IOExcep // by the reference tracker. String primaryKeyValue = csvReader.get(table.getKeyFieldIndex(fieldsFoundInZip)); Set primaryKeyErrors = referenceTracker - .checkReferencesAndUniqueness(primaryKeyValue, lineNumber, field, val, table); + .checkReferencesAndUniqueness(primaryKeyValue, lineNumber, fieldContext.getField(), fieldContext.getValue(), table); // Merging will be based on route_short_name/stop_code in the active and future datasets. All // matching route_short_names/stop_codes between the datasets shall be considered same route/stop. Any // route_short_name/stop_code in active data not present in the future will be appended to the @@ -341,7 +341,7 @@ protected void checkRoutesAndStopsIds(Set idErrors) throws IOExcep // route/stop with already encountered matching // short name/stop code. String[] strings = rowValuesForStopOrRouteId.get( - String.join(":", keyField, val) + String.join(":", keyField, fieldContext.getValue()) ); String keyForMatchingAltId = strings[0]; if (!keyForMatchingAltId.equals(currentPrimaryKey)) { @@ -360,7 +360,7 @@ protected void checkRoutesAndStopsIds(Set idErrors) throws IOExcep if (!skipRecord && !referenceTracker.transitIds.contains(String.join(":", keyField, keyValue)) && hasDuplicateError(primaryKeyErrors)) { // Modify route_id and ensure that referencing trips // have route_id updated. - updateAndRemapValue(); + updateAndRemapOutput(); } } else { // Key field has defaulted to the standard primary key field @@ -369,16 +369,16 @@ protected void checkRoutesAndStopsIds(Set idErrors) throws IOExcep if (hasDuplicateError(idErrors)) skipRecord = true; } - if (newAgencyId != null && field.name.equals(AGENCY_ID)) { + if (newAgencyId != null && fieldNameEquals(AGENCY_ID)) { LOG.info( "Updating route#agency_id to (auto-generated) {} for route={}", newAgencyId, keyValue); - val = newAgencyId; + fieldContext.setValue(newAgencyId); } } private boolean hasBlankPrimaryKey() { - return "".equals(keyValue) && field.name.equals(table.getKeyFieldName()); + return "".equals(keyValue) && fieldNameEquals(table.getKeyFieldName()); } private boolean useAltKey() { @@ -386,8 +386,8 @@ private boolean useAltKey() { } public boolean updateAgencyIdIfNeeded() { - if (newAgencyId != null && field.name.equals(AGENCY_ID) && job.mergeType.equals(REGIONAL)) { - if (val.equals("") && table.name.equals("agency") && lineNumber > 0) { + if (newAgencyId != null && fieldNameEquals(AGENCY_ID) && job.mergeType.equals(REGIONAL)) { + if (fieldContext.getValue().equals("") && table.name.equals("agency") && lineNumber > 0) { // If there is no agency_id value for a second (or greater) agency // record, return null which will trigger a failed merge feed job. job.failMergeJob(String.format( @@ -397,20 +397,23 @@ public boolean updateAgencyIdIfNeeded() { return false; } LOG.info("Updating {}#agency_id to (auto-generated) {} for ID {}", table.name, newAgencyId, keyValue); - val = newAgencyId; + fieldContext.setValue(newAgencyId); } return true; } public void startNewField(int specFieldIndex) throws IOException { - this.field = sharedSpecFields.get(specFieldIndex); + Field field = sharedSpecFields.get(specFieldIndex); // Get index of field from GTFS spec as it appears in feed - index = fieldsFoundList.indexOf(field); - val = csvReader.get(index); + int index = fieldsFoundList.indexOf(field); // Default value to write is unchanged from value found in csv (i.e. val). Note: if looking to // modify the value that is written in the merged file, you must update valueToWrite (e.g., // updating this feed's end_date or accounting for cases where IDs conflict). - valueToWrite = val; + fieldContext = new FieldContext( + field, + index, + csvReader.get(index) + ); } public boolean storeRowAndStopValues() { @@ -463,11 +466,11 @@ public void checkFirstLineConditions() throws IOException { } public void scopeValueIfNeeded() { - boolean isKeyField = field.isForeignReference() || keyField.equals(field.name); - if (job.mergeType.equals(REGIONAL) && isKeyField && !val.isEmpty()) { + boolean isKeyField = fieldContext.getField().isForeignReference() || fieldNameEquals(keyField); + if (job.mergeType.equals(REGIONAL) && isKeyField && !fieldContext.getValue().isEmpty()) { // For regional merge, if field is a GTFS identifier (e.g., route_id, // stop_id, etc.), add scoped prefix. - valueToWrite = String.join(":", idScope, val); + fieldContext.setValueToWrite(String.join(":", idScope, fieldContext.getValue())); } } @@ -557,7 +560,7 @@ public boolean constructRowValues() throws IOException { // record. Likewise, if the reference has been modified, ensure that the value written to the // merged result is correctly updated. if (!areForeignRefsOk()) continue; - rowValues[specFieldIndex] = valueToWrite; + rowValues[specFieldIndex] = fieldContext.getValueToWrite(); } return true; } @@ -613,8 +616,12 @@ protected String getIdScope() { return idScope; } - protected Field getField() { - return field; + protected FieldContext getFieldContext() { + return fieldContext; + } + + protected boolean fieldNameEquals(String value) { + return fieldContext.getField().name.equals(value); } protected int getLineNumber() { @@ -639,13 +646,15 @@ protected LocalDate getCsvDate(String fieldName) throws IOException { /** * Updates output for the current field and remaps the record id. */ - protected void updateAndRemapValue(boolean updateKeyValue) { - valueToWrite = String.join(":", idScope, val); + protected void updateAndRemapOutput(boolean updateKeyValue) { + String value = fieldContext.getValue(); + String valueToWrite = String.join(":", idScope, value); + fieldContext.setValueToWrite(valueToWrite); if (updateKeyValue) { keyValue = valueToWrite; } mergeFeedsResult.remappedIds.put( - getTableScopedValue(table, idScope, val), + getTableScopedValue(table, idScope, value), valueToWrite ); } @@ -653,7 +662,7 @@ protected void updateAndRemapValue(boolean updateKeyValue) { /** * Shorthand for the above method. */ - protected void updateAndRemapValue() { - updateAndRemapValue(false); + protected void updateAndRemapOutput() { + updateAndRemapOutput(false); } } \ No newline at end of file diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/ShapesMergeLineContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/ShapesMergeLineContext.java index 07b363310..439b84ebd 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/ShapesMergeLineContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/ShapesMergeLineContext.java @@ -30,7 +30,8 @@ private void checkShapeIds(Set idErrors) { // shape_id:shape_pt_sequence values from both datasets (e.g., if future dataset contains // sequences 1,2,3,10 and active contains 1,2,7,9,10; the merged set will contain // 1,2,3,7,9,10). - if (getField().name.equals("shape_id")) { + if (fieldNameEquals("shape_id")) { + String val = getFieldContext().getValue(); if (isHandlingFutureFeed()) { // Track shape_id if working on future feed. shapeIdsInFutureFeed.add(val); @@ -38,12 +39,18 @@ private void checkShapeIds(Set idErrors) { // For the active feed, if the shape_id was already processed from the // future feed, we need to add the feed-scope to avoid weird, hybrid shapes // with points from both feeds. - updateAndRemapValue(true); + updateAndRemapOutput(true); // Re-check refs and uniqueness after changing shape_id value. (Note: this // probably won't have any impact, but there's not much harm in including it.) idErrors = referenceTracker - .checkReferencesAndUniqueness(keyValue, getLineNumber(), getField(), valueToWrite, - table, keyField, table.getOrderFieldName()); + .checkReferencesAndUniqueness( + keyValue, + getLineNumber(), + getFieldContext().getField(), + getFieldContext().getValueToWrite(), + table, + keyField, + table.getOrderFieldName()); } } // Skip record if normal duplicate errors are found. diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/TripsMergeLineContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/TripsMergeLineContext.java index ac91f8273..d3e17d809 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/TripsMergeLineContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/TripsMergeLineContext.java @@ -29,12 +29,12 @@ private void checkTripIds(Set idErrors) { if (job.tripIdsToSkipForActiveFeed.contains(keyValue)) { skipRecord = true; } else if (job.tripIdsToModifyForActiveFeed.contains(keyValue)) { - updateAndRemapValue(true); + updateAndRemapOutput(true); } } for (NewGTFSError error : idErrors) { if (error.errorType.equals(NewGTFSErrorType.DUPLICATE_ID)) { - updateAndRemapValue(true); + updateAndRemapOutput(true); } } } From 20c39fc1a63cde9db2968c556dcecc8a590c91b7 Mon Sep 17 00:00:00 2001 From: binh-dam-ibigroup <56846598+binh-dam-ibigroup@users.noreply.github.com> Date: Fri, 5 Nov 2021 18:00:58 -0400 Subject: [PATCH 054/122] refactor(MergeLineContext): Fix improper side effect. Fix broken test. --- .../datatools/manager/jobs/feedmerge/MergeLineContext.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java index 6f210a8f2..01c446bd6 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java @@ -229,8 +229,7 @@ public boolean areForeignRefsOk() throws IOException { String key = getTableScopedValue(field.referenceTable, idScope, fieldContext.getValue()); // Check if we're performing a service period merge, this ref field is a service_id, and it // is not found in the list of service_ids (e.g., it was removed). - String valueToWrite = fieldContext.getValueToWrite(); - boolean isValidServiceId = mergeFeedsResult.serviceIds.contains(valueToWrite); + boolean isValidServiceId = mergeFeedsResult.serviceIds.contains(fieldContext.getValueToWrite()); // If the current foreign ref points to another record that has // been skipped or is a ref to a non-existent service_id during a service period merge, skip @@ -241,7 +240,7 @@ public boolean areForeignRefsOk() throws IOException { // valid service_ids found in calendar_dates, do not skip that record for both the // calendar_date and any related trips. if (fieldNameEquals(SERVICE_ID) && isValidServiceId) { - LOG.warn("Not skipping valid service_id {} for {} {}", valueToWrite, table.name, keyValue); + LOG.warn("Not skipping valid service_id {} for {} {}", fieldContext.getValueToWrite(), table.name, keyValue); } else { String skippedKey = getTableScopedValue(table, idScope, keyValue); if (orderField != null) { @@ -258,7 +257,7 @@ public boolean areForeignRefsOk() throws IOException { if (mergeFeedsResult.remappedIds.containsKey(key)) { mergeFeedsResult.remappedReferences++; // If the value has been remapped update the value to write. - valueToWrite = mergeFeedsResult.remappedIds.get(key); + fieldContext.setValueToWrite(mergeFeedsResult.remappedIds.get(key)); } } return true; From 4065b90067199b894ee0b76a41ee187c3d676334 Mon Sep 17 00:00:00 2001 From: binh-dam-ibigroup <56846598+binh-dam-ibigroup@users.noreply.github.com> Date: Mon, 8 Nov 2021 09:07:43 -0500 Subject: [PATCH 055/122] refactor(MergeLineContext): Make various refactors. --- .../feedmerge/AgencyMergeLineContext.java | 8 +-- .../manager/jobs/feedmerge/FieldContext.java | 5 +- .../jobs/feedmerge/MergeLineContext.java | 60 ++++++++++++------- .../jobs/feedmerge/StopsMergeLineContext.java | 4 +- 4 files changed, 42 insertions(+), 35 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/AgencyMergeLineContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/AgencyMergeLineContext.java index 34255a240..c6db04056 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/AgencyMergeLineContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/AgencyMergeLineContext.java @@ -40,12 +40,8 @@ private void checkForMissingAgencyId() { newAgencyId = UUID.randomUUID().toString(); if (keyFieldMissing) { // Only add agency_id field if it is missing in table. - List fieldsList = new ArrayList<>(Arrays.asList(fieldsFoundInZip)); - fieldsList.add(Table.AGENCY.fields[0]); - fieldsFoundInZip = fieldsList.toArray(fieldsFoundInZip); - allFields.add(Table.AGENCY.fields[0]); + addField(Table.AGENCY.fields[0]); } - fieldsFoundList = Arrays.asList(fieldsFoundInZip); } } @@ -79,7 +75,7 @@ private boolean checkMismatchedAgency() { )); return true; } - LOG.warn("Skipping {} file for feed {}/{} (future file preferred)", table.name, feedIndex, feedsToMerge.size()); + LOG.warn("Skipping {} file for feed {}/{} (future file preferred)", table.name, getFeedIndex(), feedsToMerge.size()); skipFile = true; } return false; diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/FieldContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/FieldContext.java index 1615ef654..e829dfa8e 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/FieldContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/FieldContext.java @@ -8,14 +8,11 @@ */ public class FieldContext { private final Field field; - private final int index; private String value; private String valueToWrite; - public FieldContext(Field field, int index, String value) { + public FieldContext(Field field, String value) { this.field = field; - // Get index of field from GTFS spec as it appears in feed - this.index = index; // Default value to write is unchanged from value found in csv (i.e. val). Note: if looking to // modify the value that is written in the merged file, you must update valueToWrite (e.g., // updating this feed's end_date or accounting for cases where IDs conflict). diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java index 01c446bd6..3e14019aa 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java @@ -16,6 +16,7 @@ import java.io.IOException; import java.io.OutputStreamWriter; import java.time.LocalDate; +import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; @@ -44,7 +45,7 @@ public class MergeLineContext { private static final Logger LOG = LoggerFactory.getLogger(MergeLineContext.class); protected final MergeFeedsJob job; private final ZipOutputStream out; - protected final Set allFields; // try to make private + private final Set allFields; protected LocalDate futureFirstCalendarStartDate; protected final LocalDate activeFeedFirstDate; protected LocalDate futureFeedFirstDate; // try to make private @@ -68,14 +69,14 @@ public class MergeLineContext { protected final MergeFeedsResult mergeFeedsResult; protected final List feedsToMerge; protected int keyFieldIndex; - protected Field[] fieldsFoundInZip; // try to make private - protected List fieldsFoundList; + private Field[] fieldsFoundInZip; + private List fieldsFoundList; // Set up objects for tracking the rows encountered private final Map rowValuesForStopOrRouteId = new HashMap<>(); private final Set rowStrings = new HashSet<>(); private List sharedSpecFields; private FieldContext fieldContext; - protected int feedIndex; + private int feedIndex; public FeedVersion version; public FeedSource feedSource; @@ -301,7 +302,7 @@ protected void checkRoutesAndStopsIds(Set idErrors) throws IOExcep // in case the stop_code or route_short_name are being used. This // must occur unconditionally because each record must be tracked // by the reference tracker. - String primaryKeyValue = csvReader.get(table.getKeyFieldIndex(fieldsFoundInZip)); + String primaryKeyValue = csvReader.get(getKeyFieldIndex()); Set primaryKeyErrors = referenceTracker .checkReferencesAndUniqueness(primaryKeyValue, lineNumber, fieldContext.getField(), fieldContext.getValue(), table); // Merging will be based on route_short_name/stop_code in the active and future datasets. All @@ -401,20 +402,6 @@ public boolean updateAgencyIdIfNeeded() { return true; } - public void startNewField(int specFieldIndex) throws IOException { - Field field = sharedSpecFields.get(specFieldIndex); - // Get index of field from GTFS spec as it appears in feed - int index = fieldsFoundList.indexOf(field); - // Default value to write is unchanged from value found in csv (i.e. val). Note: if looking to - // modify the value that is written in the merged file, you must update valueToWrite (e.g., - // updating this feed's end_date or accounting for cases where IDs conflict). - fieldContext = new FieldContext( - field, - index, - csvReader.get(index) - ); - } - public boolean storeRowAndStopValues() { String newLine = String.join(",", rowValues); switch (table.name) { @@ -534,7 +521,14 @@ public boolean constructRowValues() throws IOException { // There is nothing to do in this loop if it has already been determined that the record should // be skipped. if (skipRecord) break; - startNewField(specFieldIndex); + Field field = sharedSpecFields.get(specFieldIndex); + // Default value to write is unchanged from value found in csv (i.e. val). Note: if looking to + // modify the value that is written in the merged file, you must update valueToWrite (e.g., + // updating this feed's end_date or accounting for cases where IDs conflict). + fieldContext = new FieldContext( + field, + csvReader.get(fieldsFoundList.indexOf(field)) + ); // Handle filling in agency_id if missing when merging regional feeds. If false is returned, // the job has encountered a failing condition (the method handles failing the job itself). if (!updateAgencyIdIfNeeded()) { @@ -548,17 +542,17 @@ public boolean constructRowValues() throws IOException { // track references for a large number of feeds (e.g., every feed in New // York State). if (job.mergeType.equals(SERVICE_PERIOD)) { - Set idErrors = getIdErrors(); + Set idErrors = getIdErrors(); // FIXME: Rename. This method is imperative. // Store values for key fields that have been encountered and update any key values that need modification due // to conflicts. - checkFieldsForMergeConflicts(idErrors); + checkFieldsForMergeConflicts(idErrors); // FIXME: This method changes skipRecord; if (skipRecord) continue; } // If the current field is a foreign reference, check if the reference has been removed in the // merged result. If this is the case (or other conditions are met), we will need to skip this // record. Likewise, if the reference has been modified, ensure that the value written to the // merged result is correctly updated. - if (!areForeignRefsOk()) continue; + if (!areForeignRefsOk()) continue; // FIXME: This method changes skipRecord; rowValues[specFieldIndex] = fieldContext.getValueToWrite(); } return true; @@ -619,6 +613,8 @@ protected FieldContext getFieldContext() { return fieldContext; } + protected int getFeedIndex() { return feedIndex; } + protected boolean fieldNameEquals(String value) { return fieldContext.getField().name.equals(value); } @@ -664,4 +660,22 @@ protected void updateAndRemapOutput(boolean updateKeyValue) { protected void updateAndRemapOutput() { updateAndRemapOutput(false); } + + /** + * Add the specified field once record reading has started. + */ + protected void addField(Field field) { + List fieldsList = new ArrayList<>(Arrays.asList(fieldsFoundInZip)); + fieldsList.add(field); + fieldsFoundInZip = fieldsList.toArray(fieldsFoundInZip); + allFields.add(field); + fieldsFoundList = Arrays.asList(fieldsFoundInZip); + } + + /** + * Helper method to get the key field position. + */ + protected int getKeyFieldIndex() { + return table.getKeyFieldIndex(fieldsFoundInZip); + } } \ No newline at end of file diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/StopsMergeLineContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/StopsMergeLineContext.java index 2805bd1b7..7816896c4 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/StopsMergeLineContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/StopsMergeLineContext.java @@ -70,10 +70,10 @@ private void checkStopCodeStuff() throws IOException { // not require stop_code), we simply default to merging on stop_id. LOG.warn( "stop_code is not present in file {}/{}. Reverting to stop_id", - feedIndex + 1, feedsToMerge.size()); + getFeedIndex() + 1, feedsToMerge.size()); // If the key value for stop_code is not present, revert to stop_id. keyField = table.getKeyFieldName(); - keyFieldIndex = table.getKeyFieldIndex(fieldsFoundInZip); + keyFieldIndex = getKeyFieldIndex(); keyValue = getCsvReader().get(keyFieldIndex); // When all stops missing stop_code for the first feed, there's nothing to do (i.e., // no failure condition has been triggered yet). Just indicate this in the flag and From b87b5756f90c6a2c721f65018a7e3c8cc4ef67cb Mon Sep 17 00:00:00 2001 From: binh-dam-ibigroup <56846598+binh-dam-ibigroup@users.noreply.github.com> Date: Mon, 8 Nov 2021 14:12:32 -0500 Subject: [PATCH 056/122] refactor(MergeFeedsJob): Extract FeedMergeContext. --- .../datatools/manager/jobs/MergeFeedsJob.java | 108 +++++++--------- .../jobs/feedmerge/FeedMergeContext.java | 116 ++++++++++++++++++ .../manager/jobs/feedmerge/MergeStrategy.java | 6 - .../manager/jobs/MergeFeedsJobTest.java | 8 +- 4 files changed, 164 insertions(+), 74 deletions(-) create mode 100644 src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/FeedMergeContext.java diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java b/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java index 53f618715..68891e262 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java @@ -6,6 +6,7 @@ import com.conveyal.datatools.manager.DataManager; import com.conveyal.datatools.manager.auth.Auth0UserProfile; import com.conveyal.datatools.manager.gtfsplus.tables.GtfsPlusTable; +import com.conveyal.datatools.manager.jobs.feedmerge.FeedMergeContext; import com.conveyal.datatools.manager.jobs.feedmerge.FeedToMerge; import com.conveyal.datatools.manager.jobs.feedmerge.MergeFeedsResult; import com.conveyal.datatools.manager.jobs.feedmerge.MergeFeedsType; @@ -139,7 +140,8 @@ public class MergeFeedsJob extends FeedSourceJob { public Set serviceIdsToExtend = new HashSet<>(); @JsonIgnore @BsonIgnore public Set serviceIdsToCloneAndRename = new HashSet<>(); - private List feedsToMerge; + // Variables used for a service period merge. + private FeedMergeContext feedMergeContext; public MergeFeedsJob(Auth0UserProfile owner, Set feedVersions, String file, MergeFeedsType mergeType) { this(owner, feedVersions, file, mergeType, true); @@ -148,13 +150,13 @@ public MergeFeedsJob(Auth0UserProfile owner, Set feedVersions, Stri /** Shorthand method to get the future feed during a service period merge */ @BsonIgnore @JsonIgnore public FeedToMerge getFutureFeed() { - return feedsToMerge.get(0); + return feedMergeContext.futureFeedToMerge; } /** Shorthand method to get the active feed during a service period merge */ @BsonIgnore @JsonIgnore public FeedToMerge getActiveFeed() { - return feedsToMerge.get(1); + return feedMergeContext.activeFeedToMerge; } /** @@ -210,7 +212,7 @@ public Set getFeedVersions() { @BsonIgnore @JsonIgnore public List getFeedsToMerge() { - return this.feedsToMerge; + return this.feedMergeContext.feedsToMerge; } /** @@ -251,21 +253,36 @@ public void jobLogic() { // Create the zipfile with try with resources so that it is always closed. try (ZipOutputStream out = new ZipOutputStream(new FileOutputStream(mergedTempFile))) { LOG.info("Created merge file: {}", mergedTempFile.getAbsolutePath()); - feedsToMerge = collectAndSortFeeds(feedVersions, owner); + feedMergeContext = new FeedMergeContext(feedVersions, owner); // Determine which tables to merge (only merge GTFS+ tables for MTC extension). final List
tablesToMerge = getTablesToMerge(); int numberOfTables = tablesToMerge.size(); + + // Skip merging process altogether if the failing condition is met. + FeedMergeContext.TripMismatchedServiceIds serviceIdMismatch; + if (feedMergeContext.areTripIdsMatchingButNotServiceIds()) { + failMergeJob("Feed merge failed because the trip_ids are identical in the future and active feeds. A new service requires unique trip_ids for merging."); + return; + } else if ((serviceIdMismatch = feedMergeContext.shouldFailJobDueToMatchingTripIds()) != null) { + // We cannot account for the case where service_ids do not match! It would be a bit too complicated + // to handle this unique case, so instead just include in the failure reasons and use failure + // strategy. + failMergeJob( + String.format("Shared trip_id (%s) had mismatched service id between two feeds (active: %s, future: %s)", + serviceIdMismatch.tripId, + serviceIdMismatch.activeServiceId, + serviceIdMismatch.futureServiceId + ) + ); + return; + } + // Before initiating the merge process, get the merge strategy to use, which runs some pre-processing to // check for id conflicts for certain tables (e.g., trips and calendars). if (mergeType.equals(SERVICE_PERIOD)) { mergeFeedsResult.mergeStrategy = getMergeStrategy(); } - // Skip merging process altogether if the failing condition is met. - if (MergeStrategy.FAIL_DUE_TO_MATCHING_TRIP_IDS.equals(mergeFeedsResult.mergeStrategy)) { - failMergeJob("Feed merge failed because the trip_ids are identical in the future and active feeds. A new service requires unique trip_ids for merging."); - return; - } // Loop over GTFS tables and merge each feed one table at a time. for (int i = 0; i < numberOfTables; i++) { Table table = tablesToMerge.get(i); @@ -274,7 +291,7 @@ public void jobLogic() { status.update("Merging " + table.name, percentComplete); // Perform the merge. LOG.info("Writing {} to merged feed", table.name); - int mergedLineNumber = constructMergedTable(table, feedsToMerge, out); + int mergedLineNumber = constructMergedTable(table, feedMergeContext.feedsToMerge, out); if (mergedLineNumber == 0) { LOG.warn("Skipping {} table. No entries found in zip files.", table.name); } else if (mergedLineNumber == -1) { @@ -286,12 +303,10 @@ public void jobLogic() { logAndReportToBugsnag(e, message); status.fail(message, e); } finally { - for (FeedToMerge feed : feedsToMerge) { - try { - feed.close(); - } catch (IOException e) { - logAndReportToBugsnag(e, "Error closing FeedToMerge object"); - } + try { + feedMergeContext.close(); + } catch (IOException e) { + logAndReportToBugsnag(e, "Error closing FeedMergeContext object"); } } if (!mergeFeedsResult.failed) { @@ -447,39 +462,27 @@ private int constructMergedTable(Table table, List feedsToMerge, Zi * Get the merge strategy to use for MTC service period merges by checking the active and future feeds for various * combinations of matching trip and service IDs. */ - private MergeStrategy getMergeStrategy() throws IOException { - boolean shouldFailJob = false; - FeedToMerge futureFeedToMerge = getFutureFeed(); - FeedToMerge activeFeedToMerge = getActiveFeed(); - futureFeedToMerge.collectTripAndServiceIds(); - activeFeedToMerge.collectTripAndServiceIds(); - Set activeTripIds = activeFeedToMerge.idsForTable.get(Table.TRIPS); - Set futureTripIds = futureFeedToMerge.idsForTable.get(Table.TRIPS); - // Determine whether service and trip IDs are exact matches. - boolean serviceIdsMatch = activeFeedToMerge.serviceIds.equals(futureFeedToMerge.serviceIds); - boolean tripIdsMatch = activeTripIds.equals(futureTripIds); - if (tripIdsMatch) { + private MergeStrategy getMergeStrategy() { + if (feedMergeContext.tripIdsMatch) { // Effectively this exact match condition means that the future feed will be used as is // (including stops, routes, etc.), the only modification being service date ranges. // This is Condition 2 in the docs. - // If only trip IDs match, do not permit merge to continue. - return serviceIdsMatch ? EXTEND_FUTURE : MergeStrategy.FAIL_DUE_TO_MATCHING_TRIP_IDS; + // (If only trip IDs match, do not permit merge to continue + // - that is handled by method shouldFailJobDueToMatchingTripIds.) + return feedMergeContext.serviceIdsMatch ? EXTEND_FUTURE : MergeStrategy.DEFAULT; } - if (serviceIdsMatch) { + if (feedMergeContext.serviceIdsMatch) { // If just the service_ids are an exact match, check the that the stop_times having matching signatures // between the two feeds (i.e., each stop time in the ordered list is identical between the two feeds). - Feed futureFeed = new Feed(DataManager.GTFS_DATA_SOURCE, futureFeedToMerge.version.namespace); - Feed activeFeed = new Feed(DataManager.GTFS_DATA_SOURCE, activeFeedToMerge.version.namespace); - Set sharedTripIds = Sets.intersection(activeTripIds, futureTripIds); - for (String tripId : sharedTripIds) { - if (compareStopTimes(tripId, futureFeed, activeFeed)) { - shouldFailJob = true; - } + Feed futureFeed = feedMergeContext.futureFeed; + Feed activeFeed = feedMergeContext.activeFeed; + for (String tripId : feedMergeContext.sharedTripIds) { + compareStopTimesAndCollectTripAndServiceIds(tripId, futureFeed, activeFeed); } // If a trip only in the active feed references a service_id that is set to be extended, that // service_id needs to be cloned and renamed to differentiate it from the same service_id in // the future feed. (The trip in question will be linked to the cloned service_id.) - Set tripsOnlyInActiveFeed = Sets.difference(activeTripIds, futureTripIds); + Set tripsOnlyInActiveFeed = Sets.difference(feedMergeContext.activeTripIds, feedMergeContext.futureTripIds); tripsOnlyInActiveFeed.stream() .map(tripId -> activeFeed.trips.get(tripId).service_id) .filter(serviceId -> serviceIdsToExtend.contains(serviceId)) @@ -487,14 +490,13 @@ private MergeStrategy getMergeStrategy() throws IOException { // If a trip only in the future feed references a service_id that is set to be extended, that // service_id needs to be cloned and renamed to differentiate it from the same service_id in // the future feed. (The trip in question will be linked to the cloned service_id.) - Set tripsOnlyInFutureFeed = Sets.difference(futureTripIds, activeTripIds); + Set tripsOnlyInFutureFeed = Sets.difference(feedMergeContext.futureTripIds, feedMergeContext.activeTripIds); tripsOnlyInFutureFeed.stream() .map(tripId -> futureFeed.trips.get(tripId).service_id) .filter(serviceId -> serviceIdsToExtend.contains(serviceId)) .forEach(serviceId -> serviceIdsToCloneAndRename.add(serviceId)); - // If a failure was encountered above, use failure strategy. Otherwise, use check stop times to proceed with - // feed merge. - return shouldFailJob ? MergeStrategy.FAIL_DUE_TO_MATCHING_TRIP_IDS : CHECK_STOP_TIMES; + + return CHECK_STOP_TIMES; } // If neither the trips or services are exact matches, use the default merge strategy. return MergeStrategy.DEFAULT; @@ -503,29 +505,14 @@ private MergeStrategy getMergeStrategy() throws IOException { /** * Compare stop times for the given tripId between the future and active feeds. The comparison will inform whether * trip and/or service IDs should be modified in the output merged feed. - * @return true if an error was encountered during the check. false if no error was encountered. */ - private boolean compareStopTimes(String tripId, Feed futureFeed, Feed activeFeed) { + private void compareStopTimesAndCollectTripAndServiceIds(String tripId, Feed futureFeed, Feed activeFeed) { // Fetch all ordered stop_times for each shared trip_id and compare the two sets for the // future and active feed. If the stop_times are an exact match, include one instance of the trip // (ignoring the other identical one). If they do not match, modify the active trip_id and include. List futureStopTimes = Lists.newArrayList(futureFeed.stopTimes.getOrdered(tripId)); List activeStopTimes = Lists.newArrayList(activeFeed.stopTimes.getOrdered(tripId)); String futureServiceId = futureFeed.trips.get(tripId).service_id; - String activeServiceId = activeFeed.trips.get(tripId).service_id; - if (!futureServiceId.equals(activeServiceId)) { - // We cannot account for the case where service_ids do not match! It would be a bit too complicated - // to handle this unique case, so instead just include in the failure reasons and use failure - // strategy. - failMergeJob( - String.format("Shared trip_id (%s) had mismatched service id between two feeds (active: %s, future: %s)", - tripId, - activeServiceId, - futureServiceId - ) - ); - return true; - } if (!stopTimesMatch(futureStopTimes, activeStopTimes)) { // If stop_times or services do not match, the trip will be cloned. Also, track the service_id // (it will need to be cloned and renamed for both active feeds). @@ -538,7 +525,6 @@ private boolean compareStopTimes(String tripId, Feed futureFeed, Feed activeFeed tripIdsToSkipForActiveFeed.add(tripId); serviceIdsToExtend.add(futureServiceId); } - return false; } public String getFeedSourceId() { diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/FeedMergeContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/FeedMergeContext.java new file mode 100644 index 000000000..812769fbb --- /dev/null +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/FeedMergeContext.java @@ -0,0 +1,116 @@ +package com.conveyal.datatools.manager.jobs.feedmerge; + +import com.conveyal.datatools.manager.DataManager; +import com.conveyal.datatools.manager.auth.Auth0UserProfile; +import com.conveyal.datatools.manager.models.FeedVersion; +import com.conveyal.datatools.manager.utils.MergeFeedUtils; +import com.conveyal.gtfs.loader.Feed; +import com.conveyal.gtfs.loader.Table; +import com.google.common.collect.Sets; + +import java.io.Closeable; +import java.io.IOException; +import java.util.List; +import java.util.Set; + +/** + * Contains merge information between an active feed and a future feed. + */ +public class FeedMergeContext implements Closeable { + public final List feedsToMerge; + public final FeedToMerge activeFeedToMerge; + public final FeedToMerge futureFeedToMerge; + public final Set activeTripIds; + public final Set futureTripIds; + public final boolean serviceIdsMatch; + public final boolean tripIdsMatch; + public final Feed futureFeed; + public final Feed activeFeed; + /** Trip ids shared between the active and future feed. */ + public final Set sharedTripIds; + + + public FeedMergeContext(Set feedVersions, Auth0UserProfile owner) throws IOException { + feedsToMerge = MergeFeedUtils.collectAndSortFeeds(feedVersions, owner); + activeFeedToMerge = feedsToMerge.get(1); + futureFeedToMerge = feedsToMerge.get(0); + futureFeedToMerge.collectTripAndServiceIds(); + activeFeedToMerge.collectTripAndServiceIds(); + activeTripIds = activeFeedToMerge.idsForTable.get(Table.TRIPS); + futureTripIds = futureFeedToMerge.idsForTable.get(Table.TRIPS); + + // Determine whether service and trip IDs are exact matches. + serviceIdsMatch = activeFeedToMerge.serviceIds.equals(futureFeedToMerge.serviceIds); + tripIdsMatch = activeTripIds.equals(futureTripIds); + + futureFeed = new Feed(DataManager.GTFS_DATA_SOURCE, futureFeedToMerge.version.namespace); + activeFeed = new Feed(DataManager.GTFS_DATA_SOURCE, activeFeedToMerge.version.namespace); + sharedTripIds = Sets.intersection(activeTripIds, futureTripIds); + } + + @Override + public void close() throws IOException { + for (FeedToMerge feed : feedsToMerge) { + feed.close(); + } + } + + /** + * Partially handles MTC Requirement to detect matching trip ids linked to different service ids. + * @return true if trip ids match but not service ids (in such situation, merge should fail). + */ + public boolean areTripIdsMatchingButNotServiceIds() { + // If only trip IDs match and not service IDs, do not permit merge to continue. + return tripIdsMatch && !serviceIdsMatch; + } + + /** + * Determines whether there a same trip id is linked to different service ids in the active and future feed + * (MTC requirement). + * @return the first {@link TripMismatchedServiceIds} whose trip id is linked to different service ids, + * or null if nothing found. + */ + public TripMismatchedServiceIds shouldFailJobDueToMatchingTripIds() { + if (serviceIdsMatch) { + // If just the service_ids are an exact match, check the that the stop_times having matching signatures + // between the two feeds (i.e., each stop time in the ordered list is identical between the two feeds). + for (String tripId : sharedTripIds) { + TripMismatchedServiceIds mismatchInfo = tripIdHasMismatchedServiceIds(tripId, futureFeed, activeFeed); + if (mismatchInfo.hasMismatch) { + return mismatchInfo; + } + } + } + + return null; + } + + /** + * Compare stop times for the given tripId between the future and active feeds. The comparison will inform whether + * trip and/or service IDs should be modified in the output merged feed. + * @return A {@link TripMismatchedServiceIds} with info on whether the given tripId is found in + * different service ids in the active and future feed. + */ + public static TripMismatchedServiceIds tripIdHasMismatchedServiceIds(String tripId, Feed futureFeed, Feed activeFeed) { + String futureServiceId = futureFeed.trips.get(tripId).service_id; + String activeServiceId = activeFeed.trips.get(tripId).service_id; + return new TripMismatchedServiceIds(tripId, !futureServiceId.equals(activeServiceId), activeServiceId, futureServiceId); + } + + /** + * Holds the status of a trip service id mismatch determination. + */ + public static class TripMismatchedServiceIds { + public final String tripId; + public final String activeServiceId; + public final String futureServiceId; + public final boolean hasMismatch; + + TripMismatchedServiceIds(String tripId, boolean hasMismatch, String activeServiceId, String futureServiceId) { + this.tripId = tripId; + this.hasMismatch = hasMismatch; + this.activeServiceId = activeServiceId; + this.futureServiceId = futureServiceId; + } + } +} diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeStrategy.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeStrategy.java index fd26ff2d4..50f26b218 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeStrategy.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeStrategy.java @@ -18,12 +18,6 @@ public enum MergeStrategy { * should be the merged date. All files from the future feed only shall be used in the merged feed. */ EXTEND_FUTURE, - /** - * If trip_ids provided in active and future feeds are the same but the service_ids are unique then merge - * functionality shall reject feeds from merging. The user shall be notified that a new service requires unique - * trip_ids for merging. - */ - FAIL_DUE_TO_MATCHING_TRIP_IDS, /** * If service_ids in active and future feed exactly match but only some of the trip_ids match then the merge * strategy shall handle the following three cases: diff --git a/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java b/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java index deb27564d..c64b3b1c1 100644 --- a/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java +++ b/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java @@ -293,8 +293,7 @@ public void canMergeRegionalWithOnlyCalendarFeed () throws SQLException { } /** - * Ensures that an MTC merge of feeds that has exactly matching trips but mismatched services fails according to the - * strategy {@link MergeStrategy#FAIL_DUE_TO_MATCHING_TRIP_IDS}. + * Ensures that an MTC merge of feeds that has exactly matching trips but mismatched services fails. */ @Test public void mergeMTCShouldFailOnDuplicateTripsButMismatchedServices() { @@ -304,11 +303,6 @@ public void mergeMTCShouldFailOnDuplicateTripsButMismatchedServices() { MergeFeedsJob mergeFeedsJob = new MergeFeedsJob(user, versions, "merged_output", MergeFeedsType.SERVICE_PERIOD); // Run the job in this thread (we're not concerned about concurrency here). mergeFeedsJob.run(); - // Check that correct strategy was used. - assertEquals( - MergeStrategy.FAIL_DUE_TO_MATCHING_TRIP_IDS, - mergeFeedsJob.mergeFeedsResult.mergeStrategy - ); // Result should fail. assertTrue( mergeFeedsJob.mergeFeedsResult.failed, From bde820d1f677a61c439b8b59552d4786f34053af Mon Sep 17 00:00:00 2001 From: binh-dam-ibigroup <56846598+binh-dam-ibigroup@users.noreply.github.com> Date: Mon, 8 Nov 2021 15:10:34 -0500 Subject: [PATCH 057/122] refactor: Did other refactors. --- .../datatools/manager/jobs/MergeFeedsJob.java | 5 ++++ .../feedmerge/AgencyMergeLineContext.java | 6 +--- .../CalendarDatesMergeLineContext.java | 8 +++-- .../feedmerge/CalendarMergeLineContext.java | 7 +++-- .../jobs/feedmerge/FeedMergeContext.java | 29 +++++++++++++++++++ .../jobs/feedmerge/MergeLineContext.java | 21 ++++---------- .../jobs/feedmerge/StopsMergeLineContext.java | 2 +- 7 files changed, 50 insertions(+), 28 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java b/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java index 68891e262..1221dcf07 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java @@ -535,4 +535,9 @@ private void logAndReportToBugsnag(Exception e, String message, Object... args) LOG.error(message, args, e); ErrorUtils.reportToBugsnag(e, "datatools", message, owner); } + + @BsonIgnore @JsonIgnore + public FeedMergeContext getFeedMergeContext() { + return feedMergeContext; + } } diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/AgencyMergeLineContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/AgencyMergeLineContext.java index c6db04056..a186e0ff0 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/AgencyMergeLineContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/AgencyMergeLineContext.java @@ -1,15 +1,11 @@ package com.conveyal.datatools.manager.jobs.feedmerge; import com.conveyal.datatools.manager.jobs.MergeFeedsJob; -import com.conveyal.gtfs.loader.Field; import com.conveyal.gtfs.loader.Table; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; import java.util.UUID; import java.util.zip.ZipOutputStream; @@ -75,7 +71,7 @@ private boolean checkMismatchedAgency() { )); return true; } - LOG.warn("Skipping {} file for feed {}/{} (future file preferred)", table.name, getFeedIndex(), feedsToMerge.size()); + LOG.warn("Skipping {} file for feed {}/{} (future file preferred)", table.name, getFeedIndex(), feedMergeContext.feedsToMerge.size()); skipFile = true; } return false; diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarDatesMergeLineContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarDatesMergeLineContext.java index 4c8f51ac5..aafc2fde2 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarDatesMergeLineContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarDatesMergeLineContext.java @@ -36,6 +36,7 @@ private void checkCalendarDatesIds() throws IOException { // Drop any calendar_dates.txt records from the existing feed for dates that are // not before the first date of the future feed. LocalDate date = getCsvDate("date"); + LocalDate futureFeedFirstDate = feedMergeContext.getFutureFeedFirstDate(); if (isHandlingActiveFeed() && !date.isBefore(futureFeedFirstDate)) { LOG.warn( "Skipping calendar_dates entry {} because it operates in the time span of future feed (i.e., after or on {}).", @@ -52,16 +53,17 @@ private void checkCalendarDatesIds() throws IOException { } private void updateFutureFeedFirstDate() { + LocalDate futureFirstCalendarStartDate = feedMergeContext.getFutureFirstCalendarStartDate(); if ( isHandlingActiveFeed() && job.mergeType.equals(SERVICE_PERIOD) && - futureFirstCalendarStartDate.isBefore(LocalDate.MAX) && - futureFeedFirstDate.isBefore(futureFirstCalendarStartDate) + futureFirstCalendarStartDate.isBefore(LocalDate.MAX) && + feedMergeContext.getFutureFeedFirstDate().isBefore(futureFirstCalendarStartDate) ) { // If the future feed's first date is before its first calendar start date, // override the future feed first date with the calendar start date for use when checking // MTC calendar_dates and calendar records for modification/exclusion. - futureFeedFirstDate = futureFirstCalendarStartDate; + feedMergeContext.setFutureFeedFirstDate(futureFirstCalendarStartDate); } } } \ No newline at end of file diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarMergeLineContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarMergeLineContext.java index 75e8a3475..622a75756 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarMergeLineContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarMergeLineContext.java @@ -49,8 +49,8 @@ private void checkCalendarIds(Set idErrors) throws IOException { if (isHandlingFutureFeed()) { // For the future feed, check if the calendar's start date is earlier than the // previous earliest value and update if so. - if (futureFirstCalendarStartDate.isAfter(startDate)) { - futureFirstCalendarStartDate = startDate; + if (feedMergeContext.getFutureFirstCalendarStartDate().isAfter(startDate)) { + feedMergeContext.setFutureFirstCalendarStartDate(startDate); } // FIXME: Move this below so that a cloned service doesn't get prematurely // modified? (do we want the cloned record to have the original values?) @@ -59,7 +59,7 @@ private void checkCalendarIds(Set idErrors) throws IOException { // start date if the merge strategy dictates. The justification for this logic is that the active feed's // service_id will be modified to a different unique value and the trips shared between the future/active // service are exactly matching. - getFieldContext().resetValue(activeFeedFirstDate.format(GTFS_DATE_FORMATTER)); + getFieldContext().resetValue(feedMergeContext.activeFeedFirstDate.format(GTFS_DATE_FORMATTER)); } } else { // If a service_id from the active calendar has both the @@ -68,6 +68,7 @@ private void checkCalendarIds(Set idErrors) throws IOException { // calendar_dates, and calendar_attributes referencing this // service_id shall also be removed/ignored. Stop_time records // for the ignored trips shall also be removed. + LocalDate futureFeedFirstDate = feedMergeContext.getFutureFeedFirstDate(); if (!startDate.isBefore(futureFeedFirstDate)) { LOG.warn( "Skipping calendar entry {} because it operates fully within the time span of future feed.", diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/FeedMergeContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/FeedMergeContext.java index 812769fbb..c2360db4e 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/FeedMergeContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/FeedMergeContext.java @@ -10,6 +10,7 @@ import java.io.Closeable; import java.io.IOException; +import java.time.LocalDate; import java.util.List; import java.util.Set; @@ -26,6 +27,9 @@ public class FeedMergeContext implements Closeable { public final boolean tripIdsMatch; public final Feed futureFeed; public final Feed activeFeed; + public final LocalDate activeFeedFirstDate; + private LocalDate futureFeedFirstDate; + private LocalDate futureFirstCalendarStartDate = LocalDate.MAX; /** Trip ids shared between the active and future feed. */ public final Set sharedTripIds; @@ -46,6 +50,15 @@ public FeedMergeContext(Set feedVersions, Auth0UserProfile owner) t futureFeed = new Feed(DataManager.GTFS_DATA_SOURCE, futureFeedToMerge.version.namespace); activeFeed = new Feed(DataManager.GTFS_DATA_SOURCE, activeFeedToMerge.version.namespace); sharedTripIds = Sets.intersection(activeTripIds, futureTripIds); + + // Initialize future and active feed's first date to the first calendar date from validation result. + // This is equivalent to either the earliest date of service defined for a calendar_date record or the + // earliest start_date value for a calendars.txt record. For MTC, however, they require that GTFS + // providers use calendars.txt entries and prefer that this value (which is used to determine cutoff + // dates for the active feed when merging with the future) be strictly assigned the earliest + // calendar#start_date (unless that table for some reason does not exist). + activeFeedFirstDate = activeFeedToMerge.version.validationResult.firstCalendarDate; + futureFeedFirstDate = futureFeedToMerge.version.validationResult.firstCalendarDate; } @Override @@ -97,6 +110,22 @@ public static TripMismatchedServiceIds tripIdHasMismatchedServiceIds(String trip return new TripMismatchedServiceIds(tripId, !futureServiceId.equals(activeServiceId), activeServiceId, futureServiceId); } + public LocalDate getFutureFirstCalendarStartDate() { + return futureFirstCalendarStartDate; + } + + public void setFutureFirstCalendarStartDate(LocalDate futureFirstCalendarStartDate) { + this.futureFirstCalendarStartDate = futureFirstCalendarStartDate; + } + + public LocalDate getFutureFeedFirstDate() { + return futureFeedFirstDate; + } + + public void setFutureFeedFirstDate(LocalDate futureFeedFirstDate) { + this.futureFeedFirstDate = futureFeedFirstDate; + } + /** * Holds the status of a trip service id mismatch determination. */ diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java index 3e14019aa..29fe5117f 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java @@ -46,9 +46,6 @@ public class MergeLineContext { protected final MergeFeedsJob job; private final ZipOutputStream out; private final Set allFields; - protected LocalDate futureFirstCalendarStartDate; - protected final LocalDate activeFeedFirstDate; - protected LocalDate futureFeedFirstDate; // try to make private private boolean handlingActiveFeed; private boolean handlingFutureFeed; private String idScope; @@ -67,7 +64,7 @@ public class MergeLineContext { protected String keyField; private String orderField; protected final MergeFeedsResult mergeFeedsResult; - protected final List feedsToMerge; + protected final FeedMergeContext feedMergeContext; protected int keyFieldIndex; private Field[] fieldsFoundInZip; private List fieldsFoundList; @@ -107,22 +104,13 @@ public static MergeLineContext create(MergeFeedsJob job, Table table, ZipOutputS protected MergeLineContext(MergeFeedsJob job, Table table, ZipOutputStream out) throws IOException { this.job = job; this.table = table; - this.feedsToMerge = job.getFeedsToMerge(); + this.feedMergeContext = job.getFeedMergeContext(); // Get shared fields between all feeds being merged. This is used to filter the spec fields so that only // fields found in the collection of feeds are included in the merged table. - allFields = getAllFields(feedsToMerge, table); + allFields = getAllFields(feedMergeContext.feedsToMerge, table); this.mergeFeedsResult = job.mergeFeedsResult; this.writer = new CsvListWriter(new OutputStreamWriter(out), CsvPreference.STANDARD_PREFERENCE); this.out = out; - // Initialize future and active feed's first date to the first calendar date from validation result. - // This is equivalent to either the earliest date of service defined for a calendar_date record or the - // earliest start_date value for a calendars.txt record. For MTC, however, they require that GTFS - // providers use calendars.txt entries and prefer that this value (which is used to determine cutoff - // dates for the active feed when merging with the future) be strictly assigned the earliest - // calendar#start_date (unless that table for some reason does not exist). - futureFeedFirstDate = job.getFutureFeed().version.validationResult.firstCalendarDate; - activeFeedFirstDate = job.getActiveFeed().version.validationResult.firstCalendarDate; - futureFirstCalendarStartDate = LocalDate.MAX; } public void startNewFeed(int feedIndex) throws IOException { @@ -130,7 +118,7 @@ public void startNewFeed(int feedIndex) throws IOException { handlingActiveFeed = feedIndex > 0; handlingFutureFeed = feedIndex == 0; this.feedIndex = feedIndex; - this.feed = feedsToMerge.get(feedIndex); + this.feed = feedMergeContext.feedsToMerge.get(feedIndex); this.version = feed.version; this.feedSource = version.parentFeedSource(); keyField = getMergeKeyField(table, job.mergeType); @@ -274,6 +262,7 @@ private boolean serviceIdHasOrShouldBeSkipped(String key, boolean isValidService /** * Overridable method whose default behavior below is to skip a record if it creates a duplicate id. + * @throws IOException Some overrides throw IOException. */ public void checkFieldsForMergeConflicts(Set idErrors) throws IOException { if (hasDuplicateError(idErrors)) skipRecord = true; diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/StopsMergeLineContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/StopsMergeLineContext.java index 7816896c4..4a3c54f42 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/StopsMergeLineContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/StopsMergeLineContext.java @@ -70,7 +70,7 @@ private void checkStopCodeStuff() throws IOException { // not require stop_code), we simply default to merging on stop_id. LOG.warn( "stop_code is not present in file {}/{}. Reverting to stop_id", - getFeedIndex() + 1, feedsToMerge.size()); + getFeedIndex() + 1, feedMergeContext.feedsToMerge.size()); // If the key value for stop_code is not present, revert to stop_id. keyField = table.getKeyFieldName(); keyFieldIndex = getKeyFieldIndex(); From e80950ce7e1251fa5d40595afb2bd91081c1da16 Mon Sep 17 00:00:00 2001 From: binh-dam-ibigroup <56846598+binh-dam-ibigroup@users.noreply.github.com> Date: Mon, 8 Nov 2021 16:06:17 -0500 Subject: [PATCH 058/122] fix(MergeLineContext): Move newAgencyId to FeedMergeContext for persisted value. --- .../jobs/feedmerge/AgencyMergeLineContext.java | 2 +- .../jobs/feedmerge/FeedMergeContext.java | 17 +++++++++++++++-- .../jobs/feedmerge/MergeLineContext.java | 7 ++++--- 3 files changed, 20 insertions(+), 6 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/AgencyMergeLineContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/AgencyMergeLineContext.java index a186e0ff0..6354b6209 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/AgencyMergeLineContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/AgencyMergeLineContext.java @@ -33,7 +33,7 @@ private void checkForMissingAgencyId() { // agency_id is optional if only one agency is present, but that will // cause issues for the feed merge, so we need to insert an agency_id // for the single entry. - newAgencyId = UUID.randomUUID().toString(); + feedMergeContext.setNewAgencyId(UUID.randomUUID().toString()); if (keyFieldMissing) { // Only add agency_id field if it is missing in table. addField(Table.AGENCY.fields[0]); diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/FeedMergeContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/FeedMergeContext.java index c2360db4e..6e4100fe1 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/FeedMergeContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/FeedMergeContext.java @@ -30,9 +30,14 @@ public class FeedMergeContext implements Closeable { public final LocalDate activeFeedFirstDate; private LocalDate futureFeedFirstDate; private LocalDate futureFirstCalendarStartDate = LocalDate.MAX; - /** Trip ids shared between the active and future feed. */ + /** + * Trip ids shared between the active and future feed. + */ public final Set sharedTripIds; - + /** + * Holds the auto-generated agency id to be updated if none was provided. + */ + private String newAgencyId; public FeedMergeContext(Set feedVersions, Auth0UserProfile owner) throws IOException { feedsToMerge = MergeFeedUtils.collectAndSortFeeds(feedVersions, owner); @@ -126,6 +131,14 @@ public void setFutureFeedFirstDate(LocalDate futureFeedFirstDate) { this.futureFeedFirstDate = futureFeedFirstDate; } + public String getNewAgencyId() { + return newAgencyId; + } + + public void setNewAgencyId(String newAgencyId) { + this.newAgencyId = newAgencyId; + } + /** * Holds the status of a trip service id mismatch determination. */ diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java index 29fe5117f..189c8cbb8 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java @@ -53,7 +53,6 @@ public class MergeLineContext { private final CsvListWriter writer; private CsvReader csvReader; protected boolean skipRecord; - protected String newAgencyId; // move protected boolean keyFieldMissing; private String[] rowValues; private int lineNumber = 0; @@ -126,8 +125,8 @@ public void startNewFeed(int feedIndex) throws IOException { keyFieldMissing = false; // Use for a new agency ID for use if the feed does not contain one. Initialize to // null. If the value becomes non-null, the agency_id is missing and needs to be - // replaced with the generated value stored in this variable. - newAgencyId = null; + // replaced in other affected tables with the generated value stored in this variable. + feedMergeContext.setNewAgencyId(null); // Generate ID prefix to scope GTFS identifiers to avoid conflicts. idScope = getCleanName(feedSource.name) + version.version; csvReader = table.getCsvReader(feed.zipFile, null); @@ -358,6 +357,7 @@ protected void checkRoutesAndStopsIds(Set idErrors) throws IOExcep if (hasDuplicateError(idErrors)) skipRecord = true; } + String newAgencyId = feedMergeContext.getNewAgencyId(); if (newAgencyId != null && fieldNameEquals(AGENCY_ID)) { LOG.info( "Updating route#agency_id to (auto-generated) {} for route={}", @@ -375,6 +375,7 @@ private boolean useAltKey() { } public boolean updateAgencyIdIfNeeded() { + String newAgencyId = feedMergeContext.getNewAgencyId(); if (newAgencyId != null && fieldNameEquals(AGENCY_ID) && job.mergeType.equals(REGIONAL)) { if (fieldContext.getValue().equals("") && table.name.equals("agency") && lineNumber > 0) { // If there is no agency_id value for a second (or greater) agency From d6e63dc8a097578040a7639ae604c0f5eea2cfdd Mon Sep 17 00:00:00 2001 From: binh-dam-ibigroup <56846598+binh-dam-ibigroup@users.noreply.github.com> Date: Mon, 8 Nov 2021 16:27:10 -0500 Subject: [PATCH 059/122] docs(MergeFeedsJob): Replace condition number with requirement description. --- .../com/conveyal/datatools/manager/jobs/MergeFeedsJob.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java b/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java index 1221dcf07..1ed8fcb27 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java @@ -464,11 +464,10 @@ private int constructMergedTable(Table table, List feedsToMerge, Zi */ private MergeStrategy getMergeStrategy() { if (feedMergeContext.tripIdsMatch) { + // If trip ids and service ids match, these ids will be extended to the future per MTC requirements. // Effectively this exact match condition means that the future feed will be used as is // (including stops, routes, etc.), the only modification being service date ranges. - // This is Condition 2 in the docs. - // (If only trip IDs match, do not permit merge to continue - // - that is handled by method shouldFailJobDueToMatchingTripIds.) + // (If service ids mismatch, shouldFailJobDueToMatchingTripIds will fail the merge.) return feedMergeContext.serviceIdsMatch ? EXTEND_FUTURE : MergeStrategy.DEFAULT; } if (feedMergeContext.serviceIdsMatch) { From 6dd613b4b5e2fef601c887af782477b719f875fa Mon Sep 17 00:00:00 2001 From: binh-dam-ibigroup <56846598+binh-dam-ibigroup@users.noreply.github.com> Date: Wed, 10 Nov 2021 17:27:30 -0500 Subject: [PATCH 060/122] fix: Update feed merge logic (needs tests) --- .../datatools/manager/jobs/MergeFeedsJob.java | 137 ++++++++++++++---- .../CalendarDatesMergeLineContext.java | 11 +- .../feedmerge/CalendarMergeLineContext.java | 106 ++++++++------ .../jobs/feedmerge/FeedMergeContext.java | 22 ++- .../manager/jobs/feedmerge/FieldContext.java | 7 + .../jobs/feedmerge/MergeLineContext.java | 65 ++++----- .../manager/jobs/feedmerge/MergeStrategy.java | 1 + .../feedmerge/RoutesMergeLineContext.java | 4 +- .../feedmerge/ShapesMergeLineContext.java | 16 +- .../jobs/feedmerge/StopsMergeLineContext.java | 4 +- .../jobs/feedmerge/TripsMergeLineContext.java | 40 ++--- 11 files changed, 266 insertions(+), 147 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java b/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java index 1ed8fcb27..fe3a24c23 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java @@ -23,7 +23,6 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.Lists; -import com.google.common.collect.Sets; import org.bson.codecs.pojo.annotations.BsonIgnore; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -33,6 +32,7 @@ import java.io.FileOutputStream; import java.io.IOException; import java.nio.file.Files; +import java.util.ArrayList; import java.util.Arrays; import java.util.HashSet; import java.util.List; @@ -43,7 +43,6 @@ import static com.conveyal.datatools.manager.jobs.feedmerge.MergeFeedsType.SERVICE_PERIOD; import static com.conveyal.datatools.manager.jobs.feedmerge.MergeFeedsType.REGIONAL; import static com.conveyal.datatools.manager.jobs.feedmerge.MergeStrategy.CHECK_STOP_TIMES; -import static com.conveyal.datatools.manager.jobs.feedmerge.MergeStrategy.EXTEND_FUTURE; import static com.conveyal.datatools.manager.models.FeedRetrievalMethod.REGIONAL_MERGE; import static com.conveyal.datatools.manager.utils.MergeFeedUtils.*; @@ -133,13 +132,18 @@ public class MergeFeedsJob extends FeedSourceJob { */ final FeedVersion mergedVersion; @JsonIgnore @BsonIgnore - public Set tripIdsToModifyForActiveFeed = new HashSet<>(); + public Set sharedTripIdsWithInconsistentSignature = new HashSet<>(); @JsonIgnore @BsonIgnore - public Set tripIdsToSkipForActiveFeed = new HashSet<>(); + public Set sharedTripIdsWithConsistentSignature = new HashSet<>(); @JsonIgnore @BsonIgnore public Set serviceIdsToExtend = new HashSet<>(); @JsonIgnore @BsonIgnore public Set serviceIdsToCloneAndRename = new HashSet<>(); + @JsonIgnore @BsonIgnore + public Set serviceIdsToInsert = new HashSet<>(); + + private List sharedConsistentTripAndCalendarIds = new ArrayList<>(); + // Variables used for a service period merge. private FeedMergeContext feedMergeContext; @@ -259,7 +263,8 @@ public void jobLogic() { final List
tablesToMerge = getTablesToMerge(); int numberOfTables = tablesToMerge.size(); - // Skip merging process altogether if the failing condition is met. +/* + // Skip merging process altogether if the failing condition is met FeedMergeContext.TripMismatchedServiceIds serviceIdMismatch; if (feedMergeContext.areTripIdsMatchingButNotServiceIds()) { failMergeJob("Feed merge failed because the trip_ids are identical in the future and active feeds. A new service requires unique trip_ids for merging."); @@ -278,11 +283,28 @@ public void jobLogic() { return; } + */ + // Before initiating the merge process, get the merge strategy to use, which runs some pre-processing to // check for id conflicts for certain tables (e.g., trips and calendars). if (mergeType.equals(SERVICE_PERIOD)) { - mergeFeedsResult.mergeStrategy = getMergeStrategy(); + determineMergeStrategy(); + + // "If a single trip signature does not match the merge process shall stop with the following + // error message along with matching trip_ids with differing trip signatures." + // (MTC revised merge Step 2) + Set tripIdsWithInconsistentSignature = getSharedTripIdsWithInconsistentSignature(); + if (!tripIdsWithInconsistentSignature.isEmpty()) { + failMergeJob( + String.format("Trips %s in new feed have differing makeup from matching trips in active feed." + + "If a trip characteristic has changed, a new trip_id must be assigned.", + String.join(", ", tripIdsWithInconsistentSignature) + ) + ); + return; + } } + // Loop over GTFS tables and merge each feed one table at a time. for (int i = 0; i < numberOfTables; i++) { Table table = tablesToMerge.get(i); @@ -324,6 +346,13 @@ public void jobLogic() { } } + /** + * Obtains trip ids whose entries in the stop_times table differ between the active and future feed. + */ + private Set getSharedTripIdsWithInconsistentSignature() { + return sharedTripIdsWithInconsistentSignature; + } + private List
getTablesToMerge() { List
tablesToMerge = Arrays.stream(Table.tablesInOrder) .filter(Table::isSpecTable) @@ -462,15 +491,36 @@ private int constructMergedTable(Table table, List feedsToMerge, Zi * Get the merge strategy to use for MTC service period merges by checking the active and future feeds for various * combinations of matching trip and service IDs. */ - private MergeStrategy getMergeStrategy() { - if (feedMergeContext.tripIdsMatch) { - // If trip ids and service ids match, these ids will be extended to the future per MTC requirements. - // Effectively this exact match condition means that the future feed will be used as is - // (including stops, routes, etc.), the only modification being service date ranges. - // (If service ids mismatch, shouldFailJobDueToMatchingTripIds will fail the merge.) - return feedMergeContext.serviceIdsMatch ? EXTEND_FUTURE : MergeStrategy.DEFAULT; - } - if (feedMergeContext.serviceIdsMatch) { + private void determineMergeStrategy() { + // Revised merge logic + // Step 1: TDM Merge functionality shall start with first comparing trip_ids + // between active and future GTFS feed. + if (feedMergeContext.areActiveAndFutureTripIdsDisjoint()) { + // If none of the trip_ids in active GTFS feed match with the trip_ids + // available in future GTFS feed, then proceed to Step 3; otherwise continue to the next step [Step 2]. + // Step 3: When the complete set of trip_ids between active and future GTFS feeds is different, + // all trip records from both feeds shall be added to the merged feed as per the following rule + // and the merge process will exit. + // If a service_id from an active calendar has an end date in the future, + // the end_date shall be set to one day prior to the earliest start_date in the future dataset + // before appending the calendar record to the merged file. + // The merge process shall end here by publishing the merge feed and inform the user + // that trip_ids were unique which successfully created a merge feed. + + // => Step 3 is the existing DEFAULT merge strategy. + mergeFeedsResult.mergeStrategy = MergeStrategy.DEFAULT; + } else { + // Step 2: If matching trip_ids are provided in active and future GTFS feed, for those matching trips, + // trip signatures – a combination of arrival_time, departure_time, stop_id, + // and stop_sequence – in stop_times.txt file should be compared. + // If all the matching trip_ids contain the same trip signatures, the merge process shall proceed + // to step 4. If a single trip signature does not match + // the merge process shall stop with the following error message + // along with matching trip_ids with differing trip signatures. + // Error Message: Trips [trip_id] in new feed have differing makeup from matching trips in active feed. + // If a trip character has changed, new trip_id must be assigned. + + // => Step 2 is the CHECK_STOP_TIMES strategy // If just the service_ids are an exact match, check the that the stop_times having matching signatures // between the two feeds (i.e., each stop time in the ordered list is identical between the two feeds). Feed futureFeed = feedMergeContext.futureFeed; @@ -478,6 +528,17 @@ private MergeStrategy getMergeStrategy() { for (String tripId : feedMergeContext.sharedTripIds) { compareStopTimesAndCollectTripAndServiceIds(tripId, futureFeed, activeFeed); } + + // Build the set of calendars to be cloned/renamed/extended from trip ids present + // in both active/future feeds and that have consistent signature. + // These trips will be linked to the new service_ids. + serviceIdsToCloneAndRename.addAll( + sharedTripIdsWithConsistentSignature.stream() + .map(tripId -> activeFeed.trips.get(tripId).service_id) + .collect(Collectors.toList()) + ); + +/* // If a trip only in the active feed references a service_id that is set to be extended, that // service_id needs to be cloned and renamed to differentiate it from the same service_id in // the future feed. (The trip in question will be linked to the cloned service_id.) @@ -494,11 +555,9 @@ private MergeStrategy getMergeStrategy() { .map(tripId -> futureFeed.trips.get(tripId).service_id) .filter(serviceId -> serviceIdsToExtend.contains(serviceId)) .forEach(serviceId -> serviceIdsToCloneAndRename.add(serviceId)); - - return CHECK_STOP_TIMES; +*/ + mergeFeedsResult.mergeStrategy = CHECK_STOP_TIMES; } - // If neither the trips or services are exact matches, use the default merge strategy. - return MergeStrategy.DEFAULT; } /** @@ -511,18 +570,18 @@ private void compareStopTimesAndCollectTripAndServiceIds(String tripId, Feed fut // (ignoring the other identical one). If they do not match, modify the active trip_id and include. List futureStopTimes = Lists.newArrayList(futureFeed.stopTimes.getOrdered(tripId)); List activeStopTimes = Lists.newArrayList(activeFeed.stopTimes.getOrdered(tripId)); + String activeServiceId = activeFeed.trips.get(tripId).service_id; String futureServiceId = futureFeed.trips.get(tripId).service_id; if (!stopTimesMatch(futureStopTimes, activeStopTimes)) { - // If stop_times or services do not match, the trip will be cloned. Also, track the service_id - // (it will need to be cloned and renamed for both active feeds). - tripIdsToModifyForActiveFeed.add(tripId); - serviceIdsToCloneAndRename.add(futureServiceId); + // If stop_times or services do not match, merge will fail and no other action will be taken. + sharedTripIdsWithInconsistentSignature.add(tripId); } else { // If the trip's stop_times are an exact match, we can safely include just the - // future trip and exclude the active one. Also, track the service_id (it will need to be - // extended to the full time range). - tripIdsToSkipForActiveFeed.add(tripId); - serviceIdsToExtend.add(futureServiceId); + // future trip and exclude the active one. Also, mark the service_id for cloning, + // the cloned service id will need to be extended to the full time range. + sharedTripIdsWithConsistentSignature.add(tripId); + serviceIdsToCloneAndRename.add(futureServiceId); + sharedConsistentTripAndCalendarIds.add(new TripAndCalendars(tripId, activeServiceId, futureServiceId)); } } @@ -539,4 +598,28 @@ private void logAndReportToBugsnag(Exception e, String message, Object... args) public FeedMergeContext getFeedMergeContext() { return feedMergeContext; } + + /** + * @return true if the specified calendar id is listed as the active calendar id in any of the trips/calendar items. + */ + public boolean isActiveCalendarOfSharedTripId(String calendarId) { + for (TripAndCalendars item : sharedConsistentTripAndCalendarIds) { + if (item.activeCalendarId.equals(calendarId)) { + return true; + } + } + return false; + } + + private static class TripAndCalendars { + public final String tripId; + public final String activeCalendarId; + public final String futureCalendarId; + + public TripAndCalendars(String tripId, String activeCalendarId, String futureCalendarId) { + this.tripId = tripId; + this.activeCalendarId = activeCalendarId; + this.futureCalendarId = futureCalendarId; + } + } } diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarDatesMergeLineContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarDatesMergeLineContext.java index aafc2fde2..92aa05ab3 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarDatesMergeLineContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarDatesMergeLineContext.java @@ -22,8 +22,8 @@ public CalendarDatesMergeLineContext(MergeFeedsJob job, Table table, ZipOutputSt } @Override - public void checkFieldsForMergeConflicts(Set idErrors) throws IOException { - checkCalendarDatesIds(); + public void checkFieldsForMergeConflicts(Set idErrors, FieldContext fieldContext) throws IOException { + checkCalendarDatesIds(fieldContext); } @Override @@ -32,7 +32,7 @@ public void startNewRow() throws IOException { updateFutureFeedFirstDate(); } - private void checkCalendarDatesIds() throws IOException { + private void checkCalendarDatesIds(FieldContext fieldContext) throws IOException { // Drop any calendar_dates.txt records from the existing feed for dates that are // not before the first date of the future feed. LocalDate date = getCsvDate("date"); @@ -49,10 +49,13 @@ private void checkCalendarDatesIds() throws IOException { // Track service ID because we want to avoid removing trips that may reference this // service_id when the service_id is used by calendar.txt records that operate in // the valid date range, i.e., before the future feed's first date. - if (!skipRecord && fieldNameEquals(SERVICE_ID)) mergeFeedsResult.serviceIds.add(getFieldContext().getValueToWrite()); + if (!skipRecord && fieldContext.nameEquals(SERVICE_ID)) mergeFeedsResult.serviceIds.add(fieldContext.getValueToWrite()); } + // FIXME: move this (almost seems irrelevant) private void updateFutureFeedFirstDate() { + // This will be populated because calendar dates is processed + // after calendar, which populates FutureFirstCalendarStartDate. LocalDate futureFirstCalendarStartDate = feedMergeContext.getFutureFirstCalendarStartDate(); if ( isHandlingActiveFeed() && diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarMergeLineContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarMergeLineContext.java index 622a75756..8f7644765 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarMergeLineContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarMergeLineContext.java @@ -26,11 +26,65 @@ public CalendarMergeLineContext(MergeFeedsJob job, Table table, ZipOutputStream } @Override - public void checkFieldsForMergeConflicts(Set idErrors) throws IOException { - checkCalendarIds(idErrors); + public void checkFieldsForMergeConflicts(Set idErrors, FieldContext fieldContext) throws IOException { + checkCalendarIds(idErrors, fieldContext); } - private void checkCalendarIds(Set idErrors) throws IOException { + private void checkCalendarIds(Set idErrors, FieldContext fieldContext) throws IOException { + if (isHandlingActiveFeed()) { + LocalDate startDate = getCsvDate("start_date"); + // If a service_id from the active calendar has both the + // start_date and end_date in the future, the service will be + // excluded from the merged file. Records in trips, + // calendar_dates, and calendar_attributes referencing this + // service_id shall also be removed/ignored. Stop_time records + // for the ignored trips shall also be removed. + LocalDate futureFeedFirstDate = feedMergeContext.getFutureFeedFirstDate(); + if (!startDate.isBefore(futureFeedFirstDate)) { + LOG.warn( + "Skipping calendar entry {} because it operates fully within the time span of future feed.", + keyValue); + String key = getTableScopedValue(table, getIdScope(), keyValue); + mergeFeedsResult.skippedIds.add(key); + skipRecord = true; + } else { + // In the MTC revised feed merge logic: + // - If trip ids in active and future feed are disjoint, + // - calendar entries from the active feed will be inserted, + // but the ending date will be set to the day before the earliest **calendar start date** from the new feed. + // - If some trip ids are found in both active/future feed, + // - new calendar entries are created for those trips + // that span from active feed’s start date to the future feed’s end date. + // - calendar entries for other trip ids in the active feed are inserted in the merged feed, + // but the ending date will be set to the day before the **start date of the new feed**. + LocalDate endDate = getCsvDate("end_date"); + LocalDate futureStartDate = null; + boolean activeAndFutureTripIdsDisjoint = job.sharedTripIdsWithConsistentSignature.isEmpty(); + if (activeAndFutureTripIdsDisjoint) { + futureStartDate = feedMergeContext.getFutureFirstCalendarStartDate(); + } else { + if (job.isActiveCalendarOfSharedTripId(keyValue)) { + // New calendar entry is already flagged for insertion from getMergeStrategy. + // Insert this calendar record for other trip ids that may reference it. + } else { + futureStartDate = futureFeedFirstDate; + } + } + + if (fieldContext.nameEquals("end_date")) { + if (futureStartDate != null && !endDate.isBefore(futureStartDate)) { + fieldContext.resetValue(futureStartDate + .minus(1, ChronoUnit.DAYS) + .format(GTFS_DATE_FORMATTER)); + } + } + } + } else if (isHandlingFutureFeed()) { + // In the MTC revised feed merge logic: + // - Calendar entries from the future feed will be inserted as is in the merged feed. + // so no additional processing needed here. + } + // If any service_id in the active feed matches with the future // feed, it should be modified and all associated trip records // must also be changed with the modified service_id. @@ -42,63 +96,29 @@ private void checkCalendarIds(Set idErrors) throws IOException { if (hasDuplicateError(idErrors)) { // Modify service_id and ensure that referencing trips // have service_id updated. - updateAndRemapOutput(); + updateAndRemapOutput(fieldContext); } - LocalDate startDate = getCsvDate("start_date"); if (isHandlingFutureFeed()) { - // For the future feed, check if the calendar's start date is earlier than the - // previous earliest value and update if so. - if (feedMergeContext.getFutureFirstCalendarStartDate().isAfter(startDate)) { - feedMergeContext.setFutureFirstCalendarStartDate(startDate); - } // FIXME: Move this below so that a cloned service doesn't get prematurely // modified? (do we want the cloned record to have the original values?) - if (shouldUpdateFutureFeedStartDate()) { + if (shouldUpdateFutureFeedStartDate(fieldContext)) { // Update start_date to extend service through the active feed's // start date if the merge strategy dictates. The justification for this logic is that the active feed's // service_id will be modified to a different unique value and the trips shared between the future/active // service are exactly matching. - getFieldContext().resetValue(feedMergeContext.activeFeedFirstDate.format(GTFS_DATE_FORMATTER)); + fieldContext.resetValue(feedMergeContext.activeFeedFirstDate.format(GTFS_DATE_FORMATTER)); } } else { - // If a service_id from the active calendar has both the - // start_date and end_date in the future, the service will be - // excluded from the merged file. Records in trips, - // calendar_dates, and calendar_attributes referencing this - // service_id shall also be removed/ignored. Stop_time records - // for the ignored trips shall also be removed. - LocalDate futureFeedFirstDate = feedMergeContext.getFutureFeedFirstDate(); - if (!startDate.isBefore(futureFeedFirstDate)) { - LOG.warn( - "Skipping calendar entry {} because it operates fully within the time span of future feed.", - keyValue); - String key = getTableScopedValue(table, getIdScope(), keyValue); - mergeFeedsResult.skippedIds.add(key); - skipRecord = true; - } else { - // If a service_id from the active calendar has only the - // end_date in the future, the end_date shall be set to one - // day prior to the earliest start_date in future dataset - // before appending the calendar record to the merged file. - if (fieldNameEquals("end_date")) { - LocalDate endDate = getCsvDate("end_date"); - if (!endDate.isBefore(futureFeedFirstDate)) { - getFieldContext().resetValue(futureFeedFirstDate - .minus(1, ChronoUnit.DAYS) - .format(GTFS_DATE_FORMATTER)); - } - } - } } // Track service ID because we want to avoid removing trips that may reference this // service_id when the service_id is used by calendar_dates that operate in the valid // date range, i.e., before the future feed's first date. - if (!skipRecord && fieldNameEquals(SERVICE_ID)) mergeFeedsResult.serviceIds.add(getFieldContext().getValueToWrite()); + if (!skipRecord && fieldContext.nameEquals(SERVICE_ID)) mergeFeedsResult.serviceIds.add(fieldContext.getValueToWrite()); } - private boolean shouldUpdateFutureFeedStartDate() { - return fieldNameEquals("start_date") && + private boolean shouldUpdateFutureFeedStartDate(FieldContext fieldContext) { + return fieldContext.nameEquals("start_date") && ( EXTEND_FUTURE == mergeFeedsResult.mergeStrategy || ( diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/FeedMergeContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/FeedMergeContext.java index 6e4100fe1..385e17ae9 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/FeedMergeContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/FeedMergeContext.java @@ -6,6 +6,7 @@ import com.conveyal.datatools.manager.utils.MergeFeedUtils; import com.conveyal.gtfs.loader.Feed; import com.conveyal.gtfs.loader.Table; +import com.conveyal.gtfs.model.Calendar; import com.google.common.collect.Sets; import java.io.Closeable; @@ -64,6 +65,13 @@ public FeedMergeContext(Set feedVersions, Auth0UserProfile owner) t // calendar#start_date (unless that table for some reason does not exist). activeFeedFirstDate = activeFeedToMerge.version.validationResult.firstCalendarDate; futureFeedFirstDate = futureFeedToMerge.version.validationResult.firstCalendarDate; + + // Initialize, before processing rows, the calendar start dates from the future feed. + for (Calendar c : futureFeed.calendars.getAll()) { + if (futureFirstCalendarStartDate.isAfter(c.start_date)) { + futureFirstCalendarStartDate = c.start_date; + } + } } @Override @@ -74,6 +82,16 @@ public void close() throws IOException { } /** + * Partially handles the Revised MTC Feed Merge Requirement + * to detect disjoint trip ids between the active/future feeds. + * @return true if no trip ids from the active feed is found in the future feed, and vice-versa. + */ + public boolean areActiveAndFutureTripIdsDisjoint() { + return sharedTripIds.isEmpty(); + } + + /** + * FIXME: Remove - this is from the old merge logic. * Partially handles MTC Requirement to detect matching trip ids linked to different service ids. * @return true if trip ids match but not service ids (in such situation, merge should fail). */ @@ -119,10 +137,6 @@ public LocalDate getFutureFirstCalendarStartDate() { return futureFirstCalendarStartDate; } - public void setFutureFirstCalendarStartDate(LocalDate futureFirstCalendarStartDate) { - this.futureFirstCalendarStartDate = futureFirstCalendarStartDate; - } - public LocalDate getFutureFeedFirstDate() { return futureFeedFirstDate; } diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/FieldContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/FieldContext.java index e829dfa8e..0c33ad92b 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/FieldContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/FieldContext.java @@ -45,4 +45,11 @@ public void setValueToWrite(String newValue) { public void resetValue(String newValue) { value = valueToWrite = newValue; } + + /** + * Convenience method to compare if this field name equals a specified one. + */ + public boolean nameEquals(String fieldName) { + return field.name.equals(fieldName); + } } diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java index 189c8cbb8..f2bc4eaef 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java @@ -71,7 +71,6 @@ public class MergeLineContext { private final Map rowValuesForStopOrRouteId = new HashMap<>(); private final Set rowStrings = new HashSet<>(); private List sharedSpecFields; - private FieldContext fieldContext; private int feedIndex; public FeedVersion version; @@ -211,7 +210,7 @@ public void startNewRow() throws IOException { .collect(Collectors.toList()); } - public boolean areForeignRefsOk() throws IOException { + public boolean areForeignRefsOk(FieldContext fieldContext) throws IOException { Field field = fieldContext.getField(); if (field.isForeignReference()) { String key = getTableScopedValue(field.referenceTable, idScope, fieldContext.getValue()); @@ -223,11 +222,11 @@ public boolean areForeignRefsOk() throws IOException { // been skipped or is a ref to a non-existent service_id during a service period merge, skip // this record and add its primary key to the list of skipped IDs (so that other references // can be properly omitted). - if (serviceIdHasOrShouldBeSkipped(key, isValidServiceId)) { + if (serviceIdHasOrShouldBeSkipped(fieldContext, key, isValidServiceId)) { // If a calendar#service_id has been skipped (it's listed in skippedIds), but there were // valid service_ids found in calendar_dates, do not skip that record for both the // calendar_date and any related trips. - if (fieldNameEquals(SERVICE_ID) && isValidServiceId) { + if (fieldContext.nameEquals(SERVICE_ID) && isValidServiceId) { LOG.warn("Not skipping valid service_id {} for {} {}", fieldContext.getValueToWrite(), table.name, keyValue); } else { String skippedKey = getTableScopedValue(table, idScope, keyValue); @@ -251,9 +250,9 @@ public boolean areForeignRefsOk() throws IOException { return true; } - private boolean serviceIdHasOrShouldBeSkipped(String key, boolean isValidServiceId) { + private boolean serviceIdHasOrShouldBeSkipped(FieldContext fieldContext, String key, boolean isValidServiceId) { boolean serviceIdShouldBeSkipped = job.mergeType.equals(SERVICE_PERIOD) && - fieldNameEquals(SERVICE_ID) && + fieldContext.nameEquals(SERVICE_ID) && !isValidServiceId; return mergeFeedsResult.skippedIds.contains(key) || serviceIdShouldBeSkipped; } @@ -263,17 +262,17 @@ private boolean serviceIdHasOrShouldBeSkipped(String key, boolean isValidService * Overridable method whose default behavior below is to skip a record if it creates a duplicate id. * @throws IOException Some overrides throw IOException. */ - public void checkFieldsForMergeConflicts(Set idErrors) throws IOException { + public void checkFieldsForMergeConflicts(Set idErrors, FieldContext fieldContext) throws IOException { if (hasDuplicateError(idErrors)) skipRecord = true; } - private Set getIdErrors() { + private Set collectIdErrors(FieldContext fieldContext) { Set idErrors; Field field = fieldContext.getField(); // If analyzing the second feed (active feed), the service_id always gets feed scoped. // See https://github.com/ibi-group/datatools-server/issues/244 - if (handlingActiveFeed && fieldNameEquals(SERVICE_ID)) { - updateAndRemapOutput(); + if (handlingActiveFeed && fieldContext.nameEquals(SERVICE_ID)) { + updateAndRemapOutput(fieldContext); idErrors = referenceTracker .checkReferencesAndUniqueness(keyValue, lineNumber, field, fieldContext.getValueToWrite(), table, keyField, orderField); @@ -285,7 +284,7 @@ private Set getIdErrors() { return idErrors; } - protected void checkRoutesAndStopsIds(Set idErrors) throws IOException { + protected void checkRoutesAndStopsIds(Set idErrors, FieldContext fieldContext) throws IOException { // First, check uniqueness of primary key value (i.e., stop or route ID) // in case the stop_code or route_short_name are being used. This // must occur unconditionally because each record must be tracked @@ -298,7 +297,7 @@ protected void checkRoutesAndStopsIds(Set idErrors) throws IOExcep // route_short_name/stop_code in active data not present in the future will be appended to the // future routes/stops file. if (useAltKey()) { - if (hasBlankPrimaryKey()) { + if (hasBlankPrimaryKey(fieldContext)) { // If alt key is empty (which is permitted) and primary key is duplicate, skip // checking of alt key dupe errors/re-mapping values and // simply use the primary key (route_id/stop_id). @@ -348,7 +347,7 @@ protected void checkRoutesAndStopsIds(Set idErrors) throws IOExcep if (!skipRecord && !referenceTracker.transitIds.contains(String.join(":", keyField, keyValue)) && hasDuplicateError(primaryKeyErrors)) { // Modify route_id and ensure that referencing trips // have route_id updated. - updateAndRemapOutput(); + updateAndRemapOutput(fieldContext); } } else { // Key field has defaulted to the standard primary key field @@ -358,7 +357,7 @@ protected void checkRoutesAndStopsIds(Set idErrors) throws IOExcep } String newAgencyId = feedMergeContext.getNewAgencyId(); - if (newAgencyId != null && fieldNameEquals(AGENCY_ID)) { + if (newAgencyId != null && fieldContext.nameEquals(AGENCY_ID)) { LOG.info( "Updating route#agency_id to (auto-generated) {} for route={}", newAgencyId, keyValue); @@ -366,17 +365,17 @@ protected void checkRoutesAndStopsIds(Set idErrors) throws IOExcep } } - private boolean hasBlankPrimaryKey() { - return "".equals(keyValue) && fieldNameEquals(table.getKeyFieldName()); + private boolean hasBlankPrimaryKey(FieldContext fieldContext) { + return "".equals(keyValue) && fieldContext.nameEquals(table.getKeyFieldName()); } private boolean useAltKey() { return keyField.equals("stop_code") || keyField.equals("route_short_name"); } - public boolean updateAgencyIdIfNeeded() { + public boolean updateAgencyIdIfNeeded(FieldContext fieldContext) { String newAgencyId = feedMergeContext.getNewAgencyId(); - if (newAgencyId != null && fieldNameEquals(AGENCY_ID) && job.mergeType.equals(REGIONAL)) { + if (newAgencyId != null && fieldContext.nameEquals(AGENCY_ID) && job.mergeType.equals(REGIONAL)) { if (fieldContext.getValue().equals("") && table.name.equals("agency") && lineNumber > 0) { // If there is no agency_id value for a second (or greater) agency // record, return null which will trigger a failed merge feed job. @@ -441,8 +440,8 @@ public void checkFirstLineConditions() throws IOException { // Default is to do nothing. } - public void scopeValueIfNeeded() { - boolean isKeyField = fieldContext.getField().isForeignReference() || fieldNameEquals(keyField); + public void scopeValueIfNeeded(FieldContext fieldContext) { + boolean isKeyField = fieldContext.getField().isForeignReference() || fieldContext.nameEquals(keyField); if (job.mergeType.equals(REGIONAL) && isKeyField && !fieldContext.getValue().isEmpty()) { // For regional merge, if field is a GTFS identifier (e.g., route_id, // stop_id, etc.), add scoped prefix. @@ -515,34 +514,34 @@ public boolean constructRowValues() throws IOException { // Default value to write is unchanged from value found in csv (i.e. val). Note: if looking to // modify the value that is written in the merged file, you must update valueToWrite (e.g., // updating this feed's end_date or accounting for cases where IDs conflict). - fieldContext = new FieldContext( + FieldContext fieldContext = new FieldContext( field, csvReader.get(fieldsFoundList.indexOf(field)) ); // Handle filling in agency_id if missing when merging regional feeds. If false is returned, // the job has encountered a failing condition (the method handles failing the job itself). - if (!updateAgencyIdIfNeeded()) { + if (!updateAgencyIdIfNeeded(fieldContext)) { return false; } // Determine if field is a GTFS identifier (and scope if needed). - scopeValueIfNeeded(); + scopeValueIfNeeded(fieldContext); // Only need to check for merge conflicts if using MTC merge type because // the regional merge type scopes all identifiers by default. Also, the // reference tracker will get far too large if we attempt to use it to // track references for a large number of feeds (e.g., every feed in New // York State). if (job.mergeType.equals(SERVICE_PERIOD)) { - Set idErrors = getIdErrors(); // FIXME: Rename. This method is imperative. + Set idErrors = collectIdErrors(fieldContext); // Store values for key fields that have been encountered and update any key values that need modification due // to conflicts. - checkFieldsForMergeConflicts(idErrors); // FIXME: This method changes skipRecord; + checkFieldsForMergeConflicts(idErrors, fieldContext); // FIXME: This method changes skipRecord; if (skipRecord) continue; } // If the current field is a foreign reference, check if the reference has been removed in the // merged result. If this is the case (or other conditions are met), we will need to skip this // record. Likewise, if the reference has been modified, ensure that the value written to the // merged result is correctly updated. - if (!areForeignRefsOk()) continue; // FIXME: This method changes skipRecord; + if (!areForeignRefsOk(fieldContext)) continue; // FIXME: This method changes skipRecord; rowValues[specFieldIndex] = fieldContext.getValueToWrite(); } return true; @@ -599,16 +598,8 @@ protected String getIdScope() { return idScope; } - protected FieldContext getFieldContext() { - return fieldContext; - } - protected int getFeedIndex() { return feedIndex; } - protected boolean fieldNameEquals(String value) { - return fieldContext.getField().name.equals(value); - } - protected int getLineNumber() { return lineNumber; } @@ -631,7 +622,7 @@ protected LocalDate getCsvDate(String fieldName) throws IOException { /** * Updates output for the current field and remaps the record id. */ - protected void updateAndRemapOutput(boolean updateKeyValue) { + protected void updateAndRemapOutput(FieldContext fieldContext, boolean updateKeyValue) { String value = fieldContext.getValue(); String valueToWrite = String.join(":", idScope, value); fieldContext.setValueToWrite(valueToWrite); @@ -647,8 +638,8 @@ protected void updateAndRemapOutput(boolean updateKeyValue) { /** * Shorthand for the above method. */ - protected void updateAndRemapOutput() { - updateAndRemapOutput(false); + protected void updateAndRemapOutput(FieldContext fieldContext) { + updateAndRemapOutput(fieldContext,false); } /** diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeStrategy.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeStrategy.java index 50f26b218..5d1562ab3 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeStrategy.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeStrategy.java @@ -13,6 +13,7 @@ public enum MergeStrategy { */ DEFAULT, /** + * FIXME: Remove - no longer used (was in the old MTC feed merge) * If service_ids and trip_ids in active feed are the same as future feed then the service end date for the * merged feed shall match with future feed’s service end date and the service start date for the merged feed * should be the merged date. All files from the future feed only shall be used in the merged feed. diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/RoutesMergeLineContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/RoutesMergeLineContext.java index ae976e342..1599857b7 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/RoutesMergeLineContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/RoutesMergeLineContext.java @@ -14,7 +14,7 @@ public RoutesMergeLineContext(MergeFeedsJob job, Table table, ZipOutputStream ou } @Override - public void checkFieldsForMergeConflicts(Set idErrors) throws IOException { - checkRoutesAndStopsIds(idErrors); + public void checkFieldsForMergeConflicts(Set idErrors, FieldContext fieldContext) throws IOException { + checkRoutesAndStopsIds(idErrors, fieldContext); } } \ No newline at end of file diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/ShapesMergeLineContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/ShapesMergeLineContext.java index 439b84ebd..44ccd8bbb 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/ShapesMergeLineContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/ShapesMergeLineContext.java @@ -20,18 +20,18 @@ public ShapesMergeLineContext(MergeFeedsJob job, Table table, ZipOutputStream ou } @Override - public void checkFieldsForMergeConflicts(Set idErrors) { - checkShapeIds(idErrors); + public void checkFieldsForMergeConflicts(Set idErrors, FieldContext fieldContext) { + checkShapeIds(idErrors, fieldContext); } - private void checkShapeIds(Set idErrors) { + private void checkShapeIds(Set idErrors, FieldContext fieldContext) { // If a shape_id is found in both future and active datasets, all shape points from // the active dataset must be feed-scoped. Otherwise, the merged dataset may contain // shape_id:shape_pt_sequence values from both datasets (e.g., if future dataset contains // sequences 1,2,3,10 and active contains 1,2,7,9,10; the merged set will contain // 1,2,3,7,9,10). - if (fieldNameEquals("shape_id")) { - String val = getFieldContext().getValue(); + if (fieldContext.nameEquals("shape_id")) { + String val = fieldContext.getValue(); if (isHandlingFutureFeed()) { // Track shape_id if working on future feed. shapeIdsInFutureFeed.add(val); @@ -39,15 +39,15 @@ private void checkShapeIds(Set idErrors) { // For the active feed, if the shape_id was already processed from the // future feed, we need to add the feed-scope to avoid weird, hybrid shapes // with points from both feeds. - updateAndRemapOutput(true); + updateAndRemapOutput(fieldContext,true); // Re-check refs and uniqueness after changing shape_id value. (Note: this // probably won't have any impact, but there's not much harm in including it.) idErrors = referenceTracker .checkReferencesAndUniqueness( keyValue, getLineNumber(), - getFieldContext().getField(), - getFieldContext().getValueToWrite(), + fieldContext.getField(), + fieldContext.getValueToWrite(), table, keyField, table.getOrderFieldName()); diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/StopsMergeLineContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/StopsMergeLineContext.java index 4a3c54f42..5ff2a342f 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/StopsMergeLineContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/StopsMergeLineContext.java @@ -29,8 +29,8 @@ public void checkFirstLineConditions() throws IOException { } @Override - public void checkFieldsForMergeConflicts(Set idErrors) throws IOException { - checkRoutesAndStopsIds(idErrors); + public void checkFieldsForMergeConflicts(Set idErrors, FieldContext fieldContext) throws IOException { + checkRoutesAndStopsIds(idErrors, fieldContext); } private void checkStopCodeStuff() throws IOException { diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/TripsMergeLineContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/TripsMergeLineContext.java index d3e17d809..c3fbb431f 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/TripsMergeLineContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/TripsMergeLineContext.java @@ -2,41 +2,41 @@ import com.conveyal.datatools.manager.jobs.MergeFeedsJob; import com.conveyal.gtfs.error.NewGTFSError; -import com.conveyal.gtfs.error.NewGTFSErrorType; import com.conveyal.gtfs.loader.Table; import java.io.IOException; import java.util.Set; import java.util.zip.ZipOutputStream; +import static com.conveyal.datatools.manager.jobs.feedmerge.MergeFeedsType.SERVICE_PERIOD; +import static com.conveyal.datatools.manager.utils.MergeFeedUtils.hasDuplicateError; + public class TripsMergeLineContext extends MergeLineContext { public TripsMergeLineContext(MergeFeedsJob job, Table table, ZipOutputStream out) throws IOException { super(job, table, out); } @Override - public void checkFieldsForMergeConflicts(Set idErrors) { - checkTripIds(idErrors); + public void checkFieldsForMergeConflicts(Set idErrors, FieldContext fieldContext) { + checkTripIds(idErrors, fieldContext); } - private void checkTripIds(Set idErrors) { - // trip_ids between active and future datasets must not match. The tripIdsToSkip and - // tripIdsToModify sets below are determined based on the MergeStrategy used for MTC - // service period merges. - if (isHandlingActiveFeed()) { - // Handling active feed. Skip or modify trip id if found in one of the - // respective sets. - if (job.tripIdsToSkipForActiveFeed.contains(keyValue)) { - skipRecord = true; - } else if (job.tripIdsToModifyForActiveFeed.contains(keyValue)) { - updateAndRemapOutput(true); - } + private void checkTripIds(Set idErrors, FieldContext fieldContext) { + // For the MTC revised feed merge process, + // the updated logic requires to insert all trips from both the active and future feed, + // except if they are present in both, in which case we only insert the trip entry from the future feed. + if ( + job.mergeType.equals(SERVICE_PERIOD) && + isHandlingActiveFeed() && + job.sharedTripIdsWithConsistentSignature.contains(keyValue) + ) { + // Skip this record, we will use the one from the future feed. + skipRecord = true; } - for (NewGTFSError error : idErrors) { - if (error.errorType.equals(NewGTFSErrorType.DUPLICATE_ID)) { - updateAndRemapOutput(true); - } + + // Remap duplicate trip ids. + if (hasDuplicateError(idErrors)) { + updateAndRemapOutput(fieldContext, true); } } - } \ No newline at end of file From a194147627ac38e01bdfeaf2d558ef6bca41d4f2 Mon Sep 17 00:00:00 2001 From: binh-dam-ibigroup <56846598+binh-dam-ibigroup@users.noreply.github.com> Date: Wed, 10 Nov 2021 20:15:46 -0500 Subject: [PATCH 061/122] refactor: Remove unused code. --- .../datatools/manager/jobs/MergeFeedsJob.java | 20 +-------- .../feedmerge/CalendarMergeLineContext.java | 1 - .../jobs/feedmerge/FeedMergeContext.java | 43 ------------------- 3 files changed, 1 insertion(+), 63 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java b/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java index fe3a24c23..0660c858f 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java @@ -296,7 +296,7 @@ public void jobLogic() { Set tripIdsWithInconsistentSignature = getSharedTripIdsWithInconsistentSignature(); if (!tripIdsWithInconsistentSignature.isEmpty()) { failMergeJob( - String.format("Trips %s in new feed have differing makeup from matching trips in active feed." + + String.format("Trips %s in new feed have differing makeup from matching trips in active feed. " + "If a trip characteristic has changed, a new trip_id must be assigned.", String.join(", ", tripIdsWithInconsistentSignature) ) @@ -538,24 +538,6 @@ private void determineMergeStrategy() { .collect(Collectors.toList()) ); -/* - // If a trip only in the active feed references a service_id that is set to be extended, that - // service_id needs to be cloned and renamed to differentiate it from the same service_id in - // the future feed. (The trip in question will be linked to the cloned service_id.) - Set tripsOnlyInActiveFeed = Sets.difference(feedMergeContext.activeTripIds, feedMergeContext.futureTripIds); - tripsOnlyInActiveFeed.stream() - .map(tripId -> activeFeed.trips.get(tripId).service_id) - .filter(serviceId -> serviceIdsToExtend.contains(serviceId)) - .forEach(serviceId -> serviceIdsToCloneAndRename.add(serviceId)); - // If a trip only in the future feed references a service_id that is set to be extended, that - // service_id needs to be cloned and renamed to differentiate it from the same service_id in - // the future feed. (The trip in question will be linked to the cloned service_id.) - Set tripsOnlyInFutureFeed = Sets.difference(feedMergeContext.futureTripIds, feedMergeContext.activeTripIds); - tripsOnlyInFutureFeed.stream() - .map(tripId -> futureFeed.trips.get(tripId).service_id) - .filter(serviceId -> serviceIdsToExtend.contains(serviceId)) - .forEach(serviceId -> serviceIdsToCloneAndRename.add(serviceId)); -*/ mergeFeedsResult.mergeStrategy = CHECK_STOP_TIMES; } } diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarMergeLineContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarMergeLineContext.java index 8f7644765..f762980b1 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarMergeLineContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarMergeLineContext.java @@ -109,7 +109,6 @@ private void checkCalendarIds(Set idErrors, FieldContext fieldCont // service are exactly matching. fieldContext.resetValue(feedMergeContext.activeFeedFirstDate.format(GTFS_DATE_FORMATTER)); } - } else { } // Track service ID because we want to avoid removing trips that may reference this // service_id when the service_id is used by calendar_dates that operate in the valid diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/FeedMergeContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/FeedMergeContext.java index 385e17ae9..f3e1177c0 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/FeedMergeContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/FeedMergeContext.java @@ -90,49 +90,6 @@ public boolean areActiveAndFutureTripIdsDisjoint() { return sharedTripIds.isEmpty(); } - /** - * FIXME: Remove - this is from the old merge logic. - * Partially handles MTC Requirement to detect matching trip ids linked to different service ids. - * @return true if trip ids match but not service ids (in such situation, merge should fail). - */ - public boolean areTripIdsMatchingButNotServiceIds() { - // If only trip IDs match and not service IDs, do not permit merge to continue. - return tripIdsMatch && !serviceIdsMatch; - } - - /** - * Determines whether there a same trip id is linked to different service ids in the active and future feed - * (MTC requirement). - * @return the first {@link TripMismatchedServiceIds} whose trip id is linked to different service ids, - * or null if nothing found. - */ - public TripMismatchedServiceIds shouldFailJobDueToMatchingTripIds() { - if (serviceIdsMatch) { - // If just the service_ids are an exact match, check the that the stop_times having matching signatures - // between the two feeds (i.e., each stop time in the ordered list is identical between the two feeds). - for (String tripId : sharedTripIds) { - TripMismatchedServiceIds mismatchInfo = tripIdHasMismatchedServiceIds(tripId, futureFeed, activeFeed); - if (mismatchInfo.hasMismatch) { - return mismatchInfo; - } - } - } - - return null; - } - - /** - * Compare stop times for the given tripId between the future and active feeds. The comparison will inform whether - * trip and/or service IDs should be modified in the output merged feed. - * @return A {@link TripMismatchedServiceIds} with info on whether the given tripId is found in - * different service ids in the active and future feed. - */ - public static TripMismatchedServiceIds tripIdHasMismatchedServiceIds(String tripId, Feed futureFeed, Feed activeFeed) { - String futureServiceId = futureFeed.trips.get(tripId).service_id; - String activeServiceId = activeFeed.trips.get(tripId).service_id; - return new TripMismatchedServiceIds(tripId, !futureServiceId.equals(activeServiceId), activeServiceId, futureServiceId); - } - public LocalDate getFutureFirstCalendarStartDate() { return futureFirstCalendarStartDate; } From 21afc5031e00b79e9ec25fb79b94c6e6badc874b Mon Sep 17 00:00:00 2001 From: binh-dam-ibigroup <56846598+binh-dam-ibigroup@users.noreply.github.com> Date: Mon, 15 Nov 2021 17:11:55 -0500 Subject: [PATCH 062/122] fix: Add tests for updated merge logic, change code to make these tests pass (these tests only). --- .../datatools/manager/jobs/MergeFeedsJob.java | 47 ++------ .../feedmerge/CalendarMergeLineContext.java | 14 +-- .../jobs/feedmerge/FeedMergeContext.java | 7 ++ .../jobs/feedmerge/MergeLineContext.java | 45 ++++---- .../manager/jobs/MergeFeedsJobTest.java | 104 ++++++++++++++---- .../gtfs/merge-data-added-trips/agency.txt | 2 + .../gtfs/merge-data-added-trips/calendar.txt | 3 + .../gtfs/merge-data-added-trips/feed_info.txt | 2 + .../gtfs/merge-data-added-trips/routes.txt | 3 + .../stop_attributes.txt | 3 + .../merge-data-added-trips/stop_times.txt | 9 ++ .../gtfs/merge-data-added-trips/stops.txt | 6 + .../gtfs/merge-data-added-trips/trips.txt | 4 + .../merge-data-future-unique-ids/calendar.txt | 2 +- 14 files changed, 166 insertions(+), 85 deletions(-) create mode 100755 src/test/resources/com/conveyal/datatools/gtfs/merge-data-added-trips/agency.txt create mode 100755 src/test/resources/com/conveyal/datatools/gtfs/merge-data-added-trips/calendar.txt create mode 100644 src/test/resources/com/conveyal/datatools/gtfs/merge-data-added-trips/feed_info.txt create mode 100755 src/test/resources/com/conveyal/datatools/gtfs/merge-data-added-trips/routes.txt create mode 100644 src/test/resources/com/conveyal/datatools/gtfs/merge-data-added-trips/stop_attributes.txt create mode 100755 src/test/resources/com/conveyal/datatools/gtfs/merge-data-added-trips/stop_times.txt create mode 100755 src/test/resources/com/conveyal/datatools/gtfs/merge-data-added-trips/stops.txt create mode 100755 src/test/resources/com/conveyal/datatools/gtfs/merge-data-added-trips/trips.txt diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java b/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java index 0660c858f..dfd506b58 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java @@ -140,7 +140,7 @@ public class MergeFeedsJob extends FeedSourceJob { @JsonIgnore @BsonIgnore public Set serviceIdsToCloneAndRename = new HashSet<>(); @JsonIgnore @BsonIgnore - public Set serviceIdsToInsert = new HashSet<>(); + public Set serviceIdsToTerminateEarly = new HashSet<>(); private List sharedConsistentTripAndCalendarIds = new ArrayList<>(); @@ -263,36 +263,13 @@ public void jobLogic() { final List
tablesToMerge = getTablesToMerge(); int numberOfTables = tablesToMerge.size(); -/* - // Skip merging process altogether if the failing condition is met - FeedMergeContext.TripMismatchedServiceIds serviceIdMismatch; - if (feedMergeContext.areTripIdsMatchingButNotServiceIds()) { - failMergeJob("Feed merge failed because the trip_ids are identical in the future and active feeds. A new service requires unique trip_ids for merging."); - return; - } else if ((serviceIdMismatch = feedMergeContext.shouldFailJobDueToMatchingTripIds()) != null) { - // We cannot account for the case where service_ids do not match! It would be a bit too complicated - // to handle this unique case, so instead just include in the failure reasons and use failure - // strategy. - failMergeJob( - String.format("Shared trip_id (%s) had mismatched service id between two feeds (active: %s, future: %s)", - serviceIdMismatch.tripId, - serviceIdMismatch.activeServiceId, - serviceIdMismatch.futureServiceId - ) - ); - return; - } - - */ - // Before initiating the merge process, get the merge strategy to use, which runs some pre-processing to // check for id conflicts for certain tables (e.g., trips and calendars). if (mergeType.equals(SERVICE_PERIOD)) { determineMergeStrategy(); - // "If a single trip signature does not match the merge process shall stop with the following + // Failure condition "if a single trip signature does not match the merge process shall stop with the following // error message along with matching trip_ids with differing trip signatures." - // (MTC revised merge Step 2) Set tripIdsWithInconsistentSignature = getSharedTripIdsWithInconsistentSignature(); if (!tripIdsWithInconsistentSignature.isEmpty()) { failMergeJob( @@ -538,6 +515,14 @@ private void determineMergeStrategy() { .collect(Collectors.toList()) ); + // Build the set of calendars to be shortened to the day before the future feed start date + // from trips in the active feed but not in the future feed. + serviceIdsToTerminateEarly.addAll( + feedMergeContext.getActiveTripIdsNotInFutureFeed().stream() + .map(tripId -> activeFeed.trips.get(tripId).service_id) + .collect(Collectors.toList()) + ); + mergeFeedsResult.mergeStrategy = CHECK_STOP_TIMES; } } @@ -581,18 +566,6 @@ public FeedMergeContext getFeedMergeContext() { return feedMergeContext; } - /** - * @return true if the specified calendar id is listed as the active calendar id in any of the trips/calendar items. - */ - public boolean isActiveCalendarOfSharedTripId(String calendarId) { - for (TripAndCalendars item : sharedConsistentTripAndCalendarIds) { - if (item.activeCalendarId.equals(calendarId)) { - return true; - } - } - return false; - } - private static class TripAndCalendars { public final String tripId; public final String activeCalendarId; diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarMergeLineContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarMergeLineContext.java index f762980b1..1fdf6569b 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarMergeLineContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarMergeLineContext.java @@ -59,16 +59,14 @@ private void checkCalendarIds(Set idErrors, FieldContext fieldCont // but the ending date will be set to the day before the **start date of the new feed**. LocalDate endDate = getCsvDate("end_date"); LocalDate futureStartDate = null; - boolean activeAndFutureTripIdsDisjoint = job.sharedTripIdsWithConsistentSignature.isEmpty(); - if (activeAndFutureTripIdsDisjoint) { + boolean activeAndFutureTripIdsAreDisjoint = job.sharedTripIdsWithConsistentSignature.isEmpty(); + if (activeAndFutureTripIdsAreDisjoint) { futureStartDate = feedMergeContext.getFutureFirstCalendarStartDate(); + } else if (job.serviceIdsToTerminateEarly.contains(keyValue)) { + futureStartDate = futureFeedFirstDate; } else { - if (job.isActiveCalendarOfSharedTripId(keyValue)) { - // New calendar entry is already flagged for insertion from getMergeStrategy. - // Insert this calendar record for other trip ids that may reference it. - } else { - futureStartDate = futureFeedFirstDate; - } + // New calendar entry is already flagged for insertion from getMergeStrategy. + // Insert this calendar record for other trip ids that may reference it. } if (fieldContext.nameEquals("end_date")) { diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/FeedMergeContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/FeedMergeContext.java index f3e1177c0..0c7fe36b0 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/FeedMergeContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/FeedMergeContext.java @@ -110,6 +110,13 @@ public void setNewAgencyId(String newAgencyId) { this.newAgencyId = newAgencyId; } + /** + * Obtains the active trip ids found in the active feed, but not in the future feed. + */ + public Sets.SetView getActiveTripIdsNotInFutureFeed() { + return Sets.difference(activeTripIds, futureTripIds); + } + /** * Holds the status of a trip service id mismatch determination. */ diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java index f2bc4eaef..43c0299e7 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java @@ -456,26 +456,33 @@ public void initializeRowValues() { rowValues = new String[sharedSpecFields.size()]; } + /** + * Adds a cloned service id for trips with the same signature in both the active & future feeds. + * The cloned service id spans from the start date in the active feed until the end date in the future feed. + * @throws IOException + */ public void addClonedServiceId() throws IOException { - if ((table.name.equals("calendar")) && job.serviceIdsToCloneAndRename.contains(rowValues[keyFieldIndex])) { - // FIXME: Do we need to worry about calendar_dates? - String[] clonedValues = rowValues.clone(); - String newServiceId = clonedValues[keyFieldIndex] = String.join(":", idScope, rowValues[keyFieldIndex]); - // Modify start/end date. - int startDateIndex = Table.CALENDAR.getFieldIndex("start_date"); - int endDateIndex = Table.CALENDAR.getFieldIndex("end_date"); - clonedValues[startDateIndex] = feed.version.validationResult.firstCalendarDate.format(GTFS_DATE_FORMATTER); - clonedValues[endDateIndex] = feed.version.validationResult.lastCalendarDate.format(GTFS_DATE_FORMATTER); - referenceTracker.checkReferencesAndUniqueness( - keyValue, - lineNumber, - table.fields[0], - newServiceId, - table, - keyField, - orderField - ); - writeValuesToTable(clonedValues, true); + if (table.name.equals("calendar")) { + String originalServiceId = rowValues[keyFieldIndex]; + if (job.serviceIdsToCloneAndRename.contains(originalServiceId)) { + // FIXME: Do we need to worry about calendar_dates? + String[] clonedValues = rowValues.clone(); + String newServiceId = clonedValues[keyFieldIndex] = String.join(":", idScope, originalServiceId); + // Modify start date only (preserve the end date on the future calendar entry). + int startDateIndex = Table.CALENDAR.getFieldIndex("start_date"); + clonedValues[startDateIndex] = feedMergeContext.activeFeed.calendars.get(originalServiceId).start_date + .format(GTFS_DATE_FORMATTER); + referenceTracker.checkReferencesAndUniqueness( + keyValue, + lineNumber, + table.fields[0], + newServiceId, + table, + keyField, + orderField + ); + writeValuesToTable(clonedValues, true); + } } } diff --git a/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java b/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java index c64b3b1c1..f61016adf 100644 --- a/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java +++ b/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java @@ -61,8 +61,16 @@ public class MergeFeedsJobTest extends UnitTest { private static FeedVersion fakeTransitFutureUnique; /** The base feed but with differing service_ids. */ private static FeedVersion fakeTransitModService; - /** The base feed (transposed to the future dates) but with differing trip_ids. */ - private static FeedVersion fakeTransitModTrips; + /** + * The base feed (transposed to the future dates), with some trip_ids from the base feed with different signatures + * and some added trips. + */ + private static FeedVersion fakeTransitNewSignatureTrips; + /** + * The base feed (transposed to the future dates), with some trip_ids from the base feed with the same signature, + * and some added trips, and a trip from the base feed removed. + */ + private static FeedVersion fakeTransitSameSignatureTrips; private static FeedSource napa; private static FeedSource caltrain; private static FeedSource bart; @@ -131,7 +139,8 @@ public static void setUp() throws IOException { fakeTransitFuture = createFeedVersion(fakeTransit, zipFolderFiles("merge-data-future")); fakeTransitFutureUnique = createFeedVersion(fakeTransit, zipFolderFiles("merge-data-future-unique-ids")); fakeTransitModService = createFeedVersion(fakeTransit, zipFolderFiles("merge-data-mod-services")); - fakeTransitModTrips = createFeedVersion(fakeTransit, zipFolderFiles("merge-data-mod-trips")); + fakeTransitNewSignatureTrips = createFeedVersion(fakeTransit, zipFolderFiles("merge-data-mod-trips")); + fakeTransitSameSignatureTrips = createFeedVersion(fakeTransit, zipFolderFiles("merge-data-added-trips")); } /** @@ -348,14 +357,15 @@ public void mergeMTCShouldHandleExtendFutureStrategy() throws SQLException { } /** - * Ensures that an MTC merge of feeds with exact matches of service_ids and trip_ids will utilize the + * Ensures that an MTC merge of feeds with exact matches of service_ids and trip_ids, + * trip ids having the same signature (same stop times) will utilize the * {@link MergeStrategy#CHECK_STOP_TIMES} strategy correctly. */ @Test - public void mergeMTCShouldHandleCheckStopTimesStrategy() throws SQLException { + public void mergeMTCShouldHandleMatchingTripIdsWithSameSignature() throws SQLException { Set versions = new HashSet<>(); versions.add(fakeTransitBase); - versions.add(fakeTransitModTrips); + versions.add(fakeTransitSameSignatureTrips); MergeFeedsJob mergeFeedsJob = new MergeFeedsJob(user, versions, "merged_output", MergeFeedsType.SERVICE_PERIOD); // Run the job in this thread (we're not concerned about concurrency here). mergeFeedsJob.run(); @@ -369,39 +379,79 @@ public void mergeMTCShouldHandleCheckStopTimesStrategy() throws SQLException { mergeFeedsJob.mergeFeedsResult.failed, "Merge feeds job should succeed with CHECK_STOP_TIMES strategy." ); - // assert service_ids start_dates have been extended to the start_date of the base feed. + String mergedNamespace = mergeFeedsJob.mergedVersion.namespace; // - calendar table - // expect a total of 5 records in calendar table: - // - 2 original (common_id start date extended) - // - 2 cloned for active feed (from MergeFeedsJob#serviceIdsToCloneAndRename) - // - 1 cloned and modified for future feed (from MergeFeedsJob#serviceIdsToExtend) + // expect a total of 4 records in calendar table: + // - 1 from the active feed (common_id start date is changed to one day before first start_date in future feed) + // (the other one is unused and is discarded) + // - 2 from the future feed + // - 1 cloned for the matching trip id present in both active and future feeds + // (from MergeFeedsJob#serviceIdsToCloneAndRename). assertThatSqlCountQueryYieldsExpectedCount( String.format("SELECT count(*) FROM %s.calendar", mergedNamespace), - 5 + 4 ); + // expect that 2 calendars (1 common_id extended from future and 1 Fake_Transit1:common_id from active) have // start_date pinned to start date of active feed. assertThatSqlCountQueryYieldsExpectedCount( String.format("SELECT count(*) FROM %s.calendar where start_date = '20170918'", mergedNamespace), 2 ); - // Out of 6 total trips from the input datasets, expect 5 trips in merged output. - // 1 trip from active feed skipped because it matches the trip_id from the future feed exactly. - // 1 trip from active feed is cloned/modified because it differs from its future counterpart. + // One of the calendars above should have been extended + // until the end date of that entry in the future feed. + assertThatSqlCountQueryYieldsExpectedCount( + String.format("SELECT count(*) FROM %s.calendar where start_date = '20170918' and end_date='20170925'", mergedNamespace), + 1 + ); + // The other one should have end_date set to a day before the start of the future feed start date + // (in the test data, that first date comes from the other calendar entry). + assertThatSqlCountQueryYieldsExpectedCount( + String.format("SELECT count(*) FROM %s.calendar where start_date = '20170918' and end_date='20170919'", mergedNamespace), + 1 + ); + // Out of all trips from the input datasets, expect 4 trips in merged output. + // 1 trip from active feed that is not in the future feed, + // 1 trip in both the active and future feeds, with the same signature (same stop times), + // 2 trips from the future feed not in the active feed. assertThatSqlCountQueryYieldsExpectedCount( String.format("SELECT count(*) FROM %s.trips", mergedNamespace), - 5 + 4 ); } /** - * Ensures that an MTC merge of feeds with non-matching service_ids and trip_ids will utilize the + * Ensures that an MTC merge of feeds with trip_ids matching in the active and future feed, + * but with different signatures (e.g. different stop times) fails. + */ + @Test + public void mergeMTCShouldHandleMatchingTripIdsWithDifferentSignatures() { + Set versions = new HashSet<>(); + versions.add(fakeTransitBase); + versions.add(fakeTransitNewSignatureTrips); + MergeFeedsJob mergeFeedsJob = new MergeFeedsJob(user, versions, "merged_output", MergeFeedsType.SERVICE_PERIOD); + // Run the job in this thread (we're not concerned about concurrency here). + mergeFeedsJob.run(); + // Check that correct strategy was used. + assertEquals( + MergeStrategy.CHECK_STOP_TIMES, + mergeFeedsJob.mergeFeedsResult.mergeStrategy + ); + // Result should fail. + assertTrue( + mergeFeedsJob.mergeFeedsResult.failed, + "Merge feeds job with trip ids of different signatures should fail." + ); + } + + /** + * Ensures that an MTC merge of feeds with disjoint (non-matching) trip_ids will utilize the * {@link MergeStrategy#DEFAULT} strategy correctly. */ @Test - public void mergeMTCShouldHandleDefaultStrategy() throws SQLException { + public void mergeMTCShouldHandleDisjointTripIds() throws SQLException { Set versions = new HashSet<>(); versions.add(fakeTransitBase); versions.add(fakeTransitFutureUnique); @@ -422,11 +472,25 @@ public void mergeMTCShouldHandleDefaultStrategy() throws SQLException { String mergedNamespace = mergeFeedsJob.mergedVersion.namespace; // - calendar table - // expect a total of 4 records in calendar table (all records from original files are included). + // expect a total of 3 records in calendar table + // - 2 records from future feed + // - 1 records from active feed that is used + // - the unused record from the active feed is discarded. assertThatSqlCountQueryYieldsExpectedCount( String.format("SELECT count(*) FROM %s.calendar", mergedNamespace), - 4 + 3 ); + // The one calendar entry for the active feed should end one day before the first calendar start date + // of the future feed. + final String activeCalendarNewEndDate = "20170919"; // One day before 20170920. + assertThatSqlCountQueryYieldsExpectedCount( + String.format( + "SELECT count(*) FROM %s.calendar WHERE end_date='%s' AND service_id in ('Fake_Transit1:common_id', 'Fake_Transit1:only_calendar_id')", + mergedNamespace, + activeCalendarNewEndDate), + 1 + ); + // - trips table // expect a total of 4 records in trips table (all records from original files are included). assertThatSqlCountQueryYieldsExpectedCount( diff --git a/src/test/resources/com/conveyal/datatools/gtfs/merge-data-added-trips/agency.txt b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-added-trips/agency.txt new file mode 100755 index 000000000..a916ce91b --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-added-trips/agency.txt @@ -0,0 +1,2 @@ +agency_id,agency_name,agency_url,agency_lang,agency_phone,agency_email,agency_timezone,agency_fare_url,agency_branding_url +1,Fake Transit,,,,,America/Los_Angeles,, diff --git a/src/test/resources/com/conveyal/datatools/gtfs/merge-data-added-trips/calendar.txt b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-added-trips/calendar.txt new file mode 100755 index 000000000..8d5260fe6 --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-added-trips/calendar.txt @@ -0,0 +1,3 @@ +service_id,monday,tuesday,wednesday,thursday,friday,saturday,sunday,start_date,end_date +common_id,1,1,1,1,1,1,1,20170923,20170925 +only_calendar_id,1,1,1,1,1,1,1,20170920,20170927 diff --git a/src/test/resources/com/conveyal/datatools/gtfs/merge-data-added-trips/feed_info.txt b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-added-trips/feed_info.txt new file mode 100644 index 000000000..ceac60810 --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-added-trips/feed_info.txt @@ -0,0 +1,2 @@ +feed_id,feed_publisher_name,feed_publisher_url,feed_lang,feed_version +fake_transit,Conveyal,http://www.conveyal.com,en,1.0 \ No newline at end of file diff --git a/src/test/resources/com/conveyal/datatools/gtfs/merge-data-added-trips/routes.txt b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-added-trips/routes.txt new file mode 100755 index 000000000..b13480efa --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-added-trips/routes.txt @@ -0,0 +1,3 @@ +agency_id,route_id,route_short_name,route_long_name,route_desc,route_type,route_url,route_color,route_text_color,route_branding_url +1,1,1,Route 1,,3,,7CE6E7,FFFFFF, +1,2,2,Route 2,,3,,7CE6E7,FFFFFF, diff --git a/src/test/resources/com/conveyal/datatools/gtfs/merge-data-added-trips/stop_attributes.txt b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-added-trips/stop_attributes.txt new file mode 100644 index 000000000..b77c473e0 --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-added-trips/stop_attributes.txt @@ -0,0 +1,3 @@ +stop_id,accessibility_id,cardinal_direction,relative_position,stop_city +4u6g,0,SE,FS,Scotts Valley +johv,0,SE,FS,Scotts Valley diff --git a/src/test/resources/com/conveyal/datatools/gtfs/merge-data-added-trips/stop_times.txt b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-added-trips/stop_times.txt new file mode 100755 index 000000000..04d65a948 --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-added-trips/stop_times.txt @@ -0,0 +1,9 @@ +trip_id,arrival_time,departure_time,stop_id,stop_sequence,stop_headsign,pickup_type,drop_off_type,shape_dist_traveled,timepoint +trip3,07:00:00,07:00:00,4u6g,1,,0,0,0.0000000, +trip3,07:01:00,07:01:00,johv,2,,0,0,341.4491961, +only-calendar-trip1,07:00:00,07:00:00,4u6g,1,,0,0,0.0000000, +only-calendar-trip1,07:01:00,07:01:00,johv,2,,0,0,341.4491961, +only-calendar-trip2,07:00:00,07:00:00,johv,1,,0,0,0.0000000, +only-calendar-trip2,07:01:00,07:01:00,4u6g,2,,0,0,341.4491961, +only-calendar-trip999,07:00:00,07:00:00,johv,1,,0,0,0.0000000, +only-calendar-trip999,07:01:00,07:01:00,4u6g,2,,0,0,341.4491961, diff --git a/src/test/resources/com/conveyal/datatools/gtfs/merge-data-added-trips/stops.txt b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-added-trips/stops.txt new file mode 100755 index 000000000..0db5a6d40 --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-added-trips/stops.txt @@ -0,0 +1,6 @@ +stop_id,stop_code,stop_name,stop_desc,stop_lat,stop_lon,zone_id,stop_url,location_type,parent_station,stop_timezone,wheelchair_boarding +4u6g,4u6g,Butler Ln,,37.0612132,-122.0074332,,,0,,, +johv,johv,Scotts Valley Dr & Victor Sq,,37.0590172,-122.0096058,,,0,,, +123,,Parent Station,,37.0666,-122.0777,,,1,,, +1234,1234,Child Stop,,37.06662,-122.07772,,,0,123,, +1234567,1234567,Unused stop,,37.06668,-122.07781,,,0,123,, \ No newline at end of file diff --git a/src/test/resources/com/conveyal/datatools/gtfs/merge-data-added-trips/trips.txt b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-added-trips/trips.txt new file mode 100755 index 000000000..d745f3502 --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-added-trips/trips.txt @@ -0,0 +1,4 @@ +route_id,trip_id,trip_headsign,trip_short_name,direction_id,block_id,shape_id,bikes_allowed,wheelchair_accessible,service_id +1,only-calendar-trip999,,,0,,,0,0,common_id +2,only-calendar-trip2,,,0,,,0,0,common_id +2,trip3,,,0,,,0,0,only_calendar_id \ No newline at end of file diff --git a/src/test/resources/com/conveyal/datatools/gtfs/merge-data-future-unique-ids/calendar.txt b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-future-unique-ids/calendar.txt index 5d9454336..a5a410036 100755 --- a/src/test/resources/com/conveyal/datatools/gtfs/merge-data-future-unique-ids/calendar.txt +++ b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-future-unique-ids/calendar.txt @@ -1,3 +1,3 @@ service_id,monday,tuesday,wednesday,thursday,friday,saturday,sunday,start_date,end_date -future_id,1,1,1,1,1,1,1,20170923,20170925 +future_id,1,1,1,1,1,1,1,20170920,20170925 future_id_other,1,1,1,1,1,1,1,20170924,20170927 From b3f4962c7551f51bd9a85bd0217b4de09819e171 Mon Sep 17 00:00:00 2001 From: binh-dam-ibigroup <56846598+binh-dam-ibigroup@users.noreply.github.com> Date: Mon, 15 Nov 2021 17:31:44 -0500 Subject: [PATCH 063/122] test(MergeFeedsJob): Update other tests so they pass. --- .../manager/jobs/MergeFeedsJobTest.java | 82 +++++-------------- 1 file changed, 20 insertions(+), 62 deletions(-) diff --git a/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java b/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java index f61016adf..47e420ce1 100644 --- a/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java +++ b/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java @@ -38,10 +38,9 @@ */ public class MergeFeedsJobTest extends UnitTest { private static final Logger LOG = LoggerFactory.getLogger(MergeFeedsJobTest.class); - private static Auth0UserProfile user = Auth0UserProfile.createTestAdminUser(); + private static final Auth0UserProfile user = Auth0UserProfile.createTestAdminUser(); private static FeedVersion bartVersion1; private static FeedVersion bartVersion2SameTrips; - private static FeedVersion calTrainVersion; private static FeedVersion bartVersionOldLite; private static FeedVersion bartVersionNewLite; private static FeedVersion calTrainVersionLite; @@ -71,8 +70,6 @@ public class MergeFeedsJobTest extends UnitTest { * and some added trips, and a trip from the base feed removed. */ private static FeedVersion fakeTransitSameSignatureTrips; - private static FeedSource napa; - private static FeedSource caltrain; private static FeedSource bart; /** @@ -85,7 +82,7 @@ public static void setUp() throws IOException { // Create a project, feed sources, and feed versions to merge. project = new Project(); - project.name = String.format("Test %s", new Date().toString()); + project.name = String.format("Test %s", new Date()); Persistence.projects.create(project); // Bart @@ -97,15 +94,13 @@ public static void setUp() throws IOException { bartVersionNewLite = createFeedVersionFromGtfsZip(bart, "bart_new_lite.zip"); // Caltrain - caltrain = new FeedSource("Caltrain", project.id, MANUALLY_UPLOADED); + FeedSource caltrain = new FeedSource("Caltrain", project.id, MANUALLY_UPLOADED); Persistence.feedSources.create(caltrain); - calTrainVersion = createFeedVersionFromGtfsZip(caltrain, "caltrain_gtfs.zip"); calTrainVersionLite = createFeedVersionFromGtfsZip(caltrain, "caltrain_gtfs_lite.zip"); // Napa - napa = new FeedSource("Napa", project.id, MANUALLY_UPLOADED); + FeedSource napa = new FeedSource("Napa", project.id, MANUALLY_UPLOADED); Persistence.feedSources.create(napa); - napaVersion = createFeedVersionFromGtfsZip(napa, "napa-no-agency-id.zip"); napaVersionLite = createFeedVersionFromGtfsZip(napa, "napa-no-agency-id-lite.zip"); // Fake agencies (for testing calendar service_id merges with MTC strategy). @@ -313,15 +308,15 @@ public void mergeMTCShouldFailOnDuplicateTripsButMismatchedServices() { // Run the job in this thread (we're not concerned about concurrency here). mergeFeedsJob.run(); // Result should fail. - assertTrue( + assertFalse( mergeFeedsJob.mergeFeedsResult.failed, - "Merge feeds job should fail if feeds have exactly matching trips but mismatched services." + "If feeds have exactly matching trips but mismatched services, new service ids should be created that span both feeds." ); } /** * Ensures that an MTC merge of feeds with exact matches of service_ids and trip_ids will utilize the - * {@link MergeStrategy#EXTEND_FUTURE} strategy correctly. + * {@link MergeStrategy#CHECK_STOP_TIMES} strategy correctly. */ @Test public void mergeMTCShouldHandleExtendFutureStrategy() throws SQLException { @@ -334,22 +329,23 @@ public void mergeMTCShouldHandleExtendFutureStrategy() throws SQLException { // Result should fail. assertFalse( mergeFeedsJob.mergeFeedsResult.failed, - "Merge feeds job should succeed with EXTEND_FEED strategy." + "Merge feeds job should succeed with CHECK_STOP_TIMES strategy." ); assertEquals( - MergeStrategy.EXTEND_FUTURE, + MergeStrategy.CHECK_STOP_TIMES, mergeFeedsJob.mergeFeedsResult.mergeStrategy ); // assert service_ids start_dates have been extended to the start_date of the base feed. String mergedNamespace = mergeFeedsJob.mergedVersion.namespace; // - calendar table - // expect a total of 2 records in calendar table + // expect a total of 5 records in calendar table assertThatSqlCountQueryYieldsExpectedCount( String.format("SELECT count(*) FROM %s.calendar", mergedNamespace), - 2 + 5 ); - // expect that both records in calendar table have the correct start_date + // expect that two records in calendar table have the correct start_date + // (one for the original calendar entry, one for the extended service id for trips with same signature) assertThatSqlCountQueryYieldsExpectedCount( String.format("SELECT count(*) FROM %s.calendar where start_date = '20170918' and monday = 1", mergedNamespace), 2 @@ -557,59 +553,21 @@ public void canMergeBARTFeeds() throws SQLException { } /** - * Tests that the MTC merge strategy will successfully merge BART feeds. + * Tests that BART feeds with trips of same id but different signatures + * between active and future feeds cannot be merged per MTC revised merge logic. */ @Test - public void canMergeBARTFeedsSameTrips() throws SQLException { + public void shouldNotMergeBARTFeedsSameTrips() { Set versions = new HashSet<>(); versions.add(bartVersion1); versions.add(bartVersion2SameTrips); MergeFeedsJob mergeFeedsJob = new MergeFeedsJob(user, versions, "merged_output", MergeFeedsType.SERVICE_PERIOD); // Result should succeed this time. mergeFeedsJob.run(); - assertFeedMergeSucceeded(mergeFeedsJob); - // Check GTFS+ line numbers. - assertEquals( - 2, // Magic number represents expected number of lines after merge. - mergeFeedsJob.mergeFeedsResult.linesPerTable.get("directions").intValue(), - "Merged directions count should equal expected value." - ); - assertEquals( - 2, // Magic number represents the number of stop_attributes in the merged BART feed. - mergeFeedsJob.mergeFeedsResult.linesPerTable.get("stop_attributes").intValue(), - "Merged feed stop_attributes count should equal expected value." - ); - // Check GTFS file line numbers. - assertEquals( - 4629, // Magic number represents the number of trips in the merged BART feed. - mergeFeedsJob.mergedVersion.feedLoadResult.trips.rowCount, - "Merged feed trip count should equal expected value." - ); - assertEquals( - 9, // Magic number represents the number of routes in the merged BART feed. - mergeFeedsJob.mergedVersion.feedLoadResult.routes.rowCount, - "Merged feed route count should equal expected value." - ); - assertEquals( - // During merge, if identical shape_id is found in both feeds, active feed shape_id should be feed-scoped. - bartVersion1.feedLoadResult.shapes.rowCount + bartVersion2SameTrips.feedLoadResult.shapes.rowCount, - mergeFeedsJob.mergedVersion.feedLoadResult.shapes.rowCount, - "Merged feed shapes count should equal expected value." - ); - // Expect that two calendar dates are excluded from the active feed (because they occur after the first date of - // the future feed). - int expectedCalendarDatesCount = bartVersion1.feedLoadResult.calendarDates.rowCount + bartVersion2SameTrips.feedLoadResult.calendarDates.rowCount - 2; - assertEquals( - // During merge, if identical shape_id is found in both feeds, active feed shape_id should be feed-scoped. - expectedCalendarDatesCount, - mergeFeedsJob.mergedVersion.feedLoadResult.calendarDates.rowCount, - "Merged feed calendar_dates count should equal expected value." - ); - // Ensure there are no referential integrity errors or duplicate ID errors. - assertThatFeedHasNoErrorsOfType( - mergeFeedsJob.mergedVersion.namespace, - NewGTFSErrorType.REFERENTIAL_INTEGRITY.toString(), - NewGTFSErrorType.DUPLICATE_ID.toString() + // Result should fail. + assertTrue( + mergeFeedsJob.mergeFeedsResult.failed, + "Merge feeds job with trips of different signatures should fail." ); } From cba91414f7ba12158ff3477a6eca61d80d3ece34 Mon Sep 17 00:00:00 2001 From: binh-dam-ibigroup <56846598+binh-dam-ibigroup@users.noreply.github.com> Date: Mon, 15 Nov 2021 17:44:05 -0500 Subject: [PATCH 064/122] refactor: Remove unused fields. --- .../datatools/manager/jobs/MergeFeedsJob.java | 2 - .../feedmerge/CalendarMergeLineContext.java | 52 ++++++------------- .../jobs/feedmerge/MergeLineContext.java | 3 +- .../manager/jobs/feedmerge/MergeStrategy.java | 7 --- .../manager/jobs/MergeFeedsJobTest.java | 1 - 5 files changed, 16 insertions(+), 49 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java b/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java index dfd506b58..90050f7d0 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java @@ -136,8 +136,6 @@ public class MergeFeedsJob extends FeedSourceJob { @JsonIgnore @BsonIgnore public Set sharedTripIdsWithConsistentSignature = new HashSet<>(); @JsonIgnore @BsonIgnore - public Set serviceIdsToExtend = new HashSet<>(); - @JsonIgnore @BsonIgnore public Set serviceIdsToCloneAndRename = new HashSet<>(); @JsonIgnore @BsonIgnore public Set serviceIdsToTerminateEarly = new HashSet<>(); diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarMergeLineContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarMergeLineContext.java index 1fdf6569b..e73896208 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarMergeLineContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarMergeLineContext.java @@ -12,8 +12,6 @@ import java.util.Set; import java.util.zip.ZipOutputStream; -import static com.conveyal.datatools.manager.jobs.feedmerge.MergeStrategy.CHECK_STOP_TIMES; -import static com.conveyal.datatools.manager.jobs.feedmerge.MergeStrategy.EXTEND_FUTURE; import static com.conveyal.datatools.manager.utils.MergeFeedUtils.getTableScopedValue; import static com.conveyal.datatools.manager.utils.MergeFeedUtils.hasDuplicateError; import static com.conveyal.gtfs.loader.DateField.GTFS_DATE_FORMATTER; @@ -64,24 +62,26 @@ private void checkCalendarIds(Set idErrors, FieldContext fieldCont futureStartDate = feedMergeContext.getFutureFirstCalendarStartDate(); } else if (job.serviceIdsToTerminateEarly.contains(keyValue)) { futureStartDate = futureFeedFirstDate; - } else { - // New calendar entry is already flagged for insertion from getMergeStrategy. - // Insert this calendar record for other trip ids that may reference it. } + // In other cases not covered above, new calendar entry is already flagged for insertion + // from getMergeStrategy, so that trip ids may reference it. - if (fieldContext.nameEquals("end_date")) { - if (futureStartDate != null && !endDate.isBefore(futureStartDate)) { - fieldContext.resetValue(futureStartDate - .minus(1, ChronoUnit.DAYS) - .format(GTFS_DATE_FORMATTER)); - } + + if ( + fieldContext.nameEquals("end_date") && + futureStartDate != null && + !endDate.isBefore(futureStartDate) + ) { + fieldContext.resetValue(futureStartDate + .minus(1, ChronoUnit.DAYS) + .format(GTFS_DATE_FORMATTER)); } } - } else if (isHandlingFutureFeed()) { - // In the MTC revised feed merge logic: - // - Calendar entries from the future feed will be inserted as is in the merged feed. - // so no additional processing needed here. } + // If handling the future feed, the MTC revised feed merge logic is as follows: + // - Calendar entries from the future feed will be inserted as is in the merged feed. + // so no additional processing needed here. + // If any service_id in the active feed matches with the future // feed, it should be modified and all associated trip records @@ -97,31 +97,9 @@ private void checkCalendarIds(Set idErrors, FieldContext fieldCont updateAndRemapOutput(fieldContext); } - if (isHandlingFutureFeed()) { - // FIXME: Move this below so that a cloned service doesn't get prematurely - // modified? (do we want the cloned record to have the original values?) - if (shouldUpdateFutureFeedStartDate(fieldContext)) { - // Update start_date to extend service through the active feed's - // start date if the merge strategy dictates. The justification for this logic is that the active feed's - // service_id will be modified to a different unique value and the trips shared between the future/active - // service are exactly matching. - fieldContext.resetValue(feedMergeContext.activeFeedFirstDate.format(GTFS_DATE_FORMATTER)); - } - } // Track service ID because we want to avoid removing trips that may reference this // service_id when the service_id is used by calendar_dates that operate in the valid // date range, i.e., before the future feed's first date. if (!skipRecord && fieldContext.nameEquals(SERVICE_ID)) mergeFeedsResult.serviceIds.add(fieldContext.getValueToWrite()); } - - private boolean shouldUpdateFutureFeedStartDate(FieldContext fieldContext) { - return fieldContext.nameEquals("start_date") && - ( - EXTEND_FUTURE == mergeFeedsResult.mergeStrategy || - ( - CHECK_STOP_TIMES == mergeFeedsResult.mergeStrategy && - job.serviceIdsToExtend.contains(keyValue) - ) - ); - } } \ No newline at end of file diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java index 43c0299e7..0ae018f36 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java @@ -29,7 +29,6 @@ import static com.conveyal.datatools.manager.jobs.feedmerge.MergeFeedsType.REGIONAL; import static com.conveyal.datatools.manager.jobs.feedmerge.MergeFeedsType.SERVICE_PERIOD; -import static com.conveyal.datatools.manager.jobs.feedmerge.MergeStrategy.EXTEND_FUTURE; import static com.conveyal.datatools.manager.utils.MergeFeedUtils.containsField; import static com.conveyal.datatools.manager.utils.MergeFeedUtils.getAllFields; import static com.conveyal.datatools.manager.utils.MergeFeedUtils.getMergeKeyField; @@ -155,7 +154,7 @@ public boolean shouldSkipFile() { if (handlingActiveFeed && job.mergeType.equals(SERVICE_PERIOD)) { // Always prefer the "future" file for the feed_info table, which means // we can skip any iterations following the first one. - return EXTEND_FUTURE.equals(mergeFeedsResult.mergeStrategy) || table.name.equals("feed_info"); + return table.name.equals("feed_info"); } return false; } diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeStrategy.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeStrategy.java index 5d1562ab3..c3b783152 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeStrategy.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeStrategy.java @@ -12,13 +12,6 @@ public enum MergeStrategy { * the merged file. It shall be ensured that trip_ids between active and future datasets must not match. */ DEFAULT, - /** - * FIXME: Remove - no longer used (was in the old MTC feed merge) - * If service_ids and trip_ids in active feed are the same as future feed then the service end date for the - * merged feed shall match with future feed’s service end date and the service start date for the merged feed - * should be the merged date. All files from the future feed only shall be used in the merged feed. - */ - EXTEND_FUTURE, /** * If service_ids in active and future feed exactly match but only some of the trip_ids match then the merge * strategy shall handle the following three cases: diff --git a/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java b/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java index 47e420ce1..14e75f330 100644 --- a/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java +++ b/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java @@ -45,7 +45,6 @@ public class MergeFeedsJobTest extends UnitTest { private static FeedVersion bartVersionNewLite; private static FeedVersion calTrainVersionLite; private static Project project; - private static FeedVersion napaVersion; private static FeedVersion napaVersionLite; private static FeedVersion bothCalendarFilesVersion; private static FeedVersion bothCalendarFilesVersion2; From c1224bf3b163826e0c5d9b38a76311512a82d4d4 Mon Sep 17 00:00:00 2001 From: binh-dam-ibigroup <56846598+binh-dam-ibigroup@users.noreply.github.com> Date: Mon, 15 Nov 2021 18:43:31 -0500 Subject: [PATCH 065/122] fix(MergeFeedUtils): Use simplified stop times comparison (MTD req) --- .../datatools/manager/jobs/MergeFeedsJob.java | 25 +++++++++++-------- .../jobs/feedmerge/MergeLineContext.java | 2 +- .../manager/utils/MergeFeedUtils.java | 16 +++++++++--- 3 files changed, 29 insertions(+), 14 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java b/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java index 90050f7d0..192892178 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java @@ -136,7 +136,7 @@ public class MergeFeedsJob extends FeedSourceJob { @JsonIgnore @BsonIgnore public Set sharedTripIdsWithConsistentSignature = new HashSet<>(); @JsonIgnore @BsonIgnore - public Set serviceIdsToCloneAndRename = new HashSet<>(); + public Set serviceIdsToCloneRenameAndExtend = new HashSet<>(); @JsonIgnore @BsonIgnore public Set serviceIdsToTerminateEarly = new HashSet<>(); @@ -507,24 +507,29 @@ private void determineMergeStrategy() { // Build the set of calendars to be cloned/renamed/extended from trip ids present // in both active/future feeds and that have consistent signature. // These trips will be linked to the new service_ids. - serviceIdsToCloneAndRename.addAll( - sharedTripIdsWithConsistentSignature.stream() - .map(tripId -> activeFeed.trips.get(tripId).service_id) - .collect(Collectors.toList()) + serviceIdsToCloneRenameAndExtend.addAll( + getActiveServiceIds(this.sharedTripIdsWithConsistentSignature) ); // Build the set of calendars to be shortened to the day before the future feed start date // from trips in the active feed but not in the future feed. serviceIdsToTerminateEarly.addAll( - feedMergeContext.getActiveTripIdsNotInFutureFeed().stream() - .map(tripId -> activeFeed.trips.get(tripId).service_id) - .collect(Collectors.toList()) + getActiveServiceIds(feedMergeContext.getActiveTripIdsNotInFutureFeed()) ); mergeFeedsResult.mergeStrategy = CHECK_STOP_TIMES; } } + /** + * Obtains the service ids corresponding to the provided trip ids. + */ + private List getActiveServiceIds(Set tripIds) { + return tripIds.stream() + .map(tripId -> feedMergeContext.activeFeed.trips.get(tripId).service_id) + .collect(Collectors.toList()); + } + /** * Compare stop times for the given tripId between the future and active feeds. The comparison will inform whether * trip and/or service IDs should be modified in the output merged feed. @@ -537,7 +542,7 @@ private void compareStopTimesAndCollectTripAndServiceIds(String tripId, Feed fut List activeStopTimes = Lists.newArrayList(activeFeed.stopTimes.getOrdered(tripId)); String activeServiceId = activeFeed.trips.get(tripId).service_id; String futureServiceId = futureFeed.trips.get(tripId).service_id; - if (!stopTimesMatch(futureStopTimes, activeStopTimes)) { + if (!stopTimesMatchSimplified(futureStopTimes, activeStopTimes)) { // If stop_times or services do not match, merge will fail and no other action will be taken. sharedTripIdsWithInconsistentSignature.add(tripId); } else { @@ -545,7 +550,7 @@ private void compareStopTimesAndCollectTripAndServiceIds(String tripId, Feed fut // future trip and exclude the active one. Also, mark the service_id for cloning, // the cloned service id will need to be extended to the full time range. sharedTripIdsWithConsistentSignature.add(tripId); - serviceIdsToCloneAndRename.add(futureServiceId); + serviceIdsToCloneRenameAndExtend.add(futureServiceId); sharedConsistentTripAndCalendarIds.add(new TripAndCalendars(tripId, activeServiceId, futureServiceId)); } } diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java index 0ae018f36..127e2f860 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java @@ -463,7 +463,7 @@ public void initializeRowValues() { public void addClonedServiceId() throws IOException { if (table.name.equals("calendar")) { String originalServiceId = rowValues[keyFieldIndex]; - if (job.serviceIdsToCloneAndRename.contains(originalServiceId)) { + if (job.serviceIdsToCloneRenameAndExtend.contains(originalServiceId)) { // FIXME: Do we need to worry about calendar_dates? String[] clonedValues = rowValues.clone(); String newServiceId = clonedValues[keyFieldIndex] = String.join(":", idScope, originalServiceId); diff --git a/src/main/java/com/conveyal/datatools/manager/utils/MergeFeedUtils.java b/src/main/java/com/conveyal/datatools/manager/utils/MergeFeedUtils.java index fa8fe6252..87c856037 100644 --- a/src/main/java/com/conveyal/datatools/manager/utils/MergeFeedUtils.java +++ b/src/main/java/com/conveyal/datatools/manager/utils/MergeFeedUtils.java @@ -150,14 +150,24 @@ public static String getTableScopedValue(Table table, String prefix, String id) } /** - * Checks whether the future and active stop_times for a particular trip_id are an exact match. + * Checks whether the future and active stop_times for a particular trip_id are an exact match, + * using these criteria only: arrival_time, departure_time, stop_id, and stop_sequence + * instead of StopTime::equals (Revised MTC feed merge requirement). */ - public static boolean stopTimesMatch(List futureStopTimes, List activeStopTimes) { + public static boolean stopTimesMatchSimplified(List futureStopTimes, List activeStopTimes) { if (futureStopTimes.size() != activeStopTimes.size()) { return false; } for (int i = 0; i < activeStopTimes.size(); i++) { - if (!activeStopTimes.get(i).equals(futureStopTimes.get(i))) { + StopTime activeTime = activeStopTimes.get(i); + StopTime futureTime = futureStopTimes.get(i); + + if ( + activeTime.arrival_time != futureTime.arrival_time || + activeTime.departure_time != futureTime.departure_time || + activeTime.stop_sequence != futureTime.stop_sequence || + !activeTime.stop_id.equals(futureTime.stop_id) + ) { return false; } } From daa18a3c6319efc180e729fde4904474447bc374 Mon Sep 17 00:00:00 2001 From: binh-dam-ibigroup <56846598+binh-dam-ibigroup@users.noreply.github.com> Date: Mon, 15 Nov 2021 19:23:18 -0500 Subject: [PATCH 066/122] refactor: Remove unused code. --- .../CalendarDatesMergeLineContext.java | 32 ++++++++------- .../feedmerge/CalendarMergeLineContext.java | 7 ++-- .../jobs/feedmerge/FeedMergeContext.java | 39 +++---------------- .../jobs/feedmerge/MergeLineContext.java | 4 +- 4 files changed, 28 insertions(+), 54 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarDatesMergeLineContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarDatesMergeLineContext.java index 92aa05ab3..b8cc8d87e 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarDatesMergeLineContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarDatesMergeLineContext.java @@ -17,6 +17,8 @@ public class CalendarDatesMergeLineContext extends MergeLineContext { private static final Logger LOG = LoggerFactory.getLogger(CalendarDatesMergeLineContext.class); + private LocalDate futureFeedFirstDateForCalendarValidity; + public CalendarDatesMergeLineContext(MergeFeedsJob job, Table table, ZipOutputStream out) throws IOException { super(job, table, out); } @@ -29,19 +31,18 @@ public void checkFieldsForMergeConflicts(Set idErrors, FieldContex @Override public void startNewRow() throws IOException { super.startNewRow(); - updateFutureFeedFirstDate(); + futureFeedFirstDateForCalendarValidity = getFutureFeedFirstDateForCheckingCalendarValidity(); } private void checkCalendarDatesIds(FieldContext fieldContext) throws IOException { // Drop any calendar_dates.txt records from the existing feed for dates that are // not before the first date of the future feed. LocalDate date = getCsvDate("date"); - LocalDate futureFeedFirstDate = feedMergeContext.getFutureFeedFirstDate(); - if (isHandlingActiveFeed() && !date.isBefore(futureFeedFirstDate)) { + if (isHandlingActiveFeed() && !date.isBefore(futureFeedFirstDateForCalendarValidity)) { LOG.warn( "Skipping calendar_dates entry {} because it operates in the time span of future feed (i.e., after or on {}).", keyValue, - futureFeedFirstDate); + futureFeedFirstDateForCalendarValidity); String key = getTableScopedValue(table, getIdScope(), keyValue); mergeFeedsResult.skippedIds.add(key); skipRecord = true; @@ -52,21 +53,22 @@ private void checkCalendarDatesIds(FieldContext fieldContext) throws IOException if (!skipRecord && fieldContext.nameEquals(SERVICE_ID)) mergeFeedsResult.serviceIds.add(fieldContext.getValueToWrite()); } - // FIXME: move this (almost seems irrelevant) - private void updateFutureFeedFirstDate() { - // This will be populated because calendar dates is processed - // after calendar, which populates FutureFirstCalendarStartDate. - LocalDate futureFirstCalendarStartDate = feedMergeContext.getFutureFirstCalendarStartDate(); + /** + * Obtains the future feed start date to use + * if the future feed's first date is before its first calendar start date, + * when checking MTC calendar_dates and calendar records for modification/exclusion. + */ + private LocalDate getFutureFeedFirstDateForCheckingCalendarValidity() { + LocalDate futureFirstCalendarStartDate = feedMergeContext.futureFirstCalendarStartDate; + LocalDate futureFeedFirstDate = feedMergeContext.futureFeedFirstDate; if ( isHandlingActiveFeed() && - job.mergeType.equals(SERVICE_PERIOD) && + job.mergeType.equals(SERVICE_PERIOD) && futureFirstCalendarStartDate.isBefore(LocalDate.MAX) && - feedMergeContext.getFutureFeedFirstDate().isBefore(futureFirstCalendarStartDate) + futureFeedFirstDate.isBefore(futureFirstCalendarStartDate) ) { - // If the future feed's first date is before its first calendar start date, - // override the future feed first date with the calendar start date for use when checking - // MTC calendar_dates and calendar records for modification/exclusion. - feedMergeContext.setFutureFeedFirstDate(futureFirstCalendarStartDate); + return futureFirstCalendarStartDate; } + return futureFeedFirstDate; } } \ No newline at end of file diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarMergeLineContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarMergeLineContext.java index e73896208..22d3698ee 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarMergeLineContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarMergeLineContext.java @@ -37,8 +37,7 @@ private void checkCalendarIds(Set idErrors, FieldContext fieldCont // calendar_dates, and calendar_attributes referencing this // service_id shall also be removed/ignored. Stop_time records // for the ignored trips shall also be removed. - LocalDate futureFeedFirstDate = feedMergeContext.getFutureFeedFirstDate(); - if (!startDate.isBefore(futureFeedFirstDate)) { + if (!startDate.isBefore(feedMergeContext.futureFeedFirstDate)) { LOG.warn( "Skipping calendar entry {} because it operates fully within the time span of future feed.", keyValue); @@ -59,9 +58,9 @@ private void checkCalendarIds(Set idErrors, FieldContext fieldCont LocalDate futureStartDate = null; boolean activeAndFutureTripIdsAreDisjoint = job.sharedTripIdsWithConsistentSignature.isEmpty(); if (activeAndFutureTripIdsAreDisjoint) { - futureStartDate = feedMergeContext.getFutureFirstCalendarStartDate(); + futureStartDate = feedMergeContext.futureFirstCalendarStartDate; } else if (job.serviceIdsToTerminateEarly.contains(keyValue)) { - futureStartDate = futureFeedFirstDate; + futureStartDate = feedMergeContext.futureFeedFirstDate; } // In other cases not covered above, new calendar entry is already flagged for insertion // from getMergeStrategy, so that trip ids may reference it. diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/FeedMergeContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/FeedMergeContext.java index 0c7fe36b0..8ed6358fd 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/FeedMergeContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/FeedMergeContext.java @@ -29,8 +29,8 @@ public class FeedMergeContext implements Closeable { public final Feed futureFeed; public final Feed activeFeed; public final LocalDate activeFeedFirstDate; - private LocalDate futureFeedFirstDate; - private LocalDate futureFirstCalendarStartDate = LocalDate.MAX; + public final LocalDate futureFeedFirstDate; + public final LocalDate futureFirstCalendarStartDate; /** * Trip ids shared between the active and future feed. */ @@ -67,11 +67,13 @@ public FeedMergeContext(Set feedVersions, Auth0UserProfile owner) t futureFeedFirstDate = futureFeedToMerge.version.validationResult.firstCalendarDate; // Initialize, before processing rows, the calendar start dates from the future feed. + LocalDate futureFirstCalStartDate = LocalDate.MAX; for (Calendar c : futureFeed.calendars.getAll()) { - if (futureFirstCalendarStartDate.isAfter(c.start_date)) { - futureFirstCalendarStartDate = c.start_date; + if (futureFirstCalStartDate.isAfter(c.start_date)) { + futureFirstCalStartDate = c.start_date; } } + this.futureFirstCalendarStartDate = futureFirstCalStartDate; } @Override @@ -90,18 +92,6 @@ public boolean areActiveAndFutureTripIdsDisjoint() { return sharedTripIds.isEmpty(); } - public LocalDate getFutureFirstCalendarStartDate() { - return futureFirstCalendarStartDate; - } - - public LocalDate getFutureFeedFirstDate() { - return futureFeedFirstDate; - } - - public void setFutureFeedFirstDate(LocalDate futureFeedFirstDate) { - this.futureFeedFirstDate = futureFeedFirstDate; - } - public String getNewAgencyId() { return newAgencyId; } @@ -116,21 +106,4 @@ public void setNewAgencyId(String newAgencyId) { public Sets.SetView getActiveTripIdsNotInFutureFeed() { return Sets.difference(activeTripIds, futureTripIds); } - - /** - * Holds the status of a trip service id mismatch determination. - */ - public static class TripMismatchedServiceIds { - public final String tripId; - public final String activeServiceId; - public final String futureServiceId; - public final boolean hasMismatch; - - TripMismatchedServiceIds(String tripId, boolean hasMismatch, String activeServiceId, String futureServiceId) { - this.tripId = tripId; - this.hasMismatch = hasMismatch; - this.activeServiceId = activeServiceId; - this.futureServiceId = futureServiceId; - } - } } diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java index 127e2f860..fda033d4d 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java @@ -221,7 +221,7 @@ public boolean areForeignRefsOk(FieldContext fieldContext) throws IOException { // been skipped or is a ref to a non-existent service_id during a service period merge, skip // this record and add its primary key to the list of skipped IDs (so that other references // can be properly omitted). - if (serviceIdHasOrShouldBeSkipped(fieldContext, key, isValidServiceId)) { + if (serviceIdHasKeyOrShouldBeSkipped(fieldContext, key, isValidServiceId)) { // If a calendar#service_id has been skipped (it's listed in skippedIds), but there were // valid service_ids found in calendar_dates, do not skip that record for both the // calendar_date and any related trips. @@ -249,7 +249,7 @@ public boolean areForeignRefsOk(FieldContext fieldContext) throws IOException { return true; } - private boolean serviceIdHasOrShouldBeSkipped(FieldContext fieldContext, String key, boolean isValidServiceId) { + private boolean serviceIdHasKeyOrShouldBeSkipped(FieldContext fieldContext, String key, boolean isValidServiceId) { boolean serviceIdShouldBeSkipped = job.mergeType.equals(SERVICE_PERIOD) && fieldContext.nameEquals(SERVICE_ID) && !isValidServiceId; From 213d5b284bda57d017c2aea76ce7d2ae03053171 Mon Sep 17 00:00:00 2001 From: binh-dam-ibigroup <56846598+binh-dam-ibigroup@users.noreply.github.com> Date: Tue, 16 Nov 2021 08:24:08 -0500 Subject: [PATCH 067/122] refactor(MergeLineContext): Abstract row post-write. --- .../CalendarDatesMergeLineContext.java | 5 +- .../feedmerge/CalendarMergeLineContext.java | 36 +++++++++++++++ .../jobs/feedmerge/MergeLineContext.java | 46 +++++-------------- 3 files changed, 51 insertions(+), 36 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarDatesMergeLineContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarDatesMergeLineContext.java index b8cc8d87e..f44be0492 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarDatesMergeLineContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarDatesMergeLineContext.java @@ -17,6 +17,7 @@ public class CalendarDatesMergeLineContext extends MergeLineContext { private static final Logger LOG = LoggerFactory.getLogger(CalendarDatesMergeLineContext.class); + /** Holds the date used to check calendar validity */ private LocalDate futureFeedFirstDateForCalendarValidity; public CalendarDatesMergeLineContext(MergeFeedsJob job, Table table, ZipOutputStream out) throws IOException { @@ -29,8 +30,8 @@ public void checkFieldsForMergeConflicts(Set idErrors, FieldContex } @Override - public void startNewRow() throws IOException { - super.startNewRow(); + public void startNewFeed(int feedIndex) throws IOException { + super.startNewFeed(feedIndex); futureFeedFirstDateForCalendarValidity = getFutureFeedFirstDateForCheckingCalendarValidity(); } diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarMergeLineContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarMergeLineContext.java index 22d3698ee..7a6582af6 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarMergeLineContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarMergeLineContext.java @@ -28,6 +28,14 @@ public void checkFieldsForMergeConflicts(Set idErrors, FieldContex checkCalendarIds(idErrors, fieldContext); } + @Override + public void afterRowWrite() throws IOException { + // If the current row is for a calendar service_id that is marked for cloning/renaming, clone the + // values, change the ID, extend the start/end dates to the feed's full range, and write the + // additional line to the file. + addClonedServiceId(); + } + private void checkCalendarIds(Set idErrors, FieldContext fieldContext) throws IOException { if (isHandlingActiveFeed()) { LocalDate startDate = getCsvDate("start_date"); @@ -101,4 +109,32 @@ private void checkCalendarIds(Set idErrors, FieldContext fieldCont // date range, i.e., before the future feed's first date. if (!skipRecord && fieldContext.nameEquals(SERVICE_ID)) mergeFeedsResult.serviceIds.add(fieldContext.getValueToWrite()); } + + /** + * Adds a cloned service id for trips with the same signature in both the active & future feeds. + * The cloned service id spans from the start date in the active feed until the end date in the future feed. + * @throws IOException + */ + public void addClonedServiceId() throws IOException { + String originalServiceId = getRowValues()[keyFieldIndex]; + if (job.serviceIdsToCloneRenameAndExtend.contains(originalServiceId)) { + // FIXME: Do we need to worry about calendar_dates? + String[] clonedValues = getRowValues().clone(); + String newServiceId = clonedValues[keyFieldIndex] = String.join(":", getIdScope(), originalServiceId); + // Modify start date only (preserve the end date on the future calendar entry). + int startDateIndex = Table.CALENDAR.getFieldIndex("start_date"); + clonedValues[startDateIndex] = feedMergeContext.activeFeed.calendars.get(originalServiceId).start_date + .format(GTFS_DATE_FORMATTER); + referenceTracker.checkReferencesAndUniqueness( + keyValue, + getLineNumber(), + table.fields[0], + newServiceId, + table, + keyField, + table.getOrderFieldName() + ); + writeValuesToTable(clonedValues, true); + } + } } \ No newline at end of file diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java index fda033d4d..99e6eae5f 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java @@ -439,6 +439,13 @@ public void checkFirstLineConditions() throws IOException { // Default is to do nothing. } + /** + * Overridable placeholder for additional processing after writing the current row. + */ + public void afterRowWrite() throws IOException { + // Default is to do nothing. + } + public void scopeValueIfNeeded(FieldContext fieldContext) { boolean isKeyField = fieldContext.getField().isForeignReference() || fieldContext.nameEquals(keyField); if (job.mergeType.equals(REGIONAL) && isKeyField && !fieldContext.getValue().isEmpty()) { @@ -455,36 +462,6 @@ public void initializeRowValues() { rowValues = new String[sharedSpecFields.size()]; } - /** - * Adds a cloned service id for trips with the same signature in both the active & future feeds. - * The cloned service id spans from the start date in the active feed until the end date in the future feed. - * @throws IOException - */ - public void addClonedServiceId() throws IOException { - if (table.name.equals("calendar")) { - String originalServiceId = rowValues[keyFieldIndex]; - if (job.serviceIdsToCloneRenameAndExtend.contains(originalServiceId)) { - // FIXME: Do we need to worry about calendar_dates? - String[] clonedValues = rowValues.clone(); - String newServiceId = clonedValues[keyFieldIndex] = String.join(":", idScope, originalServiceId); - // Modify start date only (preserve the end date on the future calendar entry). - int startDateIndex = Table.CALENDAR.getFieldIndex("start_date"); - clonedValues[startDateIndex] = feedMergeContext.activeFeed.calendars.get(originalServiceId).start_date - .format(GTFS_DATE_FORMATTER); - referenceTracker.checkReferencesAndUniqueness( - keyValue, - lineNumber, - table.fields[0], - newServiceId, - table, - keyField, - orderField - ); - writeValuesToTable(clonedValues, true); - } - } - } - public void writeValuesToTable(String[] values, boolean incrementLineNumbers) throws IOException { writer.write(values); if (incrementLineNumbers) { @@ -570,10 +547,9 @@ public void finishRowAndWriteToZip() throws IOException { } // Write line to table. writeValuesToTable(rowValues, true); - // If the current row is for a calendar service_id that is marked for cloning/renaming, clone the - // values, change the ID, extend the start/end dates to the feed's full range, and write the - // additional line to the file. - addClonedServiceId(); + + // Optional table-specific additional processing. + afterRowWrite(); } public boolean lineIsBlank() throws IOException { @@ -610,6 +586,8 @@ protected int getLineNumber() { return lineNumber; } + protected String[] getRowValues() { return rowValues; } + /** * Retrieves the value for the specified CSV field. */ From 06b2156db917fa518ba09f157a55369277cc7aa7 Mon Sep 17 00:00:00 2001 From: binh-dam-ibigroup <56846598+binh-dam-ibigroup@users.noreply.github.com> Date: Tue, 16 Nov 2021 10:18:58 -0500 Subject: [PATCH 068/122] refactor: Address PR comments. --- .../datatools/manager/jobs/MergeFeedsJob.java | 81 ++----------------- .../jobs/feedmerge/MergeFeedsResult.java | 2 - .../jobs/feedmerge/MergeLineContext.java | 25 +++--- .../jobs/feedmerge/StopsMergeLineContext.java | 8 +- .../manager/utils/MergeFeedUtils.java | 26 +++--- 5 files changed, 40 insertions(+), 102 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java b/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java index 1ed8fcb27..3c0e693a5 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java @@ -55,7 +55,8 @@ * found in any other feed version. Note: There is absolutely no attempt to merge * entities based on either expected shared IDs or entity location (e.g., stop * coordinates). - * - {@link MergeFeedsType#SERVICE_PERIOD}: this strategy is defined in detail at https://github.com/conveyal/datatools-server/issues/185, + * - {@link MergeFeedsType#SERVICE_PERIOD}: + * this strategy is defined in detail at https://github.com/conveyal/datatools-server/issues/185, * but in essence, this strategy attempts to merge an active and future feed into * a combined file. For certain entities (specifically stops and routes) it uses * alternate fields as primary keys (stop_code and route_short_name) if they are @@ -66,55 +67,6 @@ * prefer entities from the active version, so that entities edited in Data Tools would override the values found * in the "future" file, which may have limited data attributes due to being exported from scheduling software with * limited GTFS support. - * - * Reproduced from https://github.com/conveyal/datatools-server/issues/185 on 2019/04/23: - * - * 1. When a new GTFS+ feed is loaded in TDM, check as part of the loading and validation process if - * the dataset is for a future date. (If all services start in the future, consider the dataset - * to be for the future). - * 2. If it is a future dataset, automatically notify the user that the feed needs to be merged with - * most recent active version or a selected one in order to further process the feed. - * 3. Use the chosen version to merge the future feed. The merging process needs to be efficient so - * that the user doesn’t need to wait more than a tolerable time. - * 4. The merge process shall compare the active and future datasets, validate the following rules - * and generate the Merge Validation Report: - * i. Merging will be based on route_short_name in the active and future datasets. All matching - * route_short_names between the datasets shall be considered same route. Any route_short_name - * in active data not present in the future will be appended to the future routes file. - * ii. Future feed_info.txt file should get priority over active feed file when difference is - * identified. - * iii. When difference is found in agency.txt file between active and future feeds, the future - * agency.txt file data should be used. Possible issue with missing agency_id referenced by routes - * iv. When stop_code is included, stop merging will be based on that. If stop_code is not - * included, it will be based on stop_id. All stops in future data will be carried forward and - * any stops found in active data that are not in the future data shall be appended. If one - * of the feed is missing stop_code, merge fails with a notification to the user with - * suggestion that the feed with missing stop_code must be fixed with stop_code. - * v. If any service_id in the active feed matches with the future feed, it should be modified - * and all associated trip records must also be changed with the modified service_id. - * If a service_id from the active calendar has both the start_date and end_date in the - * future, the service shall not be appended to the merged file. Records in trips, - * calendar_dates, and calendar_attributes referencing this service_id shall also be - * removed/ignored. Stop_time records for the ignored trips shall also be removed. - * If a service_id from the active calendar has only the end_date in the future, the end_date - * shall be set to one day prior to the earliest start_date in future dataset before appending - * the calendar record to the merged file. - * trip_ids between active and future datasets must not match. If any trip_id is found to be - * matching, the merge should fail with appropriate notification to user with the cause of the - * failure. Notification should include all matched trip_ids. - * vi. New shape_ids in the future datasets should be appended in the merged feed. - * vii. Merging fare_attributes will be based on fare_id in the active and future datasets. All - * matching fare_ids between the datasets shall be considered same fare. Any fare_id in active - * data not present in the future will be appended to the future fare_attributes file. - * viii. All fare rules from the future dataset will be included. Any identical fare rules from - * the active dataset will be discarded. Any fare rules unique to the active dataset will be - * appended to the future file. - * ix. All transfers.txt entries with unique stop pairs (from - to) from both the future and - * active datasets will be included in the merged file. Entries with duplicate stop pairs from - * the active dataset will be discarded. - * x. All GTFS+ files should be merged based on how the associated base GTFS file is merged. For - * example, directions for routes that are not in the future routes.txt file should be appended - * to the future directions.txt file in the merged feed. */ public class MergeFeedsJob extends FeedSourceJob { @@ -143,31 +95,13 @@ public class MergeFeedsJob extends FeedSourceJob { // Variables used for a service period merge. private FeedMergeContext feedMergeContext; - public MergeFeedsJob(Auth0UserProfile owner, Set feedVersions, String file, MergeFeedsType mergeType) { - this(owner, feedVersions, file, mergeType, true); - } - - /** Shorthand method to get the future feed during a service period merge */ - @BsonIgnore @JsonIgnore - public FeedToMerge getFutureFeed() { - return feedMergeContext.futureFeedToMerge; - } - - /** Shorthand method to get the active feed during a service period merge */ - @BsonIgnore @JsonIgnore - public FeedToMerge getActiveFeed() { - return feedMergeContext.activeFeedToMerge; - } - /** * @param owner user ID that initiated job * @param feedVersions set of feed versions to merge * @param file resulting merge filename (without .zip) * @param mergeType the type of merge to perform {@link MergeFeedsType} - * @param storeNewVersion whether to store merged feed as new version */ - public MergeFeedsJob(Auth0UserProfile owner, Set feedVersions, String file, - MergeFeedsType mergeType, boolean storeNewVersion) { + public MergeFeedsJob(Auth0UserProfile owner, Set feedVersions, String file, MergeFeedsType mergeType) { super(owner, mergeType.equals(REGIONAL) ? "Merging project feeds" : "Merging feed versions", JobType.MERGE_FEED_VERSIONS); this.feedVersions = feedVersions; @@ -182,7 +116,7 @@ public MergeFeedsJob(Auth0UserProfile owner, Set feedVersions, Stri // Grab parent feed source depending on merge type. FeedSource regionalFeedSource = null; // If storing a regional merge as a new version, find the feed source designated by the project. - if (mergeType.equals(REGIONAL) && storeNewVersion) { + if (mergeType.equals(REGIONAL)) { regionalFeedSource = Persistence.feedSources.getById(project.regionalFeedSourceId); // Create new feed source if this is the first regional merge. if (regionalFeedSource == null) { @@ -201,7 +135,7 @@ public MergeFeedsJob(Auth0UserProfile owner, Set feedVersions, Stri : feedVersions.iterator().next().parentFeedSource(); // Assuming job is successful, mergedVersion will contain the resulting feed version. // Merged version will be null if the new version should not be stored. - this.mergedVersion = getMergedVersion(this, storeNewVersion); + this.mergedVersion = getMergedVersion(this, true); this.mergeFeedsResult = new MergeFeedsResult(mergeType); } @@ -210,11 +144,6 @@ public Set getFeedVersions() { return this.feedVersions; } - @BsonIgnore @JsonIgnore - public List getFeedsToMerge() { - return this.feedMergeContext.feedsToMerge; - } - /** * The final stage handles clean up (deleting temp file) and adding the next job to process the * new merged version (assuming the merge did not fail). diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeFeedsResult.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeFeedsResult.java index a5a732c69..4a0c8f6bd 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeFeedsResult.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeFeedsResult.java @@ -18,8 +18,6 @@ public class MergeFeedsResult implements Serializable { /** Type of merge operation performed */ public MergeFeedsType type; public MergeStrategy mergeStrategy = MergeStrategy.DEFAULT; - /** Contains a set of strings for which there were error-causing duplicate values */ - public Set idConflicts = new HashSet<>(); /** Contains the set of IDs for records that were excluded in the merged feed */ public Set skippedIds = new HashSet<>(); /** diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java index 189c8cbb8..60081fbd7 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java @@ -41,7 +41,6 @@ public class MergeLineContext { protected static final String AGENCY_ID = "agency_id"; protected static final String SERVICE_ID = "service_id"; - private static final String STOPS = "stops"; private static final Logger LOG = LoggerFactory.getLogger(MergeLineContext.class); protected final MergeFeedsJob job; private final ZipOutputStream out; @@ -91,7 +90,7 @@ public static MergeLineContext create(MergeFeedsJob job, Table table, ZipOutputS return new RoutesMergeLineContext(job, table, out); case "shapes": return new ShapesMergeLineContext(job, table, out); - case STOPS: + case "stops": return new StopsMergeLineContext(job, table, out); case "trips": return new TripsMergeLineContext(job, table, out); @@ -268,21 +267,17 @@ public void checkFieldsForMergeConflicts(Set idErrors) throws IOEx } private Set getIdErrors() { - Set idErrors; - Field field = fieldContext.getField(); + String fieldValue; // If analyzing the second feed (active feed), the service_id always gets feed scoped. // See https://github.com/ibi-group/datatools-server/issues/244 if (handlingActiveFeed && fieldNameEquals(SERVICE_ID)) { updateAndRemapOutput(); - idErrors = referenceTracker - .checkReferencesAndUniqueness(keyValue, lineNumber, field, fieldContext.getValueToWrite(), - table, keyField, orderField); + fieldValue = fieldContext.getValueToWrite(); } else { - idErrors = referenceTracker - .checkReferencesAndUniqueness(keyValue, lineNumber, field, fieldContext.getValue(), - table, keyField, orderField); + fieldValue = fieldContext.getValue(); } - return idErrors; + return referenceTracker.checkReferencesAndUniqueness(keyValue, lineNumber, fieldContext.getField(), + fieldValue, table, keyField, orderField); } protected void checkRoutesAndStopsIds(Set idErrors) throws IOException { @@ -345,7 +340,11 @@ protected void checkRoutesAndStopsIds(Set idErrors) throws IOExcep // where two routes have different short_names, but share the same route_id. We want // both of these routes to end up in the merged feed in this case because we're // matching on short name, so we must modify the route_id. - if (!skipRecord && !referenceTracker.transitIds.contains(String.join(":", keyField, keyValue)) && hasDuplicateError(primaryKeyErrors)) { + if ( + !skipRecord && + !referenceTracker.transitIds.contains(String.join(":", keyField, keyValue)) && + hasDuplicateError(primaryKeyErrors) + ) { // Modify route_id and ensure that referencing trips // have route_id updated. updateAndRemapOutput(); @@ -398,7 +397,7 @@ public boolean storeRowAndStopValues() { // Store row values for route or stop ID (or alternative ID field) in order // to check for ID conflicts. NOTE: This is only intended to be used for // routes and stops. Otherwise, this might (will) consume too much memory. - case STOPS: + case "stops": case "routes": // FIXME: This should be revised for tables with order fields, but it should work fine for its // primary purposes: to detect exact copy rows and to temporarily hold the data in case a reference diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/StopsMergeLineContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/StopsMergeLineContext.java index 4a3c54f42..21b6bc495 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/StopsMergeLineContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/StopsMergeLineContext.java @@ -25,7 +25,7 @@ public StopsMergeLineContext(MergeFeedsJob job, Table table, ZipOutputStream out @Override public void checkFirstLineConditions() throws IOException { - checkStopCodeStuff(); + checkThatStopCodesArePopulatedWhereRequired(); } @Override @@ -33,7 +33,11 @@ public void checkFieldsForMergeConflicts(Set idErrors) throws IOEx checkRoutesAndStopsIds(idErrors); } - private void checkStopCodeStuff() throws IOException { + /** + * Checks that the stop_code field of the Stop entities to merge is populated where required. + * @throws IOException + */ + private void checkThatStopCodesArePopulatedWhereRequired() throws IOException { if (shouldCheckStopCodes()) { // Before reading any lines in stops.txt, first determine whether all records contain // properly filled stop_codes. The rules governing this logic are as follows: diff --git a/src/main/java/com/conveyal/datatools/manager/utils/MergeFeedUtils.java b/src/main/java/com/conveyal/datatools/manager/utils/MergeFeedUtils.java index fa8fe6252..efc07fb01 100644 --- a/src/main/java/com/conveyal/datatools/manager/utils/MergeFeedUtils.java +++ b/src/main/java/com/conveyal/datatools/manager/utils/MergeFeedUtils.java @@ -50,11 +50,16 @@ public static Set getIdsForTable(ZipFile zipFile, Table table) throws IO LOG.warn("Table {} not found in zip file: {}", table.name, zipFile.getName()); return ids; } - Field[] fieldsFoundInZip = table.getFieldsFromFieldHeaders(csvReader.getHeaders(), null); - // Get the key field (id value) for each row. - int keyFieldIndex = getFieldIndex(fieldsFoundInZip, keyField); - while (csvReader.readRecord()) ids.add(csvReader.get(keyFieldIndex)); - csvReader.close(); + try { + Field[] fieldsFoundInZip = table.getFieldsFromFieldHeaders(csvReader.getHeaders(), null); + // Get the key field (id value) for each row. + int keyFieldIndex = getFieldIndex(fieldsFoundInZip, keyField); + while (csvReader.readRecord()) { + ids.add(csvReader.get(keyFieldIndex)); + } + } finally { + csvReader.close(); + } return ids; } @@ -117,10 +122,13 @@ public static Set getAllFields(List feedsToMerge, Table tabl if (csvReader == null) { continue; } - // Get fields found from headers and add them to the shared fields set. - Field[] fieldsFoundInZip = table.getFieldsFromFieldHeaders(csvReader.getHeaders(), null); - sharedFields.addAll(Arrays.asList(fieldsFoundInZip)); - csvReader.close(); + try { + // Get fields found from headers and add them to the shared fields set. + Field[] fieldsFoundInZip = table.getFieldsFromFieldHeaders(csvReader.getHeaders(), null); + sharedFields.addAll(Arrays.asList(fieldsFoundInZip)); + } finally { + csvReader.close(); + } } return sharedFields; } From 1806012c2ab1188993f494773430d890cccc9dd2 Mon Sep 17 00:00:00 2001 From: binh-dam-ibigroup <56846598+binh-dam-ibigroup@users.noreply.github.com> Date: Tue, 16 Nov 2021 12:48:42 -0500 Subject: [PATCH 069/122] refactor(MergeLineContext): Make skipRecord private. --- .../CalendarDatesMergeLineContext.java | 13 ++-- .../feedmerge/CalendarMergeLineContext.java | 13 ++-- .../jobs/feedmerge/MergeLineContext.java | 64 ++++++++++++------- .../feedmerge/RoutesMergeLineContext.java | 4 +- .../feedmerge/ShapesMergeLineContext.java | 14 ++-- .../jobs/feedmerge/StopsMergeLineContext.java | 4 +- .../jobs/feedmerge/TripsMergeLineContext.java | 11 ++-- 7 files changed, 77 insertions(+), 46 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarDatesMergeLineContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarDatesMergeLineContext.java index aafc2fde2..b4e67169c 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarDatesMergeLineContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarDatesMergeLineContext.java @@ -22,8 +22,8 @@ public CalendarDatesMergeLineContext(MergeFeedsJob job, Table table, ZipOutputSt } @Override - public void checkFieldsForMergeConflicts(Set idErrors) throws IOException { - checkCalendarDatesIds(); + public boolean checkFieldsForMergeConflicts(Set idErrors) throws IOException { + return checkCalendarDatesIds(); } @Override @@ -32,7 +32,8 @@ public void startNewRow() throws IOException { updateFutureFeedFirstDate(); } - private void checkCalendarDatesIds() throws IOException { + private boolean checkCalendarDatesIds() throws IOException { + boolean shouldSkipRecord = false; // Drop any calendar_dates.txt records from the existing feed for dates that are // not before the first date of the future feed. LocalDate date = getCsvDate("date"); @@ -44,12 +45,14 @@ private void checkCalendarDatesIds() throws IOException { futureFeedFirstDate); String key = getTableScopedValue(table, getIdScope(), keyValue); mergeFeedsResult.skippedIds.add(key); - skipRecord = true; + shouldSkipRecord = true; } // Track service ID because we want to avoid removing trips that may reference this // service_id when the service_id is used by calendar.txt records that operate in // the valid date range, i.e., before the future feed's first date. - if (!skipRecord && fieldNameEquals(SERVICE_ID)) mergeFeedsResult.serviceIds.add(getFieldContext().getValueToWrite()); + if (!shouldSkipRecord && fieldNameEquals(SERVICE_ID)) mergeFeedsResult.serviceIds.add(getFieldContext().getValueToWrite()); + + return !shouldSkipRecord; } private void updateFutureFeedFirstDate() { diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarMergeLineContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarMergeLineContext.java index 622a75756..62baca8c6 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarMergeLineContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarMergeLineContext.java @@ -26,11 +26,12 @@ public CalendarMergeLineContext(MergeFeedsJob job, Table table, ZipOutputStream } @Override - public void checkFieldsForMergeConflicts(Set idErrors) throws IOException { - checkCalendarIds(idErrors); + public boolean checkFieldsForMergeConflicts(Set idErrors) throws IOException { + return checkCalendarIds(idErrors); } - private void checkCalendarIds(Set idErrors) throws IOException { + private boolean checkCalendarIds(Set idErrors) throws IOException { + boolean shouldSkipRecord = false; // If any service_id in the active feed matches with the future // feed, it should be modified and all associated trip records // must also be changed with the modified service_id. @@ -75,7 +76,7 @@ private void checkCalendarIds(Set idErrors) throws IOException { keyValue); String key = getTableScopedValue(table, getIdScope(), keyValue); mergeFeedsResult.skippedIds.add(key); - skipRecord = true; + shouldSkipRecord = true; } else { // If a service_id from the active calendar has only the // end_date in the future, the end_date shall be set to one @@ -94,7 +95,9 @@ private void checkCalendarIds(Set idErrors) throws IOException { // Track service ID because we want to avoid removing trips that may reference this // service_id when the service_id is used by calendar_dates that operate in the valid // date range, i.e., before the future feed's first date. - if (!skipRecord && fieldNameEquals(SERVICE_ID)) mergeFeedsResult.serviceIds.add(getFieldContext().getValueToWrite()); + if (!shouldSkipRecord && fieldNameEquals(SERVICE_ID)) mergeFeedsResult.serviceIds.add(getFieldContext().getValueToWrite()); + + return !shouldSkipRecord; } private boolean shouldUpdateFutureFeedStartDate() { diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java index 60081fbd7..c6eab8eda 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java @@ -51,7 +51,7 @@ public class MergeLineContext { // CSV writer used to write to zip file. private final CsvListWriter writer; private CsvReader csvReader; - protected boolean skipRecord; + private boolean skipRecord; protected boolean keyFieldMissing; private String[] rowValues; private int lineNumber = 0; @@ -210,7 +210,7 @@ public void startNewRow() throws IOException { .collect(Collectors.toList()); } - public boolean areForeignRefsOk() throws IOException { + public boolean checkForeignReferences() throws IOException { Field field = fieldContext.getField(); if (field.isForeignReference()) { String key = getTableScopedValue(field.referenceTable, idScope, fieldContext.getValue()); @@ -235,7 +235,6 @@ public boolean areForeignRefsOk() throws IOException { getCsvValue(orderField)); } mergeFeedsResult.skippedIds.add(skippedKey); - skipRecord = true; return false; } } @@ -260,27 +259,26 @@ private boolean serviceIdHasOrShouldBeSkipped(String key, boolean isValidService /** * Overridable method whose default behavior below is to skip a record if it creates a duplicate id. + * @return false, if a failing condition was encountered. true, if everything was ok. * @throws IOException Some overrides throw IOException. */ - public void checkFieldsForMergeConflicts(Set idErrors) throws IOException { - if (hasDuplicateError(idErrors)) skipRecord = true; + public boolean checkFieldsForMergeConflicts(Set idErrors) throws IOException { + return !hasDuplicateError(idErrors); } private Set getIdErrors() { - String fieldValue; // If analyzing the second feed (active feed), the service_id always gets feed scoped. // See https://github.com/ibi-group/datatools-server/issues/244 - if (handlingActiveFeed && fieldNameEquals(SERVICE_ID)) { - updateAndRemapOutput(); - fieldValue = fieldContext.getValueToWrite(); - } else { - fieldValue = fieldContext.getValue(); - } + String fieldValue = handlingActiveFeed && fieldNameEquals(SERVICE_ID) + ? fieldContext.getValueToWrite() + : fieldContext.getValue(); + return referenceTracker.checkReferencesAndUniqueness(keyValue, lineNumber, fieldContext.getField(), fieldValue, table, keyField, orderField); } - protected void checkRoutesAndStopsIds(Set idErrors) throws IOException { + protected boolean checkRoutesAndStopsIds(Set idErrors) throws IOException { + boolean shouldSkipRecord = false; // First, check uniqueness of primary key value (i.e., stop or route ID) // in case the stop_code or route_short_name are being used. This // must occur unconditionally because each record must be tracked @@ -300,7 +298,7 @@ protected void checkRoutesAndStopsIds(Set idErrors) throws IOExcep // // Otherwise, allow the record to be written in output. if (hasDuplicateError(primaryKeyErrors)) { - skipRecord = true; + shouldSkipRecord = true; } } else if (hasDuplicateError(idErrors)) { // If we encounter a route/stop that shares its alt. @@ -333,7 +331,7 @@ protected void checkRoutesAndStopsIds(Set idErrors) throws IOExcep // have their references updated. mergeFeedsResult.remappedIds.put(key, keyForMatchingAltId); } - skipRecord = true; + shouldSkipRecord = true; } // Next check for regular ID conflicts (e.g., on route_id or stop_id) because any // conflicts here will actually break the feed. This essentially handles the case @@ -341,7 +339,7 @@ protected void checkRoutesAndStopsIds(Set idErrors) throws IOExcep // both of these routes to end up in the merged feed in this case because we're // matching on short name, so we must modify the route_id. if ( - !skipRecord && + !shouldSkipRecord && !referenceTracker.transitIds.contains(String.join(":", keyField, keyValue)) && hasDuplicateError(primaryKeyErrors) ) { @@ -353,7 +351,10 @@ protected void checkRoutesAndStopsIds(Set idErrors) throws IOExcep // Key field has defaulted to the standard primary key field // (stop_id or route_id), which makes the check much // simpler (just skip the duplicate record). - if (hasDuplicateError(idErrors)) skipRecord = true; + // FIXME: refactor. + if (hasDuplicateError(idErrors)) { + shouldSkipRecord = true; + } } String newAgencyId = feedMergeContext.getNewAgencyId(); @@ -363,6 +364,8 @@ protected void checkRoutesAndStopsIds(Set idErrors) throws IOExcep newAgencyId, keyValue); fieldContext.setValue(newAgencyId); } + + return !shouldSkipRecord; } private boolean hasBlankPrimaryKey() { @@ -503,13 +506,14 @@ public void writeHeaders() throws IOException { writeValuesToTable(headers, false); } + /** + * Constructs a new row value. + * @return false, if a failing condition was encountered. true, if everything was ok. + */ public boolean constructRowValues() throws IOException { // Piece together the row to write, which should look practically identical to the original // row except for the identifiers receiving a prefix to avoid ID conflicts. for (int specFieldIndex = 0; specFieldIndex < sharedSpecFields.size(); specFieldIndex++) { - // There is nothing to do in this loop if it has already been determined that the record should - // be skipped. - if (skipRecord) break; Field field = sharedSpecFields.get(specFieldIndex); // Default value to write is unchanged from value found in csv (i.e. val). Note: if looking to // modify the value that is written in the merged file, you must update valueToWrite (e.g., @@ -531,17 +535,29 @@ public boolean constructRowValues() throws IOException { // track references for a large number of feeds (e.g., every feed in New // York State). if (job.mergeType.equals(SERVICE_PERIOD)) { - Set idErrors = getIdErrors(); // FIXME: Rename. This method is imperative. + // Remap service id from active feed to distinguish them + // from entries with the same id in the future feed. + // See https://github.com/ibi-group/datatools-server/issues/244 + if (handlingActiveFeed && fieldNameEquals(SERVICE_ID)) { + updateAndRemapOutput(); + } + // Store values for key fields that have been encountered and update any key values that need modification due // to conflicts. - checkFieldsForMergeConflicts(idErrors); // FIXME: This method changes skipRecord; - if (skipRecord) continue; + // This method can change skipRecord. + if (!checkFieldsForMergeConflicts(getIdErrors())) { + skipRecord = true; + break; + } } // If the current field is a foreign reference, check if the reference has been removed in the // merged result. If this is the case (or other conditions are met), we will need to skip this // record. Likewise, if the reference has been modified, ensure that the value written to the // merged result is correctly updated. - if (!areForeignRefsOk()) continue; // FIXME: This method changes skipRecord; + if (!checkForeignReferences()) { + skipRecord = true; + break; + } rowValues[specFieldIndex] = fieldContext.getValueToWrite(); } return true; diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/RoutesMergeLineContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/RoutesMergeLineContext.java index ae976e342..ae9f9781b 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/RoutesMergeLineContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/RoutesMergeLineContext.java @@ -14,7 +14,7 @@ public RoutesMergeLineContext(MergeFeedsJob job, Table table, ZipOutputStream ou } @Override - public void checkFieldsForMergeConflicts(Set idErrors) throws IOException { - checkRoutesAndStopsIds(idErrors); + public boolean checkFieldsForMergeConflicts(Set idErrors) throws IOException { + return checkRoutesAndStopsIds(idErrors); } } \ No newline at end of file diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/ShapesMergeLineContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/ShapesMergeLineContext.java index 439b84ebd..4d21e657b 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/ShapesMergeLineContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/ShapesMergeLineContext.java @@ -20,11 +20,12 @@ public ShapesMergeLineContext(MergeFeedsJob job, Table table, ZipOutputStream ou } @Override - public void checkFieldsForMergeConflicts(Set idErrors) { - checkShapeIds(idErrors); + public boolean checkFieldsForMergeConflicts(Set idErrors) { + return checkShapeIds(idErrors); } - private void checkShapeIds(Set idErrors) { + private boolean checkShapeIds(Set idErrors) { + boolean shouldSkipRecord = false; // If a shape_id is found in both future and active datasets, all shape points from // the active dataset must be feed-scoped. Otherwise, the merged dataset may contain // shape_id:shape_pt_sequence values from both datasets (e.g., if future dataset contains @@ -54,6 +55,11 @@ private void checkShapeIds(Set idErrors) { } } // Skip record if normal duplicate errors are found. - if (hasDuplicateError(idErrors)) skipRecord = true; + // FIXME: refactor (used in super class) + if (hasDuplicateError(idErrors)) { + shouldSkipRecord = true; + } + + return !shouldSkipRecord; } } \ No newline at end of file diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/StopsMergeLineContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/StopsMergeLineContext.java index 21b6bc495..2ce5b8b46 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/StopsMergeLineContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/StopsMergeLineContext.java @@ -29,8 +29,8 @@ public void checkFirstLineConditions() throws IOException { } @Override - public void checkFieldsForMergeConflicts(Set idErrors) throws IOException { - checkRoutesAndStopsIds(idErrors); + public boolean checkFieldsForMergeConflicts(Set idErrors) throws IOException { + return checkRoutesAndStopsIds(idErrors); } /** diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/TripsMergeLineContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/TripsMergeLineContext.java index d3e17d809..d7bdcb8fa 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/TripsMergeLineContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/TripsMergeLineContext.java @@ -15,11 +15,12 @@ public TripsMergeLineContext(MergeFeedsJob job, Table table, ZipOutputStream out } @Override - public void checkFieldsForMergeConflicts(Set idErrors) { - checkTripIds(idErrors); + public boolean checkFieldsForMergeConflicts(Set idErrors) { + return checkTripIds(idErrors); } - private void checkTripIds(Set idErrors) { + private boolean checkTripIds(Set idErrors) { + boolean shouldSkipRecord = false; // trip_ids between active and future datasets must not match. The tripIdsToSkip and // tripIdsToModify sets below are determined based on the MergeStrategy used for MTC // service period merges. @@ -27,7 +28,7 @@ private void checkTripIds(Set idErrors) { // Handling active feed. Skip or modify trip id if found in one of the // respective sets. if (job.tripIdsToSkipForActiveFeed.contains(keyValue)) { - skipRecord = true; + shouldSkipRecord = true; } else if (job.tripIdsToModifyForActiveFeed.contains(keyValue)) { updateAndRemapOutput(true); } @@ -37,6 +38,8 @@ private void checkTripIds(Set idErrors) { updateAndRemapOutput(true); } } + + return !shouldSkipRecord; } } \ No newline at end of file From cb08230887d17fae2d0bcb6e8d5b1c47e5bc8522 Mon Sep 17 00:00:00 2001 From: binh-dam-ibigroup <56846598+binh-dam-ibigroup@users.noreply.github.com> Date: Tue, 16 Nov 2021 13:21:57 -0500 Subject: [PATCH 070/122] refactor: Continue fixing merge conflicts. --- .../feedmerge/CalendarMergeLineContext.java | 2 +- .../jobs/feedmerge/MergeLineContext.java | 14 ++++++------- .../feedmerge/RoutesMergeLineContext.java | 2 +- .../jobs/feedmerge/StopsMergeLineContext.java | 2 +- .../jobs/feedmerge/TripsMergeLineContext.java | 21 +++++++++---------- 5 files changed, 20 insertions(+), 21 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarMergeLineContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarMergeLineContext.java index dd9b2e5ea..69b4d5bf4 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarMergeLineContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarMergeLineContext.java @@ -36,7 +36,7 @@ public void afterRowWrite() throws IOException { addClonedServiceId(); } - private void checkCalendarIds(Set idErrors, FieldContext fieldContext) throws IOException { + private boolean checkCalendarIds(Set idErrors, FieldContext fieldContext) throws IOException { boolean shouldSkipRecord = false; if (isHandlingActiveFeed()) { LocalDate startDate = getCsvDate("start_date"); diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java index b483a1bc3..284e71c71 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java @@ -264,10 +264,10 @@ public boolean checkFieldsForMergeConflicts(Set idErrors, FieldCon return !hasDuplicateError(idErrors); } - private Set getIdErrors() { + private Set getIdErrors(FieldContext fieldContext) { // If analyzing the second feed (active feed), the service_id always gets feed scoped. // See https://github.com/ibi-group/datatools-server/issues/244 - String fieldValue = handlingActiveFeed && fieldNameEquals(SERVICE_ID) + String fieldValue = handlingActiveFeed && fieldContext.nameEquals(SERVICE_ID) ? fieldContext.getValueToWrite() : fieldContext.getValue(); @@ -275,7 +275,7 @@ private Set getIdErrors() { fieldValue, table, keyField, orderField); } - protected boolean checkRoutesAndStopsIds(Set idErrors) throws IOException { + protected boolean checkRoutesAndStopsIds(Set idErrors, FieldContext fieldContext) throws IOException { boolean shouldSkipRecord = false; // First, check uniqueness of primary key value (i.e., stop or route ID) // in case the stop_code or route_short_name are being used. This @@ -520,14 +520,14 @@ public boolean constructRowValues() throws IOException { // Remap service id from active feed to distinguish them // from entries with the same id in the future feed. // See https://github.com/ibi-group/datatools-server/issues/244 - if (handlingActiveFeed && fieldNameEquals(SERVICE_ID)) { - updateAndRemapOutput(); + if (handlingActiveFeed && fieldContext.nameEquals(SERVICE_ID)) { + updateAndRemapOutput(fieldContext); } // Store values for key fields that have been encountered and update any key values that need modification due // to conflicts. // This method can change skipRecord. - if (!checkFieldsForMergeConflicts(getIdErrors(), fieldContext)) { + if (!checkFieldsForMergeConflicts(getIdErrors(fieldContext), fieldContext)) { skipRecord = true; break; } @@ -536,7 +536,7 @@ public boolean constructRowValues() throws IOException { // merged result. If this is the case (or other conditions are met), we will need to skip this // record. Likewise, if the reference has been modified, ensure that the value written to the // merged result is correctly updated. - if (!checkForeignReferences()) { + if (!checkForeignReferences(fieldContext)) { skipRecord = true; break; } diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/RoutesMergeLineContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/RoutesMergeLineContext.java index 1287d51bb..276979c79 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/RoutesMergeLineContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/RoutesMergeLineContext.java @@ -15,6 +15,6 @@ public RoutesMergeLineContext(MergeFeedsJob job, Table table, ZipOutputStream ou @Override public boolean checkFieldsForMergeConflicts(Set idErrors, FieldContext fieldContext) throws IOException { - return checkRoutesAndStopsIds(idErrors); + return checkRoutesAndStopsIds(idErrors, fieldContext); } } \ No newline at end of file diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/StopsMergeLineContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/StopsMergeLineContext.java index 0f382b926..7a8983090 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/StopsMergeLineContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/StopsMergeLineContext.java @@ -30,7 +30,7 @@ public void checkFirstLineConditions() throws IOException { @Override public boolean checkFieldsForMergeConflicts(Set idErrors, FieldContext fieldContext) throws IOException { - return checkRoutesAndStopsIds(idErrors); + return checkRoutesAndStopsIds(idErrors, fieldContext); } /** diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/TripsMergeLineContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/TripsMergeLineContext.java index 8871ede52..697c343e8 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/TripsMergeLineContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/TripsMergeLineContext.java @@ -23,17 +23,16 @@ public boolean checkFieldsForMergeConflicts(Set idErrors, FieldCon private boolean checkTripIds(Set idErrors, FieldContext fieldContext) { boolean shouldSkipRecord = false; - // trip_ids between active and future datasets must not match. The tripIdsToSkip and - // tripIdsToModify sets below are determined based on the MergeStrategy used for MTC - // service period merges. - if (isHandlingActiveFeed()) { - // Handling active feed. Skip or modify trip id if found in one of the - // respective sets. - if (job.tripIdsToSkipForActiveFeed.contains(keyValue)) { - shouldSkipRecord = true; - } else if (job.tripIdsToModifyForActiveFeed.contains(keyValue)) { - updateAndRemapOutput(true); - } + // For the MTC revised feed merge process, + // the updated logic requires to insert all trips from both the active and future feed, + // except if they are present in both, in which case we only insert the trip entry from the future feed. + if ( + job.mergeType.equals(SERVICE_PERIOD) && + isHandlingActiveFeed() && + job.sharedTripIdsWithConsistentSignature.contains(keyValue) + ) { + // Skip this record, we will use the one from the future feed. + shouldSkipRecord = true; } // Remap duplicate trip ids. From 17a7d838d95213c77f019be57f50aa8ee6c0f4aa Mon Sep 17 00:00:00 2001 From: binh-dam-ibigroup <56846598+binh-dam-ibigroup@users.noreply.github.com> Date: Tue, 16 Nov 2021 16:31:28 -0500 Subject: [PATCH 071/122] test(MergeFeedsJob): Add test for autopopulating agency id if missing. --- .../manager/jobs/MergeFeedsJobTest.java | 50 +++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java b/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java index c64b3b1c1..3065c102d 100644 --- a/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java +++ b/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java @@ -66,6 +66,8 @@ public class MergeFeedsJobTest extends UnitTest { private static FeedSource napa; private static FeedSource caltrain; private static FeedSource bart; + private static FeedVersion noAgencyVersion1; + private static FeedVersion noAgencyVersion2; /** * Prepare and start a testing-specific web server @@ -132,6 +134,12 @@ public static void setUp() throws IOException { fakeTransitFutureUnique = createFeedVersion(fakeTransit, zipFolderFiles("merge-data-future-unique-ids")); fakeTransitModService = createFeedVersion(fakeTransit, zipFolderFiles("merge-data-mod-services")); fakeTransitModTrips = createFeedVersion(fakeTransit, zipFolderFiles("merge-data-mod-trips")); + + // Feeds with no agency id + FeedSource noAgencyIds = new FeedSource("no-agency-ids", project.id, MANUALLY_UPLOADED); + Persistence.feedSources.create(noAgencyIds); + noAgencyVersion1 = createFeedVersion(noAgencyIds, zipFolderFiles("no-agency-id-1")); + noAgencyVersion2 = createFeedVersion(noAgencyIds, zipFolderFiles("no-agency-id-2")); } /** @@ -892,6 +900,48 @@ public void canMergeBARTFeedsWithSpecialStops() throws SQLException, IOException ); } + /** + * Tests whether feeds without agency ids can be merged. + * The merged feed should have autogenerated agency ids. + */ + @Test + public void canMergeFeedsWithoutAgencyIds () throws SQLException { + Set versions = new HashSet<>(); + versions.add(noAgencyVersion1); + versions.add(noAgencyVersion2); + FeedVersion mergedVersion = regionallyMergeVersions(versions); + + String mergedNamespace = mergedVersion.namespace; + // - agency + // expect a total of 2 records + assertThatSqlCountQueryYieldsExpectedCount( + String.format("SELECT count(*) FROM %s.agency", mergedNamespace), + 2 + ); + // there shouldn't be records with blank agency_id + assertThatSqlCountQueryYieldsExpectedCount( + String.format("SELECT count(*) FROM %s.agency where agency_id = '' and agency_id is not null", mergedNamespace), + 0 + ); + // - routes + // expect a total of 2 records + assertThatSqlCountQueryYieldsExpectedCount( + String.format("SELECT count(*) FROM %s.routes", mergedNamespace), + 2 + ); + // there shouldn't be records with blank agency_id + assertThatSqlCountQueryYieldsExpectedCount( + String.format("SELECT count(*) FROM %s.routes where agency_id = '' and agency_id is not null", mergedNamespace), + 0 + ); + // - trips + // expect 4 records + assertThatSqlCountQueryYieldsExpectedCount( + String.format("SELECT count(*) FROM %s.trips", mergedNamespace), + 4 + ); + } + /** * Verifies that a completed merge feeds job did not fail. * @param mergeFeedsJob From 39db651a614de38be62f1b5c3fd2b23cb6fb70c8 Mon Sep 17 00:00:00 2001 From: binh-dam-ibigroup <56846598+binh-dam-ibigroup@users.noreply.github.com> Date: Tue, 16 Nov 2021 17:12:10 -0500 Subject: [PATCH 072/122] test(MergeLineContext): Populate autogenerated agency id if not provided. --- .../feedmerge/AgencyMergeLineContext.java | 6 +++++- .../jobs/feedmerge/FeedMergeContext.java | 21 +++++++++++++------ .../jobs/feedmerge/MergeLineContext.java | 20 ++++++++++++------ .../manager/jobs/MergeFeedsJobTest.java | 4 ++-- .../datatools/gtfs/no-agency-id-1/agency.txt | 2 ++ .../gtfs/no-agency-id-1/calendar.txt | 4 ++++ .../gtfs/no-agency-id-1/calendar_dates.txt | 1 + .../gtfs/no-agency-id-1/fare_attributes.txt | 1 + .../gtfs/no-agency-id-1/fare_rules.txt | 1 + .../gtfs/no-agency-id-1/feed_info.txt | 2 ++ .../gtfs/no-agency-id-1/frequencies.txt | 1 + .../datatools/gtfs/no-agency-id-1/routes.txt | 2 ++ .../datatools/gtfs/no-agency-id-1/shapes.txt | 3 +++ .../gtfs/no-agency-id-1/stop_times.txt | 3 +++ .../datatools/gtfs/no-agency-id-1/stops.txt | 3 +++ .../gtfs/no-agency-id-1/transfers.txt | 1 + .../datatools/gtfs/no-agency-id-1/trips.txt | 3 +++ .../datatools/gtfs/no-agency-id-2/agency.txt | 2 ++ .../gtfs/no-agency-id-2/calendar.txt | 4 ++++ .../gtfs/no-agency-id-2/calendar_dates.txt | 1 + .../gtfs/no-agency-id-2/fare_attributes.txt | 1 + .../gtfs/no-agency-id-2/fare_rules.txt | 1 + .../gtfs/no-agency-id-2/feed_info.txt | 2 ++ .../gtfs/no-agency-id-2/frequencies.txt | 1 + .../datatools/gtfs/no-agency-id-2/routes.txt | 2 ++ .../datatools/gtfs/no-agency-id-2/shapes.txt | 3 +++ .../gtfs/no-agency-id-2/stop_times.txt | 3 +++ .../datatools/gtfs/no-agency-id-2/stops.txt | 3 +++ .../gtfs/no-agency-id-2/transfers.txt | 1 + .../datatools/gtfs/no-agency-id-2/trips.txt | 3 +++ 30 files changed, 90 insertions(+), 15 deletions(-) create mode 100755 src/test/resources/com/conveyal/datatools/gtfs/no-agency-id-1/agency.txt create mode 100755 src/test/resources/com/conveyal/datatools/gtfs/no-agency-id-1/calendar.txt create mode 100755 src/test/resources/com/conveyal/datatools/gtfs/no-agency-id-1/calendar_dates.txt create mode 100755 src/test/resources/com/conveyal/datatools/gtfs/no-agency-id-1/fare_attributes.txt create mode 100755 src/test/resources/com/conveyal/datatools/gtfs/no-agency-id-1/fare_rules.txt create mode 100644 src/test/resources/com/conveyal/datatools/gtfs/no-agency-id-1/feed_info.txt create mode 100755 src/test/resources/com/conveyal/datatools/gtfs/no-agency-id-1/frequencies.txt create mode 100644 src/test/resources/com/conveyal/datatools/gtfs/no-agency-id-1/routes.txt create mode 100644 src/test/resources/com/conveyal/datatools/gtfs/no-agency-id-1/shapes.txt create mode 100644 src/test/resources/com/conveyal/datatools/gtfs/no-agency-id-1/stop_times.txt create mode 100644 src/test/resources/com/conveyal/datatools/gtfs/no-agency-id-1/stops.txt create mode 100755 src/test/resources/com/conveyal/datatools/gtfs/no-agency-id-1/transfers.txt create mode 100644 src/test/resources/com/conveyal/datatools/gtfs/no-agency-id-1/trips.txt create mode 100755 src/test/resources/com/conveyal/datatools/gtfs/no-agency-id-2/agency.txt create mode 100755 src/test/resources/com/conveyal/datatools/gtfs/no-agency-id-2/calendar.txt create mode 100755 src/test/resources/com/conveyal/datatools/gtfs/no-agency-id-2/calendar_dates.txt create mode 100755 src/test/resources/com/conveyal/datatools/gtfs/no-agency-id-2/fare_attributes.txt create mode 100755 src/test/resources/com/conveyal/datatools/gtfs/no-agency-id-2/fare_rules.txt create mode 100644 src/test/resources/com/conveyal/datatools/gtfs/no-agency-id-2/feed_info.txt create mode 100755 src/test/resources/com/conveyal/datatools/gtfs/no-agency-id-2/frequencies.txt create mode 100644 src/test/resources/com/conveyal/datatools/gtfs/no-agency-id-2/routes.txt create mode 100644 src/test/resources/com/conveyal/datatools/gtfs/no-agency-id-2/shapes.txt create mode 100644 src/test/resources/com/conveyal/datatools/gtfs/no-agency-id-2/stop_times.txt create mode 100644 src/test/resources/com/conveyal/datatools/gtfs/no-agency-id-2/stops.txt create mode 100755 src/test/resources/com/conveyal/datatools/gtfs/no-agency-id-2/transfers.txt create mode 100644 src/test/resources/com/conveyal/datatools/gtfs/no-agency-id-2/trips.txt diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/AgencyMergeLineContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/AgencyMergeLineContext.java index 6354b6209..33604c8c3 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/AgencyMergeLineContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/AgencyMergeLineContext.java @@ -33,7 +33,11 @@ private void checkForMissingAgencyId() { // agency_id is optional if only one agency is present, but that will // cause issues for the feed merge, so we need to insert an agency_id // for the single entry. - feedMergeContext.setNewAgencyId(UUID.randomUUID().toString()); + if (isHandlingActiveFeed()) { + feedMergeContext.setActiveFeedNewAgencyId(UUID.randomUUID().toString()); + } else { + feedMergeContext.setFutureFeedNewAgencyId(UUID.randomUUID().toString()); + } if (keyFieldMissing) { // Only add agency_id field if it is missing in table. addField(Table.AGENCY.fields[0]); diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/FeedMergeContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/FeedMergeContext.java index 6e4100fe1..b61276a24 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/FeedMergeContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/FeedMergeContext.java @@ -35,9 +35,10 @@ public class FeedMergeContext implements Closeable { */ public final Set sharedTripIds; /** - * Holds the auto-generated agency id to be updated if none was provided. + * Holds the auto-generated agency id to be updated for each feed if none was provided. */ - private String newAgencyId; + private String activeFeedNewAgencyId; + private String futureFeedNewAgencyId; public FeedMergeContext(Set feedVersions, Auth0UserProfile owner) throws IOException { feedsToMerge = MergeFeedUtils.collectAndSortFeeds(feedVersions, owner); @@ -131,12 +132,20 @@ public void setFutureFeedFirstDate(LocalDate futureFeedFirstDate) { this.futureFeedFirstDate = futureFeedFirstDate; } - public String getNewAgencyId() { - return newAgencyId; + public String getActiveFeedNewAgencyId() { + return activeFeedNewAgencyId; } - public void setNewAgencyId(String newAgencyId) { - this.newAgencyId = newAgencyId; + public String getFutureFeedNewAgencyId() { + return futureFeedNewAgencyId; + } + + public void setActiveFeedNewAgencyId(String newAgencyId) { + this.activeFeedNewAgencyId = newAgencyId; + } + + public void setFutureFeedNewAgencyId(String newAgencyId) { + this.futureFeedNewAgencyId = newAgencyId; } /** diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java index c6eab8eda..90be0d009 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java @@ -122,10 +122,7 @@ public void startNewFeed(int feedIndex) throws IOException { keyField = getMergeKeyField(table, job.mergeType); orderField = table.getOrderFieldName(); keyFieldMissing = false; - // Use for a new agency ID for use if the feed does not contain one. Initialize to - // null. If the value becomes non-null, the agency_id is missing and needs to be - // replaced in other affected tables with the generated value stored in this variable. - feedMergeContext.setNewAgencyId(null); + // Generate ID prefix to scope GTFS identifiers to avoid conflicts. idScope = getCleanName(feedSource.name) + version.version; csvReader = table.getCsvReader(feed.zipFile, null); @@ -357,7 +354,8 @@ protected boolean checkRoutesAndStopsIds(Set idErrors) throws IOEx } } - String newAgencyId = feedMergeContext.getNewAgencyId(); + String newAgencyId = getNewAgencyIdForFeed(); + if (newAgencyId != null && fieldNameEquals(AGENCY_ID)) { LOG.info( "Updating route#agency_id to (auto-generated) {} for route={}", @@ -368,6 +366,16 @@ protected boolean checkRoutesAndStopsIds(Set idErrors) throws IOEx return !shouldSkipRecord; } + private String getNewAgencyIdForFeed() { + String newAgencyId; + if (handlingActiveFeed) { + newAgencyId = feedMergeContext.getActiveFeedNewAgencyId(); + } else { + newAgencyId = feedMergeContext.getFutureFeedNewAgencyId(); + } + return newAgencyId; + } + private boolean hasBlankPrimaryKey() { return "".equals(keyValue) && fieldNameEquals(table.getKeyFieldName()); } @@ -377,7 +385,7 @@ private boolean useAltKey() { } public boolean updateAgencyIdIfNeeded() { - String newAgencyId = feedMergeContext.getNewAgencyId(); + String newAgencyId = getNewAgencyIdForFeed(); if (newAgencyId != null && fieldNameEquals(AGENCY_ID) && job.mergeType.equals(REGIONAL)) { if (fieldContext.getValue().equals("") && table.name.equals("agency") && lineNumber > 0) { // If there is no agency_id value for a second (or greater) agency diff --git a/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java b/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java index 3065c102d..39561943a 100644 --- a/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java +++ b/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java @@ -920,7 +920,7 @@ public void canMergeFeedsWithoutAgencyIds () throws SQLException { ); // there shouldn't be records with blank agency_id assertThatSqlCountQueryYieldsExpectedCount( - String.format("SELECT count(*) FROM %s.agency where agency_id = '' and agency_id is not null", mergedNamespace), + String.format("SELECT count(*) FROM %s.agency where agency_id = '' or agency_id is null", mergedNamespace), 0 ); // - routes @@ -931,7 +931,7 @@ public void canMergeFeedsWithoutAgencyIds () throws SQLException { ); // there shouldn't be records with blank agency_id assertThatSqlCountQueryYieldsExpectedCount( - String.format("SELECT count(*) FROM %s.routes where agency_id = '' and agency_id is not null", mergedNamespace), + String.format("SELECT count(*) FROM %s.routes where agency_id = '' or agency_id is null", mergedNamespace), 0 ); // - trips diff --git a/src/test/resources/com/conveyal/datatools/gtfs/no-agency-id-1/agency.txt b/src/test/resources/com/conveyal/datatools/gtfs/no-agency-id-1/agency.txt new file mode 100755 index 000000000..bef4d6072 --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/no-agency-id-1/agency.txt @@ -0,0 +1,2 @@ +agency_id,agency_name,agency_url,agency_lang,agency_phone,agency_email,agency_timezone,agency_fare_url,agency_branding_url +,Agency1,http://agency1.example.com/,en,888-555-1111,agency1@example.com,America/New_York,, diff --git a/src/test/resources/com/conveyal/datatools/gtfs/no-agency-id-1/calendar.txt b/src/test/resources/com/conveyal/datatools/gtfs/no-agency-id-1/calendar.txt new file mode 100755 index 000000000..7d765be25 --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/no-agency-id-1/calendar.txt @@ -0,0 +1,4 @@ +service_id,monday,tuesday,wednesday,thursday,friday,saturday,sunday,start_date,end_date +bb285e71-c906-4b67-ab68-0212fc864728,0,0,0,0,0,0,1,20180823,20200823 +bd6e404d-6e02-45c1-826c-a569ca947fce,1,1,1,1,1,0,0,20180823,20200823 +ef5a027b-353d-4071-b10b-f232e1c6b8cf,0,0,0,0,0,1,0,20180823,20200823 diff --git a/src/test/resources/com/conveyal/datatools/gtfs/no-agency-id-1/calendar_dates.txt b/src/test/resources/com/conveyal/datatools/gtfs/no-agency-id-1/calendar_dates.txt new file mode 100755 index 000000000..74c1ef632 --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/no-agency-id-1/calendar_dates.txt @@ -0,0 +1 @@ +service_id,date,exception_type diff --git a/src/test/resources/com/conveyal/datatools/gtfs/no-agency-id-1/fare_attributes.txt b/src/test/resources/com/conveyal/datatools/gtfs/no-agency-id-1/fare_attributes.txt new file mode 100755 index 000000000..8a1793839 --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/no-agency-id-1/fare_attributes.txt @@ -0,0 +1 @@ +fare_id,price,currency_type,payment_method,transfers,transfer_duration diff --git a/src/test/resources/com/conveyal/datatools/gtfs/no-agency-id-1/fare_rules.txt b/src/test/resources/com/conveyal/datatools/gtfs/no-agency-id-1/fare_rules.txt new file mode 100755 index 000000000..c7f6b54a3 --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/no-agency-id-1/fare_rules.txt @@ -0,0 +1 @@ +fare_id,route_id,origin_id,destination_id,contains_id diff --git a/src/test/resources/com/conveyal/datatools/gtfs/no-agency-id-1/feed_info.txt b/src/test/resources/com/conveyal/datatools/gtfs/no-agency-id-1/feed_info.txt new file mode 100644 index 000000000..6ab097246 --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/no-agency-id-1/feed_info.txt @@ -0,0 +1,2 @@ +feed_id,feed_publisher_name,feed_publisher_url,feed_lang,feed_version +feed1,IBI,http://www.ibigroup.com,en,1.0 \ No newline at end of file diff --git a/src/test/resources/com/conveyal/datatools/gtfs/no-agency-id-1/frequencies.txt b/src/test/resources/com/conveyal/datatools/gtfs/no-agency-id-1/frequencies.txt new file mode 100755 index 000000000..9b46cdeef --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/no-agency-id-1/frequencies.txt @@ -0,0 +1 @@ +trip_id,start_time,end_time,headway_secs,exact_times diff --git a/src/test/resources/com/conveyal/datatools/gtfs/no-agency-id-1/routes.txt b/src/test/resources/com/conveyal/datatools/gtfs/no-agency-id-1/routes.txt new file mode 100644 index 000000000..d5fe2002e --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/no-agency-id-1/routes.txt @@ -0,0 +1,2 @@ +agency_id,route_id,route_short_name,route_long_name,route_desc,route_type,route_url,route_color,route_text_color,route_branding_url +,1,1,Agency1Route1,,3,,6A478F,FFFFFF, \ No newline at end of file diff --git a/src/test/resources/com/conveyal/datatools/gtfs/no-agency-id-1/shapes.txt b/src/test/resources/com/conveyal/datatools/gtfs/no-agency-id-1/shapes.txt new file mode 100644 index 000000000..df1a9f70e --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/no-agency-id-1/shapes.txt @@ -0,0 +1,3 @@ +shape_id,shape_pt_lat,shape_pt_lon,shape_pt_sequence,shape_dist_traveled +03c47dc1-bf37-4668-8c58-9847a496f92d,38.3223235,-122.3105574,1,0.0000000 +03c47dc1-bf37-4668-8c58-9847a496f92d,38.3221392,-122.3104330,2,23.1739361 \ No newline at end of file diff --git a/src/test/resources/com/conveyal/datatools/gtfs/no-agency-id-1/stop_times.txt b/src/test/resources/com/conveyal/datatools/gtfs/no-agency-id-1/stop_times.txt new file mode 100644 index 000000000..fcd8ff8ee --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/no-agency-id-1/stop_times.txt @@ -0,0 +1,3 @@ +trip_id,arrival_time,departure_time,stop_id,stop_sequence,stop_headsign,pickup_type,drop_off_type,shape_dist_traveled,timepoint +t104-sl2-p18-r45,14:00:00,14:00:00,100,1,,0,0,13.3808067,1 +t104-sl2-p18-r45,14:03:14,14:03:14,102,2,,0,0,1122.9721799,0 \ No newline at end of file diff --git a/src/test/resources/com/conveyal/datatools/gtfs/no-agency-id-1/stops.txt b/src/test/resources/com/conveyal/datatools/gtfs/no-agency-id-1/stops.txt new file mode 100644 index 000000000..104792368 --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/no-agency-id-1/stops.txt @@ -0,0 +1,3 @@ +stop_id,stop_code,stop_name,stop_desc,stop_lat,stop_lon,zone_id,stop_url,location_type,parent_station,stop_timezone,wheelchair_boarding +100,89062,Terrace Dr at Liberty Dr,,38.2933330,-122.2694440,,http://ridethevine.rideralerts.com/InfoPoint/89062,0,,, +102,89140,Trancas St at Jefferson St,,38.3225000,-122.3011110,,http://ridethevine.rideralerts.com/InfoPoint/89140,0,,, \ No newline at end of file diff --git a/src/test/resources/com/conveyal/datatools/gtfs/no-agency-id-1/transfers.txt b/src/test/resources/com/conveyal/datatools/gtfs/no-agency-id-1/transfers.txt new file mode 100755 index 000000000..357103c47 --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/no-agency-id-1/transfers.txt @@ -0,0 +1 @@ +from_stop_id,to_stop_id,transfer_type,min_transfer_time diff --git a/src/test/resources/com/conveyal/datatools/gtfs/no-agency-id-1/trips.txt b/src/test/resources/com/conveyal/datatools/gtfs/no-agency-id-1/trips.txt new file mode 100644 index 000000000..2502fbb6a --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/no-agency-id-1/trips.txt @@ -0,0 +1,3 @@ +route_id,trip_id,trip_headsign,trip_short_name,direction_id,block_id,shape_id,bikes_allowed,wheelchair_accessible,service_id +1,t104-sl2-p18-r45,08_1,,0,802,03c47dc1-bf37-4668-8c58-9847a496f92d,0,0,bd6e404d-6e02-45c1-826c-a569ca947fce +1,t104-sl3-p17-r1B,08_1,,0,2008,03c47dc1-bf37-4668-8c58-9847a496f92d,0,0,ef5a027b-353d-4071-b10b-f232e1c6b8cf \ No newline at end of file diff --git a/src/test/resources/com/conveyal/datatools/gtfs/no-agency-id-2/agency.txt b/src/test/resources/com/conveyal/datatools/gtfs/no-agency-id-2/agency.txt new file mode 100755 index 000000000..0cb24cbd5 --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/no-agency-id-2/agency.txt @@ -0,0 +1,2 @@ +agency_id,agency_name,agency_url,agency_lang,agency_phone,agency_email,agency_timezone,agency_fare_url,agency_branding_url +,Agency2,http://agency2.example.com/,en,888-555-2222,agency2@example.com,America/New_York,, diff --git a/src/test/resources/com/conveyal/datatools/gtfs/no-agency-id-2/calendar.txt b/src/test/resources/com/conveyal/datatools/gtfs/no-agency-id-2/calendar.txt new file mode 100755 index 000000000..d6546a48e --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/no-agency-id-2/calendar.txt @@ -0,0 +1,4 @@ +service_id,monday,tuesday,wednesday,thursday,friday,saturday,sunday,start_date,end_date +2-bb285e71-c906-4b67-ab68-0212fc864728,0,0,0,0,0,0,1,20180823,20190823 +2-bd6e404d-6e02-45c1-826c-a569ca947fce,1,1,1,1,1,0,0,20180823,20190823 +2-ef5a027b-353d-4071-b10b-f232e1c6b8cf,0,0,0,0,0,1,0,20180823,20190823 diff --git a/src/test/resources/com/conveyal/datatools/gtfs/no-agency-id-2/calendar_dates.txt b/src/test/resources/com/conveyal/datatools/gtfs/no-agency-id-2/calendar_dates.txt new file mode 100755 index 000000000..74c1ef632 --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/no-agency-id-2/calendar_dates.txt @@ -0,0 +1 @@ +service_id,date,exception_type diff --git a/src/test/resources/com/conveyal/datatools/gtfs/no-agency-id-2/fare_attributes.txt b/src/test/resources/com/conveyal/datatools/gtfs/no-agency-id-2/fare_attributes.txt new file mode 100755 index 000000000..8a1793839 --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/no-agency-id-2/fare_attributes.txt @@ -0,0 +1 @@ +fare_id,price,currency_type,payment_method,transfers,transfer_duration diff --git a/src/test/resources/com/conveyal/datatools/gtfs/no-agency-id-2/fare_rules.txt b/src/test/resources/com/conveyal/datatools/gtfs/no-agency-id-2/fare_rules.txt new file mode 100755 index 000000000..c7f6b54a3 --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/no-agency-id-2/fare_rules.txt @@ -0,0 +1 @@ +fare_id,route_id,origin_id,destination_id,contains_id diff --git a/src/test/resources/com/conveyal/datatools/gtfs/no-agency-id-2/feed_info.txt b/src/test/resources/com/conveyal/datatools/gtfs/no-agency-id-2/feed_info.txt new file mode 100644 index 000000000..6ab097246 --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/no-agency-id-2/feed_info.txt @@ -0,0 +1,2 @@ +feed_id,feed_publisher_name,feed_publisher_url,feed_lang,feed_version +feed1,IBI,http://www.ibigroup.com,en,1.0 \ No newline at end of file diff --git a/src/test/resources/com/conveyal/datatools/gtfs/no-agency-id-2/frequencies.txt b/src/test/resources/com/conveyal/datatools/gtfs/no-agency-id-2/frequencies.txt new file mode 100755 index 000000000..9b46cdeef --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/no-agency-id-2/frequencies.txt @@ -0,0 +1 @@ +trip_id,start_time,end_time,headway_secs,exact_times diff --git a/src/test/resources/com/conveyal/datatools/gtfs/no-agency-id-2/routes.txt b/src/test/resources/com/conveyal/datatools/gtfs/no-agency-id-2/routes.txt new file mode 100644 index 000000000..fdd60f057 --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/no-agency-id-2/routes.txt @@ -0,0 +1,2 @@ +agency_id,route_id,route_short_name,route_long_name,route_desc,route_type,route_url,route_color,route_text_color,route_branding_url +,2,2,Agency2Route2,,3,,6A478F,FFFFFF, \ No newline at end of file diff --git a/src/test/resources/com/conveyal/datatools/gtfs/no-agency-id-2/shapes.txt b/src/test/resources/com/conveyal/datatools/gtfs/no-agency-id-2/shapes.txt new file mode 100644 index 000000000..78f8e4c59 --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/no-agency-id-2/shapes.txt @@ -0,0 +1,3 @@ +shape_id,shape_pt_lat,shape_pt_lon,shape_pt_sequence,shape_dist_traveled +2-03c47dc1-bf37-4668-8c58-9847a496f92d,38.3223235,-122.3105574,1,0.0000000 +2-03c47dc1-bf37-4668-8c58-9847a496f92d,38.3221392,-122.3104330,2,23.1739361 \ No newline at end of file diff --git a/src/test/resources/com/conveyal/datatools/gtfs/no-agency-id-2/stop_times.txt b/src/test/resources/com/conveyal/datatools/gtfs/no-agency-id-2/stop_times.txt new file mode 100644 index 000000000..1aa8dd0bf --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/no-agency-id-2/stop_times.txt @@ -0,0 +1,3 @@ +trip_id,arrival_time,departure_time,stop_id,stop_sequence,stop_headsign,pickup_type,drop_off_type,shape_dist_traveled,timepoint +2-t104-sl2-p18-r45,14:00:00,14:00:00,100,1,,0,0,13.3808067,1 +2-t104-sl2-p18-r45,14:03:14,14:03:14,102,2,,0,0,1122.9721799,0 \ No newline at end of file diff --git a/src/test/resources/com/conveyal/datatools/gtfs/no-agency-id-2/stops.txt b/src/test/resources/com/conveyal/datatools/gtfs/no-agency-id-2/stops.txt new file mode 100644 index 000000000..104792368 --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/no-agency-id-2/stops.txt @@ -0,0 +1,3 @@ +stop_id,stop_code,stop_name,stop_desc,stop_lat,stop_lon,zone_id,stop_url,location_type,parent_station,stop_timezone,wheelchair_boarding +100,89062,Terrace Dr at Liberty Dr,,38.2933330,-122.2694440,,http://ridethevine.rideralerts.com/InfoPoint/89062,0,,, +102,89140,Trancas St at Jefferson St,,38.3225000,-122.3011110,,http://ridethevine.rideralerts.com/InfoPoint/89140,0,,, \ No newline at end of file diff --git a/src/test/resources/com/conveyal/datatools/gtfs/no-agency-id-2/transfers.txt b/src/test/resources/com/conveyal/datatools/gtfs/no-agency-id-2/transfers.txt new file mode 100755 index 000000000..357103c47 --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/no-agency-id-2/transfers.txt @@ -0,0 +1 @@ +from_stop_id,to_stop_id,transfer_type,min_transfer_time diff --git a/src/test/resources/com/conveyal/datatools/gtfs/no-agency-id-2/trips.txt b/src/test/resources/com/conveyal/datatools/gtfs/no-agency-id-2/trips.txt new file mode 100644 index 000000000..7245d28c6 --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/no-agency-id-2/trips.txt @@ -0,0 +1,3 @@ +route_id,trip_id,trip_headsign,trip_short_name,direction_id,block_id,shape_id,bikes_allowed,wheelchair_accessible,service_id +2,2-t104-sl2-p18-r45,08_1,,0,20802,2-03c47dc1-bf37-4668-8c58-9847a496f92d,0,0,2-bd6e404d-6e02-45c1-826c-a569ca947fce +2,2-t104-sl3-p17-r1B,08_1,,0,22008,2-03c47dc1-bf37-4668-8c58-9847a496f92d,0,0,2-ef5a027b-353d-4071-b10b-f232e1c6b8cf \ No newline at end of file From dc643c2746916bbadf805e3f34c3ff45958353c6 Mon Sep 17 00:00:00 2001 From: binh-dam-ibigroup <56846598+binh-dam-ibigroup@users.noreply.github.com> Date: Tue, 16 Nov 2021 17:21:51 -0500 Subject: [PATCH 073/122] refactor(MergeFeedsJob): Refactor test class. --- .../manager/jobs/MergeFeedsJobTest.java | 43 ++++++++----------- 1 file changed, 18 insertions(+), 25 deletions(-) diff --git a/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java b/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java index 39561943a..069933f0c 100644 --- a/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java +++ b/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java @@ -38,15 +38,13 @@ */ public class MergeFeedsJobTest extends UnitTest { private static final Logger LOG = LoggerFactory.getLogger(MergeFeedsJobTest.class); - private static Auth0UserProfile user = Auth0UserProfile.createTestAdminUser(); + private static final Auth0UserProfile user = Auth0UserProfile.createTestAdminUser(); private static FeedVersion bartVersion1; private static FeedVersion bartVersion2SameTrips; - private static FeedVersion calTrainVersion; private static FeedVersion bartVersionOldLite; private static FeedVersion bartVersionNewLite; private static FeedVersion calTrainVersionLite; private static Project project; - private static FeedVersion napaVersion; private static FeedVersion napaVersionLite; private static FeedVersion bothCalendarFilesVersion; private static FeedVersion bothCalendarFilesVersion2; @@ -63,8 +61,6 @@ public class MergeFeedsJobTest extends UnitTest { private static FeedVersion fakeTransitModService; /** The base feed (transposed to the future dates) but with differing trip_ids. */ private static FeedVersion fakeTransitModTrips; - private static FeedSource napa; - private static FeedSource caltrain; private static FeedSource bart; private static FeedVersion noAgencyVersion1; private static FeedVersion noAgencyVersion2; @@ -79,7 +75,7 @@ public static void setUp() throws IOException { // Create a project, feed sources, and feed versions to merge. project = new Project(); - project.name = String.format("Test %s", new Date().toString()); + project.name = String.format("Test %s", new Date()); Persistence.projects.create(project); // Bart @@ -91,15 +87,13 @@ public static void setUp() throws IOException { bartVersionNewLite = createFeedVersionFromGtfsZip(bart, "bart_new_lite.zip"); // Caltrain - caltrain = new FeedSource("Caltrain", project.id, MANUALLY_UPLOADED); + FeedSource caltrain = new FeedSource("Caltrain", project.id, MANUALLY_UPLOADED); Persistence.feedSources.create(caltrain); - calTrainVersion = createFeedVersionFromGtfsZip(caltrain, "caltrain_gtfs.zip"); calTrainVersionLite = createFeedVersionFromGtfsZip(caltrain, "caltrain_gtfs_lite.zip"); // Napa - napa = new FeedSource("Napa", project.id, MANUALLY_UPLOADED); + FeedSource napa = new FeedSource("Napa", project.id, MANUALLY_UPLOADED); Persistence.feedSources.create(napa); - napaVersion = createFeedVersionFromGtfsZip(napa, "napa-no-agency-id.zip"); napaVersionLite = createFeedVersionFromGtfsZip(napa, "napa-no-agency-id-lite.zip"); // Fake agencies (for testing calendar service_id merges with MTC strategy). @@ -156,7 +150,7 @@ public static void tearDown() { * Ensures that a regional feed merge will produce a feed that includes all entities from each feed. */ @Test - public void canMergeRegional() throws SQLException { + void canMergeRegional() throws SQLException { // Set up list of feed versions to merge. Set versions = new HashSet<>(); versions.add(bartVersionOldLite); @@ -215,7 +209,7 @@ public void canMergeRegional() throws SQLException { * calendar_dates and another with only the calendar. */ @Test - public void canMergeRegionalWithOnlyCalendarFeed () throws SQLException { + void canMergeRegionalWithOnlyCalendarFeed () throws SQLException { Set versions = new HashSet<>(); versions.add(onlyCalendarDatesVersion); versions.add(onlyCalendarVersion); @@ -304,7 +298,7 @@ public void canMergeRegionalWithOnlyCalendarFeed () throws SQLException { * Ensures that an MTC merge of feeds that has exactly matching trips but mismatched services fails. */ @Test - public void mergeMTCShouldFailOnDuplicateTripsButMismatchedServices() { + void mergeMTCShouldFailOnDuplicateTripsButMismatchedServices() { Set versions = new HashSet<>(); versions.add(fakeTransitBase); versions.add(fakeTransitModService); @@ -323,7 +317,7 @@ public void mergeMTCShouldFailOnDuplicateTripsButMismatchedServices() { * {@link MergeStrategy#EXTEND_FUTURE} strategy correctly. */ @Test - public void mergeMTCShouldHandleExtendFutureStrategy() throws SQLException { + void mergeMTCShouldHandleExtendFutureStrategy() throws SQLException { Set versions = new HashSet<>(); versions.add(fakeTransitBase); versions.add(fakeTransitFuture); @@ -360,7 +354,7 @@ public void mergeMTCShouldHandleExtendFutureStrategy() throws SQLException { * {@link MergeStrategy#CHECK_STOP_TIMES} strategy correctly. */ @Test - public void mergeMTCShouldHandleCheckStopTimesStrategy() throws SQLException { + void mergeMTCShouldHandleCheckStopTimesStrategy() throws SQLException { Set versions = new HashSet<>(); versions.add(fakeTransitBase); versions.add(fakeTransitModTrips); @@ -409,7 +403,7 @@ public void mergeMTCShouldHandleCheckStopTimesStrategy() throws SQLException { * {@link MergeStrategy#DEFAULT} strategy correctly. */ @Test - public void mergeMTCShouldHandleDefaultStrategy() throws SQLException { + void mergeMTCShouldHandleDefaultStrategy() throws SQLException { Set versions = new HashSet<>(); versions.add(fakeTransitBase); versions.add(fakeTransitFutureUnique); @@ -447,7 +441,7 @@ public void mergeMTCShouldHandleDefaultStrategy() throws SQLException { * Tests that the MTC merge strategy will successfully merge BART feeds. */ @Test - public void canMergeBARTFeeds() throws SQLException { + void canMergeBARTFeeds() throws SQLException { Set versions = new HashSet<>(); versions.add(bartVersionOldLite); versions.add(bartVersionNewLite); @@ -504,7 +498,7 @@ public void canMergeBARTFeeds() throws SQLException { * Tests that the MTC merge strategy will successfully merge BART feeds. */ @Test - public void canMergeBARTFeedsSameTrips() throws SQLException { + void canMergeBARTFeedsSameTrips() throws SQLException { Set versions = new HashSet<>(); versions.add(bartVersion1); versions.add(bartVersion2SameTrips); @@ -563,7 +557,7 @@ public void canMergeBARTFeedsSameTrips() throws SQLException { * other has only the calendar file. */ @Test - public void canMergeFeedsWithMTCForServiceIds1 () throws SQLException { + void canMergeFeedsWithMTCForServiceIds1 () throws SQLException { Set versions = new HashSet<>(); versions.add(bothCalendarFilesVersion); versions.add(onlyCalendarVersion); @@ -666,7 +660,7 @@ public void canMergeFeedsWithMTCForServiceIds1 () throws SQLException { * and the other has only the calendar file. */ @Test - public void canMergeFeedsWithMTCForServiceIds2 () throws SQLException { + void canMergeFeedsWithMTCForServiceIds2 () throws SQLException { Set versions = new HashSet<>(); versions.add(onlyCalendarDatesVersion); versions.add(onlyCalendarVersion); @@ -755,7 +749,7 @@ public void canMergeFeedsWithMTCForServiceIds2 () throws SQLException { * that service_id. */ @Test - public void canMergeFeedsWithMTCForServiceIds3 () throws SQLException { + void canMergeFeedsWithMTCForServiceIds3 () throws SQLException { Set versions = new HashSet<>(); versions.add(bothCalendarFilesVersion); versions.add(bothCalendarFilesVersion2); @@ -827,7 +821,7 @@ public void canMergeFeedsWithMTCForServiceIds3 () throws SQLException { * that service_id. */ @Test - public void canMergeFeedsWithMTCForServiceIds4 () throws SQLException { + void canMergeFeedsWithMTCForServiceIds4 () throws SQLException { Set versions = new HashSet<>(); versions.add(bothCalendarFilesVersion); versions.add(bothCalendarFilesVersion3); @@ -880,7 +874,7 @@ public void canMergeFeedsWithMTCForServiceIds4 () throws SQLException { * entrances, generic nodes, etc.) handles missing stop codes correctly. */ @Test - public void canMergeBARTFeedsWithSpecialStops() throws SQLException, IOException { + void canMergeBARTFeedsWithSpecialStops() throws SQLException, IOException { // Mini BART old/new feeds are pared down versions of the zips (bart_new.zip and bart_old.zip). They each have // only one trip and its corresponding stop_times. They do contain a full set of routes and stops. The stops are // from a recent (as of August 2021) GTFS file that includes a bunch of new stop records that act as entrances). @@ -905,7 +899,7 @@ public void canMergeBARTFeedsWithSpecialStops() throws SQLException, IOException * The merged feed should have autogenerated agency ids. */ @Test - public void canMergeFeedsWithoutAgencyIds () throws SQLException { + void canMergeFeedsWithoutAgencyIds () throws SQLException { Set versions = new HashSet<>(); versions.add(noAgencyVersion1); versions.add(noAgencyVersion2); @@ -944,7 +938,6 @@ public void canMergeFeedsWithoutAgencyIds () throws SQLException { /** * Verifies that a completed merge feeds job did not fail. - * @param mergeFeedsJob */ private void assertFeedMergeSucceeded(MergeFeedsJob mergeFeedsJob) { if (mergeFeedsJob.mergedVersion.namespace == null || mergeFeedsJob.mergeFeedsResult.failed) { From 81736e8c7292c3d0a7406b17a4b448931404dec0 Mon Sep 17 00:00:00 2001 From: binh-dam-ibigroup <56846598+binh-dam-ibigroup@users.noreply.github.com> Date: Tue, 16 Nov 2021 17:52:47 -0500 Subject: [PATCH 074/122] refactor(FeedMergeContext): Extract class FeedContext. --- .../datatools/manager/jobs/MergeFeedsJob.java | 8 +- .../feedmerge/AgencyMergeLineContext.java | 10 +-- .../CalendarDatesMergeLineContext.java | 6 +- .../feedmerge/CalendarMergeLineContext.java | 4 +- .../manager/jobs/feedmerge/FeedContext.java | 50 +++++++++++++ .../jobs/feedmerge/FeedMergeContext.java | 74 +++---------------- .../jobs/feedmerge/MergeLineContext.java | 12 +-- .../feedmerge/ShapesMergeLineContext.java | 1 - 8 files changed, 78 insertions(+), 87 deletions(-) create mode 100644 src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/FeedContext.java diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java b/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java index 3c0e693a5..70a7b2f04 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java @@ -402,15 +402,15 @@ private MergeStrategy getMergeStrategy() { if (feedMergeContext.serviceIdsMatch) { // If just the service_ids are an exact match, check the that the stop_times having matching signatures // between the two feeds (i.e., each stop time in the ordered list is identical between the two feeds). - Feed futureFeed = feedMergeContext.futureFeed; - Feed activeFeed = feedMergeContext.activeFeed; + Feed futureFeed = feedMergeContext.future.feed; + Feed activeFeed = feedMergeContext.active.feed; for (String tripId : feedMergeContext.sharedTripIds) { compareStopTimesAndCollectTripAndServiceIds(tripId, futureFeed, activeFeed); } // If a trip only in the active feed references a service_id that is set to be extended, that // service_id needs to be cloned and renamed to differentiate it from the same service_id in // the future feed. (The trip in question will be linked to the cloned service_id.) - Set tripsOnlyInActiveFeed = Sets.difference(feedMergeContext.activeTripIds, feedMergeContext.futureTripIds); + Set tripsOnlyInActiveFeed = Sets.difference(feedMergeContext.active.tripIds, feedMergeContext.future.tripIds); tripsOnlyInActiveFeed.stream() .map(tripId -> activeFeed.trips.get(tripId).service_id) .filter(serviceId -> serviceIdsToExtend.contains(serviceId)) @@ -418,7 +418,7 @@ private MergeStrategy getMergeStrategy() { // If a trip only in the future feed references a service_id that is set to be extended, that // service_id needs to be cloned and renamed to differentiate it from the same service_id in // the future feed. (The trip in question will be linked to the cloned service_id.) - Set tripsOnlyInFutureFeed = Sets.difference(feedMergeContext.futureTripIds, feedMergeContext.activeTripIds); + Set tripsOnlyInFutureFeed = Sets.difference(feedMergeContext.future.tripIds, feedMergeContext.active.tripIds); tripsOnlyInFutureFeed.stream() .map(tripId -> futureFeed.trips.get(tripId).service_id) .filter(serviceId -> serviceIdsToExtend.contains(serviceId)) diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/AgencyMergeLineContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/AgencyMergeLineContext.java index 33604c8c3..bfddc4776 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/AgencyMergeLineContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/AgencyMergeLineContext.java @@ -33,11 +33,11 @@ private void checkForMissingAgencyId() { // agency_id is optional if only one agency is present, but that will // cause issues for the feed merge, so we need to insert an agency_id // for the single entry. - if (isHandlingActiveFeed()) { - feedMergeContext.setActiveFeedNewAgencyId(UUID.randomUUID().toString()); - } else { - feedMergeContext.setFutureFeedNewAgencyId(UUID.randomUUID().toString()); - } + (isHandlingActiveFeed() + ? feedMergeContext.active + : feedMergeContext.future + ).setNewAgencyId(UUID.randomUUID().toString()); + if (keyFieldMissing) { // Only add agency_id field if it is missing in table. addField(Table.AGENCY.fields[0]); diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarDatesMergeLineContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarDatesMergeLineContext.java index b4e67169c..d2d5009cb 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarDatesMergeLineContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarDatesMergeLineContext.java @@ -37,7 +37,7 @@ private boolean checkCalendarDatesIds() throws IOException { // Drop any calendar_dates.txt records from the existing feed for dates that are // not before the first date of the future feed. LocalDate date = getCsvDate("date"); - LocalDate futureFeedFirstDate = feedMergeContext.getFutureFeedFirstDate(); + LocalDate futureFeedFirstDate = feedMergeContext.future.getFeedFirstDate(); if (isHandlingActiveFeed() && !date.isBefore(futureFeedFirstDate)) { LOG.warn( "Skipping calendar_dates entry {} because it operates in the time span of future feed (i.e., after or on {}).", @@ -61,12 +61,12 @@ private void updateFutureFeedFirstDate() { isHandlingActiveFeed() && job.mergeType.equals(SERVICE_PERIOD) && futureFirstCalendarStartDate.isBefore(LocalDate.MAX) && - feedMergeContext.getFutureFeedFirstDate().isBefore(futureFirstCalendarStartDate) + feedMergeContext.future.getFeedFirstDate().isBefore(futureFirstCalendarStartDate) ) { // If the future feed's first date is before its first calendar start date, // override the future feed first date with the calendar start date for use when checking // MTC calendar_dates and calendar records for modification/exclusion. - feedMergeContext.setFutureFeedFirstDate(futureFirstCalendarStartDate); + feedMergeContext.future.setFeedFirstDate(futureFirstCalendarStartDate); } } } \ No newline at end of file diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarMergeLineContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarMergeLineContext.java index 62baca8c6..66a09ecf3 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarMergeLineContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarMergeLineContext.java @@ -60,7 +60,7 @@ private boolean checkCalendarIds(Set idErrors) throws IOException // start date if the merge strategy dictates. The justification for this logic is that the active feed's // service_id will be modified to a different unique value and the trips shared between the future/active // service are exactly matching. - getFieldContext().resetValue(feedMergeContext.activeFeedFirstDate.format(GTFS_DATE_FORMATTER)); + getFieldContext().resetValue(feedMergeContext.active.getFeedFirstDate().format(GTFS_DATE_FORMATTER)); } } else { // If a service_id from the active calendar has both the @@ -69,7 +69,7 @@ private boolean checkCalendarIds(Set idErrors) throws IOException // calendar_dates, and calendar_attributes referencing this // service_id shall also be removed/ignored. Stop_time records // for the ignored trips shall also be removed. - LocalDate futureFeedFirstDate = feedMergeContext.getFutureFeedFirstDate(); + LocalDate futureFeedFirstDate = feedMergeContext.future.getFeedFirstDate(); if (!startDate.isBefore(futureFeedFirstDate)) { LOG.warn( "Skipping calendar entry {} because it operates fully within the time span of future feed.", diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/FeedContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/FeedContext.java new file mode 100644 index 000000000..228e1fb78 --- /dev/null +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/FeedContext.java @@ -0,0 +1,50 @@ +package com.conveyal.datatools.manager.jobs.feedmerge; + +import com.conveyal.datatools.manager.DataManager; +import com.conveyal.gtfs.loader.Feed; +import com.conveyal.gtfs.loader.Table; + +import java.io.IOException; +import java.time.LocalDate; +import java.util.Set; + +/** + * Contains information related to a feed to merge. + */ +public class FeedContext { + public final FeedToMerge feedToMerge; + public final Set tripIds; + public final Feed feed; + private LocalDate feedFirstDate; + /** + * Holds the auto-generated agency id to be updated for each feed if none was provided. + */ + private String newAgencyId; + + public FeedContext(FeedToMerge givenFeedToMerge) throws IOException { + feedToMerge = givenFeedToMerge; + feedToMerge.collectTripAndServiceIds(); + tripIds = feedToMerge.idsForTable.get(Table.TRIPS); + feed = new Feed(DataManager.GTFS_DATA_SOURCE, feedToMerge.version.namespace); + + // Initialize future and active feed's first date to the first calendar date from validation result. + // This is equivalent to either the earliest date of service defined for a calendar_date record or the + // earliest start_date value for a calendars.txt record. For MTC, however, they require that GTFS + // providers use calendars.txt entries and prefer that this value (which is used to determine cutoff + // dates for the active feed when merging with the future) be strictly assigned the earliest + // calendar#start_date (unless that table for some reason does not exist). + feedFirstDate = feedToMerge.version.validationResult.firstCalendarDate; + } + + public LocalDate getFeedFirstDate() { return feedFirstDate; } + + public void setFeedFirstDate(LocalDate firstDate) { feedFirstDate = firstDate; } + + public String getNewAgencyId() { + return newAgencyId; + } + + public void setNewAgencyId(String agencyId) { + newAgencyId = agencyId; + } +} diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/FeedMergeContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/FeedMergeContext.java index b61276a24..983b5b138 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/FeedMergeContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/FeedMergeContext.java @@ -1,11 +1,9 @@ package com.conveyal.datatools.manager.jobs.feedmerge; -import com.conveyal.datatools.manager.DataManager; import com.conveyal.datatools.manager.auth.Auth0UserProfile; import com.conveyal.datatools.manager.models.FeedVersion; import com.conveyal.datatools.manager.utils.MergeFeedUtils; import com.conveyal.gtfs.loader.Feed; -import com.conveyal.gtfs.loader.Table; import com.google.common.collect.Sets; import java.io.Closeable; @@ -19,52 +17,24 @@ */ public class FeedMergeContext implements Closeable { public final List feedsToMerge; - public final FeedToMerge activeFeedToMerge; - public final FeedToMerge futureFeedToMerge; - public final Set activeTripIds; - public final Set futureTripIds; + public final FeedContext active; + public final FeedContext future; public final boolean serviceIdsMatch; public final boolean tripIdsMatch; - public final Feed futureFeed; - public final Feed activeFeed; - public final LocalDate activeFeedFirstDate; - private LocalDate futureFeedFirstDate; private LocalDate futureFirstCalendarStartDate = LocalDate.MAX; - /** - * Trip ids shared between the active and future feed. - */ public final Set sharedTripIds; - /** - * Holds the auto-generated agency id to be updated for each feed if none was provided. - */ - private String activeFeedNewAgencyId; - private String futureFeedNewAgencyId; public FeedMergeContext(Set feedVersions, Auth0UserProfile owner) throws IOException { feedsToMerge = MergeFeedUtils.collectAndSortFeeds(feedVersions, owner); - activeFeedToMerge = feedsToMerge.get(1); - futureFeedToMerge = feedsToMerge.get(0); - futureFeedToMerge.collectTripAndServiceIds(); - activeFeedToMerge.collectTripAndServiceIds(); - activeTripIds = activeFeedToMerge.idsForTable.get(Table.TRIPS); - futureTripIds = futureFeedToMerge.idsForTable.get(Table.TRIPS); + FeedToMerge activeFeedToMerge = feedsToMerge.get(1); + FeedToMerge futureFeedToMerge = feedsToMerge.get(0); + active = new FeedContext(activeFeedToMerge); + future = new FeedContext(futureFeedToMerge); // Determine whether service and trip IDs are exact matches. serviceIdsMatch = activeFeedToMerge.serviceIds.equals(futureFeedToMerge.serviceIds); - tripIdsMatch = activeTripIds.equals(futureTripIds); - - futureFeed = new Feed(DataManager.GTFS_DATA_SOURCE, futureFeedToMerge.version.namespace); - activeFeed = new Feed(DataManager.GTFS_DATA_SOURCE, activeFeedToMerge.version.namespace); - sharedTripIds = Sets.intersection(activeTripIds, futureTripIds); - - // Initialize future and active feed's first date to the first calendar date from validation result. - // This is equivalent to either the earliest date of service defined for a calendar_date record or the - // earliest start_date value for a calendars.txt record. For MTC, however, they require that GTFS - // providers use calendars.txt entries and prefer that this value (which is used to determine cutoff - // dates for the active feed when merging with the future) be strictly assigned the earliest - // calendar#start_date (unless that table for some reason does not exist). - activeFeedFirstDate = activeFeedToMerge.version.validationResult.firstCalendarDate; - futureFeedFirstDate = futureFeedToMerge.version.validationResult.firstCalendarDate; + tripIdsMatch = active.tripIds.equals(future.tripIds); + sharedTripIds = Sets.intersection(active.tripIds, future.tripIds); } @Override @@ -94,7 +64,7 @@ public TripMismatchedServiceIds shouldFailJobDueToMatchingTripIds() { // If just the service_ids are an exact match, check the that the stop_times having matching signatures // between the two feeds (i.e., each stop time in the ordered list is identical between the two feeds). for (String tripId : sharedTripIds) { - TripMismatchedServiceIds mismatchInfo = tripIdHasMismatchedServiceIds(tripId, futureFeed, activeFeed); + TripMismatchedServiceIds mismatchInfo = tripIdHasMismatchedServiceIds(tripId, future.feed, active.feed); if (mismatchInfo.hasMismatch) { return mismatchInfo; } @@ -110,7 +80,7 @@ public TripMismatchedServiceIds shouldFailJobDueToMatchingTripIds() { * @return A {@link TripMismatchedServiceIds} with info on whether the given tripId is found in * different service ids in the active and future feed. */ - public static TripMismatchedServiceIds tripIdHasMismatchedServiceIds(String tripId, Feed futureFeed, Feed activeFeed) { + private static TripMismatchedServiceIds tripIdHasMismatchedServiceIds(String tripId, Feed futureFeed, Feed activeFeed) { String futureServiceId = futureFeed.trips.get(tripId).service_id; String activeServiceId = activeFeed.trips.get(tripId).service_id; return new TripMismatchedServiceIds(tripId, !futureServiceId.equals(activeServiceId), activeServiceId, futureServiceId); @@ -124,30 +94,6 @@ public void setFutureFirstCalendarStartDate(LocalDate futureFirstCalendarStartDa this.futureFirstCalendarStartDate = futureFirstCalendarStartDate; } - public LocalDate getFutureFeedFirstDate() { - return futureFeedFirstDate; - } - - public void setFutureFeedFirstDate(LocalDate futureFeedFirstDate) { - this.futureFeedFirstDate = futureFeedFirstDate; - } - - public String getActiveFeedNewAgencyId() { - return activeFeedNewAgencyId; - } - - public String getFutureFeedNewAgencyId() { - return futureFeedNewAgencyId; - } - - public void setActiveFeedNewAgencyId(String newAgencyId) { - this.activeFeedNewAgencyId = newAgencyId; - } - - public void setFutureFeedNewAgencyId(String newAgencyId) { - this.futureFeedNewAgencyId = newAgencyId; - } - /** * Holds the status of a trip service id mismatch determination. */ diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java index 90be0d009..74feaac71 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java @@ -348,7 +348,6 @@ protected boolean checkRoutesAndStopsIds(Set idErrors) throws IOEx // Key field has defaulted to the standard primary key field // (stop_id or route_id), which makes the check much // simpler (just skip the duplicate record). - // FIXME: refactor. if (hasDuplicateError(idErrors)) { shouldSkipRecord = true; } @@ -367,13 +366,10 @@ protected boolean checkRoutesAndStopsIds(Set idErrors) throws IOEx } private String getNewAgencyIdForFeed() { - String newAgencyId; - if (handlingActiveFeed) { - newAgencyId = feedMergeContext.getActiveFeedNewAgencyId(); - } else { - newAgencyId = feedMergeContext.getFutureFeedNewAgencyId(); - } - return newAgencyId; + return (handlingActiveFeed + ? feedMergeContext.active + : feedMergeContext.future + ).getNewAgencyId(); } private boolean hasBlankPrimaryKey() { diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/ShapesMergeLineContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/ShapesMergeLineContext.java index 4d21e657b..4bb7a35d0 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/ShapesMergeLineContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/ShapesMergeLineContext.java @@ -55,7 +55,6 @@ private boolean checkShapeIds(Set idErrors) { } } // Skip record if normal duplicate errors are found. - // FIXME: refactor (used in super class) if (hasDuplicateError(idErrors)) { shouldSkipRecord = true; } From 225eb76e5e4b884cfdbe4d0e97ad2d3bceca7251 Mon Sep 17 00:00:00 2001 From: binh-dam-ibigroup <56846598+binh-dam-ibigroup@users.noreply.github.com> Date: Wed, 17 Nov 2021 08:51:36 -0500 Subject: [PATCH 075/122] refactor(MergeFeedsjob): Address PR comments --- .../com/conveyal/datatools/manager/jobs/MergeFeedsJob.java | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java b/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java index 70a7b2f04..50cf49411 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java @@ -79,10 +79,6 @@ public class MergeFeedsJob extends FeedSourceJob { public final String projectId; public final MergeFeedsType mergeType; private File mergedTempFile = null; - /** - * If {@link MergeFeedsJob} storeNewVersion variable is true, a new version will be created from the merged GTFS - * dataset. Otherwise, this will be null throughout the life of the job. - */ final FeedVersion mergedVersion; @JsonIgnore @BsonIgnore public Set tripIdsToModifyForActiveFeed = new HashSet<>(); @@ -178,7 +174,7 @@ public void jobLogic() { logAndReportToBugsnag(e, message); status.fail(message, e); } - mergedTempFile.deleteOnExit(); + // Create the zipfile with try with resources so that it is always closed. try (ZipOutputStream out = new ZipOutputStream(new FileOutputStream(mergedTempFile))) { LOG.info("Created merge file: {}", mergedTempFile.getAbsolutePath()); From 2a30ee538a2930ced0f5a96b6146e773d1479228 Mon Sep 17 00:00:00 2001 From: binh-dam-ibigroup <56846598+binh-dam-ibigroup@users.noreply.github.com> Date: Wed, 17 Nov 2021 11:00:44 -0500 Subject: [PATCH 076/122] improvement(MergeFeedJobs): Improve reporting of trip ids to check. --- .../com/conveyal/datatools/manager/jobs/MergeFeedsJob.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java b/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java index 4cab869c7..06d72f8df 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java @@ -195,11 +195,10 @@ public void jobLogic() { // error message along with matching trip_ids with differing trip signatures." Set tripIdsWithInconsistentSignature = getSharedTripIdsWithInconsistentSignature(); if (!tripIdsWithInconsistentSignature.isEmpty()) { + mergeFeedsResult.tripIdsToCheck.addAll(tripIdsWithInconsistentSignature); failMergeJob( - String.format("Trips %s in new feed have differing makeup from matching trips in active feed. " + - "If a trip characteristic has changed, a new trip_id must be assigned.", - String.join(", ", tripIdsWithInconsistentSignature) - ) + "Trips in the new feed have differing makeup from matching trips in active feed. " + + "If a trip characteristic has changed, a new trip_id must be assigned." ); return; } From 6ea596804f6c46a2c9832091c8bb47ef83402f5f Mon Sep 17 00:00:00 2001 From: binh-dam-ibigroup <56846598+binh-dam-ibigroup@users.noreply.github.com> Date: Wed, 24 Nov 2021 00:21:38 -0500 Subject: [PATCH 077/122] fix(MergeLineContext): Partially remove unused service_ids. --- .../datatools/manager/jobs/MergeFeedsJob.java | 4 +- .../feedmerge/CalendarMergeLineContext.java | 84 +++++++++++-------- .../manager/jobs/feedmerge/FeedContext.java | 2 - .../jobs/feedmerge/FeedMergeContext.java | 11 ++- .../manager/jobs/feedmerge/FeedToMerge.java | 14 ++++ .../jobs/feedmerge/MergeLineContext.java | 29 ++++++- .../manager/jobs/MergeFeedsJobTest.java | 33 +++++--- 7 files changed, 123 insertions(+), 54 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java b/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java index 06d72f8df..f7c1f614e 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java @@ -86,7 +86,7 @@ public class MergeFeedsJob extends FeedSourceJob { @JsonIgnore @BsonIgnore public Set serviceIdsToCloneRenameAndExtend = new HashSet<>(); @JsonIgnore @BsonIgnore - public Set serviceIdsToTerminateEarly = new HashSet<>(); + public Set serviceIdsFromActiveFeedToTerminateEarly = new HashSet<>(); private List sharedConsistentTripAndCalendarIds = new ArrayList<>(); @@ -437,7 +437,7 @@ private void determineMergeStrategy() { // Build the set of calendars to be shortened to the day before the future feed start date // from trips in the active feed but not in the future feed. - serviceIdsToTerminateEarly.addAll( + serviceIdsFromActiveFeedToTerminateEarly.addAll( getActiveServiceIds(feedMergeContext.getActiveTripIdsNotInFutureFeed()) ); diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarMergeLineContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarMergeLineContext.java index 38fbef4bc..5d33fccbc 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarMergeLineContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarMergeLineContext.java @@ -38,19 +38,20 @@ public void afterRowWrite() throws IOException { private boolean checkCalendarIds(Set idErrors, FieldContext fieldContext) throws IOException { boolean shouldSkipRecord = false; + String key = getTableScopedValue(table, getIdScope(), keyValue); + if (isHandlingActiveFeed()) { LocalDate startDate = getCsvDate("start_date"); - // If a service_id from the active calendar has both the - // start_date and end_date in the future, the service will be - // excluded from the merged file. Records in trips, - // calendar_dates, and calendar_attributes referencing this - // service_id shall also be removed/ignored. Stop_time records - // for the ignored trips shall also be removed. if (!startDate.isBefore(feedMergeContext.future.getFeedFirstDate())) { + // If a service_id from the active calendar has both the + // start_date and end_date in the future, the service will be + // excluded from the merged file. Records in trips, + // calendar_dates, and calendar_attributes referencing this + // service_id shall also be removed/ignored. Stop_time records + // for the ignored trips shall also be removed. LOG.warn( - "Skipping calendar entry {} because it operates fully within the time span of future feed.", + "Skipping active calendar entry {} because it operates fully within the time span of future feed.", keyValue); - String key = getTableScopedValue(table, getIdScope(), keyValue); mergeFeedsResult.skippedIds.add(key); shouldSkipRecord = true; } else { @@ -68,7 +69,7 @@ private boolean checkCalendarIds(Set idErrors, FieldContext fieldC boolean activeAndFutureTripIdsAreDisjoint = job.sharedTripIdsWithConsistentSignature.isEmpty(); if (activeAndFutureTripIdsAreDisjoint) { futureStartDate = feedMergeContext.futureFirstCalendarStartDate; - } else if (job.serviceIdsToTerminateEarly.contains(keyValue)) { + } else if (job.serviceIdsFromActiveFeedToTerminateEarly.contains(keyValue)) { futureStartDate = feedMergeContext.future.getFeedFirstDate(); } // In other cases not covered above, new calendar entry is already flagged for insertion @@ -85,10 +86,20 @@ private boolean checkCalendarIds(Set idErrors, FieldContext fieldC .format(GTFS_DATE_FORMATTER)); } } +/* } else { + // If handling the future feed, the MTC revised feed merge logic is as follows: + // - Calendar entries from the future feed will be inserted as is in the merged feed. + // so no additional processing needed here, unless the calendar entry is no longer used. + if (job.serviceIdsFromFutureFeedToRemove.contains(keyValue)) { + LOG.warn( + "Skipping future calendar entry {} because it will become unused in the merged feed.", + keyValue); + mergeFeedsResult.skippedIds.add(key); + shouldSkipRecord = true; + } + + */ } - // If handling the future feed, the MTC revised feed merge logic is as follows: - // - Calendar entries from the future feed will be inserted as is in the merged feed. - // so no additional processing needed here. // If any service_id in the active feed matches with the future @@ -98,8 +109,7 @@ private boolean checkCalendarIds(Set idErrors, FieldContext fieldC // duplicates? I think we would need to consider the // service_id:exception_type:date as the unique key and include any // all entries as long as they are unique on this key. - - if (hasDuplicateError(idErrors)) { + if (isHandlingActiveFeed() && hasDuplicateError(idErrors)) { // Modify service_id and ensure that referencing trips // have service_id updated. updateAndRemapOutput(fieldContext); @@ -108,7 +118,11 @@ private boolean checkCalendarIds(Set idErrors, FieldContext fieldC // Track service ID because we want to avoid removing trips that may reference this // service_id when the service_id is used by calendar_dates that operate in the valid // date range, i.e., before the future feed's first date. - if (!shouldSkipRecord && fieldContext.nameEquals(SERVICE_ID)) mergeFeedsResult.serviceIds.add(fieldContext.getValueToWrite()); + // + // If service is going to be cloned, add to the output service ids. + if (!shouldSkipRecord && fieldContext.nameEquals(SERVICE_ID)) { + mergeFeedsResult.serviceIds.add(fieldContext.getValueToWrite()); + } return !shouldSkipRecord; } @@ -119,25 +133,27 @@ private boolean checkCalendarIds(Set idErrors, FieldContext fieldC * @throws IOException */ public void addClonedServiceId() throws IOException { - String originalServiceId = getRowValues()[keyFieldIndex]; - if (job.serviceIdsToCloneRenameAndExtend.contains(originalServiceId)) { - // FIXME: Do we need to worry about calendar_dates? - String[] clonedValues = getRowValues().clone(); - String newServiceId = clonedValues[keyFieldIndex] = String.join(":", getIdScope(), originalServiceId); - // Modify start date only (preserve the end date on the future calendar entry). - int startDateIndex = Table.CALENDAR.getFieldIndex("start_date"); - clonedValues[startDateIndex] = feedMergeContext.active.feed.calendars.get(originalServiceId).start_date - .format(GTFS_DATE_FORMATTER); - referenceTracker.checkReferencesAndUniqueness( - keyValue, - getLineNumber(), - table.fields[0], - newServiceId, - table, - keyField, - table.getOrderFieldName() - ); - writeValuesToTable(clonedValues, true); + if (isHandlingFutureFeed()) { + String originalServiceId = keyValue; + if (job.serviceIdsToCloneRenameAndExtend.contains(originalServiceId)) { + // FIXME: Do we need to worry about calendar_dates? + String[] clonedValues = getRowValues().clone(); + String newServiceId = clonedValues[keyFieldIndex] = String.join(":", getIdScope(), originalServiceId); + // Modify start date only (preserve the end date from the future calendar entry). + int startDateIndex = Table.CALENDAR.getFieldIndex("start_date"); + clonedValues[startDateIndex] = feedMergeContext.active.feed.calendars.get(originalServiceId).start_date + .format(GTFS_DATE_FORMATTER); + referenceTracker.checkReferencesAndUniqueness( + keyValue, + getLineNumber(), + table.fields[0], + newServiceId, + table, + keyField, + table.getOrderFieldName() + ); + writeValuesToTable(clonedValues, true); + } } } } \ No newline at end of file diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/FeedContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/FeedContext.java index 228e1fb78..d76f84068 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/FeedContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/FeedContext.java @@ -38,8 +38,6 @@ public FeedContext(FeedToMerge givenFeedToMerge) throws IOException { public LocalDate getFeedFirstDate() { return feedFirstDate; } - public void setFeedFirstDate(LocalDate firstDate) { feedFirstDate = firstDate; } - public String getNewAgencyId() { return newAgencyId; } diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/FeedMergeContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/FeedMergeContext.java index 6325b3691..e3ba05ec5 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/FeedMergeContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/FeedMergeContext.java @@ -32,7 +32,7 @@ public FeedMergeContext(Set feedVersions, Auth0UserProfile owner) t future = new FeedContext(futureFeedToMerge); // Determine whether service and trip IDs are exact matches. - serviceIdsMatch = activeFeedToMerge.serviceIds.equals(futureFeedToMerge.serviceIds); + serviceIdsMatch = activeFeedToMerge.serviceIdsInUse.equals(futureFeedToMerge.serviceIdsInUse); tripIdsMatch = active.tripIds.equals(future.tripIds); sharedTripIds = Sets.intersection(active.tripIds, future.tripIds); @@ -63,9 +63,16 @@ public boolean areActiveAndFutureTripIdsDisjoint() { } /** - * Obtains the active trip ids found in the active feed, but not in the future feed. + * Obtains the trip ids found in the active feed, but not in the future feed. */ public Sets.SetView getActiveTripIdsNotInFutureFeed() { return Sets.difference(active.tripIds, future.tripIds); } + + /** + * Obtains the trip ids found in the future feed, but not in the active feed. + */ + public Sets.SetView getFutureTripIdsNotInActiveFeed() { + return Sets.difference(future.tripIds, active.tripIds); + } } diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/FeedToMerge.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/FeedToMerge.java index c703c1c26..91ba441d4 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/FeedToMerge.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/FeedToMerge.java @@ -10,6 +10,7 @@ import java.io.IOException; import java.util.HashSet; import java.util.Set; +import java.util.stream.Collectors; import java.util.zip.ZipFile; import static com.conveyal.datatools.manager.utils.MergeFeedUtils.getIdsForTable; @@ -23,6 +24,7 @@ public class FeedToMerge implements Closeable { public ZipFile zipFile; public SetMultimap idsForTable = HashMultimap.create(); public Set serviceIds = new HashSet<>(); + public Set serviceIdsInUse; private static final Set
tablesToCheck = Sets.newHashSet(Table.TRIPS, Table.CALENDAR, Table.CALENDAR_DATES); public FeedToMerge(FeedVersion version) throws IOException { @@ -37,6 +39,18 @@ public void collectTripAndServiceIds() throws IOException { } serviceIds.addAll(idsForTable.get(Table.CALENDAR)); serviceIds.addAll(idsForTable.get(Table.CALENDAR_DATES)); + + serviceIdsInUse = getServiceIdsInUse(idsForTable.get(Table.TRIPS)); + } + + /** + * Obtains the service ids corresponding to the provided trip ids. + * FIXME: Duplicate of MergeFeedsJob. + */ + private Set getServiceIdsInUse(Set tripIds) { + return tripIds.stream() + .map(tripId -> version.retrieveFeed().trips.get(tripId).service_id) + .collect(Collectors.toSet()); } public void close() throws IOException { diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java index 34e3db185..cea509996 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java @@ -144,6 +144,14 @@ public void startNewFeed(int feedIndex) throws IOException { // If there is no agency_id for agency table, create one and ensure that // route#agency_id gets set. } + + if (handlingFutureFeed) { + mergeFeedsResult.serviceIds.addAll( + job.serviceIdsToCloneRenameAndExtend.stream().map( + id -> String.join(":", idScope, id) + ).collect(Collectors.toSet()) + ); + } } public boolean shouldSkipFile() { @@ -395,6 +403,24 @@ public boolean updateAgencyIdIfNeeded(FieldContext fieldContext) { return true; } + public boolean updateServiceIdsIfNeeded(FieldContext fieldContext) { + String fieldValue = fieldContext.getValue(); + if (table.name.equals(Table.TRIPS.name) && + fieldContext.nameEquals(SERVICE_ID) && + job.serviceIdsToCloneRenameAndExtend.contains(fieldValue) && + job.mergeType.equals(SERVICE_PERIOD) + ) { + // Future trip ids not in the active feed will not get the service id remapped, + // they will use the service id as defined in the future feed instead. + if (!(handlingFutureFeed && feedMergeContext.getFutureTripIdsNotInActiveFeed().contains(keyValue))) { + String newServiceId = String.join(":", idScope, fieldValue); + LOG.info("Updating {}#service_id to (auto-generated) {} for ID {}", table.name, newServiceId, keyValue); + fieldContext.setValueToWrite(newServiceId); + } + } + return true; + } + public boolean storeRowAndStopValues() { String newLine = String.join(",", rowValues); switch (table.name) { @@ -527,9 +553,10 @@ public boolean constructRowValues() throws IOException { updateAndRemapOutput(fieldContext); } + updateServiceIdsIfNeeded(fieldContext); + // Store values for key fields that have been encountered and update any key values that need modification due // to conflicts. - // This method can change skipRecord. if (!checkFieldsForMergeConflicts(getIdErrors(fieldContext), fieldContext)) { skipRecord = true; break; diff --git a/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java b/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java index bbf096fd9..1a87122e5 100644 --- a/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java +++ b/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java @@ -387,16 +387,31 @@ void mergeMTCShouldHandleMatchingTripIdsWithSameSignature() throws SQLException // - calendar table // expect a total of 4 records in calendar table: - // - 1 from the active feed (common_id start date is changed to one day before first start_date in future feed) - // (the other one is unused and is discarded) - // - 2 from the future feed - // - 1 cloned for the matching trip id present in both active and future feeds - // (from MergeFeedsJob#serviceIdsToCloneAndRename). + // - common_id from the active feed (but start date is changed to one day before first start_date in future feed), + // - common_id from the future feed (because of one future trip not in the active feed), + // - common_id cloned and extended for the matching trip id present in both active and future feeds + // (from MergeFeedsJob#serviceIdsToCloneAndRename), + // - only_calendar_id used in the future feed. assertThatSqlCountQueryYieldsExpectedCount( String.format("SELECT count(*) FROM %s.calendar", mergedNamespace), 4 ); + // Out of all trips from the input datasets, expect 4 trips in merged output. + // 1 trip from active feed that is not in the future feed, + // 1 trip in both the active and future feeds, with the same signature (same stop times), + // 2 trips from the future feed not in the active feed. + assertThatSqlCountQueryYieldsExpectedCount( + String.format("SELECT count(*) FROM %s.trips", mergedNamespace), + 4 + ); + + // There should be no unused service ids. + assertThatSqlCountQueryYieldsExpectedCount( + String.format("SELECT count(*) FROM %s.errors where error_type = 'SERVICE_UNUSED'", mergedNamespace), + 0 + ); + // expect that 2 calendars (1 common_id extended from future and 1 Fake_Transit1:common_id from active) have // start_date pinned to start date of active feed. assertThatSqlCountQueryYieldsExpectedCount( @@ -415,14 +430,6 @@ void mergeMTCShouldHandleMatchingTripIdsWithSameSignature() throws SQLException String.format("SELECT count(*) FROM %s.calendar where start_date = '20170918' and end_date='20170919'", mergedNamespace), 1 ); - // Out of all trips from the input datasets, expect 4 trips in merged output. - // 1 trip from active feed that is not in the future feed, - // 1 trip in both the active and future feeds, with the same signature (same stop times), - // 2 trips from the future feed not in the active feed. - assertThatSqlCountQueryYieldsExpectedCount( - String.format("SELECT count(*) FROM %s.trips", mergedNamespace), - 4 - ); } /** From 942e3e6298cce688a74065509c1cbac0e88e1a07 Mon Sep 17 00:00:00 2001 From: binh-dam-ibigroup <56846598+binh-dam-ibigroup@users.noreply.github.com> Date: Wed, 24 Nov 2021 10:33:11 -0500 Subject: [PATCH 078/122] fix(MergeLineContext): Partially remove unused service ids --- .../datatools/manager/jobs/MergeFeedsJob.java | 32 ++++++- .../feedmerge/CalendarMergeLineContext.java | 18 ++-- .../jobs/feedmerge/MergeLineContext.java | 90 +++++++++++-------- .../manager/jobs/MergeFeedsJobTest.java | 73 ++++++++++++++- .../gtfs/merge-data-added-trips-2/agency.txt | 2 + .../merge-data-added-trips-2/calendar.txt | 3 + .../merge-data-added-trips-2/feed_info.txt | 2 + .../gtfs/merge-data-added-trips-2/routes.txt | 3 + .../stop_attributes.txt | 3 + .../merge-data-added-trips-2/stop_times.txt | 9 ++ .../gtfs/merge-data-added-trips-2/stops.txt | 6 ++ .../gtfs/merge-data-added-trips-2/trips.txt | 3 + 12 files changed, 195 insertions(+), 49 deletions(-) create mode 100755 src/test/resources/com/conveyal/datatools/gtfs/merge-data-added-trips-2/agency.txt create mode 100755 src/test/resources/com/conveyal/datatools/gtfs/merge-data-added-trips-2/calendar.txt create mode 100644 src/test/resources/com/conveyal/datatools/gtfs/merge-data-added-trips-2/feed_info.txt create mode 100755 src/test/resources/com/conveyal/datatools/gtfs/merge-data-added-trips-2/routes.txt create mode 100644 src/test/resources/com/conveyal/datatools/gtfs/merge-data-added-trips-2/stop_attributes.txt create mode 100755 src/test/resources/com/conveyal/datatools/gtfs/merge-data-added-trips-2/stop_times.txt create mode 100755 src/test/resources/com/conveyal/datatools/gtfs/merge-data-added-trips-2/stops.txt create mode 100755 src/test/resources/com/conveyal/datatools/gtfs/merge-data-added-trips-2/trips.txt diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java b/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java index f7c1f614e..c1e858be0 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java @@ -23,6 +23,7 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.Lists; +import com.google.common.collect.Sets; import org.bson.codecs.pojo.annotations.BsonIgnore; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -87,6 +88,10 @@ public class MergeFeedsJob extends FeedSourceJob { public Set serviceIdsToCloneRenameAndExtend = new HashSet<>(); @JsonIgnore @BsonIgnore public Set serviceIdsFromActiveFeedToTerminateEarly = new HashSet<>(); + @JsonIgnore @BsonIgnore + public Set serviceIdsFromActiveFeedToRemove = new HashSet<>(); + @JsonIgnore @BsonIgnore + public Set serviceIdsFromFutureFeedToRemove = new HashSet<>(); private List sharedConsistentTripAndCalendarIds = new ArrayList<>(); @@ -441,6 +446,20 @@ private void determineMergeStrategy() { getActiveServiceIds(feedMergeContext.getActiveTripIdsNotInFutureFeed()) ); + // Build the set of calendars ids from the future feed to be removed + // because they become no longer used after shared trips are remapped to another service id. + serviceIdsFromFutureFeedToRemove = Sets.difference( + feedMergeContext.future.feedToMerge.serviceIds, + getFutureServiceIds(feedMergeContext.getFutureTripIdsNotInActiveFeed()) + ); + + // Build the set of calendars ids from the active feed to be removed + // because they become no longer used after shared trips are remapped to another service id. + serviceIdsFromActiveFeedToRemove = Sets.difference( + feedMergeContext.active.feedToMerge.serviceIds, + getActiveServiceIds(feedMergeContext.getActiveTripIdsNotInFutureFeed()) + ); + mergeFeedsResult.mergeStrategy = CHECK_STOP_TIMES; } } @@ -448,10 +467,19 @@ private void determineMergeStrategy() { /** * Obtains the service ids corresponding to the provided trip ids. */ - private List getActiveServiceIds(Set tripIds) { + private Set getActiveServiceIds(Set tripIds) { return tripIds.stream() .map(tripId -> feedMergeContext.active.feed.trips.get(tripId).service_id) - .collect(Collectors.toList()); + .collect(Collectors.toSet()); + } + + /** + * Obtains the service ids corresponding to the provided trip ids. + */ + private Set getFutureServiceIds(Set tripIds) { + return tripIds.stream() + .map(tripId -> feedMergeContext.future.feed.trips.get(tripId).service_id) + .collect(Collectors.toSet()); } /** diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarMergeLineContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarMergeLineContext.java index 5d33fccbc..ae6566f71 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarMergeLineContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarMergeLineContext.java @@ -86,10 +86,20 @@ private boolean checkCalendarIds(Set idErrors, FieldContext fieldC .format(GTFS_DATE_FORMATTER)); } } -/* } else { + + // Remove calendar entries that are no longer used. + if (job.serviceIdsFromActiveFeedToRemove.contains(keyValue)) { + LOG.warn( + "Skipping active calendar entry {} because it will become unused in the merged feed.", + keyValue); + mergeFeedsResult.skippedIds.add(key); + shouldSkipRecord = true; + } + } else { // If handling the future feed, the MTC revised feed merge logic is as follows: // - Calendar entries from the future feed will be inserted as is in the merged feed. - // so no additional processing needed here, unless the calendar entry is no longer used. + // so no additional processing needed here, unless the calendar entry is no longer used, + // in that case we drop the calendar entry. if (job.serviceIdsFromFutureFeedToRemove.contains(keyValue)) { LOG.warn( "Skipping future calendar entry {} because it will become unused in the merged feed.", @@ -97,8 +107,6 @@ private boolean checkCalendarIds(Set idErrors, FieldContext fieldC mergeFeedsResult.skippedIds.add(key); shouldSkipRecord = true; } - - */ } @@ -137,7 +145,7 @@ public void addClonedServiceId() throws IOException { String originalServiceId = keyValue; if (job.serviceIdsToCloneRenameAndExtend.contains(originalServiceId)) { // FIXME: Do we need to worry about calendar_dates? - String[] clonedValues = getRowValues().clone(); + String[] clonedValues = getOriginalRowValues().clone(); String newServiceId = clonedValues[keyFieldIndex] = String.join(":", getIdScope(), originalServiceId); // Modify start date only (preserve the end date from the future calendar entry). int startDateIndex = Table.CALENDAR.getFieldIndex("start_date"); diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java index cea509996..136846f7b 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java @@ -52,6 +52,7 @@ public class MergeLineContext { private CsvReader csvReader; private boolean skipRecord; protected boolean keyFieldMissing; + private String[] originalRowValues; private String[] rowValues; private int lineNumber = 0; protected final Table table; @@ -198,6 +199,7 @@ public boolean iterateOverRows() throws IOException { if (!constructRowValues()) { return false; } + finishRowAndWriteToZip(); } return true; @@ -491,6 +493,7 @@ public void initializeRowValues() { skipRecord = false; // Reset the row values (this must happen after the first line is checked). rowValues = new String[sharedSpecFields.size()]; + originalRowValues = new String[sharedSpecFields.size()]; } public void writeValuesToTable(String[] values, boolean incrementLineNumbers) throws IOException { @@ -522,6 +525,7 @@ public void writeHeaders() throws IOException { * @return false, if a failing condition was encountered. true, if everything was ok. */ public boolean constructRowValues() throws IOException { + boolean result = true; // Piece together the row to write, which should look practically identical to the original // row except for the identifiers receiving a prefix to avoid ID conflicts. for (int specFieldIndex = 0; specFieldIndex < sharedSpecFields.size(); specFieldIndex++) { @@ -533,65 +537,73 @@ public boolean constructRowValues() throws IOException { field, csvReader.get(fieldsFoundList.indexOf(field)) ); - // Handle filling in agency_id if missing when merging regional feeds. If false is returned, - // the job has encountered a failing condition (the method handles failing the job itself). - if (!updateAgencyIdIfNeeded(fieldContext)) { - return false; - } - // Determine if field is a GTFS identifier (and scope if needed). - scopeValueIfNeeded(fieldContext); - // Only need to check for merge conflicts if using MTC merge type because - // the regional merge type scopes all identifiers by default. Also, the - // reference tracker will get far too large if we attempt to use it to - // track references for a large number of feeds (e.g., every feed in New - // York State). - if (job.mergeType.equals(SERVICE_PERIOD)) { - // Remap service id from active feed to distinguish them - // from entries with the same id in the future feed. - // See https://github.com/ibi-group/datatools-server/issues/244 - if (handlingActiveFeed && fieldContext.nameEquals(SERVICE_ID)) { - updateAndRemapOutput(fieldContext); + originalRowValues[specFieldIndex] = fieldContext.getValueToWrite(); + if (!skipRecord) { + // Handle filling in agency_id if missing when merging regional feeds. If false is returned, + // the job has encountered a failing condition (the method handles failing the job itself). + if (!updateAgencyIdIfNeeded(fieldContext)) { + result = false; } + // Determine if field is a GTFS identifier (and scope if needed). + scopeValueIfNeeded(fieldContext); + // Only need to check for merge conflicts if using MTC merge type because + // the regional merge type scopes all identifiers by default. Also, the + // reference tracker will get far too large if we attempt to use it to + // track references for a large number of feeds (e.g., every feed in New + // York State). + if (job.mergeType.equals(SERVICE_PERIOD)) { + // Remap service id from active feed to distinguish them + // from entries with the same id in the future feed. + // See https://github.com/ibi-group/datatools-server/issues/244 + if (handlingActiveFeed && fieldContext.nameEquals(SERVICE_ID)) { + updateAndRemapOutput(fieldContext); + } - updateServiceIdsIfNeeded(fieldContext); + updateServiceIdsIfNeeded(fieldContext); - // Store values for key fields that have been encountered and update any key values that need modification due - // to conflicts. - if (!checkFieldsForMergeConflicts(getIdErrors(fieldContext), fieldContext)) { + // Store values for key fields that have been encountered and update any key values that need modification due + // to conflicts. + if (!checkFieldsForMergeConflicts(getIdErrors(fieldContext), fieldContext)) { + skipRecord = true; + continue; + } + } + // If the current field is a foreign reference, check if the reference has been removed in the + // merged result. If this is the case (or other conditions are met), we will need to skip this + // record. Likewise, if the reference has been modified, ensure that the value written to the + // merged result is correctly updated. + if (!checkForeignReferences(fieldContext)) { skipRecord = true; - break; + continue; } + rowValues[specFieldIndex] = fieldContext.getValueToWrite(); } - // If the current field is a foreign reference, check if the reference has been removed in the - // merged result. If this is the case (or other conditions are met), we will need to skip this - // record. Likewise, if the reference has been modified, ensure that the value written to the - // merged result is correctly updated. - if (!checkForeignReferences(fieldContext)) { - skipRecord = true; - break; - } - rowValues[specFieldIndex] = fieldContext.getValueToWrite(); } - return true; + return result; } - public void finishRowAndWriteToZip() throws IOException { + private void finishRowAndWriteToZip() throws IOException { + boolean shouldWriteCurrentRow = true; // Do not write rows that are designated to be skipped. if (skipRecord && job.mergeType.equals(SERVICE_PERIOD)) { mergeFeedsResult.recordsSkipCount++; - return; + shouldWriteCurrentRow = false; } // Store row and stop values. If the return value is true, the record has been skipped and we // should skip writing the row to the merged table. if (storeRowAndStopValues()) { - return; + shouldWriteCurrentRow = false; } + // Finally, handle writing lines to zip entry. if (mergedLineNumber == 0) { writeHeaders(); } - // Write line to table. - writeValuesToTable(rowValues, true); + + if (shouldWriteCurrentRow) { + // Write line to table. + writeValuesToTable(rowValues, true); + } // Optional table-specific additional processing. afterRowWrite(); @@ -631,7 +643,7 @@ protected int getLineNumber() { return lineNumber; } - protected String[] getRowValues() { return rowValues; } + protected String[] getOriginalRowValues() { return originalRowValues; } /** * Retrieves the value for the specified CSV field. diff --git a/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java b/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java index 1a87122e5..f27eee26b 100644 --- a/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java +++ b/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java @@ -69,6 +69,11 @@ public class MergeFeedsJobTest extends UnitTest { * and some added trips, and a trip from the base feed removed. */ private static FeedVersion fakeTransitSameSignatureTrips; + /** + * The base feed (transposed to the future dates), with some trip_ids from the base feed with the same signature, + * and a trip from the base feed removed. + */ + private static FeedVersion fakeTransitSameSignatureTrips2; private static FeedSource bart; private static FeedVersion noAgencyVersion1; private static FeedVersion noAgencyVersion2; @@ -137,6 +142,7 @@ public static void setUp() throws IOException { fakeTransitModService = createFeedVersion(fakeTransit, zipFolderFiles("merge-data-mod-services")); fakeTransitNewSignatureTrips = createFeedVersion(fakeTransit, zipFolderFiles("merge-data-mod-trips")); fakeTransitSameSignatureTrips = createFeedVersion(fakeTransit, zipFolderFiles("merge-data-added-trips")); + fakeTransitSameSignatureTrips2 = createFeedVersion(fakeTransit, zipFolderFiles("merge-data-added-trips-2")); // Feeds with no agency id FeedSource noAgencyIds = new FeedSource("no-agency-ids", project.id, MANUALLY_UPLOADED); @@ -345,17 +351,24 @@ void mergeMTCShouldHandleExtendFutureStrategy() throws SQLException { // assert service_ids start_dates have been extended to the start_date of the base feed. String mergedNamespace = mergeFeedsJob.mergedVersion.namespace; + // There should be no unused service ids. + assertThatSqlCountQueryYieldsExpectedCount( + String.format("SELECT count(*) FROM %s.errors where error_type = 'SERVICE_UNUSED'", mergedNamespace), + 0 + ); + // - calendar table - // expect a total of 5 records in calendar table + // expect a total of 1 record in calendar table that + // corresponds to the trip ids present in both active and future feed. assertThatSqlCountQueryYieldsExpectedCount( String.format("SELECT count(*) FROM %s.calendar", mergedNamespace), - 5 + 1 ); // expect that two records in calendar table have the correct start_date // (one for the original calendar entry, one for the extended service id for trips with same signature) assertThatSqlCountQueryYieldsExpectedCount( String.format("SELECT count(*) FROM %s.calendar where start_date = '20170918' and monday = 1", mergedNamespace), - 2 + 1 ); } @@ -432,6 +445,60 @@ void mergeMTCShouldHandleMatchingTripIdsWithSameSignature() throws SQLException ); } + /** + * Ensures that an MTC merge of feeds with exact matches of service_ids and trip_ids, + * trip ids having the same signature (same stop times) will utilize the + * {@link MergeStrategy#CHECK_STOP_TIMES} strategy correctly and drop unused future service ids. + */ + @Test + void mergeMTCShouldHandleMatchingTripIdsAndDropUnusedFutureCalendar() throws SQLException { + Set versions = new HashSet<>(); + versions.add(fakeTransitBase); + versions.add(fakeTransitSameSignatureTrips2); + MergeFeedsJob mergeFeedsJob = new MergeFeedsJob(user, versions, "merged_output", MergeFeedsType.SERVICE_PERIOD); + // Run the job in this thread (we're not concerned about concurrency here). + mergeFeedsJob.run(); + // Check that correct strategy was used. + assertEquals( + MergeStrategy.CHECK_STOP_TIMES, + mergeFeedsJob.mergeFeedsResult.mergeStrategy + ); + // Result should succeed. + assertFalse( + mergeFeedsJob.mergeFeedsResult.failed, + "Merge feeds job should succeed with CHECK_STOP_TIMES strategy." + ); + + String mergedNamespace = mergeFeedsJob.mergedVersion.namespace; + + // - calendar table + // expect a total of 4 records in calendar table: + // - common_id from the active feed (but start date is changed to one day before first start_date in future feed), + // - common_id from the future feed (because of one future trip not in the active feed), + // - common_id cloned and extended for the matching trip id present in both active and future feeds + // (from MergeFeedsJob#serviceIdsToCloneAndRename), + // - only_calendar_id used in the future feed. + assertThatSqlCountQueryYieldsExpectedCount( + String.format("SELECT count(*) FROM %s.calendar", mergedNamespace), + 3 + ); + + // Out of all trips from the input datasets, expect 4 trips in merged output. + // 1 trip from active feed that is not in the future feed, + // 1 trip in both the active and future feeds, with the same signature (same stop times), + // 2 trips from the future feed not in the active feed. + assertThatSqlCountQueryYieldsExpectedCount( + String.format("SELECT count(*) FROM %s.trips", mergedNamespace), + 3 + ); + + // There should be no unused service ids. + assertThatSqlCountQueryYieldsExpectedCount( + String.format("SELECT count(*) FROM %s.errors where error_type = 'SERVICE_UNUSED'", mergedNamespace), + 0 + ); + } + /** * Ensures that an MTC merge of feeds with trip_ids matching in the active and future feed, * but with different signatures (e.g. different stop times) fails. diff --git a/src/test/resources/com/conveyal/datatools/gtfs/merge-data-added-trips-2/agency.txt b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-added-trips-2/agency.txt new file mode 100755 index 000000000..a916ce91b --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-added-trips-2/agency.txt @@ -0,0 +1,2 @@ +agency_id,agency_name,agency_url,agency_lang,agency_phone,agency_email,agency_timezone,agency_fare_url,agency_branding_url +1,Fake Transit,,,,,America/Los_Angeles,, diff --git a/src/test/resources/com/conveyal/datatools/gtfs/merge-data-added-trips-2/calendar.txt b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-added-trips-2/calendar.txt new file mode 100755 index 000000000..8d5260fe6 --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-added-trips-2/calendar.txt @@ -0,0 +1,3 @@ +service_id,monday,tuesday,wednesday,thursday,friday,saturday,sunday,start_date,end_date +common_id,1,1,1,1,1,1,1,20170923,20170925 +only_calendar_id,1,1,1,1,1,1,1,20170920,20170927 diff --git a/src/test/resources/com/conveyal/datatools/gtfs/merge-data-added-trips-2/feed_info.txt b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-added-trips-2/feed_info.txt new file mode 100644 index 000000000..ceac60810 --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-added-trips-2/feed_info.txt @@ -0,0 +1,2 @@ +feed_id,feed_publisher_name,feed_publisher_url,feed_lang,feed_version +fake_transit,Conveyal,http://www.conveyal.com,en,1.0 \ No newline at end of file diff --git a/src/test/resources/com/conveyal/datatools/gtfs/merge-data-added-trips-2/routes.txt b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-added-trips-2/routes.txt new file mode 100755 index 000000000..b13480efa --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-added-trips-2/routes.txt @@ -0,0 +1,3 @@ +agency_id,route_id,route_short_name,route_long_name,route_desc,route_type,route_url,route_color,route_text_color,route_branding_url +1,1,1,Route 1,,3,,7CE6E7,FFFFFF, +1,2,2,Route 2,,3,,7CE6E7,FFFFFF, diff --git a/src/test/resources/com/conveyal/datatools/gtfs/merge-data-added-trips-2/stop_attributes.txt b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-added-trips-2/stop_attributes.txt new file mode 100644 index 000000000..b77c473e0 --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-added-trips-2/stop_attributes.txt @@ -0,0 +1,3 @@ +stop_id,accessibility_id,cardinal_direction,relative_position,stop_city +4u6g,0,SE,FS,Scotts Valley +johv,0,SE,FS,Scotts Valley diff --git a/src/test/resources/com/conveyal/datatools/gtfs/merge-data-added-trips-2/stop_times.txt b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-added-trips-2/stop_times.txt new file mode 100755 index 000000000..04d65a948 --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-added-trips-2/stop_times.txt @@ -0,0 +1,9 @@ +trip_id,arrival_time,departure_time,stop_id,stop_sequence,stop_headsign,pickup_type,drop_off_type,shape_dist_traveled,timepoint +trip3,07:00:00,07:00:00,4u6g,1,,0,0,0.0000000, +trip3,07:01:00,07:01:00,johv,2,,0,0,341.4491961, +only-calendar-trip1,07:00:00,07:00:00,4u6g,1,,0,0,0.0000000, +only-calendar-trip1,07:01:00,07:01:00,johv,2,,0,0,341.4491961, +only-calendar-trip2,07:00:00,07:00:00,johv,1,,0,0,0.0000000, +only-calendar-trip2,07:01:00,07:01:00,4u6g,2,,0,0,341.4491961, +only-calendar-trip999,07:00:00,07:00:00,johv,1,,0,0,0.0000000, +only-calendar-trip999,07:01:00,07:01:00,4u6g,2,,0,0,341.4491961, diff --git a/src/test/resources/com/conveyal/datatools/gtfs/merge-data-added-trips-2/stops.txt b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-added-trips-2/stops.txt new file mode 100755 index 000000000..0db5a6d40 --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-added-trips-2/stops.txt @@ -0,0 +1,6 @@ +stop_id,stop_code,stop_name,stop_desc,stop_lat,stop_lon,zone_id,stop_url,location_type,parent_station,stop_timezone,wheelchair_boarding +4u6g,4u6g,Butler Ln,,37.0612132,-122.0074332,,,0,,, +johv,johv,Scotts Valley Dr & Victor Sq,,37.0590172,-122.0096058,,,0,,, +123,,Parent Station,,37.0666,-122.0777,,,1,,, +1234,1234,Child Stop,,37.06662,-122.07772,,,0,123,, +1234567,1234567,Unused stop,,37.06668,-122.07781,,,0,123,, \ No newline at end of file diff --git a/src/test/resources/com/conveyal/datatools/gtfs/merge-data-added-trips-2/trips.txt b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-added-trips-2/trips.txt new file mode 100755 index 000000000..216821978 --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-added-trips-2/trips.txt @@ -0,0 +1,3 @@ +route_id,trip_id,trip_headsign,trip_short_name,direction_id,block_id,shape_id,bikes_allowed,wheelchair_accessible,service_id +2,only-calendar-trip2,,,0,,,0,0,common_id +2,trip3,,,0,,,0,0,only_calendar_id \ No newline at end of file From 6256c56f7f65b02794cb40d67ff86efad605a32c Mon Sep 17 00:00:00 2001 From: binh-dam-ibigroup <56846598+binh-dam-ibigroup@users.noreply.github.com> Date: Wed, 24 Nov 2021 10:38:19 -0500 Subject: [PATCH 079/122] test(MergeFeedsJobTest): Update test comments, do some refactor. --- .../manager/jobs/MergeFeedsJobTest.java | 34 ++++++++----------- 1 file changed, 14 insertions(+), 20 deletions(-) diff --git a/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java b/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java index f27eee26b..aae6ff06d 100644 --- a/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java +++ b/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java @@ -351,11 +351,7 @@ void mergeMTCShouldHandleExtendFutureStrategy() throws SQLException { // assert service_ids start_dates have been extended to the start_date of the base feed. String mergedNamespace = mergeFeedsJob.mergedVersion.namespace; - // There should be no unused service ids. - assertThatSqlCountQueryYieldsExpectedCount( - String.format("SELECT count(*) FROM %s.errors where error_type = 'SERVICE_UNUSED'", mergedNamespace), - 0 - ); + assertNoUnusedServiceIds(mergedNamespace); // - calendar table // expect a total of 1 record in calendar table that @@ -364,14 +360,21 @@ void mergeMTCShouldHandleExtendFutureStrategy() throws SQLException { String.format("SELECT count(*) FROM %s.calendar", mergedNamespace), 1 ); - // expect that two records in calendar table have the correct start_date - // (one for the original calendar entry, one for the extended service id for trips with same signature) + // expect that the record in calendar table has the correct start_date. assertThatSqlCountQueryYieldsExpectedCount( String.format("SELECT count(*) FROM %s.calendar where start_date = '20170918' and monday = 1", mergedNamespace), 1 ); } + private void assertNoUnusedServiceIds(String mergedNamespace) throws SQLException { + // There should be no unused service ids. + assertThatSqlCountQueryYieldsExpectedCount( + String.format("SELECT count(*) FROM %s.errors where error_type = 'SERVICE_UNUSED'", mergedNamespace), + 0 + ); + } + /** * Ensures that an MTC merge of feeds with exact matches of service_ids and trip_ids, * trip ids having the same signature (same stop times) will utilize the @@ -419,11 +422,7 @@ void mergeMTCShouldHandleMatchingTripIdsWithSameSignature() throws SQLException 4 ); - // There should be no unused service ids. - assertThatSqlCountQueryYieldsExpectedCount( - String.format("SELECT count(*) FROM %s.errors where error_type = 'SERVICE_UNUSED'", mergedNamespace), - 0 - ); + assertNoUnusedServiceIds(mergedNamespace); // expect that 2 calendars (1 common_id extended from future and 1 Fake_Transit1:common_id from active) have // start_date pinned to start date of active feed. @@ -472,9 +471,8 @@ void mergeMTCShouldHandleMatchingTripIdsAndDropUnusedFutureCalendar() throws SQL String mergedNamespace = mergeFeedsJob.mergedVersion.namespace; // - calendar table - // expect a total of 4 records in calendar table: + // expect a total of 3 records in calendar table: // - common_id from the active feed (but start date is changed to one day before first start_date in future feed), - // - common_id from the future feed (because of one future trip not in the active feed), // - common_id cloned and extended for the matching trip id present in both active and future feeds // (from MergeFeedsJob#serviceIdsToCloneAndRename), // - only_calendar_id used in the future feed. @@ -486,17 +484,13 @@ void mergeMTCShouldHandleMatchingTripIdsAndDropUnusedFutureCalendar() throws SQL // Out of all trips from the input datasets, expect 4 trips in merged output. // 1 trip from active feed that is not in the future feed, // 1 trip in both the active and future feeds, with the same signature (same stop times), - // 2 trips from the future feed not in the active feed. + // 1 trip from the future feed not in the active feed. assertThatSqlCountQueryYieldsExpectedCount( String.format("SELECT count(*) FROM %s.trips", mergedNamespace), 3 ); - // There should be no unused service ids. - assertThatSqlCountQueryYieldsExpectedCount( - String.format("SELECT count(*) FROM %s.errors where error_type = 'SERVICE_UNUSED'", mergedNamespace), - 0 - ); + assertNoUnusedServiceIds(mergedNamespace); } /** From 96efba31f219d57b1b3f9fe43615239a1c8896da Mon Sep 17 00:00:00 2001 From: binh-dam-ibigroup <56846598+binh-dam-ibigroup@users.noreply.github.com> Date: Wed, 24 Nov 2021 11:47:50 -0500 Subject: [PATCH 080/122] test(MergeFeedsJobTest): Fix test data. Start some refactors. --- .../feedmerge/CalendarMergeLineContext.java | 1 + .../manager/jobs/feedmerge/FeedContext.java | 25 ++++++++++++++- .../jobs/feedmerge/FeedMergeContext.java | 5 +++ .../manager/jobs/MergeFeedsJobTest.java | 31 ++++++++++++++----- .../merge-data-added-trips-2/stop_times.txt | 2 -- 5 files changed, 53 insertions(+), 11 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarMergeLineContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarMergeLineContext.java index ae6566f71..577441b89 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarMergeLineContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarMergeLineContext.java @@ -12,6 +12,7 @@ import java.util.Set; import java.util.zip.ZipOutputStream; +import static com.conveyal.datatools.manager.jobs.feedmerge.MergeFeedsType.SERVICE_PERIOD; import static com.conveyal.datatools.manager.utils.MergeFeedUtils.getTableScopedValue; import static com.conveyal.datatools.manager.utils.MergeFeedUtils.hasDuplicateError; import static com.conveyal.gtfs.loader.DateField.GTFS_DATE_FORMATTER; diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/FeedContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/FeedContext.java index d76f84068..f43252356 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/FeedContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/FeedContext.java @@ -3,10 +3,12 @@ import com.conveyal.datatools.manager.DataManager; import com.conveyal.gtfs.loader.Feed; import com.conveyal.gtfs.loader.Table; +import com.google.common.collect.Sets; import java.io.IOException; import java.time.LocalDate; import java.util.Set; +import java.util.stream.Collectors; /** * Contains information related to a feed to merge. @@ -15,11 +17,12 @@ public class FeedContext { public final FeedToMerge feedToMerge; public final Set tripIds; public final Feed feed; - private LocalDate feedFirstDate; + private final LocalDate feedFirstDate; /** * Holds the auto-generated agency id to be updated for each feed if none was provided. */ private String newAgencyId; + private Set serviceIdsToRemove; public FeedContext(FeedToMerge givenFeedToMerge) throws IOException { feedToMerge = givenFeedToMerge; @@ -45,4 +48,24 @@ public String getNewAgencyId() { public void setNewAgencyId(String agencyId) { newAgencyId = agencyId; } + + public Set getServiceIdsToRemove() { + return serviceIdsToRemove; + } + + public void setServiceIdsToRemoveFromOtherFeed(Set idsNotInOtherFeed) { + serviceIdsToRemove = Sets.difference( + feedToMerge.serviceIds, + getServiceIds(idsNotInOtherFeed) + ); + } + + /** + * Obtains the service ids corresponding to the provided trip ids. + */ + public Set getServiceIds(Set tripIds) { + return tripIds.stream() + .map(tripId -> feed.trips.get(tripId).service_id) + .collect(Collectors.toSet()); + } } diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/FeedMergeContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/FeedMergeContext.java index e3ba05ec5..c46dd4f84 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/FeedMergeContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/FeedMergeContext.java @@ -31,6 +31,11 @@ public FeedMergeContext(Set feedVersions, Auth0UserProfile owner) t active = new FeedContext(activeFeedToMerge); future = new FeedContext(futureFeedToMerge); + // Build the set of calendars ids from the active|future feed to be removed + // because they become no longer used after shared trips are remapped to another service id. + active.setServiceIdsToRemoveFromOtherFeed(getActiveTripIdsNotInFutureFeed()); + future.setServiceIdsToRemoveFromOtherFeed(getFutureTripIdsNotInActiveFeed()); + // Determine whether service and trip IDs are exact matches. serviceIdsMatch = activeFeedToMerge.serviceIdsInUse.equals(futureFeedToMerge.serviceIdsInUse); tripIdsMatch = active.tripIds.equals(future.tripIds); diff --git a/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java b/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java index aae6ff06d..380bc4f1e 100644 --- a/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java +++ b/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java @@ -352,6 +352,7 @@ void mergeMTCShouldHandleExtendFutureStrategy() throws SQLException { String mergedNamespace = mergeFeedsJob.mergedVersion.namespace; assertNoUnusedServiceIds(mergedNamespace); + assertNoRefIntegrityErrors(mergedNamespace); // - calendar table // expect a total of 1 record in calendar table that @@ -367,14 +368,6 @@ void mergeMTCShouldHandleExtendFutureStrategy() throws SQLException { ); } - private void assertNoUnusedServiceIds(String mergedNamespace) throws SQLException { - // There should be no unused service ids. - assertThatSqlCountQueryYieldsExpectedCount( - String.format("SELECT count(*) FROM %s.errors where error_type = 'SERVICE_UNUSED'", mergedNamespace), - 0 - ); - } - /** * Ensures that an MTC merge of feeds with exact matches of service_ids and trip_ids, * trip ids having the same signature (same stop times) will utilize the @@ -423,6 +416,7 @@ void mergeMTCShouldHandleMatchingTripIdsWithSameSignature() throws SQLException ); assertNoUnusedServiceIds(mergedNamespace); + assertNoRefIntegrityErrors(mergedNamespace); // expect that 2 calendars (1 common_id extended from future and 1 Fake_Transit1:common_id from active) have // start_date pinned to start date of active feed. @@ -491,6 +485,7 @@ void mergeMTCShouldHandleMatchingTripIdsAndDropUnusedFutureCalendar() throws SQL ); assertNoUnusedServiceIds(mergedNamespace); + assertNoRefIntegrityErrors(mergedNamespace); } /** @@ -1050,4 +1045,24 @@ private FeedVersion regionallyMergeVersions(Set versions) { LOG.info("Regional merged file: {}", mergeFeedsJob.mergedVersion.retrieveGtfsFile().getAbsolutePath()); return mergeFeedsJob.mergedVersion; } + + /** + * Checks there are no unused service ids. + */ + private void assertNoUnusedServiceIds(String mergedNamespace) throws SQLException { + assertThatSqlCountQueryYieldsExpectedCount( + String.format("SELECT count(*) FROM %s.errors where error_type = 'SERVICE_UNUSED'", mergedNamespace), + 0 + ); + } + + /** + * Checks there are no referential integrity issues. + */ + private void assertNoRefIntegrityErrors(String mergedNamespace) throws SQLException { + assertThatSqlCountQueryYieldsExpectedCount( + String.format("SELECT count(*) FROM %s.errors where error_type = 'REFERENTIAL_INTEGRITY'", mergedNamespace), + 0 + ); + } } diff --git a/src/test/resources/com/conveyal/datatools/gtfs/merge-data-added-trips-2/stop_times.txt b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-added-trips-2/stop_times.txt index 04d65a948..09e46408f 100755 --- a/src/test/resources/com/conveyal/datatools/gtfs/merge-data-added-trips-2/stop_times.txt +++ b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-added-trips-2/stop_times.txt @@ -5,5 +5,3 @@ only-calendar-trip1,07:00:00,07:00:00,4u6g,1,,0,0,0.0000000, only-calendar-trip1,07:01:00,07:01:00,johv,2,,0,0,341.4491961, only-calendar-trip2,07:00:00,07:00:00,johv,1,,0,0,0.0000000, only-calendar-trip2,07:01:00,07:01:00,4u6g,2,,0,0,341.4491961, -only-calendar-trip999,07:00:00,07:00:00,johv,1,,0,0,0.0000000, -only-calendar-trip999,07:01:00,07:01:00,4u6g,2,,0,0,341.4491961, From 162fc02c1d6665626c4687f5464d65d33fb6f7cf Mon Sep 17 00:00:00 2001 From: binh-dam-ibigroup <56846598+binh-dam-ibigroup@users.noreply.github.com> Date: Wed, 24 Nov 2021 12:14:17 -0500 Subject: [PATCH 081/122] refactor(FeedContext): Move logic to get service ids to remove. --- .../datatools/manager/jobs/MergeFeedsJob.java | 18 ++---------------- .../feedmerge/CalendarMergeLineContext.java | 5 ++--- .../manager/jobs/feedmerge/FeedContext.java | 3 ++- .../jobs/feedmerge/FeedMergeContext.java | 10 +++++----- 4 files changed, 11 insertions(+), 25 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java b/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java index c1e858be0..9030f996f 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java @@ -23,7 +23,6 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.Lists; -import com.google.common.collect.Sets; import org.bson.codecs.pojo.annotations.BsonIgnore; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -88,10 +87,6 @@ public class MergeFeedsJob extends FeedSourceJob { public Set serviceIdsToCloneRenameAndExtend = new HashSet<>(); @JsonIgnore @BsonIgnore public Set serviceIdsFromActiveFeedToTerminateEarly = new HashSet<>(); - @JsonIgnore @BsonIgnore - public Set serviceIdsFromActiveFeedToRemove = new HashSet<>(); - @JsonIgnore @BsonIgnore - public Set serviceIdsFromFutureFeedToRemove = new HashSet<>(); private List sharedConsistentTripAndCalendarIds = new ArrayList<>(); @@ -446,19 +441,10 @@ private void determineMergeStrategy() { getActiveServiceIds(feedMergeContext.getActiveTripIdsNotInFutureFeed()) ); - // Build the set of calendars ids from the future feed to be removed - // because they become no longer used after shared trips are remapped to another service id. - serviceIdsFromFutureFeedToRemove = Sets.difference( - feedMergeContext.future.feedToMerge.serviceIds, - getFutureServiceIds(feedMergeContext.getFutureTripIdsNotInActiveFeed()) - ); - // Build the set of calendars ids from the active feed to be removed + // Build the set of calendars ids from the active|future feed to be removed // because they become no longer used after shared trips are remapped to another service id. - serviceIdsFromActiveFeedToRemove = Sets.difference( - feedMergeContext.active.feedToMerge.serviceIds, - getActiveServiceIds(feedMergeContext.getActiveTripIdsNotInFutureFeed()) - ); + feedMergeContext.collectServiceIdsToRemove(); mergeFeedsResult.mergeStrategy = CHECK_STOP_TIMES; } diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarMergeLineContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarMergeLineContext.java index 577441b89..6dda5c6f6 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarMergeLineContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarMergeLineContext.java @@ -12,7 +12,6 @@ import java.util.Set; import java.util.zip.ZipOutputStream; -import static com.conveyal.datatools.manager.jobs.feedmerge.MergeFeedsType.SERVICE_PERIOD; import static com.conveyal.datatools.manager.utils.MergeFeedUtils.getTableScopedValue; import static com.conveyal.datatools.manager.utils.MergeFeedUtils.hasDuplicateError; import static com.conveyal.gtfs.loader.DateField.GTFS_DATE_FORMATTER; @@ -89,7 +88,7 @@ private boolean checkCalendarIds(Set idErrors, FieldContext fieldC } // Remove calendar entries that are no longer used. - if (job.serviceIdsFromActiveFeedToRemove.contains(keyValue)) { + if (feedMergeContext.active.getServiceIdsToRemove().contains(keyValue)) { LOG.warn( "Skipping active calendar entry {} because it will become unused in the merged feed.", keyValue); @@ -101,7 +100,7 @@ private boolean checkCalendarIds(Set idErrors, FieldContext fieldC // - Calendar entries from the future feed will be inserted as is in the merged feed. // so no additional processing needed here, unless the calendar entry is no longer used, // in that case we drop the calendar entry. - if (job.serviceIdsFromFutureFeedToRemove.contains(keyValue)) { + if (feedMergeContext.future.getServiceIdsToRemove().contains(keyValue)) { LOG.warn( "Skipping future calendar entry {} because it will become unused in the merged feed.", keyValue); diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/FeedContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/FeedContext.java index f43252356..5422d59fd 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/FeedContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/FeedContext.java @@ -7,6 +7,7 @@ import java.io.IOException; import java.time.LocalDate; +import java.util.HashSet; import java.util.Set; import java.util.stream.Collectors; @@ -22,7 +23,7 @@ public class FeedContext { * Holds the auto-generated agency id to be updated for each feed if none was provided. */ private String newAgencyId; - private Set serviceIdsToRemove; + private Set serviceIdsToRemove = new HashSet<>(); public FeedContext(FeedToMerge givenFeedToMerge) throws IOException { feedToMerge = givenFeedToMerge; diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/FeedMergeContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/FeedMergeContext.java index c46dd4f84..05782adca 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/FeedMergeContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/FeedMergeContext.java @@ -31,11 +31,6 @@ public FeedMergeContext(Set feedVersions, Auth0UserProfile owner) t active = new FeedContext(activeFeedToMerge); future = new FeedContext(futureFeedToMerge); - // Build the set of calendars ids from the active|future feed to be removed - // because they become no longer used after shared trips are remapped to another service id. - active.setServiceIdsToRemoveFromOtherFeed(getActiveTripIdsNotInFutureFeed()); - future.setServiceIdsToRemoveFromOtherFeed(getFutureTripIdsNotInActiveFeed()); - // Determine whether service and trip IDs are exact matches. serviceIdsMatch = activeFeedToMerge.serviceIdsInUse.equals(futureFeedToMerge.serviceIdsInUse); tripIdsMatch = active.tripIds.equals(future.tripIds); @@ -51,6 +46,11 @@ public FeedMergeContext(Set feedVersions, Auth0UserProfile owner) t this.futureFirstCalendarStartDate = futureFirstCalStartDate; } + public void collectServiceIdsToRemove() { + active.setServiceIdsToRemoveFromOtherFeed(getActiveTripIdsNotInFutureFeed()); + future.setServiceIdsToRemoveFromOtherFeed(getFutureTripIdsNotInActiveFeed()); + } + @Override public void close() throws IOException { for (FeedToMerge feed : feedsToMerge) { From eabe6a6804f8aaa6ad99b5e48d3105e04f98ff0d Mon Sep 17 00:00:00 2001 From: binh-dam-ibigroup <56846598+binh-dam-ibigroup@users.noreply.github.com> Date: Wed, 24 Nov 2021 13:21:29 -0500 Subject: [PATCH 082/122] fix(MergeFeedsJob): Add state for headers written; preserve calendar dates. --- .../datatools/manager/jobs/MergeFeedsJob.java | 22 +------ .../CalendarDatesMergeLineContext.java | 66 ++++++++++++++++++- .../feedmerge/CalendarMergeLineContext.java | 4 +- .../jobs/feedmerge/MergeLineContext.java | 7 +- .../manager/jobs/MergeFeedsJobTest.java | 6 ++ 5 files changed, 78 insertions(+), 27 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java b/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java index 9030f996f..af8fdf4e8 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java @@ -432,13 +432,13 @@ private void determineMergeStrategy() { // in both active/future feeds and that have consistent signature. // These trips will be linked to the new service_ids. serviceIdsToCloneRenameAndExtend.addAll( - getActiveServiceIds(this.sharedTripIdsWithConsistentSignature) + feedMergeContext.active.getServiceIds(this.sharedTripIdsWithConsistentSignature) ); // Build the set of calendars to be shortened to the day before the future feed start date // from trips in the active feed but not in the future feed. serviceIdsFromActiveFeedToTerminateEarly.addAll( - getActiveServiceIds(feedMergeContext.getActiveTripIdsNotInFutureFeed()) + feedMergeContext.active.getServiceIds(feedMergeContext.getActiveTripIdsNotInFutureFeed()) ); @@ -450,24 +450,6 @@ private void determineMergeStrategy() { } } - /** - * Obtains the service ids corresponding to the provided trip ids. - */ - private Set getActiveServiceIds(Set tripIds) { - return tripIds.stream() - .map(tripId -> feedMergeContext.active.feed.trips.get(tripId).service_id) - .collect(Collectors.toSet()); - } - - /** - * Obtains the service ids corresponding to the provided trip ids. - */ - private Set getFutureServiceIds(Set tripIds) { - return tripIds.stream() - .map(tripId -> feedMergeContext.future.feed.trips.get(tripId).service_id) - .collect(Collectors.toSet()); - } - /** * Compare stop times for the given tripId between the future and active feeds. The comparison will inform whether * trip and/or service IDs should be modified in the output merged feed. diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarDatesMergeLineContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarDatesMergeLineContext.java index 93b84e669..ba6c25b3f 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarDatesMergeLineContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarDatesMergeLineContext.java @@ -29,6 +29,14 @@ public boolean checkFieldsForMergeConflicts(Set idErrors, FieldCon return checkCalendarDatesIds(fieldContext); } + @Override + public void afterRowWrite() throws IOException { + // If the current row is for a calendar service_id that is marked for cloning/renaming, clone the + // values, change the ID, extend the start/end dates to the feed's full range, and write the + // additional line to the file. + addClonedServiceId(); + } + @Override public void startNewFeed(int feedIndex) throws IOException { super.startNewFeed(feedIndex); @@ -37,6 +45,7 @@ public void startNewFeed(int feedIndex) throws IOException { private boolean checkCalendarDatesIds(FieldContext fieldContext) throws IOException { boolean shouldSkipRecord = false; + String key = getTableScopedValue(table, getIdScope(), keyValue); // Drop any calendar_dates.txt records from the existing feed for dates that are // not before the first date of the future feed. LocalDate date = getCsvDate("date"); @@ -45,11 +54,37 @@ private boolean checkCalendarDatesIds(FieldContext fieldContext) throws IOExcept "Skipping calendar_dates entry {} because it operates in the time span of future feed (i.e., after or on {}).", keyValue, futureFeedFirstDateForCalendarValidity); - String key = getTableScopedValue(table, getIdScope(), keyValue); mergeFeedsResult.skippedIds.add(key); shouldSkipRecord = true; } - // Track service ID because we want to avoid removing trips that may reference this + + if (job.mergeType.equals(SERVICE_PERIOD)) { + if (isHandlingActiveFeed()) { + // Remove calendar entries that are no longer used. + if (feedMergeContext.active.getServiceIdsToRemove().contains(keyValue)) { + LOG.warn( + "Skipping active calendar entry {} because it will become unused in the merged feed.", + keyValue); + mergeFeedsResult.skippedIds.add(key); + shouldSkipRecord = true; + } + } else { + // If handling the future feed, the MTC revised feed merge logic is as follows: + // - Calendar entries from the future feed will be inserted as is in the merged feed. + // so no additional processing needed here, unless the calendar entry is no longer used, + // in that case we drop the calendar entry. + if (feedMergeContext.future.getServiceIdsToRemove().contains(keyValue)) { + LOG.warn( + "Skipping future calendar entry {} because it will become unused in the merged feed.", + keyValue); + mergeFeedsResult.skippedIds.add(key); + shouldSkipRecord = true; + } + } + } + + + // Track service ID because we want to avoid removing trips that may reference this // service_id when the service_id is used by calendar.txt records that operate in // the valid date range, i.e., before the future feed's first date. if (!shouldSkipRecord && fieldContext.nameEquals(SERVICE_ID)) mergeFeedsResult.serviceIds.add(fieldContext.getValueToWrite()); @@ -75,4 +110,29 @@ private LocalDate getFutureFeedFirstDateForCheckingCalendarValidity() { } return futureFeedFirstDate; } -} \ No newline at end of file + + /** + * Adds a cloned service id for trips with the same signature in both the active & future feeds. + * The cloned service id spans from the start date in the active feed until the end date in the future feed. + * @throws IOException + */ + public void addClonedServiceId() throws IOException { + if (isHandlingFutureFeed() && job.mergeType.equals(SERVICE_PERIOD)) { + String originalServiceId = keyValue; + if (job.serviceIdsToCloneRenameAndExtend.contains(originalServiceId)) { + String[] clonedValues = getOriginalRowValues().clone(); + String newServiceId = clonedValues[keyFieldIndex] = String.join(":", getIdScope(), originalServiceId); + + referenceTracker.checkReferencesAndUniqueness( + keyValue, + getLineNumber(), + table.fields[0], + newServiceId, + table, + keyField, + table.getOrderFieldName() + ); + writeValuesToTable(clonedValues, true); + } + } + }} \ No newline at end of file diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarMergeLineContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarMergeLineContext.java index 6dda5c6f6..31f821606 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarMergeLineContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarMergeLineContext.java @@ -12,6 +12,7 @@ import java.util.Set; import java.util.zip.ZipOutputStream; +import static com.conveyal.datatools.manager.jobs.feedmerge.MergeFeedsType.SERVICE_PERIOD; import static com.conveyal.datatools.manager.utils.MergeFeedUtils.getTableScopedValue; import static com.conveyal.datatools.manager.utils.MergeFeedUtils.hasDuplicateError; import static com.conveyal.gtfs.loader.DateField.GTFS_DATE_FORMATTER; @@ -141,10 +142,9 @@ private boolean checkCalendarIds(Set idErrors, FieldContext fieldC * @throws IOException */ public void addClonedServiceId() throws IOException { - if (isHandlingFutureFeed()) { + if (isHandlingFutureFeed() && job.mergeType.equals(SERVICE_PERIOD)) { String originalServiceId = keyValue; if (job.serviceIdsToCloneRenameAndExtend.contains(originalServiceId)) { - // FIXME: Do we need to worry about calendar_dates? String[] clonedValues = getOriginalRowValues().clone(); String newServiceId = clonedValues[keyFieldIndex] = String.join(":", getIdScope(), originalServiceId); // Modify start date only (preserve the end date from the future calendar entry). diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java index 136846f7b..cd4c29f99 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java @@ -76,6 +76,7 @@ public class MergeLineContext { public FeedSource feedSource; public boolean skipFile; public int mergedLineNumber = 0; + private boolean headersWritten = false; public static MergeLineContext create(MergeFeedsJob job, Table table, ZipOutputStream out) throws IOException { switch (table.name) { @@ -509,7 +510,7 @@ public void flushAndClose() throws IOException { out.closeEntry(); } - public void writeHeaders() throws IOException { + private void writeHeaders() throws IOException { // Create entry for zip file. ZipEntry tableEntry = new ZipEntry(table.name + ".txt"); out.putNextEntry(tableEntry); @@ -518,6 +519,8 @@ public void writeHeaders() throws IOException { .map(f -> f.name) .toArray(String[]::new); writeValuesToTable(headers, false); + + headersWritten = true; } /** @@ -596,7 +599,7 @@ private void finishRowAndWriteToZip() throws IOException { } // Finally, handle writing lines to zip entry. - if (mergedLineNumber == 0) { + if (mergedLineNumber == 0 && !headersWritten) { writeHeaders(); } diff --git a/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java b/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java index 380bc4f1e..2035c3ac9 100644 --- a/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java +++ b/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java @@ -484,6 +484,12 @@ void mergeMTCShouldHandleMatchingTripIdsAndDropUnusedFutureCalendar() throws SQL 3 ); + // The calendar_dates entry should be preserved, but remapped to a different id. + assertThatSqlCountQueryYieldsExpectedCount( + String.format("SELECT count(*) FROM %s.calendar_dates", mergedNamespace), + 1 + ); + assertNoUnusedServiceIds(mergedNamespace); assertNoRefIntegrityErrors(mergedNamespace); } From b82a81e064ec88e5c5dc82b8cd3672952bd33def Mon Sep 17 00:00:00 2001 From: binh-dam-ibigroup <56846598+binh-dam-ibigroup@users.noreply.github.com> Date: Wed, 24 Nov 2021 13:35:39 -0500 Subject: [PATCH 083/122] test(MergeFeedsJob): Add missing test data file. --- .../datatools/gtfs/merge-data-added-trips-2/calendar_dates.txt | 2 ++ 1 file changed, 2 insertions(+) create mode 100755 src/test/resources/com/conveyal/datatools/gtfs/merge-data-added-trips-2/calendar_dates.txt diff --git a/src/test/resources/com/conveyal/datatools/gtfs/merge-data-added-trips-2/calendar_dates.txt b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-added-trips-2/calendar_dates.txt new file mode 100755 index 000000000..5b1ec55e0 --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-added-trips-2/calendar_dates.txt @@ -0,0 +1,2 @@ +service_id,date,exception_type +common_id,20190218,1 From b281dea4f739bbc46089a0561758793a9b2addb3 Mon Sep 17 00:00:00 2001 From: binh-dam-ibigroup <56846598+binh-dam-ibigroup@users.noreply.github.com> Date: Tue, 30 Nov 2021 17:48:00 -0500 Subject: [PATCH 084/122] refactor(AutoPublishJob): Prep for adding AutoPublishJob --- .../common/status/MonitorableJob.java | 3 +- .../api/FeedVersionController.java | 40 ++++--- .../manager/jobs/AutoPublishJob.java | 112 ++++++++++++++++++ .../datatools/manager/models/FeedSource.java | 6 + .../datatools/manager/models/FeedVersion.java | 16 +++ .../manager/models/FeedVersionTest.java | 44 +++++-- 6 files changed, 192 insertions(+), 29 deletions(-) create mode 100644 src/main/java/com/conveyal/datatools/manager/jobs/AutoPublishJob.java diff --git a/src/main/java/com/conveyal/datatools/common/status/MonitorableJob.java b/src/main/java/com/conveyal/datatools/common/status/MonitorableJob.java index a3e487cbd..d134b2a34 100644 --- a/src/main/java/com/conveyal/datatools/common/status/MonitorableJob.java +++ b/src/main/java/com/conveyal/datatools/common/status/MonitorableJob.java @@ -79,7 +79,8 @@ public enum JobType { MONITOR_SERVER_STATUS, MERGE_FEED_VERSIONS, RECREATE_BUILD_IMAGE, - UPDATE_PELIAS + UPDATE_PELIAS, + AUTO_PUBLISH_FEED_VERSION } public MonitorableJob(Auth0UserProfile owner, String name, JobType type) { diff --git a/src/main/java/com/conveyal/datatools/manager/controllers/api/FeedVersionController.java b/src/main/java/com/conveyal/datatools/manager/controllers/api/FeedVersionController.java index 73d22c009..be8419f24 100644 --- a/src/main/java/com/conveyal/datatools/manager/controllers/api/FeedVersionController.java +++ b/src/main/java/com/conveyal/datatools/manager/controllers/api/FeedVersionController.java @@ -1,6 +1,7 @@ package com.conveyal.datatools.manager.controllers.api; import com.conveyal.datatools.common.utils.SparkUtils; +import com.conveyal.datatools.common.utils.aws.CheckedAWSException; import com.conveyal.datatools.common.utils.aws.S3Utils; import com.conveyal.datatools.manager.DataManager; import com.conveyal.datatools.manager.auth.Auth0UserProfile; @@ -260,30 +261,33 @@ private static FeedVersion publishToExternalResource (Request req, Response res) // notify any extensions of the change try { - for (String resourceType : DataManager.feedResources.keySet()) { - DataManager.feedResources.get(resourceType).feedVersionCreated(version, null); - } - if (!DataManager.isExtensionEnabled("mtc")) { - // update published version ID on feed source - Persistence.feedSources.updateField(version.feedSourceId, "publishedVersionId", version.namespace); - return version; - } else { - // NOTE: If the MTC extension is enabled, the parent feed source's publishedVersionId will not be updated to the - // version's namespace until the FeedUpdater has successfully downloaded the feed from the share S3 bucket. - Date publishedDate = new Date(); - // Set "sent" timestamp to now and reset "processed" timestamp (in the case that it had previously been - // published as the active version. - version.sentToExternalPublisher = publishedDate; - version.processedByExternalPublisher = null; - Persistence.feedVersions.replace(version.id, version); - return version; - } + publishToExternalResource(version); + return version; } catch (Exception e) { logMessageAndHalt(req, 500, "Could not publish feed.", e); return null; } } + public static void publishToExternalResource(FeedVersion version) throws CheckedAWSException { + for (String resourceType : DataManager.feedResources.keySet()) { + DataManager.feedResources.get(resourceType).feedVersionCreated(version, null); + } + if (!DataManager.isExtensionEnabled("mtc")) { + // update published version ID on feed source + Persistence.feedSources.updateField(version.feedSourceId, "publishedVersionId", version.namespace); + } else { + // NOTE: If the MTC extension is enabled, the parent feed source's publishedVersionId will not be updated to the + // version's namespace until the FeedUpdater has successfully downloaded the feed from the share S3 bucket. + Date publishedDate = new Date(); + // Set "sent" timestamp to now and reset "processed" timestamp (in the case that it had previously been + // published as the active version. + version.sentToExternalPublisher = publishedDate; + version.processedByExternalPublisher = null; + Persistence.feedVersions.replace(version.id, version); + } + } + /** * HTTP endpoint to initiate an export of a shapefile containing the stops or routes of one or * more feed versions. NOTE: the job ID returned must be used by the requester to download the diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/AutoPublishJob.java b/src/main/java/com/conveyal/datatools/manager/jobs/AutoPublishJob.java new file mode 100644 index 000000000..8fdc64b24 --- /dev/null +++ b/src/main/java/com/conveyal/datatools/manager/jobs/AutoPublishJob.java @@ -0,0 +1,112 @@ +package com.conveyal.datatools.manager.jobs; + +import com.conveyal.datatools.common.status.MonitorableJob; +import com.conveyal.datatools.manager.auth.Auth0UserProfile; +import com.conveyal.datatools.manager.controllers.api.FeedVersionController; +import com.conveyal.datatools.manager.gtfsplus.GtfsPlusValidation; +import com.conveyal.datatools.manager.models.FeedSource; +import com.conveyal.datatools.manager.models.FeedVersion; +import com.conveyal.datatools.manager.models.Project; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; + +/** + * Auto publish the latest feed versions of a feed source if there are no blocking validation errors. + * The following other conditions are checked: + * + * 1) {@link Project#pinnedDeploymentId} is not null. + * 2) The project is not locked/already being deployed by another instance of {@link AutoPublishJob}. + * 3) The deployment is not null, has feed versions and has been previously deployed. + * 4) The deployment does not conflict with an already active deployment. + * 5) There are no related feed versions with critical errors or feed fetches in progress. + * + * If there are related feed fetches in progress auto deploy is skipped but the deployment's feed versions are still + * advanced to the latest versions. + */ +public class AutoPublishJob extends MonitorableJob { + public static final Logger LOG = LoggerFactory.getLogger(AutoPublishJob.class); + + /** + * Feed source to be considered for auto-publishing. + */ + private final FeedSource feedSource; + + /** + * A set of projects which have been locked by a instance of {@link AutoPublishJob} to prevent repeat + * auto-publishing. + */ + private static final Set lockedFeedSources = Collections.synchronizedSet(new HashSet<>()); + + /** + * Auto-publish latest feed from specific feed source. + */ + public AutoPublishJob(FeedSource feedSource, Auth0UserProfile owner) { + super(owner, "Auto-Publish Feed", JobType.AUTO_PUBLISH_FEED_VERSION); + this.feedSource = feedSource; + } + + @Override + public void jobLogic() { + // Define if project and feed source are candidates for auto publish. + if ( + lockedFeedSources.contains(feedSource.id) + ) { + String message = String.format( + "Feed source %s skipped for auto publishing (another publishing job is in progress)", + feedSource.name + ); + LOG.info(message); + status.fail(message); + return; + } + + try { + synchronized (lockedFeedSources) { + if (!lockedFeedSources.contains(feedSource.id)) { + lockedFeedSources.add(feedSource.id); + LOG.info("Auto-publish lock added for feed source id: {}", feedSource.id); + } else { + LOG.warn("Unable to acquire lock for feed source {}", feedSource.name); + status.fail(String.format("Feed source %s is locked for auto-publishing.", feedSource.name)); + return; + } + } + LOG.info("Auto-publish task running for feed source {}", feedSource.name); + + // Retrieve the latest feed version associated with the feed source of the current + // feed version set for the deployment. + FeedVersion latestFeedVersion = feedSource.retrieveLatest(); + + // Validate and check for blocking issues in the feed version to deploy. + if (latestFeedVersion.hasBlockingIssuesForPublishing()) { + status.fail("Could not publish this feed version because it contains blocking errors."); + } + + try { + GtfsPlusValidation gtfsPlusValidation = GtfsPlusValidation.validate(latestFeedVersion.id); + if (!gtfsPlusValidation.issues.isEmpty()) { + status.fail("Could not publish this feed version because it contains GTFS+ blocking errors."); + } + } catch(Exception e) { + status.fail("Could not read GTFS+ zip file", e); + } + + + // If validation successful, just execute the feed updating process. + // FIXME: move method to another class. + FeedVersionController.publishToExternalResource(latestFeedVersion); + } catch (Exception e) { + status.fail( + String.format("Could not auto-publish feed source %s!", feedSource.name), + e + ); + } finally { + lockedFeedSources.remove(feedSource.id); + LOG.info("Auto deploy lock removed for project id: {}", feedSource.id); + } + } +} diff --git a/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java b/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java index 612e45049..905b56820 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java +++ b/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java @@ -102,6 +102,12 @@ public String organizationId () { /** Is this feed deployable? */ public boolean deployable; + /** + * Determines whether this feed will be auto-published (e.g. after fetching a new version) + * if no blocking errors are found (requires MTC extension). + */ + public boolean autoPublish; + /** * How do we receive this feed? */ diff --git a/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java b/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java index 657d90d33..add59f222 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java +++ b/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java @@ -430,6 +430,22 @@ private boolean hasHighSeverityErrorTypes() { return false; } + /** + * Checks for issues that block feed publishing, consistent with UI. + */ + public boolean hasBlockingIssuesForPublishing() { + return this.validationResult.fatalException != null; + /* + if () return true; + const errorCounts = version.validationResult.error_counts + return errorCounts && + !!(errorCounts.find(ec => BLOCKING_ERROR_TYPES.indexOf(ec.type) !== -1)) + } + + + */ + } + @JsonView(JsonViews.UserInterface.class) @JsonProperty("noteCount") public int noteCount() { diff --git a/src/test/java/com/conveyal/datatools/manager/models/FeedVersionTest.java b/src/test/java/com/conveyal/datatools/manager/models/FeedVersionTest.java index 01405a204..f3d9fa646 100644 --- a/src/test/java/com/conveyal/datatools/manager/models/FeedVersionTest.java +++ b/src/test/java/com/conveyal/datatools/manager/models/FeedVersionTest.java @@ -3,6 +3,8 @@ import com.conveyal.datatools.DatatoolsTest; import com.conveyal.datatools.UnitTest; import com.conveyal.datatools.manager.persistence.Persistence; +import com.conveyal.gtfs.validator.ValidationResult; +import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; @@ -13,32 +15,54 @@ import static org.hamcrest.Matchers.not; public class FeedVersionTest extends UnitTest { + private static Project project; + private static FeedSource feedSource; /** Initialize application for tests to run. */ @BeforeAll public static void setUp() throws Exception { // start server if it isn't already running DatatoolsTest.setUp(); + + // set up project + project = new Project(); + project.name = String.format("Test project %s", new Date()); + Persistence.projects.create(project); + + feedSource = new FeedSource("Test feed source"); + feedSource.projectId = project.id; + Persistence.feedSources.create(feedSource); } + @AfterAll + public static void tearDown() { + if (project != null) { + project.delete(); + } + } /** * Make sure FeedVersionIDs are always unique, even if created at the same second. * See https://github.com/ibi-group/datatools-server/issues/251 */ @Test - public void canCreateUniqueFeedVersionIDs() { + void canCreateUniqueFeedVersionIDs() { // Create a project, feed sources, and feed versions to merge. - Project testProject = new Project(); - testProject.name = String.format("Test project %s", new Date().toString()); - Persistence.projects.create(testProject); - FeedSource testFeedsoure = new FeedSource("Test feed source"); - testFeedsoure.projectId = testProject.id; - Persistence.feedSources.create(testFeedsoure); - // create two feedVersions immediately after each other which should end up having unique IDs - FeedVersion feedVersion1 = new FeedVersion(testFeedsoure); - FeedVersion feedVersion2 = new FeedVersion(testFeedsoure); + FeedVersion feedVersion1 = new FeedVersion(feedSource); + FeedVersion feedVersion2 = new FeedVersion(feedSource); assertThat(feedVersion1.id, not(equalTo(feedVersion2.id))); } + + /** + * Detect feeds with blocking issues for publishing. + */ + @Test + void canDetectBlockingIssuesForPublishing() { + FeedVersion feedVersion1 = new FeedVersion(feedSource); + feedVersion1.validationResult = new ValidationResult(); + feedVersion1.validationResult.fatalException = "A fatal exception occurred"; + + assertThat(feedVersion1.hasBlockingIssuesForPublishing(), equalTo(true)); + } } From bc6418c60dece236ecf221172f8c10ee61bef6cf Mon Sep 17 00:00:00 2001 From: binh-dam-ibigroup <56846598+binh-dam-ibigroup@users.noreply.github.com> Date: Thu, 2 Dec 2021 12:17:02 -0500 Subject: [PATCH 085/122] feat(AutoPublishJob): Add and hook auto publish job. --- .../manager/jobs/AutoPublishJob.java | 1 + .../manager/jobs/ProcessSingleFeedJob.java | 10 ++++ .../datatools/manager/models/FeedVersion.java | 49 +++++++++++-------- .../manager/models/FeedVersionTest.java | 45 ++++++++++++++++- 4 files changed, 83 insertions(+), 22 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/AutoPublishJob.java b/src/main/java/com/conveyal/datatools/manager/jobs/AutoPublishJob.java index 8fdc64b24..8319cccfa 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/AutoPublishJob.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/AutoPublishJob.java @@ -99,6 +99,7 @@ public void jobLogic() { // If validation successful, just execute the feed updating process. // FIXME: move method to another class. FeedVersionController.publishToExternalResource(latestFeedVersion); + LOG.info("Auto-published feed source {} to external resource.", feedSource.id); } catch (Exception e) { status.fail( String.format("Could not auto-publish feed source %s!", feedSource.name), diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/ProcessSingleFeedJob.java b/src/main/java/com/conveyal/datatools/manager/jobs/ProcessSingleFeedJob.java index d83161471..35e88e0a8 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/ProcessSingleFeedJob.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/ProcessSingleFeedJob.java @@ -150,6 +150,16 @@ public void jobLogic() { ) { addNextJob(new AutoDeployJob(feedSource.retrieveProject(), owner)); } + + // If auto-publish job is enabled (MTC extension required), + // create an auto-publish job for feeds that are fetched automatically. + if ( + DataManager.isExtensionEnabled("mtc") && + feedSource.autoPublish && + feedVersion.retrievalMethod == FeedRetrievalMethod.FETCHED_AUTOMATICALLY + ) { + addNextJob(new AutoPublishJob(feedSource, owner)); + } } /** diff --git a/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java b/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java index add59f222..b5bb8b7cd 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java +++ b/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java @@ -407,10 +407,34 @@ private boolean hasFeedVersionExpired() { * @return whether high severity error types have been flagged. */ private boolean hasHighSeverityErrorTypes() { - Set highSeverityErrorTypes = Stream.of(NewGTFSErrorType.values()) - .filter(type -> type.priority == Priority.HIGH) - .map(NewGTFSErrorType::toString) - .collect(Collectors.toSet()); + return hasSpecificErrorTypes(Stream.of(NewGTFSErrorType.values()) + .filter(type -> type.priority == Priority.HIGH)); + } + + /** + * Checks for issues that block feed publishing, consistent with UI. + */ + public boolean hasBlockingIssuesForPublishing() { + if (this.validationResult.fatalException != null) return true; + + return hasSpecificErrorTypes(Stream.of( + NewGTFSErrorType.ILLEGAL_FIELD_VALUE, + NewGTFSErrorType.MISSING_COLUMN, + NewGTFSErrorType.REFERENTIAL_INTEGRITY, + NewGTFSErrorType.SERVICE_WITHOUT_DAYS_OF_WEEK, + NewGTFSErrorType.TABLE_MISSING_COLUMN_HEADERS, + NewGTFSErrorType.TABLE_IN_SUBDIRECTORY, + NewGTFSErrorType.WRONG_NUMBER_OF_FIELDS + )); + } + + /** + * Determines whether this feed has specific error types. + */ + private boolean hasSpecificErrorTypes(Stream errorTypes) { + Set highSeverityErrorTypes = errorTypes + .map(NewGTFSErrorType::toString) + .collect(Collectors.toSet()); try (Connection connection = GTFS_DATA_SOURCE.getConnection()) { String sql = String.format("select distinct error_type from %s.errors", namespace); PreparedStatement preparedStatement = connection.prepareStatement(sql); @@ -427,23 +451,8 @@ private boolean hasHighSeverityErrorTypes() { // is invalid for one reason or another. return true; } - return false; - } - - /** - * Checks for issues that block feed publishing, consistent with UI. - */ - public boolean hasBlockingIssuesForPublishing() { - return this.validationResult.fatalException != null; - /* - if () return true; - const errorCounts = version.validationResult.error_counts - return errorCounts && - !!(errorCounts.find(ec => BLOCKING_ERROR_TYPES.indexOf(ec.type) !== -1)) - } - - */ + return false; } @JsonView(JsonViews.UserInterface.class) diff --git a/src/test/java/com/conveyal/datatools/manager/models/FeedVersionTest.java b/src/test/java/com/conveyal/datatools/manager/models/FeedVersionTest.java index f3d9fa646..11889e000 100644 --- a/src/test/java/com/conveyal/datatools/manager/models/FeedVersionTest.java +++ b/src/test/java/com/conveyal/datatools/manager/models/FeedVersionTest.java @@ -3,13 +3,27 @@ import com.conveyal.datatools.DatatoolsTest; import com.conveyal.datatools.UnitTest; import com.conveyal.datatools.manager.persistence.Persistence; +import com.conveyal.gtfs.error.NewGTFSError; +import com.conveyal.gtfs.error.NewGTFSErrorType; +import com.conveyal.gtfs.error.SQLErrorStorage; +import com.conveyal.gtfs.util.InvalidNamespaceException; import com.conveyal.gtfs.validator.ValidationResult; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.EnumSource; +import org.junit.jupiter.params.provider.MethodSource; +import org.junit.jupiter.params.provider.ValueSource; +import java.sql.Connection; +import java.sql.SQLException; import java.util.Date; +import java.util.stream.Stream; +import static com.conveyal.datatools.TestUtils.createFeedVersionFromGtfsZip; +import static com.conveyal.datatools.manager.DataManager.GTFS_DATA_SOURCE; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.not; @@ -55,14 +69,41 @@ void canCreateUniqueFeedVersionIDs() { } /** - * Detect feeds with blocking issues for publishing. + * Detect feeds with fatal exceptions (a blocking issue for publishing). */ @Test - void canDetectBlockingIssuesForPublishing() { + void canDetectBlockingFatalExceptionsForPublishing() { FeedVersion feedVersion1 = new FeedVersion(feedSource); feedVersion1.validationResult = new ValidationResult(); feedVersion1.validationResult.fatalException = "A fatal exception occurred"; assertThat(feedVersion1.hasBlockingIssuesForPublishing(), equalTo(true)); } + + /** + * Detect feeds with blocking error types that prevents publishing, per + * https://github.com/ibi-group/datatools-ui/blob/dev/lib/manager/util/version.js#L79. + */ + @ParameterizedTest + @EnumSource(value = NewGTFSErrorType.class, names = { + "ILLEGAL_FIELD_VALUE", + "MISSING_COLUMN", + "REFERENTIAL_INTEGRITY", + "SERVICE_WITHOUT_DAYS_OF_WEEK", + "TABLE_MISSING_COLUMN_HEADERS", + "TABLE_IN_SUBDIRECTORY", + "WRONG_NUMBER_OF_FIELDS" + }) + void canDetectBlockingErrorTypesForPublishing(NewGTFSErrorType errorType) throws InvalidNamespaceException, SQLException { + FeedVersion feedVersion1 = createFeedVersionFromGtfsZip(feedSource, "bart_old_lite.zip"); + + // Add blocking error types to feed version + try (Connection connection = GTFS_DATA_SOURCE.getConnection()) { + SQLErrorStorage errorStorage = new SQLErrorStorage(connection, feedVersion1.namespace + ".", false); + errorStorage.storeError(NewGTFSError.forFeed(errorType, null)); + errorStorage.commitAndClose(); + } + + assertThat(feedVersion1.hasBlockingIssuesForPublishing(), equalTo(true)); + } } From 22ad6acacea9304611020286ffeb35b0bb0ad0a3 Mon Sep 17 00:00:00 2001 From: binh-dam-ibigroup <56846598+binh-dam-ibigroup@users.noreply.github.com> Date: Thu, 2 Dec 2021 16:17:37 -0500 Subject: [PATCH 086/122] test(AutoPublishJobTest): Add "end-to-end" test class. --- .../manager/jobs/AutoPublishJob.java | 17 +-- .../manager/jobs/AutoPublishJobTest.java | 106 ++++++++++++++++++ 2 files changed, 111 insertions(+), 12 deletions(-) create mode 100644 src/test/java/com/conveyal/datatools/manager/jobs/AutoPublishJobTest.java diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/AutoPublishJob.java b/src/main/java/com/conveyal/datatools/manager/jobs/AutoPublishJob.java index 8319cccfa..e46e82940 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/AutoPublishJob.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/AutoPublishJob.java @@ -6,7 +6,6 @@ import com.conveyal.datatools.manager.gtfsplus.GtfsPlusValidation; import com.conveyal.datatools.manager.models.FeedSource; import com.conveyal.datatools.manager.models.FeedVersion; -import com.conveyal.datatools.manager.models.Project; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -15,17 +14,11 @@ import java.util.Set; /** - * Auto publish the latest feed versions of a feed source if there are no blocking validation errors. - * The following other conditions are checked: - * - * 1) {@link Project#pinnedDeploymentId} is not null. - * 2) The project is not locked/already being deployed by another instance of {@link AutoPublishJob}. - * 3) The deployment is not null, has feed versions and has been previously deployed. - * 4) The deployment does not conflict with an already active deployment. - * 5) There are no related feed versions with critical errors or feed fetches in progress. - * - * If there are related feed fetches in progress auto deploy is skipped but the deployment's feed versions are still - * advanced to the latest versions. + * Auto publish the latest feed versions of a feed source if: + * - there are no blocking validation errors, and. + * - the feed source is not locked/already being published by another instance of {@link AutoPublishJob}. + * This class assumes that feed attributes such as autoPublish and retrievalMethod + * have been checked. */ public class AutoPublishJob extends MonitorableJob { public static final Logger LOG = LoggerFactory.getLogger(AutoPublishJob.class); diff --git a/src/test/java/com/conveyal/datatools/manager/jobs/AutoPublishJobTest.java b/src/test/java/com/conveyal/datatools/manager/jobs/AutoPublishJobTest.java new file mode 100644 index 000000000..69cad7aee --- /dev/null +++ b/src/test/java/com/conveyal/datatools/manager/jobs/AutoPublishJobTest.java @@ -0,0 +1,106 @@ +package com.conveyal.datatools.manager.jobs; + +import com.conveyal.datatools.DatatoolsTest; +import com.conveyal.datatools.UnitTest; +import com.conveyal.datatools.manager.auth.Auth0UserProfile; +import com.conveyal.datatools.manager.models.FeedSource; +import com.conveyal.datatools.manager.models.Project; +import com.conveyal.datatools.manager.persistence.Persistence; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import java.io.IOException; +import java.util.Date; +import java.util.stream.Stream; + +import static com.conveyal.datatools.TestUtils.createFeedVersion; +import static com.conveyal.datatools.TestUtils.createFeedVersionFromGtfsZip; +import static com.conveyal.datatools.TestUtils.zipFolderFiles; +import static com.conveyal.datatools.manager.models.FeedRetrievalMethod.FETCHED_AUTOMATICALLY; +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * Tests for the various {@link AutoPublishJob} cases. + */ +public class AutoPublishJobTest extends UnitTest { + private static final Auth0UserProfile user = Auth0UserProfile.createTestAdminUser(); + private static Project project; + private static FeedSource feedSource; + + /** + * Prepare and start a testing-specific web server + */ + @BeforeAll + public static void setUp() throws IOException { + // start server if it isn't already running + DatatoolsTest.setUp(); + + // Create a project, feed sources, and feed versions to merge. + project = new Project(); + project.name = String.format("Test %s", new Date()); + Persistence.projects.create(project); + + FeedSource fakeAgency = new FeedSource("Feed source", project.id, FETCHED_AUTOMATICALLY); + Persistence.feedSources.create(fakeAgency); + feedSource = fakeAgency; + } + + @AfterAll + public static void tearDown() { + if (project != null) { + project.delete(); + } + } + + /** + * Ensures that a feed is or is not published depending on errors in the feed. + */ + @ParameterizedTest + @MethodSource("createPublishFeedCases") + void shouldProcessFeed(String resourceName, boolean isError, String errorMessage) throws IOException { + // Add the version to the feed source + if (resourceName.endsWith(".zip")) { + createFeedVersionFromGtfsZip(feedSource, resourceName); + } else { + createFeedVersion(feedSource, zipFolderFiles(resourceName)); + } + + // Create the job + AutoPublishJob autoPublishJob = new AutoPublishJob(feedSource, user); + + // Run the job in this thread (we're not concerned about concurrency here). + autoPublishJob.run(); + + assertEquals( + isError, + autoPublishJob.status.error, + "AutoPublish job error status was incorrectly determined." + ); + if (isError) { + assertEquals(errorMessage, autoPublishJob.status.message); + } + } + + private static Stream createPublishFeedCases() { + return Stream.of( + Arguments.of( + "fake-agency-with-only-calendar-expire-in-2099-with-failed-referential-integrity", + true, + "Could not publish this feed version because it contains blocking errors." + ), + Arguments.of( + "bart_old_lite.zip", + true, + "Could not publish this feed version because it contains GTFS+ blocking errors." + ), + Arguments.of( + "bart_new_lite.zip", + false, + null + ) + ); + } +} From eddd7451f46d0e565576c14a32e11710095a2bc8 Mon Sep 17 00:00:00 2001 From: binh-dam-ibigroup <56846598+binh-dam-ibigroup@users.noreply.github.com> Date: Thu, 2 Dec 2021 17:07:48 -0500 Subject: [PATCH 087/122] refactor(MonitorableJobWithResourceLock): Extract resource lock mgmt code to separate class. --- .../datatools/manager/jobs/AutoDeployJob.java | 305 +++++++++--------- .../manager/jobs/AutoPublishJob.java | 102 ++---- .../jobs/MonitorableJobWithResourceLock.java | 86 +++++ 3 files changed, 260 insertions(+), 233 deletions(-) create mode 100644 src/main/java/com/conveyal/datatools/manager/jobs/MonitorableJobWithResourceLock.java diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/AutoDeployJob.java b/src/main/java/com/conveyal/datatools/manager/jobs/AutoDeployJob.java index bac7fca7d..89a88785b 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/AutoDeployJob.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/AutoDeployJob.java @@ -1,6 +1,5 @@ package com.conveyal.datatools.manager.jobs; -import com.conveyal.datatools.common.status.MonitorableJob; import com.conveyal.datatools.manager.auth.Auth0UserProfile; import com.conveyal.datatools.manager.models.Deployment; import com.conveyal.datatools.manager.models.FeedVersion; @@ -12,7 +11,6 @@ import org.slf4j.LoggerFactory; import java.util.Collection; -import java.util.Collections; import java.util.Date; import java.util.HashSet; import java.util.LinkedList; @@ -32,7 +30,7 @@ * If there are related feed fetches in progress auto deploy is skipped but the deployment's feed versions are still * advanced to the latest versions. */ -public class AutoDeployJob extends MonitorableJob { +public class AutoDeployJob extends MonitorableJobWithResourceLock { public static final Logger LOG = LoggerFactory.getLogger(AutoDeployJob.class); /** @@ -40,29 +38,30 @@ public class AutoDeployJob extends MonitorableJob { */ private final Project project; - /** - * A set of projects which have been locked by a instance of {@link AutoDeployJob} to prevent repeat - * deployments. - */ - private static final Set lockedProjects = Collections.synchronizedSet(new HashSet<>()); + private final Deployment deployment; /** * Auto deploy specific project. */ public AutoDeployJob(Project project, Auth0UserProfile owner) { - super(owner, "Auto Deploy Feed", JobType.AUTO_DEPLOY_FEED_VERSION); + super( + owner, + "Auto Deploy Feed", + JobType.AUTO_DEPLOY_FEED_VERSION, + project, + project.name + ); this.project = project; + deployment = Persistence.deployments.getById(project.pinnedDeploymentId); } @Override public void jobLogic() { - Deployment deployment = Persistence.deployments.getById(project.pinnedDeploymentId); // Define if project and deployment are candidates for auto deploy. if ( project.pinnedDeploymentId == null || deployment == null || - deployment.feedVersionIds.isEmpty() || - lockedProjects.contains(project.id) + deployment.feedVersionIds.isEmpty() ) { String message = String.format( "Project %s skipped for auto deployment as required criteria not met.", @@ -89,171 +88,157 @@ public void jobLogic() { return; } - try { - synchronized (lockedProjects) { - if (!lockedProjects.contains(project.id)) { - lockedProjects.add(project.id); - LOG.info("Auto deploy lock added for project id: {}", project.id); - } else { - LOG.warn("Unable to acquire lock for project {}", project.name); - status.fail(String.format("Project %s is locked for auto-deployments.", project.name)); - return; - } - } - LOG.info("Auto deploy task running for project {}", project.name); - - // Get the most recently used server. - String latestServerId = deployment.latest().serverId; - OtpServer server = Persistence.servers.getById(latestServerId); - if (server == null) { - String message = String.format( - "Server with id %s no longer exists. Skipping deployment for project %s.", - latestServerId, - project.name - ); - LOG.warn(message); - status.fail(message); - return; - } + // Super class handles lock management and will trigger innerJobLogic. + super.jobLogic(); + } - // Analyze and update feed versions in deployment. - Collection updatedFeedVersionIds = new LinkedList<>(); - List latestVersionsWithCriticalErrors = new LinkedList<>(); - List previousFeedVersions = deployment.retrieveFeedVersions(); - boolean shouldWaitForNewFeedVersions = false; + @Override + protected void innerJobLogic() { + LOG.info("Auto deploy task running for project {}", project.name); - // Production ready feed versions. - List pinnedFeedVersions = deployment.retrievePinnedFeedVersions(); - Set pinnedFeedSourceIds = new HashSet<>( - pinnedFeedVersions - .stream() - .map(pinnedFeedVersion -> pinnedFeedVersion.feedSource.id) - .collect(Collectors.toList()) + // Get the most recently used server. + String latestServerId = deployment.latest().serverId; + OtpServer server = Persistence.servers.getById(latestServerId); + if (server == null) { + String message = String.format( + "Server with id %s no longer exists. Skipping deployment for project %s.", + latestServerId, + project.name ); + LOG.warn(message); + status.fail(message); + return; + } - // Iterate through each feed version for deployment. - for ( - Deployment.SummarizedFeedVersion currentDeploymentFeedVersion : previousFeedVersions - ) { - // Retrieve the latest feed version associated with the feed source of the current - // feed version set for the deployment. - FeedVersion latestFeedVersion = currentDeploymentFeedVersion.feedSource.retrieveLatest(); - // Make sure the latest feed version is not going to supersede a pinned feed version. - if (pinnedFeedSourceIds.contains(latestFeedVersion.feedSourceId)) { - continue; - } - - // Update to the latest feed version. - updatedFeedVersionIds.add(latestFeedVersion.id); - - // Throttle this auto-deployment if needed. For projects that haven't yet been auto-deployed, don't - // wait and go ahead with the auto-deployment. But if the project has been auto-deployed before and - // if the latest feed version was created before the last time the project was auto-deployed and - // there are currently-active jobs that could result in an updated feed version being created, then - // this auto deployment should be throttled. - if ( - project.lastAutoDeploy != null && - latestFeedVersion.dateCreated.before(project.lastAutoDeploy) && - currentDeploymentFeedVersion.feedSource.hasJobsInProgress() - ) { - // Another job exists that should result in the creation of a new feed version which should then - // trigger an additional AutoDeploy job. - LOG.warn( - "Feed source {} contains an active job that should result in the creation of a new feed version. Auto deployment will be skipped until that version has fully processed.", - currentDeploymentFeedVersion.feedSource.name - ); - shouldWaitForNewFeedVersions = true; - } - - // Make sure the latest feed version has no critical errors. - if (latestFeedVersion.hasCriticalErrors()) { - latestVersionsWithCriticalErrors.add(latestFeedVersion); - } + // Analyze and update feed versions in deployment. + Collection updatedFeedVersionIds = new LinkedList<>(); + List latestVersionsWithCriticalErrors = new LinkedList<>(); + List previousFeedVersions = deployment.retrieveFeedVersions(); + boolean shouldWaitForNewFeedVersions = false; + + // Production ready feed versions. + List pinnedFeedVersions = deployment.retrievePinnedFeedVersions(); + Set pinnedFeedSourceIds = new HashSet<>( + pinnedFeedVersions + .stream() + .map(pinnedFeedVersion -> pinnedFeedVersion.feedSource.id) + .collect(Collectors.toList()) + ); + + // Iterate through each feed version for deployment. + for ( + Deployment.SummarizedFeedVersion currentDeploymentFeedVersion : previousFeedVersions + ) { + // Retrieve the latest feed version associated with the feed source of the current + // feed version set for the deployment. + FeedVersion latestFeedVersion = currentDeploymentFeedVersion.feedSource.retrieveLatest(); + // Make sure the latest feed version is not going to supersede a pinned feed version. + if (pinnedFeedSourceIds.contains(latestFeedVersion.feedSourceId)) { + continue; } - // Skip auto-deployment for this project if Data Tools should wait for a job that should create a new - // feed version to complete. - if (shouldWaitForNewFeedVersions) { - status.completeSuccessfully("Auto-Deployment will wait for new feed versions to be created from jobs in-progress"); - return; - } + // Update to the latest feed version. + updatedFeedVersionIds.add(latestFeedVersion.id); - // Skip auto-deployment for this project if any of the feed versions contained critical errors. - if (latestVersionsWithCriticalErrors.size() > 0) { - StringBuilder errorMessageBuilder = new StringBuilder( - String.format("Auto deployment for project %s has %s feed(s) with critical errors:", - project.name, - latestVersionsWithCriticalErrors.size()) + // Throttle this auto-deployment if needed. For projects that haven't yet been auto-deployed, don't + // wait and go ahead with the auto-deployment. But if the project has been auto-deployed before and + // if the latest feed version was created before the last time the project was auto-deployed and + // there are currently-active jobs that could result in an updated feed version being created, then + // this auto deployment should be throttled. + if ( + project.lastAutoDeploy != null && + latestFeedVersion.dateCreated.before(project.lastAutoDeploy) && + currentDeploymentFeedVersion.feedSource.hasJobsInProgress() + ) { + // Another job exists that should result in the creation of a new feed version which should then + // trigger an additional AutoDeploy job. + LOG.warn( + "Feed source {} contains an active job that should result in the creation of a new feed version. Auto deployment will be skipped until that version has fully processed.", + currentDeploymentFeedVersion.feedSource.name ); - for (FeedVersion version : latestVersionsWithCriticalErrors) { - errorMessageBuilder.append( - String.format( - "%s (version %s), ", - version.parentFeedSource().name, - version.id - ) - ); - } - String message = errorMessageBuilder.toString(); - LOG.warn(message); - if (!project.autoDeployWithCriticalErrors) { - NotifyUsersForSubscriptionJob.createNotification( - "deployment-updated", - project.id, - message - ); - status.fail(message); - return; - } + shouldWaitForNewFeedVersions = true; } - // Add all pinned feed versions to the list of feed versions to be deployed so that they aren't lost as part - // of this update. - for (Deployment.SummarizedFeedVersion pinnedFeedVersion : pinnedFeedVersions) { - updatedFeedVersionIds.add(pinnedFeedVersion.id); + // Make sure the latest feed version has no critical errors. + if (latestFeedVersion.hasCriticalErrors()) { + latestVersionsWithCriticalErrors.add(latestFeedVersion); } + } + + // Skip auto-deployment for this project if Data Tools should wait for a job that should create a new + // feed version to complete. + if (shouldWaitForNewFeedVersions) { + status.completeSuccessfully("Auto-Deployment will wait for new feed versions to be created from jobs in-progress"); + return; + } - // Check if the updated feed versions have any difference between the previous ones. If not, and if not - // doing a regularly scheduled update with street data, then don't bother starting a deploy job. - // TODO: add logic for street data update - Set previousFeedVersionIds = new HashSet<>( - previousFeedVersions.stream().map(feedVersion -> feedVersion.id).collect(Collectors.toList()) + // Skip auto-deployment for this project if any of the feed versions contained critical errors. + if (latestVersionsWithCriticalErrors.size() > 0) { + StringBuilder errorMessageBuilder = new StringBuilder( + String.format("Auto deployment for project %s has %s feed(s) with critical errors:", + project.name, + latestVersionsWithCriticalErrors.size()) ); - if ( - !updatedFeedVersionIds.stream() - .anyMatch(feedVersionId -> !previousFeedVersionIds.contains(feedVersionId)) - ) { - LOG.info("No updated feed versions to deploy for project {}.", project.name); - status.completeSuccessfully("No updated feed versions to deploy."); - return; + for (FeedVersion version : latestVersionsWithCriticalErrors) { + errorMessageBuilder.append( + String.format( + "%s (version %s), ", + version.parentFeedSource().name, + version.id + ) + ); } - - // Queue up the deploy job. - if (JobUtils.queueDeployJob(deployment, owner, server) != null) { - LOG.info("Last auto deploy date updated for project {}.", project.name); - // Update the deployment's feed version IDs with the latest (and pinned) feed versions. - deployment.feedVersionIds = updatedFeedVersionIds; - project.lastAutoDeploy = new Date(); - Persistence.deployments.replace(deployment.id, deployment); - Persistence.projects.replace(project.id, project); - status.completeSuccessfully("Auto deploy started new deploy job."); - } else { - String message = String.format( - "Auto-deployment to %s should occur after active deployment for project %s completes.", - server.name, - project.name + String message = errorMessageBuilder.toString(); + LOG.warn(message); + if (!project.autoDeployWithCriticalErrors) { + NotifyUsersForSubscriptionJob.createNotification( + "deployment-updated", + project.id, + message ); - LOG.info(message); - status.completeSuccessfully(message); + status.fail(message); + return; } - } catch (Exception e) { - status.fail( - String.format("Could not auto-deploy project %s!", project.name), - e + } + + // Add all pinned feed versions to the list of feed versions to be deployed so that they aren't lost as part + // of this update. + for (Deployment.SummarizedFeedVersion pinnedFeedVersion : pinnedFeedVersions) { + updatedFeedVersionIds.add(pinnedFeedVersion.id); + } + + // Check if the updated feed versions have any difference between the previous ones. If not, and if not + // doing a regularly scheduled update with street data, then don't bother starting a deploy job. + // TODO: add logic for street data update + Set previousFeedVersionIds = new HashSet<>( + previousFeedVersions.stream().map(feedVersion -> feedVersion.id).collect(Collectors.toList()) + ); + if ( + !updatedFeedVersionIds.stream() + .anyMatch(feedVersionId -> !previousFeedVersionIds.contains(feedVersionId)) + ) { + LOG.info("No updated feed versions to deploy for project {}.", project.name); + status.completeSuccessfully("No updated feed versions to deploy."); + return; + } + + // Queue up the deploy job. + if (JobUtils.queueDeployJob(deployment, owner, server) != null) { + LOG.info("Last auto deploy date updated for project {}.", project.name); + // Update the deployment's feed version IDs with the latest (and pinned) feed versions. + deployment.feedVersionIds = updatedFeedVersionIds; + project.lastAutoDeploy = new Date(); + Persistence.deployments.replace(deployment.id, deployment); + Persistence.projects.replace(project.id, project); + status.completeSuccessfully("Auto deploy started new deploy job."); + } else { + String message = String.format( + "Auto-deployment to %s should occur after active deployment for project %s completes.", + server.name, + project.name ); - } finally { - lockedProjects.remove(project.id); - LOG.info("Auto deploy lock removed for project id: {}", project.id); + LOG.info(message); + status.completeSuccessfully(message); } } } diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/AutoPublishJob.java b/src/main/java/com/conveyal/datatools/manager/jobs/AutoPublishJob.java index e46e82940..c6f1c95fd 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/AutoPublishJob.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/AutoPublishJob.java @@ -1,6 +1,5 @@ package com.conveyal.datatools.manager.jobs; -import com.conveyal.datatools.common.status.MonitorableJob; import com.conveyal.datatools.manager.auth.Auth0UserProfile; import com.conveyal.datatools.manager.controllers.api.FeedVersionController; import com.conveyal.datatools.manager.gtfsplus.GtfsPlusValidation; @@ -9,10 +8,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.Collections; -import java.util.HashSet; -import java.util.Set; - /** * Auto publish the latest feed versions of a feed source if: * - there are no blocking validation errors, and. @@ -20,87 +15,48 @@ * This class assumes that feed attributes such as autoPublish and retrievalMethod * have been checked. */ -public class AutoPublishJob extends MonitorableJob { +public class AutoPublishJob extends MonitorableJobWithResourceLock { public static final Logger LOG = LoggerFactory.getLogger(AutoPublishJob.class); - /** - * Feed source to be considered for auto-publishing. - */ - private final FeedSource feedSource; - - /** - * A set of projects which have been locked by a instance of {@link AutoPublishJob} to prevent repeat - * auto-publishing. - */ - private static final Set lockedFeedSources = Collections.synchronizedSet(new HashSet<>()); - /** * Auto-publish latest feed from specific feed source. */ public AutoPublishJob(FeedSource feedSource, Auth0UserProfile owner) { - super(owner, "Auto-Publish Feed", JobType.AUTO_PUBLISH_FEED_VERSION); - this.feedSource = feedSource; + super( + owner, + "Auto-Publish Feed", + JobType.AUTO_PUBLISH_FEED_VERSION, + feedSource, + feedSource.name + ); } @Override - public void jobLogic() { - // Define if project and feed source are candidates for auto publish. - if ( - lockedFeedSources.contains(feedSource.id) - ) { - String message = String.format( - "Feed source %s skipped for auto publishing (another publishing job is in progress)", - feedSource.name - ); - LOG.info(message); - status.fail(message); - return; - } + protected void innerJobLogic() throws Exception { + FeedSource feedSource = super.resource; + LOG.info("Auto-publish task running for feed source {}", feedSource.name); - try { - synchronized (lockedFeedSources) { - if (!lockedFeedSources.contains(feedSource.id)) { - lockedFeedSources.add(feedSource.id); - LOG.info("Auto-publish lock added for feed source id: {}", feedSource.id); - } else { - LOG.warn("Unable to acquire lock for feed source {}", feedSource.name); - status.fail(String.format("Feed source %s is locked for auto-publishing.", feedSource.name)); - return; - } - } - LOG.info("Auto-publish task running for feed source {}", feedSource.name); - - // Retrieve the latest feed version associated with the feed source of the current - // feed version set for the deployment. - FeedVersion latestFeedVersion = feedSource.retrieveLatest(); + // Retrieve the latest feed version associated with the feed source of the current + // feed version set for the deployment. + FeedVersion latestFeedVersion = feedSource.retrieveLatest(); - // Validate and check for blocking issues in the feed version to deploy. - if (latestFeedVersion.hasBlockingIssuesForPublishing()) { - status.fail("Could not publish this feed version because it contains blocking errors."); - } + // Validate and check for blocking issues in the feed version to deploy. + if (latestFeedVersion.hasBlockingIssuesForPublishing()) { + status.fail("Could not publish this feed version because it contains blocking errors."); + } - try { - GtfsPlusValidation gtfsPlusValidation = GtfsPlusValidation.validate(latestFeedVersion.id); - if (!gtfsPlusValidation.issues.isEmpty()) { - status.fail("Could not publish this feed version because it contains GTFS+ blocking errors."); - } - } catch(Exception e) { - status.fail("Could not read GTFS+ zip file", e); + try { + GtfsPlusValidation gtfsPlusValidation = GtfsPlusValidation.validate(latestFeedVersion.id); + if (!gtfsPlusValidation.issues.isEmpty()) { + status.fail("Could not publish this feed version because it contains GTFS+ blocking errors."); } - - - // If validation successful, just execute the feed updating process. - // FIXME: move method to another class. - FeedVersionController.publishToExternalResource(latestFeedVersion); - LOG.info("Auto-published feed source {} to external resource.", feedSource.id); - } catch (Exception e) { - status.fail( - String.format("Could not auto-publish feed source %s!", feedSource.name), - e - ); - } finally { - lockedFeedSources.remove(feedSource.id); - LOG.info("Auto deploy lock removed for project id: {}", feedSource.id); + } catch(Exception e) { + status.fail("Could not read GTFS+ zip file", e); } + + // If validation successful, just execute the feed updating process. + // FIXME: move method to another class. + FeedVersionController.publishToExternalResource(latestFeedVersion); + LOG.info("Auto-published feed source {} to external resource.", feedSource.id); } } diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/MonitorableJobWithResourceLock.java b/src/main/java/com/conveyal/datatools/manager/jobs/MonitorableJobWithResourceLock.java new file mode 100644 index 000000000..7873d4982 --- /dev/null +++ b/src/main/java/com/conveyal/datatools/manager/jobs/MonitorableJobWithResourceLock.java @@ -0,0 +1,86 @@ +package com.conveyal.datatools.manager.jobs; + +import com.conveyal.datatools.common.status.MonitorableJob; +import com.conveyal.datatools.manager.auth.Auth0UserProfile; +import com.conveyal.datatools.manager.models.Model; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; + +/** + * Contains logic to lock/release feeds and other objects to ensure + * that jobs on such resources are not executed concurrently. + */ +public abstract class MonitorableJobWithResourceLock extends MonitorableJob { + public static final Logger LOG = LoggerFactory.getLogger(MonitorableJobWithResourceLock.class); + + protected final T resource; + private final String resourceName; + private final String resourceClass; + private final String jobClass; + + /** + * A set of resources (ids) which have been locked by a instance of {@link MonitorableJobWithResourceLock} + * to prevent repeat auto-deploy, auto-publishing, etc. + */ + private static final Set lockedResources = Collections.synchronizedSet(new HashSet<>()); + + protected MonitorableJobWithResourceLock( + Auth0UserProfile owner, + String name, + JobType jobType, + T resource, + String resourceName + ) { + super(owner, name, jobType); + this.resource = resource; + this.resourceName = resourceName; + resourceClass = resource.getClass().getSimpleName(); + jobClass = this.getClass().getSimpleName(); + } + + protected abstract void innerJobLogic() throws Exception; + + @Override + public void jobLogic() { + // Determine if the resource is not locked for this job. + if ( + lockedResources.contains(resource.id) + ) { + String message = String.format( + "%s '%s' skipped for %s execution (another such job is in progress)", + resourceClass, + resourceName, + jobClass + ); + LOG.info(message); + status.fail(message); + return; + } + + try { + synchronized (lockedResources) { + if (!lockedResources.contains(resource.id)) { + lockedResources.add(resource.id); + LOG.info("{} lock added for {} id '{}'", jobClass, resourceClass, resource.id); + } else { + LOG.warn("Unable to acquire lock for {} '{}'", resourceClass, resourceName); + status.fail(String.format("%s '%s' is locked for %s.", resourceClass, resourceName, jobClass)); + return; + } + } + innerJobLogic(); + } catch (Exception e) { + status.fail( + String.format("%s failed for %s '%s'!", jobClass, resourceClass, resourceName), + e + ); + } finally { + lockedResources.remove(resource.id); + LOG.info("{} lock removed for {} id: '{}'", jobClass, resourceClass, resource.id); + } + } +} From 1852cd59e907614efe54997b2f2ed8e1c9128e22 Mon Sep 17 00:00:00 2001 From: binh-dam-ibigroup <56846598+binh-dam-ibigroup@users.noreply.github.com> Date: Fri, 3 Dec 2021 16:32:05 -0500 Subject: [PATCH 088/122] fix(MtcFeedResource): Avoid NullPointerException if ExternalFeedSourceProperty->AgencyId is set to n --- .../extensions/mtc/MtcFeedResource.java | 2 +- .../extensions/mtc/MtcFeedResourceTest.java | 23 +++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/conveyal/datatools/manager/extensions/mtc/MtcFeedResource.java b/src/main/java/com/conveyal/datatools/manager/extensions/mtc/MtcFeedResource.java index 0a5a18b6f..cf5c10028 100644 --- a/src/main/java/com/conveyal/datatools/manager/extensions/mtc/MtcFeedResource.java +++ b/src/main/java/com/conveyal/datatools/manager/extensions/mtc/MtcFeedResource.java @@ -193,7 +193,7 @@ public void feedVersionCreated( constructId(feedVersion.parentFeedSource(), this.getResourceType(), AGENCY_ID_FIELDNAME) ); - if(agencyIdProp == null || agencyIdProp.value.equals("null")) { + if(agencyIdProp == null || agencyIdProp.value == null || agencyIdProp.value.equals("null")) { LOG.error("Could not read {} for FeedSource {}", AGENCY_ID_FIELDNAME, feedVersion.feedSourceId); return; } diff --git a/src/test/java/com/conveyal/datatools/manager/extensions/mtc/MtcFeedResourceTest.java b/src/test/java/com/conveyal/datatools/manager/extensions/mtc/MtcFeedResourceTest.java index 8358a03c5..36f92d1d2 100644 --- a/src/test/java/com/conveyal/datatools/manager/extensions/mtc/MtcFeedResourceTest.java +++ b/src/test/java/com/conveyal/datatools/manager/extensions/mtc/MtcFeedResourceTest.java @@ -4,6 +4,7 @@ import com.conveyal.datatools.UnitTest; import com.conveyal.datatools.manager.models.ExternalFeedSourceProperty; import com.conveyal.datatools.manager.models.FeedSource; +import com.conveyal.datatools.manager.models.FeedVersion; import com.conveyal.datatools.manager.models.Project; import com.conveyal.datatools.manager.persistence.Persistence; import com.fasterxml.jackson.databind.JsonNode; @@ -15,7 +16,9 @@ import java.io.IOException; import java.util.Date; +import static com.conveyal.datatools.TestUtils.createFeedVersion; import static com.conveyal.datatools.TestUtils.parseJson; +import static com.conveyal.datatools.TestUtils.zipFolderFiles; import static com.github.tomakehurst.wiremock.client.WireMock.aResponse; import static com.github.tomakehurst.wiremock.client.WireMock.get; import static com.github.tomakehurst.wiremock.client.WireMock.urlPathEqualTo; @@ -27,6 +30,7 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; class MtcFeedResourceTest extends UnitTest { private static Project project; @@ -152,4 +156,23 @@ void canUpdateFeedExternalPropertiesToMongo() throws IOException { ExternalFeedSourceProperty removedPublicIdProp = Persistence.externalFeedSourceProperties.getById(agencyPublicIdProp.id); assertThat(removedPublicIdProp, nullValue()); } + + @Test + void shouldTolerateNullObjectInExternalPropertyAgencyId() throws IOException { + // Add an entry in the ExternalFeedSourceProperties collection + // with AgencyId value set to null. + ExternalFeedSourceProperty agencyIdProp = new ExternalFeedSourceProperty( + feedSource, + "MTC", + "AgencyId", + null + ); + agencyIdProp.feedSourceId = feedSource.id; + Persistence.externalFeedSourceProperties.create(agencyIdProp); + + // Trigger the feed update process (it should not upload anything to S3). + FeedVersion feedVersion = createFeedVersion(feedSource, zipFolderFiles("mini-bart-new")); + MtcFeedResource mtcFeedResource = new MtcFeedResource(); + assertDoesNotThrow(() -> mtcFeedResource.feedVersionCreated(feedVersion, null)); + } } From 9f92311c203988cc7c97d200d95dcda211e5b0e1 Mon Sep 17 00:00:00 2001 From: miles-grant-ibi Date: Mon, 6 Dec 2021 16:52:43 +0100 Subject: [PATCH 089/122] refactor(Deployment): move pelias update to own method and endpoint --- .../controllers/api/DeploymentController.java | 19 +++++++++++++++++++ .../datatools/manager/jobs/DeployJob.java | 10 ---------- .../manager/jobs/PeliasUpdateJob.java | 13 +++++++++++++ .../datatools/manager/models/Deployment.java | 1 - 4 files changed, 32 insertions(+), 11 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/controllers/api/DeploymentController.java b/src/main/java/com/conveyal/datatools/manager/controllers/api/DeploymentController.java index 7ef59d772..1094d7796 100644 --- a/src/main/java/com/conveyal/datatools/manager/controllers/api/DeploymentController.java +++ b/src/main/java/com/conveyal/datatools/manager/controllers/api/DeploymentController.java @@ -10,6 +10,7 @@ import com.conveyal.datatools.common.utils.aws.S3Utils; import com.conveyal.datatools.manager.auth.Auth0UserProfile; import com.conveyal.datatools.manager.jobs.DeployJob; +import com.conveyal.datatools.manager.jobs.PeliasUpdateJob; import com.conveyal.datatools.manager.models.Deployment; import com.conveyal.datatools.manager.models.EC2InstanceSummary; import com.conveyal.datatools.manager.models.FeedSource; @@ -485,6 +486,23 @@ private static String deploy (Request req, Response res) { return SparkUtils.formatJobMessage(job.jobId, "Deployment initiating."); } + /** + * Create a Pelias update job based on an existing, live deployment + */ + private static String peliasUpdate (Request req, Response res) { + Auth0UserProfile userProfile = req.attribute("user"); + Deployment deployment = getDeploymentWithPermissions(req, res); + Project project = Persistence.projects.getById(deployment.projectId); + if (project == null) { + logMessageAndHalt(req, 400, "Internal reference error. Deployment's project ID is invalid"); + } + + // Execute the pelias update job and keep track of it + PeliasUpdateJob peliasUpdateJob = new PeliasUpdateJob(userProfile, "Updating Custom Geocoder Database", deployment); + JobUtils.heavyExecutor.execute(peliasUpdateJob); + return SparkUtils.formatJobMessage(peliasUpdateJob.jobId, "Pelias update initiating."); + } + /** * Uploads a file from Spark request object to the s3 bucket of the deployment the Pelias Update Job is associated with. * Follows https://github.com/ibi-group/datatools-server/blob/dev/src/main/java/com/conveyal/datatools/editor/controllers/api/EditorController.java#L111 @@ -537,6 +555,7 @@ public static void register (String apiPrefix) { fullJson.addMixin(Deployment.class, Deployment.DeploymentWithEc2InstancesMixin.class); post(apiPrefix + "secure/deployments/:id/deploy/:target", DeploymentController::deploy, slimJson::write); + post(apiPrefix + "secure/deployments/:id/updatepelias", DeploymentController::peliasUpdate, slimJson::write); post(apiPrefix + "secure/deployments/:id/deploy/", ((request, response) -> { logMessageAndHalt(request, 400, "Must provide valid deployment target name"); return null; diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/DeployJob.java b/src/main/java/com/conveyal/datatools/manager/jobs/DeployJob.java index cf557888e..78ef55ea3 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/DeployJob.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/DeployJob.java @@ -415,16 +415,6 @@ else if ("true".equals(System.getenv("RUN_E2E"))) { status.baseUrl = otpServer.publicUrl; } - // Now that the build + deployment was successful, update Pelias - if (deployment.peliasUpdate) { - // Get log upload URI from deploy job - AmazonS3URI logUploadS3URI = getS3FolderURI(); - - // Execute the pelias update job and keep track of it - PeliasUpdateJob peliasUpdateJob = new PeliasUpdateJob(owner, "Updating Local Places Index", deployment, logUploadS3URI); - addNextJob(peliasUpdateJob); - } - status.completed = true; } diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/PeliasUpdateJob.java b/src/main/java/com/conveyal/datatools/manager/jobs/PeliasUpdateJob.java index f4d358c5f..0f127d2c4 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/PeliasUpdateJob.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/PeliasUpdateJob.java @@ -6,6 +6,7 @@ import com.conveyal.datatools.manager.auth.Auth0UserProfile; import com.conveyal.datatools.manager.models.Deployment; import com.conveyal.datatools.manager.models.FeedVersion; +import com.conveyal.datatools.manager.models.OtpServer; import com.conveyal.datatools.manager.persistence.Persistence; import com.conveyal.datatools.manager.utils.HttpUtils; import com.conveyal.datatools.manager.utils.SimpleHttpResponse; @@ -58,6 +59,18 @@ public PeliasUpdateJob(Auth0UserProfile owner, String name, Deployment deploymen this.timer = new Timer(); this.logUploadS3URI = logUploadS3URI; } + public PeliasUpdateJob(Auth0UserProfile owner, String name, Deployment deployment) { + super(owner, name, JobType.UPDATE_PELIAS); + this.deployment = deployment; + this.timer = new Timer(); + + if (deployment.deployJobSummaries.size() <= 0) { + throw new RuntimeException("Deployment must be deployed to at least one server to update Pelias!"); + } + + // Get log upload URI from deployment (the latest build artifacts folder is where the logs get uploaded to) + this.logUploadS3URI = new AmazonS3URI(deployment.deployJobSummaries.get(deployment.deployJobSummaries.size() - 1).buildArtifactsFolder); + } /** * This method must be overridden by subclasses to perform the core steps of the job. diff --git a/src/main/java/com/conveyal/datatools/manager/models/Deployment.java b/src/main/java/com/conveyal/datatools/manager/models/Deployment.java index 5acd09000..db2ce0a7c 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/Deployment.java +++ b/src/main/java/com/conveyal/datatools/manager/models/Deployment.java @@ -74,7 +74,6 @@ public class Deployment extends Model implements Serializable { private ObjectMapper otpConfigMapper = new ObjectMapper().setSerializationInclusion(Include.NON_NULL); /* Pelias fields, used to determine where/if to send data to the Pelias webhook */ - public boolean peliasUpdate; public boolean peliasResetDb; public List peliasCsvFiles = new ArrayList<>(); From e6225296ea4accc8f78749636423e1b9d3f72d89 Mon Sep 17 00:00:00 2001 From: binh-dam-ibigroup <56846598+binh-dam-ibigroup@users.noreply.github.com> Date: Mon, 6 Dec 2021 20:06:31 -0500 Subject: [PATCH 090/122] improvement(FeedUpdater): Use list of feeds to update post-publishing. --- .../extensions/mtc/MtcFeedResource.java | 12 +- .../datatools/manager/jobs/FeedUpdater.java | 166 +++++++++++++----- .../manager/jobs/AutoPublishJobTest.java | 94 +++++++++- 3 files changed, 226 insertions(+), 46 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/extensions/mtc/MtcFeedResource.java b/src/main/java/com/conveyal/datatools/manager/extensions/mtc/MtcFeedResource.java index cf5c10028..4226897f6 100644 --- a/src/main/java/com/conveyal/datatools/manager/extensions/mtc/MtcFeedResource.java +++ b/src/main/java/com/conveyal/datatools/manager/extensions/mtc/MtcFeedResource.java @@ -48,11 +48,12 @@ public class MtcFeedResource implements ExternalFeedResource { public static final Logger LOG = LoggerFactory.getLogger(MtcFeedResource.class); + public static final String TEST_AGENCY = "test-agency"; + public static final String AGENCY_ID_FIELDNAME = "AgencyId"; + public static final String RESOURCE_TYPE = "MTC"; private String rtdApi, s3Bucket, s3Prefix; - public static final String AGENCY_ID_FIELDNAME = "AgencyId"; - public static final String RESOURCE_TYPE = "MTC"; public MtcFeedResource() { rtdApi = DataManager.getExtensionPropertyAsText(RESOURCE_TYPE, "rtd_api"); s3Bucket = DataManager.getExtensionPropertyAsText(RESOURCE_TYPE, "s3_bucket"); @@ -193,11 +194,16 @@ public void feedVersionCreated( constructId(feedVersion.parentFeedSource(), this.getResourceType(), AGENCY_ID_FIELDNAME) ); - if(agencyIdProp == null || agencyIdProp.value == null || agencyIdProp.value.equals("null")) { + if (agencyIdProp == null || agencyIdProp.value == null || agencyIdProp.value.equals("null")) { LOG.error("Could not read {} for FeedSource {}", AGENCY_ID_FIELDNAME, feedVersion.feedSourceId); return; } + if (agencyIdProp.value.equals(TEST_AGENCY)) { + LOG.info("Skipping S3 upload for unit test."); + return; + } + String keyName = String.format("%s%s.zip", this.s3Prefix, agencyIdProp.value); LOG.info("Pushing to MTC S3 Bucket: s3://{}/{}", s3Bucket, keyName); File file = feedVersion.retrieveGtfsFile(); diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/FeedUpdater.java b/src/main/java/com/conveyal/datatools/manager/jobs/FeedUpdater.java index d1b72933f..67266631f 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/FeedUpdater.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/FeedUpdater.java @@ -13,6 +13,7 @@ import com.conveyal.datatools.manager.persistence.Persistence; import com.conveyal.datatools.manager.utils.HashUtils; import com.google.common.io.ByteStreams; +import org.bson.conversions.Bson; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -35,6 +36,9 @@ import static com.conveyal.datatools.common.utils.Scheduler.schedulerService; import static com.mongodb.client.model.Filters.and; import static com.mongodb.client.model.Filters.eq; +import static com.mongodb.client.model.Filters.lt; +import static com.mongodb.client.model.Filters.ne; +import static com.mongodb.client.model.Filters.or; /** * This class is used to schedule an {@link UpdateFeedsTask}, which will check the specified S3 bucket (and prefix) for @@ -52,16 +56,32 @@ * it in a “failed” folder, yet there is no check by Data Tools to see if the feed landed there. */ public class FeedUpdater { + private static final String TEST_BUCKET = "test-bucket"; + private static final String TEST_COMPLETED_FOLDER = "test-completed"; + private static final Logger LOG = LoggerFactory.getLogger(FeedUpdater.class); + private Map eTagForFeed; private final String feedBucket; private final String bucketFolder; - private static final Logger LOG = LoggerFactory.getLogger(FeedUpdater.class); + private CompletedFeedRetriever completedFeedRetriever; + private List versionsToMarkAsProcessed; + private FeedUpdater(int updateFrequencySeconds, String feedBucket, String bucketFolder) { LOG.info("Setting feed update to check every {} seconds", updateFrequencySeconds); schedulerService.scheduleAtFixedRate(new UpdateFeedsTask(), 0, updateFrequencySeconds, TimeUnit.SECONDS); this.feedBucket = feedBucket; this.bucketFolder = bucketFolder; + this.completedFeedRetriever = new DefaultCompletedFeedRetriever(); + } + + /** + * Constructor used for tests. + */ + private FeedUpdater(CompletedFeedRetriever completedFeedRetriever) { + this.feedBucket = TEST_BUCKET; + this.bucketFolder = TEST_COMPLETED_FOLDER; + this.completedFeedRetriever = completedFeedRetriever; } /** @@ -72,6 +92,13 @@ public static FeedUpdater schedule(int updateFrequencySeconds, String s3Bucket, return new FeedUpdater(updateFrequencySeconds, s3Bucket, s3Prefix); } + /** + * Helper method used in tests to create a {@link FeedUpdater}. + */ + public static FeedUpdater createForTest(CompletedFeedRetriever completedFeedRetriever) { + return new FeedUpdater(completedFeedRetriever); + } + private class UpdateFeedsTask implements Runnable { public void run() { Map updatedTags; @@ -92,54 +119,55 @@ public void run() { * objects in order to keep data-tools application in sync with external processes (for example, MTC RTD). * @return map of feedIDs to eTag values */ - private Map checkForUpdatedFeeds() { + public Map checkForUpdatedFeeds() { if (eTagForFeed == null) { // If running the check for the first time, instantiate the eTag map. LOG.info("Running initial check for feeds on S3."); eTagForFeed = new HashMap<>(); } + + // The feed versions that need to be marked as processed are versions where + // all conditions below apply: + // - sentToExternalPublisher is not null, + // - an entry for that version is in objectSummaries below, + // - processedByExternalPublisher is null or before sentToExternalPublisher. + Bson query = and( + ne("sentToExternalPublisher", null), + or( + eq("processedByExternalPublisher", null), + lt("processedByExternalPublisher", "sentToExternalPublisher") + ) + ); + versionsToMarkAsProcessed = Persistence.feedVersions.getFiltered(query) + .stream() + .map(v -> v.id) + .collect(Collectors.toList()); + LOG.debug("Checking for feeds on S3."); Map newTags = new HashMap<>(); // iterate over feeds in download_prefix folder and register to (MTC project) - ObjectListing gtfsList = null; - try { - gtfsList = S3Utils.getDefaultS3Client().listObjects(feedBucket, bucketFolder); - } catch (AmazonServiceException | CheckedAWSException e) { - LOG.error("Failed to list S3 Objects", e); + List objectSummaries = completedFeedRetriever.retrieveCompletedFeeds(); + if (objectSummaries == null) { return newTags; } - LOG.debug(eTagForFeed.toString()); - for (S3ObjectSummary objSummary : gtfsList.getObjectSummaries()) { + LOG.debug(eTagForFeed.toString()); + for (S3ObjectSummary objSummary : objectSummaries) { String eTag = objSummary.getETag(); String keyName = objSummary.getKey(); LOG.debug("{} etag = {}", keyName, eTag); - if (!eTagForFeed.containsValue(eTag)) { + + if (keyName.equals(bucketFolder)) continue; + String filename = keyName.split("/")[1]; + String feedId = filename.replace(".zip", ""); + FeedSource feedSource = getFeedSource(feedId); + + if (shouldMarkFeedAsProcessed(eTag, feedSource)) { // Don't add object if it is a dir - if (keyName.equals(bucketFolder)) continue; - String filename = keyName.split("/")[1]; - String feedId = filename.replace(".zip", ""); // Skip object if the filename is null if ("null".equals(feedId)) continue; try { LOG.info("New version found for {} at s3://{}/{}. ETag = {}.", feedId, feedBucket, keyName, eTag); - FeedSource feedSource = null; - List properties = Persistence.externalFeedSourceProperties.getFiltered( - and(eq("value", feedId), eq("name", AGENCY_ID_FIELDNAME)) - ); - if (properties.size() > 1) { - StringBuilder b = new StringBuilder(); - properties.forEach(b::append); - LOG.warn("Found multiple feed sources for {}: {}", - feedId, - properties.stream().map(p -> p.feedSourceId).collect(Collectors.joining(","))); - } - for (ExternalFeedSourceProperty prop : properties) { - // FIXME: What if there are multiple props found for different feed sources. This could happen if - // multiple projects have been synced with MTC or if the ExternalFeedSourceProperty for a feed - // source is not deleted properly when the feed source is deleted. - feedSource = Persistence.feedSources.getById(prop.feedSourceId); - } if (feedSource == null) { LOG.error("No feed source found for feed ID {}", feedId); continue; @@ -162,24 +190,46 @@ private Map checkForUpdatedFeeds() { return newTags; } + private FeedSource getFeedSource(String feedId) { + FeedSource feedSource = null; + List properties = Persistence.externalFeedSourceProperties.getFiltered( + and(eq("value", feedId), eq("name", AGENCY_ID_FIELDNAME)) + ); + if (properties.size() > 1) { + StringBuilder b = new StringBuilder(); + properties.forEach(b::append); + LOG.warn("Found multiple feed sources for {}: {}", + feedId, + properties.stream().map(p -> p.feedSourceId).collect(Collectors.joining(","))); + } + for (ExternalFeedSourceProperty prop : properties) { + // FIXME: What if there are multiple props found for different feed sources. This could happen if + // multiple projects have been synced with MTC or if the ExternalFeedSourceProperty for a feed + // source is not deleted properly when the feed source is deleted. + feedSource = Persistence.feedSources.getById(prop.feedSourceId); + } + return feedSource; + } + + /** + * @return true if the feed with the corresponding etag should be mark as processed, false otherwise. + */ + private boolean shouldMarkFeedAsProcessed(String eTag, FeedSource feedSource) { + if (eTagForFeed.containsValue(eTag)) return false; + + FeedVersion publishedVersion = getFeedVersionToUpdate(feedSource); + return versionsToMarkAsProcessed.contains(publishedVersion.id); + } + /** * Update the published feed version for the feed source. * @param feedId the unique ID used by MTC to identify a feed source * @param feedSource the feed source for which a newly published version should be registered */ private void updatePublishedFeedVersion(String feedId, FeedSource feedSource) { - // Collect the feed versions for the feed source. - Collection versions = feedSource.retrieveFeedVersions(); try { - // Get the latest published version (if there is one). NOTE: This is somewhat flawed because it presumes - // that the latest published version is guaranteed to be the one found in the "completed" folder, but it - // could be that more than one versions were recently "published" and the latest published version was a bad - // feed that failed processing by RTD. - Optional lastPublishedVersionCandidate = versions - .stream() - .min(Comparator.comparing(v -> v.sentToExternalPublisher, Comparator.nullsLast(Comparator.reverseOrder()))); - if (lastPublishedVersionCandidate.isPresent()) { - FeedVersion publishedVersion = lastPublishedVersionCandidate.get(); + FeedVersion publishedVersion = getFeedVersionToUpdate(feedSource); + if (publishedVersion != null) { if (publishedVersion.sentToExternalPublisher == null) { LOG.warn("Not updating published version for {} (version was never sent to external publisher)", feedId); return; @@ -197,6 +247,24 @@ private void updatePublishedFeedVersion(String feedId, FeedSource feedSource) { } } + /** + * Get the latest published version (if there is one). NOTE: This is somewhat flawed because it presumes + * that the latest published version is guaranteed to be the one found in the "completed" folder, but it + * could be that more than one versions were recently "published" and the latest published version was a bad + * feed that failed processing by RTD. + */ + private static FeedVersion getFeedVersionToUpdate(FeedSource feedSource) { + // Collect the feed versions for the feed source. + Collection versions = feedSource.retrieveFeedVersions(); + Optional lastPublishedVersionCandidate = versions + .stream() + .min(Comparator.comparing(v -> v.sentToExternalPublisher, Comparator.nullsLast(Comparator.reverseOrder()))); + if (lastPublishedVersionCandidate.isPresent()) { + return lastPublishedVersionCandidate.get(); + } + return null; + } + /** * NOTE: This method is not in use, but should be strongly considered as an alternative approach if/when RTD is able * to maintain md5 checksums when copying a file from "waiting" folder to "completed". @@ -238,4 +306,20 @@ private FeedVersion findMatchingFeedVersion( return matchingVersion; } + public interface CompletedFeedRetriever { + List retrieveCompletedFeeds(); + } + + public class DefaultCompletedFeedRetriever implements CompletedFeedRetriever { + @Override + public List retrieveCompletedFeeds() { + try { + ObjectListing gtfsList = S3Utils.getDefaultS3Client().listObjects(feedBucket, bucketFolder); + return gtfsList.getObjectSummaries(); + } catch (CheckedAWSException e) { + LOG.error("Failed to list S3 Objects", e); + return null; + } + } + } } diff --git a/src/test/java/com/conveyal/datatools/manager/jobs/AutoPublishJobTest.java b/src/test/java/com/conveyal/datatools/manager/jobs/AutoPublishJobTest.java index 69cad7aee..6b93fd1e6 100644 --- a/src/test/java/com/conveyal/datatools/manager/jobs/AutoPublishJobTest.java +++ b/src/test/java/com/conveyal/datatools/manager/jobs/AutoPublishJobTest.java @@ -1,31 +1,43 @@ package com.conveyal.datatools.manager.jobs; +import com.amazonaws.services.s3.model.S3ObjectSummary; import com.conveyal.datatools.DatatoolsTest; import com.conveyal.datatools.UnitTest; import com.conveyal.datatools.manager.auth.Auth0UserProfile; +import com.conveyal.datatools.manager.models.ExternalFeedSourceProperty; import com.conveyal.datatools.manager.models.FeedSource; +import com.conveyal.datatools.manager.models.FeedVersion; import com.conveyal.datatools.manager.models.Project; import com.conveyal.datatools.manager.persistence.Persistence; +import com.google.common.collect.Lists; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; import java.io.IOException; +import java.util.ArrayList; import java.util.Date; +import java.util.List; +import java.util.Map; import java.util.stream.Stream; import static com.conveyal.datatools.TestUtils.createFeedVersion; import static com.conveyal.datatools.TestUtils.createFeedVersionFromGtfsZip; import static com.conveyal.datatools.TestUtils.zipFolderFiles; +import static com.conveyal.datatools.manager.extensions.mtc.MtcFeedResource.TEST_AGENCY; import static com.conveyal.datatools.manager.models.FeedRetrievalMethod.FETCHED_AUTOMATICALLY; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; /** * Tests for the various {@link AutoPublishJob} cases. */ public class AutoPublishJobTest extends UnitTest { + private static final String TEST_COMPLETED_FOLDER = "test-completed"; private static final Auth0UserProfile user = Auth0UserProfile.createTestAdminUser(); private static Project project; private static FeedSource feedSource; @@ -46,6 +58,17 @@ public static void setUp() throws IOException { FeedSource fakeAgency = new FeedSource("Feed source", project.id, FETCHED_AUTOMATICALLY); Persistence.feedSources.create(fakeAgency); feedSource = fakeAgency; + + // Add an AgencyId entry to ExternalFeedSourceProperty + // (one-time, it will be reused for this feed source) + // but set the value to TEST_AGENCY to prevent actual S3 upload. + ExternalFeedSourceProperty agencyIdProp = new ExternalFeedSourceProperty( + feedSource, + "MTC", + "AgencyId", + TEST_AGENCY + ); + Persistence.externalFeedSourceProperties.create(agencyIdProp); } @AfterAll @@ -62,10 +85,11 @@ public static void tearDown() { @MethodSource("createPublishFeedCases") void shouldProcessFeed(String resourceName, boolean isError, String errorMessage) throws IOException { // Add the version to the feed source + FeedVersion createdVersion; if (resourceName.endsWith(".zip")) { - createFeedVersionFromGtfsZip(feedSource, resourceName); + createdVersion = createFeedVersionFromGtfsZip(feedSource, resourceName); } else { - createFeedVersion(feedSource, zipFolderFiles(resourceName)); + createdVersion = createFeedVersion(feedSource, zipFolderFiles(resourceName)); } // Create the job @@ -79,6 +103,7 @@ void shouldProcessFeed(String resourceName, boolean isError, String errorMessage autoPublishJob.status.error, "AutoPublish job error status was incorrectly determined." ); + if (isError) { assertEquals(errorMessage, autoPublishJob.status.message); } @@ -103,4 +128,69 @@ private static Stream createPublishFeedCases() { ) ); } + + @Test + void shouldUpdateFeedInfoAfterPublishComplete() { + // Add the version to the feed source + FeedVersion createdVersion = createFeedVersionFromGtfsZip(feedSource, "bart_new_lite.zip"); + + // Create the job + AutoPublishJob autoPublishJob = new AutoPublishJob(feedSource, user); + + // Run the job in this thread (we're not concerned about concurrency here). + autoPublishJob.run(); + + assertEquals(false, autoPublishJob.status.error); + + // Make sure that the publish-pending attribute has been set for the feed version in Mongo. + FeedVersion updatedFeedVersion = Persistence.feedVersions.getById(createdVersion.id); + assertNotNull(updatedFeedVersion.sentToExternalPublisher); + + // Create a test FeedUpdater instance, and simulate running the task. + TestCompletedFeedRetriever completedFeedRetriever = new TestCompletedFeedRetriever(); + FeedUpdater feedUpdater = FeedUpdater.createForTest(completedFeedRetriever); + + // The list of feeds processed externally (completed) should be empty at this point. + Map etags = feedUpdater.checkForUpdatedFeeds(); + assertTrue(etags.isEmpty()); + + // Simulate completion of feed publishing. + completedFeedRetriever.isPublishingComplete = true; + + // The etags should contain the id of the agency. + // If a feed has been republished since last check, it will have a new etag/file hash, + // and the scenario below should apply. + Map etagsAfter = feedUpdater.checkForUpdatedFeeds(); + assertEquals(1, etagsAfter.size()); + assertTrue(etagsAfter.containsValue("test-etag")); + + // Make sure that the publish-complete attribute has been set for the feed version in Mongo. + FeedVersion updatedFeedVersionAfter = Persistence.feedVersions.getById(createdVersion.id); + Date updatedDate = updatedFeedVersionAfter.processedByExternalPublisher; + String namespace = updatedFeedVersionAfter.namespace; + assertNotNull(updatedDate); + + // At the next check for updates, the metadata for the feeds completed above + // should not be updated again. + feedUpdater.checkForUpdatedFeeds(); + FeedVersion updatedFeedVersionAfter2 = Persistence.feedVersions.getById(createdVersion.id); + assertEquals(updatedDate, updatedFeedVersionAfter2.processedByExternalPublisher); + assertEquals(namespace, updatedFeedVersionAfter2.namespace); + } + + private static class TestCompletedFeedRetriever implements FeedUpdater.CompletedFeedRetriever { + public boolean isPublishingComplete; + + @Override + public List retrieveCompletedFeeds() { + if (!isPublishingComplete) { + return new ArrayList<>(); + } else { + S3ObjectSummary objSummary = new S3ObjectSummary(); + objSummary.setETag("test-etag"); + objSummary.setKey(String.format("%s/%s", TEST_COMPLETED_FOLDER, TEST_AGENCY)); + return Lists.newArrayList(objSummary); + } + } + } } From 28e80985dfd138aba00b728179079d31e4e91687 Mon Sep 17 00:00:00 2001 From: binh-dam-ibigroup <56846598+binh-dam-ibigroup@users.noreply.github.com> Date: Mon, 6 Dec 2021 20:34:00 -0500 Subject: [PATCH 091/122] refactor(FeedUpdater): Refactor per sonar lint, add/tweak comments. --- .../manager/jobs/AutoPublishJob.java | 1 - .../datatools/manager/jobs/FeedUpdater.java | 40 +++++++++++-------- .../manager/jobs/AutoPublishJobTest.java | 12 ++++-- 3 files changed, 31 insertions(+), 22 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/AutoPublishJob.java b/src/main/java/com/conveyal/datatools/manager/jobs/AutoPublishJob.java index c6f1c95fd..69eab49ad 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/AutoPublishJob.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/AutoPublishJob.java @@ -55,7 +55,6 @@ protected void innerJobLogic() throws Exception { } // If validation successful, just execute the feed updating process. - // FIXME: move method to another class. FeedVersionController.publishToExternalResource(latestFeedVersion); LOG.info("Auto-published feed source {} to external resource.", feedSource.id); } diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/FeedUpdater.java b/src/main/java/com/conveyal/datatools/manager/jobs/FeedUpdater.java index 67266631f..d363309f8 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/FeedUpdater.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/FeedUpdater.java @@ -59,11 +59,13 @@ public class FeedUpdater { private static final String TEST_BUCKET = "test-bucket"; private static final String TEST_COMPLETED_FOLDER = "test-completed"; private static final Logger LOG = LoggerFactory.getLogger(FeedUpdater.class); + public static final String SENT_TO_EXTERNAL_PUBLISHER_FIELD = "sentToExternalPublisher"; + public static final String PROCESSED_BY_EXTERNAL_PUBLISHER_FIELD = "processedByExternalPublisher"; private Map eTagForFeed; private final String feedBucket; private final String bucketFolder; - private CompletedFeedRetriever completedFeedRetriever; + private final CompletedFeedRetriever completedFeedRetriever; private List versionsToMarkAsProcessed; @@ -126,16 +128,15 @@ public Map checkForUpdatedFeeds() { eTagForFeed = new HashMap<>(); } - // The feed versions that need to be marked as processed are versions where - // all conditions below apply: + // The feed versions corresponding to entries in objectSummaries + // that need to be marked as processed should meet all conditions below: // - sentToExternalPublisher is not null, - // - an entry for that version is in objectSummaries below, // - processedByExternalPublisher is null or before sentToExternalPublisher. Bson query = and( - ne("sentToExternalPublisher", null), + ne(SENT_TO_EXTERNAL_PUBLISHER_FIELD, null), or( - eq("processedByExternalPublisher", null), - lt("processedByExternalPublisher", "sentToExternalPublisher") + eq(PROCESSED_BY_EXTERNAL_PUBLISHER_FIELD, null), + lt(PROCESSED_BY_EXTERNAL_PUBLISHER_FIELD, SENT_TO_EXTERNAL_PUBLISHER_FIELD) ) ); versionsToMarkAsProcessed = Persistence.feedVersions.getFiltered(query) @@ -190,14 +191,15 @@ public Map checkForUpdatedFeeds() { return newTags; } + /** + * Obtains the {@link FeedSource} for the given feed id (for MTC, that's the 2-letter agency code). + */ private FeedSource getFeedSource(String feedId) { FeedSource feedSource = null; List properties = Persistence.externalFeedSourceProperties.getFiltered( and(eq("value", feedId), eq("name", AGENCY_ID_FIELDNAME)) ); if (properties.size() > 1) { - StringBuilder b = new StringBuilder(); - properties.forEach(b::append); LOG.warn("Found multiple feed sources for {}: {}", feedId, properties.stream().map(p -> p.feedSourceId).collect(Collectors.joining(","))); @@ -217,7 +219,8 @@ private FeedSource getFeedSource(String feedId) { private boolean shouldMarkFeedAsProcessed(String eTag, FeedSource feedSource) { if (eTagForFeed.containsValue(eTag)) return false; - FeedVersion publishedVersion = getFeedVersionToUpdate(feedSource); + FeedVersion publishedVersion = getLatestPublishedVersion(feedSource); + if (publishedVersion == null) return false; return versionsToMarkAsProcessed.contains(publishedVersion.id); } @@ -228,7 +231,7 @@ private boolean shouldMarkFeedAsProcessed(String eTag, FeedSource feedSource) { */ private void updatePublishedFeedVersion(String feedId, FeedSource feedSource) { try { - FeedVersion publishedVersion = getFeedVersionToUpdate(feedSource); + FeedVersion publishedVersion = getLatestPublishedVersion(feedSource); if (publishedVersion != null) { if (publishedVersion.sentToExternalPublisher == null) { LOG.warn("Not updating published version for {} (version was never sent to external publisher)", feedId); @@ -236,7 +239,7 @@ private void updatePublishedFeedVersion(String feedId, FeedSource feedSource) { } // Set published namespace to the feed version and set the processedByExternalPublisher timestamp. LOG.info("Latest published version (sent at {}) for {} is {}", publishedVersion.sentToExternalPublisher, feedId, publishedVersion.id); - Persistence.feedVersions.updateField(publishedVersion.id, "processedByExternalPublisher", new Date()); + Persistence.feedVersions.updateField(publishedVersion.id, PROCESSED_BY_EXTERNAL_PUBLISHER_FIELD, new Date()); Persistence.feedSources.updateField(feedSource.id, "publishedVersionId", publishedVersion.namespace); } else { LOG.error("No published versions found for {} ({} id={})", feedId, feedSource.name, feedSource.id); @@ -253,16 +256,13 @@ private void updatePublishedFeedVersion(String feedId, FeedSource feedSource) { * could be that more than one versions were recently "published" and the latest published version was a bad * feed that failed processing by RTD. */ - private static FeedVersion getFeedVersionToUpdate(FeedSource feedSource) { + private static FeedVersion getLatestPublishedVersion(FeedSource feedSource) { // Collect the feed versions for the feed source. Collection versions = feedSource.retrieveFeedVersions(); Optional lastPublishedVersionCandidate = versions .stream() .min(Comparator.comparing(v -> v.sentToExternalPublisher, Comparator.nullsLast(Comparator.reverseOrder()))); - if (lastPublishedVersionCandidate.isPresent()) { - return lastPublishedVersionCandidate.get(); - } - return null; + return lastPublishedVersionCandidate.orElse(null); } /** @@ -306,10 +306,16 @@ private FeedVersion findMatchingFeedVersion( return matchingVersion; } + /** + * Helper interface for fetching a list of feeds deemed production-complete. + */ public interface CompletedFeedRetriever { List retrieveCompletedFeeds(); } + /** + * Implements the default behavior for above interface. + */ public class DefaultCompletedFeedRetriever implements CompletedFeedRetriever { @Override public List retrieveCompletedFeeds() { diff --git a/src/test/java/com/conveyal/datatools/manager/jobs/AutoPublishJobTest.java b/src/test/java/com/conveyal/datatools/manager/jobs/AutoPublishJobTest.java index 6b93fd1e6..d3ad4868d 100644 --- a/src/test/java/com/conveyal/datatools/manager/jobs/AutoPublishJobTest.java +++ b/src/test/java/com/conveyal/datatools/manager/jobs/AutoPublishJobTest.java @@ -30,6 +30,7 @@ import static com.conveyal.datatools.manager.extensions.mtc.MtcFeedResource.TEST_AGENCY; import static com.conveyal.datatools.manager.models.FeedRetrievalMethod.FETCHED_AUTOMATICALLY; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -85,11 +86,10 @@ public static void tearDown() { @MethodSource("createPublishFeedCases") void shouldProcessFeed(String resourceName, boolean isError, String errorMessage) throws IOException { // Add the version to the feed source - FeedVersion createdVersion; if (resourceName.endsWith(".zip")) { - createdVersion = createFeedVersionFromGtfsZip(feedSource, resourceName); + createFeedVersionFromGtfsZip(feedSource, resourceName); } else { - createdVersion = createFeedVersion(feedSource, zipFolderFiles(resourceName)); + createFeedVersion(feedSource, zipFolderFiles(resourceName)); } // Create the job @@ -140,7 +140,7 @@ void shouldUpdateFeedInfoAfterPublishComplete() { // Run the job in this thread (we're not concerned about concurrency here). autoPublishJob.run(); - assertEquals(false, autoPublishJob.status.error); + assertFalse(autoPublishJob.status.error); // Make sure that the publish-pending attribute has been set for the feed version in Mongo. FeedVersion updatedFeedVersion = Persistence.feedVersions.getById(createdVersion.id); @@ -178,6 +178,10 @@ void shouldUpdateFeedInfoAfterPublishComplete() { assertEquals(namespace, updatedFeedVersionAfter2.namespace); } + /** + * Mocks the results of an {@link S3ObjectSummary} retrieval before/after the + * external MTC publishing process is complete. + */ private static class TestCompletedFeedRetriever implements FeedUpdater.CompletedFeedRetriever { public boolean isPublishingComplete; From 1f799b8cde550b5d1242b7937b852fc09368cebd Mon Sep 17 00:00:00 2001 From: binh-dam-ibigroup <56846598+binh-dam-ibigroup@users.noreply.github.com> Date: Tue, 7 Dec 2021 08:19:17 -0500 Subject: [PATCH 092/122] test(MtcFeedResource,AutoPublishJob): Clean up after some individual tests. --- .../manager/extensions/mtc/MtcFeedResourceTest.java | 7 ++++++- .../datatools/manager/jobs/AutoPublishJobTest.java | 4 +++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/src/test/java/com/conveyal/datatools/manager/extensions/mtc/MtcFeedResourceTest.java b/src/test/java/com/conveyal/datatools/manager/extensions/mtc/MtcFeedResourceTest.java index 36f92d1d2..16ee55831 100644 --- a/src/test/java/com/conveyal/datatools/manager/extensions/mtc/MtcFeedResourceTest.java +++ b/src/test/java/com/conveyal/datatools/manager/extensions/mtc/MtcFeedResourceTest.java @@ -155,6 +155,10 @@ void canUpdateFeedExternalPropertiesToMongo() throws IOException { // Removed field AgencyPublicId from RTD should be deleted from Mongo. ExternalFeedSourceProperty removedPublicIdProp = Persistence.externalFeedSourceProperties.getById(agencyPublicIdProp.id); assertThat(removedPublicIdProp, nullValue()); + + Persistence.externalFeedSourceProperties.removeById(agencyIdProp.id); + Persistence.externalFeedSourceProperties.removeById(agencyPublicIdProp.id); + Persistence.externalFeedSourceProperties.removeById(agencyEmailProp.id); } @Test @@ -167,12 +171,13 @@ void shouldTolerateNullObjectInExternalPropertyAgencyId() throws IOException { "AgencyId", null ); - agencyIdProp.feedSourceId = feedSource.id; Persistence.externalFeedSourceProperties.create(agencyIdProp); // Trigger the feed update process (it should not upload anything to S3). FeedVersion feedVersion = createFeedVersion(feedSource, zipFolderFiles("mini-bart-new")); MtcFeedResource mtcFeedResource = new MtcFeedResource(); assertDoesNotThrow(() -> mtcFeedResource.feedVersionCreated(feedVersion, null)); + + Persistence.externalFeedSourceProperties.removeById(agencyIdProp.id); } } diff --git a/src/test/java/com/conveyal/datatools/manager/jobs/AutoPublishJobTest.java b/src/test/java/com/conveyal/datatools/manager/jobs/AutoPublishJobTest.java index d3ad4868d..3eec0d981 100644 --- a/src/test/java/com/conveyal/datatools/manager/jobs/AutoPublishJobTest.java +++ b/src/test/java/com/conveyal/datatools/manager/jobs/AutoPublishJobTest.java @@ -42,6 +42,7 @@ public class AutoPublishJobTest extends UnitTest { private static final Auth0UserProfile user = Auth0UserProfile.createTestAdminUser(); private static Project project; private static FeedSource feedSource; + private static ExternalFeedSourceProperty agencyIdProp; /** * Prepare and start a testing-specific web server @@ -63,7 +64,7 @@ public static void setUp() throws IOException { // Add an AgencyId entry to ExternalFeedSourceProperty // (one-time, it will be reused for this feed source) // but set the value to TEST_AGENCY to prevent actual S3 upload. - ExternalFeedSourceProperty agencyIdProp = new ExternalFeedSourceProperty( + agencyIdProp = new ExternalFeedSourceProperty( feedSource, "MTC", "AgencyId", @@ -77,6 +78,7 @@ public static void tearDown() { if (project != null) { project.delete(); } + Persistence.externalFeedSourceProperties.removeById(agencyIdProp.id); } /** From e383af5000bd8c2b786be7a339871be4191169d6 Mon Sep 17 00:00:00 2001 From: binh-dam-ibigroup <56846598+binh-dam-ibigroup@users.noreply.github.com> Date: Tue, 7 Dec 2021 15:15:06 -0500 Subject: [PATCH 093/122] refactor(MergeFeedsJobTest): Refactor table count assertion.. --- .../manager/jobs/MergeFeedsJobTest.java | 219 ++++++------------ 1 file changed, 71 insertions(+), 148 deletions(-) diff --git a/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java b/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java index 2035c3ac9..648a541b2 100644 --- a/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java +++ b/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java @@ -233,12 +233,9 @@ void canMergeRegionalWithOnlyCalendarFeed () throws SQLException { // assert service_ids have been feed scoped properly String mergedNamespace = mergedVersion.namespace; - // - calendar table - // expect a total of 2 records in calendar table - assertThatSqlCountQueryYieldsExpectedCount( - String.format("SELECT count(*) FROM %s.calendar", mergedNamespace), - 2 - ); + // - calendar table should have 2 records. + assertRowCountInTable(mergedNamespace, "calendar", 2); + // onlyCalendarVersion's common_id service_id should be scoped assertThatSqlCountQueryYieldsExpectedCount( String.format( @@ -256,15 +253,9 @@ void canMergeRegionalWithOnlyCalendarFeed () throws SQLException { 1 ); - // - calendar_dates table - // expect only 1 record in calendar_dates table - assertThatSqlCountQueryYieldsExpectedCount( - String.format( - "SELECT count(*) FROM %s.calendar_dates", - mergedNamespace - ), - 2 - ); + // - calendar_dates table should have 2 records. + assertRowCountInTable(mergedNamespace, "calendar_dates", 2); + // onlyCalendarDatesVersion's common_id service_id should be scoped assertThatSqlCountQueryYieldsExpectedCount( String.format( @@ -282,15 +273,9 @@ void canMergeRegionalWithOnlyCalendarFeed () throws SQLException { 1 ); - // - trips table - // expect 2 + 1 = 3 records in trips table - assertThatSqlCountQueryYieldsExpectedCount( - String.format( - "SELECT count(*) FROM %s.trips", - mergedNamespace - ), - 3 - ); + // - trips table should have 2 + 1 = 3 records. + assertRowCountInTable(mergedNamespace, "trips", 3); + // onlyCalendarDatesVersion's common_id service_id should be scoped assertThatSqlCountQueryYieldsExpectedCount( String.format( @@ -357,10 +342,8 @@ void mergeMTCShouldHandleExtendFutureStrategy() throws SQLException { // - calendar table // expect a total of 1 record in calendar table that // corresponds to the trip ids present in both active and future feed. - assertThatSqlCountQueryYieldsExpectedCount( - String.format("SELECT count(*) FROM %s.calendar", mergedNamespace), - 1 - ); + assertRowCountInTable(mergedNamespace, "calendar", 1); + // expect that the record in calendar table has the correct start_date. assertThatSqlCountQueryYieldsExpectedCount( String.format("SELECT count(*) FROM %s.calendar where start_date = '20170918' and monday = 1", mergedNamespace), @@ -401,19 +384,13 @@ void mergeMTCShouldHandleMatchingTripIdsWithSameSignature() throws SQLException // - common_id cloned and extended for the matching trip id present in both active and future feeds // (from MergeFeedsJob#serviceIdsToCloneAndRename), // - only_calendar_id used in the future feed. - assertThatSqlCountQueryYieldsExpectedCount( - String.format("SELECT count(*) FROM %s.calendar", mergedNamespace), - 4 - ); + assertRowCountInTable(mergedNamespace, "calendar", 4); // Out of all trips from the input datasets, expect 4 trips in merged output. // 1 trip from active feed that is not in the future feed, // 1 trip in both the active and future feeds, with the same signature (same stop times), // 2 trips from the future feed not in the active feed. - assertThatSqlCountQueryYieldsExpectedCount( - String.format("SELECT count(*) FROM %s.trips", mergedNamespace), - 4 - ); + assertRowCountInTable(mergedNamespace, "trips", 4); assertNoUnusedServiceIds(mergedNamespace); assertNoRefIntegrityErrors(mergedNamespace); @@ -470,25 +447,16 @@ void mergeMTCShouldHandleMatchingTripIdsAndDropUnusedFutureCalendar() throws SQL // - common_id cloned and extended for the matching trip id present in both active and future feeds // (from MergeFeedsJob#serviceIdsToCloneAndRename), // - only_calendar_id used in the future feed. - assertThatSqlCountQueryYieldsExpectedCount( - String.format("SELECT count(*) FROM %s.calendar", mergedNamespace), - 3 - ); + assertRowCountInTable(mergedNamespace, "calendar", 3); // Out of all trips from the input datasets, expect 4 trips in merged output. // 1 trip from active feed that is not in the future feed, // 1 trip in both the active and future feeds, with the same signature (same stop times), // 1 trip from the future feed not in the active feed. - assertThatSqlCountQueryYieldsExpectedCount( - String.format("SELECT count(*) FROM %s.trips", mergedNamespace), - 3 - ); + assertRowCountInTable(mergedNamespace, "trips", 3); // The calendar_dates entry should be preserved, but remapped to a different id. - assertThatSqlCountQueryYieldsExpectedCount( - String.format("SELECT count(*) FROM %s.calendar_dates", mergedNamespace), - 1 - ); + assertRowCountInTable(mergedNamespace, "calendar_dates", 1); assertNoUnusedServiceIds(mergedNamespace); assertNoRefIntegrityErrors(mergedNamespace); @@ -548,10 +516,8 @@ void mergeMTCShouldHandleDisjointTripIds() throws SQLException { // - 2 records from future feed // - 1 records from active feed that is used // - the unused record from the active feed is discarded. - assertThatSqlCountQueryYieldsExpectedCount( - String.format("SELECT count(*) FROM %s.calendar", mergedNamespace), - 3 - ); + assertRowCountInTable(mergedNamespace, "calendar", 3); + // The one calendar entry for the active feed should end one day before the first calendar start date // of the future feed. final String activeCalendarNewEndDate = "20170919"; // One day before 20170920. @@ -565,10 +531,7 @@ void mergeMTCShouldHandleDisjointTripIds() throws SQLException { // - trips table // expect a total of 4 records in trips table (all records from original files are included). - assertThatSqlCountQueryYieldsExpectedCount( - String.format("SELECT count(*) FROM %s.trips", mergedNamespace), - 4 - ); + assertRowCountInTable(mergedNamespace, "trips", 4); } /** @@ -664,12 +627,9 @@ void canMergeFeedsWithMTCForServiceIds1 () throws SQLException { // assert service_ids have been feed scoped properly String mergedNamespace = mergeFeedsJob.mergedVersion.namespace; - // - calendar table - // expect a total of 4 records in calendar table - assertThatSqlCountQueryYieldsExpectedCount( - String.format("SELECT count(*) FROM %s.calendar", mergedNamespace), - 4 - ); + // - calendar table should have 4 records. + assertRowCountInTable(mergedNamespace, "calendar", 4); + // bothCalendarFilesVersion's common_id service_id should be scoped assertThatSqlCountQueryYieldsExpectedCount( String.format( @@ -703,12 +663,9 @@ void canMergeFeedsWithMTCForServiceIds1 () throws SQLException { 1 ); - // - calendar_dates table - // expect only 2 records in calendar_dates table - assertThatSqlCountQueryYieldsExpectedCount( - String.format("SELECT count(*) FROM %s.calendar_dates", mergedNamespace), - 2 - ); + // - calendar_dates table should have only 2 records. + assertRowCountInTable(mergedNamespace, "calendar_dates", 2); + // bothCalendarFilesVersion's common_id service_id should be scoped assertThatSqlCountQueryYieldsExpectedCount( String.format( @@ -726,12 +683,9 @@ void canMergeFeedsWithMTCForServiceIds1 () throws SQLException { 1 ); - // - trips table - // expect 2 + 1 = 3 records in trips table - assertThatSqlCountQueryYieldsExpectedCount( - String.format("SELECT count(*) FROM %s.trips", mergedNamespace), - 3 - ); + // - trips should have 2 + 1 = 3 records. + assertRowCountInTable(mergedNamespace, "trips", 3); + // bothCalendarFilesVersion's common_id service_id should be scoped assertThatSqlCountQueryYieldsExpectedCount( String.format( @@ -767,12 +721,9 @@ void canMergeFeedsWithMTCForServiceIds2 () throws SQLException { // assert service_ids have been feed scoped properly String mergedNamespace = mergeFeedsJob.mergedVersion.namespace; - // - calendar table - // expect a total of 4 records in calendar table - assertThatSqlCountQueryYieldsExpectedCount( - String.format("SELECT count(*) FROM %s.calendar", mergedNamespace), - 2 - ); + // - calendar table should have 4 records. + assertRowCountInTable(mergedNamespace, "calendar", 2); + // onlyCalendarVersion's common id should not be scoped assertThatSqlCountQueryYieldsExpectedCount( String.format( @@ -790,12 +741,9 @@ void canMergeFeedsWithMTCForServiceIds2 () throws SQLException { 1 ); - // - calendar_dates table - // expect only 2 records in calendar_dates table - assertThatSqlCountQueryYieldsExpectedCount( - String.format("SELECT count(*) FROM %s.calendar_dates", mergedNamespace), - 2 - ); + // - calendar_dates table should have only 2 records. + assertRowCountInTable(mergedNamespace, "calendar_dates", 2); + // onlyCalendarDatesVersion's common_id service_id should be scoped assertThatSqlCountQueryYieldsExpectedCount( String.format( @@ -813,12 +761,9 @@ void canMergeFeedsWithMTCForServiceIds2 () throws SQLException { 1 ); - // - trips table - // expect 2 + 1 = 3 records in trips table - assertThatSqlCountQueryYieldsExpectedCount( - String.format("SELECT count(*) FROM %s.trips", mergedNamespace), - 3 - ); + // - trips table should have 2 + 1 = 3 records. + assertRowCountInTable(mergedNamespace, "trips", 3); + // bothCalendarFilesVersion's common_id service_id should be scoped assertThatSqlCountQueryYieldsExpectedCount( String.format( @@ -855,25 +800,15 @@ void canMergeFeedsWithMTCForServiceIds3 () throws SQLException { assertFeedMergeSucceeded(mergeFeedsJob); // assert service_ids have been feed scoped properly String mergedNamespace = mergeFeedsJob.mergedVersion.namespace; - // - calendar table - // expect a total of 3 records in calendar table - assertThatSqlCountQueryYieldsExpectedCount( - String.format("SELECT count(*) FROM %s.calendar", mergedNamespace), - 3 - ); - // - calendar_dates table - // expect 3 records in calendar_dates table - assertThatSqlCountQueryYieldsExpectedCount( - String.format("SELECT count(*) FROM %s.calendar_dates", mergedNamespace), - 3 - ); + // - calendar table should have 3 records. + assertRowCountInTable(mergedNamespace, "calendar", 3); + + // - calendar_dates should have 3 records. + assertRowCountInTable(mergedNamespace, "calendar_dates", 3); + + // - trips table should have 3 records. + assertRowCountInTable(mergedNamespace, "trips", 3); - // - trips table - // expect 3 records in trips table - assertThatSqlCountQueryYieldsExpectedCount( - String.format("SELECT count(*) FROM %s.trips", mergedNamespace), - 3 - ); // common_id service_id should be scoped for earlier feed version. assertThatSqlCountQueryYieldsExpectedCount( String.format( @@ -927,25 +862,16 @@ void canMergeFeedsWithMTCForServiceIds4 () throws SQLException { assertFeedMergeSucceeded(mergeFeedsJob); // assert service_ids have been feed scoped properly String mergedNamespace = mergeFeedsJob.mergedVersion.namespace; - // - calendar table - // expect a total of 3 records in calendar table - assertThatSqlCountQueryYieldsExpectedCount( - String.format("SELECT count(*) FROM %s.calendar", mergedNamespace), - 3 - ); + // - calendar table should have 3 records. + assertRowCountInTable(mergedNamespace, "calendar", 3); + // - calendar_dates table // expect 2 records in calendar_dates table (all records from future feed removed) - assertThatSqlCountQueryYieldsExpectedCount( - String.format("SELECT count(*) FROM %s.calendar_dates", mergedNamespace), - 3 - ); + assertRowCountInTable(mergedNamespace, "calendar_dates", 3); + + // - trips table should have 3 records. + assertRowCountInTable(mergedNamespace, "trips", 3); - // - trips table - // expect 3 records in trips table - assertThatSqlCountQueryYieldsExpectedCount( - String.format("SELECT count(*) FROM %s.trips", mergedNamespace), - 3 - ); // common_id service_id should be scoped for earlier feed version. assertThatSqlCountQueryYieldsExpectedCount( String.format( @@ -984,10 +910,7 @@ void canMergeBARTFeedsWithSpecialStops() throws SQLException, IOException { // Job should succeed. assertFeedMergeSucceeded(mergeFeedsJob); // Verify that the stop count is equal to the number of stops found in each of the input stops.txt files. - assertThatSqlCountQueryYieldsExpectedCount( - String.format("SELECT count(*) FROM %s.stops", mergeFeedsJob.mergedVersion.namespace), - 182 - ); + assertRowCountInTable(mergeFeedsJob.mergedVersion.namespace, "stops", 182); } /** @@ -1002,34 +925,24 @@ void canMergeFeedsWithoutAgencyIds () throws SQLException { FeedVersion mergedVersion = regionallyMergeVersions(versions); String mergedNamespace = mergedVersion.namespace; - // - agency - // expect a total of 2 records - assertThatSqlCountQueryYieldsExpectedCount( - String.format("SELECT count(*) FROM %s.agency", mergedNamespace), - 2 - ); + // - agency should have 2 records. + assertRowCountInTable(mergedNamespace, "agency", 2); + // there shouldn't be records with blank agency_id assertThatSqlCountQueryYieldsExpectedCount( String.format("SELECT count(*) FROM %s.agency where agency_id = '' or agency_id is null", mergedNamespace), 0 ); - // - routes - // expect a total of 2 records - assertThatSqlCountQueryYieldsExpectedCount( - String.format("SELECT count(*) FROM %s.routes", mergedNamespace), - 2 - ); + // - routes should have 2 records + assertRowCountInTable(mergedNamespace, "routes", 2); + // there shouldn't be records with blank agency_id assertThatSqlCountQueryYieldsExpectedCount( String.format("SELECT count(*) FROM %s.routes where agency_id = '' or agency_id is null", mergedNamespace), 0 ); - // - trips - // expect 4 records - assertThatSqlCountQueryYieldsExpectedCount( - String.format("SELECT count(*) FROM %s.trips", mergedNamespace), - 4 - ); + // - trips should have 4 records + assertRowCountInTable(mergedNamespace, "trips", 4); } /** @@ -1071,4 +984,14 @@ private void assertNoRefIntegrityErrors(String mergedNamespace) throws SQLExcept 0 ); } + + /** + * Shorthand method for asserting an expected number of rows in a table. + */ + private void assertRowCountInTable(String namespace, String tableName, int count) throws SQLException { + assertThatSqlCountQueryYieldsExpectedCount( + String.format("SELECT count(*) FROM %s.%s", namespace, tableName), + count + ); + } } From b58e73b39e12f26f6d2e411405a41e56f18bc434 Mon Sep 17 00:00:00 2001 From: binh-dam-ibigroup <56846598+binh-dam-ibigroup@users.noreply.github.com> Date: Tue, 7 Dec 2021 19:34:33 -0500 Subject: [PATCH 094/122] fix(CalendarDatesMergeLineContext): Add missing converter for GTFS+ calendar_attributes. --- .../CalendarAttributesMergeLineContext.java | 90 +++++++++++++++++++ .../CalendarDatesMergeLineContext.java | 2 +- .../jobs/feedmerge/MergeLineContext.java | 2 + .../manager/jobs/MergeFeedsJobTest.java | 7 ++ .../calendar_attributes.txt | 3 + .../merge-data-base/calendar_attributes.txt | 3 + 6 files changed, 106 insertions(+), 1 deletion(-) create mode 100644 src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarAttributesMergeLineContext.java create mode 100755 src/test/resources/com/conveyal/datatools/gtfs/merge-data-added-trips-2/calendar_attributes.txt create mode 100755 src/test/resources/com/conveyal/datatools/gtfs/merge-data-base/calendar_attributes.txt diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarAttributesMergeLineContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarAttributesMergeLineContext.java new file mode 100644 index 000000000..1d859f49d --- /dev/null +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarAttributesMergeLineContext.java @@ -0,0 +1,90 @@ +package com.conveyal.datatools.manager.jobs.feedmerge; + +import com.conveyal.datatools.manager.jobs.MergeFeedsJob; +import com.conveyal.gtfs.error.NewGTFSError; +import com.conveyal.gtfs.loader.Table; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.Set; +import java.util.zip.ZipOutputStream; + +import static com.conveyal.datatools.manager.jobs.feedmerge.MergeFeedsType.SERVICE_PERIOD; +import static com.conveyal.datatools.manager.utils.MergeFeedUtils.getTableScopedValue; +import static com.conveyal.datatools.manager.utils.MergeFeedUtils.hasDuplicateError; + +public class CalendarAttributesMergeLineContext extends MergeLineContext { + private static final Logger LOG = LoggerFactory.getLogger(CalendarAttributesMergeLineContext.class); + + public CalendarAttributesMergeLineContext(MergeFeedsJob job, Table table, ZipOutputStream out) throws IOException { + super(job, table, out); + } + + @Override + public boolean checkFieldsForMergeConflicts(Set idErrors, FieldContext fieldContext) { + return checkCalendarIds(idErrors, fieldContext); + } + + @Override + public void afterRowWrite() throws IOException { + // If the current row is for a calendar service_id that is marked for cloning/renaming, clone the + // values, change the ID, extend the start/end dates to the feed's full range, and write the + // additional line to the file. + addClonedServiceId(); + } + + private boolean checkCalendarIds(Set idErrors, FieldContext fieldContext) { + boolean shouldSkipRecord = false; + + // If any service_id in the active feed matches with the future + // feed, it should be modified and all associated trip records + // must also be changed with the modified service_id. + // TODO How can we check that calendar_dates entries are + // duplicates? I think we would need to consider the + // service_id:exception_type:date as the unique key and include any + // all entries as long as they are unique on this key. + if (isHandlingActiveFeed() && hasDuplicateError(idErrors)) { + // Modify service_id and ensure that referencing trips + // have service_id updated. + updateAndRemapOutput(fieldContext); + } + + // Skip record (based on remapped id if necessary) if it was skipped in the calendar table. + String keyInCalendarTable = getTableScopedValue(Table.CALENDAR, getIdScope(), keyValue); + if (mergeFeedsResult.skippedIds.contains(keyInCalendarTable)) { + LOG.warn( + "Skipping calendar entry {} because it was skipped in the merged calendar table.", + keyValue); + shouldSkipRecord = true; + } + + return !shouldSkipRecord; + } + + /** + * Adds a cloned service id for trips with the same signature in both the active & future feeds. + * The cloned service id spans from the start date in the active feed until the end date in the future feed. + * @throws IOException + */ + public void addClonedServiceId() throws IOException { + if (isHandlingFutureFeed() && job.mergeType.equals(SERVICE_PERIOD)) { + String originalServiceId = keyValue; + if (job.serviceIdsToCloneRenameAndExtend.contains(originalServiceId)) { + String[] clonedValues = getOriginalRowValues().clone(); + String newServiceId = clonedValues[keyFieldIndex] = String.join(":", getIdScope(), originalServiceId); + + referenceTracker.checkReferencesAndUniqueness( + keyValue, + getLineNumber(), + table.fields[0], + newServiceId, + table, + keyField, + table.getOrderFieldName() + ); + writeValuesToTable(clonedValues, true); + } + } + } +} \ No newline at end of file diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarDatesMergeLineContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarDatesMergeLineContext.java index ba6c25b3f..ce8fe327b 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarDatesMergeLineContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarDatesMergeLineContext.java @@ -84,7 +84,7 @@ private boolean checkCalendarDatesIds(FieldContext fieldContext) throws IOExcept } - // Track service ID because we want to avoid removing trips that may reference this + // Track service ID because we want to avoid removing trips that may reference this // service_id when the service_id is used by calendar.txt records that operate in // the valid date range, i.e., before the future feed's first date. if (!shouldSkipRecord && fieldContext.nameEquals(SERVICE_ID)) mergeFeedsResult.serviceIds.add(fieldContext.getValueToWrite()); diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java index cd4c29f99..197f094cc 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java @@ -84,6 +84,8 @@ public static MergeLineContext create(MergeFeedsJob job, Table table, ZipOutputS return new AgencyMergeLineContext(job, table, out); case "calendar": return new CalendarMergeLineContext(job, table, out); + case "calendar_attributes": + return new CalendarAttributesMergeLineContext(job, table, out); case "calendar_dates": return new CalendarDatesMergeLineContext(job, table, out); case "routes": diff --git a/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java b/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java index 648a541b2..5bca6e034 100644 --- a/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java +++ b/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java @@ -458,6 +458,13 @@ void mergeMTCShouldHandleMatchingTripIdsAndDropUnusedFutureCalendar() throws SQL // The calendar_dates entry should be preserved, but remapped to a different id. assertRowCountInTable(mergedNamespace, "calendar_dates", 1); + // The GTFS+ calendar_attributes table should contain the same number of entries as the calendar table. + assertEquals( + 3, + mergeFeedsJob.mergeFeedsResult.linesPerTable.get("calendar_attributes").intValue(), + "Merged calendar_dates table count should equal expected value." + ); + assertNoUnusedServiceIds(mergedNamespace); assertNoRefIntegrityErrors(mergedNamespace); } diff --git a/src/test/resources/com/conveyal/datatools/gtfs/merge-data-added-trips-2/calendar_attributes.txt b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-added-trips-2/calendar_attributes.txt new file mode 100755 index 000000000..64803a6be --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-added-trips-2/calendar_attributes.txt @@ -0,0 +1,3 @@ +service_id,service_description +common_id,Description for common_id (added trips) +only_calendar_id,Description for only_calendar_id (added trips) diff --git a/src/test/resources/com/conveyal/datatools/gtfs/merge-data-base/calendar_attributes.txt b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-base/calendar_attributes.txt new file mode 100755 index 000000000..e1ab10f8e --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-base/calendar_attributes.txt @@ -0,0 +1,3 @@ +service_id,service_description +common_id,Description for common_id +only_calendar_id,Description for only_calendar_id From 55e28a8d824f2d74c6849752401ac8c104cb6f28 Mon Sep 17 00:00:00 2001 From: binh-dam-ibigroup <56846598+binh-dam-ibigroup@users.noreply.github.com> Date: Wed, 8 Dec 2021 10:25:49 -0500 Subject: [PATCH 095/122] fix(MergeLineContext): Process GTFS+ timepoints table using logic og trips table. --- .../jobs/feedmerge/MergeLineContext.java | 2 ++ .../manager/jobs/MergeFeedsJobTest.java | 17 +++++++++++++++-- .../merge-data-added-trips-2/stop_times.txt | 2 -- .../merge-data-added-trips-2/timepoints.txt | 5 +++++ .../gtfs/merge-data-base/timepoints.txt | 5 +++++ 5 files changed, 27 insertions(+), 4 deletions(-) create mode 100644 src/test/resources/com/conveyal/datatools/gtfs/merge-data-added-trips-2/timepoints.txt create mode 100644 src/test/resources/com/conveyal/datatools/gtfs/merge-data-base/timepoints.txt diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java index 197f094cc..c332d9b22 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java @@ -95,6 +95,8 @@ public static MergeLineContext create(MergeFeedsJob job, Table table, ZipOutputS case "stops": return new StopsMergeLineContext(job, table, out); case "trips": + case "timepoints": + // Use same merge logic to filter out trips in both tables. return new TripsMergeLineContext(job, table, out); default: return new MergeLineContext(job, table, out); diff --git a/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java b/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java index 5bca6e034..6070ee731 100644 --- a/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java +++ b/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java @@ -3,6 +3,7 @@ import com.conveyal.datatools.DatatoolsTest; import com.conveyal.datatools.UnitTest; import com.conveyal.datatools.manager.auth.Auth0UserProfile; +import com.conveyal.datatools.manager.gtfsplus.GtfsPlusValidation; import com.conveyal.datatools.manager.jobs.feedmerge.MergeFeedsType; import com.conveyal.datatools.manager.jobs.feedmerge.MergeStrategy; import com.conveyal.datatools.manager.models.FeedSource; @@ -421,7 +422,7 @@ void mergeMTCShouldHandleMatchingTripIdsWithSameSignature() throws SQLException * {@link MergeStrategy#CHECK_STOP_TIMES} strategy correctly and drop unused future service ids. */ @Test - void mergeMTCShouldHandleMatchingTripIdsAndDropUnusedFutureCalendar() throws SQLException { + void mergeMTCShouldHandleMatchingTripIdsAndDropUnusedFutureCalendar() throws Exception { Set versions = new HashSet<>(); versions.add(fakeTransitBase); versions.add(fakeTransitSameSignatureTrips2); @@ -458,13 +459,25 @@ void mergeMTCShouldHandleMatchingTripIdsAndDropUnusedFutureCalendar() throws SQL // The calendar_dates entry should be preserved, but remapped to a different id. assertRowCountInTable(mergedNamespace, "calendar_dates", 1); - // The GTFS+ calendar_attributes table should contain the same number of entries as the calendar table. + // The GTFS+ calendar_attributes table should contain the same number of entries as the calendar table + // (reported by MTC). assertEquals( 3, mergeFeedsJob.mergeFeedsResult.linesPerTable.get("calendar_attributes").intValue(), "Merged calendar_dates table count should equal expected value." ); + // The GTFS+ timepoints table should not contain any trip ids not in the trips table + // (reported by MTC). + GtfsPlusValidation validation = GtfsPlusValidation.validate(mergeFeedsJob.mergedVersion.id); + assertEquals( + 0L, + validation.issues.stream().filter( + issue -> issue.tableId.equals("timepoints") && issue.fieldName.equals("trip_id") + ).count(), + "There should not be trip_id issues in the GTFS+ timepoints table." + ); + assertNoUnusedServiceIds(mergedNamespace); assertNoRefIntegrityErrors(mergedNamespace); } diff --git a/src/test/resources/com/conveyal/datatools/gtfs/merge-data-added-trips-2/stop_times.txt b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-added-trips-2/stop_times.txt index 09e46408f..e6dce4f97 100755 --- a/src/test/resources/com/conveyal/datatools/gtfs/merge-data-added-trips-2/stop_times.txt +++ b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-added-trips-2/stop_times.txt @@ -1,7 +1,5 @@ trip_id,arrival_time,departure_time,stop_id,stop_sequence,stop_headsign,pickup_type,drop_off_type,shape_dist_traveled,timepoint trip3,07:00:00,07:00:00,4u6g,1,,0,0,0.0000000, trip3,07:01:00,07:01:00,johv,2,,0,0,341.4491961, -only-calendar-trip1,07:00:00,07:00:00,4u6g,1,,0,0,0.0000000, -only-calendar-trip1,07:01:00,07:01:00,johv,2,,0,0,341.4491961, only-calendar-trip2,07:00:00,07:00:00,johv,1,,0,0,0.0000000, only-calendar-trip2,07:01:00,07:01:00,4u6g,2,,0,0,341.4491961, diff --git a/src/test/resources/com/conveyal/datatools/gtfs/merge-data-added-trips-2/timepoints.txt b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-added-trips-2/timepoints.txt new file mode 100644 index 000000000..41a7813e4 --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-added-trips-2/timepoints.txt @@ -0,0 +1,5 @@ +trip_id,stop_id +trip3,4u6g +trip3,johv +only-calendar-trip2,johv +only-calendar-trip2,4u6g \ No newline at end of file diff --git a/src/test/resources/com/conveyal/datatools/gtfs/merge-data-base/timepoints.txt b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-base/timepoints.txt new file mode 100644 index 000000000..a7ba2e023 --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-base/timepoints.txt @@ -0,0 +1,5 @@ +trip_id,stop_id +only-calendar-trip1,4u6g +only-calendar-trip1,johv +only-calendar-trip2,johv +only-calendar-trip2,4u6g \ No newline at end of file From 9a47db31b300a5a109cfb65511ef3679aa075aa3 Mon Sep 17 00:00:00 2001 From: binh-dam-ibigroup <56846598+binh-dam-ibigroup@users.noreply.github.com> Date: Wed, 8 Dec 2021 11:43:43 -0500 Subject: [PATCH 096/122] fix(TripsMergeLineContext): Remove unwritten entries from remap list. --- .../jobs/feedmerge/TripsMergeLineContext.java | 27 ++++++++++++------- .../manager/jobs/MergeFeedsJobTest.java | 11 ++++++++ 2 files changed, 28 insertions(+), 10 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/TripsMergeLineContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/TripsMergeLineContext.java index 697c343e8..81cbd10b8 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/TripsMergeLineContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/TripsMergeLineContext.java @@ -9,6 +9,7 @@ import java.util.zip.ZipOutputStream; import static com.conveyal.datatools.manager.jobs.feedmerge.MergeFeedsType.SERVICE_PERIOD; +import static com.conveyal.datatools.manager.utils.MergeFeedUtils.getTableScopedValue; import static com.conveyal.datatools.manager.utils.MergeFeedUtils.hasDuplicateError; public class TripsMergeLineContext extends MergeLineContext { @@ -22,24 +23,30 @@ public boolean checkFieldsForMergeConflicts(Set idErrors, FieldCon } private boolean checkTripIds(Set idErrors, FieldContext fieldContext) { - boolean shouldSkipRecord = false; // For the MTC revised feed merge process, // the updated logic requires to insert all trips from both the active and future feed, - // except if they are present in both, in which case we only insert the trip entry from the future feed. - if ( + // except if they are present in both, in which case + // we only insert the trip entry from the future feed and skip the one in the active feed. + boolean shouldSkipRecord = job.mergeType.equals(SERVICE_PERIOD) && isHandlingActiveFeed() && - job.sharedTripIdsWithConsistentSignature.contains(keyValue) - ) { - // Skip this record, we will use the one from the future feed. - shouldSkipRecord = true; - } + job.sharedTripIdsWithConsistentSignature.contains(keyValue); - // Remap duplicate trip ids. - if (hasDuplicateError(idErrors)) { + // Remap duplicate trip ids for records that are not skipped. + if (!shouldSkipRecord && hasDuplicateError(idErrors)) { updateAndRemapOutput(fieldContext, true); } + // Remove remapped service_ids associated to the trips table from the merge summary + // (the remapped id is already listed under the calendar/calendar_dates tables, + // so there is no need to add that foreign key again). + if (fieldContext.nameEquals(SERVICE_ID)) { + String tableScopedValue = getTableScopedValue(table, getIdScope(), fieldContext.getValue()); + if (mergeFeedsResult.remappedIds.containsKey(tableScopedValue)) { + mergeFeedsResult.remappedIds.remove(tableScopedValue); + } + } + return !shouldSkipRecord; } } \ No newline at end of file diff --git a/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java b/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java index 6070ee731..ec869a632 100644 --- a/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java +++ b/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java @@ -478,6 +478,17 @@ void mergeMTCShouldHandleMatchingTripIdsAndDropUnusedFutureCalendar() throws Exc "There should not be trip_id issues in the GTFS+ timepoints table." ); + // There should be mention of any remapped trip ids in the job summary + // because no remapped trip ids should have been written to the trips/timepoints tables + // (reported by MTC). + assertEquals( + 0L, + mergeFeedsJob.mergeFeedsResult.remappedIds.keySet().stream().filter( + key -> key.startsWith("trips:") + ).count(), + "Job summary should not mention remapped uninserted trip ids." + ); + assertNoUnusedServiceIds(mergedNamespace); assertNoRefIntegrityErrors(mergedNamespace); } From 028c6de56a07276516e4e380242b10196fbc0b6d Mon Sep 17 00:00:00 2001 From: binh-dam-ibigroup <56846598+binh-dam-ibigroup@users.noreply.github.com> Date: Thu, 9 Dec 2021 22:57:34 -0500 Subject: [PATCH 097/122] fix(MergeFeedsJob): Add some more tests and refactor. Prep for other fixes. --- .../CalendarDatesMergeLineContext.java | 61 ++++++++++++++++--- .../feedmerge/CalendarMergeLineContext.java | 3 + .../manager/jobs/feedmerge/FeedContext.java | 4 +- .../jobs/feedmerge/FeedMergeContext.java | 4 +- .../manager/jobs/MergeFeedsJobTest.java | 48 +++++++++++++-- .../gtfs/merge-data-base/calendar_dates.txt | 3 + 6 files changed, 106 insertions(+), 17 deletions(-) create mode 100644 src/test/resources/com/conveyal/datatools/gtfs/merge-data-base/calendar_dates.txt diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarDatesMergeLineContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarDatesMergeLineContext.java index ce8fe327b..3ab5d39b7 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarDatesMergeLineContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarDatesMergeLineContext.java @@ -3,6 +3,7 @@ import com.conveyal.datatools.manager.jobs.MergeFeedsJob; import com.conveyal.gtfs.error.NewGTFSError; import com.conveyal.gtfs.loader.Table; +import com.conveyal.gtfs.model.CalendarDate; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -13,7 +14,12 @@ import static com.conveyal.datatools.manager.jobs.feedmerge.MergeFeedsType.SERVICE_PERIOD; import static com.conveyal.datatools.manager.utils.MergeFeedUtils.getTableScopedValue; +import static com.conveyal.datatools.manager.utils.MergeFeedUtils.hasDuplicateError; +import static com.conveyal.gtfs.loader.DateField.GTFS_DATE_FORMATTER; +/** + * Contains logic for merging records in the GTFS calendar_dates table. + */ public class CalendarDatesMergeLineContext extends MergeLineContext { private static final Logger LOG = LoggerFactory.getLogger(CalendarDatesMergeLineContext.class); @@ -26,7 +32,7 @@ public CalendarDatesMergeLineContext(MergeFeedsJob job, Table table, ZipOutputSt @Override public boolean checkFieldsForMergeConflicts(Set idErrors, FieldContext fieldContext) throws IOException { - return checkCalendarDatesIds(fieldContext); + return checkCalendarDatesIds(idErrors, fieldContext); } @Override @@ -43,13 +49,23 @@ public void startNewFeed(int feedIndex) throws IOException { futureFeedFirstDateForCalendarValidity = getFutureFeedFirstDateForCheckingCalendarValidity(); } - private boolean checkCalendarDatesIds(FieldContext fieldContext) throws IOException { + private boolean checkCalendarDatesIds(Set idErrors, FieldContext fieldContext) throws IOException { boolean shouldSkipRecord = false; String key = getTableScopedValue(table, getIdScope(), keyValue); + // TODO: REfactor + String scopedId = String.join(":", getIdScope(), keyValue); // Drop any calendar_dates.txt records from the existing feed for dates that are - // not before the first date of the future feed. + // not before the first date of the future feed + // and also for service ids not in the merged calendar table + // (we can determine that because the calendar table has already been processed). LocalDate date = getCsvDate("date"); - if (isHandlingActiveFeed() && !date.isBefore(futureFeedFirstDateForCalendarValidity)) { + if ( + isHandlingActiveFeed() && + ( + //!job.mergeFeedsResult.serviceIds.contains(scopedId) || + !date.isBefore(futureFeedFirstDateForCalendarValidity) + ) + ) { LOG.warn( "Skipping calendar_dates entry {} because it operates in the time span of future feed (i.e., after or on {}).", keyValue, @@ -58,12 +74,13 @@ private boolean checkCalendarDatesIds(FieldContext fieldContext) throws IOExcept shouldSkipRecord = true; } + // TODO: refactor below. if (job.mergeType.equals(SERVICE_PERIOD)) { if (isHandlingActiveFeed()) { // Remove calendar entries that are no longer used. if (feedMergeContext.active.getServiceIdsToRemove().contains(keyValue)) { LOG.warn( - "Skipping active calendar entry {} because it will become unused in the merged feed.", + "Skipping active calendar_dates entry {} because it will become unused in the merged feed.", keyValue); mergeFeedsResult.skippedIds.add(key); shouldSkipRecord = true; @@ -75,7 +92,7 @@ private boolean checkCalendarDatesIds(FieldContext fieldContext) throws IOExcept // in that case we drop the calendar entry. if (feedMergeContext.future.getServiceIdsToRemove().contains(keyValue)) { LOG.warn( - "Skipping future calendar entry {} because it will become unused in the merged feed.", + "Skipping future calendar_dates entry {} because it will become unused in the merged feed.", keyValue); mergeFeedsResult.skippedIds.add(key); shouldSkipRecord = true; @@ -87,7 +104,9 @@ private boolean checkCalendarDatesIds(FieldContext fieldContext) throws IOExcept // Track service ID because we want to avoid removing trips that may reference this // service_id when the service_id is used by calendar.txt records that operate in // the valid date range, i.e., before the future feed's first date. - if (!shouldSkipRecord && fieldContext.nameEquals(SERVICE_ID)) mergeFeedsResult.serviceIds.add(fieldContext.getValueToWrite()); + if (!shouldSkipRecord && fieldContext.nameEquals(SERVICE_ID)) { + mergeFeedsResult.serviceIds.add(fieldContext.getValueToWrite()); + } return !shouldSkipRecord; } @@ -132,7 +151,33 @@ public void addClonedServiceId() throws IOException { keyField, table.getOrderFieldName() ); + + // Add entries in the future feed. writeValuesToTable(clonedValues, true); + + // Because this service has been extended from the future feed into the active feed, + // we need to add all entries for the original service id under the active feed + // (and of course rename service id). + for (CalendarDate calDate : feedMergeContext.active.feed.calendarDates.getAll()) { + if (calDate.service_id.equals(originalServiceId)) { + writeValuesToTable(getCalendarRowValues(calDate, newServiceId), true); + } + } } } - }} \ No newline at end of file + } + + /** + * Helper method that builds a string array from a CalendarDates object + * with a new service_id. + */ + private String[] getCalendarRowValues(CalendarDate calDate, String newServiceId) { + String[] rowValues = new String[getOriginalRowValues().length]; + rowValues[getFieldIndex(SERVICE_ID)] = newServiceId; + rowValues[getFieldIndex("date")] + = calDate.date.format(GTFS_DATE_FORMATTER); + rowValues[getFieldIndex("exception_type")] + = String.valueOf(calDate.exception_type); + return rowValues; + } +} \ No newline at end of file diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarMergeLineContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarMergeLineContext.java index 31f821606..7df3f8c58 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarMergeLineContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarMergeLineContext.java @@ -17,6 +17,9 @@ import static com.conveyal.datatools.manager.utils.MergeFeedUtils.hasDuplicateError; import static com.conveyal.gtfs.loader.DateField.GTFS_DATE_FORMATTER; +/** + * Contains logic for merging records in the GTFS calendar table. + */ public class CalendarMergeLineContext extends MergeLineContext { private static final Logger LOG = LoggerFactory.getLogger(CalendarMergeLineContext.class); diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/FeedContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/FeedContext.java index 5422d59fd..3bc91938c 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/FeedContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/FeedContext.java @@ -54,10 +54,10 @@ public Set getServiceIdsToRemove() { return serviceIdsToRemove; } - public void setServiceIdsToRemoveFromOtherFeed(Set idsNotInOtherFeed) { + public void setServiceIdsToRemoveUsingOtherFeed(Set tripIdsNotInOtherFeed) { serviceIdsToRemove = Sets.difference( feedToMerge.serviceIds, - getServiceIds(idsNotInOtherFeed) + getServiceIds(tripIdsNotInOtherFeed) ); } diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/FeedMergeContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/FeedMergeContext.java index 05782adca..371112026 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/FeedMergeContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/FeedMergeContext.java @@ -47,8 +47,8 @@ public FeedMergeContext(Set feedVersions, Auth0UserProfile owner) t } public void collectServiceIdsToRemove() { - active.setServiceIdsToRemoveFromOtherFeed(getActiveTripIdsNotInFutureFeed()); - future.setServiceIdsToRemoveFromOtherFeed(getFutureTripIdsNotInActiveFeed()); + active.setServiceIdsToRemoveUsingOtherFeed(getActiveTripIdsNotInFutureFeed()); + future.setServiceIdsToRemoveUsingOtherFeed(getFutureTripIdsNotInActiveFeed()); } @Override diff --git a/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java b/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java index ec869a632..6bcf7dfde 100644 --- a/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java +++ b/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java @@ -456,8 +456,30 @@ void mergeMTCShouldHandleMatchingTripIdsAndDropUnusedFutureCalendar() throws Exc // 1 trip from the future feed not in the active feed. assertRowCountInTable(mergedNamespace, "trips", 3); - // The calendar_dates entry should be preserved, but remapped to a different id. - assertRowCountInTable(mergedNamespace, "calendar_dates", 1); + // 3 calendar_dates entries should be in the merged feed: + // - 2 entries for the calendar item that was extended due to shared trip, + // (1 from the active feed, 1 from the future feed) + // - 1 entry for the calendar item in the active feed for trips not in the future feed. + // See also specific query for each entry underneath. + // (reported by MTC). + + // The entry from calendar_dates for the extended service_id from active feed should be present. + assertThatSqlCountQueryYieldsExpectedCount( + String.format("SELECT count(*) FROM %s.calendar_dates where service_id = 'Fake_Transit7:common_id' and date='20170919' and exception_type = 2", mergedNamespace), + 1 + ); + // One entry from calendar_dates for the service_id that is used the active feed + // and that is not in the future feed should be present. + assertThatSqlCountQueryYieldsExpectedCount( + String.format("SELECT count(*) FROM %s.calendar_dates where service_id = 'Fake_Transit1:common_id' and date='20170919' and exception_type = 2", mergedNamespace), + 1 + ); + // Unused calendar_dates service_ids in the active feed should still be dropped. + assertThatSqlCountQueryYieldsExpectedCount( + String.format("SELECT count(*) FROM %s.calendar_dates where service_id = 'Fake_Transit1:only_calendar_id' and date='20170919' and exception_type = 1", mergedNamespace), + 0 + ); + assertRowCountInTable(mergedNamespace, "calendar_dates", 3); // The GTFS+ calendar_attributes table should contain the same number of entries as the calendar table // (reported by MTC). @@ -834,8 +856,17 @@ void canMergeFeedsWithMTCForServiceIds3 () throws SQLException { // - calendar table should have 3 records. assertRowCountInTable(mergedNamespace, "calendar", 3); - // - calendar_dates should have 3 records. - assertRowCountInTable(mergedNamespace, "calendar_dates", 3); + // calendar_dates should have 1 record. + // - one for common_id from the future feed, + // Note that the common_id from the active feed is not included because it operates + // within the future feed timespan. + assertThatSqlCountQueryYieldsExpectedCount( + String.format( + "SELECT count(*) FROM %s.calendar_dates WHERE service_id = 'common_id' and date = '20170916'", + mergedNamespace + ), + 1 + ); // - trips table should have 3 records. assertRowCountInTable(mergedNamespace, "trips", 3); @@ -899,7 +930,14 @@ void canMergeFeedsWithMTCForServiceIds4 () throws SQLException { // - calendar_dates table // expect 2 records in calendar_dates table (all records from future feed removed) assertRowCountInTable(mergedNamespace, "calendar_dates", 3); - +/* + // calendar_dates table should have 2 records + // - common_id from the future feed + // - both_id from the future feed + // Entries from the active feed are discarded because they don't have a corresponding calendar entry + // (that would create a ref integrity issue) or it is after the future feed start date. + assertRowCountInTable(mergedNamespace, "calendar_dates", 2); +*/ // - trips table should have 3 records. assertRowCountInTable(mergedNamespace, "trips", 3); diff --git a/src/test/resources/com/conveyal/datatools/gtfs/merge-data-base/calendar_dates.txt b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-base/calendar_dates.txt new file mode 100644 index 000000000..2799b08b2 --- /dev/null +++ b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-base/calendar_dates.txt @@ -0,0 +1,3 @@ +service_id,date,exception_type +only_calendar_id,20170919,1 +common_id,20170919,2 From 6434ccd2404c2962202e358796169352451fc160 Mon Sep 17 00:00:00 2001 From: binh-dam-ibigroup <56846598+binh-dam-ibigroup@users.noreply.github.com> Date: Fri, 10 Dec 2021 07:37:42 -0500 Subject: [PATCH 098/122] fix(MergeFeedsJob): Write calendar_dates cloned service ids after initial table write. --- .../datatools/manager/jobs/MergeFeedsJob.java | 1 + .../CalendarDatesMergeLineContext.java | 44 ++++------ .../jobs/feedmerge/MergeLineContext.java | 17 ++++ .../manager/jobs/MergeFeedsJobTest.java | 88 +++++++++++-------- .../calendar_dates.txt | 2 +- .../gtfs/merge-data-base/calendar.txt | 1 + .../gtfs/merge-data-base/calendar_dates.txt | 5 +- .../gtfs/merge-data-base/stop_times.txt | 4 +- .../datatools/gtfs/merge-data-base/trips.txt | 3 +- .../gtfs/merge-data-future/calendar.txt | 2 +- .../gtfs/merge-data-future/stop_times.txt | 4 +- .../gtfs/merge-data-future/trips.txt | 3 +- 12 files changed, 101 insertions(+), 73 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java b/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java index af8fdf4e8..ab683b090 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/MergeFeedsJob.java @@ -369,6 +369,7 @@ private int constructMergedTable(Table table, List feedsToMerge, Zi return -1; } } + ctx.afterTableRecords(); ctx.flushAndClose(); } catch (IOException e) { List versionNames = feedVersions.stream() diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarDatesMergeLineContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarDatesMergeLineContext.java index 3ab5d39b7..40caebf27 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarDatesMergeLineContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarDatesMergeLineContext.java @@ -14,7 +14,6 @@ import static com.conveyal.datatools.manager.jobs.feedmerge.MergeFeedsType.SERVICE_PERIOD; import static com.conveyal.datatools.manager.utils.MergeFeedUtils.getTableScopedValue; -import static com.conveyal.datatools.manager.utils.MergeFeedUtils.hasDuplicateError; import static com.conveyal.gtfs.loader.DateField.GTFS_DATE_FORMATTER; /** @@ -36,11 +35,11 @@ public boolean checkFieldsForMergeConflicts(Set idErrors, FieldCon } @Override - public void afterRowWrite() throws IOException { + public void afterTableRecords() throws IOException { // If the current row is for a calendar service_id that is marked for cloning/renaming, clone the // values, change the ID, extend the start/end dates to the feed's full range, and write the // additional line to the file. - addClonedServiceId(); + addClonedServiceIds(); } @Override @@ -135,31 +134,22 @@ private LocalDate getFutureFeedFirstDateForCheckingCalendarValidity() { * The cloned service id spans from the start date in the active feed until the end date in the future feed. * @throws IOException */ - public void addClonedServiceId() throws IOException { - if (isHandlingFutureFeed() && job.mergeType.equals(SERVICE_PERIOD)) { - String originalServiceId = keyValue; - if (job.serviceIdsToCloneRenameAndExtend.contains(originalServiceId)) { - String[] clonedValues = getOriginalRowValues().clone(); - String newServiceId = clonedValues[keyFieldIndex] = String.join(":", getIdScope(), originalServiceId); - - referenceTracker.checkReferencesAndUniqueness( - keyValue, - getLineNumber(), - table.fields[0], - newServiceId, - table, - keyField, - table.getOrderFieldName() - ); - - // Add entries in the future feed. - writeValuesToTable(clonedValues, true); - - // Because this service has been extended from the future feed into the active feed, - // we need to add all entries for the original service id under the active feed - // (and of course rename service id). + public void addClonedServiceIds() throws IOException { + if (job.mergeType.equals(SERVICE_PERIOD)) { + for (String id : job.serviceIdsToCloneRenameAndExtend) { + String newServiceId = String.join(":", getClonedIdScope(), id); + + // Because this service has been extended to span both active and future feed, + // we need to add all calendar_dates entries for the original service id + // under the active AND future feed (and of course rename service id). + // TODO: refactor for (CalendarDate calDate : feedMergeContext.active.feed.calendarDates.getAll()) { - if (calDate.service_id.equals(originalServiceId)) { + if (calDate.service_id.equals(id)) { + writeValuesToTable(getCalendarRowValues(calDate, newServiceId), true); + } + } + for (CalendarDate calDate : feedMergeContext.future.feed.calendarDates.getAll()) { + if (calDate.service_id.equals(id)) { writeValuesToTable(getCalendarRowValues(calDate, newServiceId), true); } } diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java index c332d9b22..31d6c3e12 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java @@ -484,6 +484,14 @@ public void afterRowWrite() throws IOException { // Default is to do nothing. } + /** + * Overridable placeholder for additional processing after processing the table + * (whether any rows are available or not). + */ + public void afterTableRecords() throws IOException { + // Default is to do nothing. + } + public void scopeValueIfNeeded(FieldContext fieldContext) { boolean isKeyField = fieldContext.getField().isForeignReference() || fieldContext.nameEquals(keyField); if (job.mergeType.equals(REGIONAL) && isKeyField && !fieldContext.getValue().isEmpty()) { @@ -644,6 +652,15 @@ protected String getIdScope() { return idScope; } + /** + * Obtains the id scope to use for cloned items. + * It is set to the id scope corresponding to the future feed. + */ + protected String getClonedIdScope() { + // TODO: refactor name creation + return getCleanName(feedSource.name) + this.feedMergeContext.future.feedToMerge.version.version; + } + protected int getFeedIndex() { return feedIndex; } protected int getLineNumber() { diff --git a/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java b/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java index 6bcf7dfde..008b9e448 100644 --- a/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java +++ b/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java @@ -340,10 +340,8 @@ void mergeMTCShouldHandleExtendFutureStrategy() throws SQLException { assertNoUnusedServiceIds(mergedNamespace); assertNoRefIntegrityErrors(mergedNamespace); - // - calendar table - // expect a total of 1 record in calendar table that - // corresponds to the trip ids present in both active and future feed. - assertRowCountInTable(mergedNamespace, "calendar", 1); + // calendar table should have 2 records (all calendar ids are used and extended) + assertRowCountInTable(mergedNamespace, "calendar", 2); // expect that the record in calendar table has the correct start_date. assertThatSqlCountQueryYieldsExpectedCount( @@ -387,8 +385,10 @@ void mergeMTCShouldHandleMatchingTripIdsWithSameSignature() throws SQLException // - only_calendar_id used in the future feed. assertRowCountInTable(mergedNamespace, "calendar", 4); - // Out of all trips from the input datasets, expect 4 trips in merged output. - // 1 trip from active feed that is not in the future feed, + // Expect 4 trips in merged output: + // 1 trip from active feed that are not in the future feed, + // (the active trip for only_calendar_id is not included because that service id + // starts after the future feed start date) // 1 trip in both the active and future feeds, with the same signature (same stop times), // 2 trips from the future feed not in the active feed. assertRowCountInTable(mergedNamespace, "trips", 4); @@ -450,36 +450,52 @@ void mergeMTCShouldHandleMatchingTripIdsAndDropUnusedFutureCalendar() throws Exc // - only_calendar_id used in the future feed. assertRowCountInTable(mergedNamespace, "calendar", 3); - // Out of all trips from the input datasets, expect 4 trips in merged output. - // 1 trip from active feed that is not in the future feed, + // Expect 3 trips in merged output: + // 1 trip from active feed that are not in the future feed, + // (the active trip for only_calendar_dates is discarded because that service id + // starts after the future feed start date) // 1 trip in both the active and future feeds, with the same signature (same stop times), // 1 trip from the future feed not in the active feed. assertRowCountInTable(mergedNamespace, "trips", 3); - // 3 calendar_dates entries should be in the merged feed: - // - 2 entries for the calendar item that was extended due to shared trip, - // (1 from the active feed, 1 from the future feed) - // - 1 entry for the calendar item in the active feed for trips not in the future feed. + // 5 calendar_dates entries should be in the merged feed: + // - only_calendar_id: + // 1 from future feed, + // 0 from active feed + // (in the active feed, that service id starts after the future feed start date) + // - common_id: + // 2 from active feed for the calendar item that was extended due to shared trip, + // 2 from active feed for the active trip not in the future feed. // See also specific query for each entry underneath. // (reported by MTC). - // The entry from calendar_dates for the extended service_id from active feed should be present. + // The entries from calendar_dates for the service_id that is used the active feed + // and that is not in the future feed should be present. assertThatSqlCountQueryYieldsExpectedCount( - String.format("SELECT count(*) FROM %s.calendar_dates where service_id = 'Fake_Transit7:common_id' and date='20170919' and exception_type = 2", mergedNamespace), - 1 + String.format("SELECT count(*) FROM %s.calendar_dates where service_id = 'Fake_Transit1:common_id'", mergedNamespace), + 2 ); - // One entry from calendar_dates for the service_id that is used the active feed - // and that is not in the future feed should be present. + + // The entries from calendar_dates for the extended service_id from active feed should be present. + assertThatSqlCountQueryYieldsExpectedCount( + String.format("SELECT count(*) FROM %s.calendar_dates where service_id = 'Fake_Transit7:common_id'", mergedNamespace), + 2 + ); + + // The entries from calendar_dates for the active service_id that is entirely in the future feed + // should be present (that service id is not scoped). assertThatSqlCountQueryYieldsExpectedCount( - String.format("SELECT count(*) FROM %s.calendar_dates where service_id = 'Fake_Transit1:common_id' and date='20170919' and exception_type = 2", mergedNamespace), + String.format("SELECT count(*) FROM %s.calendar_dates where service_id = 'only_calendar_id'", mergedNamespace), 1 ); + // Unused calendar_dates service_ids in the active feed should still be dropped. assertThatSqlCountQueryYieldsExpectedCount( - String.format("SELECT count(*) FROM %s.calendar_dates where service_id = 'Fake_Transit1:only_calendar_id' and date='20170919' and exception_type = 1", mergedNamespace), + String.format("SELECT count(*) FROM %s.calendar_dates where service_id = 'Fake_Transit1:dropped_calendar_id' and date='20170919' and exception_type = 1", mergedNamespace), 0 ); - assertRowCountInTable(mergedNamespace, "calendar_dates", 3); + + assertRowCountInTable(mergedNamespace, "calendar_dates", 5); // The GTFS+ calendar_attributes table should contain the same number of entries as the calendar table // (reported by MTC). @@ -565,25 +581,26 @@ void mergeMTCShouldHandleDisjointTripIds() throws SQLException { String mergedNamespace = mergeFeedsJob.mergedVersion.namespace; // - calendar table - // expect a total of 3 records in calendar table - // - 2 records from future feed - // - 1 records from active feed that is used - // - the unused record from the active feed is discarded. - assertRowCountInTable(mergedNamespace, "calendar", 3); + // expect a total of 4 records in calendar table + // - 2 records from future feed, including only_calendar_dates which absorbs its active counterpart, + // - 1 record from active feed that is used + // - 1 unused record from the active feed that is NOT discarded (default strategy). + assertRowCountInTable(mergedNamespace, "calendar", 4); - // The one calendar entry for the active feed should end one day before the first calendar start date + // The calendar entry for the active feed ending 20170920 should end one day before the first calendar start date // of the future feed. final String activeCalendarNewEndDate = "20170919"; // One day before 20170920. assertThatSqlCountQueryYieldsExpectedCount( String.format( - "SELECT count(*) FROM %s.calendar WHERE end_date='%s' AND service_id in ('Fake_Transit1:common_id', 'Fake_Transit1:only_calendar_id')", + "SELECT count(*) FROM %s.calendar WHERE end_date='%s' AND service_id in ('Fake_Transit1:common_id')", mergedNamespace, activeCalendarNewEndDate), 1 ); - // - trips table - // expect a total of 4 records in trips table (all records from original files are included). + // trips table should have 4 records + // (all records from original files except the active trip for only_calendar_trips, + // which is skipped because it operates in the future feed). assertRowCountInTable(mergedNamespace, "trips", 4); } @@ -927,17 +944,10 @@ void canMergeFeedsWithMTCForServiceIds4 () throws SQLException { // - calendar table should have 3 records. assertRowCountInTable(mergedNamespace, "calendar", 3); - // - calendar_dates table - // expect 2 records in calendar_dates table (all records from future feed removed) + // calendar_dates table should have 3 records: + // all records from future feed and keep_one from the active feed. assertRowCountInTable(mergedNamespace, "calendar_dates", 3); -/* - // calendar_dates table should have 2 records - // - common_id from the future feed - // - both_id from the future feed - // Entries from the active feed are discarded because they don't have a corresponding calendar entry - // (that would create a ref integrity issue) or it is after the future feed start date. - assertRowCountInTable(mergedNamespace, "calendar_dates", 2); -*/ + // - trips table should have 3 records. assertRowCountInTable(mergedNamespace, "trips", 3); diff --git a/src/test/resources/com/conveyal/datatools/gtfs/merge-data-added-trips-2/calendar_dates.txt b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-added-trips-2/calendar_dates.txt index 5b1ec55e0..f33301897 100755 --- a/src/test/resources/com/conveyal/datatools/gtfs/merge-data-added-trips-2/calendar_dates.txt +++ b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-added-trips-2/calendar_dates.txt @@ -1,2 +1,2 @@ service_id,date,exception_type -common_id,20190218,1 +only_calendar_id,20190218,1 diff --git a/src/test/resources/com/conveyal/datatools/gtfs/merge-data-base/calendar.txt b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-base/calendar.txt index 0e75afb73..b8b64fe7a 100755 --- a/src/test/resources/com/conveyal/datatools/gtfs/merge-data-base/calendar.txt +++ b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-base/calendar.txt @@ -1,3 +1,4 @@ service_id,monday,tuesday,wednesday,thursday,friday,saturday,sunday,start_date,end_date common_id,1,1,1,1,1,1,1,20170918,20170920 only_calendar_id,1,1,1,1,1,1,1,20170921,20170922 +dropped_calendar_id,1,1,1,1,1,1,1,20170918,20170919 diff --git a/src/test/resources/com/conveyal/datatools/gtfs/merge-data-base/calendar_dates.txt b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-base/calendar_dates.txt index 2799b08b2..e23e5f104 100644 --- a/src/test/resources/com/conveyal/datatools/gtfs/merge-data-base/calendar_dates.txt +++ b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-base/calendar_dates.txt @@ -1,3 +1,6 @@ service_id,date,exception_type -only_calendar_id,20170919,1 +dropped_calendar_id,20170919,1 +only_calendar_id,20170921,1 +only_calendar_id,20170922,1 +common_id,20170917,1 common_id,20170919,2 diff --git a/src/test/resources/com/conveyal/datatools/gtfs/merge-data-base/stop_times.txt b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-base/stop_times.txt index 646706c5c..977736d8c 100755 --- a/src/test/resources/com/conveyal/datatools/gtfs/merge-data-base/stop_times.txt +++ b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-base/stop_times.txt @@ -2,4 +2,6 @@ trip_id,arrival_time,departure_time,stop_id,stop_sequence,stop_headsign,pickup_t only-calendar-trip1,07:00:00,07:00:00,4u6g,1,,0,0,0.0000000, only-calendar-trip1,07:01:00,07:01:00,johv,2,,0,0,341.4491961, only-calendar-trip2,07:00:00,07:00:00,johv,1,,0,0,0.0000000, -only-calendar-trip2,07:01:00,07:01:00,4u6g,2,,0,0,341.4491961, \ No newline at end of file +only-calendar-trip2,07:01:00,07:01:00,4u6g,2,,0,0,341.4491961, +only-calendar-trip3,07:10:00,07:10:00,johv,1,,0,0,0.0000000, +only-calendar-trip3,07:11:00,07:11:00,4u6g,2,,0,0,341.4491961, \ No newline at end of file diff --git a/src/test/resources/com/conveyal/datatools/gtfs/merge-data-base/trips.txt b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-base/trips.txt index 387b076cd..d8c0d7031 100755 --- a/src/test/resources/com/conveyal/datatools/gtfs/merge-data-base/trips.txt +++ b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-base/trips.txt @@ -1,3 +1,4 @@ route_id,trip_id,trip_headsign,trip_short_name,direction_id,block_id,shape_id,bikes_allowed,wheelchair_accessible,service_id 1,only-calendar-trip1,,,0,,,0,0,common_id -2,only-calendar-trip2,,,0,,,0,0,common_id \ No newline at end of file +2,only-calendar-trip2,,,0,,,0,0,common_id +2,only-calendar-trip3,,,0,,,0,0,only_calendar_id diff --git a/src/test/resources/com/conveyal/datatools/gtfs/merge-data-future/calendar.txt b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-future/calendar.txt index e97e3d013..8d5260fe6 100755 --- a/src/test/resources/com/conveyal/datatools/gtfs/merge-data-future/calendar.txt +++ b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-future/calendar.txt @@ -1,3 +1,3 @@ service_id,monday,tuesday,wednesday,thursday,friday,saturday,sunday,start_date,end_date common_id,1,1,1,1,1,1,1,20170923,20170925 -only_calendar_id,1,1,1,1,1,1,1,20170924,20170927 +only_calendar_id,1,1,1,1,1,1,1,20170920,20170927 diff --git a/src/test/resources/com/conveyal/datatools/gtfs/merge-data-future/stop_times.txt b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-future/stop_times.txt index 646706c5c..977736d8c 100755 --- a/src/test/resources/com/conveyal/datatools/gtfs/merge-data-future/stop_times.txt +++ b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-future/stop_times.txt @@ -2,4 +2,6 @@ trip_id,arrival_time,departure_time,stop_id,stop_sequence,stop_headsign,pickup_t only-calendar-trip1,07:00:00,07:00:00,4u6g,1,,0,0,0.0000000, only-calendar-trip1,07:01:00,07:01:00,johv,2,,0,0,341.4491961, only-calendar-trip2,07:00:00,07:00:00,johv,1,,0,0,0.0000000, -only-calendar-trip2,07:01:00,07:01:00,4u6g,2,,0,0,341.4491961, \ No newline at end of file +only-calendar-trip2,07:01:00,07:01:00,4u6g,2,,0,0,341.4491961, +only-calendar-trip3,07:10:00,07:10:00,johv,1,,0,0,0.0000000, +only-calendar-trip3,07:11:00,07:11:00,4u6g,2,,0,0,341.4491961, \ No newline at end of file diff --git a/src/test/resources/com/conveyal/datatools/gtfs/merge-data-future/trips.txt b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-future/trips.txt index 387b076cd..d8c0d7031 100755 --- a/src/test/resources/com/conveyal/datatools/gtfs/merge-data-future/trips.txt +++ b/src/test/resources/com/conveyal/datatools/gtfs/merge-data-future/trips.txt @@ -1,3 +1,4 @@ route_id,trip_id,trip_headsign,trip_short_name,direction_id,block_id,shape_id,bikes_allowed,wheelchair_accessible,service_id 1,only-calendar-trip1,,,0,,,0,0,common_id -2,only-calendar-trip2,,,0,,,0,0,common_id \ No newline at end of file +2,only-calendar-trip2,,,0,,,0,0,common_id +2,only-calendar-trip3,,,0,,,0,0,only_calendar_id From e14651e5e08a89c59fc8d89e22ada320b8d48014 Mon Sep 17 00:00:00 2001 From: binh-dam-ibigroup <56846598+binh-dam-ibigroup@users.noreply.github.com> Date: Fri, 10 Dec 2021 09:37:08 -0500 Subject: [PATCH 099/122] refactor(MergeLineContext): Remove redundant code. --- .../CalendarAttributesMergeLineContext.java | 36 +---- .../CalendarDatesMergeLineContext.java | 61 ++------- .../feedmerge/CalendarMergeLineContext.java | 60 +-------- .../jobs/feedmerge/MergeLineContext.java | 124 +++++++++++++++--- .../jobs/feedmerge/TripsMergeLineContext.java | 8 +- .../manager/utils/MergeFeedUtils.java | 8 -- 6 files changed, 129 insertions(+), 168 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarAttributesMergeLineContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarAttributesMergeLineContext.java index 1d859f49d..123ddb1cc 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarAttributesMergeLineContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarAttributesMergeLineContext.java @@ -10,10 +10,11 @@ import java.util.Set; import java.util.zip.ZipOutputStream; -import static com.conveyal.datatools.manager.jobs.feedmerge.MergeFeedsType.SERVICE_PERIOD; -import static com.conveyal.datatools.manager.utils.MergeFeedUtils.getTableScopedValue; import static com.conveyal.datatools.manager.utils.MergeFeedUtils.hasDuplicateError; +/** + * Holds the logic for merging entries from the GTFS+ calendar_attributes table. + */ public class CalendarAttributesMergeLineContext extends MergeLineContext { private static final Logger LOG = LoggerFactory.getLogger(CalendarAttributesMergeLineContext.class); @@ -28,9 +29,6 @@ public boolean checkFieldsForMergeConflicts(Set idErrors, FieldCon @Override public void afterRowWrite() throws IOException { - // If the current row is for a calendar service_id that is marked for cloning/renaming, clone the - // values, change the ID, extend the start/end dates to the feed's full range, and write the - // additional line to the file. addClonedServiceId(); } @@ -51,7 +49,7 @@ private boolean checkCalendarIds(Set idErrors, FieldContext fieldC } // Skip record (based on remapped id if necessary) if it was skipped in the calendar table. - String keyInCalendarTable = getTableScopedValue(Table.CALENDAR, getIdScope(), keyValue); + String keyInCalendarTable = getTableScopedValue(Table.CALENDAR, keyValue); if (mergeFeedsResult.skippedIds.contains(keyInCalendarTable)) { LOG.warn( "Skipping calendar entry {} because it was skipped in the merged calendar table.", @@ -61,30 +59,4 @@ private boolean checkCalendarIds(Set idErrors, FieldContext fieldC return !shouldSkipRecord; } - - /** - * Adds a cloned service id for trips with the same signature in both the active & future feeds. - * The cloned service id spans from the start date in the active feed until the end date in the future feed. - * @throws IOException - */ - public void addClonedServiceId() throws IOException { - if (isHandlingFutureFeed() && job.mergeType.equals(SERVICE_PERIOD)) { - String originalServiceId = keyValue; - if (job.serviceIdsToCloneRenameAndExtend.contains(originalServiceId)) { - String[] clonedValues = getOriginalRowValues().clone(); - String newServiceId = clonedValues[keyFieldIndex] = String.join(":", getIdScope(), originalServiceId); - - referenceTracker.checkReferencesAndUniqueness( - keyValue, - getLineNumber(), - table.fields[0], - newServiceId, - table, - keyField, - table.getOrderFieldName() - ); - writeValuesToTable(clonedValues, true); - } - } - } } \ No newline at end of file diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarDatesMergeLineContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarDatesMergeLineContext.java index 40caebf27..a682ef814 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarDatesMergeLineContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarDatesMergeLineContext.java @@ -4,6 +4,7 @@ import com.conveyal.gtfs.error.NewGTFSError; import com.conveyal.gtfs.loader.Table; import com.conveyal.gtfs.model.CalendarDate; +import com.google.common.collect.Iterables; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -13,7 +14,6 @@ import java.util.zip.ZipOutputStream; import static com.conveyal.datatools.manager.jobs.feedmerge.MergeFeedsType.SERVICE_PERIOD; -import static com.conveyal.datatools.manager.utils.MergeFeedUtils.getTableScopedValue; import static com.conveyal.gtfs.loader.DateField.GTFS_DATE_FORMATTER; /** @@ -31,7 +31,7 @@ public CalendarDatesMergeLineContext(MergeFeedsJob job, Table table, ZipOutputSt @Override public boolean checkFieldsForMergeConflicts(Set idErrors, FieldContext fieldContext) throws IOException { - return checkCalendarDatesIds(idErrors, fieldContext); + return checkCalendarDatesIds(fieldContext); } @Override @@ -48,22 +48,14 @@ public void startNewFeed(int feedIndex) throws IOException { futureFeedFirstDateForCalendarValidity = getFutureFeedFirstDateForCheckingCalendarValidity(); } - private boolean checkCalendarDatesIds(Set idErrors, FieldContext fieldContext) throws IOException { + private boolean checkCalendarDatesIds(FieldContext fieldContext) throws IOException { boolean shouldSkipRecord = false; - String key = getTableScopedValue(table, getIdScope(), keyValue); - // TODO: REfactor - String scopedId = String.join(":", getIdScope(), keyValue); + String key = getTableScopedValue(keyValue); // Drop any calendar_dates.txt records from the existing feed for dates that are - // not before the first date of the future feed - // and also for service ids not in the merged calendar table - // (we can determine that because the calendar table has already been processed). + // not before the first date of the future feed. LocalDate date = getCsvDate("date"); if ( - isHandlingActiveFeed() && - ( - //!job.mergeFeedsResult.serviceIds.contains(scopedId) || - !date.isBefore(futureFeedFirstDateForCalendarValidity) - ) + isHandlingActiveFeed() && !date.isBefore(futureFeedFirstDateForCalendarValidity) ) { LOG.warn( "Skipping calendar_dates entry {} because it operates in the time span of future feed (i.e., after or on {}).", @@ -73,33 +65,10 @@ private boolean checkCalendarDatesIds(Set idErrors, FieldContext f shouldSkipRecord = true; } - // TODO: refactor below. - if (job.mergeType.equals(SERVICE_PERIOD)) { - if (isHandlingActiveFeed()) { - // Remove calendar entries that are no longer used. - if (feedMergeContext.active.getServiceIdsToRemove().contains(keyValue)) { - LOG.warn( - "Skipping active calendar_dates entry {} because it will become unused in the merged feed.", - keyValue); - mergeFeedsResult.skippedIds.add(key); - shouldSkipRecord = true; - } - } else { - // If handling the future feed, the MTC revised feed merge logic is as follows: - // - Calendar entries from the future feed will be inserted as is in the merged feed. - // so no additional processing needed here, unless the calendar entry is no longer used, - // in that case we drop the calendar entry. - if (feedMergeContext.future.getServiceIdsToRemove().contains(keyValue)) { - LOG.warn( - "Skipping future calendar_dates entry {} because it will become unused in the merged feed.", - keyValue); - mergeFeedsResult.skippedIds.add(key); - shouldSkipRecord = true; - } - } + if (job.mergeType.equals(SERVICE_PERIOD) && isServiceIdUnused()) { + shouldSkipRecord = true; } - // Track service ID because we want to avoid removing trips that may reference this // service_id when the service_id is used by calendar.txt records that operate in // the valid date range, i.e., before the future feed's first date. @@ -132,23 +101,19 @@ private LocalDate getFutureFeedFirstDateForCheckingCalendarValidity() { /** * Adds a cloned service id for trips with the same signature in both the active & future feeds. * The cloned service id spans from the start date in the active feed until the end date in the future feed. - * @throws IOException */ public void addClonedServiceIds() throws IOException { if (job.mergeType.equals(SERVICE_PERIOD)) { for (String id : job.serviceIdsToCloneRenameAndExtend) { - String newServiceId = String.join(":", getClonedIdScope(), id); + String newServiceId = getIdWithScope(id, getClonedIdScope()); // Because this service has been extended to span both active and future feed, // we need to add all calendar_dates entries for the original service id // under the active AND future feed (and of course rename service id). - // TODO: refactor - for (CalendarDate calDate : feedMergeContext.active.feed.calendarDates.getAll()) { - if (calDate.service_id.equals(id)) { - writeValuesToTable(getCalendarRowValues(calDate, newServiceId), true); - } - } - for (CalendarDate calDate : feedMergeContext.future.feed.calendarDates.getAll()) { + for (CalendarDate calDate : Iterables.concat( + feedMergeContext.active.feed.calendarDates.getAll(), + feedMergeContext.future.feed.calendarDates.getAll() + )) { if (calDate.service_id.equals(id)) { writeValuesToTable(getCalendarRowValues(calDate, newServiceId), true); } diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarMergeLineContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarMergeLineContext.java index 7df3f8c58..ab8902531 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarMergeLineContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarMergeLineContext.java @@ -12,8 +12,6 @@ import java.util.Set; import java.util.zip.ZipOutputStream; -import static com.conveyal.datatools.manager.jobs.feedmerge.MergeFeedsType.SERVICE_PERIOD; -import static com.conveyal.datatools.manager.utils.MergeFeedUtils.getTableScopedValue; import static com.conveyal.datatools.manager.utils.MergeFeedUtils.hasDuplicateError; import static com.conveyal.gtfs.loader.DateField.GTFS_DATE_FORMATTER; @@ -34,15 +32,12 @@ public boolean checkFieldsForMergeConflicts(Set idErrors, FieldCon @Override public void afterRowWrite() throws IOException { - // If the current row is for a calendar service_id that is marked for cloning/renaming, clone the - // values, change the ID, extend the start/end dates to the feed's full range, and write the - // additional line to the file. addClonedServiceId(); } private boolean checkCalendarIds(Set idErrors, FieldContext fieldContext) throws IOException { boolean shouldSkipRecord = false; - String key = getTableScopedValue(table, getIdScope(), keyValue); + String key = getTableScopedValue(keyValue); if (isHandlingActiveFeed()) { LocalDate startDate = getCsvDate("start_date"); @@ -90,29 +85,11 @@ private boolean checkCalendarIds(Set idErrors, FieldContext fieldC .format(GTFS_DATE_FORMATTER)); } } - - // Remove calendar entries that are no longer used. - if (feedMergeContext.active.getServiceIdsToRemove().contains(keyValue)) { - LOG.warn( - "Skipping active calendar entry {} because it will become unused in the merged feed.", - keyValue); - mergeFeedsResult.skippedIds.add(key); - shouldSkipRecord = true; - } - } else { - // If handling the future feed, the MTC revised feed merge logic is as follows: - // - Calendar entries from the future feed will be inserted as is in the merged feed. - // so no additional processing needed here, unless the calendar entry is no longer used, - // in that case we drop the calendar entry. - if (feedMergeContext.future.getServiceIdsToRemove().contains(keyValue)) { - LOG.warn( - "Skipping future calendar entry {} because it will become unused in the merged feed.", - keyValue); - mergeFeedsResult.skippedIds.add(key); - shouldSkipRecord = true; - } } + if (isServiceIdUnused()) { + shouldSkipRecord = true; + } // If any service_id in the active feed matches with the future // feed, it should be modified and all associated trip records @@ -138,33 +115,4 @@ private boolean checkCalendarIds(Set idErrors, FieldContext fieldC return !shouldSkipRecord; } - - /** - * Adds a cloned service id for trips with the same signature in both the active & future feeds. - * The cloned service id spans from the start date in the active feed until the end date in the future feed. - * @throws IOException - */ - public void addClonedServiceId() throws IOException { - if (isHandlingFutureFeed() && job.mergeType.equals(SERVICE_PERIOD)) { - String originalServiceId = keyValue; - if (job.serviceIdsToCloneRenameAndExtend.contains(originalServiceId)) { - String[] clonedValues = getOriginalRowValues().clone(); - String newServiceId = clonedValues[keyFieldIndex] = String.join(":", getIdScope(), originalServiceId); - // Modify start date only (preserve the end date from the future calendar entry). - int startDateIndex = Table.CALENDAR.getFieldIndex("start_date"); - clonedValues[startDateIndex] = feedMergeContext.active.feed.calendars.get(originalServiceId).start_date - .format(GTFS_DATE_FORMATTER); - referenceTracker.checkReferencesAndUniqueness( - keyValue, - getLineNumber(), - table.fields[0], - newServiceId, - table, - keyField, - table.getOrderFieldName() - ); - writeValuesToTable(clonedValues, true); - } - } - } } \ No newline at end of file diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java index 31d6c3e12..b5e549ee4 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java @@ -32,7 +32,6 @@ import static com.conveyal.datatools.manager.utils.MergeFeedUtils.containsField; import static com.conveyal.datatools.manager.utils.MergeFeedUtils.getAllFields; import static com.conveyal.datatools.manager.utils.MergeFeedUtils.getMergeKeyField; -import static com.conveyal.datatools.manager.utils.MergeFeedUtils.getTableScopedValue; import static com.conveyal.datatools.manager.utils.MergeFeedUtils.hasDuplicateError; import static com.conveyal.datatools.manager.utils.StringUtils.getCleanName; import static com.conveyal.gtfs.loader.DateField.GTFS_DATE_FORMATTER; @@ -127,8 +126,7 @@ public void startNewFeed(int feedIndex) throws IOException { orderField = table.getOrderFieldName(); keyFieldMissing = false; - // Generate ID prefix to scope GTFS identifiers to avoid conflicts. - idScope = getCleanName(feedSource.name) + version.version; + idScope = makeIdScope(version); csvReader = table.getCsvReader(feed.zipFile, null); // If csv reader is null, the table was not found in the zip file. There is no need // to handle merging this table for this zip file. @@ -154,12 +152,27 @@ public void startNewFeed(int feedIndex) throws IOException { if (handlingFutureFeed) { mergeFeedsResult.serviceIds.addAll( job.serviceIdsToCloneRenameAndExtend.stream().map( - id -> String.join(":", idScope, id) + this::getIdWithScope ).collect(Collectors.toSet()) ); } } + /** + * Returns a scoped identifier of the form e.g. FeedName3:some_id + * (to distinguish an id when used in multiple tables). + */ + protected String getIdWithScope(String id, String scope) { + return String.join(":", scope, id); + } + + /** + * Shorthand for above using current idScope. + */ + protected String getIdWithScope(String id) { + return getIdWithScope(id, idScope); + } + public boolean shouldSkipFile() { if (handlingActiveFeed && job.mergeType.equals(SERVICE_PERIOD)) { // Always prefer the "future" file for the feed_info table, which means @@ -223,7 +236,7 @@ public void startNewRow() throws IOException { public boolean checkForeignReferences(FieldContext fieldContext) throws IOException { Field field = fieldContext.getField(); if (field.isForeignReference()) { - String key = getTableScopedValue(field.referenceTable, idScope, fieldContext.getValue()); + String key = getTableScopedValue(field.referenceTable, fieldContext.getValue()); // Check if we're performing a service period merge, this ref field is a service_id, and it // is not found in the list of service_ids (e.g., it was removed). boolean isValidServiceId = mergeFeedsResult.serviceIds.contains(fieldContext.getValueToWrite()); @@ -239,10 +252,9 @@ public boolean checkForeignReferences(FieldContext fieldContext) throws IOExcept if (fieldContext.nameEquals(SERVICE_ID) && isValidServiceId) { LOG.warn("Not skipping valid service_id {} for {} {}", fieldContext.getValueToWrite(), table.name, keyValue); } else { - String skippedKey = getTableScopedValue(table, idScope, keyValue); + String skippedKey = getTableScopedValue(keyValue); if (orderField != null) { - skippedKey = String.join(":", skippedKey, - getCsvValue(orderField)); + skippedKey = String.join(":", skippedKey, getCsvValue(orderField)); } mergeFeedsResult.skippedIds.add(skippedKey); return false; @@ -327,7 +339,7 @@ protected boolean checkRoutesAndStopsIds(Set idErrors, FieldContex String currentPrimaryKey = rowValues[0]; // Get unique key to check for remapped ID when // writing values to file. - String key = getTableScopedValue(table, idScope, currentPrimaryKey); + String key = getTableScopedValue(currentPrimaryKey); // Extract the route/stop ID value used for the // route/stop with already encountered matching // short name/stop code. @@ -410,7 +422,7 @@ public boolean updateAgencyIdIfNeeded(FieldContext fieldContext) { return true; } - public boolean updateServiceIdsIfNeeded(FieldContext fieldContext) { + private void updateServiceIdsIfNeeded(FieldContext fieldContext) { String fieldValue = fieldContext.getValue(); if (table.name.equals(Table.TRIPS.name) && fieldContext.nameEquals(SERVICE_ID) && @@ -420,12 +432,11 @@ public boolean updateServiceIdsIfNeeded(FieldContext fieldContext) { // Future trip ids not in the active feed will not get the service id remapped, // they will use the service id as defined in the future feed instead. if (!(handlingFutureFeed && feedMergeContext.getFutureTripIdsNotInActiveFeed().contains(keyValue))) { - String newServiceId = String.join(":", idScope, fieldValue); + String newServiceId = getIdWithScope(fieldValue); LOG.info("Updating {}#service_id to (auto-generated) {} for ID {}", table.name, newServiceId, keyValue); fieldContext.setValueToWrite(newServiceId); } } - return true; } public boolean storeRowAndStopValues() { @@ -497,7 +508,7 @@ public void scopeValueIfNeeded(FieldContext fieldContext) { if (job.mergeType.equals(REGIONAL) && isKeyField && !fieldContext.getValue().isEmpty()) { // For regional merge, if field is a GTFS identifier (e.g., route_id, // stop_id, etc.), add scoped prefix. - fieldContext.setValueToWrite(String.join(":", idScope, fieldContext.getValue())); + fieldContext.setValueToWrite(getIdWithScope(fieldContext.getValue())); } } @@ -648,8 +659,26 @@ protected int getFieldIndex(String fieldName) { return Field.getFieldIndex(fieldsFoundInZip, fieldName); } - protected String getIdScope() { - return idScope; + /** + * Generate ID prefix to scope GTFS identifiers to avoid conflicts. + */ + private String makeIdScope(FeedVersion version) { + return getCleanName(feedSource.name) + version.version; + } + + /** Get table-scoped value used for key when remapping references for a particular feed. */ + protected String getTableScopedValue(Table table, String id) { + return String.join( + ":", + table.name, + idScope, + id + ); + } + + /** Shorthand for above using ambient table. */ + protected String getTableScopedValue(String id) { + return getTableScopedValue(table, id); } /** @@ -657,8 +686,7 @@ protected String getIdScope() { * It is set to the id scope corresponding to the future feed. */ protected String getClonedIdScope() { - // TODO: refactor name creation - return getCleanName(feedSource.name) + this.feedMergeContext.future.feedToMerge.version.version; + return makeIdScope(feedMergeContext.future.feedToMerge.version); } protected int getFeedIndex() { return feedIndex; } @@ -689,13 +717,13 @@ protected LocalDate getCsvDate(String fieldName) throws IOException { */ protected void updateAndRemapOutput(FieldContext fieldContext, boolean updateKeyValue) { String value = fieldContext.getValue(); - String valueToWrite = String.join(":", idScope, value); + String valueToWrite = getIdWithScope(value); fieldContext.setValueToWrite(valueToWrite); if (updateKeyValue) { keyValue = valueToWrite; } mergeFeedsResult.remappedIds.put( - getTableScopedValue(table, idScope, value), + getTableScopedValue(value), valueToWrite ); } @@ -724,4 +752,62 @@ protected void addField(Field field) { protected int getKeyFieldIndex() { return table.getKeyFieldIndex(fieldsFoundInZip); } + + /** + * Helper method that determines whether a service id for the + * current calendar-related table is unused or not. + */ + protected boolean isServiceIdUnused() { + boolean isUnused = false; + FeedContext feedContext = handlingActiveFeed ? feedMergeContext.active : feedMergeContext.future; + + if (feedContext.getServiceIdsToRemove().contains(keyValue)) { + String activeOrFuture = handlingActiveFeed ? "active" : "future"; + LOG.warn( + "Skipping {} {} entry {} because it will become unused in the merged feed.", + activeOrFuture, + table.name, + keyValue + ); + + mergeFeedsResult.skippedIds.add(getTableScopedValue(keyValue)); + + isUnused = true; + } + + return isUnused; + } + + /** + * Adds a cloned service id for trips with the same signature in both the active & future feeds. + * The cloned service id spans from the start date in the active feed until the end date in the future feed. + * If dealing with the calendar table, this will update the start_date field accordingly. + */ + public void addClonedServiceId() throws IOException { + if (isHandlingFutureFeed() && job.mergeType.equals(SERVICE_PERIOD)) { + String originalServiceId = keyValue; + if (job.serviceIdsToCloneRenameAndExtend.contains(originalServiceId)) { + String[] clonedValues = getOriginalRowValues().clone(); + String newServiceId = clonedValues[keyFieldIndex] = getIdWithScope(originalServiceId); + + if (table.name.equals(Table.CALENDAR.name)) { + // Modify start date only (preserve the end date from the future calendar entry). + int startDateIndex = Table.CALENDAR.getFieldIndex("start_date"); + clonedValues[startDateIndex] = feedMergeContext.active.feed.calendars.get(originalServiceId) + .start_date.format(GTFS_DATE_FORMATTER); + } + + referenceTracker.checkReferencesAndUniqueness( + keyValue, + getLineNumber(), + table.fields[0], + newServiceId, + table, + keyField, + table.getOrderFieldName() + ); + writeValuesToTable(clonedValues, true); + } + } + } } \ No newline at end of file diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/TripsMergeLineContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/TripsMergeLineContext.java index 81cbd10b8..859708bb7 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/TripsMergeLineContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/TripsMergeLineContext.java @@ -9,7 +9,6 @@ import java.util.zip.ZipOutputStream; import static com.conveyal.datatools.manager.jobs.feedmerge.MergeFeedsType.SERVICE_PERIOD; -import static com.conveyal.datatools.manager.utils.MergeFeedUtils.getTableScopedValue; import static com.conveyal.datatools.manager.utils.MergeFeedUtils.hasDuplicateError; public class TripsMergeLineContext extends MergeLineContext { @@ -41,10 +40,9 @@ private boolean checkTripIds(Set idErrors, FieldContext fieldConte // (the remapped id is already listed under the calendar/calendar_dates tables, // so there is no need to add that foreign key again). if (fieldContext.nameEquals(SERVICE_ID)) { - String tableScopedValue = getTableScopedValue(table, getIdScope(), fieldContext.getValue()); - if (mergeFeedsResult.remappedIds.containsKey(tableScopedValue)) { - mergeFeedsResult.remappedIds.remove(tableScopedValue); - } + mergeFeedsResult.remappedIds.remove( + getTableScopedValue(fieldContext.getValue()) + ); } return !shouldSkipRecord; diff --git a/src/main/java/com/conveyal/datatools/manager/utils/MergeFeedUtils.java b/src/main/java/com/conveyal/datatools/manager/utils/MergeFeedUtils.java index 6a1bcca1c..0205ef003 100644 --- a/src/main/java/com/conveyal/datatools/manager/utils/MergeFeedUtils.java +++ b/src/main/java/com/conveyal/datatools/manager/utils/MergeFeedUtils.java @@ -149,14 +149,6 @@ public static boolean hasDuplicateError(Set errors) { return false; } - /** Get table-scoped value used for key when remapping references for a particular feed. */ - public static String getTableScopedValue(Table table, String prefix, String id) { - return String.join(":", - table.name, - prefix, - id); - } - /** * Checks whether the future and active stop_times for a particular trip_id are an exact match, * using these criteria only: arrival_time, departure_time, stop_id, and stop_sequence From 960406aefb8b753dd18951c7ec2e3d398edd6c57 Mon Sep 17 00:00:00 2001 From: binh-dam-ibigroup <56846598+binh-dam-ibigroup@users.noreply.github.com> Date: Fri, 10 Dec 2021 10:40:06 -0500 Subject: [PATCH 100/122] refactor(CalendarDatesMergeLineContext): Refactor code. --- .../CalendarDatesMergeLineContext.java | 67 +++++++++++++------ 1 file changed, 45 insertions(+), 22 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarDatesMergeLineContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarDatesMergeLineContext.java index a682ef814..a772f8075 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarDatesMergeLineContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarDatesMergeLineContext.java @@ -4,13 +4,17 @@ import com.conveyal.gtfs.error.NewGTFSError; import com.conveyal.gtfs.loader.Table; import com.conveyal.gtfs.model.CalendarDate; -import com.google.common.collect.Iterables; +import com.google.common.collect.Lists; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; import java.time.LocalDate; +import java.util.ArrayList; +import java.util.List; import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.StreamSupport; import java.util.zip.ZipOutputStream; import static com.conveyal.datatools.manager.jobs.feedmerge.MergeFeedsType.SERVICE_PERIOD; @@ -50,23 +54,26 @@ public void startNewFeed(int feedIndex) throws IOException { private boolean checkCalendarDatesIds(FieldContext fieldContext) throws IOException { boolean shouldSkipRecord = false; - String key = getTableScopedValue(keyValue); - // Drop any calendar_dates.txt records from the existing feed for dates that are - // not before the first date of the future feed. - LocalDate date = getCsvDate("date"); - if ( - isHandlingActiveFeed() && !date.isBefore(futureFeedFirstDateForCalendarValidity) - ) { - LOG.warn( - "Skipping calendar_dates entry {} because it operates in the time span of future feed (i.e., after or on {}).", - keyValue, - futureFeedFirstDateForCalendarValidity); - mergeFeedsResult.skippedIds.add(key); - shouldSkipRecord = true; - } + if (job.mergeType.equals(SERVICE_PERIOD)) { + // Drop any calendar_dates.txt records from the existing feed for dates that are + // not before the first date of the future feed. + LocalDate date = getCsvDate("date"); + if ( + isHandlingActiveFeed() && !isBeforeFutureFeedStartDate(date) + ) { + String key = getTableScopedValue(keyValue); + LOG.warn( + "Skipping calendar_dates entry {} because it operates in the time span of future feed (i.e., after or on {}).", + keyValue, + futureFeedFirstDateForCalendarValidity + ); + mergeFeedsResult.skippedIds.add(key); + shouldSkipRecord = true; + } - if (job.mergeType.equals(SERVICE_PERIOD) && isServiceIdUnused()) { - shouldSkipRecord = true; + if (isServiceIdUnused()) { + shouldSkipRecord = true; + } } // Track service ID because we want to avoid removing trips that may reference this @@ -98,22 +105,38 @@ private LocalDate getFutureFeedFirstDateForCheckingCalendarValidity() { return futureFeedFirstDate; } + private boolean isBeforeFutureFeedStartDate(LocalDate date) { + return date.isBefore(futureFeedFirstDateForCalendarValidity); + } + /** * Adds a cloned service id for trips with the same signature in both the active & future feeds. * The cloned service id spans from the start date in the active feed until the end date in the future feed. */ public void addClonedServiceIds() throws IOException { if (job.mergeType.equals(SERVICE_PERIOD)) { + String clonedIdScope = getClonedIdScope(); + + // Retrieve all active and future calendar dates ahead + // to avoid repeat database get-all queries, + // and exclude active entries with a date after the future feed start date. + List allCalendarDates = new ArrayList<>(); + allCalendarDates.addAll(Lists.newArrayList( + StreamSupport.stream(feedMergeContext.active.feed.calendarDates.spliterator(), false) + .filter(calDate -> isBeforeFutureFeedStartDate(calDate.date)) + .collect(Collectors.toList()) + )); + allCalendarDates.addAll(Lists.newArrayList( + feedMergeContext.future.feed.calendarDates.getAll() + )); + for (String id : job.serviceIdsToCloneRenameAndExtend) { - String newServiceId = getIdWithScope(id, getClonedIdScope()); + String newServiceId = getIdWithScope(id, clonedIdScope); // Because this service has been extended to span both active and future feed, // we need to add all calendar_dates entries for the original service id // under the active AND future feed (and of course rename service id). - for (CalendarDate calDate : Iterables.concat( - feedMergeContext.active.feed.calendarDates.getAll(), - feedMergeContext.future.feed.calendarDates.getAll() - )) { + for (CalendarDate calDate : allCalendarDates) { if (calDate.service_id.equals(id)) { writeValuesToTable(getCalendarRowValues(calDate, newServiceId), true); } From 515db2d1453b2ed703b088417636384d1f412182 Mon Sep 17 00:00:00 2001 From: binh-dam-ibigroup <56846598+binh-dam-ibigroup@users.noreply.github.com> Date: Fri, 10 Dec 2021 15:59:38 -0500 Subject: [PATCH 101/122] refactor(SqlAssert): Extract test util class. --- .../manager/jobs/MergeFeedsJobTest.java | 453 +++++------------- .../datatools/manager/utils/SqlAssert.java | 74 +++ 2 files changed, 186 insertions(+), 341 deletions(-) create mode 100644 src/test/java/com/conveyal/datatools/manager/utils/SqlAssert.java diff --git a/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java b/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java index 008b9e448..ddcc4b2b1 100644 --- a/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java +++ b/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java @@ -10,6 +10,7 @@ import com.conveyal.datatools.manager.models.FeedVersion; import com.conveyal.datatools.manager.models.Project; import com.conveyal.datatools.manager.persistence.Persistence; +import com.conveyal.datatools.manager.utils.SqlAssert; import com.conveyal.gtfs.error.NewGTFSErrorType; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; @@ -24,7 +25,6 @@ import java.util.Set; import static com.conveyal.datatools.TestUtils.assertThatFeedHasNoErrorsOfType; -import static com.conveyal.datatools.TestUtils.assertThatSqlCountQueryYieldsExpectedCount; import static com.conveyal.datatools.TestUtils.createFeedVersion; import static com.conveyal.datatools.TestUtils.createFeedVersionFromGtfsZip; import static com.conveyal.datatools.TestUtils.zipFolderFiles; @@ -230,69 +230,34 @@ void canMergeRegionalWithOnlyCalendarFeed () throws SQLException { versions.add(onlyCalendarDatesVersion); versions.add(onlyCalendarVersion); FeedVersion mergedVersion = regionallyMergeVersions(versions); - - // assert service_ids have been feed scoped properly - String mergedNamespace = mergedVersion.namespace; + SqlAssert sqlAssert = new SqlAssert(mergedVersion); // - calendar table should have 2 records. - assertRowCountInTable(mergedNamespace, "calendar", 2); + sqlAssert.calendar.assertCount(2); // onlyCalendarVersion's common_id service_id should be scoped - assertThatSqlCountQueryYieldsExpectedCount( - String.format( - "SELECT count(*) FROM %s.calendar WHERE service_id='Fake_Agency2:common_id'", - mergedNamespace - ), - 1 - ); + sqlAssert.calendar.assertCount(1, "service_id='Fake_Agency2:common_id'"); + // onlyCalendarVersion's only_calendar_id service_id should be scoped - assertThatSqlCountQueryYieldsExpectedCount( - String.format( - "SELECT count(*) FROM %s.calendar WHERE service_id='Fake_Agency2:only_calendar_id'", - mergedNamespace - ), - 1 - ); + sqlAssert.calendar.assertCount(1, "service_id='Fake_Agency2:only_calendar_id'"); // - calendar_dates table should have 2 records. - assertRowCountInTable(mergedNamespace, "calendar_dates", 2); + sqlAssert.calendarDates.assertCount(2); // onlyCalendarDatesVersion's common_id service_id should be scoped - assertThatSqlCountQueryYieldsExpectedCount( - String.format( - "SELECT count(*) FROM %s.calendar_dates WHERE service_id='Fake_Agency3:common_id'", - mergedNamespace - ), - 1 - ); + sqlAssert.calendarDates.assertCount(1, "service_id='Fake_Agency3:common_id'"); + // onlyCalendarDatesVersion's only_calendar_dates_id service_id should be scoped - assertThatSqlCountQueryYieldsExpectedCount( - String.format( - "SELECT count(*) FROM %s.calendar_dates WHERE service_id='Fake_Agency3:only_calendar_dates_id'", - mergedNamespace - ), - 1 - ); + sqlAssert.calendarDates.assertCount(1, "service_id='Fake_Agency3:only_calendar_dates_id'"); // - trips table should have 2 + 1 = 3 records. - assertRowCountInTable(mergedNamespace, "trips", 3); + sqlAssert.trips.assertCount(3); // onlyCalendarDatesVersion's common_id service_id should be scoped - assertThatSqlCountQueryYieldsExpectedCount( - String.format( - "SELECT count(*) FROM %s.trips WHERE service_id='Fake_Agency3:common_id'", - mergedNamespace - ), - 1 - ); + sqlAssert.trips.assertCount(1, "service_id='Fake_Agency3:common_id'"); + // 2 trips with onlyCalendarVersion's common_id service_id should be scoped - assertThatSqlCountQueryYieldsExpectedCount( - String.format( - "SELECT count(*) FROM %s.trips WHERE service_id='Fake_Agency2:common_id'", - mergedNamespace - ), - 2 - ); + sqlAssert.trips.assertCount(2, "service_id='Fake_Agency2:common_id'"); } /** @@ -334,20 +299,15 @@ void mergeMTCShouldHandleExtendFutureStrategy() throws SQLException { MergeStrategy.CHECK_STOP_TIMES, mergeFeedsJob.mergeFeedsResult.mergeStrategy ); - // assert service_ids start_dates have been extended to the start_date of the base feed. - String mergedNamespace = mergeFeedsJob.mergedVersion.namespace; - - assertNoUnusedServiceIds(mergedNamespace); - assertNoRefIntegrityErrors(mergedNamespace); + SqlAssert sqlAssert = new SqlAssert(mergeFeedsJob.mergedVersion); + sqlAssert.assertNoUnusedServiceIds(); + sqlAssert.assertNoRefIntegrityErrors(); // calendar table should have 2 records (all calendar ids are used and extended) - assertRowCountInTable(mergedNamespace, "calendar", 2); + sqlAssert.calendar.assertCount(2); // expect that the record in calendar table has the correct start_date. - assertThatSqlCountQueryYieldsExpectedCount( - String.format("SELECT count(*) FROM %s.calendar where start_date = '20170918' and monday = 1", mergedNamespace), - 1 - ); + sqlAssert.calendar.assertCount(1, "start_date='20170918' and monday=1"); } /** @@ -373,8 +333,9 @@ void mergeMTCShouldHandleMatchingTripIdsWithSameSignature() throws SQLException mergeFeedsJob.mergeFeedsResult.failed, "Merge feeds job should succeed with CHECK_STOP_TIMES strategy." ); - - String mergedNamespace = mergeFeedsJob.mergedVersion.namespace; + SqlAssert sqlAssert = new SqlAssert(mergeFeedsJob.mergedVersion); + sqlAssert.assertNoUnusedServiceIds(); + sqlAssert.assertNoRefIntegrityErrors(); // - calendar table // expect a total of 4 records in calendar table: @@ -383,7 +344,7 @@ void mergeMTCShouldHandleMatchingTripIdsWithSameSignature() throws SQLException // - common_id cloned and extended for the matching trip id present in both active and future feeds // (from MergeFeedsJob#serviceIdsToCloneAndRename), // - only_calendar_id used in the future feed. - assertRowCountInTable(mergedNamespace, "calendar", 4); + sqlAssert.calendar.assertCount(4); // Expect 4 trips in merged output: // 1 trip from active feed that are not in the future feed, @@ -391,29 +352,19 @@ void mergeMTCShouldHandleMatchingTripIdsWithSameSignature() throws SQLException // starts after the future feed start date) // 1 trip in both the active and future feeds, with the same signature (same stop times), // 2 trips from the future feed not in the active feed. - assertRowCountInTable(mergedNamespace, "trips", 4); - - assertNoUnusedServiceIds(mergedNamespace); - assertNoRefIntegrityErrors(mergedNamespace); + sqlAssert.trips.assertCount(4); // expect that 2 calendars (1 common_id extended from future and 1 Fake_Transit1:common_id from active) have // start_date pinned to start date of active feed. - assertThatSqlCountQueryYieldsExpectedCount( - String.format("SELECT count(*) FROM %s.calendar where start_date = '20170918'", mergedNamespace), - 2 - ); + sqlAssert.calendar.assertCount(2, "start_date='20170918'"); + // One of the calendars above should have been extended // until the end date of that entry in the future feed. - assertThatSqlCountQueryYieldsExpectedCount( - String.format("SELECT count(*) FROM %s.calendar where start_date = '20170918' and end_date='20170925'", mergedNamespace), - 1 - ); + sqlAssert.calendar.assertCount(1, "start_date='20170918' and end_date='20170925'"); + // The other one should have end_date set to a day before the start of the future feed start date // (in the test data, that first date comes from the other calendar entry). - assertThatSqlCountQueryYieldsExpectedCount( - String.format("SELECT count(*) FROM %s.calendar where start_date = '20170918' and end_date='20170919'", mergedNamespace), - 1 - ); + sqlAssert.calendar.assertCount(1, "start_date = '20170918' and end_date='20170919'"); } /** @@ -440,7 +391,9 @@ void mergeMTCShouldHandleMatchingTripIdsAndDropUnusedFutureCalendar() throws Exc "Merge feeds job should succeed with CHECK_STOP_TIMES strategy." ); - String mergedNamespace = mergeFeedsJob.mergedVersion.namespace; + SqlAssert sqlAssert = new SqlAssert(mergeFeedsJob.mergedVersion); + sqlAssert.assertNoUnusedServiceIds(); + sqlAssert.assertNoRefIntegrityErrors(); // - calendar table // expect a total of 3 records in calendar table: @@ -448,7 +401,7 @@ void mergeMTCShouldHandleMatchingTripIdsAndDropUnusedFutureCalendar() throws Exc // - common_id cloned and extended for the matching trip id present in both active and future feeds // (from MergeFeedsJob#serviceIdsToCloneAndRename), // - only_calendar_id used in the future feed. - assertRowCountInTable(mergedNamespace, "calendar", 3); + sqlAssert.calendar.assertCount(3); // Expect 3 trips in merged output: // 1 trip from active feed that are not in the future feed, @@ -456,46 +409,22 @@ void mergeMTCShouldHandleMatchingTripIdsAndDropUnusedFutureCalendar() throws Exc // starts after the future feed start date) // 1 trip in both the active and future feeds, with the same signature (same stop times), // 1 trip from the future feed not in the active feed. - assertRowCountInTable(mergedNamespace, "trips", 3); + sqlAssert.trips.assertCount(3); // 5 calendar_dates entries should be in the merged feed: + // (reported by MTC). + sqlAssert.calendarDates.assertCount(5); // - only_calendar_id: - // 1 from future feed, + // 1 from future feed (that service id is not scoped), + sqlAssert.calendarDates.assertCount(1, "service_id = 'only_calendar_id'"); // 0 from active feed // (in the active feed, that service id starts after the future feed start date) + sqlAssert.calendarDates.assertCount(0, "service_id = 'Fake_Transit1:dropped_calendar_id'"); // - common_id: // 2 from active feed for the calendar item that was extended due to shared trip, + sqlAssert.calendarDates.assertCount(2, "service_id = 'Fake_Transit7:common_id'"); // 2 from active feed for the active trip not in the future feed. - // See also specific query for each entry underneath. - // (reported by MTC). - - // The entries from calendar_dates for the service_id that is used the active feed - // and that is not in the future feed should be present. - assertThatSqlCountQueryYieldsExpectedCount( - String.format("SELECT count(*) FROM %s.calendar_dates where service_id = 'Fake_Transit1:common_id'", mergedNamespace), - 2 - ); - - // The entries from calendar_dates for the extended service_id from active feed should be present. - assertThatSqlCountQueryYieldsExpectedCount( - String.format("SELECT count(*) FROM %s.calendar_dates where service_id = 'Fake_Transit7:common_id'", mergedNamespace), - 2 - ); - - // The entries from calendar_dates for the active service_id that is entirely in the future feed - // should be present (that service id is not scoped). - assertThatSqlCountQueryYieldsExpectedCount( - String.format("SELECT count(*) FROM %s.calendar_dates where service_id = 'only_calendar_id'", mergedNamespace), - 1 - ); - - // Unused calendar_dates service_ids in the active feed should still be dropped. - assertThatSqlCountQueryYieldsExpectedCount( - String.format("SELECT count(*) FROM %s.calendar_dates where service_id = 'Fake_Transit1:dropped_calendar_id' and date='20170919' and exception_type = 1", mergedNamespace), - 0 - ); - - assertRowCountInTable(mergedNamespace, "calendar_dates", 5); + sqlAssert.calendarDates.assertCount(2, "service_id = 'Fake_Transit1:common_id'"); // The GTFS+ calendar_attributes table should contain the same number of entries as the calendar table // (reported by MTC). @@ -526,9 +455,6 @@ void mergeMTCShouldHandleMatchingTripIdsAndDropUnusedFutureCalendar() throws Exc ).count(), "Job summary should not mention remapped uninserted trip ids." ); - - assertNoUnusedServiceIds(mergedNamespace); - assertNoRefIntegrityErrors(mergedNamespace); } /** @@ -577,31 +503,23 @@ void mergeMTCShouldHandleDisjointTripIds() throws SQLException { mergeFeedsJob.mergeFeedsResult.failed, "Merge feeds job should utilize DEFAULT strategy." ); - // assert service_ids start_dates have been extended to the start_date of the base feed. - String mergedNamespace = mergeFeedsJob.mergedVersion.namespace; - // - calendar table - // expect a total of 4 records in calendar table + SqlAssert sqlAssert = new SqlAssert(mergeFeedsJob.mergedVersion); + + // calendar table should have 4 records // - 2 records from future feed, including only_calendar_dates which absorbs its active counterpart, // - 1 record from active feed that is used // - 1 unused record from the active feed that is NOT discarded (default strategy). - assertRowCountInTable(mergedNamespace, "calendar", 4); + sqlAssert.calendar.assertCount(4); // The calendar entry for the active feed ending 20170920 should end one day before the first calendar start date // of the future feed. - final String activeCalendarNewEndDate = "20170919"; // One day before 20170920. - assertThatSqlCountQueryYieldsExpectedCount( - String.format( - "SELECT count(*) FROM %s.calendar WHERE end_date='%s' AND service_id in ('Fake_Transit1:common_id')", - mergedNamespace, - activeCalendarNewEndDate), - 1 - ); + sqlAssert.calendar.assertCount(1, "end_date='20170919' AND service_id in ('Fake_Transit1:common_id')"); // trips table should have 4 records // (all records from original files except the active trip for only_calendar_trips, // which is skipped because it operates in the future feed). - assertRowCountInTable(mergedNamespace, "trips", 4); + sqlAssert.trips.assertCount(4); } /** @@ -694,88 +612,44 @@ void canMergeFeedsWithMTCForServiceIds1 () throws SQLException { // Run the job in this thread (we're not concerned about concurrency here). mergeFeedsJob.run(); assertFeedMergeSucceeded(mergeFeedsJob); - // assert service_ids have been feed scoped properly - String mergedNamespace = mergeFeedsJob.mergedVersion.namespace; + SqlAssert sqlAssert = new SqlAssert(mergeFeedsJob.mergedVersion); // - calendar table should have 4 records. - assertRowCountInTable(mergedNamespace, "calendar", 4); + sqlAssert.calendar.assertCount(4); // bothCalendarFilesVersion's common_id service_id should be scoped - assertThatSqlCountQueryYieldsExpectedCount( - String.format( - "SELECT count(*) FROM %s.calendar WHERE service_id='Fake_Agency1:common_id'", - mergedNamespace - ), - 1 - ); + sqlAssert.calendar.assertCount(1, "service_id='Fake_Agency1:common_id'"); + // bothCalendarFilesVersion's both_id service_id should be scoped - assertThatSqlCountQueryYieldsExpectedCount( - String.format( - "SELECT count(*) FROM %s.calendar WHERE service_id='Fake_Agency1:both_id'", - mergedNamespace - ), - 1 - ); + sqlAssert.calendar.assertCount(1, "service_id='Fake_Agency1:both_id'"); + // onlyCalendarVersion's common id should not be scoped - assertThatSqlCountQueryYieldsExpectedCount( - String.format( - "SELECT count(*) FROM %s.calendar WHERE service_id='common_id'", - mergedNamespace - ), - 1 - ); + sqlAssert.calendar.assertCount(1, "service_id='common_id'"); + // onlyCalendarVersion's only_calendar_id service_id should not be scoped - assertThatSqlCountQueryYieldsExpectedCount( - String.format( - "SELECT count(*) FROM %s.calendar WHERE service_id='only_calendar_id'", - mergedNamespace - ), - 1 - ); + sqlAssert.calendar.assertCount(1, "service_id='only_calendar_id'"); // - calendar_dates table should have only 2 records. - assertRowCountInTable(mergedNamespace, "calendar_dates", 2); + sqlAssert.calendarDates.assertCount(2); // bothCalendarFilesVersion's common_id service_id should be scoped - assertThatSqlCountQueryYieldsExpectedCount( - String.format( - "SELECT count(*) FROM %s.calendar_dates WHERE service_id='Fake_Agency1:common_id'", - mergedNamespace - ), - 1 - ); + sqlAssert.calendarDates.assertCount(1, "service_id='Fake_Agency1:common_id'"); + // bothCalendarFilesVersion's both_id service_id should be scoped - assertThatSqlCountQueryYieldsExpectedCount( - String.format( - "SELECT count(*) FROM %s.calendar_dates WHERE service_id='Fake_Agency1:both_id'", - mergedNamespace - ), - 1 - ); + sqlAssert.calendarDates.assertCount(1, "service_id='Fake_Agency1:both_id'"); // - trips should have 2 + 1 = 3 records. - assertRowCountInTable(mergedNamespace, "trips", 3); + sqlAssert.trips.assertCount(3); // bothCalendarFilesVersion's common_id service_id should be scoped - assertThatSqlCountQueryYieldsExpectedCount( - String.format( - "SELECT count(*) FROM %s.trips WHERE service_id='Fake_Agency1:common_id'", - mergedNamespace - ), - 1 - ); + sqlAssert.trips.assertCount(1, "service_id='Fake_Agency1:common_id'"); + // 2 trips with onlyCalendarVersion's common_id service_id should not be scoped - assertThatSqlCountQueryYieldsExpectedCount( - String.format( - "SELECT count(*) FROM %s.trips WHERE service_id='common_id'", - mergedNamespace - ), - 2 - ); + sqlAssert.trips.assertCount(2, "service_id='common_id'"); } /** - * Tests whether a MTC feed merge of two feed versions correctly feed scopes the service_id's of the feed that is + * Tests whether an MTC feed merge of two feed versions correctly feed scopes the service_id's of the feed that is * chronologically before the other one. This tests two feeds where one of them has only the calendar_dates files, * and the other has only the calendar file. */ @@ -788,68 +662,35 @@ void canMergeFeedsWithMTCForServiceIds2 () throws SQLException { // Run the job in this thread (we're not concerned about concurrency here). mergeFeedsJob.run(); assertFeedMergeSucceeded(mergeFeedsJob); - // assert service_ids have been feed scoped properly - String mergedNamespace = mergeFeedsJob.mergedVersion.namespace; + SqlAssert sqlAssert = new SqlAssert(mergeFeedsJob.mergedVersion); - // - calendar table should have 4 records. - assertRowCountInTable(mergedNamespace, "calendar", 2); + // - calendar table should have 2 records. + sqlAssert.calendar.assertCount(2); // onlyCalendarVersion's common id should not be scoped - assertThatSqlCountQueryYieldsExpectedCount( - String.format( - "SELECT count(*) FROM %s.calendar WHERE service_id='common_id'", - mergedNamespace - ), - 1 - ); + sqlAssert.calendar.assertCount(1, "service_id='common_id'"); + // onlyCalendarVersion's only_calendar_id service_id should not be scoped - assertThatSqlCountQueryYieldsExpectedCount( - String.format( - "SELECT count(*) FROM %s.calendar WHERE service_id='only_calendar_id'", - mergedNamespace - ), - 1 - ); + sqlAssert.calendar.assertCount(1, "service_id='only_calendar_id'"); // - calendar_dates table should have only 2 records. - assertRowCountInTable(mergedNamespace, "calendar_dates", 2); + sqlAssert.calendarDates.assertCount(2); // onlyCalendarDatesVersion's common_id service_id should be scoped - assertThatSqlCountQueryYieldsExpectedCount( - String.format( - "SELECT count(*) FROM %s.calendar_dates WHERE service_id='Fake_Agency3:common_id'", - mergedNamespace - ), - 1 - ); + sqlAssert.calendarDates.assertCount(1, "service_id='Fake_Agency3:common_id'"); + // onlyCalendarDatesVersion's only_calendar_dates_id service_id should be scoped - assertThatSqlCountQueryYieldsExpectedCount( - String.format( - "SELECT count(*) FROM %s.calendar_dates WHERE service_id='Fake_Agency3:only_calendar_dates_id'", - mergedNamespace - ), - 1 - ); + sqlAssert.calendarDates.assertCount(1, "service_id='Fake_Agency3:only_calendar_dates_id'"); // - trips table should have 2 + 1 = 3 records. - assertRowCountInTable(mergedNamespace, "trips", 3); + sqlAssert.trips.assertCount(3); // bothCalendarFilesVersion's common_id service_id should be scoped - assertThatSqlCountQueryYieldsExpectedCount( - String.format( - "SELECT count(*) FROM %s.trips WHERE service_id='Fake_Agency3:common_id'", - mergedNamespace - ), - 1 - ); + sqlAssert.trips.assertCount(1, "service_id='Fake_Agency3:common_id'"); + // 2 trips with onlyCalendarVersion's common_id service_id should not be scoped - assertThatSqlCountQueryYieldsExpectedCount( - String.format( - "SELECT count(*) FROM %s.trips WHERE service_id='common_id'", - mergedNamespace - ), - 2 - ); + sqlAssert.trips.assertCount(2, "service_id='common_id'"); + // This fails, but if remappedReferences isn't actually needed maybe the current implementation is good-to-go // assertThat(mergeFeedsJob.mergeFeedsResult.remappedReferences, equalTo(1)); } @@ -868,61 +709,34 @@ void canMergeFeedsWithMTCForServiceIds3 () throws SQLException { // Run the job in this thread (we're not concerned about concurrency here). mergeFeedsJob.run(); assertFeedMergeSucceeded(mergeFeedsJob); - // assert service_ids have been feed scoped properly - String mergedNamespace = mergeFeedsJob.mergedVersion.namespace; + SqlAssert sqlAssert = new SqlAssert(mergeFeedsJob.mergedVersion); + // - calendar table should have 3 records. - assertRowCountInTable(mergedNamespace, "calendar", 3); + sqlAssert.calendar.assertCount(3); // calendar_dates should have 1 record. // - one for common_id from the future feed, // Note that the common_id from the active feed is not included because it operates // within the future feed timespan. - assertThatSqlCountQueryYieldsExpectedCount( - String.format( - "SELECT count(*) FROM %s.calendar_dates WHERE service_id = 'common_id' and date = '20170916'", - mergedNamespace - ), - 1 - ); + sqlAssert.calendarDates.assertCount(1, "service_id='common_id' and date='20170916'"); // - trips table should have 3 records. - assertRowCountInTable(mergedNamespace, "trips", 3); + sqlAssert.trips.assertCount(3); // common_id service_id should be scoped for earlier feed version. - assertThatSqlCountQueryYieldsExpectedCount( - String.format( - "SELECT count(*) FROM %s.trips WHERE service_id='Fake_Agency4:common_id'", - mergedNamespace - ), - 1 - ); + sqlAssert.trips.assertCount(1, "service_id='Fake_Agency4:common_id'"); + // cal_to_remove service_id should be scoped for earlier feed version. - assertThatSqlCountQueryYieldsExpectedCount( - String.format( - "SELECT count(*) FROM %s.trips WHERE service_id='Fake_Agency4:cal_to_remove'", - mergedNamespace - ), - 1 - ); + sqlAssert.trips.assertCount(1, "service_id='Fake_Agency4:cal_to_remove'"); + // Amended calendar record from earlier feed version should also have a modified end date (one day before the // earliest start_date from the future feed). - assertThatSqlCountQueryYieldsExpectedCount( - String.format( - "SELECT count(*) FROM %s.calendar WHERE service_id='Fake_Agency4:common_id' AND end_date='20170914'", - mergedNamespace - ), - 1 - ); + sqlAssert.calendar.assertCount(1, "service_id='Fake_Agency4:common_id' AND end_date='20170914'"); + // Modified cal_to_remove should still exist in calendar_dates. It is modified even though it does not exist in // the future feed due to the MTC requirement to update all service_ids in the active feed. // See https://github.com/ibi-group/datatools-server/issues/244 - assertThatSqlCountQueryYieldsExpectedCount( - String.format( - "SELECT count(*) FROM %s.calendar_dates WHERE service_id='Fake_Agency4:cal_to_remove'", - mergedNamespace - ), - 1 - ); + sqlAssert.calendarDates.assertCount(1, "service_id='Fake_Agency4:cal_to_remove'"); } /** @@ -939,35 +753,24 @@ void canMergeFeedsWithMTCForServiceIds4 () throws SQLException { // Run the job in this thread (we're not concerned about concurrency here). mergeFeedsJob.run(); assertFeedMergeSucceeded(mergeFeedsJob); - // assert service_ids have been feed scoped properly - String mergedNamespace = mergeFeedsJob.mergedVersion.namespace; + SqlAssert sqlAssert = new SqlAssert(mergeFeedsJob.mergedVersion); + // - calendar table should have 3 records. - assertRowCountInTable(mergedNamespace, "calendar", 3); + sqlAssert.calendar.assertCount(3); // calendar_dates table should have 3 records: // all records from future feed and keep_one from the active feed. - assertRowCountInTable(mergedNamespace, "calendar_dates", 3); + sqlAssert.calendarDates.assertCount(3); // - trips table should have 3 records. - assertRowCountInTable(mergedNamespace, "trips", 3); + sqlAssert.trips.assertCount(3); // common_id service_id should be scoped for earlier feed version. - assertThatSqlCountQueryYieldsExpectedCount( - String.format( - "SELECT count(*) FROM %s.trips WHERE service_id='Fake_Agency5:common_id'", - mergedNamespace - ), - 1 - ); + sqlAssert.trips.assertCount(1, "service_id='Fake_Agency5:common_id'"); + // Amended calendar record from earlier feed version should also have a modified end date (one day before the // earliest start_date from the future feed). - assertThatSqlCountQueryYieldsExpectedCount( - String.format( - "SELECT count(*) FROM %s.calendar WHERE service_id='Fake_Agency5:common_id' AND end_date='20170914'", - mergedNamespace - ), - 1 - ); + sqlAssert.calendar.assertCount(1, "service_id='Fake_Agency5:common_id' AND end_date='20170914'"); } /** @@ -989,7 +792,8 @@ void canMergeBARTFeedsWithSpecialStops() throws SQLException, IOException { // Job should succeed. assertFeedMergeSucceeded(mergeFeedsJob); // Verify that the stop count is equal to the number of stops found in each of the input stops.txt files. - assertRowCountInTable(mergeFeedsJob.mergedVersion.namespace, "stops", 182); + SqlAssert sqlAssert = new SqlAssert(mergeFeedsJob.mergedVersion); + sqlAssert.stops.assertCount(182); } /** @@ -1002,26 +806,23 @@ void canMergeFeedsWithoutAgencyIds () throws SQLException { versions.add(noAgencyVersion1); versions.add(noAgencyVersion2); FeedVersion mergedVersion = regionallyMergeVersions(versions); + SqlAssert sqlAssert = new SqlAssert(mergedVersion); + final String agencyIdIsBlankOrNull = "agency_id='' or agency_id is null"; - String mergedNamespace = mergedVersion.namespace; // - agency should have 2 records. - assertRowCountInTable(mergedNamespace, "agency", 2); + sqlAssert.agency.assertCount(2); // there shouldn't be records with blank agency_id - assertThatSqlCountQueryYieldsExpectedCount( - String.format("SELECT count(*) FROM %s.agency where agency_id = '' or agency_id is null", mergedNamespace), - 0 - ); + sqlAssert.agency.assertCount(0, agencyIdIsBlankOrNull); + // - routes should have 2 records - assertRowCountInTable(mergedNamespace, "routes", 2); + sqlAssert.routes.assertCount(2); + + // there shouldn't be route records with blank agency_id + sqlAssert.routes.assertCount(0, agencyIdIsBlankOrNull); - // there shouldn't be records with blank agency_id - assertThatSqlCountQueryYieldsExpectedCount( - String.format("SELECT count(*) FROM %s.routes where agency_id = '' or agency_id is null", mergedNamespace), - 0 - ); // - trips should have 4 records - assertRowCountInTable(mergedNamespace, "trips", 4); + sqlAssert.trips.assertCount(4); } /** @@ -1043,34 +844,4 @@ private FeedVersion regionallyMergeVersions(Set versions) { LOG.info("Regional merged file: {}", mergeFeedsJob.mergedVersion.retrieveGtfsFile().getAbsolutePath()); return mergeFeedsJob.mergedVersion; } - - /** - * Checks there are no unused service ids. - */ - private void assertNoUnusedServiceIds(String mergedNamespace) throws SQLException { - assertThatSqlCountQueryYieldsExpectedCount( - String.format("SELECT count(*) FROM %s.errors where error_type = 'SERVICE_UNUSED'", mergedNamespace), - 0 - ); - } - - /** - * Checks there are no referential integrity issues. - */ - private void assertNoRefIntegrityErrors(String mergedNamespace) throws SQLException { - assertThatSqlCountQueryYieldsExpectedCount( - String.format("SELECT count(*) FROM %s.errors where error_type = 'REFERENTIAL_INTEGRITY'", mergedNamespace), - 0 - ); - } - - /** - * Shorthand method for asserting an expected number of rows in a table. - */ - private void assertRowCountInTable(String namespace, String tableName, int count) throws SQLException { - assertThatSqlCountQueryYieldsExpectedCount( - String.format("SELECT count(*) FROM %s.%s", namespace, tableName), - count - ); - } } diff --git a/src/test/java/com/conveyal/datatools/manager/utils/SqlAssert.java b/src/test/java/com/conveyal/datatools/manager/utils/SqlAssert.java new file mode 100644 index 000000000..dd91d2ba9 --- /dev/null +++ b/src/test/java/com/conveyal/datatools/manager/utils/SqlAssert.java @@ -0,0 +1,74 @@ +package com.conveyal.datatools.manager.utils; + +import com.conveyal.datatools.manager.models.FeedVersion; +import com.conveyal.gtfs.loader.Table; +import com.google.common.base.Strings; + +import java.sql.SQLException; + +import static com.conveyal.datatools.TestUtils.assertThatSqlCountQueryYieldsExpectedCount; + +/** + * This class contains helper methods to assert against various PSQL tables + * of a given namespace. + */ +public class SqlAssert { + private final FeedVersion version; + public final SqlTableAssert agency = new SqlTableAssert(Table.AGENCY); + public final SqlTableAssert calendar = new SqlTableAssert(Table.CALENDAR); + public final SqlTableAssert calendarDates = new SqlTableAssert(Table.CALENDAR_DATES); + public final SqlTableAssert errors = new SqlTableAssert("errors"); + public final SqlTableAssert routes = new SqlTableAssert(Table.ROUTES); + public final SqlTableAssert trips = new SqlTableAssert(Table.TRIPS); + public final SqlTableAssert stops = new SqlTableAssert(Table.STOPS); + + public SqlAssert(FeedVersion version) { + this.version = version; + } + + /** + * Checks there are no unused service ids. + */ + public void assertNoUnusedServiceIds() throws SQLException { + errors.assertCount(0, "error_type='SERVICE_UNUSED'"); + } + + /** + * Checks there are no referential integrity issues. + */ + public void assertNoRefIntegrityErrors() throws SQLException { + errors.assertCount(0, "error_type = 'REFERENTIAL_INTEGRITY'"); + } + + /** + * Helper class to assert against a particular PSQL table. + */ + public class SqlTableAssert { + private final String tableName; + private SqlTableAssert(Table table) { + this.tableName = table.name; + } + + private SqlTableAssert(String tableName) { + this.tableName = tableName; + } + + /** + * Helper method to assert a row count on a simple WHERE clause. + */ + public void assertCount(int count, String condition) throws SQLException { + assertThatSqlCountQueryYieldsExpectedCount( + String.format("SELECT count(*) FROM %s.%s %s", version.namespace, tableName, + Strings.isNullOrEmpty(condition) ? "" : "WHERE " + condition), + count + ); + } + + /** + * Helper method to assert a row count on the entire table. + */ + public void assertCount(int count) throws SQLException { + assertCount(count, null); + } + } +} From c34970cd2d40a95875a1eea5076fb4a43d5e0881 Mon Sep 17 00:00:00 2001 From: binh-dam-ibigroup <56846598+binh-dam-ibigroup@users.noreply.github.com> Date: Mon, 13 Dec 2021 17:29:32 -0500 Subject: [PATCH 102/122] fix(CalendarDatesMergeLineContext): Fix ref integrity errors --- .../CalendarDatesMergeLineContext.java | 10 ++++-- .../manager/jobs/MergeFeedsJobTest.java | 31 ++++++++++++------- 2 files changed, 27 insertions(+), 14 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarDatesMergeLineContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarDatesMergeLineContext.java index a772f8075..32809e70f 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarDatesMergeLineContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/CalendarDatesMergeLineContext.java @@ -56,10 +56,16 @@ private boolean checkCalendarDatesIds(FieldContext fieldContext) throws IOExcept boolean shouldSkipRecord = false; if (job.mergeType.equals(SERVICE_PERIOD)) { // Drop any calendar_dates.txt records from the existing feed for dates that are - // not before the first date of the future feed. + // not before the first date of the future feed, + // or for corresponding calendar entries that have been dropped. LocalDate date = getCsvDate("date"); + String calendarKey = getTableScopedValue(Table.CALENDAR, keyValue); if ( - isHandlingActiveFeed() && !isBeforeFutureFeedStartDate(date) + isHandlingActiveFeed() && + ( + job.mergeFeedsResult.skippedIds.contains(calendarKey) || + !isBeforeFutureFeedStartDate(date) + ) ) { String key = getTableScopedValue(keyValue); LOG.warn( diff --git a/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java b/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java index ddcc4b2b1..b926ff5ea 100644 --- a/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java +++ b/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java @@ -231,6 +231,7 @@ void canMergeRegionalWithOnlyCalendarFeed () throws SQLException { versions.add(onlyCalendarVersion); FeedVersion mergedVersion = regionallyMergeVersions(versions); SqlAssert sqlAssert = new SqlAssert(mergedVersion); + sqlAssert.assertNoRefIntegrityErrors(); // - calendar table should have 2 records. sqlAssert.calendar.assertCount(2); @@ -416,15 +417,15 @@ void mergeMTCShouldHandleMatchingTripIdsAndDropUnusedFutureCalendar() throws Exc sqlAssert.calendarDates.assertCount(5); // - only_calendar_id: // 1 from future feed (that service id is not scoped), - sqlAssert.calendarDates.assertCount(1, "service_id = 'only_calendar_id'"); + sqlAssert.calendarDates.assertCount(1, "service_id='only_calendar_id'"); // 0 from active feed // (in the active feed, that service id starts after the future feed start date) - sqlAssert.calendarDates.assertCount(0, "service_id = 'Fake_Transit1:dropped_calendar_id'"); + sqlAssert.calendarDates.assertCount(0, "service_id='Fake_Transit1:dropped_calendar_id'"); // - common_id: // 2 from active feed for the calendar item that was extended due to shared trip, - sqlAssert.calendarDates.assertCount(2, "service_id = 'Fake_Transit7:common_id'"); + sqlAssert.calendarDates.assertCount(2, "service_id='Fake_Transit7:common_id'"); // 2 from active feed for the active trip not in the future feed. - sqlAssert.calendarDates.assertCount(2, "service_id = 'Fake_Transit1:common_id'"); + sqlAssert.calendarDates.assertCount(2, "service_id='Fake_Transit1:common_id'"); // The GTFS+ calendar_attributes table should contain the same number of entries as the calendar table // (reported by MTC). @@ -505,6 +506,7 @@ void mergeMTCShouldHandleDisjointTripIds() throws SQLException { ); SqlAssert sqlAssert = new SqlAssert(mergeFeedsJob.mergedVersion); + sqlAssert.assertNoRefIntegrityErrors(); // calendar table should have 4 records // - 2 records from future feed, including only_calendar_dates which absorbs its active counterpart, @@ -613,6 +615,7 @@ void canMergeFeedsWithMTCForServiceIds1 () throws SQLException { mergeFeedsJob.run(); assertFeedMergeSucceeded(mergeFeedsJob); SqlAssert sqlAssert = new SqlAssert(mergeFeedsJob.mergedVersion); + sqlAssert.assertNoRefIntegrityErrors(); // - calendar table should have 4 records. sqlAssert.calendar.assertCount(4); @@ -663,6 +666,7 @@ void canMergeFeedsWithMTCForServiceIds2 () throws SQLException { mergeFeedsJob.run(); assertFeedMergeSucceeded(mergeFeedsJob); SqlAssert sqlAssert = new SqlAssert(mergeFeedsJob.mergedVersion); + sqlAssert.assertNoRefIntegrityErrors(); // - calendar table should have 2 records. sqlAssert.calendar.assertCount(2); @@ -710,6 +714,7 @@ void canMergeFeedsWithMTCForServiceIds3 () throws SQLException { mergeFeedsJob.run(); assertFeedMergeSucceeded(mergeFeedsJob); SqlAssert sqlAssert = new SqlAssert(mergeFeedsJob.mergedVersion); + sqlAssert.assertNoRefIntegrityErrors(); // - calendar table should have 3 records. sqlAssert.calendar.assertCount(3); @@ -720,23 +725,23 @@ void canMergeFeedsWithMTCForServiceIds3 () throws SQLException { // within the future feed timespan. sqlAssert.calendarDates.assertCount(1, "service_id='common_id' and date='20170916'"); - // - trips table should have 3 records. - sqlAssert.trips.assertCount(3); + // trips table should have 2 records. + // - this includes all trips from both feed except the trip associated + // with cal_to_remove, which calendar operates within the future feed. + sqlAssert.trips.assertCount(2); // common_id service_id should be scoped for earlier feed version. sqlAssert.trips.assertCount(1, "service_id='Fake_Agency4:common_id'"); - // cal_to_remove service_id should be scoped for earlier feed version. - sqlAssert.trips.assertCount(1, "service_id='Fake_Agency4:cal_to_remove'"); + // trips for cal_to_remove service_id should be removed. + sqlAssert.trips.assertCount(0, "service_id='Fake_Agency4:cal_to_remove'"); // Amended calendar record from earlier feed version should also have a modified end date (one day before the // earliest start_date from the future feed). sqlAssert.calendar.assertCount(1, "service_id='Fake_Agency4:common_id' AND end_date='20170914'"); - // Modified cal_to_remove should still exist in calendar_dates. It is modified even though it does not exist in - // the future feed due to the MTC requirement to update all service_ids in the active feed. - // See https://github.com/ibi-group/datatools-server/issues/244 - sqlAssert.calendarDates.assertCount(1, "service_id='Fake_Agency4:cal_to_remove'"); + // cal_to_remove should be removed from calendar_dates. + sqlAssert.calendarDates.assertCount(0, "service_id='Fake_Agency4:cal_to_remove'"); } /** @@ -754,6 +759,8 @@ void canMergeFeedsWithMTCForServiceIds4 () throws SQLException { mergeFeedsJob.run(); assertFeedMergeSucceeded(mergeFeedsJob); SqlAssert sqlAssert = new SqlAssert(mergeFeedsJob.mergedVersion); + // FIXME: "version3" contains ref integrity errors... was hat intentional? + // sqlAssert.assertNoRefIntegrityErrors(); // - calendar table should have 3 records. sqlAssert.calendar.assertCount(3); From 1af2690e619bf6748d3ee2dd8438dd90feb80227 Mon Sep 17 00:00:00 2001 From: Philip Cline Date: Thu, 16 Dec 2021 13:33:40 -0500 Subject: [PATCH 103/122] Add IBI Group to readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 862ad0585..53620da17 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Transit Data Manager -The core application for Conveyal's transit data tools suite. +The core application for IBI Group's transit data tools suite. ## Documentation From 15bb206cc5df267c042a1bbbf735349b31ea1d27 Mon Sep 17 00:00:00 2001 From: miles-grant-ibi Date: Fri, 17 Dec 2021 13:37:01 +0100 Subject: [PATCH 104/122] refactor: update Pelias "branding" --- .../datatools/manager/controllers/api/DeploymentController.java | 2 +- .../com/conveyal/datatools/manager/jobs/PeliasUpdateJob.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/controllers/api/DeploymentController.java b/src/main/java/com/conveyal/datatools/manager/controllers/api/DeploymentController.java index 1094d7796..7600bf69b 100644 --- a/src/main/java/com/conveyal/datatools/manager/controllers/api/DeploymentController.java +++ b/src/main/java/com/conveyal/datatools/manager/controllers/api/DeploymentController.java @@ -498,7 +498,7 @@ private static String peliasUpdate (Request req, Response res) { } // Execute the pelias update job and keep track of it - PeliasUpdateJob peliasUpdateJob = new PeliasUpdateJob(userProfile, "Updating Custom Geocoder Database", deployment); + PeliasUpdateJob peliasUpdateJob = new PeliasUpdateJob(userProfile, "Updating Local Places Index", deployment); JobUtils.heavyExecutor.execute(peliasUpdateJob); return SparkUtils.formatJobMessage(peliasUpdateJob.jobId, "Pelias update initiating."); } diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/PeliasUpdateJob.java b/src/main/java/com/conveyal/datatools/manager/jobs/PeliasUpdateJob.java index 0f127d2c4..964f75fed 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/PeliasUpdateJob.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/PeliasUpdateJob.java @@ -77,7 +77,7 @@ public PeliasUpdateJob(Auth0UserProfile owner, String name, Deployment deploymen */ @Override public void jobLogic() throws Exception { - status.message = "Launching custom geocoder update request"; + status.message = "Launching Local Places Index update request"; workerId = this.makeWebhookRequest(); status.percentComplete = 1.0; From 8c9ad6b7d9dcd8458d92b78b97a17d068ac0cd40 Mon Sep 17 00:00:00 2001 From: binh-dam-ibigroup <56846598+binh-dam-ibigroup@users.noreply.github.com> Date: Tue, 28 Dec 2021 16:34:32 -0500 Subject: [PATCH 105/122] fix(FeedUpdater): Handle unknown feed ids from MTC. --- .../datatools/manager/jobs/FeedUpdater.java | 8 +-- .../manager/jobs/AutoPublishJobTest.java | 61 ++++++++++++++----- 2 files changed, 49 insertions(+), 20 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/FeedUpdater.java b/src/main/java/com/conveyal/datatools/manager/jobs/FeedUpdater.java index d363309f8..fbb4a9f4a 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/FeedUpdater.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/FeedUpdater.java @@ -162,6 +162,10 @@ public Map checkForUpdatedFeeds() { String filename = keyName.split("/")[1]; String feedId = filename.replace(".zip", ""); FeedSource feedSource = getFeedSource(feedId); + if (feedSource == null) { + LOG.error("No feed source found for feed ID {}", feedId); + continue; + } if (shouldMarkFeedAsProcessed(eTag, feedSource)) { // Don't add object if it is a dir @@ -169,10 +173,6 @@ public Map checkForUpdatedFeeds() { if ("null".equals(feedId)) continue; try { LOG.info("New version found for {} at s3://{}/{}. ETag = {}.", feedId, feedBucket, keyName, eTag); - if (feedSource == null) { - LOG.error("No feed source found for feed ID {}", feedId); - continue; - } updatePublishedFeedVersion(feedId, feedSource); // TODO: Explore if MD5 checksum can be used to find matching feed version. // findMatchingFeedVersion(md5, feedId, feedSource); diff --git a/src/test/java/com/conveyal/datatools/manager/jobs/AutoPublishJobTest.java b/src/test/java/com/conveyal/datatools/manager/jobs/AutoPublishJobTest.java index 3eec0d981..f6ad8361c 100644 --- a/src/test/java/com/conveyal/datatools/manager/jobs/AutoPublishJobTest.java +++ b/src/test/java/com/conveyal/datatools/manager/jobs/AutoPublishJobTest.java @@ -12,7 +12,6 @@ import com.google.common.collect.Lists; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; @@ -32,6 +31,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; /** @@ -131,8 +131,9 @@ private static Stream createPublishFeedCases() { ); } - @Test - void shouldUpdateFeedInfoAfterPublishComplete() { + @ParameterizedTest + @MethodSource("createUpdateFeedInfoCases") + void shouldUpdateFeedInfoAfterPublishComplete(String agencyId, boolean isUnknownFeedId) { // Add the version to the feed source FeedVersion createdVersion = createFeedVersionFromGtfsZip(feedSource, "bart_new_lite.zip"); @@ -149,7 +150,7 @@ void shouldUpdateFeedInfoAfterPublishComplete() { assertNotNull(updatedFeedVersion.sentToExternalPublisher); // Create a test FeedUpdater instance, and simulate running the task. - TestCompletedFeedRetriever completedFeedRetriever = new TestCompletedFeedRetriever(); + TestCompletedFeedRetriever completedFeedRetriever = new TestCompletedFeedRetriever(agencyId); FeedUpdater feedUpdater = FeedUpdater.createForTest(completedFeedRetriever); // The list of feeds processed externally (completed) should be empty at this point. @@ -163,21 +164,44 @@ void shouldUpdateFeedInfoAfterPublishComplete() { // If a feed has been republished since last check, it will have a new etag/file hash, // and the scenario below should apply. Map etagsAfter = feedUpdater.checkForUpdatedFeeds(); - assertEquals(1, etagsAfter.size()); - assertTrue(etagsAfter.containsValue("test-etag")); - // Make sure that the publish-complete attribute has been set for the feed version in Mongo. FeedVersion updatedFeedVersionAfter = Persistence.feedVersions.getById(createdVersion.id); Date updatedDate = updatedFeedVersionAfter.processedByExternalPublisher; String namespace = updatedFeedVersionAfter.namespace; - assertNotNull(updatedDate); - - // At the next check for updates, the metadata for the feeds completed above - // should not be updated again. - feedUpdater.checkForUpdatedFeeds(); - FeedVersion updatedFeedVersionAfter2 = Persistence.feedVersions.getById(createdVersion.id); - assertEquals(updatedDate, updatedFeedVersionAfter2.processedByExternalPublisher); - assertEquals(namespace, updatedFeedVersionAfter2.namespace); + + if (!isUnknownFeedId) { + // Regular scenario: updating a known/existing feed. + assertEquals(1, etagsAfter.size()); + assertTrue(etagsAfter.containsValue("test-etag")); + + // Make sure that the publish-complete attribute has been set for the feed version in Mongo. + assertNotNull(updatedDate); + + // At the next check for updates, the metadata for the feeds completed above + // should not be updated again. + feedUpdater.checkForUpdatedFeeds(); + FeedVersion updatedFeedVersionAfter2 = Persistence.feedVersions.getById(createdVersion.id); + assertEquals(updatedDate, updatedFeedVersionAfter2.processedByExternalPublisher); + assertEquals(namespace, updatedFeedVersionAfter2.namespace); + } else { + // Edge case: an unknown feed id was provided, + // so no update of the feed should be happening (and there should not be an exception). + assertEquals(0, etagsAfter.size()); + assertNull(updatedDate); + } + } + + private static Stream createUpdateFeedInfoCases() { + return Stream.of( + Arguments.of( + TEST_AGENCY, + false + ), + Arguments.of( + "12345", + true + ) + ); } /** @@ -185,8 +209,13 @@ void shouldUpdateFeedInfoAfterPublishComplete() { * external MTC publishing process is complete. */ private static class TestCompletedFeedRetriever implements FeedUpdater.CompletedFeedRetriever { + private final String agencyId; public boolean isPublishingComplete; + public TestCompletedFeedRetriever(String agencyId) { + this.agencyId = agencyId; + } + @Override public List retrieveCompletedFeeds() { if (!isPublishingComplete) { @@ -194,7 +223,7 @@ public List retrieveCompletedFeeds() { } else { S3ObjectSummary objSummary = new S3ObjectSummary(); objSummary.setETag("test-etag"); - objSummary.setKey(String.format("%s/%s", TEST_COMPLETED_FOLDER, TEST_AGENCY)); + objSummary.setKey(String.format("%s/%s", TEST_COMPLETED_FOLDER, agencyId)); return Lists.newArrayList(objSummary); } } From 3f80a65d00bcf404f27ae564624c1917fee543cb Mon Sep 17 00:00:00 2001 From: binh-dam-ibigroup <56846598+binh-dam-ibigroup@users.noreply.github.com> Date: Thu, 6 Jan 2022 16:19:32 -0500 Subject: [PATCH 106/122] fix(AutoPublishJob): Prevent publishing if errors were found. --- .../manager/jobs/AutoPublishJob.java | 22 ++++++++++--------- .../manager/jobs/AutoPublishJobTest.java | 9 ++++++-- 2 files changed, 19 insertions(+), 12 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/AutoPublishJob.java b/src/main/java/com/conveyal/datatools/manager/jobs/AutoPublishJob.java index 69eab49ad..d198fc381 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/AutoPublishJob.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/AutoPublishJob.java @@ -43,19 +43,21 @@ protected void innerJobLogic() throws Exception { // Validate and check for blocking issues in the feed version to deploy. if (latestFeedVersion.hasBlockingIssuesForPublishing()) { status.fail("Could not publish this feed version because it contains blocking errors."); - } - - try { - GtfsPlusValidation gtfsPlusValidation = GtfsPlusValidation.validate(latestFeedVersion.id); - if (!gtfsPlusValidation.issues.isEmpty()) { - status.fail("Could not publish this feed version because it contains GTFS+ blocking errors."); + } else { + try { + GtfsPlusValidation gtfsPlusValidation = GtfsPlusValidation.validate(latestFeedVersion.id); + if (!gtfsPlusValidation.issues.isEmpty()) { + status.fail("Could not publish this feed version because it contains GTFS+ blocking errors."); + } + } catch(Exception e) { + status.fail("Could not read GTFS+ zip file", e); } - } catch(Exception e) { - status.fail("Could not read GTFS+ zip file", e); } // If validation successful, just execute the feed updating process. - FeedVersionController.publishToExternalResource(latestFeedVersion); - LOG.info("Auto-published feed source {} to external resource.", feedSource.id); + if (!status.error) { + FeedVersionController.publishToExternalResource(latestFeedVersion); + LOG.info("Auto-published feed source {} to external resource.", feedSource.id); + } } } diff --git a/src/test/java/com/conveyal/datatools/manager/jobs/AutoPublishJobTest.java b/src/test/java/com/conveyal/datatools/manager/jobs/AutoPublishJobTest.java index f6ad8361c..2b4057c27 100644 --- a/src/test/java/com/conveyal/datatools/manager/jobs/AutoPublishJobTest.java +++ b/src/test/java/com/conveyal/datatools/manager/jobs/AutoPublishJobTest.java @@ -88,10 +88,11 @@ public static void tearDown() { @MethodSource("createPublishFeedCases") void shouldProcessFeed(String resourceName, boolean isError, String errorMessage) throws IOException { // Add the version to the feed source + FeedVersion originalFeedVersion; if (resourceName.endsWith(".zip")) { - createFeedVersionFromGtfsZip(feedSource, resourceName); + originalFeedVersion = createFeedVersionFromGtfsZip(feedSource, resourceName); } else { - createFeedVersion(feedSource, zipFolderFiles(resourceName)); + originalFeedVersion = createFeedVersion(feedSource, zipFolderFiles(resourceName)); } // Create the job @@ -108,6 +109,10 @@ void shouldProcessFeed(String resourceName, boolean isError, String errorMessage if (isError) { assertEquals(errorMessage, autoPublishJob.status.message); + + // In case of error, the sentToExternalPublisher flag should not be set. + FeedVersion updatedFeedVersion = Persistence.feedVersions.getById(originalFeedVersion.id); + assertNull(updatedFeedVersion.sentToExternalPublisher); } } From a6a17d878b5cf5d4ce85ca3fe3ca31a032076677 Mon Sep 17 00:00:00 2001 From: binh-dam-ibigroup <56846598+binh-dam-ibigroup@users.noreply.github.com> Date: Wed, 12 Jan 2022 15:43:31 -0500 Subject: [PATCH 107/122] fix(FeedUpdater): Prevent incorrect published status update on server startup. --- .../datatools/manager/jobs/FeedUpdater.java | 64 +++++++++++-------- .../manager/jobs/AutoPublishJobTest.java | 64 ++++++++++++++++++- 2 files changed, 101 insertions(+), 27 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/FeedUpdater.java b/src/main/java/com/conveyal/datatools/manager/jobs/FeedUpdater.java index fbb4a9f4a..501eb56f4 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/FeedUpdater.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/FeedUpdater.java @@ -158,6 +158,7 @@ public Map checkForUpdatedFeeds() { String keyName = objSummary.getKey(); LOG.debug("{} etag = {}", keyName, eTag); + // Don't add object if it is a dir if (keyName.equals(bucketFolder)) continue; String filename = keyName.split("/")[1]; String feedId = filename.replace(".zip", ""); @@ -166,16 +167,20 @@ public Map checkForUpdatedFeeds() { LOG.error("No feed source found for feed ID {}", feedId); continue; } + // Skip object if the filename is null + if ("null".equals(feedId)) continue; - if (shouldMarkFeedAsProcessed(eTag, feedSource)) { - // Don't add object if it is a dir - // Skip object if the filename is null - if ("null".equals(feedId)) continue; + FeedVersion latestVersionSentForPublishing = getLatestVersionSentForPublishing(feedId, feedSource); + if (shouldMarkFeedAsProcessed(eTag, latestVersionSentForPublishing)) { try { - LOG.info("New version found for {} at s3://{}/{}. ETag = {}.", feedId, feedBucket, keyName, eTag); - updatePublishedFeedVersion(feedId, feedSource); - // TODO: Explore if MD5 checksum can be used to find matching feed version. - // findMatchingFeedVersion(md5, feedId, feedSource); + // Don't mark a feed version as published if previous published version is before sentToExternalPublisher. + if (!objSummary.getLastModified().before(latestVersionSentForPublishing.sentToExternalPublisher)) { + LOG.info("New version found for {} at s3://{}/{}. ETag = {}.", feedId, feedBucket, keyName, eTag); + updatePublishedFeedVersion(feedId, latestVersionSentForPublishing); + // TODO: Explore if MD5 checksum can be used to find matching feed version. + // findMatchingFeedVersion(md5, feedId, feedSource); + } + } catch (Exception e) { LOG.warn("Could not load feed " + keyName, e); } finally { @@ -200,7 +205,7 @@ private FeedSource getFeedSource(String feedId) { and(eq("value", feedId), eq("name", AGENCY_ID_FIELDNAME)) ); if (properties.size() > 1) { - LOG.warn("Found multiple feed sources for {}: {}", + LOG.warn("Found multiple feed sources for {}: {}. The published status on some feed versions will be incorrect.", feedId, properties.stream().map(p -> p.feedSourceId).collect(Collectors.joining(","))); } @@ -216,22 +221,20 @@ private FeedSource getFeedSource(String feedId) { /** * @return true if the feed with the corresponding etag should be mark as processed, false otherwise. */ - private boolean shouldMarkFeedAsProcessed(String eTag, FeedSource feedSource) { + private boolean shouldMarkFeedAsProcessed(String eTag, FeedVersion publishedVersion) { if (eTagForFeed.containsValue(eTag)) return false; - - FeedVersion publishedVersion = getLatestPublishedVersion(feedSource); if (publishedVersion == null) return false; + return versionsToMarkAsProcessed.contains(publishedVersion.id); } /** * Update the published feed version for the feed source. * @param feedId the unique ID used by MTC to identify a feed source - * @param feedSource the feed source for which a newly published version should be registered + * @param publishedVersion the feed version to be registered */ - private void updatePublishedFeedVersion(String feedId, FeedSource feedSource) { + private void updatePublishedFeedVersion(String feedId, FeedVersion publishedVersion) { try { - FeedVersion publishedVersion = getLatestPublishedVersion(feedSource); if (publishedVersion != null) { if (publishedVersion.sentToExternalPublisher == null) { LOG.warn("Not updating published version for {} (version was never sent to external publisher)", feedId); @@ -240,13 +243,18 @@ private void updatePublishedFeedVersion(String feedId, FeedSource feedSource) { // Set published namespace to the feed version and set the processedByExternalPublisher timestamp. LOG.info("Latest published version (sent at {}) for {} is {}", publishedVersion.sentToExternalPublisher, feedId, publishedVersion.id); Persistence.feedVersions.updateField(publishedVersion.id, PROCESSED_BY_EXTERNAL_PUBLISHER_FIELD, new Date()); - Persistence.feedSources.updateField(feedSource.id, "publishedVersionId", publishedVersion.namespace); + Persistence.feedSources.updateField(publishedVersion.feedSourceId, "publishedVersionId", publishedVersion.namespace); } else { - LOG.error("No published versions found for {} ({} id={})", feedId, feedSource.name, feedSource.id); + LOG.error( + "No published versions found for {} ({} id={})", + feedId, + publishedVersion.parentFeedSource().name, + publishedVersion.feedSourceId + ); } } catch (Exception e) { e.printStackTrace(); - LOG.error("Error encountered while checking for latest published version for {}", feedId); + LOG.error("Error encountered while updating the latest published version for {}", feedId); } } @@ -256,13 +264,19 @@ private void updatePublishedFeedVersion(String feedId, FeedSource feedSource) { * could be that more than one versions were recently "published" and the latest published version was a bad * feed that failed processing by RTD. */ - private static FeedVersion getLatestPublishedVersion(FeedSource feedSource) { - // Collect the feed versions for the feed source. - Collection versions = feedSource.retrieveFeedVersions(); - Optional lastPublishedVersionCandidate = versions - .stream() - .min(Comparator.comparing(v -> v.sentToExternalPublisher, Comparator.nullsLast(Comparator.reverseOrder()))); - return lastPublishedVersionCandidate.orElse(null); + private static FeedVersion getLatestVersionSentForPublishing(String feedId, FeedSource feedSource) { + try { + // Collect the feed versions for the feed source. + Collection versions = feedSource.retrieveFeedVersions(); + Optional lastPublishedVersionCandidate = versions + .stream() + .min(Comparator.comparing(v -> v.sentToExternalPublisher, Comparator.nullsLast(Comparator.reverseOrder()))); + return lastPublishedVersionCandidate.orElse(null); + } catch (Exception e) { + e.printStackTrace(); + LOG.error("Error encountered while checking for latest published version for {}", feedId); + return null; + } } /** diff --git a/src/test/java/com/conveyal/datatools/manager/jobs/AutoPublishJobTest.java b/src/test/java/com/conveyal/datatools/manager/jobs/AutoPublishJobTest.java index 2b4057c27..3c21a1599 100644 --- a/src/test/java/com/conveyal/datatools/manager/jobs/AutoPublishJobTest.java +++ b/src/test/java/com/conveyal/datatools/manager/jobs/AutoPublishJobTest.java @@ -12,6 +12,7 @@ import com.google.common.collect.Lists; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; @@ -163,7 +164,7 @@ void shouldUpdateFeedInfoAfterPublishComplete(String agencyId, boolean isUnknown assertTrue(etags.isEmpty()); // Simulate completion of feed publishing. - completedFeedRetriever.isPublishingComplete = true; + completedFeedRetriever.makePublished(); // The etags should contain the id of the agency. // If a feed has been republished since last check, it will have a new etag/file hash, @@ -209,13 +210,62 @@ private static Stream createUpdateFeedInfoCases() { ); } + /** + * This test ensures that, upon server startup, + * feeds that meet all these criteria should not be updated/marked as published: + * - the feed has been sent to publisher (RTD), + * - the publisher has not published the feed, + * - a previous version of the feed was already published. + */ + @Test + void shouldNotUpdateFromAPreviouslyPublishedVersionOnStartup() { + final int TWO_DAYS_MILLIS = 48 * 3600000; + + // Set up a test FeedUpdater instance that fakes an external published date in the past. + TestCompletedFeedRetriever completedFeedRetriever = new TestCompletedFeedRetriever(TEST_AGENCY); + FeedUpdater feedUpdater = FeedUpdater.createForTest(completedFeedRetriever); + completedFeedRetriever.makePublished(new Date(System.currentTimeMillis() - TWO_DAYS_MILLIS)); + + // Add the version to the feed source, with + // sentToExternalPublisher set to a date after a previous publish date. + FeedVersion createdVersion = createFeedVersionFromGtfsZip(feedSource, "bart_new_lite.zip"); + createdVersion.sentToExternalPublisher = new Date(); + Persistence.feedVersions.replace(createdVersion.id, createdVersion); + + // The list of feeds processed externally (completed) should contain an entry for the agency we want. + Map etags = feedUpdater.checkForUpdatedFeeds(); + assertNotNull(etags.get(TEST_AGENCY)); + + // Make sure that the feed remains unpublished. + FeedVersion updatedFeedVersion = Persistence.feedVersions.getById(createdVersion.id); + assertNull(updatedFeedVersion.processedByExternalPublisher); + + // Now perform publishing. + AutoPublishJob autoPublishJob = new AutoPublishJob(feedSource, user); + autoPublishJob.run(); + assertFalse(autoPublishJob.status.error); + + // Simulate another publishing process + completedFeedRetriever.makePublished(new Date()); + + // The list of feeds processed externally (completed) should contain an entry for the agency we want. + Map etagsAfter = feedUpdater.checkForUpdatedFeeds(); + assertNotNull(etagsAfter.get(TEST_AGENCY)); + + // The feed should be published. + FeedVersion publishedFeedVersion = Persistence.feedVersions.getById(createdVersion.id); + assertNotNull(publishedFeedVersion.processedByExternalPublisher); + + } + /** * Mocks the results of an {@link S3ObjectSummary} retrieval before/after the * external MTC publishing process is complete. */ private static class TestCompletedFeedRetriever implements FeedUpdater.CompletedFeedRetriever { private final String agencyId; - public boolean isPublishingComplete; + private boolean isPublishingComplete; + private Date publishDate; public TestCompletedFeedRetriever(String agencyId) { this.agencyId = agencyId; @@ -229,8 +279,18 @@ public List retrieveCompletedFeeds() { S3ObjectSummary objSummary = new S3ObjectSummary(); objSummary.setETag("test-etag"); objSummary.setKey(String.format("%s/%s", TEST_COMPLETED_FOLDER, agencyId)); + objSummary.setLastModified(publishDate); return Lists.newArrayList(objSummary); } } + + public void makePublished() { + makePublished(new Date()); + } + + public void makePublished(Date publishDate) { + isPublishingComplete = true; + this.publishDate = publishDate; + } } } From d988113ac479af78266d6676ac801d66b3446fbe Mon Sep 17 00:00:00 2001 From: binh-dam-ibigroup <56846598+binh-dam-ibigroup@users.noreply.github.com> Date: Thu, 20 Jan 2022 14:26:56 -0500 Subject: [PATCH 108/122] fix(FeedSource): Allow refetching a feed after a version upload. --- .../controllers/api/FeedVersionController.java | 4 ++-- .../conveyal/datatools/manager/models/FeedSource.java | 11 ++++++++--- .../datatools/manager/models/FeedVersion.java | 10 ++++++++++ 3 files changed, 20 insertions(+), 5 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/controllers/api/FeedVersionController.java b/src/main/java/com/conveyal/datatools/manager/controllers/api/FeedVersionController.java index 71ae18254..eac4ba000 100644 --- a/src/main/java/com/conveyal/datatools/manager/controllers/api/FeedVersionController.java +++ b/src/main/java/com/conveyal/datatools/manager/controllers/api/FeedVersionController.java @@ -120,8 +120,8 @@ private static String createFeedVersionViaUpload(Request req, Response res) { LOG.info("Last modified: {}", new Date(newGtfsFile.lastModified())); // Check that the hashes of the feeds don't match, i.e. that the feed has changed since the last version. - // (as long as there is a latest version, i.e. the feed source is not completely new) - if (latestVersion != null && latestVersion.hash.equals(newFeedVersion.hash)) { + // (as long as there is a latest version, the feed source is not completely new) + if (newFeedVersion.isSameAs(latestVersion)) { // Uploaded feed matches latest. Delete GTFS file because it is a duplicate. LOG.error("Upload version {} matches latest version {}.", newFeedVersion.id, latestVersion.id); newGtfsFile.delete(); diff --git a/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java b/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java index 905b56820..a7b787652 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java +++ b/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java @@ -45,6 +45,7 @@ import java.util.Objects; import java.util.stream.Collectors; +import static com.conveyal.datatools.manager.models.FeedRetrievalMethod.FETCHED_AUTOMATICALLY; import static com.conveyal.datatools.manager.utils.StringUtils.getCleanName; import static com.mongodb.client.model.Filters.and; import static com.mongodb.client.model.Filters.eq; @@ -214,7 +215,7 @@ public FeedVersion fetch (MonitorableJob.Status status, String optionalUrlOverri // We create a new FeedVersion now, so that the fetched date is (milliseconds) before // fetch occurs. That way, in the highly unlikely event that a feed is updated while we're // fetching it, we will not miss a new feed. - FeedVersion version = new FeedVersion(this, FeedRetrievalMethod.FETCHED_AUTOMATICALLY); + FeedVersion version = new FeedVersion(this, FETCHED_AUTOMATICALLY); // build the URL from which to fetch URL url = null; @@ -246,8 +247,12 @@ public FeedVersion fetch (MonitorableJob.Status status, String optionalUrlOverri // Get latest version to check that the fetched version does not duplicate a feed already loaded. FeedVersion latest = retrieveLatest(); // lastFetched is set to null when the URL changes and when latest feed version is deleted - if (latest != null && this.lastFetched != null) + if (latest != null && latest.retrievalMethod.equals(FETCHED_AUTOMATICALLY) && this.lastFetched != null) { + // If the last feed was automatically fetched, + // set a modified threshold to skip download unless there is a more recent feed + // (if the source server supports it)/ conn.setIfModifiedSince(Math.min(latest.updated.getTime(), this.lastFetched.getTime())); + } File newGtfsFile; @@ -305,7 +310,7 @@ public FeedVersion fetch (MonitorableJob.Status status, String optionalUrlOverri e.printStackTrace(); return null; } - if (latest != null && version.hash.equals(latest.hash)) { + if (version.isSameAs(latest)) { // If new version hash equals the hash for the latest version, do not error. Simply indicate that server // operators should add If-Modified-Since support to avoid wasting bandwidth. String message = String.format("Feed %s was fetched but has not changed; server operators should add If-Modified-Since support to avoid wasting bandwidth", this.name); diff --git a/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java b/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java index b5bb8b7cd..456a7d3b5 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java +++ b/src/main/java/com/conveyal/datatools/manager/models/FeedVersion.java @@ -531,4 +531,14 @@ public void assignGtfsFileAttributes(File newGtfsFile, Long lastModifiedOverride public void assignGtfsFileAttributes(File newGtfsFile) { assignGtfsFileAttributes(newGtfsFile, null); } + + /** + * Determines whether this feed version matches another one specified, i.e., + * whether the otherVersion doesn't have a different hash, thus has not changed, compared to this one. + * @param otherVersion The version to compare the hash to. + * @return true if the otherVersion hash is the same, false if the hashes differ or the otherVersion is null. + */ + public boolean isSameAs(FeedVersion otherVersion) { + return otherVersion != null && this.hash.equals(otherVersion.hash); + } } From 791072af4eaa7c342fe6d0fcc73d9c3eeff4175d Mon Sep 17 00:00:00 2001 From: binh-dam-ibigroup <56846598+binh-dam-ibigroup@users.noreply.github.com> Date: Thu, 20 Jan 2022 18:06:53 -0500 Subject: [PATCH 109/122] test(FetchLoadFeedCombination): Add mock downloads. --- .../datatools/manager/models/FeedSource.java | 66 +++-- .../utils/connections/ConnectionResponse.java | 19 ++ .../HttpURLConnectionResponse.java | 36 +++ .../jobs/FetchLoadFeedCombinationTest.java | 226 ++++++++++++++++++ .../datatools/gtfs/__files/bart_new_lite.zip | Bin 0 -> 4153 bytes 5 files changed, 328 insertions(+), 19 deletions(-) create mode 100644 src/main/java/com/conveyal/datatools/manager/utils/connections/ConnectionResponse.java create mode 100644 src/main/java/com/conveyal/datatools/manager/utils/connections/HttpURLConnectionResponse.java create mode 100644 src/test/java/com/conveyal/datatools/manager/jobs/FetchLoadFeedCombinationTest.java create mode 100644 src/test/resources/com/conveyal/datatools/gtfs/__files/bart_new_lite.zip diff --git a/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java b/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java index a7b787652..ee997e077 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java +++ b/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java @@ -18,6 +18,8 @@ import com.conveyal.datatools.manager.models.transform.FeedTransformation; import com.conveyal.datatools.manager.persistence.Persistence; import com.conveyal.datatools.manager.utils.JobUtils; +import com.conveyal.datatools.manager.utils.connections.ConnectionResponse; +import com.conveyal.datatools.manager.utils.connections.HttpURLConnectionResponse; import com.conveyal.gtfs.GTFS; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; @@ -217,6 +219,33 @@ public FeedVersion fetch (MonitorableJob.Status status, String optionalUrlOverri // fetching it, we will not miss a new feed. FeedVersion version = new FeedVersion(this, FETCHED_AUTOMATICALLY); + // Get latest version to check that the fetched version does not duplicate a feed already loaded. + FeedVersion latest = retrieveLatest(); + + Long modifiedThreshold = null; + // lastFetched is set to null when the URL changes and when latest feed version is deleted + if (latest != null && latest.retrievalMethod.equals(FETCHED_AUTOMATICALLY) && this.lastFetched != null) { + // If the last feed was automatically fetched, + // set a modified threshold to skip download unless there is a more recent feed + // (if the source server supports it). + modifiedThreshold = Math.min(latest.updated.getTime(), this.lastFetched.getTime()); + } + HttpURLConnection conn = makeHttpURLConnection(status, optionalUrlOverride, modifiedThreshold); + if (conn == null) return null; + + try { + conn.connect(); + return fetch(status, optionalUrlOverride, version, latest, new HttpURLConnectionResponse(conn)); + } catch (IOException e) { + String message = String.format("Unable to connect to %s; not fetching %s feed", conn.getURL(), this.name); // url, this.name); + LOG.error(message); + status.fail(message); + e.printStackTrace(); + return null; + } + } + + private HttpURLConnection makeHttpURLConnection(MonitorableJob.Status status, String optionalUrlOverride, Long modifiedThreshold) { // build the URL from which to fetch URL url = null; try { @@ -244,22 +273,21 @@ public FeedVersion fetch (MonitorableJob.Status status, String optionalUrlOverri } conn.setDefaultUseCaches(true); - // Get latest version to check that the fetched version does not duplicate a feed already loaded. - FeedVersion latest = retrieveLatest(); - // lastFetched is set to null when the URL changes and when latest feed version is deleted - if (latest != null && latest.retrievalMethod.equals(FETCHED_AUTOMATICALLY) && this.lastFetched != null) { - // If the last feed was automatically fetched, - // set a modified threshold to skip download unless there is a more recent feed - // (if the source server supports it)/ - conn.setIfModifiedSince(Math.min(latest.updated.getTime(), this.lastFetched.getTime())); - } - File newGtfsFile; + if (modifiedThreshold != null) conn.setIfModifiedSince(modifiedThreshold); + + return conn; + } + /** + * Processes the given response. + * @return true if a new FeedVersion was created from the response, false otherwise. + */ + public FeedVersion fetch(MonitorableJob.Status status, String optionalUrlOverride, FeedVersion version, FeedVersion latest, ConnectionResponse response) { + File newGtfsFile; try { - conn.connect(); String message; - int responseCode = conn.getResponseCode(); + int responseCode = response.getResponseCode(); LOG.info("Fetch feed response code={}", responseCode); switch (responseCode) { case HttpURLConnection.HTTP_NOT_MODIFIED: @@ -274,17 +302,17 @@ public FeedVersion fetch (MonitorableJob.Status status, String optionalUrlOverri status.update(message, 75.0); // Create new file from input stream (this also handles hashing the file and other version fields // calculated from the GTFS file. - newGtfsFile = version.newGtfsFile(conn.getInputStream()); + newGtfsFile = version.newGtfsFile(response.getInputStream()); break; case HttpURLConnection.HTTP_MOVED_TEMP: case HttpURLConnection.HTTP_MOVED_PERM: case HttpURLConnection.HTTP_SEE_OTHER: // Get redirect url from "location" header field - String newUrl = conn.getHeaderField("Location"); + String redirectUrl = response.getRedirectUrl(); if (optionalUrlOverride != null) { // Only permit recursion one level deep. If more than one redirect is detected, fail the job and // suggest that user try again with new URL. - message = String.format("More than one redirects for fetch URL detected. Please try fetch again with latest URL: %s", newUrl); + message = String.format("More than one redirects for fetch URL detected. Please try fetch again with latest URL: %s", redirectUrl); LOG.error(message); status.fail(message); return null; @@ -292,13 +320,13 @@ public FeedVersion fetch (MonitorableJob.Status status, String optionalUrlOverri // If override URL is null, this is the zeroth fetch. Recursively call fetch, but only one time // to prevent multiple (possibly infinite?) redirects. Any more redirects than one should // probably be met with user action to update the fetch URL. - LOG.info("Recursively calling fetch feed with new URL: {}", newUrl); - return fetch(status, newUrl); + LOG.info("Recursively calling fetch feed with new URL: {}", redirectUrl); + return fetch(status, redirectUrl); } default: // Any other HTTP codes result in failure. // FIXME Are there "success" codes we're not accounting for? - message = String.format("HTTP status (%d: %s) retrieving %s feed", responseCode, conn.getResponseMessage(), this.name); + message = String.format("HTTP status (%d: %s) retrieving %s feed", responseCode, response.getResponseMessage(), this.name); LOG.error(message); status.fail(message); return null; @@ -330,7 +358,7 @@ public FeedVersion fetch (MonitorableJob.Status status, String optionalUrlOverri Persistence.feedSources.updateField(this.id, "lastFetched", version.updated); // Set file timestamp according to last modified header from connection - version.fileTimestamp = conn.getLastModified(); + version.fileTimestamp = response.getLastModified(); String message = String.format("Fetch complete for %s", this.name); LOG.info(message); status.completeSuccessfully(message); diff --git a/src/main/java/com/conveyal/datatools/manager/utils/connections/ConnectionResponse.java b/src/main/java/com/conveyal/datatools/manager/utils/connections/ConnectionResponse.java new file mode 100644 index 000000000..3ab0b42f5 --- /dev/null +++ b/src/main/java/com/conveyal/datatools/manager/utils/connections/ConnectionResponse.java @@ -0,0 +1,19 @@ +package com.conveyal.datatools.manager.utils.connections; + +import java.io.IOException; +import java.io.InputStream; + +/** + * An interface for getting HTTP connection response data. + */ +public interface ConnectionResponse { + int getResponseCode() throws IOException; + + String getResponseMessage() throws IOException; + + String getRedirectUrl(); + + InputStream getInputStream() throws IOException; + + Long getLastModified(); +} diff --git a/src/main/java/com/conveyal/datatools/manager/utils/connections/HttpURLConnectionResponse.java b/src/main/java/com/conveyal/datatools/manager/utils/connections/HttpURLConnectionResponse.java new file mode 100644 index 000000000..b316fac55 --- /dev/null +++ b/src/main/java/com/conveyal/datatools/manager/utils/connections/HttpURLConnectionResponse.java @@ -0,0 +1,36 @@ +package com.conveyal.datatools.manager.utils.connections; + +import java.io.IOException; +import java.io.InputStream; +import java.net.HttpURLConnection; + +/** + * Builds a {@link ConnectionResponse} instance sent to FeedSource from an {@link HttpURLConnection} instance. + */ +public class HttpURLConnectionResponse implements ConnectionResponse { + private final HttpURLConnection connection; + + public HttpURLConnectionResponse(HttpURLConnection conn) { + this.connection = conn; + } + + public int getResponseCode() throws IOException { + return connection.getResponseCode(); + } + + public InputStream getInputStream() throws IOException { + return connection.getInputStream(); + } + + public String getResponseMessage() throws IOException { + return connection.getResponseMessage(); + } + + public String getRedirectUrl() { + return connection.getHeaderField("Location"); + } + + public Long getLastModified() { + return connection.getLastModified(); + } +} diff --git a/src/test/java/com/conveyal/datatools/manager/jobs/FetchLoadFeedCombinationTest.java b/src/test/java/com/conveyal/datatools/manager/jobs/FetchLoadFeedCombinationTest.java new file mode 100644 index 000000000..afa52b246 --- /dev/null +++ b/src/test/java/com/conveyal/datatools/manager/jobs/FetchLoadFeedCombinationTest.java @@ -0,0 +1,226 @@ +package com.conveyal.datatools.manager.jobs; + +import com.conveyal.datatools.DatatoolsTest; +import com.conveyal.datatools.UnitTest; +import com.conveyal.datatools.common.status.MonitorableJob; +import com.conveyal.datatools.manager.auth.Auth0UserProfile; +import com.conveyal.datatools.manager.models.FeedSource; +import com.conveyal.datatools.manager.models.FeedVersion; +import com.conveyal.datatools.manager.models.Project; +import com.conveyal.datatools.manager.persistence.Persistence; +import com.conveyal.datatools.manager.utils.connections.ConnectionResponse; +import com.github.tomakehurst.wiremock.WireMockServer; +import io.restassured.response.Response; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.io.InputStream; +import java.net.HttpURLConnection; +import java.util.Date; +import java.util.List; + +import static com.conveyal.datatools.TestUtils.createFeedVersionFromGtfsZip; +import static com.conveyal.datatools.manager.models.FeedRetrievalMethod.FETCHED_AUTOMATICALLY; +import static com.conveyal.datatools.manager.models.FeedRetrievalMethod.MANUALLY_UPLOADED; +import static com.github.tomakehurst.wiremock.client.WireMock.aResponse; +import static com.github.tomakehurst.wiremock.client.WireMock.configureFor; +import static com.github.tomakehurst.wiremock.client.WireMock.get; +import static com.github.tomakehurst.wiremock.client.WireMock.urlPathEqualTo; +import static com.github.tomakehurst.wiremock.core.WireMockConfiguration.options; +import static com.mongodb.client.model.Filters.eq; +import static io.restassured.RestAssured.given; +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * Tests for the various combinations of {@link FetchSingleFeedJob} and {@link LoadFeedJob} cases. + */ +public class FetchLoadFeedCombinationTest extends UnitTest { + private static final Auth0UserProfile user = Auth0UserProfile.createTestAdminUser(); + private static Project project; + private static final String MOCKED_HOST = "fakehost.com"; + private static final String MOCKED_FETCH_URL = "/dev/schedules/google_transit.zip"; + + private static WireMockServer wireMockServer; + private FeedSource feedSource; + + /** + * Prepare and start a testing-specific web server + */ + @BeforeAll + public static void setUp() throws IOException { + // start server if it isn't already running + DatatoolsTest.setUp(); + + // Create a project and feed sources. + project = new Project(); + project.name = String.format("Test %s", new Date()); + Persistence.projects.create(project); + + // This sets up a mock server that accepts requests and sends predefined responses to mock a GTFS file download. + configureFor(MOCKED_HOST, 80); + wireMockServer = new WireMockServer( + options() + .usingFilesUnderDirectory("src/test/resources/com/conveyal/datatools/gtfs/") + ); + wireMockServer.start(); + } + + @AfterAll + public static void tearDown() { + wireMockServer.stop(); + if (project != null) { + project.delete(); + } + } + + @BeforeEach + public void setUpEach() { + feedSource = new FeedSource("Feed source", project.id, MANUALLY_UPLOADED); + Persistence.feedSources.create(feedSource); + + // Create wiremock stub for gtfs download. + wireMockServer.stubFor( + get(urlPathEqualTo(MOCKED_FETCH_URL)) + .willReturn( + aResponse() + .withBodyFile("bart_new_lite.zip") + ) + ); + } + + /** + * Refetching should be allowed in the following scenario: + * 1. Feed is fetched as Version 1. + * 2. Another feed version is uploaded as Version 2. + * 3. Feed is refetched as Version 3. + */ + @Test + void shouldRefetchAfterFetchAndManualUpload() { + // Simulate the first job for the initial fetch. + simulateFetch(); + + // Assert Version 1 is created. + assertVersionCount(1); + + // Create the second job for manual upload. + FeedVersion uploadedFeedVersion = createFeedVersionFromGtfsZip(feedSource, "bart_old_lite.zip"); + new LoadFeedJob(uploadedFeedVersion, user, true).run(); + + // Assert Version 2 is created. + assertVersionCount(2); + + // Simulate the third job for the refetch. + simulateFetch(); + + // Assert Version 3 is created. + assertVersionCount(3); + } + + /** + * Refetching should not happen when doing two successive fetches (existing functionality). + * 1. Feed is fetched as Version 1. + * 2. Feed is refetched but no version is created, because either + * 304 NOT MODIFIED was returned, or the exact same file was downloaded again. + */ + @Test + void shouldNotLoadVersionIdenticalToPrevious() { + // Simulate the first fetch. + simulateFetch(); + + // Assert Version 1 is created. + assertVersionCount(1); + + // Simulate the second fetch with the response as "unchanged". + simulateFetch(); + + // Assert no version 2 is created. + assertVersionCount(1); + + // create wiremock stub for get users endpoint + wireMockServer.stubFor( + get(urlPathEqualTo(MOCKED_FETCH_URL)) + .willReturn( + aResponse() + .withStatus(HttpURLConnection.HTTP_NOT_MODIFIED) + ) + ); + + // Simulate the second fetch with the response as "unchanged". + simulateFetch(); + + // Assert no version 2 is created. + assertVersionCount(1); + } + + /** + * Simulates a fetch on the feed source. + */ + private void simulateFetch() { + MockConnectionResponse response = new MockConnectionResponse( + given() + .get(MOCKED_FETCH_URL) + .then() + .extract() + .response() + ); + FeedVersion newVersion = new FeedVersion(feedSource, FETCHED_AUTOMATICALLY); + newVersion = feedSource.fetch( + new MonitorableJob.Status(), + null, + newVersion, + feedSource.retrieveLatest(), + response + ); + if (newVersion != null) { + new ProcessSingleFeedJob(newVersion, user, true).run(); + } + } + + /** + * Assert feed version count. + */ + private void assertVersionCount(int size) { + // Fetch versions. + List versions = Persistence.feedVersions.getFiltered( + eq("feedSourceId", feedSource.id) + ); + assertEquals(size, versions.size()); + } + + /** + * Simulates a {@link ConnectionResponse} instance sent to FeedSource + * from a mock {@link Response} instance. + * TODO: Handle mock redirects. + */ + public static class MockConnectionResponse implements ConnectionResponse { + private final Response response; + + public MockConnectionResponse(Response resp) { + this.response = resp; + } + + public int getResponseCode() { + return response.statusCode(); + } + + public InputStream getInputStream() { + return response.asInputStream(); + } + + public Long getLastModified() { + return response.time(); + } + + public String getResponseMessage() { + return response.statusLine(); + } + + @Override + public String getRedirectUrl() { + return response.getHeader("Location"); + } + } +} diff --git a/src/test/resources/com/conveyal/datatools/gtfs/__files/bart_new_lite.zip b/src/test/resources/com/conveyal/datatools/gtfs/__files/bart_new_lite.zip new file mode 100644 index 0000000000000000000000000000000000000000..493105c7bd1d501bb60be042aa64cc547ea5ad99 GIT binary patch literal 4153 zcmb7G2|SePAAbik7$)N?nyloC21CNO@pqPSlm@evWB!9tB*PdCMFyLz!;#o(Ly_1L zm8__+$hb1emQ<^C)H*vvo88*y#h7=qyPxgx`OP!$J0IWg?|Po!*Tqp(OaYPre|*Of zXJ}y^0y`F34bgWqf_IQ4j?FAA-zz71#t4 z9_sRXsrnj~)LYZkOMX05x@D8SKC9Z~;pIt}vFU!gMsHbmWL=)v`X{jgHv&%o>N-f1 z)SP~zG>qO{>*-=`I>4*(P$;@mPmi#uWnRKi!>sV#Tfpj8i*Kp9d{kc7w_xnIO}2fLnS&I=;4JL0Fcg=#n_Iz)xn&GKmw{0H0$2=R zb^tS2Sdbk<6aQF{Rs%iqTse3|2WNdy37cmj$|Fi6?}k=n`$0P~8|y85ls)Hs$Uftj zx)!j&O!GcKptVIgtrU*$wTx}x?bpEMf%(%}3>uxyW(D|!FLZ`^vmrK)B#j0=tfjiW zp1=_e)h}?au<6b`Z^OKt)iz*9iO|Upd@cR6`_A7(OmxrPYatFB^|^{Tk$3s+`{?>T_kO9h+~mxb zc*1S+>gL(+za8n&P*F55SrwM-!@d3fAx10Voe5q2W$e*{!KoXlXM-jIIONEvrCMl^yy@$}%P4~a^@ zUq7ZZjP8}Mb=meY_&K{&%sB5jddC6iw`TiiyH#4gEW2+K$@!k3enC9Q@PfAJ(AsF( z-m(~)!>`TnyE{e;GOn$UAay>%cDj+?8(r%Vhfe`#dc&P);5O{~-v^}&evyHLVlhI) z!KD;{IL!Zc5Oi9J$>1dG=#z=MDF(cD=N@v0BbQ`Ek#}vCUpJ8Enw1t+J=Z!}jNM7v zfCXj}mQFgdm6CqRBOw>O?E!`b#$pOO#`#DGwt<9CoQ$9B7rlrpGPNrFqBbSpdsDWF z*QvUbdc5X`RWrqUX2nbkOmmpR#UQEhq^&ZvmF}dmBGy|`ZL6^xAD{Yk$+vdhHHkiH zzrG?*{9WdD;)*ZF+TJJz+c!!&XrHLudFkt%AIU0u+XwcFwbUn15sD}dJ6-qlHpZ6b zE@CE!AJ8ONr-OBukJx(c6*>{F1>vnB6PrH)cIVuu4l~HA6akVNk@mN-0`z|#`iHfJO zy#inqjDPFj8RV|khW{^BQ135@~|ZRPNL=qK6X4r33JRM}(3 z$DVbZ;_k~E#hwT#KbNh$vNe0xAnqdf*oLXAw6br0I??jz@ z$0DIZEwBw!*(Q}%cJKVt$$M{xP&yW&Yy3rAGFE6iY*t(4hvyiX9coQ#3e26&!Vj25 zqc=_m&MxmOwrW2FsX~LXIZrN|wwyLmENfazEqI(-kt!#7_4REtg2yWcLnlLg+%Vc;nn7@S3L=Z4I2 zn5P+He0fs>#+kQV$W=?YHi^9#-wWruGZwjtR#fz6p8Z z>ULGl29hw08po@BKnXo_3vQ%Tk}CsKfJgAj#sd?~gT>Kwv>w}FDQ!C$e^72$^xMMN zc#c6R;ZfWZVsB8+`?nL+#A#!LZ@J3GuCbcFugqoQ)T64X6!K=pRMpbdqc46-@VGzV zR97LRYr`tc)=WRkYt9XlV|dVM=~FR31Z=j5ZO}N=S-fnb>W+NBwilO2Y;^1!K7FdG zTxV}x(H^5LlMFU}%Y)mm&_cIu-AgHtO4w#B;BNPV+jE5rqAyO=>?3VBkXM^Y^o<1Z z{#7dZQc4bu7>q)7`B*z`CUUfB#WP8cMh&rf?23Gqb+_rscP4)7(eg4`-lgY4FFA+t z4Vrcg&+t@y5}vzGMayhrQkzrT6vaI>d{YacSly{Y#`Vtc$m&u1K2{r?%lQtq30#;c z5d~49kVFbJSV#)iI2Zo*-(@8d7nXNOxdh8Q!5Zf_fUln~_K^7S3`J@fSab>2IQKA0 zi2q*(DiRsiKuBc*YaqcYdQ=I8_1f1`8+B|$+-kfHXE zze3&*P=cK;b;uunMv^QQFh~*)(FKx4IfKNXeKQ`@3h)FwZ%y=w03X>hNDTnX7r`3m z-h}Z#N6a8|4_6gA+F=nPSmWGGF#!E}k^-5RrRo5gm*r@rRzT)Nr#hPf=RmL@2-hSC zav~_~(#eg){s~|U=QtAEsj{*x3t$WOR_o?hpJFea&q!=1%%{^DiQo1{mJj0#_5(~) zGzvT;^RjeyBJuI!^Z0W!012J4^3D(pE!aQghR#tTp_dLWB=iJ;{>vCc!jEYF@f7Yv z!Je|$E*5@9!Y`dyNO-CQVtOH=lWuQ(4SO!wkGYp5een7#^wNQYgnkL2KOH|vhP1;U z%q(|4KwY+@-Kz#n-;CCXQ+NE8Vd-o@G8BW^`{|@WGC)cjZpGO>Kz+6D?R}Vm+q5=# VzH!i?A>aXthaQ8r+X8D~=ucK&Y<2(u literal 0 HcmV?d00001 From d02fb9f7fc3b08be40185bf4a63c4c225ea8dc43 Mon Sep 17 00:00:00 2001 From: binh-dam-ibigroup <56846598+binh-dam-ibigroup@users.noreply.github.com> Date: Thu, 20 Jan 2022 18:47:35 -0500 Subject: [PATCH 110/122] refactor(FeedSource, FetchLoadFeedCombinationTest): Tweak code and comments. --- .../datatools/manager/models/FeedSource.java | 41 +++++++++++++------ .../jobs/FetchLoadFeedCombinationTest.java | 7 ++-- 2 files changed, 33 insertions(+), 15 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java b/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java index ee997e077..4aa8b73c9 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java +++ b/src/main/java/com/conveyal/datatools/manager/models/FeedSource.java @@ -222,20 +222,12 @@ public FeedVersion fetch (MonitorableJob.Status status, String optionalUrlOverri // Get latest version to check that the fetched version does not duplicate a feed already loaded. FeedVersion latest = retrieveLatest(); - Long modifiedThreshold = null; - // lastFetched is set to null when the URL changes and when latest feed version is deleted - if (latest != null && latest.retrievalMethod.equals(FETCHED_AUTOMATICALLY) && this.lastFetched != null) { - // If the last feed was automatically fetched, - // set a modified threshold to skip download unless there is a more recent feed - // (if the source server supports it). - modifiedThreshold = Math.min(latest.updated.getTime(), this.lastFetched.getTime()); - } - HttpURLConnection conn = makeHttpURLConnection(status, optionalUrlOverride, modifiedThreshold); + HttpURLConnection conn = makeHttpURLConnection(status, optionalUrlOverride, getModifiedThreshold(latest)); if (conn == null) return null; try { conn.connect(); - return fetch(status, optionalUrlOverride, version, latest, new HttpURLConnectionResponse(conn)); + return processFetchResponse(status, optionalUrlOverride, version, latest, new HttpURLConnectionResponse(conn)); } catch (IOException e) { String message = String.format("Unable to connect to %s; not fetching %s feed", conn.getURL(), this.name); // url, this.name); LOG.error(message); @@ -245,6 +237,25 @@ public FeedVersion fetch (MonitorableJob.Status status, String optionalUrlOverri } } + /** + * Computes the modified time to set to the HttpURLConnection + * so that if a version has not been published since the last fetch, + * then download can be skipped. + * @return The computed threshold if the latest feed version exists and was auto-fetched + * and there is a record for the last fetch action, null otherwise. + */ + private Long getModifiedThreshold(FeedVersion latest) { + Long modifiedThreshold = null; + // lastFetched is set to null when the URL changes and when latest feed version is deleted + if (latest != null && latest.retrievalMethod.equals(FETCHED_AUTOMATICALLY) && this.lastFetched != null) { + modifiedThreshold = Math.min(latest.updated.getTime(), this.lastFetched.getTime()); + } + return modifiedThreshold; + } + + /** + * Builds an {@link HttpURLConnection}. + */ private HttpURLConnection makeHttpURLConnection(MonitorableJob.Status status, String optionalUrlOverride, Long modifiedThreshold) { // build the URL from which to fetch URL url = null; @@ -280,10 +291,16 @@ private HttpURLConnection makeHttpURLConnection(MonitorableJob.Status status, St } /** - * Processes the given response. + * Processes the given fetch response. * @return true if a new FeedVersion was created from the response, false otherwise. */ - public FeedVersion fetch(MonitorableJob.Status status, String optionalUrlOverride, FeedVersion version, FeedVersion latest, ConnectionResponse response) { + public FeedVersion processFetchResponse( + MonitorableJob.Status status, + String optionalUrlOverride, + FeedVersion version, + FeedVersion latest, + ConnectionResponse response + ) { File newGtfsFile; try { String message; diff --git a/src/test/java/com/conveyal/datatools/manager/jobs/FetchLoadFeedCombinationTest.java b/src/test/java/com/conveyal/datatools/manager/jobs/FetchLoadFeedCombinationTest.java index afa52b246..56c99d32c 100644 --- a/src/test/java/com/conveyal/datatools/manager/jobs/FetchLoadFeedCombinationTest.java +++ b/src/test/java/com/conveyal/datatools/manager/jobs/FetchLoadFeedCombinationTest.java @@ -139,7 +139,8 @@ void shouldNotLoadVersionIdenticalToPrevious() { // Assert no version 2 is created. assertVersionCount(1); - // create wiremock stub for get users endpoint + // Some servers support a 304 (not modified) response, + // and that should also result in no new version created. wireMockServer.stubFor( get(urlPathEqualTo(MOCKED_FETCH_URL)) .willReturn( @@ -148,7 +149,7 @@ void shouldNotLoadVersionIdenticalToPrevious() { ) ); - // Simulate the second fetch with the response as "unchanged". + // Simulate the re-fetch with the response as "unchanged". simulateFetch(); // Assert no version 2 is created. @@ -167,7 +168,7 @@ private void simulateFetch() { .response() ); FeedVersion newVersion = new FeedVersion(feedSource, FETCHED_AUTOMATICALLY); - newVersion = feedSource.fetch( + newVersion = feedSource.processFetchResponse( new MonitorableJob.Status(), null, newVersion, From e4beec0b0a68e6a40d556a66c51c638d727ed90f Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Wed, 26 Jan 2022 07:27:02 +0000 Subject: [PATCH 111/122] refactor(MergeFeedsJobTest.java): Added tests to check parent station reference update --- .../conveyal/datatools/manager/jobs/MergeFeedsJobTest.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java b/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java index b926ff5ea..efcadf01f 100644 --- a/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java +++ b/src/test/java/com/conveyal/datatools/manager/jobs/MergeFeedsJobTest.java @@ -259,6 +259,12 @@ void canMergeRegionalWithOnlyCalendarFeed () throws SQLException { // 2 trips with onlyCalendarVersion's common_id service_id should be scoped sqlAssert.trips.assertCount(2, "service_id='Fake_Agency2:common_id'"); + + // 2 parent stations should reference the updated stop_id for Fake_Agency2 + sqlAssert.stops.assertCount(2, "parent_station='Fake_Agency2:123'"); + + // 2 parent stations should reference the updated stop_id for Fake_Agency3 + sqlAssert.stops.assertCount(2, "parent_station='Fake_Agency3:123'"); } /** From 943b1cf6047e542ba7f17953f08320a22d0dd003 Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Wed, 26 Jan 2022 09:10:10 +0000 Subject: [PATCH 112/122] refactor(Update to check field references): Refactor to allow for internal table references to be co --- .../jobs/feedmerge/MergeLineContext.java | 11 ++++++++++ .../jobs/feedmerge/StopsMergeLineContext.java | 21 +++++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java index b5e549ee4..8cacad4d1 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java @@ -503,6 +503,14 @@ public void afterTableRecords() throws IOException { // Default is to do nothing. } + /** + * Overridable placeholder for checking internal table references. E.g. parent_station references stop_id. It is + * illegal to have a self reference within a {@link Table} configuration. + */ + public void checkFieldsForReferences(FieldContext fieldContext) { + // Default is to do nothing. + } + public void scopeValueIfNeeded(FieldContext fieldContext) { boolean isKeyField = fieldContext.getField().isForeignReference() || fieldContext.nameEquals(keyField); if (job.mergeType.equals(REGIONAL) && isKeyField && !fieldContext.getValue().isEmpty()) { @@ -594,6 +602,9 @@ public boolean constructRowValues() throws IOException { continue; } } + + checkFieldsForReferences(fieldContext); + // If the current field is a foreign reference, check if the reference has been removed in the // merged result. If this is the case (or other conditions are met), we will need to skip this // record. Likewise, if the reference has been modified, ensure that the value written to the diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/StopsMergeLineContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/StopsMergeLineContext.java index 7a8983090..e62369ec4 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/StopsMergeLineContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/StopsMergeLineContext.java @@ -2,6 +2,7 @@ import com.conveyal.datatools.manager.jobs.MergeFeedsJob; import com.conveyal.gtfs.error.NewGTFSError; +import com.conveyal.gtfs.loader.Field; import com.conveyal.gtfs.loader.Table; import com.csvreader.CsvReader; import org.slf4j.Logger; @@ -33,6 +34,26 @@ public boolean checkFieldsForMergeConflicts(Set idErrors, FieldCon return checkRoutesAndStopsIds(idErrors, fieldContext); } + @Override + public void checkFieldsForReferences(FieldContext fieldContext) { + updateParentStationReference(fieldContext); + } + + /** + * If there is a parent station reference, update to include the scope stop_id. + */ + private void updateParentStationReference(FieldContext fieldContext) { + Field field = fieldContext.getField(); + if (field.name.equals("parent_station")) { + String parentStation = fieldContext.getValue(); + if (!"".equals(parentStation)) { + LOG.debug("Updating parent station to: " + getIdWithScope(parentStation)); + fieldContext.resetValue(parentStation); + updateAndRemapOutput(fieldContext); + } + } + } + /** * Checks that the stop_code field of the Stop entities to merge is populated where required. * @throws IOException From 1e98025119352a8b13e2495a43efe033278c520e Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Fri, 4 Feb 2022 11:00:37 +0000 Subject: [PATCH 113/122] refactor(StopsMergeLineContext.java): Update to use shorthand method --- .../manager/jobs/feedmerge/StopsMergeLineContext.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/StopsMergeLineContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/StopsMergeLineContext.java index e62369ec4..eb33e6142 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/StopsMergeLineContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/StopsMergeLineContext.java @@ -43,8 +43,7 @@ public void checkFieldsForReferences(FieldContext fieldContext) { * If there is a parent station reference, update to include the scope stop_id. */ private void updateParentStationReference(FieldContext fieldContext) { - Field field = fieldContext.getField(); - if (field.name.equals("parent_station")) { + if (fieldContext.nameEquals("parent_station")) { String parentStation = fieldContext.getValue(); if (!"".equals(parentStation)) { LOG.debug("Updating parent station to: " + getIdWithScope(parentStation)); From 70fd7545faa8582ef246b460239f711d93ef3efa Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Fri, 4 Feb 2022 11:11:21 +0000 Subject: [PATCH 114/122] refactor(StopsMergeLineContext.java): Updated logging style --- .../datatools/manager/jobs/feedmerge/StopsMergeLineContext.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/StopsMergeLineContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/StopsMergeLineContext.java index eb33e6142..acd4e19e8 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/StopsMergeLineContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/StopsMergeLineContext.java @@ -46,7 +46,7 @@ private void updateParentStationReference(FieldContext fieldContext) { if (fieldContext.nameEquals("parent_station")) { String parentStation = fieldContext.getValue(); if (!"".equals(parentStation)) { - LOG.debug("Updating parent station to: " + getIdWithScope(parentStation)); + LOG.debug("Updating parent station to: {}", getIdWithScope(parentStation)); fieldContext.resetValue(parentStation); updateAndRemapOutput(fieldContext); } From 9ac239c259d6e00a96eb275e66b1cf9a2bdf6ba6 Mon Sep 17 00:00:00 2001 From: Philip Cline Date: Mon, 7 Feb 2022 13:20:45 -0500 Subject: [PATCH 115/122] fix(deps): bump gtfs-lib to v7.0.4 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 8f5f4e35a..2025b72a0 100644 --- a/pom.xml +++ b/pom.xml @@ -270,7 +270,7 @@ com.github.conveyal gtfs-lib - 7.0.3 + 7.0.4 From 29e135704187bd8c1543691182f009d6869f1f8f Mon Sep 17 00:00:00 2001 From: miles-grant-ibi Date: Mon, 7 Feb 2022 13:13:49 -0800 Subject: [PATCH 116/122] fix(StopsMergeLineContext): remap output before wrting new value --- .../datatools/manager/jobs/feedmerge/StopsMergeLineContext.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/StopsMergeLineContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/StopsMergeLineContext.java index acd4e19e8..4e4893f7c 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/StopsMergeLineContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/StopsMergeLineContext.java @@ -46,9 +46,9 @@ private void updateParentStationReference(FieldContext fieldContext) { if (fieldContext.nameEquals("parent_station")) { String parentStation = fieldContext.getValue(); if (!"".equals(parentStation)) { + updateAndRemapOutput(fieldContext); LOG.debug("Updating parent station to: {}", getIdWithScope(parentStation)); fieldContext.resetValue(parentStation); - updateAndRemapOutput(fieldContext); } } } From e08419aacceb73b358e9f4eb28c1d9328e079f49 Mon Sep 17 00:00:00 2001 From: miles-grant-ibi Date: Mon, 7 Feb 2022 13:21:42 -0800 Subject: [PATCH 117/122] fix(StopsMergeLineContext): add missing stop ID update --- .../manager/jobs/feedmerge/StopsMergeLineContext.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/StopsMergeLineContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/StopsMergeLineContext.java index 4e4893f7c..9509ce685 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/StopsMergeLineContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/StopsMergeLineContext.java @@ -46,9 +46,9 @@ private void updateParentStationReference(FieldContext fieldContext) { if (fieldContext.nameEquals("parent_station")) { String parentStation = fieldContext.getValue(); if (!"".equals(parentStation)) { - updateAndRemapOutput(fieldContext); LOG.debug("Updating parent station to: {}", getIdWithScope(parentStation)); - fieldContext.resetValue(parentStation); + updateAndRemapOutput(fieldContext); + fieldContext.resetValue(getIdWithScope(parentStation)); } } } From b3b7339db2089e6dc2811416d9a28013dc9941b7 Mon Sep 17 00:00:00 2001 From: miles-grant-ibi Date: Mon, 7 Feb 2022 13:36:28 -0800 Subject: [PATCH 118/122] refactor(MergeLineContext): only check fields for references when references change --- .../datatools/manager/jobs/feedmerge/MergeLineContext.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java index 8cacad4d1..a97d08d46 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java @@ -601,9 +601,13 @@ public boolean constructRowValues() throws IOException { skipRecord = true; continue; } + } else { + // TODO: this method renames a lot of fields which don't seem to be updated + // when using SERVICE_PERIOD. However, it may do other things which are needed even when + // using SERVICE_PERIOD + checkFieldsForReferences(fieldContext); } - checkFieldsForReferences(fieldContext); // If the current field is a foreign reference, check if the reference has been removed in the // merged result. If this is the case (or other conditions are met), we will need to skip this From 5e1f152233d1833095c5895c58fb5d36dbe26adf Mon Sep 17 00:00:00 2001 From: miles-grant-ibi Date: Mon, 7 Feb 2022 13:51:19 -0800 Subject: [PATCH 119/122] refactor(StopsMergeLineContext): move updateAndRemapOutput back to original location --- .../datatools/manager/jobs/feedmerge/StopsMergeLineContext.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/StopsMergeLineContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/StopsMergeLineContext.java index 9509ce685..0d9f18aa9 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/StopsMergeLineContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/StopsMergeLineContext.java @@ -47,8 +47,8 @@ private void updateParentStationReference(FieldContext fieldContext) { String parentStation = fieldContext.getValue(); if (!"".equals(parentStation)) { LOG.debug("Updating parent station to: {}", getIdWithScope(parentStation)); - updateAndRemapOutput(fieldContext); fieldContext.resetValue(getIdWithScope(parentStation)); + updateAndRemapOutput(fieldContext); } } } From c0749bc63234dbf79928c8966949307f8f2a4070 Mon Sep 17 00:00:00 2001 From: miles-grant-ibi Date: Mon, 7 Feb 2022 13:54:07 -0800 Subject: [PATCH 120/122] Revert "refactor(StopsMergeLineContext): move updateAndRemapOutput back to original location" This reverts commit 5e1f152233d1833095c5895c58fb5d36dbe26adf. --- .../datatools/manager/jobs/feedmerge/StopsMergeLineContext.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/StopsMergeLineContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/StopsMergeLineContext.java index 0d9f18aa9..9509ce685 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/StopsMergeLineContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/StopsMergeLineContext.java @@ -47,8 +47,8 @@ private void updateParentStationReference(FieldContext fieldContext) { String parentStation = fieldContext.getValue(); if (!"".equals(parentStation)) { LOG.debug("Updating parent station to: {}", getIdWithScope(parentStation)); - fieldContext.resetValue(getIdWithScope(parentStation)); updateAndRemapOutput(fieldContext); + fieldContext.resetValue(getIdWithScope(parentStation)); } } } From 0684aa93cc1156c5932e0aef694ce3a7b1a386dc Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Tue, 8 Feb 2022 10:05:46 +0000 Subject: [PATCH 121/122] refactor(Check fields for reference updates): Update to be applied only if merge type is regional --- .../manager/jobs/feedmerge/MergeLineContext.java | 9 ++++----- .../manager/jobs/feedmerge/StopsMergeLineContext.java | 1 - 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java index a97d08d46..78da13dff 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java @@ -601,14 +601,13 @@ public boolean constructRowValues() throws IOException { skipRecord = true; continue; } - } else { - // TODO: this method renames a lot of fields which don't seem to be updated - // when using SERVICE_PERIOD. However, it may do other things which are needed even when - // using SERVICE_PERIOD + } else if (job.mergeType.equals(REGIONAL)){ + // If merging feed versions from different agencies, the reference id is updated to avoid conflicts. + // e.g. stop_id becomes Fake_Agency2:123 instead of 123. This method allows referencing fields to be + // updated to the newer id. checkFieldsForReferences(fieldContext); } - // If the current field is a foreign reference, check if the reference has been removed in the // merged result. If this is the case (or other conditions are met), we will need to skip this // record. Likewise, if the reference has been modified, ensure that the value written to the diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/StopsMergeLineContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/StopsMergeLineContext.java index 9509ce685..4644c8c3a 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/StopsMergeLineContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/StopsMergeLineContext.java @@ -48,7 +48,6 @@ private void updateParentStationReference(FieldContext fieldContext) { if (!"".equals(parentStation)) { LOG.debug("Updating parent station to: {}", getIdWithScope(parentStation)); updateAndRemapOutput(fieldContext); - fieldContext.resetValue(getIdWithScope(parentStation)); } } } From 73baceb0af6f153489409ef3f6466c581908295c Mon Sep 17 00:00:00 2001 From: Robin Beer Date: Tue, 8 Feb 2022 15:16:20 +0000 Subject: [PATCH 122/122] refactor(MergeLineContext.java): Added missing space Added missing space --- .../datatools/manager/jobs/feedmerge/MergeLineContext.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java index 78da13dff..5fb7edd0f 100644 --- a/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java +++ b/src/main/java/com/conveyal/datatools/manager/jobs/feedmerge/MergeLineContext.java @@ -601,7 +601,7 @@ public boolean constructRowValues() throws IOException { skipRecord = true; continue; } - } else if (job.mergeType.equals(REGIONAL)){ + } else if (job.mergeType.equals(REGIONAL)) { // If merging feed versions from different agencies, the reference id is updated to avoid conflicts. // e.g. stop_id becomes Fake_Agency2:123 instead of 123. This method allows referencing fields to be // updated to the newer id.