From 94591062b0cb04c65c01bcfbf4b8cd7677b190c3 Mon Sep 17 00:00:00 2001
From: Haitam Laarabi <10712736+htmlrb@users.noreply.github.com>
Date: Thu, 10 Mar 2022 00:27:36 -0800
Subject: [PATCH 01/58] adding python scripts

---
 .../freight/frism_to_beam_freight_plans.py    | 112 ++++++++++++++++++
 src/main/python/gemini/debugging.py           |  28 +++++
 .../nrel_to_beam_charging_infrastruture.py    | 110 +++++++++++++++++
 .../unlimited_charging_infrastructure.py      |  22 ++--
 4 files changed, 261 insertions(+), 11 deletions(-)
 create mode 100644 src/main/python/freight/frism_to_beam_freight_plans.py
 create mode 100644 src/main/python/gemini/debugging.py
 create mode 100644 src/main/python/gemini/nrel_to_beam_charging_infrastruture.py

diff --git a/src/main/python/freight/frism_to_beam_freight_plans.py b/src/main/python/freight/frism_to_beam_freight_plans.py
new file mode 100644
index 00000000000..01d9dd43cfb
--- /dev/null
+++ b/src/main/python/freight/frism_to_beam_freight_plans.py
@@ -0,0 +1,112 @@
+import pandas as pd
+import os
+from pathlib import Path
+
+def read_csv_file(filename):
+    compression = None
+    if filename.endswith(".gz"):
+        compression = 'gzip'
+    return pd.read_csv(filename, sep=",", index_col=None, header=0, compression=compression)
+
+
+def add_prefix(prefix, column, row, toNum = True):
+    if toNum:
+        old = str(int(row[column]))
+    else:
+        old = str(row[column])
+    new = f"{prefix}{old}"
+    return new
+
+
+directory_input = os.path.expanduser('~/Data/FREIGHT/Outputs_All_SF_0223_2022')
+directory_output = os.path.expanduser('~/Data/FREIGHT/Outputs_All_SF_0223_2022_merged')
+Path(directory_output).mkdir(parents=True, exist_ok=True)
+carriers = None
+payload_plans = None
+tours = None
+
+for filename in os.listdir(directory_input):
+    filepath = f'{directory_input}/{filename}'
+    parts = filename.split('_', 2)
+    unique_id = parts[0].lower()
+    county = parts[1].lower()
+    filetype = parts[2].lower()
+    
+    if filetype.startswith('carrier_'):
+        df = pd.read_csv(filepath)
+        df['carrierId'] = df.apply(lambda row: add_prefix(f'{unique_id}-{county}-', 'carrierId', row), axis=1)
+        df['vehicleId'] = df.apply(lambda row: add_prefix(f'{unique_id}-{county}-', 'vehicleId', row), axis=1)
+        df['vehicleTypeId'] = df.apply(lambda row: add_prefix('FREIGHT-', 'vehicleTypeId', row), axis=1)
+        df['tourId'] = df.apply(lambda row: add_prefix(f'{unique_id}-{county}-', 'tourId', row), axis=1)
+        if carriers is None:
+            carriers = df
+        else:
+            carriers = pd.concat([carriers, df])
+    elif filetype.startswith('freight_tours_'):
+        df = pd.read_csv(filepath)
+        df['tour_id'] = df.apply(lambda row: add_prefix(f'{unique_id}-{county}-', 'tour_id', row), axis=1)
+        if tours is None:
+            tours = df
+        else:
+            tours = pd.concat([tours, df])
+    elif filetype.startswith('payload_'):
+        df = pd.read_csv(filepath)
+        df['tourId'] = df.apply(lambda row: add_prefix(f'{unique_id}-{county}-', 'tourId', row), axis=1)
+        df['payloadId'] = df.apply(lambda row: add_prefix(f'{unique_id}-{county}-', 'payloadId', row, False), axis=1)
+        if payload_plans is None:
+            payload_plans = df
+        else:
+            payload_plans = pd.concat([payload_plans, df])
+    else:
+        print(f'SKIPPING {filetype}')
+
+
+# In[9]:
+
+
+# carrierId,tourId,vehicleId,vehicleTypeId,warehouseZone,warehouseX,warehouseY,MESOZONE,BoundaryZONE
+# carrierId,tourId,vehicleId,vehicleTypeId,warehouseZone,warehouseX,warehouseY,MESOZONE,BoundaryZONE
+carriers_renames = {
+    'depot_zone': 'warehouseZone',
+    'depot_zone_x': 'warehouseX',
+    'depot_zone_y': 'warehouseY'
+}
+carriers_drop = ['x', 'y']
+carriers.rename(columns=carriers_renames, inplace=True)
+carriers.drop(carriers_drop, axis=1, inplace=True)
+carriers.to_csv(f'{directory_output}/freight-merged-carriers.csv', index=False)
+
+
+# In[10]:
+
+
+# tourId,departureTimeInSec,departureLocationZone,maxTourDurationInSec,departureLocationX,departureLocationY
+# tourId,departureTimeInSec,departureLocationZone,maxTourDurationInSec,departureLocationX,departureLocationY
+tours_renames = {
+    'tour_id': 'tourId',
+    'departureLocation_zone': 'departureLocationZone',
+    'departureLocation_x': 'departureLocationX',
+    'departureLocation_y': 'departureLocationY'
+}
+tours.rename(columns=tours_renames, inplace=True)
+tours.to_csv(f'{directory_output}/freight-merged-tours.csv', index=False)
+
+
+# In[11]:
+
+
+# payloadId,sequenceRank,tourId,payloadType,weightInKg,requestType,locationZone,estimatedTimeOfArrivalInSec,arrivalTimeWindowInSecLower,arrivalTimeWindowInSecUpper,operationDurationInSec,locationX,locationY
+# payloadId,sequenceRank,tourId,payloadType,weightInKg,requestType,locationZone,estimatedTimeOfArrivalInSec,arrivalTimeWindowInSecLower,arrivalTimeWindowInSecUpper,operationDurationInSec,locationX,locationY
+payload_plans_renames = {
+    'arrivalTimeWindowInSec_lower': 'arrivalTimeWindowInSecLower',
+    'arrivalTimeWindowInSec_upper': 'arrivalTimeWindowInSecUpper',
+    'locationZone_x': 'locationX',
+    'locationZone_y': 'locationY'
+}
+payload_plans_drop = ['weightInlb', 'cummulativeWeightInlb']
+payload_plans['weightInKg'] = abs(payload_plans['weightInlb'].astype(int)) * 0.45359237
+payload_plans.rename(columns=payload_plans_renames, inplace=True)
+payload_plans.drop(payload_plans_drop, axis=1, inplace=True)
+payload_plans.to_csv(f'{directory_output}/freight-merged-payload-plans.csv', index=False)
+
+print("END")
\ No newline at end of file
diff --git a/src/main/python/gemini/debugging.py b/src/main/python/gemini/debugging.py
new file mode 100644
index 00000000000..82a58c5435c
--- /dev/null
+++ b/src/main/python/gemini/debugging.py
@@ -0,0 +1,28 @@
+import os
+
+mnl_lines = ["mnlStatus,requestId,parkingZoneId,chargingPointType,parkingType,costInDollars\n"]
+mnl_search_lines = ["parkingZoneId,geoId,parkingType,chargingPointType,pricingModel,reservedFor,stallsAvailable,"
+                    "maxStalls,vehicleId,parkingDuration,activityType,valueOfTime,requestId,isEV,rideHailFastChargingOnly,"
+                    "validChargingCapability,hasAvailability,validParkingType,isValidTime,isValidVehicleManager\n"]
+mnl_param_lines = ["parkingZoneId,geoId,parkingType,chargingPointType,pricingModel,reservedFor,stallsAvailable,"
+                   "maxStalls,vehicleId,parkingDuration,activityType,valueOfTime,requestId,costInDollars,RangeAnxietyCost,"
+                   "WalkingEgressCost,ParkingTicketCost,HomeActivityPrefersResidentialParking\n"]
+output_log_file = os.path.expanduser('~/Data/GEMINI/2022Mars-Calibration/beamLog.out.txt')
+with open(output_log_file) as infile:
+    for line in infile:
+        if "SAMPLED: " in line:
+            mnl_lines.append("sampled,"+line.split("SAMPLED: ")[1])
+        elif "CHOSEN: " in line:
+            mnl_lines.append("chosen,"+line.split("CHOSEN: ")[1])
+        elif "SEARCH: " in line:
+            mnl_search_lines.append(line.split("SEARCH: ")[1])
+        elif "PARAM: " in line:
+            mnl_param_lines.append(line.split("PARAM: ")[1])
+
+with open(os.path.expanduser('~/Data/GEMINI/2022Mars-Calibration/mnl-beamLog.csv'), 'w') as f:
+    f.writelines(mnl_lines)
+with open(os.path.expanduser('~/Data/GEMINI/2022Mars-Calibration/mnl-search-beamLog.csv'), 'w') as f:
+    f.writelines(mnl_search_lines)
+with open(os.path.expanduser('~/Data/GEMINI/2022Mars-Calibration/mnl-param-beamLog.csv'), 'w') as f:
+    f.writelines(mnl_param_lines)
+print("END")
diff --git a/src/main/python/gemini/nrel_to_beam_charging_infrastruture.py b/src/main/python/gemini/nrel_to_beam_charging_infrastruture.py
new file mode 100644
index 00000000000..c9621cf5bb8
--- /dev/null
+++ b/src/main/python/gemini/nrel_to_beam_charging_infrastruture.py
@@ -0,0 +1,110 @@
+import pandas as pd
+import os
+import random
+from tqdm import tqdm
+
+def read_csv_file(filename):
+    compression = None
+    if filename.endswith(".gz"):
+        compression = 'gzip'
+    return pd.read_csv(filename, sep=",", index_col=None, header=0, compression=compression)
+
+nrel_file_input = os.path.expanduser('~/Data/GEMINI/2022Feb/siting/init1-7_2022_Feb_03_wgs84.csv')
+smart_file_input = os.path.expanduser("~/Data/GEMINI/stations/taz-parking-sparse-fast-limited-l2-150-lowtech-b.csv")
+nrel_file_converted_input = os.path.expanduser(nrel_file_input.split(".")[0] + "_converted.csv")
+smart_file_updated_input = os.path.expanduser(smart_file_input.split(".")[0] + "_updated.csv")
+smart_file_with_fees_input = os.path.expanduser(nrel_file_input.split(".")[0] + "_withFees.csv.gz")
+
+
+def convert_nrel_data(nrel_file, nrel_file_converted):
+    if not os.path.exists(nrel_file_converted):
+        data = read_csv_file(nrel_file)
+        data2 = data[["subSpace", "pType", "chrgType", "field_1", "household_id", "X", "Y", "housingTypes", "propertytype", "propertysubtype", "county"]]
+        data2 = data2.rename(columns={
+            "chrgType": "chargingPointType",
+            "pType": "parkingType",
+            "subSpace": "taz",
+            "X": "locationX",
+            "Y": "locationY",
+            "housingTypes": "housingType",
+            "propertytype": "propertyType",
+            "propertysubtype": "propertySubType",
+            "county": "county"
+        })
+        data2["parkingZoneId"] = ""
+        data2["reservedFor"] = "Any"
+        data2["pricingModel"] = "Block"
+        data2["feeInCents"] = 0
+        data2["numStalls"] = 1
+        data2.loc[data2["household_id"].notna(), ['reservedFor']] = data2.loc[data2["household_id"].notna()].apply(
+            lambda row1: "household(" + str(int(row1["household_id"])) + ")", axis=1)
+        data2.loc[data2["field_1"].notna(), ['parkingZoneId']] = data2.loc[data2["field_1"].notna()].apply(
+            lambda row2: "PEV-" + str(int(row2["taz"])) + "-" + str(int(row2["field_1"])), axis=1)
+        nrel_data = data2.drop(columns=['household_id', 'field_1'])
+        nrel_data.to_csv(nrel_file_converted, index=False)
+        print("Reading nrel infrastructure done!")
+        return nrel_data
+    else:
+        return read_csv_file(nrel_file_converted)
+
+
+# Reading fees
+def reading_sf_bay_fees(smart_file, smart_file_updated):
+    if not os.path.exists(smart_file_updated):
+        smart_data = read_csv_file(smart_file)
+        smart_data["chargingPointType"] = "NoCharger"
+        smart_data.loc[(smart_data["chargingType"] == "WorkLevel2(7.2|AC)") & (smart_data["parkingType"] == "Public"), ['chargingPointType']] = "publiclevel2(7.2|AC)"
+        smart_data.loc[(smart_data["chargingType"] == "WorkLevel2(7.2|AC)") & (smart_data["parkingType"] == "Workplace"), ['chargingPointType']] = "worklevel2(7.2|AC)"
+        smart_data.loc[smart_data["chargingType"] == "Custom(150.0|DC)", ['chargingPointType']] = "publicfc(150.0|DC)"
+        smart_data.loc[smart_data["chargingType"] == "HomeLevel2(7.2|AC)", ['chargingPointType']] = "homelevel2(7.2|AC)"
+        smart_data.loc[smart_data["chargingType"] == "HomeLevel1(1.8|AC)", ['chargingPointType']] = "homelevel1(1.8|AC)"
+        smart_data = smart_data.drop(columns=["chargingType"])
+        smart_data = smart_data.rename(columns={"ReservedFor": "reservedFor"})
+        smart_data.to_csv(smart_file_updated, index=False)
+        print("Reading Fees done!")
+        return smart_data
+    else:
+        return read_csv_file(smart_file_updated)
+
+
+def assign_fees_to_infrastructure(nrel_data, fees_data, smart_file_with_fees):
+    df_dict = nrel_data.to_dict('records')
+    for row in tqdm(df_dict):
+        charging_type_arg = row["chargingPointType"]
+        if "fc" in charging_type_arg:
+            charging_type_arg = "publicfc(150.0|DC)"
+        filtered = fees_data.loc[(fees_data["taz"] == row["taz"]) &
+                                 (fees_data["parkingType"] == row["parkingType"]) &
+                                 (fees_data["chargingPointType"] == charging_type_arg)]
+        if len(filtered.index) == 0:
+            filtered = fees_data.loc[(fees_data["parkingType"] == row["parkingType"]) &
+                                     (fees_data["chargingPointType"] == charging_type_arg)]
+        pd.options.mode.chained_assignment = None
+        filtered.loc[:, "numStalls"] = filtered.loc[:, "numStalls"].astype('int')
+        tot_stalls = filtered["numStalls"].sum()
+        cumulated = 0.0
+        memorized_fee = 0.0
+        rd_prob = random.uniform(0, 1)
+        for row2 in filtered.itertuples():
+            memorized_fee = row2.feeInCents
+            cumulated = cumulated + float(row2.numStalls) / float(tot_stalls)
+            if cumulated >= rd_prob:
+                break
+        if "(150.0|DC)" in row["chargingPointType"]:
+            memorized_fee = memorized_fee * 1.6
+        elif "(250.0|DC)" in row["chargingPointType"]:
+            memorized_fee = memorized_fee * 2.2
+        elif "(400.0|DC)" in row["chargingPointType"]:
+            memorized_fee = memorized_fee * 3.1
+        row["feeInCents"] = memorized_fee
+    output = pd.DataFrame.from_dict(df_dict)
+    output.reset_index(drop=True, inplace=True)
+    output.to_csv(smart_file_with_fees, index=False)
+
+
+nrel_data_output = convert_nrel_data(nrel_file_input, nrel_file_converted_input)
+print("convert_nrel_data done!")
+fees_data_output = reading_sf_bay_fees(smart_file_input, smart_file_updated_input)
+print("reading_sf_bay_fees done!")
+assign_fees_to_infrastructure(nrel_data_output, fees_data_output, smart_file_with_fees_input)
+print("END")
diff --git a/src/main/python/gemini/unlimited_charging_infrastructure.py b/src/main/python/gemini/unlimited_charging_infrastructure.py
index d0936fe6dd6..b31c9661042 100644
--- a/src/main/python/gemini/unlimited_charging_infrastructure.py
+++ b/src/main/python/gemini/unlimited_charging_infrastructure.py
@@ -14,24 +14,24 @@
     csv_writer.write(headerfile+"\n")
 
     for x in range(1, 1455):
-        csv_writer.write(f"{x},Residential,Block,HomeLevel1(1.8|AC),9999999,50,Any" + "\n")
-        csv_writer.write(f"{x},Residential,Block,HomeLevel2(7.2|AC),9999999,200,Any" + "\n")
+        csv_writer.write(f"{x},Residential,Block,HomeLevel1(1.8|AC),9999999,0.45,Any" + "\n")
+        csv_writer.write(f"{x},Residential,Block,HomeLevel2(7.2|AC),9999999,1.8,Any" + "\n")
 
-        csv_writer.write(f"{x},Workplace,Block,WorkLevel2(7.2|AC),9999999,200,Any" + "\n")
+        csv_writer.write(f"{x},Workplace,Block,WorkLevel2(7.2|AC),9999999,14.4,Any" + "\n")
 
-        csv_writer.write(f"{x},Public,Block,PublicLevel2(7.2|AC),9999999,200,Any" + "\n")
-        csv_writer.write(f"{x},Public,Block,PublicFC(50|DC),9999999,1600,Any" + "\n")
-        csv_writer.write(f"{x},Public,Block,PublicFC(150|DC),9999999,4800,Any" + "\n")
-        csv_writer.write(f"{x},Public,Block,PublicXFC(250|DC),9999999,9500,Any" + "\n")
-        csv_writer.write(f"{x},Public,Block,PublicXFC(400|DC),9999999,15200,Any" + "\n")
+        csv_writer.write(f"{x},Public,Block,PublicLevel2(7.2|AC),9999999,19.3,Any" + "\n")
+        csv_writer.write(f"{x},Public,Block,PublicFC(50|DC),9999999,306.73,Any" + "\n")
+        csv_writer.write(f"{x},Public,Block,PublicFC(150|DC),9999999,490.98,Any" + "\n")
+        csv_writer.write(f"{x},Public,Block,PublicXFC(250|DC),9999999,675.05,Any" + "\n")
+        csv_writer.write(f"{x},Public,Block,PublicXFC(400|DC),9999999,950.52,Any" + "\n")
 
 
 with open('gemini_taz_unlimited_depots.csv', mode='w') as csv_writer:
     csv_writer.write(headerfile+"\n")
 
     for x in range(1, 1455):
-        csv_writer.write(f"{x},Public,FlatFee,DepotFC(150.0|DC),9999999,4800,ridehail(GlobalRHM)" + "\n")
-        csv_writer.write(f"{x},Public,FlatFee,DepotXFC(250.0|DC),9999999,9500,ridehail(GlobalRHM)" + "\n")
-        csv_writer.write(f"{x},Public,FlatFee,DepotXFC(400.0|DC),9999999,15200,ridehail(GlobalRHM)" + "\n")
+        csv_writer.write(f"{x},Public,FlatFee,DepotFC(150.0|DC),9999999,490.98,ridehail(GlobalRHM)" + "\n")
+        csv_writer.write(f"{x},Public,FlatFee,DepotXFC(250.0|DC),9999999,675.05,ridehail(GlobalRHM)" + "\n")
+        csv_writer.write(f"{x},Public,FlatFee,DepotXFC(400.0|DC),9999999,950.52,ridehail(GlobalRHM)" + "\n")
 
 

From 06c7e8ccf8b1b54df36d190cfb57823b4bda8842 Mon Sep 17 00:00:00 2001
From: Haitam Laarabi <10712736+htmlrb@users.noreply.github.com>
Date: Thu, 10 Mar 2022 01:49:49 -0800
Subject: [PATCH 02/58] updating script

---
 .../python/freight/frism_to_beam_freight_plans.py    | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/src/main/python/freight/frism_to_beam_freight_plans.py b/src/main/python/freight/frism_to_beam_freight_plans.py
index 01d9dd43cfb..6e48543772f 100644
--- a/src/main/python/freight/frism_to_beam_freight_plans.py
+++ b/src/main/python/freight/frism_to_beam_freight_plans.py
@@ -74,6 +74,7 @@ def add_prefix(prefix, column, row, toNum = True):
 carriers_drop = ['x', 'y']
 carriers.rename(columns=carriers_renames, inplace=True)
 carriers.drop(carriers_drop, axis=1, inplace=True)
+carriers['warehouseZone'] = carriers['warehouseZone'].astype(int)
 carriers.to_csv(f'{directory_output}/freight-merged-carriers.csv', index=False)
 
 
@@ -89,6 +90,9 @@ def add_prefix(prefix, column, row, toNum = True):
     'departureLocation_y': 'departureLocationY'
 }
 tours.rename(columns=tours_renames, inplace=True)
+tours['departureTimeInSec'] = tours['departureTimeInSec'].astype(int)
+tours['maxTourDurationInSec'] = tours['maxTourDurationInSec'].astype(int)
+tours['departureLocationZone'] = tours['departureLocationZone'].astype(int)
 tours.to_csv(f'{directory_output}/freight-merged-tours.csv', index=False)
 
 
@@ -107,6 +111,14 @@ def add_prefix(prefix, column, row, toNum = True):
 payload_plans['weightInKg'] = abs(payload_plans['weightInlb'].astype(int)) * 0.45359237
 payload_plans.rename(columns=payload_plans_renames, inplace=True)
 payload_plans.drop(payload_plans_drop, axis=1, inplace=True)
+payload_plans['sequenceRank'] = payload_plans['sequenceRank'].astype(int)
+payload_plans['payloadType'] = payload_plans['payloadType'].astype(int)
+payload_plans['requestType'] = payload_plans['requestType'].astype(int)
+payload_plans['estimatedTimeOfArrivalInSec'] = payload_plans['estimatedTimeOfArrivalInSec'].astype(int)
+payload_plans['arrivalTimeWindowInSecLower'] = payload_plans['arrivalTimeWindowInSecLower'].astype(int)
+payload_plans['arrivalTimeWindowInSecUpper'] = payload_plans['arrivalTimeWindowInSecUpper'].astype(int)
+payload_plans['operationDurationInSec'] = payload_plans['operationDurationInSec'].astype(int)
+payload_plans['locationZone'] = payload_plans['locationZone'].astype(int)
 payload_plans.to_csv(f'{directory_output}/freight-merged-payload-plans.csv', index=False)
 
 print("END")
\ No newline at end of file

From a62850bdd5cc9984747bf340edf6bf0338722493 Mon Sep 17 00:00:00 2001
From: Haitam Laarabi <10712736+htmlrb@users.noreply.github.com>
Date: Thu, 10 Mar 2022 09:08:24 -0800
Subject: [PATCH 03/58] merging rudvik's freight adjustment

---
 build.gradle                                        |  2 +-
 .../scala/beam/router/r5/CarWeightCalculator.scala  | 13 +++++++++++--
 src/main/scala/beam/router/r5/R5Wrapper.scala       | 11 +++++++++--
 3 files changed, 21 insertions(+), 5 deletions(-)

diff --git a/build.gradle b/build.gradle
index f0199365510..89b15e53d10 100755
--- a/build.gradle
+++ b/build.gradle
@@ -136,7 +136,7 @@ configurations.all {
 
 dependencies {
 
-    implementation(group: 'com.github.LBNL-UCB-STI', name: 'beam-utilities', version: 'v0.2.15') {
+    implementation(group: 'com.github.LBNL-UCB-STI', name: 'beam-utilities', version: 'v0.2.16') {
         exclude group: 'com.github.LBNL-UCB-STI', module: 'r5'
         exclude group: 'org.matsim', module: 'matsim'
     }
diff --git a/src/main/scala/beam/router/r5/CarWeightCalculator.scala b/src/main/scala/beam/router/r5/CarWeightCalculator.scala
index a97b137706c..6d1c48f0a26 100644
--- a/src/main/scala/beam/router/r5/CarWeightCalculator.scala
+++ b/src/main/scala/beam/router/r5/CarWeightCalculator.scala
@@ -5,6 +5,7 @@ import org.matsim.core.router.util.TravelTime
 
 import java.util.concurrent.ThreadLocalRandom
 import java.util.concurrent.atomic.AtomicInteger
+import scala.util.Try
 
 class CarWeightCalculator(workerParams: R5Parameters, travelTimeNoiseFraction: Double = 0d) {
   private val networkHelper = workerParams.networkHelper
@@ -32,7 +33,8 @@ class CarWeightCalculator(workerParams: R5Parameters, travelTimeNoiseFraction: D
     travelTime: TravelTime,
     vehicleType: Option[BeamVehicleType],
     time: Double,
-    shouldAddNoise: Boolean
+    shouldAddNoise: Boolean,
+    hgv: Boolean = false
   ): Double = {
     val link = networkHelper.getLinkUnsafe(linkId)
     assert(link != null)
@@ -54,6 +56,13 @@ class CarWeightCalculator(workerParams: R5Parameters, travelTimeNoiseFraction: D
         physSimTravelTime * travelTimeNoises(idx)
       }
     val linkTravelTime = Math.max(physSimTravelTimeWithNoise, minTravelTime)
-    Math.min(linkTravelTime, maxTravelTime)
+    val result = Math.min(linkTravelTime, maxTravelTime)
+
+    // TODO this is only prototype
+    val isLinkHgv = Try(link.getAttributes.getAttribute("hgv")).map(_.asInstanceOf[Boolean]).getOrElse(false)
+    if (hgv) {
+      if (isLinkHgv) result / 10 else result * 10
+    } else
+      result
   }
 }
diff --git a/src/main/scala/beam/router/r5/R5Wrapper.scala b/src/main/scala/beam/router/r5/R5Wrapper.scala
index c43457b5f25..91c97075149 100644
--- a/src/main/scala/beam/router/r5/R5Wrapper.scala
+++ b/src/main/scala/beam/router/r5/R5Wrapper.scala
@@ -1,7 +1,7 @@
 package beam.router.r5
 
 import beam.agentsim.agents.choice.mode.DrivingCost
-import beam.agentsim.agents.vehicles.BeamVehicleType
+import beam.agentsim.agents.vehicles.{BeamVehicleType, VehicleCategory}
 import beam.agentsim.agents.vehicles.VehicleProtocol.StreetVehicle
 import beam.agentsim.events.SpaceTime
 import beam.router.BeamRouter.{RoutingRequest, RoutingResponse, _}
@@ -1135,7 +1135,14 @@ class R5Wrapper(workerParams: R5Parameters, travelTime: TravelTime, travelTimeNo
       val maxSpeed: Double = vehicleType.maxVelocity.getOrElse(profileRequest.getSpeedForMode(streetMode))
       val minTravelTime = edge.getLengthM / maxSpeed
       if (streetMode == StreetMode.CAR) {
-        carWeightCalculator.calcTravelTime(linkId, travelTime, Some(vehicleType), time, shouldAddNoise)
+        carWeightCalculator.calcTravelTime(
+          linkId,
+          travelTime,
+          Some(vehicleType),
+          time,
+          shouldAddNoise,
+          vehicleType.vehicleCategory == VehicleCategory.HeavyDutyTruck
+        )
       } else if (streetMode == StreetMode.BICYCLE && shouldApplyBicycleScaleFactor) {
         val scaleFactor = bikeLanesAdjustment.scaleFactor(vehicleType, linkId)
         minTravelTime * scaleFactor

From 64aafd2868cc20ed397daf3bda0910f38a6e9c6d Mon Sep 17 00:00:00 2001
From: Haitam Laarabi <10712736+htmlrb@users.noreply.github.com>
Date: Fri, 11 Mar 2022 11:55:17 -0800
Subject: [PATCH 04/58] logging

---
 .../agentsim/agents/freight/input/GenericFreightReader.scala  | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/main/scala/beam/agentsim/agents/freight/input/GenericFreightReader.scala b/src/main/scala/beam/agentsim/agents/freight/input/GenericFreightReader.scala
index 25cf0f5f509..474841608fb 100644
--- a/src/main/scala/beam/agentsim/agents/freight/input/GenericFreightReader.scala
+++ b/src/main/scala/beam/agentsim/agents/freight/input/GenericFreightReader.scala
@@ -167,7 +167,9 @@ class GenericFreightReader(
             throw new IllegalArgumentException(
               s"Vehicle type ${firstRow.vehicleTypeId} for vehicle $vehicleId has no payloadCapacityInKg defined"
             )
-          createFreightVehicle(vehicleId, vehicleType, carrierId, warehouseLocationUTM, rnd.nextInt())
+          val vehicle = createFreightVehicle(vehicleId, vehicleType, carrierId, warehouseLocationUTM, rnd.nextInt())
+          logger.info(s"create vehicle ${vehicle.id} and vehicle type is ${vehicle.beamVehicleType}")
+          vehicle
         }
         .toIndexedSeq
       vehicles

From 6afd66afc3d39ae7afc51ca4ec9d553e4db2f9e5 Mon Sep 17 00:00:00 2001
From: Haitam Laarabi <10712736+htmlrb@users.noreply.github.com>
Date: Fri, 11 Mar 2022 13:28:37 -0800
Subject: [PATCH 05/58] testing a fix

---
 .../agentsim/agents/freight/input/GenericFreightReader.scala | 4 +---
 .../beam/agentsim/agents/modalbehaviors/ChoosesMode.scala    | 5 ++++-
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/src/main/scala/beam/agentsim/agents/freight/input/GenericFreightReader.scala b/src/main/scala/beam/agentsim/agents/freight/input/GenericFreightReader.scala
index 474841608fb..25cf0f5f509 100644
--- a/src/main/scala/beam/agentsim/agents/freight/input/GenericFreightReader.scala
+++ b/src/main/scala/beam/agentsim/agents/freight/input/GenericFreightReader.scala
@@ -167,9 +167,7 @@ class GenericFreightReader(
             throw new IllegalArgumentException(
               s"Vehicle type ${firstRow.vehicleTypeId} for vehicle $vehicleId has no payloadCapacityInKg defined"
             )
-          val vehicle = createFreightVehicle(vehicleId, vehicleType, carrierId, warehouseLocationUTM, rnd.nextInt())
-          logger.info(s"create vehicle ${vehicle.id} and vehicle type is ${vehicle.beamVehicleType}")
-          vehicle
+          createFreightVehicle(vehicleId, vehicleType, carrierId, warehouseLocationUTM, rnd.nextInt())
         }
         .toIndexedSeq
       vehicles
diff --git a/src/main/scala/beam/agentsim/agents/modalbehaviors/ChoosesMode.scala b/src/main/scala/beam/agentsim/agents/modalbehaviors/ChoosesMode.scala
index 0068f1f20cd..d715f535a65 100755
--- a/src/main/scala/beam/agentsim/agents/modalbehaviors/ChoosesMode.scala
+++ b/src/main/scala/beam/agentsim/agents/modalbehaviors/ChoosesMode.scala
@@ -269,11 +269,14 @@ trait ChoosesMode {
         implicit val executionContext: ExecutionContext = context.system.dispatcher
         plansModeOption match {
           case Some(CAR | DRIVE_TRANSIT) =>
+            val category = if(this.id.toString.toLowerCase.startsWith("FREIGHT")) {
+              VehicleCategory.HeavyDutyTruck
+            } else VehicleCategory.Car
             requestAvailableVehicles(
               vehicleFleets,
               currentLocation,
               _experiencedBeamPlan.activities(currentActivityIndex),
-              Some(VehicleCategory.Car)
+              Some(category)
             ) pipeTo self
           case Some(BIKE | BIKE_TRANSIT) =>
             requestAvailableVehicles(

From 70bbbfe65787c8733ad90452d981be5ae2c26e1c Mon Sep 17 00:00:00 2001
From: Haitam Laarabi <10712736+htmlrb@users.noreply.github.com>
Date: Fri, 11 Mar 2022 13:55:01 -0800
Subject: [PATCH 06/58] trying a fix

---
 .../agents/household/HouseholdActor.scala     | 13 +++-
 .../household/HouseholdFleetManager.scala     | 59 +++++++++----------
 2 files changed, 39 insertions(+), 33 deletions(-)

diff --git a/src/main/scala/beam/agentsim/agents/household/HouseholdActor.scala b/src/main/scala/beam/agentsim/agents/household/HouseholdActor.scala
index 1c25958ce19..75d1cdc8cf1 100755
--- a/src/main/scala/beam/agentsim/agents/household/HouseholdActor.scala
+++ b/src/main/scala/beam/agentsim/agents/household/HouseholdActor.scala
@@ -539,7 +539,6 @@ object HouseholdActor {
 
   class EmergencyHouseholdVehicleGenerator(
     household: Household,
-    homeCoordFromPlans: Coord,
     beamScenario: BeamScenario,
     vehiclesAdjustment: VehiclesAdjustment,
     defaultCategory: VehicleCategory
@@ -550,7 +549,13 @@ object HouseholdActor {
     private val generateEmergencyHousehold =
       beamScenario.beamConfig.beam.agentsim.agents.vehicles.generateEmergencyHouseholdVehicleWhenPlansRequireIt
 
-    def createVehicle(personId: Id[Person], vehicleIndex: Int, category: VehicleCategory): Option[BeamVehicle] = {
+    def createVehicle(
+      personId: Id[Person],
+      vehicleIndex: Int,
+      category: VehicleCategory,
+      whenWhere: SpaceTime,
+      manager: ActorRef
+    ): Option[BeamVehicle] = {
       val vehicleTypeMaybe =
         if (generateEmergencyHousehold && defaultCategory == category) {
           category match {
@@ -562,7 +567,7 @@ object HouseholdActor {
                   household.getIncome.getIncome,
                   household.getMemberIds.size(),
                   householdPopulation = null,
-                  homeCoordFromPlans,
+                  whenWhere.loc,
                   realDistribution
                 )
                 .headOption
@@ -599,6 +604,8 @@ object HouseholdActor {
         vehicle.initializeFuelLevelsFromUniformDistribution(
           beamScenario.beamConfig.beam.agentsim.agents.vehicles.meanPrivateVehicleStartingSOC
         )
+        vehicle.setManager(Some(manager))
+        vehicle.spaceTime = whenWhere
         vehicle
       }
     }
diff --git a/src/main/scala/beam/agentsim/agents/household/HouseholdFleetManager.scala b/src/main/scala/beam/agentsim/agents/household/HouseholdFleetManager.scala
index ebd7062290b..1ada1b28bbc 100644
--- a/src/main/scala/beam/agentsim/agents/household/HouseholdFleetManager.scala
+++ b/src/main/scala/beam/agentsim/agents/household/HouseholdFleetManager.scala
@@ -10,7 +10,7 @@ import beam.agentsim.agents.InitializeTrigger
 import beam.agentsim.agents.household.HouseholdActor._
 import beam.agentsim.agents.household.HouseholdFleetManager.ResolvedParkingResponses
 import beam.agentsim.agents.modalbehaviors.DrivesVehicle.ActualVehicle
-import beam.agentsim.agents.vehicles.BeamVehicle
+import beam.agentsim.agents.vehicles.{BeamVehicle, VehicleCategory}
 import beam.agentsim.events.SpaceTime
 import beam.agentsim.infrastructure.{ParkingInquiry, ParkingInquiryResponse}
 import beam.agentsim.scheduler.BeamAgentScheduler.CompletionNotice
@@ -39,6 +39,7 @@ class HouseholdFleetManager(
   private val vehiclesInternal: collection.mutable.Map[Id[BeamVehicle], BeamVehicle] =
     collection.mutable.Map(vehicles.toSeq: _*)
 
+  private val freightVehicleCategories: Array[VehicleCategory.VehicleCategory] = Array(VehicleCategory.HeavyDutyTruck, VehicleCategory.LightDutyTruck)
   private var availableVehicles: List[BeamVehicle] = Nil
   var triggerSender: Option[ActorRef] = None
 
@@ -109,40 +110,38 @@ class HouseholdFleetManager(
         for {
           neededVehicleCategory              <- requireVehicleCategoryAvailable
           emergencyHouseholdVehicleGenerator <- maybeEmergencyHouseholdVehicleGenerator
-          vehicle                            <- emergencyHouseholdVehicleGenerator.createVehicle(personId, nextVehicleIndex, neededVehicleCategory)
+          vehicle <- emergencyHouseholdVehicleGenerator.createVehicle(
+            personId,
+            nextVehicleIndex,
+            neededVehicleCategory,
+            whenWhere,
+            self
+          )
         } yield {
-          val vehicleCreatedOutOfThinAir: Boolean = if (availableVehicles.isEmpty) {
+          if (availableVehicles.isEmpty) {
+            // Create a vehicle out of thin air
+            nextVehicleIndex += 1
+            val mobilityRequester = sender()
+            vehiclesInternal(vehicle.id) = vehicle
+
+            // Pipe my car through the parking manager
+            // and complete initialization only when I got them all.
+            val responseFuture = parkingManager ? ParkingInquiry.init(
+              whenWhere,
+              "wherever",
+              triggerId = triggerId
+            )
             logger.warn(
               s"No vehicles available for category ${neededVehicleCategory} available for person ${personId.toString}, creating a new vehicle with id ${vehicle.id.toString}"
             )
-            emergencyHouseholdVehicleGenerator.createVehicle(personId, nextVehicleIndex, neededVehicleCategory) match {
-              case Some(vehicle) =>
-                nextVehicleIndex += 1
-                vehicle.setManager(Some(self))
-                vehicle.spaceTime = whenWhere
-                val mobilityRequester = sender()
-                vehiclesInternal(vehicle.id) = vehicle
-
-                // Pipe my car through the parking manager
-                // and complete initialization only when I got them all.
-                val responseFuture = parkingManager ? ParkingInquiry.init(
-                  whenWhere,
-                  "wherever",
-                  triggerId = triggerId
-                )
-                responseFuture.collect { case ParkingInquiryResponse(stall, _, otherTriggerId) =>
-                  vehicle.useParkingStall(stall)
-                  logger.debug("Vehicle {} is now taken, which was just created", vehicle.id)
-                  vehicle.becomeDriver(mobilityRequester)
-                  MobilityStatusResponse(Vector(ActualVehicle(vehicle)), otherTriggerId)
-                } pipeTo mobilityRequester
-                true
-              case _ =>
-                false
-            }
-          } else false
 
-          if (!vehicleCreatedOutOfThinAir) {
+            responseFuture.collect { case ParkingInquiryResponse(stall, _, otherTriggerId) =>
+              vehicle.useParkingStall(stall)
+              logger.debug("Vehicle {} is now taken, which was just created", vehicle.id)
+              vehicle.becomeDriver(mobilityRequester)
+              MobilityStatusResponse(Vector(ActualVehicle(vehicle)), otherTriggerId)
+            } pipeTo mobilityRequester
+          } else {
             availableVehicles = availableVehicles match {
               case firstVehicle :: rest =>
                 logger.debug("Vehicle {} is now taken", firstVehicle.id)

From 1c511b83387d6587a665c4863235ad06e51fe6c2 Mon Sep 17 00:00:00 2001
From: Haitam Laarabi <10712736+htmlrb@users.noreply.github.com>
Date: Fri, 11 Mar 2022 13:59:23 -0800
Subject: [PATCH 07/58] logging

---
 .../beam/agentsim/agents/household/HouseholdFleetManager.scala   | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/main/scala/beam/agentsim/agents/household/HouseholdFleetManager.scala b/src/main/scala/beam/agentsim/agents/household/HouseholdFleetManager.scala
index 1ada1b28bbc..2e8d82ec4a0 100644
--- a/src/main/scala/beam/agentsim/agents/household/HouseholdFleetManager.scala
+++ b/src/main/scala/beam/agentsim/agents/household/HouseholdFleetManager.scala
@@ -155,6 +155,7 @@ class HouseholdFleetManager(
           }
         }
       }.getOrElse {
+        availableVehicles.foreach(vehicle => logger.info(s"person ${personId} has this vehicle available: ${vehicle.id} - ${vehicle.beamVehicleType}"))
         availableVehicles = availableVehicles match {
           case firstVehicle :: rest =>
             logger.debug("Vehicle {} is now taken", firstVehicle.id)

From 7f06837d9faaca5d492124cd287df5f2fe3ffb35 Mon Sep 17 00:00:00 2001
From: Haitam Laarabi <10712736+htmlrb@users.noreply.github.com>
Date: Fri, 11 Mar 2022 14:12:21 -0800
Subject: [PATCH 08/58] more logging

---
 .../beam/agentsim/agents/household/HouseholdFleetManager.scala | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/main/scala/beam/agentsim/agents/household/HouseholdFleetManager.scala b/src/main/scala/beam/agentsim/agents/household/HouseholdFleetManager.scala
index 2e8d82ec4a0..31bba19a727 100644
--- a/src/main/scala/beam/agentsim/agents/household/HouseholdFleetManager.scala
+++ b/src/main/scala/beam/agentsim/agents/household/HouseholdFleetManager.scala
@@ -142,6 +142,7 @@ class HouseholdFleetManager(
               MobilityStatusResponse(Vector(ActualVehicle(vehicle)), otherTriggerId)
             } pipeTo mobilityRequester
           } else {
+            availableVehicles.foreach(vehicle => logger.info(s"1- person ${personId} has this vehicle available: ${vehicle.id} - ${vehicle.beamVehicleType}"))
             availableVehicles = availableVehicles match {
               case firstVehicle :: rest =>
                 logger.debug("Vehicle {} is now taken", firstVehicle.id)
@@ -155,7 +156,7 @@ class HouseholdFleetManager(
           }
         }
       }.getOrElse {
-        availableVehicles.foreach(vehicle => logger.info(s"person ${personId} has this vehicle available: ${vehicle.id} - ${vehicle.beamVehicleType}"))
+        availableVehicles.foreach(vehicle => logger.info(s"2- person ${personId} has this vehicle available: ${vehicle.id} - ${vehicle.beamVehicleType}"))
         availableVehicles = availableVehicles match {
           case firstVehicle :: rest =>
             logger.debug("Vehicle {} is now taken", firstVehicle.id)

From fcee26b4f8a45721ef9f0e234fa6e1a1a3b23b45 Mon Sep 17 00:00:00 2001
From: Haitam Laarabi <10712736+htmlrb@users.noreply.github.com>
Date: Fri, 11 Mar 2022 14:22:44 -0800
Subject: [PATCH 09/58] fixing compilation error

---
 .../agents/household/HouseholdActor.scala        | 16 ++++++----------
 1 file changed, 6 insertions(+), 10 deletions(-)

diff --git a/src/main/scala/beam/agentsim/agents/household/HouseholdActor.scala b/src/main/scala/beam/agentsim/agents/household/HouseholdActor.scala
index 75d1cdc8cf1..7fe4b4afe6e 100755
--- a/src/main/scala/beam/agentsim/agents/household/HouseholdActor.scala
+++ b/src/main/scala/beam/agentsim/agents/household/HouseholdActor.scala
@@ -187,6 +187,7 @@ object HouseholdActor {
     private var cavPassengerSchedules: Map[BeamVehicle, PassengerSchedule] = Map()
     private var personAndActivityToCav: Map[(Id[Person], Activity), BeamVehicle] = Map()
     private var personAndActivityToLegs: Map[(Id[Person], Activity), List[BeamLeg]] = Map()
+    private val vehicleCategories = List(Car, Bike)
 
     private val realDistribution: UniformRealDistribution = new UniformRealDistribution()
     realDistribution.reseedRandomGenerator(beamScenario.beamConfig.matsim.modules.global.randomSeed)
@@ -209,23 +210,18 @@ object HouseholdActor {
 
         //We should create a vehicle manager for cars and bikes for all households in case they are generated during the simulation
 
-        val vehiclesByAllCategories = List(Car, Bike)
+        val vehiclesByAllCategories = vehicleCategories
           .map(cat => cat -> vehiclesByCategory.getOrElse(cat, Map[Id[BeamVehicle], BeamVehicle]()))
           .toMap
-        val fleetManagers = vehiclesByAllCategories.map { case (category, vehicleMap) =>
-          val emergencyGenerator = new EmergencyHouseholdVehicleGenerator(
-            household,
-            homeCoordFromPlans,
-            beamScenario,
-            vehiclesAdjustment,
-            category
-          )
+        val fleetManagers = vehiclesByAllCategories.map { case (category, vehiclesInCategory) =>
+          val emergencyGenerator =
+            new EmergencyHouseholdVehicleGenerator(household, beamScenario, vehiclesAdjustment, category)
           val fleetManager =
             context.actorOf(
               Props(
                 new HouseholdFleetManager(
                   parkingManager,
-                  vehicleMap,
+                  vehiclesInCategory,
                   homeCoordFromPlans,
                   Some(emergencyGenerator),
                   beamServices.beamConfig.beam.debug

From 2ca9d1d21fb655d42f742818dad81f5212e4d4eb Mon Sep 17 00:00:00 2001
From: Haitam Laarabi <10712736+htmlrb@users.noreply.github.com>
Date: Fri, 11 Mar 2022 19:17:05 -0800
Subject: [PATCH 10/58] testing a fix

---
 .../beam/agentsim/agents/household/HouseholdActor.scala   | 8 ++++----
 test/input/beamville/freight/freight-carriers.csv         | 4 ++--
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/src/main/scala/beam/agentsim/agents/household/HouseholdActor.scala b/src/main/scala/beam/agentsim/agents/household/HouseholdActor.scala
index 7fe4b4afe6e..7b6f9c7e541 100755
--- a/src/main/scala/beam/agentsim/agents/household/HouseholdActor.scala
+++ b/src/main/scala/beam/agentsim/agents/household/HouseholdActor.scala
@@ -187,7 +187,7 @@ object HouseholdActor {
     private var cavPassengerSchedules: Map[BeamVehicle, PassengerSchedule] = Map()
     private var personAndActivityToCav: Map[(Id[Person], Activity), BeamVehicle] = Map()
     private var personAndActivityToLegs: Map[(Id[Person], Activity), List[BeamLeg]] = Map()
-    private val vehicleCategories = List(Car, Bike)
+    private val basicVehicleCategories = List(Car, Bike)
 
     private val realDistribution: UniformRealDistribution = new UniformRealDistribution()
     realDistribution.reseedRandomGenerator(beamScenario.beamConfig.matsim.modules.global.randomSeed)
@@ -210,9 +210,9 @@ object HouseholdActor {
 
         //We should create a vehicle manager for cars and bikes for all households in case they are generated during the simulation
 
-        val vehiclesByAllCategories = vehicleCategories
-          .map(cat => cat -> vehiclesByCategory.getOrElse(cat, Map[Id[BeamVehicle], BeamVehicle]()))
-          .toMap
+        val vehiclesByAllCategories = basicVehicleCategories
+          .map(cat => cat -> Map[Id[BeamVehicle], BeamVehicle]())
+          .toMap ++ vehiclesByCategory
         val fleetManagers = vehiclesByAllCategories.map { case (category, vehiclesInCategory) =>
           val emergencyGenerator =
             new EmergencyHouseholdVehicleGenerator(household, beamScenario, vehiclesAdjustment, category)
diff --git a/test/input/beamville/freight/freight-carriers.csv b/test/input/beamville/freight/freight-carriers.csv
index 2c0cb915cf6..be2a76367ce 100644
--- a/test/input/beamville/freight/freight-carriers.csv
+++ b/test/input/beamville/freight/freight-carriers.csv
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f63f1808beb4ca83adea70df13c2e758d4c6f1bc5842d71b6d5bf67c7d0dd458
-size 171
+oid sha256:f86d7ff97e39083e319b32da3f949e9df267c19d7e531eec9ef1fcebe1ee606f
+size 211

From 235e9d84d0279675e3bc85af506deb75976bacb6 Mon Sep 17 00:00:00 2001
From: Haitam Laarabi <10712736+htmlrb@users.noreply.github.com>
Date: Sat, 12 Mar 2022 20:07:57 -0800
Subject: [PATCH 11/58] making households equivalent to carriers

---
 .../agents/freight/FreightEntities.scala      |  6 +--
 .../agents/freight/FreightReplanner.scala     | 24 ++++------
 .../agents/freight/input/FreightReader.scala  | 34 +++++++-------
 .../freight/input/GenericFreightReader.scala  | 42 ++++++++---------
 .../household/HouseholdFleetManager.scala     | 17 +++++--
 .../agents/modalbehaviors/ChoosesMode.scala   |  2 +-
 .../agents/modalbehaviors/DrivesVehicle.scala |  4 +-
 .../agents/parking/ChoosesParking.scala       |  4 +-
 src/main/scala/beam/sim/BeamHelper.scala      | 46 +++++++++----------
 .../agents/freight/FreightReplannerSpec.scala |  2 +-
 .../input/GenericFreightReaderSpec.scala      |  6 +--
 .../beamville/freight/freight-carriers.csv    |  4 +-
 12 files changed, 93 insertions(+), 98 deletions(-)

diff --git a/src/main/scala/beam/agentsim/agents/freight/FreightEntities.scala b/src/main/scala/beam/agentsim/agents/freight/FreightEntities.scala
index 050460bbdd2..779e6ea5507 100644
--- a/src/main/scala/beam/agentsim/agents/freight/FreightEntities.scala
+++ b/src/main/scala/beam/agentsim/agents/freight/FreightEntities.scala
@@ -25,8 +25,6 @@ object FreightRequestType extends Enum[FreightRequestType] {
 case class FreightTour(
   tourId: Id[FreightTour],
   departureTimeInSec: Int,
-  warehouseLocationTaz: Option[Id[TAZ]],
-  warehouseLocationUTM: Coord,
   maxTourDurationInSec: Int
 )
 
@@ -51,5 +49,7 @@ case class FreightCarrier(
   tourMap: Map[Id[BeamVehicle], IndexedSeq[FreightTour]],
   payloadPlans: Map[Id[PayloadPlan], PayloadPlan],
   fleet: Map[Id[BeamVehicle], BeamVehicle],
-  plansPerTour: Map[Id[FreightTour], IndexedSeq[PayloadPlan]]
+  plansPerTour: Map[Id[FreightTour], IndexedSeq[PayloadPlan]],
+  warehouseLocationTaz: Option[Id[TAZ]],
+  warehouseLocationUTM: Coord
 )
diff --git a/src/main/scala/beam/agentsim/agents/freight/FreightReplanner.scala b/src/main/scala/beam/agentsim/agents/freight/FreightReplanner.scala
index d97375be9d1..a4c0a1dd064 100644
--- a/src/main/scala/beam/agentsim/agents/freight/FreightReplanner.scala
+++ b/src/main/scala/beam/agentsim/agents/freight/FreightReplanner.scala
@@ -63,7 +63,7 @@ class FreightReplanner(
   ): Iterable[Plan] = {
     routes.groupBy(_.vehicle.id).map { case (vehicleIdStr, routes) =>
       val vehicleId = Id.createVehicleId(vehicleIdStr)
-      val person = population.get(freightReader.createPersonId(vehicleId))
+      val person = population.get(freightReader.createPersonId(freightCarrier.carrierId, vehicleId))
       val toursAndPlans = routes.zipWithIndex.map { case (route, i) =>
         convertToFreightTourWithPayloadPlans(
           s"freight-tour-${route.vehicle.id}-$i".createId,
@@ -73,7 +73,7 @@ class FreightReplanner(
       }
       val tours = toursAndPlans.map(_._1)
       val plansPerTour = toursAndPlans.map { case (tour, plans) => tour.tourId -> plans }.toMap
-      freightReader.createPersonPlan(tours, plansPerTour, person)
+      freightReader.createPersonPlan(freightCarrier, tours, plansPerTour, person)
     }
   }
 
@@ -82,13 +82,7 @@ class FreightReplanner(
     route: Route,
     payloadPlans: Map[Id[PayloadPlan], PayloadPlan]
   ): (FreightTour, IndexedSeq[PayloadPlan]) = {
-    val tour = FreightTour(
-      tourId,
-      route.startTime,
-      None,
-      route.startLocation,
-      route.duration * 2
-    )
+    val tour = FreightTour(tourId, route.startTime, route.duration * 2)
 
     val plans = route.activities.zipWithIndex.map { case (activity, i) =>
       val requestType: FreightRequestType = activity.service match {
@@ -126,8 +120,8 @@ class FreightReplanner(
   private implicit def toLocation(coord: Coord): Location = Location(coord.getX, coord.getY)
   private implicit def toCoord(location: Location): Coord = new Coord(location.x, location.y)
 
-  private def getVehicleHouseholdLocation(vehicle: BeamVehicle): Location = {
-    val householdIdStr = freightReader.createHouseholdId(vehicle.id).toString
+  private def getVehicleHouseholdLocation(carrierId: Id[FreightCarrier]): Location = {
+    val householdIdStr = freightReader.createHouseholdId(carrierId).toString
     val x = beamServices.matsimServices.getScenario.getHouseholds.getHouseholdAttributes
       .getAttribute(householdIdStr, "homecoordx")
       .asInstanceOf[Double]
@@ -178,10 +172,10 @@ class FreightReplanner(
       }
     }
 
-    def toJspritVehicle(beamVehicle: BeamVehicle, departureTime: Int) = {
+    def toJspritVehicle(carrierId: Id[FreightCarrier], beamVehicle: BeamVehicle, departureTime: Int) = {
       Vehicle(
         beamVehicle.id.toString,
-        getVehicleHouseholdLocation(beamVehicle),
+        getVehicleHouseholdLocation(carrierId),
         beamVehicle.beamVehicleType.payloadCapacityInKg.get,
         departureTime
       )
@@ -198,7 +192,7 @@ class FreightReplanner(
         freightCarrier.fleet.values
           .map(beamVehicle => {
             val departure = randomTimeAround(departureTime)
-            toJspritVehicle(beamVehicle, departure)
+            toJspritVehicle(freightCarrier.carrierId, beamVehicle, departure)
           })
           .toIndexedSeq
       val services = freightCarrier.payloadPlans.values.map(toService).toIndexedSeq
@@ -219,7 +213,7 @@ class FreightReplanner(
         beamVehicle = freightCarrier.fleet(vehicleId)
         tour <- tours
         services = freightCarrier.plansPerTour(tour.tourId).map(toService)
-        vehicles = IndexedSeq(toJspritVehicle(beamVehicle, tour.departureTimeInSec))
+        vehicles = IndexedSeq(toJspritVehicle(freightCarrier.carrierId, beamVehicle, tour.departureTimeInSec))
       } yield JspritWrapper.solve(Problem(vehicles, services, Some(calculateCost)))
 
       Monoid.combineAll(tourSolutions)
diff --git a/src/main/scala/beam/agentsim/agents/freight/input/FreightReader.scala b/src/main/scala/beam/agentsim/agents/freight/input/FreightReader.scala
index 9a88c610ff9..8d35f3740ad 100644
--- a/src/main/scala/beam/agentsim/agents/freight/input/FreightReader.scala
+++ b/src/main/scala/beam/agentsim/agents/freight/input/FreightReader.scala
@@ -16,10 +16,11 @@ import com.conveyal.r5.streets.StreetLayer
 import org.matsim.api.core.v01.population._
 import org.matsim.api.core.v01.{Coord, Id}
 import org.matsim.core.population.PopulationUtils
-import org.matsim.households.{Household, HouseholdsFactory, Income, IncomeImpl}
+import org.matsim.households.{Household, Households, HouseholdsFactory, Income, IncomeImpl}
 import org.matsim.vehicles.Vehicle
 
 import java.util.concurrent.atomic.AtomicReference
+import scala.collection.JavaConverters._
 import scala.util.Random
 
 trait FreightReader {
@@ -30,9 +31,9 @@ trait FreightReader {
 
   def readPayloadPlans(): Map[Id[PayloadPlan], PayloadPlan]
 
-  def createPersonId(vehicleId: Id[BeamVehicle]): Id[Person]
+  def createPersonId(carrierId: Id[FreightCarrier], vehicleId: Id[BeamVehicle]): Id[Person]
 
-  def createHouseholdId(vehicleId: Id[BeamVehicle]): Id[Household]
+  def createHouseholdId(carrierId: Id[FreightCarrier]): Id[Household]
 
   def readFreightCarriers(
     allTours: Map[Id[FreightTour], FreightTour],
@@ -49,13 +50,14 @@ trait FreightReader {
   }
 
   def createPersonPlan(
+    carrier: FreightCarrier,
     tours: IndexedSeq[FreightTour],
     plansPerTour: Map[Id[FreightTour], IndexedSeq[PayloadPlan]],
     person: Person
   ): Plan = {
     val allToursPlanElements = tours.flatMap { tour =>
       val tourInitialActivity =
-        createFreightActivity("Warehouse", tour.warehouseLocationUTM, tour.departureTimeInSec, None)
+        createFreightActivity("Warehouse", carrier.warehouseLocationUTM, tour.departureTimeInSec, None)
       val firstLeg: Leg = createFreightLeg(tour.departureTimeInSec)
 
       val plans: IndexedSeq[PayloadPlan] = plansPerTour.get(tour.tourId) match {
@@ -82,7 +84,7 @@ trait FreightReader {
       elements
     }
 
-    val finalActivity = createFreightActivity("Warehouse", tours.head.warehouseLocationUTM, -1, None)
+    val finalActivity = createFreightActivity("Warehouse", carrier.warehouseLocationUTM, -1, None)
     val allPlanElements: IndexedSeq[PlanElement] = allToursPlanElements :+ finalActivity
 
     val currentPlan = PopulationUtils.createPlan(person)
@@ -96,26 +98,22 @@ trait FreightReader {
 
   def generatePopulation(
     carriers: IndexedSeq[FreightCarrier],
-    personFactory: PopulationFactory,
+    populationFactory: PopulationFactory,
     householdsFactory: HouseholdsFactory
-  ): IndexedSeq[(Household, Plan)] = {
+  ): IndexedSeq[(FreightCarrier, Household, Plan)] = {
     carriers.flatMap { carrier =>
+      val freightHouseholdId = createHouseholdId(carrier.carrierId)
+      val household = householdsFactory.createHousehold(freightHouseholdId)
+      household.setIncome(new IncomeImpl(0, Income.IncomePeriod.year))
       carrier.tourMap.map { case (vehicleId, tours) =>
-        val personId = createPersonId(vehicleId)
-        val person = personFactory.createPerson(personId)
-
-        val currentPlan: Plan = createPersonPlan(tours, carrier.plansPerTour, person)
-
+        val personId = createPersonId(carrier.carrierId, vehicleId)
+        val person = populationFactory.createPerson(personId)
+        val currentPlan: Plan = createPersonPlan(carrier, tours, carrier.plansPerTour, person)
         person.addPlan(currentPlan)
         person.setSelectedPlan(currentPlan)
-
-        val freightHouseholdId = createHouseholdId(vehicleId)
-        val household: Household = householdsFactory.createHousehold(freightHouseholdId)
-        household.setIncome(new IncomeImpl(44444, Income.IncomePeriod.year))
         household.getMemberIds.add(personId)
         household.getVehicleIds.add(vehicleId)
-
-        (household, currentPlan)
+        (carrier, household, currentPlan)
       }
     }
   }
diff --git a/src/main/scala/beam/agentsim/agents/freight/input/GenericFreightReader.scala b/src/main/scala/beam/agentsim/agents/freight/input/GenericFreightReader.scala
index 25cf0f5f509..f75f8cad7dc 100644
--- a/src/main/scala/beam/agentsim/agents/freight/input/GenericFreightReader.scala
+++ b/src/main/scala/beam/agentsim/agents/freight/input/GenericFreightReader.scala
@@ -53,15 +53,7 @@ class GenericFreightReader(
           row.get("departureLocationZone")
         ) match {
           case (departureLocationZoneMaybe, Some(departureLocationUTMOnMap)) =>
-            Some(
-              FreightTour(
-                tourId,
-                departureTimeInSec,
-                departureLocationZoneMaybe,
-                departureLocationUTMOnMap,
-                maxTourDurationInSec
-              )
-            )
+            Some(FreightTour(tourId, departureTimeInSec, maxTourDurationInSec))
           case _ =>
             logger.error(f"Following freight tour row discarded because departure location is not reachable: $row")
             None
@@ -175,6 +167,7 @@ class GenericFreightReader(
 
     def createCarrier(carrierId: Id[FreightCarrier], carrierRows: IndexedSeq[FreightCarrierRow]) = {
       val warehouseLocationUTM: Coord = carrierRows.head.warehouseLocationUTM
+      val warehouseLocationZone: Option[Id[TAZ]] = carrierRows.head.warehouseLocationZone
       val vehicles: scala.IndexedSeq[BeamVehicle] = createCarrierVehicles(carrierId, carrierRows, warehouseLocationUTM)
       val vehicleMap: Map[Id[BeamVehicle], BeamVehicle] = vehicles.map(vehicle => vehicle.id -> vehicle).toMap
 
@@ -183,7 +176,7 @@ class GenericFreightReader(
         .mapValues { rows =>
           rows
             //setting the tour warehouse location to be the carrier warehouse location
-            .map(row => tours(row.tourId).copy(warehouseLocationUTM = warehouseLocationUTM))
+            .map(row => tours(row.tourId))
             .sortBy(_.departureTimeInSec)
         }
 
@@ -194,7 +187,15 @@ class GenericFreightReader(
       val carrierPlanIds: Set[Id[PayloadPlan]] = plansPerTour.values.flatten.map(_.payloadId).toSet
       val payloadMap = plans.filterKeys(carrierPlanIds)
 
-      FreightCarrier(carrierId, tourMap, payloadMap, vehicleMap, plansPerTour)
+      FreightCarrier(
+        carrierId,
+        tourMap,
+        payloadMap,
+        vehicleMap,
+        plansPerTour,
+        warehouseLocationZone,
+        warehouseLocationUTM
+      )
     }
 
     val maybeCarrierRows = GenericCsvReader.readAsSeq[Option[FreightCarrierRow]](config.carriersFilePath) { row =>
@@ -252,21 +253,16 @@ class GenericFreightReader(
   }
 
   @Override
-  def createPersonId(vehicleId: Id[BeamVehicle]): Id[Person] = {
-    if (vehicleId.toString.startsWith(freightIdPrefix)) {
-      Id.createPersonId(s"$vehicleId-agent")
-    } else {
-      Id.createPersonId(s"$freightIdPrefix-$vehicleId-agent")
-    }
+  def createPersonId(carrierId: Id[FreightCarrier], vehicleId: Id[BeamVehicle]): Id[Person] = {
+    val updatedCarrierId = carrierId.toString.replace(freightIdPrefix + "-", "")
+    val updatedVehicleId = vehicleId.toString.replace(freightIdPrefix + "-", "")
+    Id.createPersonId(s"$freightIdPrefix-$updatedCarrierId-$updatedVehicleId-agent")
   }
 
   @Override
-  def createHouseholdId(vehicleId: Id[BeamVehicle]): Id[Household] = {
-    if (vehicleId.toString.startsWith(freightIdPrefix)) {
-      s"$vehicleId-household".createId
-    } else {
-      s"$freightIdPrefix-$vehicleId-household".createId
-    }
+  def createHouseholdId(carrierId: Id[FreightCarrier]): Id[Household] = {
+    val updatedCarrierId = carrierId.toString.replace(freightIdPrefix + "-", "")
+    s"$freightIdPrefix-$updatedCarrierId-household".createId
   }
 
 }
diff --git a/src/main/scala/beam/agentsim/agents/household/HouseholdFleetManager.scala b/src/main/scala/beam/agentsim/agents/household/HouseholdFleetManager.scala
index 31bba19a727..942d8e2d0df 100644
--- a/src/main/scala/beam/agentsim/agents/household/HouseholdFleetManager.scala
+++ b/src/main/scala/beam/agentsim/agents/household/HouseholdFleetManager.scala
@@ -39,7 +39,8 @@ class HouseholdFleetManager(
   private val vehiclesInternal: collection.mutable.Map[Id[BeamVehicle], BeamVehicle] =
     collection.mutable.Map(vehicles.toSeq: _*)
 
-  private val freightVehicleCategories: Array[VehicleCategory.VehicleCategory] = Array(VehicleCategory.HeavyDutyTruck, VehicleCategory.LightDutyTruck)
+  private val freightVehicleCategories: Array[VehicleCategory.VehicleCategory] =
+    Array(VehicleCategory.HeavyDutyTruck, VehicleCategory.LightDutyTruck)
   private var availableVehicles: List[BeamVehicle] = Nil
   var triggerSender: Option[ActorRef] = None
 
@@ -142,7 +143,13 @@ class HouseholdFleetManager(
               MobilityStatusResponse(Vector(ActualVehicle(vehicle)), otherTriggerId)
             } pipeTo mobilityRequester
           } else {
-            availableVehicles.foreach(vehicle => logger.info(s"1- person ${personId} has this vehicle available: ${vehicle.id} - ${vehicle.beamVehicleType}"))
+            availableVehicles
+              .filter(_.beamVehicleType.toString.startsWith("FREIGHT"))
+              .foreach(vehicle =>
+                logger.info(
+                  s"1- person ${personId} has this vehicle available: ${vehicle.id} - ${vehicle.beamVehicleType}"
+                )
+              )
             availableVehicles = availableVehicles match {
               case firstVehicle :: rest =>
                 logger.debug("Vehicle {} is now taken", firstVehicle.id)
@@ -156,7 +163,11 @@ class HouseholdFleetManager(
           }
         }
       }.getOrElse {
-        availableVehicles.foreach(vehicle => logger.info(s"2- person ${personId} has this vehicle available: ${vehicle.id} - ${vehicle.beamVehicleType}"))
+        availableVehicles
+          .filter(_.beamVehicleType.toString.startsWith("FREIGHT"))
+          .foreach(vehicle =>
+            logger.info(s"2- person ${personId} has this vehicle available: ${vehicle.id} - ${vehicle.beamVehicleType}")
+          )
         availableVehicles = availableVehicles match {
           case firstVehicle :: rest =>
             logger.debug("Vehicle {} is now taken", firstVehicle.id)
diff --git a/src/main/scala/beam/agentsim/agents/modalbehaviors/ChoosesMode.scala b/src/main/scala/beam/agentsim/agents/modalbehaviors/ChoosesMode.scala
index d715f535a65..e80f0d0fa62 100755
--- a/src/main/scala/beam/agentsim/agents/modalbehaviors/ChoosesMode.scala
+++ b/src/main/scala/beam/agentsim/agents/modalbehaviors/ChoosesMode.scala
@@ -269,7 +269,7 @@ trait ChoosesMode {
         implicit val executionContext: ExecutionContext = context.system.dispatcher
         plansModeOption match {
           case Some(CAR | DRIVE_TRANSIT) =>
-            val category = if(this.id.toString.toLowerCase.startsWith("FREIGHT")) {
+            val category = if (this.id.toString.toLowerCase.startsWith("FREIGHT")) {
               VehicleCategory.HeavyDutyTruck
             } else VehicleCategory.Car
             requestAvailableVehicles(
diff --git a/src/main/scala/beam/agentsim/agents/modalbehaviors/DrivesVehicle.scala b/src/main/scala/beam/agentsim/agents/modalbehaviors/DrivesVehicle.scala
index 6ed9d63cb33..3335c5251aa 100755
--- a/src/main/scala/beam/agentsim/agents/modalbehaviors/DrivesVehicle.scala
+++ b/src/main/scala/beam/agentsim/agents/modalbehaviors/DrivesVehicle.scala
@@ -3,7 +3,6 @@ package beam.agentsim.agents.modalbehaviors
 import akka.actor.FSM.Failure
 import akka.actor.{ActorRef, Stash}
 import beam.agentsim.Resource.{NotifyVehicleIdle, ReleaseParkingStall}
-import beam.agentsim.agents.{BeamAgent, PersonAgent}
 import beam.agentsim.agents.PersonAgent._
 import beam.agentsim.agents.modalbehaviors.DrivesVehicle._
 import beam.agentsim.agents.parking.ChoosesParking.{handleUseParkingSpot, ConnectingToChargingPoint}
@@ -12,6 +11,7 @@ import beam.agentsim.agents.vehicles.AccessErrorCodes.VehicleFullError
 import beam.agentsim.agents.vehicles.BeamVehicle.{BeamVehicleState, FuelConsumed}
 import beam.agentsim.agents.vehicles.VehicleProtocol._
 import beam.agentsim.agents.vehicles._
+import beam.agentsim.agents.{BeamAgent, PersonAgent}
 import beam.agentsim.events.RefuelSessionEvent.NotApplicable
 import beam.agentsim.events._
 import beam.agentsim.infrastructure.ChargingNetworkManager._
@@ -22,7 +22,7 @@ import beam.agentsim.scheduler.Trigger.TriggerWithId
 import beam.agentsim.scheduler.{HasTriggerId, Trigger}
 import beam.router.Modes.BeamMode
 import beam.router.Modes.BeamMode.{HOV2_TELEPORTATION, HOV3_TELEPORTATION, WALK}
-import beam.router.model.{BeamLeg, BeamPath, EmbodiedBeamLeg}
+import beam.router.model.{BeamLeg, BeamPath}
 import beam.router.osm.TollCalculator
 import beam.router.skim.event.TransitCrowdingSkimmerEvent
 import beam.sim.common.GeoUtils
diff --git a/src/main/scala/beam/agentsim/agents/parking/ChoosesParking.scala b/src/main/scala/beam/agentsim/agents/parking/ChoosesParking.scala
index 77a1b5e072e..b043b848de8 100755
--- a/src/main/scala/beam/agentsim/agents/parking/ChoosesParking.scala
+++ b/src/main/scala/beam/agentsim/agents/parking/ChoosesParking.scala
@@ -11,12 +11,12 @@ import beam.agentsim.agents.freight.input.FreightReader.FREIGHT_REQUEST_TYPE
 import beam.agentsim.agents.modalbehaviors.DrivesVehicle.StartLegTrigger
 import beam.agentsim.agents.parking.ChoosesParking._
 import beam.agentsim.agents.vehicles.VehicleProtocol.StreetVehicle
-import beam.agentsim.agents.vehicles.{BeamVehicle, PassengerSchedule, VehicleCategory, VehicleManager}
+import beam.agentsim.agents.vehicles.{BeamVehicle, PassengerSchedule, VehicleManager}
 import beam.agentsim.events.{LeavingParkingEvent, ParkingEvent, SpaceTime}
 import beam.agentsim.infrastructure.ChargingNetworkManager._
+import beam.agentsim.infrastructure.ParkingInquiry.ParkingSearchMode
 import beam.agentsim.infrastructure.charging.{ChargingPointType, ElectricCurrentType}
 import beam.agentsim.infrastructure.parking.PricingModel
-import beam.agentsim.infrastructure.ParkingInquiry.ParkingSearchMode
 import beam.agentsim.infrastructure.taz.TAZTreeMap
 import beam.agentsim.infrastructure.{ParkingInquiry, ParkingInquiryResponse, ParkingStall}
 import beam.agentsim.scheduler.BeamAgentScheduler.{CompletionNotice, ScheduleTrigger}
diff --git a/src/main/scala/beam/sim/BeamHelper.scala b/src/main/scala/beam/sim/BeamHelper.scala
index e5f0e5cd9d1..b4ca3f00e9a 100755
--- a/src/main/scala/beam/sim/BeamHelper.scala
+++ b/src/main/scala/beam/sim/BeamHelper.scala
@@ -50,7 +50,7 @@ import com.typesafe.config.{ConfigFactory, Config => TypesafeConfig}
 import com.typesafe.scalalogging.LazyLogging
 import kamon.Kamon
 import org.matsim.api.core.v01.network.Link
-import org.matsim.api.core.v01.population.{Activity, Plan, Population}
+import org.matsim.api.core.v01.population.{Activity, Population}
 import org.matsim.api.core.v01.{Id, Scenario}
 import org.matsim.core.api.experimental.events.EventsManager
 import org.matsim.core.config.groups.TravelTimeCalculatorConfigGroup
@@ -61,7 +61,7 @@ import org.matsim.core.events.ParallelEventsManagerImpl
 import org.matsim.core.scenario.{MutableScenario, ScenarioBuilder, ScenarioByInstanceModule, ScenarioUtils}
 import org.matsim.core.trafficmonitoring.TravelTimeCalculator
 import org.matsim.core.utils.collections.QuadTree
-import org.matsim.households.{Household, Households}
+import org.matsim.households.Households
 import org.matsim.utils.objectattributes.AttributeConverter
 import org.matsim.vehicles.Vehicle
 
@@ -916,30 +916,28 @@ trait BeamHelper extends LazyLogging {
     beamScenario.freightCarriers
       .flatMap(_.fleet)
       .foreach { case (id, vehicle) => beamScenario.privateVehicles.put(id, vehicle) }
-
-    val plans: IndexedSeq[(Household, Plan)] = converter.generatePopulation(
-      beamScenario.freightCarriers,
-      population.getFactory,
-      households.getFactory
-    )
-
     val allowedModes = Seq(BeamMode.CAR.value)
-    plans.foreach { case (household, plan) =>
-      households.getHouseholds.put(household.getId, household)
-      population.addPerson(plan.getPerson)
-      AvailableModeUtils.setAvailableModesForPerson_v2(
-        beamScenario,
-        plan.getPerson,
-        household,
-        population,
-        allowedModes
+    converter
+      .generatePopulation(
+        beamScenario.freightCarriers,
+        population.getFactory,
+        households.getFactory
       )
-      val freightVehicle = beamScenario.privateVehicles(household.getVehicleIds.get(0))
-      households.getHouseholdAttributes
-        .putAttribute(household.getId.toString, "homecoordx", freightVehicle.spaceTime.loc.getX)
-      households.getHouseholdAttributes
-        .putAttribute(household.getId.toString, "homecoordy", freightVehicle.spaceTime.loc.getY)
-    }
+      .foreach { case (carrier, household, plan) =>
+        households.getHouseholdAttributes
+          .putAttribute(household.getId.toString, "homecoordx", carrier.warehouseLocationUTM.getX)
+        households.getHouseholdAttributes
+          .putAttribute(household.getId.toString, "homecoordy", carrier.warehouseLocationUTM.getY)
+        households.getHouseholds.put(household.getId, household)
+        population.addPerson(plan.getPerson)
+        AvailableModeUtils.setAvailableModesForPerson_v2(
+          beamScenario,
+          plan.getPerson,
+          household,
+          population,
+          allowedModes
+        )
+      }
   }
 
   def setupBeamWithConfig(
diff --git a/src/test/scala/beam/agentsim/agents/freight/FreightReplannerSpec.scala b/src/test/scala/beam/agentsim/agents/freight/FreightReplannerSpec.scala
index b68fa905a13..eb8a1ea2d2b 100644
--- a/src/test/scala/beam/agentsim/agents/freight/FreightReplannerSpec.scala
+++ b/src/test/scala/beam/agentsim/agents/freight/FreightReplannerSpec.scala
@@ -122,7 +122,7 @@ class FreightReplannerSpec extends AnyWordSpecLike with Matchers with BeamHelper
         beamServices.beamScenario.freightCarriers.find(_.carrierId == "freight-carrier-1".createId[FreightCarrier]).get
       replanner.replan(carrier)
       val person = beamServices.matsimServices.getScenario.getPopulation.getPersons
-        .get(Id.createPersonId("freight-vehicle-1-agent"))
+        .get(Id.createPersonId("freight-carrier-1-vehicle-1-agent"))
       val plan = person.getSelectedPlan
       plan.getPlanElements should have size 9
       plan.getPlanElements.get(0) shouldBe a[Activity]
diff --git a/src/test/scala/beam/agentsim/agents/freight/input/GenericFreightReaderSpec.scala b/src/test/scala/beam/agentsim/agents/freight/input/GenericFreightReaderSpec.scala
index 12042d72cb8..6c57dc2c347 100644
--- a/src/test/scala/beam/agentsim/agents/freight/input/GenericFreightReaderSpec.scala
+++ b/src/test/scala/beam/agentsim/agents/freight/input/GenericFreightReaderSpec.scala
@@ -70,7 +70,6 @@ class GenericFreightReaderSpec extends AnyWordSpecLike with Matchers {
       val tour3 = tours("tour-3".createId)
       tour3.tourId should be("tour-3".createId)
       tour3.departureTimeInSec should be(15000)
-      tour3.warehouseLocationUTM should be(new Coord(170308.4, 2964.6))
       tour3.maxTourDurationInSec should be(36000)
     }
 
@@ -88,7 +87,6 @@ class GenericFreightReaderSpec extends AnyWordSpecLike with Matchers {
       carrier1.tourMap(Id.createVehicleId("freight-vehicle-2")).head should have(
         'tourId ("tour-1".createId[FreightTour]),
         'departureTimeInSec (1000),
-        'warehouseLocationUTM (new Coord(169637.3661199976, 3030.52756066406)),
         'maxTourDurationInSec (36000)
       )
       carrier1.plansPerTour should have size 3
@@ -136,7 +134,7 @@ class GenericFreightReaderSpec extends AnyWordSpecLike with Matchers {
       )
 
       personPlans should have size 3
-      val plan1 = personPlans(Id.createPersonId("freight-vehicle-1-agent"))
+      val plan1 = personPlans(Id.createPersonId("freight-carrier-1-vehicle-1-agent"))
       plan1.getPlanElements should have size 15
       plan1.getPlanElements.get(2).asInstanceOf[Activity].getCoord should be(
         new Coord(169567.3017564815, 836.6518909569604)
@@ -144,7 +142,7 @@ class GenericFreightReaderSpec extends AnyWordSpecLike with Matchers {
       plan1.getPlanElements.get(12).asInstanceOf[Activity].getCoord should be(
         new Coord(169576.80444138843, 3380.0075111142937)
       )
-      val plan4 = personPlans(Id.createPersonId("freight-vehicle-3-agent"))
+      val plan4 = personPlans(Id.createPersonId("freight-carrier-2-vehicle-3-agent"))
       plan4.getPlanElements should have size 5
       plan4.getPlanElements.get(2).asInstanceOf[Activity].getCoord should be(
         new Coord(169900.11498160253, 3510.2356380579545)
diff --git a/test/input/beamville/freight/freight-carriers.csv b/test/input/beamville/freight/freight-carriers.csv
index be2a76367ce..2c0cb915cf6 100644
--- a/test/input/beamville/freight/freight-carriers.csv
+++ b/test/input/beamville/freight/freight-carriers.csv
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f86d7ff97e39083e319b32da3f949e9df267c19d7e531eec9ef1fcebe1ee606f
-size 211
+oid sha256:f63f1808beb4ca83adea70df13c2e758d4c6f1bc5842d71b6d5bf67c7d0dd458
+size 171

From f1b68af3bea106bfa55936f725b8eaadc0137681 Mon Sep 17 00:00:00 2001
From: Haitam Laarabi <10712736+htmlrb@users.noreply.github.com>
Date: Sat, 12 Mar 2022 20:53:47 -0800
Subject: [PATCH 12/58] additional fix to freight

---
 .../agents/freight/input/FreightReader.scala  |  3 +--
 .../agents/household/HouseholdActor.scala     | 26 ++++++++++++-------
 .../household/HouseholdFleetManager.scala     |  4 +--
 .../agents/modalbehaviors/ChoosesMode.scala   |  7 ++---
 4 files changed, 21 insertions(+), 19 deletions(-)

diff --git a/src/main/scala/beam/agentsim/agents/freight/input/FreightReader.scala b/src/main/scala/beam/agentsim/agents/freight/input/FreightReader.scala
index 8d35f3740ad..1c3f25f9686 100644
--- a/src/main/scala/beam/agentsim/agents/freight/input/FreightReader.scala
+++ b/src/main/scala/beam/agentsim/agents/freight/input/FreightReader.scala
@@ -16,11 +16,10 @@ import com.conveyal.r5.streets.StreetLayer
 import org.matsim.api.core.v01.population._
 import org.matsim.api.core.v01.{Coord, Id}
 import org.matsim.core.population.PopulationUtils
-import org.matsim.households.{Household, Households, HouseholdsFactory, Income, IncomeImpl}
+import org.matsim.households.{Household, HouseholdsFactory, Income, IncomeImpl}
 import org.matsim.vehicles.Vehicle
 
 import java.util.concurrent.atomic.AtomicReference
-import scala.collection.JavaConverters._
 import scala.util.Random
 
 trait FreightReader {
diff --git a/src/main/scala/beam/agentsim/agents/household/HouseholdActor.scala b/src/main/scala/beam/agentsim/agents/household/HouseholdActor.scala
index 7b6f9c7e541..228a14c55d2 100755
--- a/src/main/scala/beam/agentsim/agents/household/HouseholdActor.scala
+++ b/src/main/scala/beam/agentsim/agents/household/HouseholdActor.scala
@@ -18,7 +18,7 @@ import beam.agentsim.agents.ridehail.RideHailAgent.{
 }
 import beam.agentsim.agents.ridehail.RideHailManager.RoutingResponses
 import beam.agentsim.agents.vehicles.EnergyEconomyAttributes.Powertrain
-import beam.agentsim.agents.vehicles.VehicleCategory.{Bike, Car, VehicleCategory}
+import beam.agentsim.agents.vehicles.VehicleCategory.{VehicleCategory, _}
 import beam.agentsim.agents.vehicles._
 import beam.agentsim.events.SpaceTime
 import beam.agentsim.infrastructure.{ParkingInquiry, ParkingInquiryResponse}
@@ -182,12 +182,15 @@ object HouseholdActor {
 
     private var members: Map[Id[Person], PersonIdWithActorRef] = Map()
 
+    private val isFreightCarrier: Boolean = household.getId.toString.toLowerCase.contains("freight")
+
     // Data need to execute CAV dispatch
     private val cavPlans: mutable.ListBuffer[CAVSchedule] = mutable.ListBuffer()
     private var cavPassengerSchedules: Map[BeamVehicle, PassengerSchedule] = Map()
     private var personAndActivityToCav: Map[(Id[Person], Activity), BeamVehicle] = Map()
     private var personAndActivityToLegs: Map[(Id[Person], Activity), List[BeamLeg]] = Map()
-    private val basicVehicleCategories = List(Car, Bike)
+    private val householdVehicleCategories = List(Car, Bike)
+    private val freightCarrierVehicleCategories = List(HeavyDutyTruck, LightDutyTruck)
 
     private val realDistribution: UniformRealDistribution = new UniformRealDistribution()
     realDistribution.reseedRandomGenerator(beamScenario.beamConfig.matsim.modules.global.randomSeed)
@@ -197,9 +200,10 @@ object HouseholdActor {
       case TriggerWithId(InitializeTrigger(tick), triggerId) =>
         val homeCoordFromPlans = household.members
           .flatMap(person =>
-            person.getSelectedPlan.getPlanElements.asScala.flatMap {
-              case act: Activity if act.getType == "Home" => Some(act.getCoord)
-              case _                                      => None
+            person.getSelectedPlan.getPlanElements.asScala.headOption.flatMap {
+              case act: Activity if isFreightCarrier && act.getType == "Warehouse" => Some(act.getCoord)
+              case act: Activity if !isFreightCarrier && act.getType == "Home"     => Some(act.getCoord)
+              case _                                                               => None
             }
           )
           .headOption
@@ -209,10 +213,14 @@ object HouseholdActor {
           vehicles.filter(_._2.beamVehicleType.automationLevel <= 3).groupBy(_._2.beamVehicleType.vehicleCategory)
 
         //We should create a vehicle manager for cars and bikes for all households in case they are generated during the simulation
-
-        val vehiclesByAllCategories = basicVehicleCategories
-          .map(cat => cat -> Map[Id[BeamVehicle], BeamVehicle]())
-          .toMap ++ vehiclesByCategory
+        val vehiclesByAllCategories = (if (isFreightCarrier)
+                                         freightCarrierVehicleCategories
+                                           .map(cat => cat -> Map[Id[BeamVehicle], BeamVehicle]())
+                                           .toMap
+                                       else
+                                         householdVehicleCategories
+                                           .map(cat => cat -> Map[Id[BeamVehicle], BeamVehicle]())
+                                           .toMap) ++ vehiclesByCategory
         val fleetManagers = vehiclesByAllCategories.map { case (category, vehiclesInCategory) =>
           val emergencyGenerator =
             new EmergencyHouseholdVehicleGenerator(household, beamScenario, vehiclesAdjustment, category)
diff --git a/src/main/scala/beam/agentsim/agents/household/HouseholdFleetManager.scala b/src/main/scala/beam/agentsim/agents/household/HouseholdFleetManager.scala
index 942d8e2d0df..3b0bdd5fbb3 100644
--- a/src/main/scala/beam/agentsim/agents/household/HouseholdFleetManager.scala
+++ b/src/main/scala/beam/agentsim/agents/household/HouseholdFleetManager.scala
@@ -10,7 +10,7 @@ import beam.agentsim.agents.InitializeTrigger
 import beam.agentsim.agents.household.HouseholdActor._
 import beam.agentsim.agents.household.HouseholdFleetManager.ResolvedParkingResponses
 import beam.agentsim.agents.modalbehaviors.DrivesVehicle.ActualVehicle
-import beam.agentsim.agents.vehicles.{BeamVehicle, VehicleCategory}
+import beam.agentsim.agents.vehicles.BeamVehicle
 import beam.agentsim.events.SpaceTime
 import beam.agentsim.infrastructure.{ParkingInquiry, ParkingInquiryResponse}
 import beam.agentsim.scheduler.BeamAgentScheduler.CompletionNotice
@@ -39,8 +39,6 @@ class HouseholdFleetManager(
   private val vehiclesInternal: collection.mutable.Map[Id[BeamVehicle], BeamVehicle] =
     collection.mutable.Map(vehicles.toSeq: _*)
 
-  private val freightVehicleCategories: Array[VehicleCategory.VehicleCategory] =
-    Array(VehicleCategory.HeavyDutyTruck, VehicleCategory.LightDutyTruck)
   private var availableVehicles: List[BeamVehicle] = Nil
   var triggerSender: Option[ActorRef] = None
 
diff --git a/src/main/scala/beam/agentsim/agents/modalbehaviors/ChoosesMode.scala b/src/main/scala/beam/agentsim/agents/modalbehaviors/ChoosesMode.scala
index e80f0d0fa62..b99c76cfde5 100755
--- a/src/main/scala/beam/agentsim/agents/modalbehaviors/ChoosesMode.scala
+++ b/src/main/scala/beam/agentsim/agents/modalbehaviors/ChoosesMode.scala
@@ -10,8 +10,8 @@ import beam.agentsim.agents.modalbehaviors.ChoosesMode._
 import beam.agentsim.agents.modalbehaviors.DrivesVehicle.{ActualVehicle, Token, VehicleOrToken}
 import beam.agentsim.agents.ridehail.{RideHailInquiry, RideHailRequest, RideHailResponse}
 import beam.agentsim.agents.vehicles.AccessErrorCodes.RideHailNotRequestedError
-import beam.agentsim.agents.vehicles.VehicleCategory.VehicleCategory
 import beam.agentsim.agents.vehicles.EnergyEconomyAttributes.Powertrain
+import beam.agentsim.agents.vehicles.VehicleCategory.VehicleCategory
 import beam.agentsim.agents.vehicles.VehicleProtocol.StreetVehicle
 import beam.agentsim.agents.vehicles._
 import beam.agentsim.events.{ModeChoiceEvent, SpaceTime}
@@ -269,14 +269,11 @@ trait ChoosesMode {
         implicit val executionContext: ExecutionContext = context.system.dispatcher
         plansModeOption match {
           case Some(CAR | DRIVE_TRANSIT) =>
-            val category = if (this.id.toString.toLowerCase.startsWith("FREIGHT")) {
-              VehicleCategory.HeavyDutyTruck
-            } else VehicleCategory.Car
             requestAvailableVehicles(
               vehicleFleets,
               currentLocation,
               _experiencedBeamPlan.activities(currentActivityIndex),
-              Some(category)
+              Some(VehicleCategory.Car)
             ) pipeTo self
           case Some(BIKE | BIKE_TRANSIT) =>
             requestAvailableVehicles(

From 5cff79abd6184fbd1ac1d0a97269da1fca873910 Mon Sep 17 00:00:00 2001
From: Haitam Laarabi <10712736+htmlrb@users.noreply.github.com>
Date: Sat, 12 Mar 2022 23:57:23 -0800
Subject: [PATCH 13/58] testing freight selection

---
 .../agents/household/HouseholdActor.scala     | 125 ++++++++++--------
 .../household/HouseholdFleetManager.scala     |  58 ++------
 2 files changed, 84 insertions(+), 99 deletions(-)

diff --git a/src/main/scala/beam/agentsim/agents/household/HouseholdActor.scala b/src/main/scala/beam/agentsim/agents/household/HouseholdActor.scala
index 228a14c55d2..f02630ee473 100755
--- a/src/main/scala/beam/agentsim/agents/household/HouseholdActor.scala
+++ b/src/main/scala/beam/agentsim/agents/household/HouseholdActor.scala
@@ -223,7 +223,13 @@ object HouseholdActor {
                                            .toMap) ++ vehiclesByCategory
         val fleetManagers = vehiclesByAllCategories.map { case (category, vehiclesInCategory) =>
           val emergencyGenerator =
-            new EmergencyHouseholdVehicleGenerator(household, beamScenario, vehiclesAdjustment, category)
+            new EmergencyHouseholdVehicleGenerator(
+              household,
+              beamScenario,
+              vehiclesAdjustment,
+              category,
+              isFreightCarrier
+            )
           val fleetManager =
             context.actorOf(
               Props(
@@ -545,7 +551,8 @@ object HouseholdActor {
     household: Household,
     beamScenario: BeamScenario,
     vehiclesAdjustment: VehiclesAdjustment,
-    defaultCategory: VehicleCategory
+    defaultCategory: VehicleCategory,
+    isFreightCarrier: Boolean
   ) extends LazyLogging {
     private val realDistribution: UniformRealDistribution = new UniformRealDistribution()
     realDistribution.reseedRandomGenerator(beamScenario.beamConfig.matsim.modules.global.randomSeed)
@@ -558,59 +565,67 @@ object HouseholdActor {
       vehicleIndex: Int,
       category: VehicleCategory,
       whenWhere: SpaceTime,
-      manager: ActorRef
-    ): Option[BeamVehicle] = {
-      val vehicleTypeMaybe =
-        if (generateEmergencyHousehold && defaultCategory == category) {
-          category match {
-            case VehicleCategory.Car =>
-              vehiclesAdjustment
-                .sampleVehicleTypesForHousehold(
-                  1,
-                  VehicleCategory.Car,
-                  household.getIncome.getIncome,
-                  household.getMemberIds.size(),
-                  householdPopulation = null,
-                  whenWhere.loc,
-                  realDistribution
-                )
-                .headOption
-                .orElse {
-                  beamScenario.vehicleTypes.get(
-                    Id.create(
-                      beamScenario.beamConfig.beam.agentsim.agents.vehicles.dummySharedCar.vehicleTypeId,
-                      classOf[BeamVehicleType]
-                    )
-                  )
-                }
-            case VehicleCategory.Bike =>
-              beamScenario.vehicleTypes
-                .get(
-                  Id.create(
-                    beamScenario.beamConfig.beam.agentsim.agents.vehicles.dummySharedBike.vehicleTypeId,
-                    classOf[BeamVehicleType]
-                  )
-                )
-            case _ =>
-              logger.warn(
-                s"Person $personId is requiring a vehicle that belongs to category $category that is neither Car nor Bike"
-              )
-              None
-          }
-        } else None
-      vehicleTypeMaybe map { vehicleType =>
-        val vehicle = new BeamVehicle(
-          Id.createVehicleId(personId.toString + "-emergency-" + vehicleIndex),
-          new Powertrain(vehicleType.primaryFuelConsumptionInJoulePerMeter),
-          vehicleType
-        )
-        beamScenario.privateVehicles.put(vehicle.id, vehicle)
-        vehicle.initializeFuelLevelsFromUniformDistribution(
-          beamScenario.beamConfig.beam.agentsim.agents.vehicles.meanPrivateVehicleStartingSOC
-        )
-        vehicle.setManager(Some(manager))
-        vehicle.spaceTime = whenWhere
-        vehicle
+      manager: ActorRef,
+      availableVehicles: List[BeamVehicle]
+    ): Option[(BeamVehicle, Int)] = {
+      if (availableVehicles.nonEmpty) {
+        if (isFreightCarrier) {
+          availableVehicles.foreach(vehicle =>
+            logger.info(s"person ${personId} has this vehicle available: ${vehicle.id} - ${vehicle.beamVehicleType}")
+          )
+          availableVehicles.zipWithIndex.find(pair => personId.toString.contains(pair._1.id.toString))
+        } else availableVehicles.zipWithIndex.headOption
+      } else {
+        (if (generateEmergencyHousehold && defaultCategory == category) {
+           category match {
+             case VehicleCategory.Car =>
+               vehiclesAdjustment
+                 .sampleVehicleTypesForHousehold(
+                   1,
+                   VehicleCategory.Car,
+                   household.getIncome.getIncome,
+                   household.getMemberIds.size(),
+                   householdPopulation = null,
+                   whenWhere.loc,
+                   realDistribution
+                 )
+                 .headOption
+                 .orElse {
+                   beamScenario.vehicleTypes.get(
+                     Id.create(
+                       beamScenario.beamConfig.beam.agentsim.agents.vehicles.dummySharedCar.vehicleTypeId,
+                       classOf[BeamVehicleType]
+                     )
+                   )
+                 }
+             case VehicleCategory.Bike =>
+               beamScenario.vehicleTypes
+                 .get(
+                   Id.create(
+                     beamScenario.beamConfig.beam.agentsim.agents.vehicles.dummySharedBike.vehicleTypeId,
+                     classOf[BeamVehicleType]
+                   )
+                 )
+             case _ =>
+               logger.warn(
+                 s"Person $personId is requiring a vehicle that belongs to category $category that is neither Car nor Bike"
+               )
+               None
+           }
+         } else None) map { vehicleType =>
+          val vehicle = new BeamVehicle(
+            Id.createVehicleId(personId.toString + "-emergency-" + vehicleIndex),
+            new Powertrain(vehicleType.primaryFuelConsumptionInJoulePerMeter),
+            vehicleType
+          )
+          beamScenario.privateVehicles.put(vehicle.id, vehicle)
+          vehicle.initializeFuelLevelsFromUniformDistribution(
+            beamScenario.beamConfig.beam.agentsim.agents.vehicles.meanPrivateVehicleStartingSOC
+          )
+          vehicle.setManager(Some(manager))
+          vehicle.spaceTime = whenWhere
+          (vehicle, -1)
+        }
       }
     }
   }
diff --git a/src/main/scala/beam/agentsim/agents/household/HouseholdFleetManager.scala b/src/main/scala/beam/agentsim/agents/household/HouseholdFleetManager.scala
index 3b0bdd5fbb3..8a37ef08bba 100644
--- a/src/main/scala/beam/agentsim/agents/household/HouseholdFleetManager.scala
+++ b/src/main/scala/beam/agentsim/agents/household/HouseholdFleetManager.scala
@@ -109,31 +109,27 @@ class HouseholdFleetManager(
         for {
           neededVehicleCategory              <- requireVehicleCategoryAvailable
           emergencyHouseholdVehicleGenerator <- maybeEmergencyHouseholdVehicleGenerator
-          vehicle <- emergencyHouseholdVehicleGenerator.createVehicle(
+          (vehicle, index) <- emergencyHouseholdVehicleGenerator.createVehicle(
             personId,
             nextVehicleIndex,
             neededVehicleCategory,
             whenWhere,
-            self
+            self,
+            availableVehicles
           )
         } yield {
-          if (availableVehicles.isEmpty) {
+          if (index < 0) {
             // Create a vehicle out of thin air
             nextVehicleIndex += 1
             val mobilityRequester = sender()
             vehiclesInternal(vehicle.id) = vehicle
-
             // Pipe my car through the parking manager
             // and complete initialization only when I got them all.
-            val responseFuture = parkingManager ? ParkingInquiry.init(
-              whenWhere,
-              "wherever",
-              triggerId = triggerId
-            )
+            val responseFuture = parkingManager ? ParkingInquiry.init(whenWhere, "wherever", triggerId = triggerId)
             logger.warn(
-              s"No vehicles available for category ${neededVehicleCategory} available for person ${personId.toString}, creating a new vehicle with id ${vehicle.id.toString}"
+              s"No vehicles available for category ${neededVehicleCategory} available " +
+              s"for person ${personId.toString}, creating a new vehicle with id ${vehicle.id.toString}"
             )
-
             responseFuture.collect { case ParkingInquiryResponse(stall, _, otherTriggerId) =>
               vehicle.useParkingStall(stall)
               logger.debug("Vehicle {} is now taken, which was just created", vehicle.id)
@@ -141,42 +137,16 @@ class HouseholdFleetManager(
               MobilityStatusResponse(Vector(ActualVehicle(vehicle)), otherTriggerId)
             } pipeTo mobilityRequester
           } else {
-            availableVehicles
-              .filter(_.beamVehicleType.toString.startsWith("FREIGHT"))
-              .foreach(vehicle =>
-                logger.info(
-                  s"1- person ${personId} has this vehicle available: ${vehicle.id} - ${vehicle.beamVehicleType}"
-                )
-              )
-            availableVehicles = availableVehicles match {
-              case firstVehicle :: rest =>
-                logger.debug("Vehicle {} is now taken", firstVehicle.id)
-                firstVehicle.becomeDriver(sender)
-                sender() ! MobilityStatusResponse(Vector(ActualVehicle(firstVehicle)), triggerId)
-                rest
-              case _ =>
-                logger.error(s"THE LIST OF VEHICLES SHOULDN'T BE EMPTY")
-                Nil
-            }
+            logger.debug("Vehicle {} is now taken", vehicle.id)
+            vehicle.becomeDriver(sender)
+            sender() ! MobilityStatusResponse(Vector(ActualVehicle(vehicle)), triggerId)
+            availableVehicles = availableVehicles.drop(index)
           }
         }
       }.getOrElse {
-        availableVehicles
-          .filter(_.beamVehicleType.toString.startsWith("FREIGHT"))
-          .foreach(vehicle =>
-            logger.info(s"2- person ${personId} has this vehicle available: ${vehicle.id} - ${vehicle.beamVehicleType}")
-          )
-        availableVehicles = availableVehicles match {
-          case firstVehicle :: rest =>
-            logger.debug("Vehicle {} is now taken", firstVehicle.id)
-            firstVehicle.becomeDriver(sender)
-            sender() ! MobilityStatusResponse(Vector(ActualVehicle(firstVehicle)), triggerId)
-            rest
-          case Nil =>
-            logger.debug(s"Not returning vehicle because no default is defined")
-            sender() ! MobilityStatusResponse(Vector(), triggerId)
-            Nil
-        }
+        logger.debug(s"Not returning vehicle because no default is defined")
+        sender() ! MobilityStatusResponse(Vector(), triggerId)
+        Nil
       }
 
     case Finish =>

From b3e0b54d8aa949b1e931fa26da51be6893cbf770 Mon Sep 17 00:00:00 2001
From: Haitam Laarabi <10712736+htmlrb@users.noreply.github.com>
Date: Sun, 13 Mar 2022 19:05:37 -0700
Subject: [PATCH 14/58] new fix

---
 .../agents/freight/input/FreightReader.scala  |   4 +-
 .../freight/input/GenericFreightReader.scala  |   2 +-
 .../agents/household/HouseholdActor.scala     | 173 +++++++++---------
 .../household/HouseholdFleetManager.scala     | 104 +++++++----
 src/main/scala/beam/sim/BeamHelper.scala      |   3 +-
 5 files changed, 156 insertions(+), 130 deletions(-)

diff --git a/src/main/scala/beam/agentsim/agents/freight/input/FreightReader.scala b/src/main/scala/beam/agentsim/agents/freight/input/FreightReader.scala
index 1c3f25f9686..4305c9a2ec2 100644
--- a/src/main/scala/beam/agentsim/agents/freight/input/FreightReader.scala
+++ b/src/main/scala/beam/agentsim/agents/freight/input/FreightReader.scala
@@ -99,7 +99,7 @@ trait FreightReader {
     carriers: IndexedSeq[FreightCarrier],
     populationFactory: PopulationFactory,
     householdsFactory: HouseholdsFactory
-  ): IndexedSeq[(FreightCarrier, Household, Plan)] = {
+  ): IndexedSeq[(FreightCarrier, Household, Plan, Id[Person], Id[BeamVehicle])] = {
     carriers.flatMap { carrier =>
       val freightHouseholdId = createHouseholdId(carrier.carrierId)
       val household = householdsFactory.createHousehold(freightHouseholdId)
@@ -112,7 +112,7 @@ trait FreightReader {
         person.setSelectedPlan(currentPlan)
         household.getMemberIds.add(personId)
         household.getVehicleIds.add(vehicleId)
-        (carrier, household, currentPlan)
+        (carrier, household, currentPlan, personId, vehicleId)
       }
     }
   }
diff --git a/src/main/scala/beam/agentsim/agents/freight/input/GenericFreightReader.scala b/src/main/scala/beam/agentsim/agents/freight/input/GenericFreightReader.scala
index f75f8cad7dc..cf096f6278c 100644
--- a/src/main/scala/beam/agentsim/agents/freight/input/GenericFreightReader.scala
+++ b/src/main/scala/beam/agentsim/agents/freight/input/GenericFreightReader.scala
@@ -52,7 +52,7 @@ class GenericFreightReader(
           row.get("departureLocationY"),
           row.get("departureLocationZone")
         ) match {
-          case (departureLocationZoneMaybe, Some(departureLocationUTMOnMap)) =>
+          case (_, Some(_)) =>
             Some(FreightTour(tourId, departureTimeInSec, maxTourDurationInSec))
           case _ =>
             logger.error(f"Following freight tour row discarded because departure location is not reachable: $row")
diff --git a/src/main/scala/beam/agentsim/agents/household/HouseholdActor.scala b/src/main/scala/beam/agentsim/agents/household/HouseholdActor.scala
index f02630ee473..a254d417868 100755
--- a/src/main/scala/beam/agentsim/agents/household/HouseholdActor.scala
+++ b/src/main/scala/beam/agentsim/agents/household/HouseholdActor.scala
@@ -190,7 +190,8 @@ object HouseholdActor {
     private var personAndActivityToCav: Map[(Id[Person], Activity), BeamVehicle] = Map()
     private var personAndActivityToLegs: Map[(Id[Person], Activity), List[BeamLeg]] = Map()
     private val householdVehicleCategories = List(Car, Bike)
-    private val freightCarrierVehicleCategories = List(HeavyDutyTruck, LightDutyTruck)
+
+    private var whoDrivesThisVehicle: Map[Id[BeamVehicle], Id[Person]] = Map()
 
     private val realDistribution: UniformRealDistribution = new UniformRealDistribution()
     realDistribution.reseedRandomGenerator(beamScenario.beamConfig.matsim.modules.global.randomSeed)
@@ -198,38 +199,38 @@ object HouseholdActor {
     override def loggedReceive: Receive = {
 
       case TriggerWithId(InitializeTrigger(tick), triggerId) =>
+        val vehiclesByCategory =
+          vehicles.filter(_._2.beamVehicleType.automationLevel <= 3).groupBy(_._2.beamVehicleType.vehicleCategory)
+        var vehiclesByAllCategories = vehiclesByCategory
+        if (!isFreightCarrier) {
+          //We should create a vehicle manager for cars and bikes for all households in case they are generated during the simulation
+          vehiclesByAllCategories = householdVehicleCategories
+            .map(cat => cat -> Map[Id[BeamVehicle], BeamVehicle]())
+            .toMap ++ vehiclesByCategory
+        }
         val homeCoordFromPlans = household.members
-          .flatMap(person =>
+          .flatMap { person =>
+            if (isFreightCarrier) {
+              val vehicleIdFromPlans = Id.create(
+                beamServices.matsimServices.getScenario.getPopulation.getPersonAttributes
+                  .getAttribute(person.getId.toString, "vehicle")
+                  .toString,
+                classOf[BeamVehicle]
+              )
+              whoDrivesThisVehicle = whoDrivesThisVehicle + (vehicleIdFromPlans -> person.getId)
+            }
             person.getSelectedPlan.getPlanElements.asScala.headOption.flatMap {
               case act: Activity if isFreightCarrier && act.getType == "Warehouse" => Some(act.getCoord)
               case act: Activity if !isFreightCarrier && act.getType == "Home"     => Some(act.getCoord)
               case _                                                               => None
             }
-          )
+          }
           .headOption
           .getOrElse(fallbackHomeCoord)
 
-        val vehiclesByCategory =
-          vehicles.filter(_._2.beamVehicleType.automationLevel <= 3).groupBy(_._2.beamVehicleType.vehicleCategory)
-
-        //We should create a vehicle manager for cars and bikes for all households in case they are generated during the simulation
-        val vehiclesByAllCategories = (if (isFreightCarrier)
-                                         freightCarrierVehicleCategories
-                                           .map(cat => cat -> Map[Id[BeamVehicle], BeamVehicle]())
-                                           .toMap
-                                       else
-                                         householdVehicleCategories
-                                           .map(cat => cat -> Map[Id[BeamVehicle], BeamVehicle]())
-                                           .toMap) ++ vehiclesByCategory
         val fleetManagers = vehiclesByAllCategories.map { case (category, vehiclesInCategory) =>
           val emergencyGenerator =
-            new EmergencyHouseholdVehicleGenerator(
-              household,
-              beamScenario,
-              vehiclesAdjustment,
-              category,
-              isFreightCarrier
-            )
+            new EmergencyHouseholdVehicleGenerator(household, beamScenario, vehiclesAdjustment, category)
           val fleetManager =
             context.actorOf(
               Props(
@@ -238,6 +239,7 @@ object HouseholdActor {
                   vehiclesInCategory,
                   homeCoordFromPlans,
                   Some(emergencyGenerator),
+                  whoDrivesThisVehicle,
                   beamServices.beamConfig.beam.debug
                 )
               ),
@@ -551,8 +553,7 @@ object HouseholdActor {
     household: Household,
     beamScenario: BeamScenario,
     vehiclesAdjustment: VehiclesAdjustment,
-    defaultCategory: VehicleCategory,
-    isFreightCarrier: Boolean
+    defaultCategory: VehicleCategory
   ) extends LazyLogging {
     private val realDistribution: UniformRealDistribution = new UniformRealDistribution()
     realDistribution.reseedRandomGenerator(beamScenario.beamConfig.matsim.modules.global.randomSeed)
@@ -560,73 +561,69 @@ object HouseholdActor {
     private val generateEmergencyHousehold =
       beamScenario.beamConfig.beam.agentsim.agents.vehicles.generateEmergencyHouseholdVehicleWhenPlansRequireIt
 
-    def createVehicle(
+    def sampleVehicleTypeForEmergencyUse(
       personId: Id[Person],
-      vehicleIndex: Int,
       category: VehicleCategory,
-      whenWhere: SpaceTime,
-      manager: ActorRef,
-      availableVehicles: List[BeamVehicle]
-    ): Option[(BeamVehicle, Int)] = {
-      if (availableVehicles.nonEmpty) {
-        if (isFreightCarrier) {
-          availableVehicles.foreach(vehicle =>
-            logger.info(s"person ${personId} has this vehicle available: ${vehicle.id} - ${vehicle.beamVehicleType}")
-          )
-          availableVehicles.zipWithIndex.find(pair => personId.toString.contains(pair._1.id.toString))
-        } else availableVehicles.zipWithIndex.headOption
-      } else {
-        (if (generateEmergencyHousehold && defaultCategory == category) {
-           category match {
-             case VehicleCategory.Car =>
-               vehiclesAdjustment
-                 .sampleVehicleTypesForHousehold(
-                   1,
-                   VehicleCategory.Car,
-                   household.getIncome.getIncome,
-                   household.getMemberIds.size(),
-                   householdPopulation = null,
-                   whenWhere.loc,
-                   realDistribution
-                 )
-                 .headOption
-                 .orElse {
-                   beamScenario.vehicleTypes.get(
-                     Id.create(
-                       beamScenario.beamConfig.beam.agentsim.agents.vehicles.dummySharedCar.vehicleTypeId,
-                       classOf[BeamVehicleType]
-                     )
-                   )
-                 }
-             case VehicleCategory.Bike =>
-               beamScenario.vehicleTypes
-                 .get(
-                   Id.create(
-                     beamScenario.beamConfig.beam.agentsim.agents.vehicles.dummySharedBike.vehicleTypeId,
-                     classOf[BeamVehicleType]
-                   )
-                 )
-             case _ =>
-               logger.warn(
-                 s"Person $personId is requiring a vehicle that belongs to category $category that is neither Car nor Bike"
-               )
-               None
-           }
-         } else None) map { vehicleType =>
-          val vehicle = new BeamVehicle(
-            Id.createVehicleId(personId.toString + "-emergency-" + vehicleIndex),
-            new Powertrain(vehicleType.primaryFuelConsumptionInJoulePerMeter),
-            vehicleType
-          )
-          beamScenario.privateVehicles.put(vehicle.id, vehicle)
-          vehicle.initializeFuelLevelsFromUniformDistribution(
-            beamScenario.beamConfig.beam.agentsim.agents.vehicles.meanPrivateVehicleStartingSOC
-          )
-          vehicle.setManager(Some(manager))
-          vehicle.spaceTime = whenWhere
-          (vehicle, -1)
+      whenWhere: SpaceTime
+    ): Option[BeamVehicleType] = {
+      if (generateEmergencyHousehold && defaultCategory == category) {
+        category match {
+          case VehicleCategory.Car =>
+            vehiclesAdjustment
+              .sampleVehicleTypesForHousehold(
+                1,
+                VehicleCategory.Car,
+                household.getIncome.getIncome,
+                household.getMemberIds.size(),
+                householdPopulation = null,
+                whenWhere.loc,
+                realDistribution
+              )
+              .headOption
+              .orElse {
+                beamScenario.vehicleTypes.get(
+                  Id.create(
+                    beamScenario.beamConfig.beam.agentsim.agents.vehicles.dummySharedCar.vehicleTypeId,
+                    classOf[BeamVehicleType]
+                  )
+                )
+              }
+          case VehicleCategory.Bike =>
+            beamScenario.vehicleTypes
+              .get(
+                Id.create(
+                  beamScenario.beamConfig.beam.agentsim.agents.vehicles.dummySharedBike.vehicleTypeId,
+                  classOf[BeamVehicleType]
+                )
+              )
+          case _ =>
+            logger.warn(
+              s"Person $personId is requiring a vehicle that belongs to category $category that is neither Car nor Bike"
+            )
+            None
         }
-      }
+      } else None
+    }
+
+    def createAndAddVehicle(
+      vehicleType: BeamVehicleType,
+      personId: Id[Person],
+      vehicleIndex: Int,
+      whenWhere: SpaceTime,
+      manager: ActorRef
+    ): BeamVehicle = {
+      val vehicle = new BeamVehicle(
+        Id.createVehicleId(personId.toString + "-emergency-" + vehicleIndex),
+        new Powertrain(vehicleType.primaryFuelConsumptionInJoulePerMeter),
+        vehicleType
+      )
+      vehicle.initializeFuelLevelsFromUniformDistribution(
+        beamScenario.beamConfig.beam.agentsim.agents.vehicles.meanPrivateVehicleStartingSOC
+      )
+      beamScenario.privateVehicles.put(vehicle.id, vehicle)
+      vehicle.setManager(Some(manager))
+      vehicle.spaceTime = whenWhere
+      vehicle
     }
   }
 }
diff --git a/src/main/scala/beam/agentsim/agents/household/HouseholdFleetManager.scala b/src/main/scala/beam/agentsim/agents/household/HouseholdFleetManager.scala
index 8a37ef08bba..c9c0d44547d 100644
--- a/src/main/scala/beam/agentsim/agents/household/HouseholdFleetManager.scala
+++ b/src/main/scala/beam/agentsim/agents/household/HouseholdFleetManager.scala
@@ -19,6 +19,7 @@ import beam.agentsim.scheduler.Trigger.TriggerWithId
 import beam.sim.config.BeamConfig.Beam.Debug
 import beam.utils.logging.pattern.ask
 import beam.utils.logging.{ExponentialLazyLogging, LoggingMessageActor}
+import org.matsim.api.core.v01.population.Person
 import org.matsim.api.core.v01.{Coord, Id}
 
 import java.util.concurrent.TimeUnit
@@ -29,6 +30,7 @@ class HouseholdFleetManager(
   vehicles: Map[Id[BeamVehicle], BeamVehicle],
   homeCoord: Coord,
   maybeEmergencyHouseholdVehicleGenerator: Option[EmergencyHouseholdVehicleGenerator],
+  whoDrivesThisVehicle: Map[Id[BeamVehicle], Id[Person]], // so far only freight module is using this collection
   implicit val debug: Debug
 ) extends LoggingMessageActor
     with ExponentialLazyLogging {
@@ -104,49 +106,31 @@ class HouseholdFleetManager(
     case GetVehicleTypes(triggerId) =>
       sender() ! VehicleTypesResponse(vehicles.values.map(_.beamVehicleType).toSet, triggerId)
 
-    case MobilityStatusInquiry(personId, whenWhere, _, requireVehicleCategoryAvailable, triggerId) =>
-      {
-        for {
-          neededVehicleCategory              <- requireVehicleCategoryAvailable
-          emergencyHouseholdVehicleGenerator <- maybeEmergencyHouseholdVehicleGenerator
-          (vehicle, index) <- emergencyHouseholdVehicleGenerator.createVehicle(
-            personId,
-            nextVehicleIndex,
-            neededVehicleCategory,
-            whenWhere,
-            self,
-            availableVehicles
-          )
-        } yield {
-          if (index < 0) {
-            // Create a vehicle out of thin air
-            nextVehicleIndex += 1
-            val mobilityRequester = sender()
-            vehiclesInternal(vehicle.id) = vehicle
-            // Pipe my car through the parking manager
-            // and complete initialization only when I got them all.
-            val responseFuture = parkingManager ? ParkingInquiry.init(whenWhere, "wherever", triggerId = triggerId)
-            logger.warn(
-              s"No vehicles available for category ${neededVehicleCategory} available " +
-              s"for person ${personId.toString}, creating a new vehicle with id ${vehicle.id.toString}"
-            )
-            responseFuture.collect { case ParkingInquiryResponse(stall, _, otherTriggerId) =>
-              vehicle.useParkingStall(stall)
-              logger.debug("Vehicle {} is now taken, which was just created", vehicle.id)
-              vehicle.becomeDriver(mobilityRequester)
-              MobilityStatusResponse(Vector(ActualVehicle(vehicle)), otherTriggerId)
-            } pipeTo mobilityRequester
-          } else {
+    case MobilityStatusInquiry(_, _, _, `None`, triggerId) =>
+      logger.debug(s"Not returning vehicle because no default is defined")
+      sender() ! MobilityStatusResponse(Vector(), triggerId)
+
+    case inquiry @ MobilityStatusInquiry(personId, _, _, Some(_), triggerId) =>
+      if (availableVehicles.nonEmpty || createAnEmergencyVehicle(inquiry).isEmpty) {
+        whoDrivesThisVehicle
+          .filter(_._2 == personId)
+          .flatMap { case (vehicleId, _) => availableVehicles.zipWithIndex.find(_._1.id == vehicleId) }
+          .headOption orElse {
+          availableVehicles.zipWithIndex.headOption.orElse {
+            logger.error(s"THE LIST OF VEHICLES SHOULD NOT BE EMPTY")
+            None
+          }
+        } match {
+          case Some((vehicle, index)) =>
             logger.debug("Vehicle {} is now taken", vehicle.id)
             vehicle.becomeDriver(sender)
             sender() ! MobilityStatusResponse(Vector(ActualVehicle(vehicle)), triggerId)
             availableVehicles = availableVehicles.drop(index)
-          }
+          case _ =>
+            logger.debug(s"Not returning vehicle because no default is defined")
+            sender() ! MobilityStatusResponse(Vector(), triggerId)
+            Nil
         }
-      }.getOrElse {
-        logger.debug(s"Not returning vehicle because no default is defined")
-        sender() ! MobilityStatusResponse(Vector(), triggerId)
-        Nil
       }
 
     case Finish =>
@@ -159,6 +143,50 @@ class HouseholdFleetManager(
     case x =>
       logger.warn(s"No handler for $x")
   }
+
+  /**
+    * @param inquiry
+    * @return
+    */
+  private def createAnEmergencyVehicle(inquiry: MobilityStatusInquiry): Option[BeamVehicle] = {
+    for {
+      category    <- inquiry.requireVehicleCategoryAvailable
+      emergency   <- maybeEmergencyHouseholdVehicleGenerator
+      vehicleType <- emergency.sampleVehicleTypeForEmergencyUse(inquiry.personId, category, inquiry.whereWhen)
+    } yield {
+      val vehicle = emergency.createAndAddVehicle(
+        vehicleType,
+        inquiry.personId,
+        nextVehicleIndex,
+        inquiry.whereWhen,
+        self
+      )
+      // Create a vehicle out of thin air
+      nextVehicleIndex += 1
+      val mobilityRequester = sender()
+      vehiclesInternal(vehicle.id) = vehicle
+
+      // Pipe my car through the parking manager
+      // and complete initialization only when I got them all.
+      val responseFuture = parkingManager ? ParkingInquiry.init(
+        inquiry.whereWhen,
+        "wherever",
+        triggerId = inquiry.triggerId
+      )
+      logger.warn(
+        s"No vehicles available for category ${category} available for " +
+        s"person ${inquiry.personId.toString}, creating a new vehicle with id ${vehicle.id.toString}"
+      )
+
+      responseFuture.collect { case ParkingInquiryResponse(stall, _, otherTriggerId) =>
+        vehicle.useParkingStall(stall)
+        logger.debug("Vehicle {} is now taken, which was just created", vehicle.id)
+        vehicle.becomeDriver(mobilityRequester)
+        MobilityStatusResponse(Vector(ActualVehicle(vehicle)), otherTriggerId)
+      } pipeTo mobilityRequester
+      vehicle
+    }
+  }
 }
 
 object HouseholdFleetManager {
diff --git a/src/main/scala/beam/sim/BeamHelper.scala b/src/main/scala/beam/sim/BeamHelper.scala
index b4ca3f00e9a..6d31e45775b 100755
--- a/src/main/scala/beam/sim/BeamHelper.scala
+++ b/src/main/scala/beam/sim/BeamHelper.scala
@@ -923,11 +923,12 @@ trait BeamHelper extends LazyLogging {
         population.getFactory,
         households.getFactory
       )
-      .foreach { case (carrier, household, plan) =>
+      .foreach { case (carrier, household, plan, personId, vehicleId) =>
         households.getHouseholdAttributes
           .putAttribute(household.getId.toString, "homecoordx", carrier.warehouseLocationUTM.getX)
         households.getHouseholdAttributes
           .putAttribute(household.getId.toString, "homecoordy", carrier.warehouseLocationUTM.getY)
+        population.getPersonAttributes.putAttribute(personId.toString, "vehicle", vehicleId.toString)
         households.getHouseholds.put(household.getId, household)
         population.addPerson(plan.getPerson)
         AvailableModeUtils.setAvailableModesForPerson_v2(

From 032acce2c9158af6dda4d7ebc3592615ce56816d Mon Sep 17 00:00:00 2001
From: Haitam Laarabi <10712736+htmlrb@users.noreply.github.com>
Date: Wed, 16 Mar 2022 21:44:40 -0700
Subject: [PATCH 15/58] additional changes

---
 .../household/HouseholdFleetManager.scala     | 48 +++++++++----------
 1 file changed, 24 insertions(+), 24 deletions(-)

diff --git a/src/main/scala/beam/agentsim/agents/household/HouseholdFleetManager.scala b/src/main/scala/beam/agentsim/agents/household/HouseholdFleetManager.scala
index c9c0d44547d..dabe8ae7ede 100644
--- a/src/main/scala/beam/agentsim/agents/household/HouseholdFleetManager.scala
+++ b/src/main/scala/beam/agentsim/agents/household/HouseholdFleetManager.scala
@@ -106,31 +106,31 @@ class HouseholdFleetManager(
     case GetVehicleTypes(triggerId) =>
       sender() ! VehicleTypesResponse(vehicles.values.map(_.beamVehicleType).toSet, triggerId)
 
-    case MobilityStatusInquiry(_, _, _, `None`, triggerId) =>
-      logger.debug(s"Not returning vehicle because no default is defined")
-      sender() ! MobilityStatusResponse(Vector(), triggerId)
-
-    case inquiry @ MobilityStatusInquiry(personId, _, _, Some(_), triggerId) =>
-      if (availableVehicles.nonEmpty || createAnEmergencyVehicle(inquiry).isEmpty) {
-        whoDrivesThisVehicle
-          .filter(_._2 == personId)
-          .flatMap { case (vehicleId, _) => availableVehicles.zipWithIndex.find(_._1.id == vehicleId) }
-          .headOption orElse {
-          availableVehicles.zipWithIndex.headOption.orElse {
+    case inquiry @ MobilityStatusInquiry(personId, _, _, requireVehicleCategoryAvailable, triggerId) =>
+      val availableVehicleMaybe: Option[(BeamVehicle, Int)] = requireVehicleCategoryAvailable match {
+        case Some(_) if personId.toString.contains("freight") =>
+          whoDrivesThisVehicle
+            .filter(_._2 == personId)
+            .flatMap { case (vehicleId, _) => availableVehicles.zipWithIndex.find(_._1.id == vehicleId) }
+            .headOption
+        case Some(requireVehicleCategory) =>
+          availableVehicles.zipWithIndex.find(_._1.beamVehicleType.vehicleCategory == requireVehicleCategory)
+        case _ => availableVehicles.zipWithIndex.headOption
+      }
+
+      availableVehicleMaybe match {
+        case Some((availableVehicle, index)) =>
+          logger.debug("Vehicle {} is now taken", availableVehicle.id)
+          availableVehicle.becomeDriver(sender)
+          sender() ! MobilityStatusResponse(Vector(ActualVehicle(availableVehicle)), triggerId)
+          availableVehicles = availableVehicles.drop(index)
+        case None if createAnEmergencyVehicle(inquiry).nonEmpty =>
+          logger.debug(s"An emergency vehicle has been created!")
+        case _ =>
+          if (availableVehicles.isEmpty)
             logger.error(s"THE LIST OF VEHICLES SHOULD NOT BE EMPTY")
-            None
-          }
-        } match {
-          case Some((vehicle, index)) =>
-            logger.debug("Vehicle {} is now taken", vehicle.id)
-            vehicle.becomeDriver(sender)
-            sender() ! MobilityStatusResponse(Vector(ActualVehicle(vehicle)), triggerId)
-            availableVehicles = availableVehicles.drop(index)
-          case _ =>
-            logger.debug(s"Not returning vehicle because no default is defined")
-            sender() ! MobilityStatusResponse(Vector(), triggerId)
-            Nil
-        }
+          logger.debug(s"Not returning vehicle because no default for  is defined")
+          sender() ! MobilityStatusResponse(Vector(), triggerId)
       }
 
     case Finish =>

From abc3f7f18d6fe18eb703f8d0657133bd04245766 Mon Sep 17 00:00:00 2001
From: Haitam Laarabi <10712736+htmlrb@users.noreply.github.com>
Date: Wed, 16 Mar 2022 23:03:16 -0700
Subject: [PATCH 16/58] another fix

---
 .../gemini/unlimited_charging_infrastructure.py | 17 +++++++++--------
 .../household/HouseholdFleetManager.scala       | 12 ++++++------
 2 files changed, 15 insertions(+), 14 deletions(-)

diff --git a/src/main/python/gemini/unlimited_charging_infrastructure.py b/src/main/python/gemini/unlimited_charging_infrastructure.py
index b31c9661042..f03ececc4a4 100644
--- a/src/main/python/gemini/unlimited_charging_infrastructure.py
+++ b/src/main/python/gemini/unlimited_charging_infrastructure.py
@@ -1,7 +1,7 @@
 #ADD MANY 50DC 150DC 250DC 350DC
 headerfile = "taz,parkingType,pricingModel,chargingPointType,numStalls,feeInCents,reservedFor"
 
-with open('gemini_taz_unlimited_parking_stalls.csv', mode='w') as csv_writer:
+with open('sfbay_taz_unlimited_parking_stalls.csv', mode='w') as csv_writer:
     csv_writer.write(headerfile+"\n")
 
     for x in range(1, 1455):
@@ -10,7 +10,7 @@
         csv_writer.write(f"{x},Public,Block,NoCharger,9999999,0,Any" + "\n")
 
 
-with open('gemini_taz_unlimited_charging_point.csv', mode='w') as csv_writer:
+with open('sfbay_taz_unlimited_charging_point.csv', mode='w') as csv_writer:
     csv_writer.write(headerfile+"\n")
 
     for x in range(1, 1455):
@@ -20,18 +20,19 @@
         csv_writer.write(f"{x},Workplace,Block,WorkLevel2(7.2|AC),9999999,14.4,Any" + "\n")
 
         csv_writer.write(f"{x},Public,Block,PublicLevel2(7.2|AC),9999999,19.3,Any" + "\n")
-        csv_writer.write(f"{x},Public,Block,PublicFC(50|DC),9999999,306.73,Any" + "\n")
         csv_writer.write(f"{x},Public,Block,PublicFC(150|DC),9999999,490.98,Any" + "\n")
-        csv_writer.write(f"{x},Public,Block,PublicXFC(250|DC),9999999,675.05,Any" + "\n")
-        csv_writer.write(f"{x},Public,Block,PublicXFC(400|DC),9999999,950.52,Any" + "\n")
+        csv_writer.write(f"{x},Public,Block,PublicFC(200|DC),9999999,654.64,Any" + "\n")
+        csv_writer.write(f"{x},Public,Block,PublicXFC(300|DC),9999999,981.96,Any" + "\n")
+        csv_writer.write(f"{x},Public,Block,PublicXFC(400|DC),9999999,1309.28,Any" + "\n")
 
 
-with open('gemini_taz_unlimited_depots.csv', mode='w') as csv_writer:
+with open('sfbay_taz_unlimited_depots.csv', mode='w') as csv_writer:
     csv_writer.write(headerfile+"\n")
 
     for x in range(1, 1455):
         csv_writer.write(f"{x},Public,FlatFee,DepotFC(150.0|DC),9999999,490.98,ridehail(GlobalRHM)" + "\n")
-        csv_writer.write(f"{x},Public,FlatFee,DepotXFC(250.0|DC),9999999,675.05,ridehail(GlobalRHM)" + "\n")
-        csv_writer.write(f"{x},Public,FlatFee,DepotXFC(400.0|DC),9999999,950.52,ridehail(GlobalRHM)" + "\n")
+        csv_writer.write(f"{x},Public,FlatFee,DepotXFC(200.0|DC),9999999,654.64,ridehail(GlobalRHM)" + "\n")
+        csv_writer.write(f"{x},Public,FlatFee,DepotXFC(300.0|DC),9999999,981.96,ridehail(GlobalRHM)" + "\n")
+        csv_writer.write(f"{x},Public,FlatFee,DepotXFC(400.0|DC),9999999,1309.28,ridehail(GlobalRHM)" + "\n")
 
 
diff --git a/src/main/scala/beam/agentsim/agents/household/HouseholdFleetManager.scala b/src/main/scala/beam/agentsim/agents/household/HouseholdFleetManager.scala
index dabe8ae7ede..98aa94799d1 100644
--- a/src/main/scala/beam/agentsim/agents/household/HouseholdFleetManager.scala
+++ b/src/main/scala/beam/agentsim/agents/household/HouseholdFleetManager.scala
@@ -107,23 +107,23 @@ class HouseholdFleetManager(
       sender() ! VehicleTypesResponse(vehicles.values.map(_.beamVehicleType).toSet, triggerId)
 
     case inquiry @ MobilityStatusInquiry(personId, _, _, requireVehicleCategoryAvailable, triggerId) =>
-      val availableVehicleMaybe: Option[(BeamVehicle, Int)] = requireVehicleCategoryAvailable match {
+      val availableVehicleMaybe: Option[BeamVehicle] = requireVehicleCategoryAvailable match {
         case Some(_) if personId.toString.contains("freight") =>
           whoDrivesThisVehicle
             .filter(_._2 == personId)
-            .flatMap { case (vehicleId, _) => availableVehicles.zipWithIndex.find(_._1.id == vehicleId) }
+            .flatMap { case (vehicleId, _) => availableVehicles.find(_.id == vehicleId) }
             .headOption
         case Some(requireVehicleCategory) =>
-          availableVehicles.zipWithIndex.find(_._1.beamVehicleType.vehicleCategory == requireVehicleCategory)
-        case _ => availableVehicles.zipWithIndex.headOption
+          availableVehicles.find(_.beamVehicleType.vehicleCategory == requireVehicleCategory)
+        case _ => availableVehicles.headOption
       }
 
       availableVehicleMaybe match {
-        case Some((availableVehicle, index)) =>
+        case Some(availableVehicle) =>
           logger.debug("Vehicle {} is now taken", availableVehicle.id)
           availableVehicle.becomeDriver(sender)
           sender() ! MobilityStatusResponse(Vector(ActualVehicle(availableVehicle)), triggerId)
-          availableVehicles = availableVehicles.drop(index)
+          availableVehicles = availableVehicles.filter(_ != availableVehicle)
         case None if createAnEmergencyVehicle(inquiry).nonEmpty =>
           logger.debug(s"An emergency vehicle has been created!")
         case _ =>

From 887c36608d645ea5a152554c8fbdca6896afeb6a Mon Sep 17 00:00:00 2001
From: Haitam Laarabi <10712736+htmlrb@users.noreply.github.com>
Date: Thu, 17 Mar 2022 10:03:55 -0700
Subject: [PATCH 17/58] fix

---
 .../agentsim/infrastructure/parking/ParkingZoneSearch.scala     | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/scala/beam/agentsim/infrastructure/parking/ParkingZoneSearch.scala b/src/main/scala/beam/agentsim/infrastructure/parking/ParkingZoneSearch.scala
index 7749313afe3..47ec1dbc848 100644
--- a/src/main/scala/beam/agentsim/infrastructure/parking/ParkingZoneSearch.scala
+++ b/src/main/scala/beam/agentsim/infrastructure/parking/ParkingZoneSearch.scala
@@ -262,7 +262,7 @@ object ParkingZoneSearch {
       override def lookupParkingZonesInNextSearchAreaUnlessThresholdReached(
         zoneQuadTree: QuadTree[GEO]
       ): Option[List[GEO]] = {
-        if (thisInnerRadius > searchMaxRadius) None
+        if (thisInnerRadius >= searchMaxRadius) None
         else {
           val result = zoneQuadTree
             .getRing(destinationUTM.getX, destinationUTM.getY, thisInnerRadius, thisOuterRadius)

From 99b3897d49ffa8eb8a5a903204c16ab6d57f275d Mon Sep 17 00:00:00 2001
From: Haitam Laarabi <10712736+htmlrb@users.noreply.github.com>
Date: Thu, 17 Mar 2022 10:04:20 -0700
Subject: [PATCH 18/58] correcting fix

---
 .../agentsim/infrastructure/parking/ParkingZoneSearch.scala   | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/main/scala/beam/agentsim/infrastructure/parking/ParkingZoneSearch.scala b/src/main/scala/beam/agentsim/infrastructure/parking/ParkingZoneSearch.scala
index 47ec1dbc848..d8738f5f720 100644
--- a/src/main/scala/beam/agentsim/infrastructure/parking/ParkingZoneSearch.scala
+++ b/src/main/scala/beam/agentsim/infrastructure/parking/ParkingZoneSearch.scala
@@ -262,7 +262,7 @@ object ParkingZoneSearch {
       override def lookupParkingZonesInNextSearchAreaUnlessThresholdReached(
         zoneQuadTree: QuadTree[GEO]
       ): Option[List[GEO]] = {
-        if (thisInnerRadius >= searchMaxRadius) None
+        if (thisInnerRadius > searchMaxRadius) None
         else {
           val result = zoneQuadTree
             .getRing(destinationUTM.getX, destinationUTM.getY, thisInnerRadius, thisOuterRadius)
@@ -289,7 +289,7 @@ object ParkingZoneSearch {
       override def lookupParkingZonesInNextSearchAreaUnlessThresholdReached(
         zoneQuadTree: QuadTree[GEO]
       ): Option[List[GEO]] = {
-        if (thisInnerDistance > maxDistance) None
+        if (thisInnerDistance >= maxDistance) None
         else {
           val result = zoneQuadTree
             .getElliptical(originUTM.getX, originUTM.getY, destinationUTM.getX, destinationUTM.getY, thisInnerDistance)

From 1b5ed9923c63194b9aae72180d66bf71a50832bd Mon Sep 17 00:00:00 2001
From: Rutvik Patel <heyrutvik@gmail.com>
Date: Sat, 19 Mar 2022 13:53:48 +0530
Subject: [PATCH 19/58] changes dep

---
 build.gradle | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build.gradle b/build.gradle
index 89b15e53d10..b3ce9129d29 100755
--- a/build.gradle
+++ b/build.gradle
@@ -199,7 +199,7 @@ dependencies {
     implementation "com.typesafe.scala-logging:scala-logging_${scalaBinaryVersion}:3.9.0"
     implementation "org.slf4j:log4j-over-slf4j:${slf4jVersion}"
 
-    implementation(group: 'com.github.LBNL-UCB-STI', name: 'r5', version: 'v4.5.3_linkRadiusMeters') {
+    implementation(group: 'com.github.LBNL-UCB-STI', name: 'r5', version: 'v4.5.3_unconstrainedSpatialSearch') {
         exclude group: 'ch.qos.logback', module: 'logback-classic'
         exclude group: 'org.slf4j', module: 'slf4j-simple'
     }

From ddf8f18e60de3ea521f6fd746291a4ac62e0c91e Mon Sep 17 00:00:00 2001
From: Rutvik Patel <heyrutvik@gmail.com>
Date: Sat, 19 Mar 2022 15:34:08 +0530
Subject: [PATCH 20/58] skip out of bound points

---
 src/main/scala/beam/sim/common/GeoUtils.scala | 25 +++++++++++--------
 1 file changed, 14 insertions(+), 11 deletions(-)

diff --git a/src/main/scala/beam/sim/common/GeoUtils.scala b/src/main/scala/beam/sim/common/GeoUtils.scala
index 7c480e4dbaf..4ba2bf335a6 100755
--- a/src/main/scala/beam/sim/common/GeoUtils.scala
+++ b/src/main/scala/beam/sim/common/GeoUtils.scala
@@ -122,19 +122,22 @@ trait GeoUtils extends ExponentialLazyLogging {
     maxRadius: Double,
     streetMode: StreetMode = StreetMode.WALK
   ): Split = {
+    val isWithinBbox = streetLayer.envelope.contains(coord.getX, coord.getY)
     var radius = 10.0
     var theSplit: Split = null
-    while (theSplit == null && radius <= maxRadius) {
-      theSplit = streetLayer.findSplit(coord.getY, coord.getX, radius, streetMode)
-      radius = radius * 10
-    }
-    if (theSplit == null) {
-      theSplit = streetLayer.findSplit(coord.getY, coord.getX, maxRadius, streetMode)
-    }
-    if (theSplit == null) {
-      notExponentialLogger.warn(
-        s"The split is `null` for StreetLayer.BoundingBox: ${streetLayer.getEnvelope}, coord: $coord, maxRadius: $maxRadius, street mode $streetMode"
-      )
+    if (isWithinBbox) {
+      while (theSplit == null && radius <= maxRadius) {
+        theSplit = streetLayer.findSplit(coord.getY, coord.getX, radius, streetMode)
+        radius = radius * 10
+      }
+      if (theSplit == null) {
+        theSplit = streetLayer.findSplit(coord.getY, coord.getX, maxRadius, streetMode)
+      }
+      if (theSplit == null) {
+        notExponentialLogger.warn(
+          s"The split is `null` for StreetLayer.BoundingBox: ${streetLayer.getEnvelope}, coord: $coord, maxRadius: $maxRadius, street mode $streetMode"
+        )
+      }
     }
     theSplit
   }

From 1dcf79e1d60f7ac06dfcb6f2d2399a1e947720e3 Mon Sep 17 00:00:00 2001
From: Rutvik Patel <heyrutvik@gmail.com>
Date: Sat, 19 Mar 2022 17:30:22 +0530
Subject: [PATCH 21/58] adds memoized split method

---
 src/main/scala/beam/sim/common/GeoUtils.scala | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/src/main/scala/beam/sim/common/GeoUtils.scala b/src/main/scala/beam/sim/common/GeoUtils.scala
index 4ba2bf335a6..a41c2a58960 100755
--- a/src/main/scala/beam/sim/common/GeoUtils.scala
+++ b/src/main/scala/beam/sim/common/GeoUtils.scala
@@ -17,6 +17,8 @@ import org.matsim.api.core.v01.network.Link
 import org.matsim.core.utils.geometry.transformations.GeotoolsTransformation
 import org.slf4j.LoggerFactory
 
+import scala.collection.concurrent.TrieMap
+
 case class EdgeWithCoord(edgeIndex: Int, wgsCoord: Coordinate)
 
 /**
@@ -29,6 +31,10 @@ trait GeoUtils extends ExponentialLazyLogging {
   def localCRS: String
   private lazy val notExponentialLogger = Logger(LoggerFactory.getLogger(getClass.getName))
 
+  type Input = (StreetMode, Double, Coord)
+  type Func = Input => Option[Split]
+  val splitStore = TrieMap[Input, Option[Split]]()
+
   lazy val utm2Wgs: GeotoolsTransformation =
     new GeotoolsTransformation(localCRS, "EPSG:4326")
 
@@ -122,6 +128,18 @@ trait GeoUtils extends ExponentialLazyLogging {
     maxRadius: Double,
     streetMode: StreetMode = StreetMode.WALK
   ): Split = {
+    splitStore
+      .getOrElseUpdate((streetMode, maxRadius, coord), Option(_getR5Split(streetLayer, coord, maxRadius, streetMode)))
+      .orNull
+  }
+
+  private def _getR5Split(
+    streetLayer: StreetLayer,
+    coord: Coord,
+    maxRadius: Double,
+    streetMode: StreetMode = StreetMode.WALK
+  ): Split = {
+    logger.info("Called _getR5Split with {}, {}, {}", streetMode, maxRadius, coord)
     val isWithinBbox = streetLayer.envelope.contains(coord.getX, coord.getY)
     var radius = 10.0
     var theSplit: Split = null

From 717b806ee3e62e5876dd908d546a9218951c83b8 Mon Sep 17 00:00:00 2001
From: Haitam Laarabi <10712736+htmlrb@users.noreply.github.com>
Date: Tue, 22 Mar 2022 16:58:54 -0700
Subject: [PATCH 22/58] updating frism plans

---
 src/main/python/freight/frism_to_beam_freight_plans.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/main/python/freight/frism_to_beam_freight_plans.py b/src/main/python/freight/frism_to_beam_freight_plans.py
index 6e48543772f..98eb27648dd 100644
--- a/src/main/python/freight/frism_to_beam_freight_plans.py
+++ b/src/main/python/freight/frism_to_beam_freight_plans.py
@@ -18,8 +18,8 @@ def add_prefix(prefix, column, row, toNum = True):
     return new
 
 
-directory_input = os.path.expanduser('~/Data/FREIGHT/Outputs_All_SF_0223_2022')
-directory_output = os.path.expanduser('~/Data/FREIGHT/Outputs_All_SF_0223_2022_merged')
+directory_input = os.path.expanduser('~/Data/FREIGHT/Outputs(All SF)_0322_2022')
+directory_output = os.path.expanduser('~/Data/FREIGHT/Outputs(All SF)_0322_2022_merged')
 Path(directory_output).mkdir(parents=True, exist_ok=True)
 carriers = None
 payload_plans = None

From 0443c5541a103ed024394316ba02d34f9eec0a45 Mon Sep 17 00:00:00 2001
From: Zach Needell <zneedell@gmail.com>
Date: Thu, 24 Mar 2022 10:06:56 -0700
Subject: [PATCH 23/58] switch to concurrent hashmap

---
 .../Hao2018CaccRoadCapacityAdjustmentFunction.java           | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/main/java/beam/physsim/jdeqsim/cacc/roadcapacityadjustmentfunctions/Hao2018CaccRoadCapacityAdjustmentFunction.java b/src/main/java/beam/physsim/jdeqsim/cacc/roadcapacityadjustmentfunctions/Hao2018CaccRoadCapacityAdjustmentFunction.java
index ea61cb5453b..6996a6e24da 100644
--- a/src/main/java/beam/physsim/jdeqsim/cacc/roadcapacityadjustmentfunctions/Hao2018CaccRoadCapacityAdjustmentFunction.java
+++ b/src/main/java/beam/physsim/jdeqsim/cacc/roadcapacityadjustmentfunctions/Hao2018CaccRoadCapacityAdjustmentFunction.java
@@ -17,6 +17,7 @@
 import java.util.HashMap;
 import java.util.Map;
 import java.util.Optional;
+import java.util.concurrent.ConcurrentHashMap;
 
 /*
 
@@ -36,8 +37,8 @@ public class Hao2018CaccRoadCapacityAdjustmentFunction implements RoadCapacityAd
     private final boolean writeGraphs;
     private final OutputDirectoryHierarchy controllerIO;
     private final MultiValuedMap<Double, Double> caccCapacityIncrease = new HashSetValuedHashMap<>();
-    private final Map<String, Double> caccLinkCapacityIncrease = new HashMap<>();
-    private final Map<String, Double> allLinksCapacityIncrease = new HashMap<>();
+    private final Map<String, Double> caccLinkCapacityIncrease = new ConcurrentHashMap<>();
+    private final Map<String, Double> allLinksCapacityIncrease = new ConcurrentHashMap<>();
     private final Optional<ICsvMapWriter> csvWriter;
 
     private int numberOfMixedVehicleTypeEncountersOnCACCCategoryRoads = 0;

From 3c345a7d4aad79918a3a96c7e2911da0f639864c Mon Sep 17 00:00:00 2001
From: Grigory D <grigory.derzhavets@simrise.tech>
Date: Sun, 27 Mar 2022 12:50:53 +0400
Subject: [PATCH 24/58] Issue-3495 add a configurable seed for population
 sampling

---
 src/main/scala/beam/sim/BeamHelper.scala                   | 1 +
 src/main/scala/beam/sim/config/BeamConfig.scala            | 6 ++++++
 src/main/scala/beam/sim/population/PopulationScaling.scala | 2 +-
 3 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/src/main/scala/beam/sim/BeamHelper.scala b/src/main/scala/beam/sim/BeamHelper.scala
index e5f0e5cd9d1..fcf518e59bd 100755
--- a/src/main/scala/beam/sim/BeamHelper.scala
+++ b/src/main/scala/beam/sim/BeamHelper.scala
@@ -966,6 +966,7 @@ trait BeamHelper extends LazyLogging {
       Kamon.init(config.withFallback(ConfigFactory.load()))
     }
 
+    logger.info("Agentsim random seed for population scaling is set to {}.", beamConfig.beam.agentsim.randomSeed)
     logger.info("Starting beam on branch {} at commit {}.", BashUtils.getBranch, BashUtils.getCommitHash)
 
     logger.info(
diff --git a/src/main/scala/beam/sim/config/BeamConfig.scala b/src/main/scala/beam/sim/config/BeamConfig.scala
index 57f17ef5f07..8568f5fcb37 100644
--- a/src/main/scala/beam/sim/config/BeamConfig.scala
+++ b/src/main/scala/beam/sim/config/BeamConfig.scala
@@ -2,6 +2,8 @@
 
 package beam.sim.config
 
+import scala.util.Random
+
 case class BeamConfig(
   beam: BeamConfig.Beam,
   matsim: BeamConfig.Matsim
@@ -37,6 +39,7 @@ object BeamConfig {
   object Beam {
 
     case class Agentsim(
+      randomSeed: scala.Int,
       agentSampleSizeAsFractionOfPopulation: scala.Double,
       agents: BeamConfig.Beam.Agentsim.Agents,
       chargingNetworkManager: BeamConfig.Beam.Agentsim.ChargingNetworkManager,
@@ -2159,6 +2162,9 @@ object BeamConfig {
 
       def apply(c: com.typesafe.config.Config): BeamConfig.Beam.Agentsim = {
         BeamConfig.Beam.Agentsim(
+          randomSeed = if (c.hasPathOrNull("randomSeed"))
+            c.getInt("randomSeed")
+            else new Random().nextInt(),
           agentSampleSizeAsFractionOfPopulation =
             if (c.hasPathOrNull("agentSampleSizeAsFractionOfPopulation"))
               c.getDouble("agentSampleSizeAsFractionOfPopulation")
diff --git a/src/main/scala/beam/sim/population/PopulationScaling.scala b/src/main/scala/beam/sim/population/PopulationScaling.scala
index b8fa57900ec..90e6efda565 100644
--- a/src/main/scala/beam/sim/population/PopulationScaling.scala
+++ b/src/main/scala/beam/sim/population/PopulationScaling.scala
@@ -126,7 +126,7 @@ class PopulationScaling extends LazyLogging {
     val numAgents = math.round(
       beamConfig.beam.agentsim.agentSampleSizeAsFractionOfPopulation * scenario.getPopulation.getPersons.size()
     )
-    val rand = new Random(beamServices.beamConfig.matsim.modules.global.randomSeed)
+    val rand = new Random(beamServices.beamConfig.beam.agentsim.randomSeed)
     val notSelectedHouseholdIds = mutable.Set[Id[Household]]()
     val notSelectedVehicleIds = mutable.Set[Id[Vehicle]]()
     val notSelectedPersonIds = mutable.Set[Id[Person]]()

From bef20768a6f02b3f2c97345bcc002b08f43c14d6 Mon Sep 17 00:00:00 2001
From: Grigory D <grigory.derzhavets@simrise.tech>
Date: Tue, 29 Mar 2022 17:45:07 +0400
Subject: [PATCH 25/58] Issue-3495 add tests

---
 .../population/PopulationSamplingSpec.scala   | 116 ++++++++++++++++++
 1 file changed, 116 insertions(+)
 create mode 100644 src/test/scala/beam/sim/population/PopulationSamplingSpec.scala

diff --git a/src/test/scala/beam/sim/population/PopulationSamplingSpec.scala b/src/test/scala/beam/sim/population/PopulationSamplingSpec.scala
new file mode 100644
index 00000000000..a868f171701
--- /dev/null
+++ b/src/test/scala/beam/sim/population/PopulationSamplingSpec.scala
@@ -0,0 +1,116 @@
+package beam.sim.population
+
+import beam.integration.IntegrationSpecCommon
+import beam.sim.config.{BeamConfig, MatSimBeamConfigBuilder}
+import beam.sim.{BeamHelper, BeamScenario, BeamServices}
+import beam.utils.FileUtils
+import com.typesafe.config.ConfigValueFactory
+import org.matsim.core.scenario.{MutableScenario, ScenarioUtils}
+import org.scalatest.matchers.must.Matchers
+import org.scalatest.matchers.should.Matchers.convertToAnyShouldWrapper
+import org.scalatest.wordspec.AnyWordSpecLike
+
+import java.nio.file.Files
+
+class PopulationSamplingSpec extends AnyWordSpecLike with Matchers with BeamHelper with IntegrationSpecCommon {
+
+  private def getObjectsForPopulationSampling(
+    seed:Integer = 0
+  ): (MutableScenario, BeamScenario, BeamConfig, BeamServices, String) = {
+    if (seed.equals(0)) {
+      return prepareObjectsForPopulationSampling(getConfigWithRandomSeed)
+    }
+    prepareObjectsForPopulationSampling(getConfigWithSeed(seed))
+  }
+
+  private def getConfigWithRandomSeed: BeamConfig = {
+    BeamConfig(
+      baseConfig
+        .withValue(
+          "beam.agentsim.agentSampleSizeAsFractionOfPopulation",
+          ConfigValueFactory.fromAnyRef(0.5)
+        )
+        .resolve()
+    )
+  }
+
+  private def getConfigWithSeed(seed: Integer): BeamConfig = {
+    BeamConfig(
+      baseConfig
+        .withValue(
+          "beam.agentsim.agentSampleSizeAsFractionOfPopulation",
+          ConfigValueFactory.fromAnyRef(0.5)
+        )
+        .withValue(
+          "beam.agentsim.randomSeed",
+          ConfigValueFactory.fromAnyRef(seed))
+        .resolve()
+    )
+  }
+
+  private def prepareObjectsForPopulationSampling(
+    config: BeamConfig
+  ): (MutableScenario, BeamScenario, BeamConfig, BeamServices, String) = {
+    val beamScenario = loadScenario(config)
+    val configBuilder = new MatSimBeamConfigBuilder(baseConfig)
+    val matsimConfig = configBuilder.buildMatSimConf()
+    matsimConfig.planCalcScore().setMemorizingExperiencedPlans(true)
+    FileUtils.setConfigOutputFile(config, matsimConfig)
+
+    val scenario = ScenarioUtils.loadScenario(matsimConfig).asInstanceOf[MutableScenario]
+    scenario.setNetwork(beamScenario.network)
+
+    val injector = org.matsim.core.controler.Injector.createInjector(
+      scenario.getConfig,
+      module(baseConfig, config, scenario, beamScenario)
+    )
+
+    val beamServices: BeamServices = injector.getInstance(classOf[BeamServices])
+    val temporaryOutputDirectory = Files.createTempDirectory("dummyOutputDirectory").toString
+    (scenario, beamScenario, config, beamServices, temporaryOutputDirectory)
+  }
+
+  "PopulationSampling" must {
+    "sample the same agents for the same randomSeed value" in {
+      val samplingDataOne = getObjectsForPopulationSampling(1441)
+      val samplingDataTwo = getObjectsForPopulationSampling(1441)
+      val agentsBeforeSamplingOneSize = samplingDataOne._1.getPopulation.getPersons.keySet().size()
+      PopulationScaling.samplePopulation(samplingDataOne._1, samplingDataOne._2, samplingDataOne._3, samplingDataOne._4, samplingDataOne._5)
+
+      agentsBeforeSamplingOneSize should be <= 50
+      val agentsAfterSamplingOne = samplingDataOne._1.getPopulation.getPersons.keySet().stream().map(a => a.leftSide).toArray
+
+      PopulationScaling.samplePopulation(samplingDataTwo._1, samplingDataTwo._2, samplingDataTwo._3, samplingDataTwo._4, samplingDataTwo._5)
+      val agentsAfterSamplingTwo = samplingDataTwo._1.getPopulation.getPersons.keySet().stream().map(a => a.leftSide).toArray
+
+      agentsAfterSamplingOne shouldBe agentsAfterSamplingTwo
+    }
+
+    "sample different agents if randomSeed is not set" in {
+      val samplingDataOne = getObjectsForPopulationSampling()
+      val samplingDataTwo = getObjectsForPopulationSampling()
+      val agentsBeforeSamplingOneSize = samplingDataOne._1.getPopulation.getPersons.keySet().size()
+      PopulationScaling.samplePopulation(samplingDataOne._1, samplingDataOne._2, samplingDataOne._3, samplingDataOne._4, samplingDataOne._5)
+      PopulationScaling.samplePopulation(samplingDataTwo._1, samplingDataTwo._2, samplingDataTwo._3, samplingDataTwo._4, samplingDataTwo._5)
+
+      agentsBeforeSamplingOneSize should be <= 50
+      val agentsAfterSamplingOne = samplingDataOne._1.getPopulation.getPersons.keySet().stream().map(a => a.leftSide).toArray
+      val agentsAfterSamplingTwo = samplingDataTwo._1.getPopulation.getPersons.keySet().stream().map(a => a.leftSide).toArray
+
+      agentsAfterSamplingOne should not equal agentsAfterSamplingTwo
+    }
+    "sample different agents for different randomSeeds" in {
+      val samplingDataOne = getObjectsForPopulationSampling(1441)
+      val samplingDataTwo = getObjectsForPopulationSampling(23)
+      val agentsBeforeSamplingOneSize = samplingDataOne._1.getPopulation.getPersons.keySet().size()
+      PopulationScaling.samplePopulation(samplingDataOne._1, samplingDataOne._2, samplingDataOne._3, samplingDataOne._4, samplingDataOne._5)
+      PopulationScaling.samplePopulation(samplingDataTwo._1, samplingDataTwo._2, samplingDataTwo._3, samplingDataTwo._4, samplingDataTwo._5)
+
+      agentsBeforeSamplingOneSize should be <= 50
+      val agentsAfterSamplingOne = samplingDataOne._1.getPopulation.getPersons.keySet().stream().map(a => a.leftSide).toArray
+      val agentsAfterSamplingTwo = samplingDataTwo._1.getPopulation.getPersons.keySet().stream().map(a => a.leftSide).toArray
+
+      agentsAfterSamplingOne should not equal agentsAfterSamplingTwo
+    }
+  }
+}

From 8f5f63619cf27d574421a55d1c293e2658cbbe7f Mon Sep 17 00:00:00 2001
From: Grigory D <grigory.derzhavets@simrise.tech>
Date: Tue, 29 Mar 2022 17:46:40 +0400
Subject: [PATCH 26/58] Issue-3495 scalafmt

---
 .../scala/beam/sim/config/BeamConfig.scala    |  5 +-
 .../population/PopulationSamplingSpec.scala   | 75 ++++++++++++++-----
 2 files changed, 61 insertions(+), 19 deletions(-)

diff --git a/src/main/scala/beam/sim/config/BeamConfig.scala b/src/main/scala/beam/sim/config/BeamConfig.scala
index 8568f5fcb37..7e37da13d85 100644
--- a/src/main/scala/beam/sim/config/BeamConfig.scala
+++ b/src/main/scala/beam/sim/config/BeamConfig.scala
@@ -2162,8 +2162,9 @@ object BeamConfig {
 
       def apply(c: com.typesafe.config.Config): BeamConfig.Beam.Agentsim = {
         BeamConfig.Beam.Agentsim(
-          randomSeed = if (c.hasPathOrNull("randomSeed"))
-            c.getInt("randomSeed")
+          randomSeed =
+            if (c.hasPathOrNull("randomSeed"))
+              c.getInt("randomSeed")
             else new Random().nextInt(),
           agentSampleSizeAsFractionOfPopulation =
             if (c.hasPathOrNull("agentSampleSizeAsFractionOfPopulation"))
diff --git a/src/test/scala/beam/sim/population/PopulationSamplingSpec.scala b/src/test/scala/beam/sim/population/PopulationSamplingSpec.scala
index a868f171701..b040893a92d 100644
--- a/src/test/scala/beam/sim/population/PopulationSamplingSpec.scala
+++ b/src/test/scala/beam/sim/population/PopulationSamplingSpec.scala
@@ -15,7 +15,7 @@ import java.nio.file.Files
 class PopulationSamplingSpec extends AnyWordSpecLike with Matchers with BeamHelper with IntegrationSpecCommon {
 
   private def getObjectsForPopulationSampling(
-    seed:Integer = 0
+    seed: Integer = 0
   ): (MutableScenario, BeamScenario, BeamConfig, BeamServices, String) = {
     if (seed.equals(0)) {
       return prepareObjectsForPopulationSampling(getConfigWithRandomSeed)
@@ -41,9 +41,7 @@ class PopulationSamplingSpec extends AnyWordSpecLike with Matchers with BeamHelp
           "beam.agentsim.agentSampleSizeAsFractionOfPopulation",
           ConfigValueFactory.fromAnyRef(0.5)
         )
-        .withValue(
-          "beam.agentsim.randomSeed",
-          ConfigValueFactory.fromAnyRef(seed))
+        .withValue("beam.agentsim.randomSeed", ConfigValueFactory.fromAnyRef(seed))
         .resolve()
     )
   }
@@ -75,13 +73,27 @@ class PopulationSamplingSpec extends AnyWordSpecLike with Matchers with BeamHelp
       val samplingDataOne = getObjectsForPopulationSampling(1441)
       val samplingDataTwo = getObjectsForPopulationSampling(1441)
       val agentsBeforeSamplingOneSize = samplingDataOne._1.getPopulation.getPersons.keySet().size()
-      PopulationScaling.samplePopulation(samplingDataOne._1, samplingDataOne._2, samplingDataOne._3, samplingDataOne._4, samplingDataOne._5)
+      PopulationScaling.samplePopulation(
+        samplingDataOne._1,
+        samplingDataOne._2,
+        samplingDataOne._3,
+        samplingDataOne._4,
+        samplingDataOne._5
+      )
 
       agentsBeforeSamplingOneSize should be <= 50
-      val agentsAfterSamplingOne = samplingDataOne._1.getPopulation.getPersons.keySet().stream().map(a => a.leftSide).toArray
-
-      PopulationScaling.samplePopulation(samplingDataTwo._1, samplingDataTwo._2, samplingDataTwo._3, samplingDataTwo._4, samplingDataTwo._5)
-      val agentsAfterSamplingTwo = samplingDataTwo._1.getPopulation.getPersons.keySet().stream().map(a => a.leftSide).toArray
+      val agentsAfterSamplingOne =
+        samplingDataOne._1.getPopulation.getPersons.keySet().stream().map(a => a.leftSide).toArray
+
+      PopulationScaling.samplePopulation(
+        samplingDataTwo._1,
+        samplingDataTwo._2,
+        samplingDataTwo._3,
+        samplingDataTwo._4,
+        samplingDataTwo._5
+      )
+      val agentsAfterSamplingTwo =
+        samplingDataTwo._1.getPopulation.getPersons.keySet().stream().map(a => a.leftSide).toArray
 
       agentsAfterSamplingOne shouldBe agentsAfterSamplingTwo
     }
@@ -90,25 +102,54 @@ class PopulationSamplingSpec extends AnyWordSpecLike with Matchers with BeamHelp
       val samplingDataOne = getObjectsForPopulationSampling()
       val samplingDataTwo = getObjectsForPopulationSampling()
       val agentsBeforeSamplingOneSize = samplingDataOne._1.getPopulation.getPersons.keySet().size()
-      PopulationScaling.samplePopulation(samplingDataOne._1, samplingDataOne._2, samplingDataOne._3, samplingDataOne._4, samplingDataOne._5)
-      PopulationScaling.samplePopulation(samplingDataTwo._1, samplingDataTwo._2, samplingDataTwo._3, samplingDataTwo._4, samplingDataTwo._5)
+      PopulationScaling.samplePopulation(
+        samplingDataOne._1,
+        samplingDataOne._2,
+        samplingDataOne._3,
+        samplingDataOne._4,
+        samplingDataOne._5
+      )
+      PopulationScaling.samplePopulation(
+        samplingDataTwo._1,
+        samplingDataTwo._2,
+        samplingDataTwo._3,
+        samplingDataTwo._4,
+        samplingDataTwo._5
+      )
 
       agentsBeforeSamplingOneSize should be <= 50
-      val agentsAfterSamplingOne = samplingDataOne._1.getPopulation.getPersons.keySet().stream().map(a => a.leftSide).toArray
-      val agentsAfterSamplingTwo = samplingDataTwo._1.getPopulation.getPersons.keySet().stream().map(a => a.leftSide).toArray
+      val agentsAfterSamplingOne =
+        samplingDataOne._1.getPopulation.getPersons.keySet().stream().map(a => a.leftSide).toArray
+      val agentsAfterSamplingTwo =
+        samplingDataTwo._1.getPopulation.getPersons.keySet().stream().map(a => a.leftSide).toArray
 
       agentsAfterSamplingOne should not equal agentsAfterSamplingTwo
     }
+
     "sample different agents for different randomSeeds" in {
       val samplingDataOne = getObjectsForPopulationSampling(1441)
       val samplingDataTwo = getObjectsForPopulationSampling(23)
       val agentsBeforeSamplingOneSize = samplingDataOne._1.getPopulation.getPersons.keySet().size()
-      PopulationScaling.samplePopulation(samplingDataOne._1, samplingDataOne._2, samplingDataOne._3, samplingDataOne._4, samplingDataOne._5)
-      PopulationScaling.samplePopulation(samplingDataTwo._1, samplingDataTwo._2, samplingDataTwo._3, samplingDataTwo._4, samplingDataTwo._5)
+      PopulationScaling.samplePopulation(
+        samplingDataOne._1,
+        samplingDataOne._2,
+        samplingDataOne._3,
+        samplingDataOne._4,
+        samplingDataOne._5
+      )
+      PopulationScaling.samplePopulation(
+        samplingDataTwo._1,
+        samplingDataTwo._2,
+        samplingDataTwo._3,
+        samplingDataTwo._4,
+        samplingDataTwo._5
+      )
 
       agentsBeforeSamplingOneSize should be <= 50
-      val agentsAfterSamplingOne = samplingDataOne._1.getPopulation.getPersons.keySet().stream().map(a => a.leftSide).toArray
-      val agentsAfterSamplingTwo = samplingDataTwo._1.getPopulation.getPersons.keySet().stream().map(a => a.leftSide).toArray
+      val agentsAfterSamplingOne =
+        samplingDataOne._1.getPopulation.getPersons.keySet().stream().map(a => a.leftSide).toArray
+      val agentsAfterSamplingTwo =
+        samplingDataTwo._1.getPopulation.getPersons.keySet().stream().map(a => a.leftSide).toArray
 
       agentsAfterSamplingOne should not equal agentsAfterSamplingTwo
     }

From 054793c450471cbb29e90582cf659ccbad139dbf Mon Sep 17 00:00:00 2001
From: Haitam Laarabi <10712736+htmlrb@users.noreply.github.com>
Date: Tue, 29 Mar 2022 21:22:02 -0700
Subject: [PATCH 27/58] updating scripts

---
 src/main/R/common/helpers.R             | 144 +++++++++++-
 src/main/R/freight/freight-processing.R | 285 +++++++++++++++++-------
 2 files changed, 343 insertions(+), 86 deletions(-)

diff --git a/src/main/R/common/helpers.R b/src/main/R/common/helpers.R
index 5c53f2af497..f032b1d614a 100644
--- a/src/main/R/common/helpers.R
+++ b/src/main/R/common/helpers.R
@@ -11,18 +11,18 @@ mergefiles <- function(directory){
   for (file in list.files(directory)){
     # if the merged dataset doesn't exist, create it
     filepath <- paste(directory, file, sep="/")
-
+    
     if (!exists("dataset")){
       dataset <- readCsv(filepath)
     }
-
+    
     # if the merged dataset does exist, append to it
     if (exists("dataset")){
       temp_dataset <-readCsv(filepath)
       dataset<-rbind(dataset, temp_dataset)
       rm(temp_dataset)
     }
-
+    
   }
   return(dataset)
 }
@@ -66,3 +66,141 @@ sankeyDiagram <- function(source, target, value, title) {
 }
 
 
+clusteringFreightBy <- function(data,cols,dataCbg,numClusters,labelData) {
+  data[,hour:=as.integer(arrivalTime/3600)%%24]
+  data_asSf <- st_as_sf(data,coords=cols,crs=4326,agr="constant")
+  data_withCBG_asSf <- st_intersection(data_asSf,dataCbg)
+  data_withCBG_asSf$X <- st_coordinates(data_withCBG_asSf$geometry)[,1]
+  data_withCBG_asSf$Y <- st_coordinates(data_withCBG_asSf$geometry)[,2]
+  data_withCBG <- data.table::as.data.table(data_withCBG_asSf)
+  data_withCBG[,cluster:=kmeans(data_withCBG[,.(X,Y)],numClusters)$cluster]
+  result <- data_withCBG[,.(count=.N,x2=mean(X),y2=mean(Y)),by=.(hour,cluster)]
+  result$label <- labelData
+  result
+}
+
+assignPostMilesGeometries <- function(TRUCK_DATA, POSTMILES_SHP) {
+  getElementWithoutFlag <- function(list_elements, flag) {
+    i <- 1
+    while(list_elements[i] == '') {
+      i <- i+1
+    }
+    return(list_elements[i])
+  }
+  TRUCK_DATA$X <- 0.0
+  TRUCK_DATA$Y <- 0.0
+  ### GEOCODING ###
+  #TRUCK_DATA$lon <- 0.0
+  #TRUCK_DATA$lat <- 0.0
+  #TRUCK_DATA$geoAddress <- ""
+  #nrow(TRUCK_DATA)
+  postmiles <- st_read(POSTMILES_SHP)
+  for(i in 1:nrow(TRUCK_DATA)) {
+    county <- TRUCK_DATA$COUNTY[i]
+    cnty <- TRUCK_DATA$CNTY[i]
+    dist <- TRUCK_DATA$DIST[i]
+    rte <- TRUCK_DATA$RTE[i]
+    leg <- TRUCK_DATA$LEG[i]
+    pm <- TRUCK_DATA$POST_MILE[i]
+    filteredPM <- postmiles %>% filter(County == cnty, 
+                                       District == dist, 
+                                       Route == rte, 
+                                       startsWith(as.character(PMc), as.character(pm)))
+    if(nrow(filteredPM) == 0) {
+      filteredPM <- postmiles %>% filter(County == cnty, District == dist, Route == rte)
+    }
+    if(nrow(filteredPM) > 1) {
+      pm_numeric <- 0
+      if(!is.na(as.numeric(pm))) {
+        pm_numeric <- as.numeric(pm)
+      } else {
+        element <- getElementWithoutFlag(unlist(strsplit(pm, "R")), "R")
+        element <- getElementWithoutFlag(unlist(strsplit(element, "L")), "L")
+        element <- getElementWithoutFlag(unlist(strsplit(element, "T")), "T")
+        element <- getElementWithoutFlag(unlist(strsplit(element, "M")), "M")
+        pm_numeric <- as.numeric(element)
+      }
+      filteredPM <- filteredPM %>% rowwise() %>% mutate(diff = abs(as.numeric(PM) - pm_numeric))
+      filteredPM <- filteredPM %>% filter(diff == min(filteredPM$diff))
+    }
+    if(nrow(filteredPM) > 0) {
+      TRUCK_DATA$X[i] <- st_coordinates(filteredPM$geometry[1])[1]
+      TRUCK_DATA$Y[i] <- st_coordinates(filteredPM$geometry[1])[2]
+    }
+    # address <- paste(county, " County, California, USA", sep="")
+    # descriptions <- unlist(strsplit(TRUCK_DATA$DESCRIPTION[i],","))
+    # if(length(descriptions) > 1) {
+    #   address <- paste(str_trim(descriptions[2]), ", ", str_trim(descriptions[1]), ", ", address, sep="")
+    # } else {
+    #   address <- paste(str_trim(descriptions), ", ", address, sep="")
+    # }
+    # address <- ", CA"
+    # address <- paste(TRUCK_DATA$DESCRIPTION[i], ", ", address, sep="")
+    # address <- gsub("RTE.", "route", address)
+    # address <- gsub("JCT.", "", address)
+    # print(address)
+    # result <- geocode(address, output = "latlona", source = "google")
+    # TRUCK_DATA$lon[i] <- as.numeric(result[1])
+    # TRUCK_DATA$lat[i] <- as.numeric(result[2])
+    # TRUCK_DATA$geoAddress[i] <- as.character(result[3])
+    # print(TRUCK_DATA[i])
+  }
+  data.table::fwrite(TRUCK_DATA, normalizePath(pp(freightDir,"/validation/2017_truck_aadtt_geocoded.csv")), quote=T)
+  print("END OF assignPostMilesGeometries")
+  return(TRUCK_DATA)
+}
+
+assignLinkIdToTruckAADTT <- function(NETWORK_CLEANED, NETWORK_CRS, TRUCK_AADTT, MAX_DISTANCE_IN_METER, EXPANSION_FACTOR) {
+  network_sf <- st_transform(st_as_sf(
+    NETWORK_CLEANED,
+    coords = c("fromLocationX", "fromLocationY"),
+    crs = NETWORK_CRS,
+    agr = "constant"), 4326)
+  
+  truck_aadtt_sf <- st_as_sf(
+    TRUCK_AADTT,
+    coords = c("X", "Y"),
+    crs = 4326,
+    agr = "constant")
+  
+  network_sf$tempDistance <- NA
+  truck_aadtt_sf$linkId <- NA
+  expansionFactor <- EXPANSION_FACTOR
+  maxDistanceToSearch <- MAX_DISTANCE_IN_METER
+  counter1 <- 0
+  counter2 <- 0
+  for (row in 1:nrow(truck_aadtt_sf)) {
+    if(row %% 100) {
+      print(paste(row, " entrees have been processed so far!"))
+    }
+    current_pm <- truck_aadtt_sf[row,]
+    currentDist <- 20
+    current_pm_links <- st_is_within_distance(current_pm, network_sf, dist = currentDist)
+    list_of_links <- current_pm_links[1][[1]]
+    while(length(list_of_links) < 1 & currentDist < maxDistanceToSearch) {
+      currentDist <- currentDist * 2
+      current_pm_links <- st_is_within_distance(current_pm, network_sf, dist = currentDist)
+      list_of_links = current_pm_links[1][[1]]
+    }
+    selected_links <- network_sf[list_of_links,]
+    if (length(list_of_links) == 0){
+      counter1 <- counter1 + 1
+      print("no links!!!")
+    } else if (length(list_of_links) == 1){
+      truck_aadtt_sf$linkId[row] <- network_sf[list_of_links,]$linkId
+      network_sf <- network_sf[-list_of_links,]
+      counter2 <- counter2 + 1
+    } else {
+      selected_pm <- truck_aadtt_sf[row,]
+      selected_network <- network_sf[selected_links,]
+      selected_network$tempDistance <- st_distance(selected_network, selected_pm)
+      selected_network_filtered <- slice(selected_network, which.min(tempDistance))
+      truck_aadtt_sf$linkId[row] <- selected_network_filtered$linkId
+      network_sf <- network_sf %>% filter(linkId != selected_network_filtered$linkId)
+      counter2 <- counter2 + 1
+    }
+  }
+  print(paste("# unmatched links ", counter1))
+  print(paste("# matched links ", counter2))
+  return(truck_aadtt_sf)
+}
\ No newline at end of file
diff --git a/src/main/R/freight/freight-processing.R b/src/main/R/freight/freight-processing.R
index 03e5a2c885e..86bfcf13f4e 100644
--- a/src/main/R/freight/freight-processing.R
+++ b/src/main/R/freight/freight-processing.R
@@ -10,23 +10,44 @@ library(ggmap)
 library(sf)
 library(stringr)
 
-workDir <- normalizePath("~/Data/SMART")
+
 activitySimDir <- normalizePath("~/Data/ACTIVITYSIM")
+freightDir <- normalizePath("~/Data/FREIGHT")
+freightWorkDir <- normalizePath(paste(freightDir,"/2022-03-09/output",sep=""))
+
+source("~/Documents/Workspace/scripts/common/keys.R")
+register_google(key = google_api_key_1)
+oaklandMap <- ggmap::get_googlemap("oakland california", zoom = 13, maptype = "roadmap")
+shpFile <- pp(activitySimDir, "/__San_Francisco_Bay_Region_2010_Census_Block_Groups-shp/641aa0d4-ce5b-4a81-9c30-8790c4ab8cfb202047-1-wkkklf.j5ouj.shp")
+sfBayCbg <- st_read(shpFile)
 
-events <- readCsv(pp(workDir, "/5.events.csv.gz"))
-unloading <- events[actType=="Unloading"]
-nrow(unloading[type=="actstart"])
-nrow(unloading[type=="actend"])
-warehouse <- events[actType=="Warehouse"]
-nrow(warehouse[type=="actstart"])
-nrow(warehouse[type=="actend"])
+
+events <- readCsv(pp(freightWorkDir, "/0.events.csv.gz"))
 pt <- events[type=="PathTraversal"][,c("time","type","vehicleType","vehicle","secondaryFuelLevel",
                                        "primaryFuelLevel","driver","mode","seatingCapacity","startX",
                                        "startY", "endX", "endY", "capacity", "arrivalTime", "departureTime",
                                        "secondaryFuel", "secondaryFuelType", "primaryFuelType",
                                        "numPassengers", "length", "primaryFuel")]
 freight_pt <- pt[startsWith(vehicle,"freight")]
-nrow(freight_pt)
+
+## DEBUG
+# nrow(freight_pt)
+# nrow(freight_pt[grepl("-emergency-",vehicle)])
+# all_pt_x <- data.table::as.data.table(rbind(b2b_pt,b2c_pt)[,c("time","vehicle","departureTime","arrivalTime","label")])
+# all_pt_1 <- all_pt_x[,-c("arrivalTime")][order(time),`:=`(IDX = 1:.N),by=vehicle]
+# all_pt_2 <- all_pt_x[,-c("departureTime")][order(time),`:=`(IDX = 1+1:.N),by=vehicle]
+# all_pt <- all_pt_1[all_pt_2, on=c("vehicle", "IDX", "label")][!is.na(arrivalTime)&!is.na(departureTime)]
+# all_pt[,`:=`(stopDuration = departureTime - arrivalTime)]
+# all_pt[,.(mean(stopDuration)),by=.(label)]
+# unloading <- events[actType=="Unloading"]
+# warehouse <- events[actType=="Warehouse"]
+# nrow(unloading[type=="actstart"])
+# nrow(unloading[type=="actend"])
+# nrow(warehouse[type=="actstart"])
+# nrow(warehouse[type=="actend"])
+##
+
+## MODE SPLIT
 # pt$mode2 <- "Transit"
 # pt[mode=="car"]$mode2 <- "Car"
 # pt[mode=="car"&startsWith(vehicle,"rideHailVehicle")]$mode2 <- "Ridehail"
@@ -34,76 +55,41 @@ nrow(freight_pt)
 # pt[mode=="walk"]$mode2 <- "Walk"
 # pt[mode=="bike"]$mode2 <- "Bike"
 # summary <- pt[,.(VMTInMiles=mean(length)/1609.34,fuelInKW=(mean(primaryFuel+secondaryFuel))*2.77778e-7),by=.(mode2)]
-# 
-
-
-factor.remap <- c('Walk'='Walk','Bike'='Bike','Ridehail'='Ridehail','Car'='Car','Transit'='Public Transit', 'Freight'='Freight')
-factor.colors <- c('Walk'='#669900','Bike'='#FFB164','Ridehail'='#B30C0C','Car'='#8A8A8A','Transit'='#0066CC','Freight'="#660099")
-factor.colors.remapped <- factor.colors
-ggplot(summary, aes(x="",y=VMTInMiles,fill=mode2))+
-  geom_bar(stat='identity')+
-  labs(y='',x='Scenario',fill='Mode',title='Mobility Metrics')+
-  theme_marain()+
-  theme(axis.text.x = element_text(angle = 0, hjust=0.5),strip.text = element_text(size=rel(1.2)))+
-  scale_fill_manual(values = factor.colors.remapped)
-
-
-source("~/Documents/Workspace/scripts/common/keys.R")
-register_google(key = google_api_key_1)
-oaklandMap <- ggmap::get_googlemap("oakland california", zoom = 13, maptype = "roadmap")
-shpFile <- pp(activitySimDir, "/__San_Francisco_Bay_Region_2010_Census_Block_Groups-shp/641aa0d4-ce5b-4a81-9c30-8790c4ab8cfb202047-1-wkkklf.j5ouj.shp")
-sfBayCbg <- st_read(shpFile)
-# ggplot(data = oaklandCbg) + geom_sf()+
-#   coord_sf( xlim = c(-130, -60),ylim = c(20, 50))
-
-
-
-freight_pt[,hour:=as.integer(arrivalTime/3600)%%24]
-##1
-# freight_pt_withCBG <- data.table::as.data.table(st_intersection(freight_pt_asSf,sfBayCbg))
-# freight_pt_summary <- freight_pt_withCBG[,.(count=.N),by=.(blkgrpid)]
-# freight_pt_withCBG_asSf <- st_join(sfBayCbg, freight_pt_asSf)
-# data <- freight_pt_withCBG_asSf %>%
-#   group_by(blkgrpid) %>% 
-#   summarize(geometry = st_union(geometry))
-# ggplot() + 
-#   geom_sf(data = data, aes(fill = blkgrpid)) +
-#   theme(legend.position = "none")
-
-##2
+# factor.remap <- c('Walk'='Walk','Bike'='Bike','Ridehail'='Ridehail','Car'='Car','Transit'='Public Transit', 'Freight'='Freight')
+# factor.colors <- c('Walk'='#669900','Bike'='#FFB164','Ridehail'='#B30C0C','Car'='#8A8A8A','Transit'='#0066CC','Freight'="#660099")
+# factor.colors.remapped <- factor.colors
+# ggplot(summary, aes(x="",y=VMTInMiles,fill=mode2))+
+#   geom_bar(stat='identity')+
+#   labs(y='',x='Scenario',fill='Mode',title='Mobility Metrics')+
+#   theme_marain()+
+#   theme(axis.text.x = element_text(angle = 0, hjust=0.5),strip.text = element_text(size=rel(1.2)))+
+#   scale_fill_manual(values = factor.colors.remapped)
+
+
+# ***************************
+# B2B vs B2C
+# ***************************
 #countyNames <- c('Alameda County','Contra Costa County','Marin County','Napa County','Santa Clara County','San Francisco County','San Mateo County','Sonoma County','Solano County')
 source("~/Documents/Workspace/scripts/common/keys.R")
 register_google(key = google_api_key_1)
 oakland_map <- ggmap::get_googlemap("alameda california", zoom = 9, maptype = "roadmap",color = "bw", style = c(feature = "all", element = "labels", visibility = "off"))
 
-clusteringFreightBy <- function(data,cols,dataCbg,numClusters,labelData) {
-  data_asSf <- st_as_sf(data,coords=cols,crs=4326,agr="constant")
-  data_withCBG_asSf <- st_intersection(data_asSf,dataCbg)
-  data_withCBG_asSf$X <- st_coordinates(data_withCBG_asSf$geometry)[,1]
-  data_withCBG_asSf$Y <- st_coordinates(data_withCBG_asSf$geometry)[,2]
-  data_withCBG <- data.table::as.data.table(data_withCBG_asSf)
-  data_withCBG[,cluster:=kmeans(data_withCBG[,.(X,Y)],numClusters)$cluster]
-  result <- data_withCBG[,.(count=.N,x2=mean(X),y2=mean(Y)),by=.(hour,cluster)]
-  result$label <- labelData
-  result
-}
-
 b2b_pt <- freight_pt[grepl("b2b",vehicle)][,label:="B2B"]
 b2c_pt <- freight_pt[grepl("b2c",vehicle)][,label:="B2C"]
 
 b2b_pt_stops <- clusteringFreightBy(b2b_pt,c("endX","endY"),sfBayCbg,50,"B2B")
 b2c_pt_stops <- clusteringFreightBy(b2c_pt,c("endX","endY"),sfBayCbg,50,"B2C")
 
-# Plot it
+## FREIGHT B2C GEO DISTRIBUTION OF STOPS
 to_plot <- rbind(b2c_pt_stops)
-hours_to_show <- c(8, 12, 16)
+hours_to_show <- c(8, 14, 20)
 toplot <- to_plot[hour %in% hours_to_show]
 toplot$hour.label <- ""
 toplot[hour==8]$hour.label <- "8am"
-toplot[hour==12]$hour.label <- "12pm"
-toplot[hour==16]$hour.label <- "4pm"
+toplot[hour==14]$hour.label <- "2pm"
+toplot[hour==20]$hour.label <- "8pm"
 #toplot[hour==20]$hour.label <- "8pm"
-hour.label_order <- c("8am", "12pm", "4pm")
+hour.label_order <- c("8am", "2pm", "8pm")
 # counties <- data.table(urbnmapr::counties)[county_name%in%countyNames]
 #,xlim=c(-122.36,-122.20),ylim=c(37.70,37.81)
 p <- oakland_map %>% 
@@ -123,9 +109,10 @@ p <- oakland_map %>%
         strip.text.x = element_text(size = 10)) +
   facet_wrap(~factor(hour.label, levels=hour.label_order)) +
   guides(color= guide_legend(), size=guide_legend())
-ggsave(pp(workDir,'/b2c-stops.png'),p,width=9,height=5,units='in')
+ggsave(pp(freightWorkDir,'/b2c-stops.png'),p,width=9,height=5,units='in')
 
 
+## FREIGHT B2B GEO DISTRIBUTION OF STOPS
 toplot <- rbind(b2b_pt_stops)
 p <- oakland_map %>% 
   ggmap() +
@@ -142,7 +129,7 @@ p <- oakland_map %>%
         axis.ticks.x = element_blank(),
         axis.ticks.y = element_blank(),
         strip.text.x = element_text(size = 10))
-ggsave(pp(workDir,'/b2b-stops.png'),p,width=4,height=5,units='in')
+ggsave(pp(freightWorkDir,'/b2b-stops.png'),p,width=4,height=5,units='in')
 
 
 to_plot <- rbind(b2c_pt_stops,b2b_pt_stops)[,.(stopsPerHour=sum(cluster)),by=.(label,hour)]
@@ -152,8 +139,9 @@ to_plot <- rbind(b2b_pt,b2c_pt)
 to_plot$timeBin <- as.integer(to_plot$time/1800)
 
 
+## FREIGHT ACTIVITY
 to_plot <- rbind(b2b_pt,b2c_pt)
-p <- to_plot[,.N,by=.(timeBin=as.POSIXct(cut(toDateTime(arrivalTime),"15 min")), label)] %>% 
+p <- to_plot[,time24:=arrivalTime%%(24*3600),][,.N,by=.(timeBin=as.POSIXct(cut(toDateTime(time24),"30 min")), label)] %>% 
   ggplot(aes(timeBin, N, colour=label)) +
   geom_line() + 
   scale_x_datetime("Hour", 
@@ -165,12 +153,10 @@ p <- to_plot[,.N,by=.(timeBin=as.POSIXct(cut(toDateTime(arrivalTime),"15 min")),
   theme(legend.title = element_text(size = 10),
         legend.text = element_text(size = 10),
         axis.text.x = element_text(angle = 0, hjust = 1))
-ggsave(pp(workDir,'/freight-activity.png'),p,width=6,height=3,units='in')
-
+ggsave(pp(freightWorkDir,'/freight-activity.png'),p,width=6,height=3,units='in')
 
+## FREIGHT AVG VMT
 to_plot <- rbind(b2b_pt,b2c_pt)[,.(VMT=mean(length)/1609.3),by=.(label)]
-ggplot(to_plot, )
-
 p <- ggplot(to_plot, aes(x=label,y=VMT,fill=label))+
   geom_bar(stat='identity')+
   labs(y='Miles',x='',title='Avg VMT')+
@@ -180,26 +166,159 @@ p <- ggplot(to_plot, aes(x=label,y=VMT,fill=label))+
         axis.text.x = element_blank(),
         legend.title = element_text(size = 10),
         legend.text = element_text(size = 10))
-ggsave(pp(workDir,'/freight-avg-vmt.png'),p,width=4,height=3,units='in')
+ggsave(pp(freightWorkDir,'/freight-avg-vmt.png'),p,width=4,height=3,units='in')
 
 
-to_plot <- rbind(b2b_pt,b2c_pt)[,.(VMT=mean(primaryFuel+secondaryFuel)*2.77778e-7),by=.(label)]
+# ***************************
+# LDT vs HDT
+# ***************************
+ldt_pt <- freight_pt[vehicleType == "FREIGHT-1"][,category:="LightDutyTruck"]
+hdt_pt <- freight_pt[vehicleType == "FREIGHT-2"][,category:="HeavyDutyTruck"]
 
-all_pt_x <- data.table::as.data.table(rbind(b2b_pt,b2c_pt)[,c("time","vehicle","departureTime","arrivalTime","label")])
-all_pt_1 <- all_pt_x[,-c("arrivalTime")][order(time),`:=`(IDX = 1:.N),by=vehicle]
-all_pt_2 <- all_pt_x[,-c("departureTime")][order(time),`:=`(IDX = 1+1:.N),by=vehicle]
-
-all_pt <- all_pt_1[all_pt_2, on=c("vehicle", "IDX", "label")][!is.na(arrivalTime)&!is.na(departureTime)]
-all_pt[,`:=`(stopDuration = departureTime - arrivalTime)]
-all_pt[,.(mean(stopDuration)),by=.(label)]
+## FREIGHT ACTIVITY BY TRUCK CATEGORY
+to_plot <- rbind(ldt_pt,hdt_pt)
+p <- to_plot[,time24:=arrivalTime%%(24*3600),][,.N,by=.(timeBin=as.POSIXct(cut(toDateTime(time24),"30 min")), category)] %>% 
+  ggplot(aes(timeBin, N, colour=category)) +
+  geom_line() + 
+  scale_x_datetime("Hour", 
+                   breaks=scales::date_breaks("2 hour"), 
+                   labels=scales::date_format("%H", tz = dateTZ)) +
+  scale_y_continuous("Activity", breaks = scales::pretty_breaks()) +
+  scale_colour_manual("Vehicle Category", values = c("#eca35b", "#20b2aa")) +
+  theme_marain() +
+  theme(legend.title = element_text(size = 10),
+        legend.text = element_text(size = 10),
+        axis.text.x = element_text(angle = 0, hjust = 1))
+ggsave(pp(freightWorkDir,'/freight-activity-by-category.png'),p,width=6,height=3,units='in')
 
-all_pt_1 <- all_pt[order(time),`:=`(IDX = 1:.N),by=vehicle]
-all_pt_2 <- all_pt[order(time),`:=`(IDX = 1:.N),by=vehicle]
+## FREIGHT AVG VMT BY TRUCK CATEGORY
+to_plot <- rbind(ldt_pt,hdt_pt)[,.(VMT=mean(length)/1609.3),by=.(category)]
+p <- ggplot(to_plot, aes(x=category,y=VMT,fill=category))+
+  geom_bar(stat='identity')+
+  labs(y='Miles',x='',title='Avg VMT')+
+  scale_fill_manual("Vehicle Category", values = c("#eca35b", "#20b2aa")) +
+  theme_marain()+
+  theme(strip.text = element_text(size = 10),
+        axis.text.x = element_blank(),
+        legend.title = element_text(size = 10),
+        legend.text = element_text(size = 10))
+ggsave(pp(freightWorkDir,'/freight-avg-vmt-by-category.png'),p,width=4,height=3,units='in')
 
-all_pt[,,by=.(vehicle)]
 
-all_pt_x[vehicle=="freight-vehicle-freightVehicle-b2b-1640"]
+# ***************************
+# FRISM
+# ***************************
+# freightWorkDir <- normalizePath(paste(freightDir,"/Outputs_All_SF_0223_2022_merged",sep=""))
+# carriers <- readCsv(pp(freightWorkDir, "/freight-merged-carriers.csv"))
+# payload <- readCsv(pp(freightWorkDir, "/freight-merged-payload-plans.csv"))
+# trous <- readCsv(pp(freightWorkDir, "/freight-merged-tours.csv"))
+# 
+# plans <- rbind(plansb2b,plansb2c)[,c("payloadId","sequenceRank","tourId","payloadType","estimatedTimeOfArrivalInSec","arrivalTimeWindowInSec_lower","business")]
+# tours <- rbind(toursb2b,toursb2c)[,c("tourId","departureTimeInSec","business")]
+# 
+# p <- plans[tours, on=c("tourId","business")]
+# pRank2 <- p[sequenceRank==2]
+# pRank2$negativeTravelTime <- pRank2$estimatedTimeOfArrivalInSec - pRank2$departureTimeInSec - pRank2$arrivalTimeWindowInSec_lower
+# p2 <- p[,diff:=estimatedTimeOfArrivalInSec-arrivalTimeWindowInSec_lower-departureTimeInSec]
+# 
+# write.csv(
+#   pRank2[negativeTravelTime<0],
+#   file = pp(freightDir, "/10percent/negativeTravelTime.csv"),
+#   row.names=FALSE,
+#   quote=FALSE,
+#   na="0")
+# 
+# toursb2$hour <- toursb2c$departureTimeInSec/3600.0
+# ggplot(toursb2b[,.N,by=hour]) + geom_line(aes(hour, N))
+# 
+# payload$business <- "B2B"
+# payload[startsWith(payloadId,"b2c")]$business <- "B2C"
+# payload[,time24:=estimatedTimeOfArrivalInSec%%(24*3600),][,.N,by=.(timeBin=as.POSIXct(cut(toDateTime(time24),"30 min")), business)] %>% 
+#   ggplot(aes(timeBin, N, colour=business)) +
+#   geom_line() + 
+#   scale_x_datetime("Hour", 
+#                    breaks=scales::date_breaks("2 hour"), 
+#                    labels=scales::date_format("%H", tz = dateTZ)) +
+#   scale_y_continuous("Activity", breaks = scales::pretty_breaks()) +
+#   scale_colour_manual("Supply Chain", values = c("#eca35b", "#20b2aa")) +
+#   theme_marain() +
+#   theme(legend.title = element_text(size = 10),
+#         legend.text = element_text(size = 10),
+#         axis.text.x = element_text(angle = 0, hjust = 1))
+
+
+################ ***************************
+################ validation
+################ ***************************
+
+##### PREPARING NETWORK AND MATCH IT WITH POSTMILE AND TRUCK AADTT DATA
+#"primary","secondary","tertiary"
+network <- readCsv(normalizePath(paste(freightDir,"/validation/network.csv.gz",sep="")))
+network_cleaned <- network[
+  linkModes %in% c("car;bike", "car;walk;bike") & attributeOrigType %in% c("motorway","trunk","primary", "secondary")][
+    ,-c("numberOfLanes", "attributeOrigId", "fromNodeId", "toNodeId", "toLocationX", "toLocationY")]
+counties <- data.table::data.table(
+  COUNTY = c("Alameda", "Contra Costa", "Marin", "Napa", "Santa Clara", 
+             "San Francisco", "San Mateo", "Solano", "Sonoma"),
+  CNTY=c("ALA", "CC", "MRN", "NAP", "SCL", "SF", "SM", "SOL", "SON")
+)
+
+data.table::fwrite(network_cleaned, pp(freightDir,"/validation/network_cleaned.csv"), quote=F)
+
+# truck_aadtt_2017 <- readCsv(normalizePath(paste(freightDir,"/validation/2017_truck_aadtt.csv",sep="")))
+# truck_aadtt_2017_sfbay <- assignPostMilesGeometries(truck_aadtt_2017[counties, on=c("CNTY"="CNTY")],
+#                           pp(freightDir, "/validation/ds1901_shp/ds1901.shp"))
+truck_aadtt_2017_sfbay <- data.table::fread(
+  normalizePath(paste(freightDir,"/validation/2017_truck_aadtt_geocoded.csv",sep="")), 
+  header=T, 
+  sep=",")
+
+truck_aadtt_with_linkId <- assignLinkIdToTruckAADTT(network_cleaned, 26910, truck_aadtt_2017_sfbay, 500, 2)
+truck_aadtt_with_linkData <- merge(data.table::as.data.table(truck_aadtt_with_linkId), network_cleaned, by="linkId")
+
+
+## ********************
+linkstats_noFreight <- readCsv(normalizePath(paste(freightDir,"/validation/beam/0.linkstats.nofreight.csv.gz",sep="")))
+linkstats_wFreight <- readCsv(normalizePath(paste(freightDir,"/validation/beam/0.linkstats.withfreight.csv.gz",sep="")))
+#totVolume <- sum(linkstats_wFreight$volume) - sum(linkstats_noFreight$volume)
+#totAADTT <- sum(truck_aadtt_2017_bay_area$TRUCK_AADT)
+lsWFreight <- linkstats_wFreight[,.(volumeWithFreight=sum(volume)),by=.(link)]
+lsNoFreight <- linkstats_noFreight[,.(volumeNoFreight=sum(volume)),by=.(link)]
+linkStats <- lsWFreight[lsNoFreight, on=c("link")]
+linkStats$truck_volume <- linkStats$volumeWithFreight - linkStats$volumeNoFreight
+linkStats$vehicle_volume <- linkStats$volumeWithFreight
+linkStats$truck_share <- linkStats$truck_volume/linkStats$vehicle_volume
+linkStats <- linkStats[,-c("volumeWithFreight","volumeNoFreight")]
+linkStats[is.na(truck_share)]$truck_share <- 0.0
+linkStats[is.infinite(truck_share)]$truck_share <- 0.0
+LinkStatsWithLocation <- linkStats[network_cleaned, on=c("link"="linkId")]
+
+LinkStats_as_sf <- st_transform(st_as_sf(
+  LinkStatsWithLocation,
+  coords = c("fromLocationX", "fromLocationY"),
+  crs = 26910,
+  agr = "constant"), 4326)
+
+
+LinkStats_withTaz <- data.table::as.data.table(st_intersection(LinkStats_as_sf, st_buffer(sfBayTAZs, 0)))
+LinkStats_withTaz <- LinkStats_withTaz[counties, on="county"]
+
+countyStats <- LinkStats_withTaz[,.(truck_volume=sum(truck_volume),vehicle_volume=sum(vehicle_volume)),by=.(county)][counties, on="county"]
+unique_counties <- unique(countyStats$cnty)
+
+SF_truck_aadtt_2017 <- truck_aadtt_2017[CNTY %in% unique_counties]
+dist_truck_aadtt_2017 <- SF_truck_aadtt_2017[,.(TRUCK_AADT=mean(TRUCK_AADT),VEHICLE_AADT=mean(VEHICLE_AADT)),by=.(DIST)]
+
+county_truck_aadtt_2017 <- SF_truck_aadtt_2017[,.(TRUCK_AADT=sum(TRUCK_AADT),VEHICLE_AADT=sum(VEHICLE_AADT)),by=.(CNTY)]
+
+truckBEAM_truckAADTT <- countyStats[county_truck_aadtt_2017, on=c("cnty"="CNTY")]
+
+truckBEAM_truckAADTT %>% ggplot(aes(county)) +
+  geom_bar(truck_volume, )
 
+source("~/Documents/Workspace/scripts/common/keys.R")
+register_google(key = google_api_key_1)
+sfBayTAZs <- st_read(pp(freightDir, "/validation/TAZs/Transportation_Analysis_Zones.shp"))
 
 
 

From 9bfb8f3ba8253753425924a56824682c03ada235 Mon Sep 17 00:00:00 2001
From: Haitam Laarabi <10712736+htmlrb@users.noreply.github.com>
Date: Tue, 29 Mar 2022 21:39:10 -0700
Subject: [PATCH 28/58] more scripts

---
 src/main/R/freight/freight-processing.R | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/src/main/R/freight/freight-processing.R b/src/main/R/freight/freight-processing.R
index 86bfcf13f4e..46f4998019a 100644
--- a/src/main/R/freight/freight-processing.R
+++ b/src/main/R/freight/freight-processing.R
@@ -277,7 +277,7 @@ truck_aadtt_with_linkId <- assignLinkIdToTruckAADTT(network_cleaned, 26910, truc
 truck_aadtt_with_linkData <- merge(data.table::as.data.table(truck_aadtt_with_linkId), network_cleaned, by="linkId")
 
 
-## ********************
+##### PREPARING BEAM/LINKSTAT DATA
 linkstats_noFreight <- readCsv(normalizePath(paste(freightDir,"/validation/beam/0.linkstats.nofreight.csv.gz",sep="")))
 linkstats_wFreight <- readCsv(normalizePath(paste(freightDir,"/validation/beam/0.linkstats.withfreight.csv.gz",sep="")))
 #totVolume <- sum(linkstats_wFreight$volume) - sum(linkstats_noFreight$volume)
@@ -291,6 +291,11 @@ linkStats$truck_share <- linkStats$truck_volume/linkStats$vehicle_volume
 linkStats <- linkStats[,-c("volumeWithFreight","volumeNoFreight")]
 linkStats[is.na(truck_share)]$truck_share <- 0.0
 linkStats[is.infinite(truck_share)]$truck_share <- 0.0
+
+
+##### MERGING
+truck_aadtt_with_linkStats <- merge(truck_aadtt_with_linkData, linkStats, by.x="linkId", by.y="link")
+
 LinkStatsWithLocation <- linkStats[network_cleaned, on=c("link"="linkId")]
 
 LinkStats_as_sf <- st_transform(st_as_sf(

From 767e49680b49ddbb6eadea478130f98a32e9d4b2 Mon Sep 17 00:00:00 2001
From: Dmitry Openkov <dimaopen@gmail.com>
Date: Thu, 31 Mar 2022 19:40:55 +0300
Subject: [PATCH 29/58] Producing missed parking skimmer event

---
 .../scala/beam/agentsim/agents/household/HouseholdActor.scala   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/scala/beam/agentsim/agents/household/HouseholdActor.scala b/src/main/scala/beam/agentsim/agents/household/HouseholdActor.scala
index 1c25958ce19..c28a256c34d 100755
--- a/src/main/scala/beam/agentsim/agents/household/HouseholdActor.scala
+++ b/src/main/scala/beam/agentsim/agents/household/HouseholdActor.scala
@@ -209,7 +209,7 @@ object HouseholdActor {
 
         //We should create a vehicle manager for cars and bikes for all households in case they are generated during the simulation
 
-        val vehiclesByAllCategories = List(Car, Bike)
+        val vehiclesByAllCategories = (List(Car, Bike) ++ vehiclesByCategory.keys).distinct
           .map(cat => cat -> vehiclesByCategory.getOrElse(cat, Map[Id[BeamVehicle], BeamVehicle]()))
           .toMap
         val fleetManagers = vehiclesByAllCategories.map { case (category, vehicleMap) =>

From 93f677c0ad51fe7703f231de50ea3cacfd176c81 Mon Sep 17 00:00:00 2001
From: Dmitry Openkov <dimaopen@gmail.com>
Date: Thu, 31 Mar 2022 19:41:13 +0300
Subject: [PATCH 30/58] Added Truck volumes to linkstats

---
 build.gradle                                  |  2 +-
 .../physsim/PhyssimCalcLinkStats.java         | 37 ++++----
 .../AgentSimToPhysSimPlanConverter.java       |  7 +-
 .../java/beam/utils/VolumesAnalyzerFixed.java | 47 +++++++++-
 .../LinkStatsWithVehicleCategory.scala        | 88 +++++++++++++++++++
 .../beam/physsim/jdeqsim/JDEQSimRunner.scala  |  3 +-
 .../physsim/PhyssimCalcLinkStatsTest.java     |  2 +-
 .../physsim/BeamCalcLinkStatsSpec.scala       |  2 +-
 test/input/sf-light/freight/freight-tours.csv |  4 +-
 9 files changed, 164 insertions(+), 28 deletions(-)
 create mode 100644 src/main/scala/beam/physsim/analysis/LinkStatsWithVehicleCategory.scala

diff --git a/build.gradle b/build.gradle
index f0199365510..248bf1880b3 100755
--- a/build.gradle
+++ b/build.gradle
@@ -136,7 +136,7 @@ configurations.all {
 
 dependencies {
 
-    implementation(group: 'com.github.LBNL-UCB-STI', name: 'beam-utilities', version: 'v0.2.15') {
+    implementation(group: 'com.github.LBNL-UCB-STI', name: 'beam-utilities', version: 'v0.2.17') {
         exclude group: 'com.github.LBNL-UCB-STI', module: 'r5'
         exclude group: 'org.matsim', module: 'matsim'
     }
diff --git a/src/main/java/beam/analysis/physsim/PhyssimCalcLinkStats.java b/src/main/java/beam/analysis/physsim/PhyssimCalcLinkStats.java
index 117ac09d524..89408d65a7a 100755
--- a/src/main/java/beam/analysis/physsim/PhyssimCalcLinkStats.java
+++ b/src/main/java/beam/analysis/physsim/PhyssimCalcLinkStats.java
@@ -1,10 +1,11 @@
 package beam.analysis.physsim;
 
+import beam.agentsim.agents.vehicles.BeamVehicle;
 import beam.analysis.plots.GraphUtils;
+import beam.physsim.analysis.LinkStatsWithVehicleCategory;
 import beam.sim.BeamConfigChangesObservable;
 import beam.sim.BeamConfigChangesObserver;
 import beam.sim.config.BeamConfig;
-import beam.utils.BeamCalcLinkStats;
 import beam.utils.VolumesAnalyzerFixed;
 import org.jfree.chart.*;
 import org.jfree.chart.plot.CategoryPlot;
@@ -12,6 +13,7 @@
 import org.jfree.data.category.CategoryDataset;
 import org.jfree.data.general.DatasetUtilities;
 import org.matsim.analysis.VolumesAnalyzer;
+import org.matsim.api.core.v01.Id;
 import org.matsim.api.core.v01.network.Link;
 import org.matsim.api.core.v01.network.Network;
 import org.matsim.core.api.experimental.events.EventsManager;
@@ -21,7 +23,7 @@
 import org.matsim.core.utils.misc.Time;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
-import scala.Tuple2;
+
 import java.awt.*;
 import java.io.IOException;
 import java.util.*;
@@ -56,14 +58,18 @@ public class PhyssimCalcLinkStats implements BeamConfigChangesObserver {
     private BeamConfig beamConfig;
     private final Network network;
     private final OutputDirectoryHierarchy controllerIO;
-    private final BeamCalcLinkStats linkStats;
     private VolumesAnalyzer volumes;
+    private TravelTimeCalculatorConfigGroup ttcConfigGroup;
+    private Map<Id<BeamVehicle>, BeamVehicle> vehicleMap;
 
     public PhyssimCalcLinkStats(Network network, OutputDirectoryHierarchy controlerIO, BeamConfig beamConfig,
-                                TravelTimeCalculatorConfigGroup ttcConfigGroup, BeamConfigChangesObservable beamConfigChangesObservable) {
+                                TravelTimeCalculatorConfigGroup ttcConfigGroup, BeamConfigChangesObservable beamConfigChangesObservable,
+                                Map<Id<BeamVehicle>, BeamVehicle> vehicleMap) {
         this.network = network;
         this.controllerIO = controlerIO;
         this.beamConfig = beamConfig;
+        this.ttcConfigGroup = ttcConfigGroup;
+        this.vehicleMap = vehicleMap;
 
         if (isNotTestMode()) {
             binSize = this.beamConfig.beam().physsim().linkStatsBinSize();
@@ -76,15 +82,15 @@ public PhyssimCalcLinkStats(Network network, OutputDirectoryHierarchy controlerI
         }
         beamConfigChangesObservable.addObserver(this);
 
-        linkStats = new BeamCalcLinkStats(network, ttcConfigGroup);
     }
 
     public void notifyIterationEnds(int iteration, TravelTime travelTime) {
-        linkStats.addData(volumes, travelTime);
         processData(iteration, travelTime);
         if (this.controllerIO != null) {
             if (isNotTestMode() && writeLinkStats(iteration)) {
-                linkStats.writeFile(this.controllerIO.getIterationFilename(iteration, "linkstats_unmodified.csv.gz"));
+                String filePath = this.controllerIO.getIterationFilename(iteration, "linkstats_unmodified.csv.gz");
+                LinkStatsWithVehicleCategory linkStats = new LinkStatsWithVehicleCategory(network, ttcConfigGroup);
+                linkStats.writeLinkStatsWithTruckVolumes(volumes, travelTime, filePath);
             }
             if (beamConfig.beam().outputs().writeGraphs()) {
                 CategoryDataset dataset = buildAndGetGraphCategoryDataset();
@@ -125,7 +131,7 @@ private void processData(int iteration, TravelTime travelTime) {
 
                 double averageSpeedToFreeSpeedRatio = averageSpeed / freeSpeed;
 
-                double relativeSpeed = Math.max((Math.round(averageSpeedToFreeSpeedRatio * 50.0) / 10),minSpeed);
+                double relativeSpeed = Math.max((Math.round(averageSpeedToFreeSpeedRatio * 50.0) / 10), minSpeed);
 
                 Map<Integer, Integer> hoursDataMap = relativeSpeedFrequenciesPerBin.get(relativeSpeed);
 
@@ -175,13 +181,13 @@ private double[][] buildModesFrequencyDataset() {
 
         Optional<Double> optionalMaxRelativeSpeedsCategories = relativeSpeedsCategoriesList.stream().max(Comparator.naturalOrder());
 
-        if(optionalMaxRelativeSpeedsCategories.isPresent()) {
+        if (optionalMaxRelativeSpeedsCategories.isPresent()) {
             int maxRelativeSpeedsCategories = optionalMaxRelativeSpeedsCategories.get().intValue();
-            dataset = new double[maxRelativeSpeedsCategories+1][noOfBins];
+            dataset = new double[maxRelativeSpeedsCategories + 1][noOfBins];
 
             for (int i = 0; i <= maxRelativeSpeedsCategories; i++) {
 
-                Map<Integer, Integer> relativeSpeedBins = relativeSpeedFrequenciesPerBin.getOrDefault((double)i, new HashMap<>());
+                Map<Integer, Integer> relativeSpeedBins = relativeSpeedFrequenciesPerBin.getOrDefault((double) i, new HashMap<>());
 
                 double[] relativeSpeedFrequencyPerHour = new double[noOfBins];
                 int index = 0;
@@ -218,7 +224,7 @@ private void createModesFrequencyGraph(CategoryDataset dataset, int iterationNum
 
         int max = Collections.max(relativeSpeedsCategoriesList).intValue();
 
-        for (int i = 0; i <= max ; i++) {
+        for (int i = 0; i <= max; i++) {
             legendItems.add(new LegendItem(String.valueOf(i), getColor(i)));
             plot.getRenderer().setSeriesPaint(i, getColor(i));
         }
@@ -250,16 +256,11 @@ private Color getRandomColor() {
     }
 
     public void notifyIterationStarts(EventsManager eventsManager, TravelTimeCalculatorConfigGroup travelTimeCalculatorConfigGroup) {
-        this.linkStats.reset();
-        volumes = new VolumesAnalyzerFixed(3600, travelTimeCalculatorConfigGroup.getMaxTime() - 1, network);
+        volumes = new VolumesAnalyzerFixed(3600, travelTimeCalculatorConfigGroup.getMaxTime() - 1, network, vehicleMap);
         eventsManager.addHandler(volumes);
         this.relativeSpeedFrequenciesPerBin.clear();
     }
 
-    public void clean(){
-        this.linkStats.reset();
-    }
-
     @Override
     public void update(BeamConfigChangesObservable observable, BeamConfig updatedBeamConfig) {
         this.beamConfig = updatedBeamConfig;
diff --git a/src/main/java/beam/physsim/jdeqsim/AgentSimToPhysSimPlanConverter.java b/src/main/java/beam/physsim/jdeqsim/AgentSimToPhysSimPlanConverter.java
index 4932055e182..c02dc705ab2 100755
--- a/src/main/java/beam/physsim/jdeqsim/AgentSimToPhysSimPlanConverter.java
+++ b/src/main/java/beam/physsim/jdeqsim/AgentSimToPhysSimPlanConverter.java
@@ -10,6 +10,7 @@
 import beam.analysis.physsim.PhyssimNetworkComparisonEuclideanVsLengthAttribute;
 import beam.analysis.physsim.PhyssimNetworkLinkLengthDistribution;
 import beam.calibration.impl.example.CountsObjectiveFunction;
+import beam.physsim.analysis.LinkStatsWithVehicleCategory;
 import beam.physsim.cchRoutingAssignment.OsmInfoHolder;
 import beam.physsim.cchRoutingAssignment.RoutingFrameworkTravelTimeCalculator;
 import beam.physsim.cchRoutingAssignment.RoutingFrameworkWrapperImpl;
@@ -315,9 +316,9 @@ private void writeTravelTime(TravelTime travelTimeForR5, VolumesAnalyzer volumes
         int endTimeInSeconds = (int) Time.parseTime(beamConfig.beam().agentsim().endTime());
         cfg.setMaxTime(endTimeInSeconds);
         Network network = agentSimScenario.getNetwork();
-        BeamCalcLinkStats linkStats = new BeamCalcLinkStats(network, cfg);
-        linkStats.addData(volumesAnalyzer, travelTimeForR5);
-        linkStats.writeFile(controlerIO.getIterationFilename(iterationEndsEvent.getIteration(), "linkstats.csv.gz"));
+        LinkStatsWithVehicleCategory linkStats = new LinkStatsWithVehicleCategory(network, cfg);
+        String filePath = controlerIO.getIterationFilename(iterationEndsEvent.getIteration(), "linkstats.csv.gz");
+        linkStats.writeLinkStatsWithTruckVolumes(volumesAnalyzer, travelTimeForR5, filePath);
     }
 
     private boolean shouldWritePlans(int iterationNumber) {
diff --git a/src/main/java/beam/utils/VolumesAnalyzerFixed.java b/src/main/java/beam/utils/VolumesAnalyzerFixed.java
index f4fbb0eb96c..663c4faf7ef 100644
--- a/src/main/java/beam/utils/VolumesAnalyzerFixed.java
+++ b/src/main/java/beam/utils/VolumesAnalyzerFixed.java
@@ -1,12 +1,15 @@
 package beam.utils;
 
+import beam.agentsim.agents.vehicles.BeamVehicle;
 import org.matsim.analysis.VolumesAnalyzer;
 import org.matsim.api.core.v01.Id;
+import org.matsim.api.core.v01.events.VehicleEntersTrafficEvent;
 import org.matsim.api.core.v01.network.Link;
 import org.matsim.api.core.v01.network.Network;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.util.Map;
 import java.util.concurrent.TimeUnit;
 
 public class VolumesAnalyzerFixed extends VolumesAnalyzer {
@@ -17,14 +20,37 @@ public class VolumesAnalyzerFixed extends VolumesAnalyzer {
     private final int timeBinSizeInSeconds;
     private final int maxSlotIndex;
     private final int maxHour;
+    private Map<Id<BeamVehicle>, BeamVehicle> vehicleMap;
 
-    public VolumesAnalyzerFixed(int timeBinSizeInSeconds, int maxTime, Network network) {
+    public VolumesAnalyzerFixed(int timeBinSizeInSeconds, int maxTime, Network network, Map<Id<BeamVehicle>, BeamVehicle> vehicleMap) {
         super(timeBinSizeInSeconds, maxTime, network);
         this.timeBinSizeInSeconds = timeBinSizeInSeconds;
+        this.vehicleMap = vehicleMap;
         this.maxSlotIndex = (maxTime / timeBinSizeInSeconds) + 1;
         this.maxHour = (int) TimeUnit.SECONDS.toHours(maxTime + 1);
     }
 
+    @Override
+    public void handleEvent(VehicleEntersTrafficEvent event) {
+        if (vehicleMap != null) {
+            BeamVehicle vehicle = vehicleMap.get(event.getVehicleId());
+            String category = vehicleCategory(vehicle);
+
+            VehicleEntersTrafficEvent eventWithMode = new VehicleEntersTrafficEvent(event.getTime(), event.getPersonId(),
+                    event.getLinkId(), event.getVehicleId(), category, event.getRelativePositionOnLink());
+            super.handleEvent(eventWithMode);
+        } else {
+            super.handleEvent(event);
+        }
+    }
+
+    private String vehicleCategory(BeamVehicle vehicle) {
+        if (vehicle == null) {
+            return null;
+        }
+        return vehicle.beamVehicleType().vehicleCategory().toString();
+    }
+
     @Override
     public double[] getVolumesPerHourForLink(final Id<Link> linkId) {
         if (SECONDS_PER_HOUR % timeBinSizeInSeconds != 0) {
@@ -49,4 +75,23 @@ public double[] getVolumesPerHourForLink(final Id<Link> linkId) {
         return volumes;
     }
 
+    public double[] getVolumesPerHourForLink(final Id<Link> linkId, String mode) {
+        if (SECONDS_PER_HOUR % timeBinSizeInSeconds != 0)
+            log.error("Volumes per hour and per link probably not correct! TimeBinSize: " + timeBinSizeInSeconds);
+
+        double[] volumes = new double[maxHour];
+
+        int[] volumesForLink = this.getVolumesForLink(linkId, mode);
+        if (volumesForLink == null) return volumes;
+
+        final int slotsPerHour = SECONDS_PER_HOUR / timeBinSizeInSeconds;
+        int slotIndex = 0;
+        for (int hour = 0; hour < maxHour; hour++) {
+            for (int i = 0; i < slotsPerHour; i++) {
+                volumes[hour] += volumesForLink[Math.min(maxSlotIndex, slotIndex++)];
+            }
+        }
+        return volumes;
+    }
+
 }
diff --git a/src/main/scala/beam/physsim/analysis/LinkStatsWithVehicleCategory.scala b/src/main/scala/beam/physsim/analysis/LinkStatsWithVehicleCategory.scala
new file mode 100644
index 00000000000..31f1fa6c8f5
--- /dev/null
+++ b/src/main/scala/beam/physsim/analysis/LinkStatsWithVehicleCategory.scala
@@ -0,0 +1,88 @@
+package beam.physsim.analysis
+
+import beam.utils.BeamCalcLinkStats
+import beam.utils.BeamCalcLinkStats.LinkData
+import beam.utils.csv.CsvWriter
+import org.matsim.analysis.VolumesAnalyzer
+import org.matsim.api.core.v01.Id
+import org.matsim.api.core.v01.network.{Link, Network}
+import org.matsim.core.config.groups.TravelTimeCalculatorConfigGroup
+import org.matsim.core.router.util.TravelTime
+
+import scala.collection.JavaConverters._
+import scala.util.Try
+
+/**
+  * @author Dmitry Openkov
+  */
+class LinkStatsWithVehicleCategory(
+  network: Network,
+  ttConfigGroup: TravelTimeCalculatorConfigGroup
+) {
+
+  def calculateLinkData(
+    volumesAnalyzer: VolumesAnalyzer,
+    travelTimeForR5: TravelTime,
+    categories: IndexedSeq[String]
+  ): (Map[Id[Link], LinkData], Map[String, Map[Id[Link], LinkData]], Int) = {
+    val calc = new BeamCalcLinkStats(network, ttConfigGroup)
+    calc.reset()
+    calc.addData(volumesAnalyzer, travelTimeForR5)
+    val totalLinkData = calc.getLinkData.asScala.toMap
+    val linkData = categories.map { category =>
+      calc.reset()
+      calc.addData(volumesAnalyzer, travelTimeForR5, category)
+      category -> calc.getLinkData.asScala.toMap
+    }.toMap
+    (totalLinkData, linkData, calc.getNofHours)
+  }
+
+  def writeToFile(
+    totalLinkData: Map[Id[Link], BeamCalcLinkStats.LinkData],
+    linkData: Map[String, Map[Id[Link], BeamCalcLinkStats.LinkData]],
+    nofHours: Int,
+    aggregation: Seq[(Seq[String], String)],
+    filePath: String
+  ): Try[Unit] = {
+    val header = Seq("link", "from", "to", "hour", "length", "freespeed", "capacity", "stat", "volume") ++
+      aggregation.map(_._2) :+ "traveltime"
+    val csvWriter = new CsvWriter(filePath, header)
+    val rows = totalLinkData.view.flatMap { case (linkId, data) =>
+      val link = network.getLinks.get(linkId)
+      for {
+        hour <- 0 until nofHours
+      } yield {
+        val categoryVolumes = aggregation.map { case (categories, _) =>
+          categories.map(category => linkData(category).get(linkId).map(_.getSumVolume(hour)).getOrElse(0.0)).sum
+        }
+        Seq(
+          linkId,
+          link.getFromNode.getId,
+          link.getToNode.getId,
+          hour,
+          link.getLength,
+          link.getFreespeed,
+          link.getCapacity,
+          "AVG",
+          totalLinkData.get(linkId).map(_.getSumVolume(hour)).getOrElse(0.0)
+        ) ++ categoryVolumes ++ Seq(data.calculateAverageTravelTime(hour))
+      }
+
+    }
+    csvWriter.writeAllAndClose(rows)
+  }
+
+  def writeLinkStatsWithTruckVolumes(
+    volumesAnalyzer: VolumesAnalyzer,
+    travelTimeForR5: TravelTime,
+    filePath: String
+  ): Try[Unit] = {
+    val categoryMapping = IndexedSeq(
+      Seq("LightDutyTruck", "HeavyDutyTruck") -> "TruckVolume",
+      Seq("HeavyDutyTruck")                   -> "HDTruckVolume"
+    )
+    val (totalLinkData, linkData, nofHours) =
+      calculateLinkData(volumesAnalyzer, travelTimeForR5, categoryMapping.flatMap(_._1))
+    writeToFile(totalLinkData, linkData, nofHours, categoryMapping, filePath)
+  }
+}
diff --git a/src/main/scala/beam/physsim/jdeqsim/JDEQSimRunner.scala b/src/main/scala/beam/physsim/jdeqsim/JDEQSimRunner.scala
index 7a594f4df57..8a8db0352eb 100644
--- a/src/main/scala/beam/physsim/jdeqsim/JDEQSimRunner.scala
+++ b/src/main/scala/beam/physsim/jdeqsim/JDEQSimRunner.scala
@@ -63,7 +63,8 @@ class JDEQSimRunner(
       controlerIO,
       beamServices.beamConfig,
       jdeqSimScenario.getConfig.travelTimeCalculator,
-      beamConfigChangesObservable
+      beamConfigChangesObservable,
+      beamServices.beamScenario.privateVehicles.view.toMap.asJava
     )
     linkStatsGraph.notifyIterationStarts(jdeqsimEvents, jdeqSimScenario.getConfig.travelTimeCalculator)
 
diff --git a/src/test/java/beam/analysis/physsim/PhyssimCalcLinkStatsTest.java b/src/test/java/beam/analysis/physsim/PhyssimCalcLinkStatsTest.java
index 8e9cf361c58..7f2c5b73c4f 100755
--- a/src/test/java/beam/analysis/physsim/PhyssimCalcLinkStatsTest.java
+++ b/src/test/java/beam/analysis/physsim/PhyssimCalcLinkStatsTest.java
@@ -47,7 +47,7 @@ public static void createDummySimWithXML() {
         eventsManager.addHandler(travelTimeCalculator);
 
         BeamConfig beamConfig = BeamConfig.apply(TestConfigUtils.testConfig("test/input/equil-square/equil-0.001k.conf").resolve().withValue("beam.physsim.quick_fix_minCarSpeedInMetersPerSecond", ConfigValueFactory.fromAnyRef(0.0)));
-        physsimCalcLinkStats = new PhyssimCalcLinkStats(network, null,  beamConfig, defaultTravelTimeCalculator, new BeamConfigChangesObservable(beamConfig, Option.empty()) );
+        physsimCalcLinkStats = new PhyssimCalcLinkStats(network, null,  beamConfig, defaultTravelTimeCalculator, new BeamConfigChangesObservable(beamConfig, Option.empty()), null);
 
         //physsimCalcLinkStats = new PhyssimCalcLinkStats(network, null, null);
 
diff --git a/src/test/scala/beam/analysis/physsim/BeamCalcLinkStatsSpec.scala b/src/test/scala/beam/analysis/physsim/BeamCalcLinkStatsSpec.scala
index 8c837094b7c..e4a5cdeb9e0 100644
--- a/src/test/scala/beam/analysis/physsim/BeamCalcLinkStatsSpec.scala
+++ b/src/test/scala/beam/analysis/physsim/BeamCalcLinkStatsSpec.scala
@@ -55,7 +55,7 @@ class BeamCalcLinkStatsSpec extends AnyWordSpecLike with Matchers with BeforeAnd
 
     beamCalcLinkStats = new BeamCalcLinkStats(network, ttccg)
     beamCalcLinkStats.reset()
-    val volumes = new VolumesAnalyzerFixed(3600, ttccg.getMaxTime() - 1, network)
+    val volumes = new VolumesAnalyzerFixed(3600, ttccg.getMaxTime() - 1, network, null)
     events.addHandler(volumes)
 
     val reader = new MatsimEventsReader(events)
diff --git a/test/input/sf-light/freight/freight-tours.csv b/test/input/sf-light/freight/freight-tours.csv
index 8a7cf95ccc6..af9f07c0269 100644
--- a/test/input/sf-light/freight/freight-tours.csv
+++ b/test/input/sf-light/freight/freight-tours.csv
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f6c6a974285e0154ecbb05246c0999c22b85cac9c068c03717e403c6b64a006b
-size 153
+oid sha256:e7be0b20871409f3e90a10e445e374831a4a63366a4a16910cb2739c3677e4dc
+size 161

From d3a58240677809e5d163c4a1c48696cc92448de7 Mon Sep 17 00:00:00 2001
From: Haitam Laarabi <10712736+htmlrb@users.noreply.github.com>
Date: Thu, 31 Mar 2022 17:22:50 -0700
Subject: [PATCH 31/58] test

---
 src/main/R/freight/freight-processing.R | 29 ++++++++++++++++++-------
 1 file changed, 21 insertions(+), 8 deletions(-)

diff --git a/src/main/R/freight/freight-processing.R b/src/main/R/freight/freight-processing.R
index 46f4998019a..2a7b43e89a4 100644
--- a/src/main/R/freight/freight-processing.R
+++ b/src/main/R/freight/freight-processing.R
@@ -14,9 +14,10 @@ library(stringr)
 activitySimDir <- normalizePath("~/Data/ACTIVITYSIM")
 freightDir <- normalizePath("~/Data/FREIGHT")
 freightWorkDir <- normalizePath(paste(freightDir,"/2022-03-09/output",sep=""))
-
 source("~/Documents/Workspace/scripts/common/keys.R")
 register_google(key = google_api_key_1)
+
+
 oaklandMap <- ggmap::get_googlemap("oakland california", zoom = 13, maptype = "roadmap")
 shpFile <- pp(activitySimDir, "/__San_Francisco_Bay_Region_2010_Census_Block_Groups-shp/641aa0d4-ce5b-4a81-9c30-8790c4ab8cfb202047-1-wkkklf.j5ouj.shp")
 sfBayCbg <- st_read(shpFile)
@@ -248,7 +249,7 @@ ggsave(pp(freightWorkDir,'/freight-avg-vmt-by-category.png'),p,width=4,height=3,
 
 
 ################ ***************************
-################ validation
+################ validation CALTrans
 ################ ***************************
 
 ##### PREPARING NETWORK AND MATCH IT WITH POSTMILE AND TRUCK AADTT DATA
@@ -263,7 +264,7 @@ counties <- data.table::data.table(
   CNTY=c("ALA", "CC", "MRN", "NAP", "SCL", "SF", "SM", "SOL", "SON")
 )
 
-data.table::fwrite(network_cleaned, pp(freightDir,"/validation/network_cleaned.csv"), quote=F)
+#data.table::fwrite(network_cleaned, pp(freightDir,"/validation/network_cleaned.csv"), quote=F)
 
 # truck_aadtt_2017 <- readCsv(normalizePath(paste(freightDir,"/validation/2017_truck_aadtt.csv",sep="")))
 # truck_aadtt_2017_sfbay <- assignPostMilesGeometries(truck_aadtt_2017[counties, on=c("CNTY"="CNTY")],
@@ -274,6 +275,7 @@ truck_aadtt_2017_sfbay <- data.table::fread(
   sep=",")
 
 truck_aadtt_with_linkId <- assignLinkIdToTruckAADTT(network_cleaned, 26910, truck_aadtt_2017_sfbay, 500, 2)
+ata.table::fwrite(truck_aadtt_with_linkId, pp(freightDir,"/validation/truck_aadtt_with_linkId.csv"), quote=F)
 truck_aadtt_with_linkData <- merge(data.table::as.data.table(truck_aadtt_with_linkId), network_cleaned, by="linkId")
 
 
@@ -282,7 +284,7 @@ linkstats_noFreight <- readCsv(normalizePath(paste(freightDir,"/validation/beam/
 linkstats_wFreight <- readCsv(normalizePath(paste(freightDir,"/validation/beam/0.linkstats.withfreight.csv.gz",sep="")))
 #totVolume <- sum(linkstats_wFreight$volume) - sum(linkstats_noFreight$volume)
 #totAADTT <- sum(truck_aadtt_2017_bay_area$TRUCK_AADT)
-lsWFreight <- linkstats_wFreight[,.(volumeWithFreight=sum(volume)),by=.(link)]
+lsWFreight <- linkstats_wFreight[,.(volumeWithFreight=sum(volume),lengthInMeter=first(length)),by=.(link)]
 lsNoFreight <- linkstats_noFreight[,.(volumeNoFreight=sum(volume)),by=.(link)]
 linkStats <- lsWFreight[lsNoFreight, on=c("link")]
 linkStats$truck_volume <- linkStats$volumeWithFreight - linkStats$volumeNoFreight
@@ -296,6 +298,9 @@ linkStats[is.infinite(truck_share)]$truck_share <- 0.0
 ##### MERGING
 truck_aadtt_with_linkStats <- merge(truck_aadtt_with_linkData, linkStats, by.x="linkId", by.y="link")
 
+sum(truck_aadtt_with_linkStats$TRUCK_AADT)
+sum(truck_aadtt_with_linkStats$truck_volume)
+
 LinkStatsWithLocation <- linkStats[network_cleaned, on=c("link"="linkId")]
 
 LinkStats_as_sf <- st_transform(st_as_sf(
@@ -321,13 +326,21 @@ truckBEAM_truckAADTT <- countyStats[county_truck_aadtt_2017, on=c("cnty"="CNTY")
 truckBEAM_truckAADTT %>% ggplot(aes(county)) +
   geom_bar(truck_volume, )
 
-source("~/Documents/Workspace/scripts/common/keys.R")
-register_google(key = google_api_key_1)
-sfBayTAZs <- st_read(pp(freightDir, "/validation/TAZs/Transportation_Analysis_Zones.shp"))
-
 
+sfBayTAZs <- st_read(pp(freightDir, "/validation/TAZs/Transportation_Analysis_Zones.shp"))
 
 
+################ ***************************
+################ validation HPMS
+################ ***************************
 
+source("~/Documents/Workspace/scripts/common/keys.R")
+sf_hpms <- st_read(pp(freightDir, "/validation/sf_hpms_inventory_clipped.geojson"))
 
+Volume_beam <- sum(linkStats$truck_volume)
+Volume_hpms <- sum(sf_hpms$AADT_Combi+sf_hpms$AADT_Singl)
+VMT_beam <- sum(linkStats$truck_volume * linkStats$lengthInMeter/1609)
+VMT_hpms <- (sum((sf_hpms$AADT_Combi+sf_hpms$AADT_Singl) * as.numeric(st_length(sf_hpms))/1609))
 
+st_length(sf_hpms[1,])/1609
+sf_hpms[1,]$Shape_Leng

From 3e2084ecaa8ad9933dd6e5cc08d65ad9f7897b44 Mon Sep 17 00:00:00 2001
From: Dmitry Openkov <dimaopen@gmail.com>
Date: Fri, 1 Apr 2022 13:14:06 +0300
Subject: [PATCH 32/58] Save volume column in linkstats as: "all volume" -
 "truck volume"

---
 .../LinkStatsWithVehicleCategory.scala         | 18 ++++++++++++------
 1 file changed, 12 insertions(+), 6 deletions(-)

diff --git a/src/main/scala/beam/physsim/analysis/LinkStatsWithVehicleCategory.scala b/src/main/scala/beam/physsim/analysis/LinkStatsWithVehicleCategory.scala
index 31f1fa6c8f5..bdf29247eff 100644
--- a/src/main/scala/beam/physsim/analysis/LinkStatsWithVehicleCategory.scala
+++ b/src/main/scala/beam/physsim/analysis/LinkStatsWithVehicleCategory.scala
@@ -52,9 +52,15 @@ class LinkStatsWithVehicleCategory(
       for {
         hour <- 0 until nofHours
       } yield {
-        val categoryVolumes = aggregation.map { case (categories, _) =>
-          categories.map(category => linkData(category).get(linkId).map(_.getSumVolume(hour)).getOrElse(0.0)).sum
+        val categories = aggregation.flatMap(_._1).distinct
+        val categoryToVolume = categories.map { category =>
+          category -> linkData(category).get(linkId).map(_.getSumVolume(hour)).getOrElse(0.0)
+        }.toMap
+        val aggregatedVolumes = aggregation.map { case (categoryGroup, _) =>
+          categoryGroup.map(category => categoryToVolume(category)).sum
         }
+        val categoryVolumeSum = categoryToVolume.values.reduceOption(_ + _).getOrElse(0.0)
+        val otherVolume = totalLinkData.get(linkId).map(_.getSumVolume(hour)).getOrElse(0.0) - categoryVolumeSum
         Seq(
           linkId,
           link.getFromNode.getId,
@@ -64,8 +70,8 @@ class LinkStatsWithVehicleCategory(
           link.getFreespeed,
           link.getCapacity,
           "AVG",
-          totalLinkData.get(linkId).map(_.getSumVolume(hour)).getOrElse(0.0)
-        ) ++ categoryVolumes ++ Seq(data.calculateAverageTravelTime(hour))
+          otherVolume
+        ) ++ aggregatedVolumes ++ Seq(data.calculateAverageTravelTime(hour))
       }
 
     }
@@ -81,8 +87,8 @@ class LinkStatsWithVehicleCategory(
       Seq("LightDutyTruck", "HeavyDutyTruck") -> "TruckVolume",
       Seq("HeavyDutyTruck")                   -> "HDTruckVolume"
     )
-    val (totalLinkData, linkData, nofHours) =
-      calculateLinkData(volumesAnalyzer, travelTimeForR5, categoryMapping.flatMap(_._1))
+    val categories = categoryMapping.flatMap(_._1).distinct
+    val (totalLinkData, linkData, nofHours) = calculateLinkData(volumesAnalyzer, travelTimeForR5, categories)
     writeToFile(totalLinkData, linkData, nofHours, categoryMapping, filePath)
   }
 }

From 45800448329e983447f20af7e267868c6c18d7df Mon Sep 17 00:00:00 2001
From: Dmitry Openkov <dimaopen@gmail.com>
Date: Fri, 1 Apr 2022 16:55:20 +0300
Subject: [PATCH 33/58] Modified LinkTravelTimeContainer so that it can read
 linkstats with an arbitrary number of columns

---
 .../beam/router/LinkTravelTimeContainer.scala | 47 ++++++++++---------
 .../router/LinkTravelTimeContainerSpec.scala  | 28 +++++++++++
 .../beam/router/0.linkstats.csv.gz            |  4 +-
 3 files changed, 55 insertions(+), 24 deletions(-)
 create mode 100644 src/test/scala/beam/router/LinkTravelTimeContainerSpec.scala

diff --git a/src/main/scala/beam/router/LinkTravelTimeContainer.scala b/src/main/scala/beam/router/LinkTravelTimeContainer.scala
index 1fda41de1d3..caf58b1131f 100755
--- a/src/main/scala/beam/router/LinkTravelTimeContainer.scala
+++ b/src/main/scala/beam/router/LinkTravelTimeContainer.scala
@@ -1,8 +1,9 @@
 package beam.router
 
-import beam.utils.FileUtils.using
-import beam.utils.{FileUtils, TravelTimeCalculatorHelper}
+import beam.utils.csv.GenericCsvReader
+import beam.utils.{MathUtils, TravelTimeCalculatorHelper}
 import com.typesafe.scalalogging.LazyLogging
+import org.apache.commons.lang3.StringUtils.isBlank
 import org.matsim.api.core.v01.network.Link
 import org.matsim.api.core.v01.population.Person
 import org.matsim.core.router.util.TravelTime
@@ -10,6 +11,7 @@ import org.matsim.vehicles.Vehicle
 
 import scala.collection.JavaConverters._
 import scala.collection.mutable
+import scala.util.Try
 
 class LinkTravelTimeContainer(fileName: String, timeBinSizeInSeconds: Int, maxHour: Int)
     extends TravelTime
@@ -23,34 +25,35 @@ class LinkTravelTimeContainer(fileName: String, timeBinSizeInSeconds: Int, maxHo
     val linkTravelTimeMap: mutable.HashMap[String, Array[Double]] = mutable.HashMap()
     logger.info(s"Stats fileName [$fileName] is being loaded")
 
-    using(FileUtils.readerFromFile(fileName)) { bufferedReader =>
-      var line: String = null
-      while ({
-        line = bufferedReader.readLine
-        line != null
-      }) {
-        val linkStats = line.split(",")
-        if (linkStats.length == 10 && "avg".equalsIgnoreCase(linkStats(7))) {
-          val linkId = linkStats(0)
-          val hour = linkStats(3).toDouble.toInt
-          val travelTime = linkStats(9).toDouble
-          linkTravelTimeMap.get(linkId) match {
-            case Some(travelTimePerHourArr) =>
-              travelTimePerHourArr.update(hour, travelTime)
-            case None =>
-              val travelTimePerHourArr = Array.ofDim[Double](maxHour)
-              travelTimePerHourArr.update(hour, travelTime)
-              linkTravelTimeMap.put(linkId, travelTimePerHourArr)
-          }
-        }
+    val (iterator, closable) =
+      GenericCsvReader.readAs(fileName, mapper, (row: (String, Int, Double)) => !isBlank(row._1))
+    try {
+      iterator.foreach { case (linkId, hour, travelTime) =>
+        val travelTimePerHourArr = linkTravelTimeMap.getOrElseUpdate(linkId, Array.ofDim[Double](maxHour))
+        travelTimePerHourArr.update(hour, travelTime)
       }
+    } finally {
+      Try(closable.close())
     }
+
     val end = System.currentTimeMillis()
     logger.info("LinkTravelTimeMap is initialized in {} ms", end - start)
 
     linkTravelTimeMap
   }
 
+  def mapper(row: java.util.Map[String, String]): (String, Int, Double) = {
+    val linkId = row.get("link")
+    val stat = row.get("stat")
+    if (isBlank(linkId) || isBlank(stat) || !stat.equalsIgnoreCase("avg")) {
+      ("", 0, 0.0)
+    } else {
+      val hour = MathUtils.doubleToInt(row.get("hour").toDouble)
+      val travelTime = row.get("traveltime").toDouble
+      (linkId, hour, travelTime)
+    }
+  }
+
   def getLinkTravelTime(link: Link, time: Double, person: Person, vehicle: Vehicle): Double = {
     travelTimeCalculator.getLinkTravelTime(link, time, person, vehicle)
   }
diff --git a/src/test/scala/beam/router/LinkTravelTimeContainerSpec.scala b/src/test/scala/beam/router/LinkTravelTimeContainerSpec.scala
new file mode 100644
index 00000000000..f6bf048858f
--- /dev/null
+++ b/src/test/scala/beam/router/LinkTravelTimeContainerSpec.scala
@@ -0,0 +1,28 @@
+package beam.router
+
+import org.matsim.api.core.v01.Id
+import org.matsim.api.core.v01.network.Link
+import org.matsim.api.core.v01.population.Person
+import org.matsim.vehicles.Vehicle
+import org.scalatest.matchers.should.Matchers
+import org.scalatest.wordspec.AnyWordSpecLike
+import org.mockito.Mockito.{mock, when}
+
+/**
+  * @author Dmitry Openkov
+  */
+class LinkTravelTimeContainerSpec extends AnyWordSpecLike with Matchers {
+  "LinkTravelTimeContainer" when {
+    "provided with a valid linkstats file" should {
+      "read travel time correctly" in {
+        val container = new LinkTravelTimeContainer("test/test-resources/beam/router/0.linkstats.csv.gz", 3600, 30)
+        val link = mock(classOf[Link])
+        when(link.getId).thenReturn(Id.createLinkId(233))
+        val person = mock(classOf[Person])
+        val vehicle = mock(classOf[Vehicle])
+        val travelTime = container.getLinkTravelTime(link, 4000, person, vehicle)
+        travelTime shouldBe 0.355 +- 0.0001
+      }
+    }
+  }
+}
diff --git a/test/test-resources/beam/router/0.linkstats.csv.gz b/test/test-resources/beam/router/0.linkstats.csv.gz
index 5ee3fca9a7e..9238d49d25a 100644
--- a/test/test-resources/beam/router/0.linkstats.csv.gz
+++ b/test/test-resources/beam/router/0.linkstats.csv.gz
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:298c5a2730f3b0c9d9d654b86e6283cf9d37c7b7ba555a8ceb934de71bdd9341
-size 40571
+oid sha256:df8f992e0640a656a48b91f9f06456bb3e07573c2c7b46e6a8e0b9131ab77d82
+size 31764

From de286627e0d68ef1aa5fe271b5866537810a9f64 Mon Sep 17 00:00:00 2001
From: Rutvik Patel <heyrutvik@gmail.com>
Date: Mon, 4 Apr 2022 14:27:21 +0530
Subject: [PATCH 34/58] adds scenario loader helper with snap coord ability

---
 src/main/scala/beam/sim/BeamHelper.scala      |  14 +-
 .../beam/utils/SnapCoordinateUtils.scala      |  64 +++++++++
 .../utils/scenario/BeamScenarioLoader.scala   |  95 ++++++------
 .../utils/scenario/ScenarioLoaderHelper.scala | 135 ++++++++++++++++++
 .../scenario/UrbanSimScenarioLoader.scala     |  76 ++--------
 5 files changed, 278 insertions(+), 106 deletions(-)
 create mode 100644 src/main/scala/beam/utils/SnapCoordinateUtils.scala
 create mode 100644 src/main/scala/beam/utils/scenario/ScenarioLoaderHelper.scala

diff --git a/src/main/scala/beam/sim/BeamHelper.scala b/src/main/scala/beam/sim/BeamHelper.scala
index e5f0e5cd9d1..8a1c2527258 100755
--- a/src/main/scala/beam/sim/BeamHelper.scala
+++ b/src/main/scala/beam/sim/BeamHelper.scala
@@ -800,10 +800,9 @@ trait BeamHelper extends LazyLogging {
     matsimConfig: MatsimConfig
   ): (MutableScenario, BeamScenario, Boolean) = {
     val scenarioConfig = beamConfig.beam.exchange.scenario
-
     val src = scenarioConfig.source.toLowerCase
-
     val fileFormat = scenarioConfig.fileFormat
+    val outputDir = matsimConfig.controler().getOutputDirectory
 
     val (scenario, beamScenario, plansMerged) =
       ProfilingUtils.timed(s"Load scenario using $src/$fileFormat", x => logger.info(x)) {
@@ -862,7 +861,8 @@ trait BeamHelper extends LazyLogging {
                 beamScenario,
                 source,
                 new GeoUtilsImpl(beamConfig),
-                Some(merger)
+                Some(merger),
+                Some(outputDir)
               ).loadScenario()
             if (src == "urbansim_v2") {
               new ScenarioAdjuster(
@@ -884,7 +884,13 @@ trait BeamHelper extends LazyLogging {
                   rdr = readers.BeamCsvScenarioReader
                 )
                 val scenarioBuilder = ScenarioBuilder(matsimConfig, beamScenario.network)
-                new BeamScenarioLoader(scenarioBuilder, beamScenario, source, new GeoUtilsImpl(beamConfig))
+                new BeamScenarioLoader(
+                  scenarioBuilder,
+                  beamScenario,
+                  source,
+                  new GeoUtilsImpl(beamConfig),
+                  Some(outputDir)
+                )
                   .loadScenario()
               }.asInstanceOf[MutableScenario]
               (scenario, beamScenario, false)
diff --git a/src/main/scala/beam/utils/SnapCoordinateUtils.scala b/src/main/scala/beam/utils/SnapCoordinateUtils.scala
new file mode 100644
index 00000000000..8f0ef293db1
--- /dev/null
+++ b/src/main/scala/beam/utils/SnapCoordinateUtils.scala
@@ -0,0 +1,64 @@
+package beam.utils
+
+import beam.sim.common.GeoUtils
+import beam.utils.csv.CsvWriter
+import beam.utils.scenario.PlanElement
+import com.conveyal.r5.streets.StreetLayer
+import com.typesafe.scalalogging.LazyLogging
+import org.matsim.api.core.v01.Coord
+
+import scala.collection.concurrent.TrieMap
+
+object SnapCoordinateUtils extends LazyLogging {
+
+  trait Result
+
+  object Result {
+    final case object OutOfBoundingBoxError extends Result
+    final case object R5SplitNullError extends Result
+    final case class Succeed(splitCoord: Coord) extends Result
+  }
+
+  final case class SnapLocationHelper(geo: GeoUtils, streetLayer: StreetLayer, maxRadius: Double) {
+    private val store: TrieMap[Coord, Option[Coord]] = TrieMap.empty
+
+    def computeResult[A](utmCoord: Coord): Result = {
+      val wgsCoord = geo.utm2Wgs(utmCoord)
+      if (streetLayer.envelope.contains(wgsCoord.getX, wgsCoord.getY)) {
+        val snapCoordOpt = store.getOrElseUpdate(
+          utmCoord,
+          Option(geo.getR5Split(streetLayer, wgsCoord, maxRadius)).map { split =>
+            val updatedPlanCoord = geo.splitToCoord(split)
+            geo.wgs2Utm(updatedPlanCoord)
+          }
+        )
+        snapCoordOpt.fold[Result](Result.R5SplitNullError)(Result.Succeed)
+      } else Result.OutOfBoundingBoxError
+    }
+  }
+
+  object Error {
+    val OutOfBoundingBox = "OutOfBoundingBox"
+    val R5SplitNull = "R5SplitNull"
+  }
+
+  object Category {
+    val ScenarioPerson = "Person"
+    val ScenarioHousehold = "Household"
+    val FreightTour = "Tour"
+    val FreightPayloadPlan = "PayloadPlan"
+    val FreightCarrier = "Carrier"
+  }
+
+  final case class ErrorInfo(id: String, category: String, error: String, planX: Double, planY: Double)
+  final case class Processed[A](data: Seq[A] = Seq.empty, errors: Seq[ErrorInfo] = Seq.empty)
+
+  def writeToCsv(path: String, errors: Seq[ErrorInfo]): Unit = {
+    new CsvWriter(path, "id", "category", "error", "x", "y")
+      .writeAllAndClose(
+        errors.map(error => List(error.id, error.category, error.error, error.planX, error.planY))
+      )
+    logger.info("See location error info at {}.", path)
+  }
+
+}
diff --git a/src/main/scala/beam/utils/scenario/BeamScenarioLoader.scala b/src/main/scala/beam/utils/scenario/BeamScenarioLoader.scala
index d7c2161e653..8793846879b 100644
--- a/src/main/scala/beam/utils/scenario/BeamScenarioLoader.scala
+++ b/src/main/scala/beam/utils/scenario/BeamScenarioLoader.scala
@@ -5,9 +5,9 @@ import beam.agentsim.agents.vehicles.{BeamVehicle, BeamVehicleType, VehicleManag
 import beam.router.Modes.BeamMode
 import beam.sim.BeamScenario
 import beam.sim.common.GeoUtils
-import beam.utils.logging.ExponentialLazyLogging
 import beam.utils.plan.sampling.AvailableModeUtils
 import com.google.common.annotations.VisibleForTesting
+import com.typesafe.scalalogging.LazyLogging
 import org.matsim.api.core.v01.network.Link
 import org.matsim.api.core.v01.population._
 import org.matsim.api.core.v01.{Coord, Id, Scenario}
@@ -19,30 +19,30 @@ import org.matsim.vehicles.{Vehicle, VehicleType, VehicleUtils}
 
 import java.util
 import java.util.concurrent.atomic.AtomicReference
+import scala.collection.Iterable
 import scala.collection.JavaConverters._
+import scala.concurrent.duration._
+import scala.concurrent.{Await, ExecutionContext, Future}
 import scala.util.Random
 
 class BeamScenarioLoader(
   val scenarioBuilder: ScenarioBuilder,
   var beamScenario: BeamScenario,
   val scenarioSource: ScenarioSource,
-  val geo: GeoUtils
-) extends ExponentialLazyLogging {
+  val geo: GeoUtils,
+  val outputDirOpt: Option[String]
+) extends ScenarioLoaderHelper {
 
   import BeamScenarioLoader._
 
   type IdToAttributes = Map[String, Seq[(String, Double)]]
 
+  private implicit val ex: ExecutionContext = scala.concurrent.ExecutionContext.Implicits.global
+
   private val availableModes: Seq[String] = BeamMode.allModes.map(_.value)
 
   private val rand: Random = new Random(beamScenario.beamConfig.matsim.modules.global.randomSeed)
 
-  private lazy val plans: Iterable[PlanElement] = {
-    val r = scenarioSource.getPlans
-    logger.info(s"Read ${r.size} plans")
-    r
-  }
-
   private val scenario: MutableScenario = scenarioBuilder.build
 
   private def replaceHouseholdsAttributes(
@@ -61,22 +61,35 @@ class BeamScenarioLoader(
   def loadScenario(): Scenario = {
     logger.info("The scenario loading started...")
 
-    val personsWithPlans = {
+    val plansF = Future {
+      val plans = scenarioSource.getPlans
+      logger.info(s"Read ${plans.size} plans")
+      validatePlans(plans)
+    }
+
+    val personsF = Future {
       val persons: Iterable[PersonInfo] = scenarioSource.getPersons
-      val personIdsWithPlanTmp = plans.map(_.personId).toSet
-      val result = persons.filter(person => personIdsWithPlanTmp.contains(person.personId))
-      logger.info(s"There are ${persons.size} people. ${result.size} have plans")
-      result
+      logger.info(s"Read ${persons.size} persons")
+      persons
     }
 
-    val vehicles = scenarioSource.getVehicles
+    val householdsF = Future {
+      val households = scenarioSource.getHousehold
+      logger.info(s"Read ${households.size} households")
+      validateHouseholds(households)
+    }
 
-    val loadedHouseholds = scenarioSource.getHousehold
+    val plans = Await.result(plansF, 1800.seconds)
+    logger.info(s"Reading plans done.")
+    val persons = Await.result(personsF, 1800.seconds)
+    logger.info(s"Reading persons done.")
+    val households = Await.result(householdsF, 1800.seconds)
+    logger.info(s"Reading households done.")
 
-    val newHouseholds: Iterable[Household] =
-      buildMatsimHouseholds(loadedHouseholds, personsWithPlans, vehicles)
+    val vehicles = scenarioSource.getVehicles
 
-    val households: Households = replaceHouseholds(scenario.getHouseholds, newHouseholds)
+    val personsWithPlans = getPersonsWithPlan(persons, plans, households)
+    logger.info(s"There are ${personsWithPlans.size} persons with plans")
 
     beamScenario.privateVehicles.clear()
     beamScenario.privateVehicleInitialSoc.clear()
@@ -88,15 +101,16 @@ class BeamScenarioLoader(
       vehicleInfo.initialSoc.foreach(beamScenario.privateVehicleInitialSoc.put(vehicle.id, _))
     }
 
+    val newHouseholds: Iterable[Household] = buildMatsimHouseholds(households, personsWithPlans, vehicles)
+    val matsimHouseholds: Households = replaceHouseholds(scenario.getHouseholds, newHouseholds)
+    val loadedAttributes = buildAttributesCoordinates(households)
+    replaceHouseholdsAttributes(matsimHouseholds, loadedAttributes)
+
     val scenarioPopulation: Population = buildPopulation(personsWithPlans)
     scenario.setPopulation(scenarioPopulation)
     updateAvailableModesForPopulation(scenario)
-
     replacePlansFromPopulation(scenarioPopulation, plans)
 
-    val loadedAttributes = buildAttributesCoordinates(loadedHouseholds)
-    replaceHouseholdsAttributes(households, loadedAttributes)
-
     logger.info("The scenario loading is completed.")
     scenario
   }
@@ -197,23 +211,24 @@ class BeamScenarioLoader(
       listOfElementsGroupedByPerson.groupBy(_.planIndex).foreach {
         case (_, listOfElementsGroupedByPlan) if listOfElementsGroupedByPlan.nonEmpty =>
           val person = population.getPersons.get(Id.createPersonId(personId.id))
+          if (person != null) {
+            val currentPlan = PopulationUtils.createPlan(person)
+            currentPlan.setScore(listOfElementsGroupedByPlan.head.planScore)
+            person.addPlan(currentPlan)
+
+            val personWithoutSelectedPlan = person.getSelectedPlan == null
+            val isCurrentPlanIndexSelected = listOfElementsGroupedByPlan.head.planSelected
+            val isLastPlanIteration = person.getPlans.size() == listOfElementsGroupedByPerson.size
+            if (personWithoutSelectedPlan && (isCurrentPlanIndexSelected || isLastPlanIteration)) {
+              person.setSelectedPlan(currentPlan)
+            }
 
-          val currentPlan = PopulationUtils.createPlan(person)
-          currentPlan.setScore(listOfElementsGroupedByPlan.head.planScore)
-          person.addPlan(currentPlan)
-
-          val personWithoutSelectedPlan = person.getSelectedPlan == null
-          val isCurrentPlanIndexSelected = listOfElementsGroupedByPlan.head.planSelected
-          val isLastPlanIteration = person.getPlans.size() == listOfElementsGroupedByPerson.size
-          if (personWithoutSelectedPlan && (isCurrentPlanIndexSelected || isLastPlanIteration)) {
-            person.setSelectedPlan(currentPlan)
-          }
-
-          listOfElementsGroupedByPlan.foreach { planElement =>
-            if (planElement.planElementType == PlanElement.Leg) {
-              buildAndAddLegToPlan(currentPlan, planElement)
-            } else if (planElement.planElementType == PlanElement.Activity) {
-              buildAndAddActivityToPlan(currentPlan, planElement)
+            listOfElementsGroupedByPlan.foreach { planElement =>
+              if (planElement.planElementType == PlanElement.Leg) {
+                buildAndAddLegToPlan(currentPlan, planElement)
+              } else if (planElement.planElementType == PlanElement.Activity) {
+                buildAndAddActivityToPlan(currentPlan, planElement)
+              }
             }
           }
       }
@@ -276,7 +291,7 @@ class BeamScenarioLoader(
   }
 }
 
-object BeamScenarioLoader extends ExponentialLazyLogging {
+object BeamScenarioLoader extends LazyLogging {
 
   private[utils] def buildMatsimHouseholds(
     households: Iterable[HouseholdInfo],
diff --git a/src/main/scala/beam/utils/scenario/ScenarioLoaderHelper.scala b/src/main/scala/beam/utils/scenario/ScenarioLoaderHelper.scala
new file mode 100644
index 00000000000..9bfb6fa95a1
--- /dev/null
+++ b/src/main/scala/beam/utils/scenario/ScenarioLoaderHelper.scala
@@ -0,0 +1,135 @@
+package beam.utils.scenario
+
+import beam.sim.BeamScenario
+import beam.sim.common.GeoUtils
+import beam.utils.SnapCoordinateUtils
+import beam.utils.SnapCoordinateUtils.{Category, Error, ErrorInfo, Processed, Result, SnapLocationHelper}
+import com.typesafe.scalalogging.LazyLogging
+import org.matsim.api.core.v01.Coord
+
+import scala.collection.Iterable
+
+trait ScenarioLoaderHelper extends LazyLogging {
+
+  def beamScenario: BeamScenario
+  def geo: GeoUtils
+  def outputDirOpt: Option[String]
+
+  private val snapLocationHelper: SnapLocationHelper = SnapLocationHelper(
+    geo,
+    beamScenario.transportNetwork.streetLayer,
+    beamScenario.beamConfig.beam.routing.r5.linkRadiusMeters
+  )
+
+  private def validatePersonPlans(personId: PersonId, plans: Iterable[PlanElement]): Processed[PlanElement] = {
+    plans.foldLeft[Processed[PlanElement]](Processed()) {
+      case (processed, planElement) if planElement.planElementType == PlanElement.Leg =>
+        processed.copy(data = processed.data :+ planElement)
+      case (processed, planElement) if planElement.planElementType == PlanElement.Activity =>
+        val utmCoord = new Coord(planElement.activityLocationX.get, planElement.activityLocationY.get)
+        snapLocationHelper.computeResult(utmCoord) match {
+          case Result.Succeed(splitCoord) =>
+            val updatedPlan = planElement
+              .copy(activityLocationX = Some(splitCoord.getX), activityLocationY = Some(splitCoord.getY))
+            processed.copy(data = processed.data :+ updatedPlan)
+          case Result.OutOfBoundingBoxError =>
+            processed.copy(errors =
+              processed.errors :+ ErrorInfo(
+                personId.id,
+                Category.ScenarioPerson,
+                Error.OutOfBoundingBox,
+                utmCoord.getX,
+                utmCoord.getY
+              )
+            )
+          case Result.R5SplitNullError =>
+            processed.copy(errors =
+              processed.errors :+ ErrorInfo(
+                personId.id,
+                Category.ScenarioPerson,
+                Error.R5SplitNull,
+                utmCoord.getX,
+                utmCoord.getY
+              )
+            )
+        }
+    }
+  }
+
+  private def snapLocationPlans(plans: Iterable[PlanElement]): Processed[PlanElement] = {
+    plans.groupBy(_.personId).foldLeft[Processed[PlanElement]](Processed()) {
+      case (processed, (personId, personPlans)) =>
+        val Processed(updatedPlans, errors) = validatePersonPlans(personId, personPlans)
+        if (updatedPlans.size == personPlans.size) {
+          processed.copy(data = processed.data ++ updatedPlans, errors = processed.errors ++ errors)
+        } else {
+          processed.copy(errors = processed.errors ++ errors)
+        }
+    }
+  }
+
+  private def snapLocationHouseholds(households: Iterable[HouseholdInfo]): Processed[HouseholdInfo] = {
+    households.foldLeft[Processed[HouseholdInfo]](Processed()) { case (processed, household) =>
+      val householdId = household.householdId.id
+      val utmCoord = new Coord(household.locationX, household.locationY)
+      snapLocationHelper.computeResult(utmCoord) match {
+        case Result.Succeed(splitCoord) =>
+          val updatedHousehold = household.copy(locationX = splitCoord.getX, locationY = splitCoord.getY)
+          processed.copy(data = processed.data :+ updatedHousehold)
+        case Result.OutOfBoundingBoxError =>
+          processed.copy(errors =
+            processed.errors :+ ErrorInfo(
+              householdId,
+              Category.ScenarioHousehold,
+              Error.OutOfBoundingBox,
+              utmCoord.getX,
+              utmCoord.getY
+            )
+          )
+        case Result.R5SplitNullError =>
+          processed.copy(errors =
+            processed.errors :+ ErrorInfo(
+              householdId,
+              Category.ScenarioHousehold,
+              Error.R5SplitNull,
+              utmCoord.getX,
+              utmCoord.getY
+            )
+          )
+      }
+    }
+  }
+
+  protected def getPersonsWithPlan(
+    persons: Iterable[PersonInfo],
+    plans: Iterable[PlanElement],
+    households: Iterable[HouseholdInfo]
+  ): Iterable[PersonInfo] = {
+    val personIdsWithPlan = plans.map(_.personId).toSet
+    val householdIds = households.map(_.householdId).toSet
+    persons.filter(person => personIdsWithPlan.contains(person.personId) && householdIds.contains(person.householdId))
+  }
+
+  protected def validatePlans(plans: Iterable[PlanElement]): Iterable[PlanElement] = {
+    val Processed(validPlans, errors) = snapLocationPlans(plans)
+    outputDirOpt.foreach(path => SnapCoordinateUtils.writeToCsv(s"$path/snapLocationPlanErrors.csv", errors))
+
+    val filteredCnt = plans.size - validPlans.size
+    if (filteredCnt > 0) {
+      logger.info(s"Filtered out $filteredCnt plans. Total number of plans: ${validPlans.size}")
+    }
+    validPlans
+  }
+
+  protected def validateHouseholds(households: Iterable[HouseholdInfo]): Iterable[HouseholdInfo] = {
+    val Processed(validHouseholds, errors) = snapLocationHouseholds(households)
+    outputDirOpt.foreach(path => SnapCoordinateUtils.writeToCsv(s"$path/snapLocationHouseholdErrors.csv", errors))
+
+    val filteredCnt = households.size - validHouseholds.size
+    if (filteredCnt > 0) {
+      logger.info(s"Filtered out $filteredCnt plans. Total number of plans: ${validHouseholds.size}")
+    }
+    validHouseholds
+  }
+
+}
diff --git a/src/main/scala/beam/utils/scenario/UrbanSimScenarioLoader.scala b/src/main/scala/beam/utils/scenario/UrbanSimScenarioLoader.scala
index 725bf5e1fda..615f07dcb50 100644
--- a/src/main/scala/beam/utils/scenario/UrbanSimScenarioLoader.scala
+++ b/src/main/scala/beam/utils/scenario/UrbanSimScenarioLoader.scala
@@ -9,7 +9,6 @@ import beam.sim.vehicles.VehiclesAdjustment
 import beam.utils.SequenceUtils
 import beam.utils.plan.sampling.AvailableModeUtils
 import beam.utils.scenario.urbansim.HOVModeTransformer
-import com.typesafe.scalalogging.LazyLogging
 import org.apache.commons.math3.distribution.UniformRealDistribution
 import org.matsim.api.core.v01.population.Population
 import org.matsim.api.core.v01.{Coord, Id, Scenario}
@@ -31,8 +30,9 @@ class UrbanSimScenarioLoader(
   val beamScenario: BeamScenario,
   val scenarioSource: ScenarioSource,
   val geo: GeoUtils,
-  val previousRunPlanMerger: Option[PreviousRunPlanMerger] = None
-) extends LazyLogging {
+  val previousRunPlanMerger: Option[PreviousRunPlanMerger] = None,
+  val outputDirOpt: Option[String]
+) extends ScenarioLoaderHelper {
 
   private implicit val ex: ExecutionContext = scala.concurrent.ExecutionContext.Implicits.global
 
@@ -45,85 +45,45 @@ class UrbanSimScenarioLoader(
   def loadScenario(): (Scenario, Boolean) = {
     clear()
 
-    val wereCoordinatesInWGS = beamScenario.beamConfig.beam.exchange.scenario.convertWgs2Utm
-
     val plansF = Future {
       val plans = scenarioSource.getPlans
       logger.info(s"Read ${plans.size} plans")
-      val activities = plans.view.filter { p =>
-        p.activityType.exists(actType => actType.toLowerCase == "home")
-      }
-
-      val personIdsWithinRange =
-        activities
-          .filter { act =>
-            val actCoord = new Coord(act.activityLocationX.get, act.activityLocationY.get)
-            val wgsCoord = if (wereCoordinatesInWGS) geo.utm2Wgs(actCoord) else actCoord
-            beamScenario.transportNetwork.streetLayer.envelope.contains(wgsCoord.getX, wgsCoord.getY)
-          }
-          .map { act =>
-            act.personId
-          }
-          .toSet
-      val planWithinRange = plans.filter(p => personIdsWithinRange.contains(p.personId))
-      val filteredCnt = plans.size - planWithinRange.size
-      if (filteredCnt > 0) {
-        logger.info(s"Filtered out $filteredCnt plans. Total number of plans: ${planWithinRange.size}")
-      }
-      planWithinRange
+      validatePlans(plans)
     }
+
     val personsF = Future {
       val persons: Iterable[PersonInfo] = scenarioSource.getPersons
       logger.info(s"Read ${persons.size} persons")
       persons
     }
+
     val householdsF = Future {
       val households = scenarioSource.getHousehold
       logger.info(s"Read ${households.size} households")
-      val householdIdsWithinBoundingBox = households.view
-        .filter { hh =>
-          val coord = new Coord(hh.locationX, hh.locationY)
-          val wgsCoord = if (wereCoordinatesInWGS) geo.utm2Wgs(coord) else coord
-          beamScenario.transportNetwork.streetLayer.envelope.contains(wgsCoord.getX, wgsCoord.getY)
-        }
-        .map { hh =>
-          hh.householdId
-        }
-        .toSet
-
-      val householdsInsideBoundingBox =
-        households.filter(household => householdIdsWithinBoundingBox.contains(household.householdId))
-      val filteredCnt = households.size - householdsInsideBoundingBox.size
-      if (filteredCnt > 0) {
-        logger.info(
-          s"Filtered out $filteredCnt households. Total number of households: ${householdsInsideBoundingBox.size}"
-        )
-      }
-      householdsInsideBoundingBox
+      validateHouseholds(households)
     }
-    val inputPlans = Await.result(plansF, 1800.seconds)
+
+    val validPlans = Await.result(plansF, 1800.seconds)
     logger.info(s"Reading plans done.")
     val persons = Await.result(personsF, 1800.seconds)
     logger.info(s"Reading persons done.")
-    val households = Await.result(householdsF, 1800.seconds)
+    val validHouseholds = Await.result(householdsF, 1800.seconds)
     logger.info(s"Reading households done.")
 
-    val (mergedPlans, plansMerged) = previousRunPlanMerger.map(_.merge(inputPlans)).getOrElse(inputPlans -> false)
+    val (mergedPlans, plansMerged) = previousRunPlanMerger.map(_.merge(validPlans)).getOrElse(validPlans -> false)
 
     val plans = {
       HOVModeTransformer.reseedRandomGenerator(beamScenario.beamConfig.matsim.modules.global.randomSeed)
       HOVModeTransformer.transformHOVtoHOVCARorHOVTeleportation(mergedPlans)
     }
 
-    val householdIds = households.map(_.householdId.id).toSet
-
-    val personsWithPlans = getPersonsWithPlan(persons, plans)
-      .filter(p => householdIds.contains(p.householdId.id))
+    val personsWithPlans = getPersonsWithPlan(persons, plans, validHouseholds)
     logger.info(s"There are ${personsWithPlans.size} persons with plans")
 
     val householdIdToPersons: Map[HouseholdId, Iterable[PersonInfo]] = personsWithPlans.groupBy(_.householdId)
 
-    val householdsWithMembers = households.filter(household => householdIdToPersons.contains(household.householdId))
+    val householdsWithMembers =
+      validHouseholds.filter(household => householdIdToPersons.contains(household.householdId))
     logger.info(s"There are ${householdsWithMembers.size} non-empty households")
 
     logger.info("Applying households...")
@@ -152,14 +112,6 @@ class UrbanSimScenarioLoader(
     beamScenario.privateVehicleInitialSoc.clear()
   }
 
-  private[utils] def getPersonsWithPlan(
-    persons: Iterable[PersonInfo],
-    plans: Iterable[PlanElement]
-  ): Iterable[PersonInfo] = {
-    val personIdsWithPlan = plans.map(_.personId).toSet
-    persons.filter(person => personIdsWithPlan.contains(person.personId))
-  }
-
   private[utils] def applyHousehold(
     households: Iterable[HouseholdInfo],
     householdIdToPersons: Map[HouseholdId, Iterable[PersonInfo]],

From 20d8306a78145b242f0b2646e359135f20b54716 Mon Sep 17 00:00:00 2001
From: Rutvik Patel <heyrutvik@gmail.com>
Date: Mon, 4 Apr 2022 14:31:00 +0530
Subject: [PATCH 35/58] updates variable names

---
 .../beam/utils/scenario/BeamScenarioLoader.scala     | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/src/main/scala/beam/utils/scenario/BeamScenarioLoader.scala b/src/main/scala/beam/utils/scenario/BeamScenarioLoader.scala
index 8793846879b..37292467d10 100644
--- a/src/main/scala/beam/utils/scenario/BeamScenarioLoader.scala
+++ b/src/main/scala/beam/utils/scenario/BeamScenarioLoader.scala
@@ -79,16 +79,16 @@ class BeamScenarioLoader(
       validateHouseholds(households)
     }
 
-    val plans = Await.result(plansF, 1800.seconds)
+    val validPlans = Await.result(plansF, 1800.seconds)
     logger.info(s"Reading plans done.")
     val persons = Await.result(personsF, 1800.seconds)
     logger.info(s"Reading persons done.")
-    val households = Await.result(householdsF, 1800.seconds)
+    val validHouseholds = Await.result(householdsF, 1800.seconds)
     logger.info(s"Reading households done.")
 
     val vehicles = scenarioSource.getVehicles
 
-    val personsWithPlans = getPersonsWithPlan(persons, plans, households)
+    val personsWithPlans = getPersonsWithPlan(persons, validPlans, validHouseholds)
     logger.info(s"There are ${personsWithPlans.size} persons with plans")
 
     beamScenario.privateVehicles.clear()
@@ -101,15 +101,15 @@ class BeamScenarioLoader(
       vehicleInfo.initialSoc.foreach(beamScenario.privateVehicleInitialSoc.put(vehicle.id, _))
     }
 
-    val newHouseholds: Iterable[Household] = buildMatsimHouseholds(households, personsWithPlans, vehicles)
+    val newHouseholds: Iterable[Household] = buildMatsimHouseholds(validHouseholds, personsWithPlans, vehicles)
     val matsimHouseholds: Households = replaceHouseholds(scenario.getHouseholds, newHouseholds)
-    val loadedAttributes = buildAttributesCoordinates(households)
+    val loadedAttributes = buildAttributesCoordinates(validHouseholds)
     replaceHouseholdsAttributes(matsimHouseholds, loadedAttributes)
 
     val scenarioPopulation: Population = buildPopulation(personsWithPlans)
     scenario.setPopulation(scenarioPopulation)
     updateAvailableModesForPopulation(scenario)
-    replacePlansFromPopulation(scenarioPopulation, plans)
+    replacePlansFromPopulation(scenarioPopulation, validPlans)
 
     logger.info("The scenario loading is completed.")
     scenario

From 6c745272de1d40a3f2bde874220442b1fc1294ba Mon Sep 17 00:00:00 2001
From: Dmitry Openkov <dimaopen@gmail.com>
Date: Mon, 4 Apr 2022 15:32:14 +0300
Subject: [PATCH 36/58] Created LinkStatsWithVehicleCategorySpec

---
 .../LinkStatsWithVehicleCategory.scala        |   3 +-
 .../LinkStatsWithVehicleCategorySpec.scala    | 145 ++++++++++++++++++
 2 files changed, 147 insertions(+), 1 deletion(-)
 create mode 100644 src/test/scala/beam/physsim/analysis/LinkStatsWithVehicleCategorySpec.scala

diff --git a/src/main/scala/beam/physsim/analysis/LinkStatsWithVehicleCategory.scala b/src/main/scala/beam/physsim/analysis/LinkStatsWithVehicleCategory.scala
index bdf29247eff..9e146b40ea6 100644
--- a/src/main/scala/beam/physsim/analysis/LinkStatsWithVehicleCategory.scala
+++ b/src/main/scala/beam/physsim/analysis/LinkStatsWithVehicleCategory.scala
@@ -82,7 +82,7 @@ class LinkStatsWithVehicleCategory(
     volumesAnalyzer: VolumesAnalyzer,
     travelTimeForR5: TravelTime,
     filePath: String
-  ): Try[Unit] = {
+  ): Try[(Map[Id[Link], LinkData], Map[String, Map[Id[Link], LinkData]], Int)] = {
     val categoryMapping = IndexedSeq(
       Seq("LightDutyTruck", "HeavyDutyTruck") -> "TruckVolume",
       Seq("HeavyDutyTruck")                   -> "HDTruckVolume"
@@ -90,5 +90,6 @@ class LinkStatsWithVehicleCategory(
     val categories = categoryMapping.flatMap(_._1).distinct
     val (totalLinkData, linkData, nofHours) = calculateLinkData(volumesAnalyzer, travelTimeForR5, categories)
     writeToFile(totalLinkData, linkData, nofHours, categoryMapping, filePath)
+      .map(_ => (totalLinkData, linkData, nofHours))
   }
 }
diff --git a/src/test/scala/beam/physsim/analysis/LinkStatsWithVehicleCategorySpec.scala b/src/test/scala/beam/physsim/analysis/LinkStatsWithVehicleCategorySpec.scala
new file mode 100644
index 00000000000..c219329f718
--- /dev/null
+++ b/src/test/scala/beam/physsim/analysis/LinkStatsWithVehicleCategorySpec.scala
@@ -0,0 +1,145 @@
+package beam.physsim.analysis
+
+import beam.agentsim.agents.vehicles.VehicleCategory.VehicleCategory
+import beam.agentsim.agents.vehicles.{BeamVehicle, BeamVehicleType, VehicleCategory}
+import beam.router.LinkTravelTimeContainer
+import beam.utils.VolumesAnalyzerFixed
+import beam.utils.matsim_conversion.MatsimPlanConversion.IdOps
+import org.matsim.api.core.v01.Id
+import org.matsim.api.core.v01.network.{Link, Network}
+import org.matsim.api.core.v01.population.Person
+import org.matsim.core.config.ConfigUtils
+import org.matsim.core.config.groups.TravelTimeCalculatorConfigGroup
+import org.matsim.core.events.{EventsUtils, MatsimEventsReader}
+import org.matsim.core.network.io.MatsimNetworkReader
+import org.matsim.core.scenario.ScenarioUtils
+import org.matsim.core.trafficmonitoring.TravelTimeCalculator
+import org.matsim.vehicles.Vehicle
+import org.mockito.Mockito.{mock, when}
+import org.scalatest.OptionValues._
+import org.scalatest.TryValues._
+import org.scalatest.matchers.should.Matchers
+import org.scalatest.wordspec.AnyWordSpecLike
+
+import scala.collection.JavaConverters._
+
+/**
+  * @author Dmitry Openkov
+  */
+class LinkStatsWithVehicleCategorySpec extends AnyWordSpecLike with Matchers {
+  private val EVENTS_FILE_PATH = "test/input/equil-square/test-data/physSimEvents-relative-speeds.xml"
+  private val NETWORK_FILE_PATH = "test/input/equil-square/test-data/physSimNetwork-relative-speeds.xml"
+  private val LINKSTATS_CSV_PATH = "output/test/linkstats-by-category.csv"
+
+  val (
+    network: Network,
+    ttConfigGroup: TravelTimeCalculatorConfigGroup,
+    travelTimeCalculator: TravelTimeCalculator,
+    volumeAnalyzer: VolumesAnalyzerFixed
+  ) = readPhysSimEvents
+
+  "LinkStatsWithVehicleCategory" when {
+    val linkStats = new LinkStatsWithVehicleCategory(network, ttConfigGroup)
+    val (totalLinkData, linkData, nofHours) = linkStats.calculateLinkData(
+      volumeAnalyzer,
+      travelTimeCalculator.getLinkTravelTimes,
+      IndexedSeq("MediumDutyPassenger", "Car")
+    )
+    "calculates volume" should {
+      "return correct volume for each vehicle category" in {
+        nofHours shouldBe 68
+        withClue(s"According to event file $EVENTS_FILE_PATH, bused 1,6 went through link 248 2 times:") {
+          val linkDataOption = linkData("MediumDutyPassenger").get(Id.createLinkId(248))
+          linkDataOption.value.getSumVolume(9) shouldBe 0.0
+          linkDataOption.value.getSumVolume(10) shouldBe 1.0
+          linkDataOption.value.getSumVolume(11) shouldBe 1.0
+          linkDataOption.value.getSumVolume(12) shouldBe 0.0
+        }
+        withClue(s"According to event file $EVENTS_FILE_PATH, cars 1-9 went through link 248 9 times:") {
+          val linkDataOption = linkData("Car").get(Id.createLinkId(248))
+          linkDataOption.value.getSumVolume(5) shouldBe 0.0
+          linkDataOption.value.getSumVolume(6) shouldBe 0.0
+          linkDataOption.value.getSumVolume(7) shouldBe 1.0
+          linkDataOption.value.getSumVolume(20) shouldBe 1.0
+          linkDataOption.value.getSumVolume(21) shouldBe 2.0
+          linkDataOption.value.getSumVolume(23) shouldBe 1.0
+          linkDataOption.value.getSumVolume(33) shouldBe 1.0
+          linkDataOption.value.getSumVolume(40) shouldBe 1.0
+          linkDataOption.value.getSumVolume(51) shouldBe 1.0
+          linkDataOption.value.getSumVolume(53) shouldBe 1.0
+          linkDataOption.value.getSumVolume(54) shouldBe 0.0
+        }
+      }
+      "return correct total volume" in {
+        withClue(s"According to event file $EVENTS_FILE_PATH, vehicles went through link 112 12 times:") {
+          val linkDataOption = totalLinkData.get(Id.createLinkId(112))
+          linkDataOption.value.getSumVolume(5) shouldBe 0.0
+          linkDataOption.value.getSumVolume(6) shouldBe 8.0
+          linkDataOption.value.getSumVolume(7) shouldBe 4.0
+          linkDataOption.value.getSumVolume(8) shouldBe 0.0
+        }
+      }
+    }
+    "saves linkstats to csv" should {
+      "save it that WarmStart could read it" in {
+        val result = linkStats.writeLinkStatsWithTruckVolumes(
+          volumeAnalyzer,
+          travelTimeCalculator.getLinkTravelTimes,
+          LINKSTATS_CSV_PATH
+        )
+        val (_, _, nofHours) = result.success.value
+        nofHours shouldBe 68
+        val container = new LinkTravelTimeContainer(LINKSTATS_CSV_PATH, 3600, nofHours)
+        val link = mock(classOf[Link])
+        when(link.getId).thenReturn(Id.createLinkId(4))
+        val person = mock(classOf[Person])
+        val vehicle = mock(classOf[Vehicle])
+        val travelTime17 = container.getLinkTravelTime(link, 17 * 3600, person, vehicle)
+        travelTime17 shouldBe 0.4 +- 0.01
+        val travelTime18 = container.getLinkTravelTime(link, 18 * 3600, person, vehicle)
+        travelTime18 shouldBe 44478.89 +- 0.01
+      }
+    }
+  }
+
+  private def readPhysSimEvents
+    : (Network, TravelTimeCalculatorConfigGroup, TravelTimeCalculator, VolumesAnalyzerFixed) = {
+    val config = ConfigUtils.createConfig
+    config.travelTimeCalculator().setMaxTime(245162)
+    val scenario = ScenarioUtils.createScenario(ConfigUtils.createConfig)
+    val network = scenario.getNetwork
+    val matsimNetworkReader = new MatsimNetworkReader(network)
+    matsimNetworkReader.readFile(NETWORK_FILE_PATH)
+
+    val ttConfigGroup: TravelTimeCalculatorConfigGroup = config.travelTimeCalculator
+    val travelTimeCalculator = new TravelTimeCalculator(network, ttConfigGroup)
+    val eventsManager = EventsUtils.createEventsManager
+    eventsManager.addHandler(travelTimeCalculator)
+    val volumeAnalyzer =
+      new VolumesAnalyzerFixed(3600, ttConfigGroup.getMaxTime - 1, network, createVehicleMap().asJava)
+    eventsManager.addHandler(volumeAnalyzer)
+
+    val matsimEventsReader = new MatsimEventsReader(eventsManager)
+    matsimEventsReader.readFile(EVENTS_FILE_PATH)
+    (network, ttConfigGroup, travelTimeCalculator, volumeAnalyzer)
+  }
+
+  def createVehicleMap(): Map[Id[BeamVehicle], BeamVehicle] = {
+    Map(
+      createVehicleWithId("bus:B2-EAST-1", VehicleCategory.MediumDutyPassenger),
+      createVehicleWithId("bus:B2-EAST-6", VehicleCategory.MediumDutyPassenger)
+    ) ++ (1 to 9).map(i => createVehicleWithId(i.toString, VehicleCategory.Car))
+  }
+
+  private def createVehicleWithId(
+    vehicleIdStr: String,
+    vehicleCategory: VehicleCategory
+  ): (Id[BeamVehicle], BeamVehicle) = {
+    val vehicleId = vehicleIdStr.createId[BeamVehicle]
+    val vehicleType = mock(classOf[BeamVehicleType])
+    when(vehicleType.vehicleCategory).thenReturn(vehicleCategory)
+    val vehicle = mock(classOf[BeamVehicle])
+    when(vehicle.beamVehicleType).thenReturn(vehicleType)
+    (vehicleId, vehicle)
+  }
+}

From a58fd1add799be083510ee8efa50003fc394f245 Mon Sep 17 00:00:00 2001
From: Rutvik Patel <heyrutvik@gmail.com>
Date: Mon, 4 Apr 2022 19:32:48 +0530
Subject: [PATCH 37/58] updates freight reader for snapping location

---
 .../agents/freight/input/FreightReader.scala  |  39 ++--
 .../freight/input/GenericFreightReader.scala  | 213 ++++++++++++++----
 src/main/scala/beam/sim/BeamHelper.scala      |  24 +-
 .../beam/utils/SnapCoordinateUtils.scala      |   1 -
 .../utils/scenario/BeamScenarioLoader.scala   |   2 +-
 .../utils/scenario/ScenarioLoaderHelper.scala |  14 +-
 .../scenario/UrbanSimScenarioLoader.scala     |   2 +-
 7 files changed, 215 insertions(+), 80 deletions(-)

diff --git a/src/main/scala/beam/agentsim/agents/freight/input/FreightReader.scala b/src/main/scala/beam/agentsim/agents/freight/input/FreightReader.scala
index 9a88c610ff9..e4981d32b8e 100644
--- a/src/main/scala/beam/agentsim/agents/freight/input/FreightReader.scala
+++ b/src/main/scala/beam/agentsim/agents/freight/input/FreightReader.scala
@@ -12,6 +12,7 @@ import beam.sim.BeamServices
 import beam.sim.common.GeoUtils
 import beam.sim.config.BeamConfig
 import beam.sim.config.BeamConfig.Beam.Agentsim.Agents.Freight
+import beam.utils.SnapCoordinateUtils.SnapLocationHelper
 import com.conveyal.r5.streets.StreetLayer
 import org.matsim.api.core.v01.population._
 import org.matsim.api.core.v01.{Coord, Id}
@@ -185,10 +186,16 @@ object FreightReader {
     beamConfig: BeamConfig,
     geoUtils: GeoUtils,
     streetLayer: StreetLayer,
-    tazMap: TAZTreeMap
+    tazMap: TAZTreeMap,
+    outputDirMaybe: Option[String]
   ): FreightReader = {
     val rand: Random = new Random(beamConfig.matsim.modules.global.randomSeed)
     val config = beamConfig.beam.agentsim.agents.freight
+    val snapLocationHelper: SnapLocationHelper = SnapLocationHelper(
+      geoUtils,
+      streetLayer,
+      beamConfig.beam.routing.r5.linkRadiusMeters
+    )
     beamConfig.beam.agentsim.agents.freight.reader match {
       case "Generic" =>
         new GenericFreightReader(
@@ -196,16 +203,22 @@ object FreightReader {
           geoUtils,
           rand,
           tazMap,
-          Some(ClosestUTMPointOnMap(streetLayer, beamConfig.beam.routing.r5.linkRadiusMeters))
+          snapLocationHelper,
+          outputDirMaybe
         )
       case s =>
         throw new RuntimeException(s"Unknown freight reader $s")
     }
   }
 
-  def apply(beamConfig: BeamConfig, geoUtils: GeoUtils, streetLayer: StreetLayer): FreightReader = {
+  def apply(
+    beamConfig: BeamConfig,
+    geoUtils: GeoUtils,
+    streetLayer: StreetLayer,
+    outputDirMaybe: Option[String]
+  ): FreightReader = {
     val tazMap = TAZTreeMap.getTazTreeMap(beamConfig.beam.agentsim.taz.filePath)
-    apply(beamConfig, geoUtils, streetLayer, tazMap)
+    apply(beamConfig, geoUtils, streetLayer, tazMap, outputDirMaybe)
   }
 
   def apply(beamServices: BeamServices): FreightReader =
@@ -213,21 +226,7 @@ object FreightReader {
       beamServices.beamConfig,
       beamServices.geo,
       beamServices.beamScenario.transportNetwork.streetLayer,
-      beamServices.beamScenario.tazTreeMap
+      beamServices.beamScenario.tazTreeMap,
+      None
     )
-
-  case class ClosestUTMPointOnMap(streetLayer: StreetLayer, r5LinkRadiusMeters: Double) {
-
-    def find(wsgCoord: Coord, geoUtils: GeoUtils): Option[Coord] = {
-      //val wsgCoord = geoUtils.utm2Wgs(utmCoord)
-      val theSplit = geoUtils.getR5Split(streetLayer, wsgCoord, r5LinkRadiusMeters)
-      if (theSplit == null) {
-        None
-      } else {
-        val wgsPointOnMap = geoUtils.splitToCoord(theSplit)
-        val utmCoord = geoUtils.wgs2Utm(wgsPointOnMap)
-        Some(utmCoord)
-      }
-    }
-  }
 }
diff --git a/src/main/scala/beam/agentsim/agents/freight/input/GenericFreightReader.scala b/src/main/scala/beam/agentsim/agents/freight/input/GenericFreightReader.scala
index 25cf0f5f509..5164c93dd82 100644
--- a/src/main/scala/beam/agentsim/agents/freight/input/GenericFreightReader.scala
+++ b/src/main/scala/beam/agentsim/agents/freight/input/GenericFreightReader.scala
@@ -1,19 +1,22 @@
 package beam.agentsim.agents.freight.input
 
 import beam.agentsim.agents.freight._
-import beam.agentsim.agents.freight.input.FreightReader.ClosestUTMPointOnMap
+import beam.agentsim.agents.freight.input.GenericFreightReader.ClosestUTMPoint
 import beam.agentsim.agents.vehicles.{BeamVehicle, BeamVehicleType}
 import beam.agentsim.infrastructure.taz.{TAZ, TAZTreeMap}
 import beam.sim.common.GeoUtils
 import beam.sim.config.BeamConfig.Beam.Agentsim.Agents.Freight
+import beam.utils.SnapCoordinateUtils
+import beam.utils.SnapCoordinateUtils.{Category, Error, ErrorInfo, Result, SnapLocationHelper}
 import beam.utils.csv.GenericCsvReader
 import beam.utils.matsim_conversion.MatsimPlanConversion.IdOps
 import com.typesafe.scalalogging.LazyLogging
+import org.apache.commons.lang3.StringUtils.isBlank
 import org.matsim.api.core.v01.population._
 import org.matsim.api.core.v01.{Coord, Id}
 import org.matsim.households.Household
-import org.apache.commons.lang3.StringUtils.isBlank
 
+import scala.collection.mutable.ListBuffer
 import scala.util.Random
 
 /**
@@ -24,7 +27,8 @@ class GenericFreightReader(
   val geoUtils: GeoUtils,
   rnd: Random,
   tazTree: TAZTreeMap,
-  closestUTMPointOnMapMaybe: Option[ClosestUTMPointOnMap]
+  val snapLocationHelper: SnapLocationHelper,
+  val outputDirMaybe: Option[String]
 ) extends LazyLogging
     with FreightReader {
 
@@ -40,6 +44,8 @@ class GenericFreightReader(
 
   @Override
   def readFreightTours(): Map[Id[FreightTour], FreightTour] = {
+    val errors: ListBuffer[ErrorInfo] = ListBuffer()
+
     val maybeTours = GenericCsvReader
       .readAsSeq[Option[FreightTour]](config.toursFilePath) { row =>
         def get(key: String): String = getRowValue(config.toursFilePath, row, key)
@@ -47,27 +53,70 @@ class GenericFreightReader(
         val tourId: Id[FreightTour] = get("tourId").createId[FreightTour]
         val departureTimeInSec = get("departureTimeInSec").toInt
         val maxTourDurationInSec = get("maxTourDurationInSec").toInt
+        val departureLocationX = row.get("departureLocationX")
+        val departureLocationY = row.get("departureLocationY")
+
+        // note: placeholder to update warehouseLocationTaz and warehouseLocationUTM later
+        val freightTour = FreightTour(
+          tourId,
+          departureTimeInSec,
+          None,
+          new Coord(),
+          maxTourDurationInSec
+        )
+
         extractCoordOrTaz(
-          row.get("departureLocationX"),
-          row.get("departureLocationY"),
+          departureLocationX,
+          departureLocationY,
           row.get("departureLocationZone")
         ) match {
-          case (departureLocationZoneMaybe, Some(departureLocationUTMOnMap)) =>
+          case (departureLocationZoneMaybe, Left(departureLocationZoneUTM)) =>
+            Some(
+              freightTour.copy(
+                warehouseLocationTaz = departureLocationZoneMaybe,
+                warehouseLocationUTM = departureLocationZoneUTM
+              )
+            )
+          case (departureLocationZoneMaybe, Right(Result.Succeed(splitCoord))) =>
             Some(
-              FreightTour(
-                tourId,
-                departureTimeInSec,
-                departureLocationZoneMaybe,
-                departureLocationUTMOnMap,
-                maxTourDurationInSec
+              freightTour.copy(
+                warehouseLocationTaz = departureLocationZoneMaybe,
+                warehouseLocationUTM = splitCoord
+              )
+            )
+          case (_, Right(Result.OutOfBoundingBoxError)) =>
+            errors.append(
+              ErrorInfo(
+                tourId.toString,
+                Category.FreightTour,
+                Error.OutOfBoundingBox,
+                departureLocationX.toDouble,
+                departureLocationY.toDouble
+              )
+            )
+            None
+          case (_, Right(Result.R5SplitNullError)) =>
+            errors.append(
+              ErrorInfo(
+                tourId.toString,
+                Category.FreightTour,
+                Error.R5SplitNull,
+                departureLocationX.toDouble,
+                departureLocationY.toDouble
               )
             )
+            None
           case _ =>
-            logger.error(f"Following freight tour row discarded because departure location is not reachable: $row")
+            logger.error(f"Following freight tour row discarded for unknown reason: $row")
             None
         }
       }
 
+    outputDirMaybe.foreach { path =>
+      if (errors.isEmpty) logger.info("No 'snap location' error to report for freight tours.")
+      else SnapCoordinateUtils.writeToCsv(s"$path/snapLocationFreightTourErrors.csv.gz", errors)
+    }
+
     maybeTours.flatten
       .groupBy(_.tourId)
       .mapValues(_.head)
@@ -75,6 +124,8 @@ class GenericFreightReader(
 
   @Override
   def readPayloadPlans(): Map[Id[PayloadPlan], PayloadPlan] = {
+    val errors: ListBuffer[ErrorInfo] = ListBuffer()
+
     val maybePlans = GenericCsvReader
       .readAsSeq[Option[PayloadPlan]](config.plansFilePath) { row =>
         def get(key: String): String = getRowValue(config.plansFilePath, row, key)
@@ -94,31 +145,66 @@ class GenericFreightReader(
         } else {
           requestType.toString
         }
-        extractCoordOrTaz(row.get("locationX"), row.get("locationY"), row.get("locationZone")) match {
-          case (locationZoneMaybe, Some(locationUTMOnMap)) =>
-            Some(
-              PayloadPlan(
-                get("payloadId").createId,
-                get("sequenceRank").toDouble.round.toInt,
-                get("tourId").createId,
-                get("payloadType").createId[PayloadType],
-                get("weightInKg").toDouble,
-                requestType,
-                activityType,
-                locationZoneMaybe,
-                locationUTMOnMap,
-                get("estimatedTimeOfArrivalInSec").toDouble.toInt,
-                get("arrivalTimeWindowInSecLower").toDouble.toInt,
-                get("arrivalTimeWindowInSecUpper").toDouble.toInt,
-                operationDurationInSec
+
+        val payloadId = get("payloadId").createId[PayloadPlan]
+        val locationX = row.get("locationX")
+        val locationY = row.get("locationY")
+
+        // note: placeholder to update locationZone and locationUTM later
+        val payloadPlan = PayloadPlan(
+          payloadId,
+          get("sequenceRank").toDouble.round.toInt,
+          get("tourId").createId,
+          get("payloadType").createId[PayloadType],
+          get("weightInKg").toDouble,
+          requestType,
+          activityType,
+          None,
+          new Coord(),
+          get("estimatedTimeOfArrivalInSec").toDouble.toInt,
+          get("arrivalTimeWindowInSecLower").toDouble.toInt,
+          get("arrivalTimeWindowInSecUpper").toDouble.toInt,
+          operationDurationInSec
+        )
+
+        extractCoordOrTaz(locationX, locationY, row.get("locationZone")) match {
+          case (locationZoneMaybe, Left(locationZoneUTM)) =>
+            Some(payloadPlan.copy(locationZone = locationZoneMaybe, locationUTM = locationZoneUTM))
+          case (locationZoneMaybe, Right(Result.Succeed(splitCoord))) =>
+            Some(payloadPlan.copy(locationZone = locationZoneMaybe, locationUTM = splitCoord))
+          case (_, Right(Result.OutOfBoundingBoxError)) =>
+            errors.append(
+              ErrorInfo(
+                payloadId.toString,
+                Category.FreightPayloadPlan,
+                Error.OutOfBoundingBox,
+                locationX.toDouble,
+                locationY.toDouble
+              )
+            )
+            None
+          case (_, Right(Result.R5SplitNullError)) =>
+            errors.append(
+              ErrorInfo(
+                payloadId.toString,
+                Category.FreightPayloadPlan,
+                Error.R5SplitNull,
+                locationX.toDouble,
+                locationY.toDouble
               )
             )
+            None
           case _ =>
-            logger.error(f"Following freight plan row discarded because zone location is not reachable: $row")
+            logger.error(f"Following freight payload plan row discarded for unknown reason: $row")
             None
         }
       }
 
+    outputDirMaybe.foreach { path =>
+      if (errors.isEmpty) logger.info("No 'snap location' error to report for freight payload plans.")
+      else SnapCoordinateUtils.writeToCsv(s"$path/snapLocationFreightPayloadPlanErrors.csv.gz", errors)
+    }
+
     maybePlans.flatten
       .groupBy(_.payloadId)
       .mapValues(_.head)
@@ -130,6 +216,7 @@ class GenericFreightReader(
     allPlans: Map[Id[PayloadPlan], PayloadPlan],
     vehicleTypes: Map[Id[BeamVehicleType], BeamVehicleType]
   ): IndexedSeq[FreightCarrier] = {
+    val errors: ListBuffer[ErrorInfo] = ListBuffer()
 
     val existingTours: Set[Id[FreightTour]] = allTours.keySet.intersect(allPlans.map(_._2.tourId).toSet)
     val plans: Map[Id[PayloadPlan], PayloadPlan] = allPlans.filter { case (_, plan) =>
@@ -208,18 +295,55 @@ class GenericFreightReader(
         logger.error(f"Following freight carrier row discarded because tour $tourId was filtered out: $row")
         None
       } else {
+        val warehouseX = row.get("warehouseX")
+        val warehouseY = row.get("warehouseY")
+
+        // note: placeholder to update warehouseLocationZone and warehouseLocationUTM later
+        val freightCarrier = FreightCarrierRow(carrierId, tourId, vehicleId, vehicleTypeId, None, new Coord())
+
         extractCoordOrTaz(row.get("warehouseX"), row.get("warehouseY"), row.get("warehouseZone")) match {
-          case (warehouseZoneMaybe, Some(warehouseUTMOnMap)) =>
-            Some(FreightCarrierRow(carrierId, tourId, vehicleId, vehicleTypeId, warehouseZoneMaybe, warehouseUTMOnMap))
-          case (_, warehouseXYMaybe) =>
-            logger.error(
-              f"Following freight carrier row discarded because warehouse location ($warehouseXYMaybe) is not reachable: $row"
+          case (warehouseZoneMaybe, Left(warehouseZoneUTM)) =>
+            Some(
+              freightCarrier.copy(warehouseLocationZone = warehouseZoneMaybe, warehouseLocationUTM = warehouseZoneUTM)
+            )
+          case (warehouseZoneMaybe, Right(Result.Succeed(splitCoord))) =>
+            Some(
+              freightCarrier.copy(warehouseLocationZone = warehouseZoneMaybe, warehouseLocationUTM = splitCoord)
+            )
+          case (_, Right(Result.OutOfBoundingBoxError)) =>
+            errors.append(
+              ErrorInfo(
+                carrierId.toString,
+                Category.FreightCarrier,
+                Error.OutOfBoundingBox,
+                warehouseX.toDouble,
+                warehouseY.toDouble
+              )
             )
             None
+          case (_, Right(Result.R5SplitNullError)) =>
+            errors.append(
+              ErrorInfo(
+                carrierId.toString,
+                Category.FreightCarrier,
+                Error.R5SplitNull,
+                warehouseX.toDouble,
+                warehouseY.toDouble
+              )
+            )
+            None
+          case _ =>
+            logger.error("Following freight carrier row is discarded for unknown reason: {}", row)
+            None
         }
       }
     }
 
+    outputDirMaybe.foreach { path =>
+      if (errors.isEmpty) logger.info("No 'snap location' error to report for freight carriers.")
+      else SnapCoordinateUtils.writeToCsv(s"$path/snapLocationFreightCarrierErrors.csv.gz", errors)
+    }
+
     val carriersWithFleet = maybeCarrierRows.flatten
       .groupBy(_.carrierId)
       .map { case (carrierId, carrierRows) =>
@@ -238,16 +362,12 @@ class GenericFreightReader(
   private def getDistributedTazLocation(taz: TAZ): Coord =
     convertedLocation(TAZTreeMap.randomLocationInTAZ(taz, rnd))
 
-  private def getLocationFromCoord(location: Coord): Option[Coord] = {
-    closestUTMPointOnMapMaybe.map(_.find(location, geoUtils)).getOrElse(Some(geoUtils.wgs2Utm(location)))
-  }
-
-  private def extractCoordOrTaz(strX: String, strY: String, strZone: String): (Option[Id[TAZ]], Option[Coord]) = {
+  private def extractCoordOrTaz(strX: String, strY: String, strZone: String): (Option[Id[TAZ]], ClosestUTMPoint) = {
     if (isBlank(strX) || isBlank(strY)) {
       val taz = getTaz(strZone)
-      (Some(taz.tazId), Some(getDistributedTazLocation(taz)))
+      (Some(taz.tazId), Left(getDistributedTazLocation(taz)))
     } else {
-      (None, getLocationFromCoord(location(strX.toDouble, strY.toDouble)))
+      (None, Right(snapLocationHelper.computeResult(location(strX.toDouble, strY.toDouble))))
     }
   }
 
@@ -270,3 +390,10 @@ class GenericFreightReader(
   }
 
 }
+
+object GenericFreightReader {
+
+  // either[tazLocation, snapLocationResult]
+  type ClosestUTMPoint = Either[Coord, Result]
+
+}
diff --git a/src/main/scala/beam/sim/BeamHelper.scala b/src/main/scala/beam/sim/BeamHelper.scala
index 8a1c2527258..3ee36ff1789 100755
--- a/src/main/scala/beam/sim/BeamHelper.scala
+++ b/src/main/scala/beam/sim/BeamHelper.scala
@@ -269,7 +269,7 @@ trait BeamHelper extends LazyLogging {
     )
   }
 
-  def loadScenario(beamConfig: BeamConfig): BeamScenario = {
+  def loadScenario(beamConfig: BeamConfig, outputDirMaybe: Option[String] = None): BeamScenario = {
     val vehicleTypes = maybeScaleTransit(beamConfig, readBeamVehicleTypeFile(beamConfig))
     val vehicleCsvReader = new VehicleCsvReader(beamConfig)
     val baseFilePath = Paths.get(beamConfig.beam.agentsim.agents.vehicles.vehicleTypesFilePath).getParent
@@ -300,7 +300,7 @@ trait BeamHelper extends LazyLogging {
     val linkToTAZMapping: Map[Link, TAZ] = LinkLevelOperations.getLinkToTazMapping(networkCoordinator.network, tazMap)
 
     val (freightCarriers, fixedActivitiesDurationsFromFreight) =
-      readFreights(beamConfig, networkCoordinator.transportNetwork.streetLayer, vehicleTypes)
+      readFreights(beamConfig, networkCoordinator.transportNetwork.streetLayer, vehicleTypes, outputDirMaybe)
 
     val fixedActivitiesDurationsFromConfig: Map[String, Double] = {
       val maybeFixedDurationsList = beamConfig.beam.agentsim.agents.activities.activityTypeToFixedDurationMap
@@ -344,13 +344,14 @@ trait BeamHelper extends LazyLogging {
   def readFreights(
     beamConfig: BeamConfig,
     streetLayer: StreetLayer,
-    vehicleTypes: Map[Id[BeamVehicleType], BeamVehicleType]
+    vehicleTypes: Map[Id[BeamVehicleType], BeamVehicleType],
+    outputDirMaybe: Option[String]
   ): (IndexedSeq[FreightCarrier], Map[String, Double]) = {
     val freightConfig = beamConfig.beam.agentsim.agents.freight
 
     if (freightConfig.enabled) {
       val geoUtils = new GeoUtilsImpl(beamConfig)
-      val freightReader = FreightReader(beamConfig, geoUtils, streetLayer)
+      val freightReader = FreightReader(beamConfig, geoUtils, streetLayer, outputDirMaybe)
       val tours = freightReader.readFreightTours()
       val plans = freightReader.readPayloadPlans()
       val carriers = freightReader.readFreightCarriers(tours, plans, vehicleTypes)
@@ -721,7 +722,8 @@ trait BeamHelper extends LazyLogging {
         beamServices.beamConfig,
         beamServices.geo,
         beamServices.beamScenario.transportNetwork.streetLayer,
-        beamServices.beamScenario.tazTreeMap
+        beamServices.beamScenario.tazTreeMap,
+        Some(outputDir)
       )
       generatePopulationForPayloadPlans(
         beamScenario,
@@ -802,12 +804,12 @@ trait BeamHelper extends LazyLogging {
     val scenarioConfig = beamConfig.beam.exchange.scenario
     val src = scenarioConfig.source.toLowerCase
     val fileFormat = scenarioConfig.fileFormat
-    val outputDir = matsimConfig.controler().getOutputDirectory
+    val outputDirOpt = Option(matsimConfig.controler().getOutputDirectory)
 
     val (scenario, beamScenario, plansMerged) =
       ProfilingUtils.timed(s"Load scenario using $src/$fileFormat", x => logger.info(x)) {
         if (src == "urbansim" || src == "urbansim_v2" || src == "generic") {
-          val beamScenario = loadScenario(beamConfig)
+          val beamScenario = loadScenario(beamConfig, outputDirOpt)
           val emptyScenario = ScenarioBuilder(matsimConfig, beamScenario.network).build
           val geoUtils = new GeoUtilsImpl(beamConfig)
           val (scenario, plansMerged) = {
@@ -862,7 +864,7 @@ trait BeamHelper extends LazyLogging {
                 source,
                 new GeoUtilsImpl(beamConfig),
                 Some(merger),
-                Some(outputDir)
+                outputDirOpt
               ).loadScenario()
             if (src == "urbansim_v2") {
               new ScenarioAdjuster(
@@ -877,7 +879,7 @@ trait BeamHelper extends LazyLogging {
         } else if (src == "beam") {
           fileFormat match {
             case "csv" =>
-              val beamScenario = loadScenario(beamConfig)
+              val beamScenario = loadScenario(beamConfig, outputDirOpt)
               val scenario = {
                 val source = new BeamScenarioSource(
                   beamConfig,
@@ -889,13 +891,13 @@ trait BeamHelper extends LazyLogging {
                   beamScenario,
                   source,
                   new GeoUtilsImpl(beamConfig),
-                  Some(outputDir)
+                  outputDirOpt
                 )
                   .loadScenario()
               }.asInstanceOf[MutableScenario]
               (scenario, beamScenario, false)
             case "xml" =>
-              val beamScenario = loadScenario(beamConfig)
+              val beamScenario = loadScenario(beamConfig, outputDirOpt)
               val scenario = {
                 val result = ScenarioUtils.loadScenario(matsimConfig).asInstanceOf[MutableScenario]
                 fixDanglingPersons(result)
diff --git a/src/main/scala/beam/utils/SnapCoordinateUtils.scala b/src/main/scala/beam/utils/SnapCoordinateUtils.scala
index 8f0ef293db1..36db7dbb044 100644
--- a/src/main/scala/beam/utils/SnapCoordinateUtils.scala
+++ b/src/main/scala/beam/utils/SnapCoordinateUtils.scala
@@ -2,7 +2,6 @@ package beam.utils
 
 import beam.sim.common.GeoUtils
 import beam.utils.csv.CsvWriter
-import beam.utils.scenario.PlanElement
 import com.conveyal.r5.streets.StreetLayer
 import com.typesafe.scalalogging.LazyLogging
 import org.matsim.api.core.v01.Coord
diff --git a/src/main/scala/beam/utils/scenario/BeamScenarioLoader.scala b/src/main/scala/beam/utils/scenario/BeamScenarioLoader.scala
index 37292467d10..76100ebf179 100644
--- a/src/main/scala/beam/utils/scenario/BeamScenarioLoader.scala
+++ b/src/main/scala/beam/utils/scenario/BeamScenarioLoader.scala
@@ -30,7 +30,7 @@ class BeamScenarioLoader(
   var beamScenario: BeamScenario,
   val scenarioSource: ScenarioSource,
   val geo: GeoUtils,
-  val outputDirOpt: Option[String]
+  val outputDirMaybe: Option[String]
 ) extends ScenarioLoaderHelper {
 
   import BeamScenarioLoader._
diff --git a/src/main/scala/beam/utils/scenario/ScenarioLoaderHelper.scala b/src/main/scala/beam/utils/scenario/ScenarioLoaderHelper.scala
index 9bfb6fa95a1..2113554ca44 100644
--- a/src/main/scala/beam/utils/scenario/ScenarioLoaderHelper.scala
+++ b/src/main/scala/beam/utils/scenario/ScenarioLoaderHelper.scala
@@ -13,7 +13,7 @@ trait ScenarioLoaderHelper extends LazyLogging {
 
   def beamScenario: BeamScenario
   def geo: GeoUtils
-  def outputDirOpt: Option[String]
+  def outputDirMaybe: Option[String]
 
   private val snapLocationHelper: SnapLocationHelper = SnapLocationHelper(
     geo,
@@ -112,7 +112,11 @@ trait ScenarioLoaderHelper extends LazyLogging {
 
   protected def validatePlans(plans: Iterable[PlanElement]): Iterable[PlanElement] = {
     val Processed(validPlans, errors) = snapLocationPlans(plans)
-    outputDirOpt.foreach(path => SnapCoordinateUtils.writeToCsv(s"$path/snapLocationPlanErrors.csv", errors))
+
+    outputDirMaybe.foreach { path =>
+      if (errors.isEmpty) logger.info("No 'snap location' error to report for scenario plans.")
+      else SnapCoordinateUtils.writeToCsv(s"$path/snapLocationPlanErrors.csv.gz", errors)
+    }
 
     val filteredCnt = plans.size - validPlans.size
     if (filteredCnt > 0) {
@@ -123,7 +127,11 @@ trait ScenarioLoaderHelper extends LazyLogging {
 
   protected def validateHouseholds(households: Iterable[HouseholdInfo]): Iterable[HouseholdInfo] = {
     val Processed(validHouseholds, errors) = snapLocationHouseholds(households)
-    outputDirOpt.foreach(path => SnapCoordinateUtils.writeToCsv(s"$path/snapLocationHouseholdErrors.csv", errors))
+
+    outputDirMaybe.foreach { path =>
+      if (errors.isEmpty) logger.info("No 'snap location' error to report for scenario households.")
+      else SnapCoordinateUtils.writeToCsv(s"$path/snapLocationHouseholdErrors.csv.gz", errors)
+    }
 
     val filteredCnt = households.size - validHouseholds.size
     if (filteredCnt > 0) {
diff --git a/src/main/scala/beam/utils/scenario/UrbanSimScenarioLoader.scala b/src/main/scala/beam/utils/scenario/UrbanSimScenarioLoader.scala
index 615f07dcb50..16268ac5564 100644
--- a/src/main/scala/beam/utils/scenario/UrbanSimScenarioLoader.scala
+++ b/src/main/scala/beam/utils/scenario/UrbanSimScenarioLoader.scala
@@ -31,7 +31,7 @@ class UrbanSimScenarioLoader(
   val scenarioSource: ScenarioSource,
   val geo: GeoUtils,
   val previousRunPlanMerger: Option[PreviousRunPlanMerger] = None,
-  val outputDirOpt: Option[String]
+  val outputDirMaybe: Option[String]
 ) extends ScenarioLoaderHelper {
 
   private implicit val ex: ExecutionContext = scala.concurrent.ExecutionContext.Implicits.global

From f4913fbb890d81412cb93e568eac5a47d2454db3 Mon Sep 17 00:00:00 2001
From: Haitam Laarabi <10712736+htmlrb@users.noreply.github.com>
Date: Mon, 4 Apr 2022 11:38:17 -0700
Subject: [PATCH 38/58] updating deploy

---
 gradle.deploy.properties                |  5 +-
 src/main/R/freight/freight-processing.R | 92 +++++++++++++------------
 2 files changed, 52 insertions(+), 45 deletions(-)

diff --git a/gradle.deploy.properties b/gradle.deploy.properties
index 0f687ec06c7..1940d898f85 100644
--- a/gradle.deploy.properties
+++ b/gradle.deploy.properties
@@ -7,9 +7,10 @@ instanceType=r5.8xlarge
 # shutdownBehaviour = stop | terminate
 shutdownBehaviour=terminate
 s3Backup=true
-maxRAM=230g
+maxRAM=740g
 #storageSize (in GiB) =  any number between 64 and 256
-storageSize=128
+storageSize=256
+profiler_type=cpumem
 
 #r5.8xlarge (32/256)
 #c5.9xlarge (36/72)    -> 5 instances -> $1.53 per Hour
diff --git a/src/main/R/freight/freight-processing.R b/src/main/R/freight/freight-processing.R
index 2a7b43e89a4..573adca159c 100644
--- a/src/main/R/freight/freight-processing.R
+++ b/src/main/R/freight/freight-processing.R
@@ -14,6 +14,8 @@ library(stringr)
 activitySimDir <- normalizePath("~/Data/ACTIVITYSIM")
 freightDir <- normalizePath("~/Data/FREIGHT")
 freightWorkDir <- normalizePath(paste(freightDir,"/2022-03-09/output",sep=""))
+
+
 source("~/Documents/Workspace/scripts/common/keys.R")
 register_google(key = google_api_key_1)
 
@@ -206,46 +208,50 @@ p <- ggplot(to_plot, aes(x=category,y=VMT,fill=category))+
 ggsave(pp(freightWorkDir,'/freight-avg-vmt-by-category.png'),p,width=4,height=3,units='in')
 
 
-# ***************************
-# FRISM
-# ***************************
-# freightWorkDir <- normalizePath(paste(freightDir,"/Outputs_All_SF_0223_2022_merged",sep=""))
-# carriers <- readCsv(pp(freightWorkDir, "/freight-merged-carriers.csv"))
-# payload <- readCsv(pp(freightWorkDir, "/freight-merged-payload-plans.csv"))
-# trous <- readCsv(pp(freightWorkDir, "/freight-merged-tours.csv"))
-# 
-# plans <- rbind(plansb2b,plansb2c)[,c("payloadId","sequenceRank","tourId","payloadType","estimatedTimeOfArrivalInSec","arrivalTimeWindowInSec_lower","business")]
-# tours <- rbind(toursb2b,toursb2c)[,c("tourId","departureTimeInSec","business")]
-# 
-# p <- plans[tours, on=c("tourId","business")]
-# pRank2 <- p[sequenceRank==2]
-# pRank2$negativeTravelTime <- pRank2$estimatedTimeOfArrivalInSec - pRank2$departureTimeInSec - pRank2$arrivalTimeWindowInSec_lower
-# p2 <- p[,diff:=estimatedTimeOfArrivalInSec-arrivalTimeWindowInSec_lower-departureTimeInSec]
-# 
-# write.csv(
-#   pRank2[negativeTravelTime<0],
-#   file = pp(freightDir, "/10percent/negativeTravelTime.csv"),
-#   row.names=FALSE,
-#   quote=FALSE,
-#   na="0")
-# 
-# toursb2$hour <- toursb2c$departureTimeInSec/3600.0
-# ggplot(toursb2b[,.N,by=hour]) + geom_line(aes(hour, N))
-# 
-# payload$business <- "B2B"
-# payload[startsWith(payloadId,"b2c")]$business <- "B2C"
-# payload[,time24:=estimatedTimeOfArrivalInSec%%(24*3600),][,.N,by=.(timeBin=as.POSIXct(cut(toDateTime(time24),"30 min")), business)] %>% 
-#   ggplot(aes(timeBin, N, colour=business)) +
-#   geom_line() + 
-#   scale_x_datetime("Hour", 
-#                    breaks=scales::date_breaks("2 hour"), 
-#                    labels=scales::date_format("%H", tz = dateTZ)) +
-#   scale_y_continuous("Activity", breaks = scales::pretty_breaks()) +
-#   scale_colour_manual("Supply Chain", values = c("#eca35b", "#20b2aa")) +
-#   theme_marain() +
-#   theme(legend.title = element_text(size = 10),
-#         legend.text = element_text(size = 10),
-#         axis.text.x = element_text(angle = 0, hjust = 1))
+#***************************
+#FRISM
+#***************************
+freightWorkDir <- normalizePath(paste(freightDir,"/Outputs(All SF)_0322_2022_merged",sep=""))
+carriers <- readCsv(pp(freightWorkDir, "/freight-merged-carriers.csv"))
+payload <- readCsv(pp(freightWorkDir, "/freight-merged-payload-plans.csv"))
+tours <- readCsv(pp(freightWorkDir, "/freight-merged-tours.csv"))
+
+print(paste("# carriers:", length(unique(carriers$carrierId))))
+print(paste("# tours:", length(unique(carriers$tourId))))
+print(paste("# vehicleId:", length(unique(carriers$vehicleId))))
+
+plans <- rbind(plansb2b,plansb2c)[,c("payloadId","sequenceRank","tourId","payloadType","estimatedTimeOfArrivalInSec","arrivalTimeWindowInSec_lower","business")]
+tours <- rbind(toursb2b,toursb2c)[,c("tourId","departureTimeInSec","business")]
+
+p <- plans[tours, on=c("tourId","business")]
+pRank2 <- p[sequenceRank==2]
+pRank2$negativeTravelTime <- pRank2$estimatedTimeOfArrivalInSec - pRank2$departureTimeInSec - pRank2$arrivalTimeWindowInSec_lower
+p2 <- p[,diff:=estimatedTimeOfArrivalInSec-arrivalTimeWindowInSec_lower-departureTimeInSec]
+
+write.csv(
+  pRank2[negativeTravelTime<0],
+  file = pp(freightDir, "/10percent/negativeTravelTime.csv"),
+  row.names=FALSE,
+  quote=FALSE,
+  na="0")
+
+toursb2$hour <- toursb2c$departureTimeInSec/3600.0
+ggplot(toursb2b[,.N,by=hour]) + geom_line(aes(hour, N))
+
+payload$business <- "B2B"
+payload[startsWith(payloadId,"b2c")]$business <- "B2C"
+payload[,time24:=estimatedTimeOfArrivalInSec%%(24*3600),][,.N,by=.(timeBin=as.POSIXct(cut(toDateTime(time24),"30 min")), business)] %>%
+  ggplot(aes(timeBin, N, colour=business)) +
+  geom_line() +
+  scale_x_datetime("Hour",
+                   breaks=scales::date_breaks("2 hour"),
+                   labels=scales::date_format("%H", tz = dateTZ)) +
+  scale_y_continuous("Activity", breaks = scales::pretty_breaks()) +
+  scale_colour_manual("Supply Chain", values = c("#eca35b", "#20b2aa")) +
+  theme_marain() +
+  theme(legend.title = element_text(size = 10),
+        legend.text = element_text(size = 10),
+        axis.text.x = element_text(angle = 0, hjust = 1))
 
 
 ################ ***************************
@@ -280,6 +286,7 @@ truck_aadtt_with_linkData <- merge(data.table::as.data.table(truck_aadtt_with_li
 
 
 ##### PREPARING BEAM/LINKSTAT DATA
+linkStats <- readCsv(normalizePath(paste(freightDir,"/validation/beam/0.linkstats.csv.gz",sep="")))
 linkstats_noFreight <- readCsv(normalizePath(paste(freightDir,"/validation/beam/0.linkstats.nofreight.csv.gz",sep="")))
 linkstats_wFreight <- readCsv(normalizePath(paste(freightDir,"/validation/beam/0.linkstats.withfreight.csv.gz",sep="")))
 #totVolume <- sum(linkstats_wFreight$volume) - sum(linkstats_noFreight$volume)
@@ -334,12 +341,11 @@ sfBayTAZs <- st_read(pp(freightDir, "/validation/TAZs/Transportation_Analysis_Zo
 ################ validation HPMS
 ################ ***************************
 
-source("~/Documents/Workspace/scripts/common/keys.R")
 sf_hpms <- st_read(pp(freightDir, "/validation/sf_hpms_inventory_clipped.geojson"))
 
-Volume_beam <- sum(linkStats$truck_volume)
+Volume_beam <- sum(linkStats$TruckVolume)
 Volume_hpms <- sum(sf_hpms$AADT_Combi+sf_hpms$AADT_Singl)
-VMT_beam <- sum(linkStats$truck_volume * linkStats$lengthInMeter/1609)
+VMT_beam <- sum(linkStats$TruckVolume * linkStats$length/1609)
 VMT_hpms <- (sum((sf_hpms$AADT_Combi+sf_hpms$AADT_Singl) * as.numeric(st_length(sf_hpms))/1609))
 
 st_length(sf_hpms[1,])/1609

From 3bfe012bd83923fdc04b157995c0edf08cf25d3c Mon Sep 17 00:00:00 2001
From: Haitam Laarabi <10712736+htmlrb@users.noreply.github.com>
Date: Mon, 4 Apr 2022 21:25:23 -0700
Subject: [PATCH 39/58] merge

---
 src/main/R/common/helpers.R             |   4 +-
 src/main/R/freight/freight-processing.R | 185 ++++++++++++++----------
 2 files changed, 107 insertions(+), 82 deletions(-)

diff --git a/src/main/R/common/helpers.R b/src/main/R/common/helpers.R
index f032b1d614a..63f96a50f46 100644
--- a/src/main/R/common/helpers.R
+++ b/src/main/R/common/helpers.R
@@ -170,8 +170,8 @@ assignLinkIdToTruckAADTT <- function(NETWORK_CLEANED, NETWORK_CRS, TRUCK_AADTT,
   counter1 <- 0
   counter2 <- 0
   for (row in 1:nrow(truck_aadtt_sf)) {
-    if(row %% 100) {
-      print(paste(row, " entrees have been processed so far!"))
+    if(row %% 100 == 0) {
+      print(paste(row, " entries have been processed so far!"))
     }
     current_pm <- truck_aadtt_sf[row,]
     currentDist <- 20
diff --git a/src/main/R/freight/freight-processing.R b/src/main/R/freight/freight-processing.R
index 573adca159c..2a98c99a86c 100644
--- a/src/main/R/freight/freight-processing.R
+++ b/src/main/R/freight/freight-processing.R
@@ -13,29 +13,32 @@ library(stringr)
 
 activitySimDir <- normalizePath("~/Data/ACTIVITYSIM")
 freightDir <- normalizePath("~/Data/FREIGHT")
-freightWorkDir <- normalizePath(paste(freightDir,"/2022-03-09/output",sep=""))
-
-
-source("~/Documents/Workspace/scripts/common/keys.R")
-register_google(key = google_api_key_1)
-
-
-oaklandMap <- ggmap::get_googlemap("oakland california", zoom = 13, maptype = "roadmap")
-shpFile <- pp(activitySimDir, "/__San_Francisco_Bay_Region_2010_Census_Block_Groups-shp/641aa0d4-ce5b-4a81-9c30-8790c4ab8cfb202047-1-wkkklf.j5ouj.shp")
-sfBayCbg <- st_read(shpFile)
-
-
-events <- readCsv(pp(freightWorkDir, "/0.events.csv.gz"))
-pt <- events[type=="PathTraversal"][,c("time","type","vehicleType","vehicle","secondaryFuelLevel",
+validationDir <- normalizePath("~/Data/FREIGHT/validation")
+freightWorkDir <- normalizePath(paste(validationDir,"/beam",sep=""))
+
+# events <- readCsv(pp(freightWorkDir, "/0.events.csv"))
+# events_filtered <- events[(actType %in% c("Warehouse", "Unloading", "Loading")) | (type=="PathTraversal" & startsWith(vehicle,"freight"))]
+# write.csv(
+#   events_filtered,
+#   file = pp(freightWorkDir, "/filtered.0.events.csv"),
+#   row.names=F,
+#   quote=T)
+events_filtered <- readCsv(pp(freightWorkDir, "/filtered.0.events.csv"))
+pt <- events_filtered[type=="PathTraversal"][,c("time","type","vehicleType","vehicle","secondaryFuelLevel",
                                        "primaryFuelLevel","driver","mode","seatingCapacity","startX",
                                        "startY", "endX", "endY", "capacity", "arrivalTime", "departureTime",
                                        "secondaryFuel", "secondaryFuelType", "primaryFuelType",
                                        "numPassengers", "length", "primaryFuel")]
 freight_pt <- pt[startsWith(vehicle,"freight")]
 
-## DEBUG
+if (nrow(freight_pt[grepl("-emergency-",vehicle)]) > 0) {
+  println("This is a bug")
+}
+print(paste("# vehicles: ", length(unique(freight_pt$vehicle))))
+print("By category: ")
+freight_pt[,.N,by=.(vehicle, vehicleType)][,.(count=.N),by=.(vehicleType)]
+
 # nrow(freight_pt)
-# nrow(freight_pt[grepl("-emergency-",vehicle)])
 # all_pt_x <- data.table::as.data.table(rbind(b2b_pt,b2c_pt)[,c("time","vehicle","departureTime","arrivalTime","label")])
 # all_pt_1 <- all_pt_x[,-c("arrivalTime")][order(time),`:=`(IDX = 1:.N),by=vehicle]
 # all_pt_2 <- all_pt_x[,-c("departureTime")][order(time),`:=`(IDX = 1+1:.N),by=vehicle]
@@ -50,6 +53,72 @@ freight_pt <- pt[startsWith(vehicle,"freight")]
 # nrow(warehouse[type=="actend"])
 ##
 
+## ***************************
+#FRISM
+## ***************************
+freightWorkDir <- normalizePath(paste(freightDir,"/Outputs(All SF)_0322_2022_merged",sep=""))
+carriers <- readCsv(pp(freightWorkDir, "/freight-merged-carriers.csv"))
+payload <- readCsv(pp(freightWorkDir, "/freight-merged-payload-plans.csv"))
+tours <- readCsv(pp(freightWorkDir, "/freight-merged-tours.csv"))
+
+print(paste("# carriers:", length(unique(carriers$carrierId))))
+print(paste("# tours:", length(unique(carriers$tourId))))
+print(paste("# vehicles:", length(unique(carriers$vehicleId))))
+print("By category: ")
+carriers[,.N,by=.(vehicleId, vehicleTypeId)][,.(count=.N),by=.(vehicleTypeId)]
+
+plans <- rbind(plansb2b,plansb2c)[,c("payloadId","sequenceRank","tourId","payloadType","estimatedTimeOfArrivalInSec","arrivalTimeWindowInSec_lower","business")]
+tours <- rbind(toursb2b,toursb2c)[,c("tourId","departureTimeInSec","business")]
+
+p <- plans[tours, on=c("tourId","business")]
+pRank2 <- p[sequenceRank==2]
+pRank2$negativeTravelTime <- pRank2$estimatedTimeOfArrivalInSec - pRank2$departureTimeInSec - pRank2$arrivalTimeWindowInSec_lower
+p2 <- p[,diff:=estimatedTimeOfArrivalInSec-arrivalTimeWindowInSec_lower-departureTimeInSec]
+
+write.csv(
+  pRank2[negativeTravelTime<0],
+  file = pp(freightDir, "/10percent/negativeTravelTime.csv"),
+  row.names=FALSE,
+  quote=FALSE,
+  na="0")
+
+toursb2$hour <- toursb2c$departureTimeInSec/3600.0
+ggplot(toursb2b[,.N,by=hour]) + geom_line(aes(hour, N))
+
+payload$business <- "B2B"
+payload[startsWith(payloadId,"b2c")]$business <- "B2C"
+payload[,time24:=estimatedTimeOfArrivalInSec%%(24*3600),][,.N,by=.(timeBin=as.POSIXct(cut(toDateTime(time24),"30 min")), business)] %>%
+  ggplot(aes(timeBin, N, colour=business)) +
+  geom_line() +
+  scale_x_datetime("Hour",
+                   breaks=scales::date_breaks("2 hour"),
+                   labels=scales::date_format("%H", tz = dateTZ)) +
+  scale_y_continuous("Activity", breaks = scales::pretty_breaks()) +
+  scale_colour_manual("Supply Chain", values = c("#eca35b", "#20b2aa")) +
+  theme_marain() +
+  theme(legend.title = element_text(size = 10),
+        legend.text = element_text(size = 10),
+        axis.text.x = element_text(angle = 0, hjust = 1))
+
+
+payload_carriers <- payload[carriers, on="tourId"]
+payload_carriers$vehicleCategory <- "LightDutyTruck"
+payload_carriers[vehicleTypeId == "FREIGHT-2"]$vehicleCategory <- "HeavyDutyTruck"
+p <- payload_carriers[,time24:=estimatedTimeOfArrivalInSec%%(24*3600),][,.N,by=.(timeBin=as.POSIXct(cut(toDateTime(time24),"30 min")), vehicleCategory)] %>%
+  ggplot(aes(timeBin, N, colour=vehicleCategory)) +
+  geom_line() +
+  scale_x_datetime("Hour",
+                   breaks=scales::date_breaks("2 hour"),
+                   labels=scales::date_format("%H", tz = dateTZ)) +
+  scale_y_continuous("Activity", breaks = scales::pretty_breaks()) +
+  scale_colour_manual("Vehicle Category", values = c("#eca35b", "#20b2aa")) +
+  theme_marain() +
+  theme(legend.title = element_text(size = 10),
+        legend.text = element_text(size = 10),
+        axis.text.x = element_text(angle = 0, hjust = 1))
+ggsave(pp(freightWorkDir,'/output/frism-activity-by-category.png'),p,width=6,height=3,units='in')
+
+
 ## MODE SPLIT
 # pt$mode2 <- "Transit"
 # pt[mode=="car"]$mode2 <- "Car"
@@ -72,18 +141,18 @@ freight_pt <- pt[startsWith(vehicle,"freight")]
 # ***************************
 # B2B vs B2C
 # ***************************
+b2b_pt <- freight_pt[grepl("b2b",vehicle)][,label:="B2B"]
+b2c_pt <- freight_pt[grepl("b2c",vehicle)][,label:="B2C"]
+
+## FREIGHT B2C GEO DISTRIBUTION OF STOPS
 #countyNames <- c('Alameda County','Contra Costa County','Marin County','Napa County','Santa Clara County','San Francisco County','San Mateo County','Sonoma County','Solano County')
 source("~/Documents/Workspace/scripts/common/keys.R")
 register_google(key = google_api_key_1)
 oakland_map <- ggmap::get_googlemap("alameda california", zoom = 9, maptype = "roadmap",color = "bw", style = c(feature = "all", element = "labels", visibility = "off"))
-
-b2b_pt <- freight_pt[grepl("b2b",vehicle)][,label:="B2B"]
-b2c_pt <- freight_pt[grepl("b2c",vehicle)][,label:="B2C"]
-
+shpFile <- pp(activitySimDir, "/__San_Francisco_Bay_Region_2010_Census_Block_Groups-shp/641aa0d4-ce5b-4a81-9c30-8790c4ab8cfb202047-1-wkkklf.j5ouj.shp")
+sfBayCbg <- st_read(shpFile)
 b2b_pt_stops <- clusteringFreightBy(b2b_pt,c("endX","endY"),sfBayCbg,50,"B2B")
 b2c_pt_stops <- clusteringFreightBy(b2c_pt,c("endX","endY"),sfBayCbg,50,"B2C")
-
-## FREIGHT B2C GEO DISTRIBUTION OF STOPS
 to_plot <- rbind(b2c_pt_stops)
 hours_to_show <- c(8, 14, 20)
 toplot <- to_plot[hour %in% hours_to_show]
@@ -112,7 +181,7 @@ p <- oakland_map %>%
         strip.text.x = element_text(size = 10)) +
   facet_wrap(~factor(hour.label, levels=hour.label_order)) +
   guides(color= guide_legend(), size=guide_legend())
-ggsave(pp(freightWorkDir,'/b2c-stops.png'),p,width=9,height=5,units='in')
+ggsave(pp(freightWorkDir,'/output/b2c-stops.png'),p,width=9,height=5,units='in')
 
 
 ## FREIGHT B2B GEO DISTRIBUTION OF STOPS
@@ -135,13 +204,6 @@ p <- oakland_map %>%
 ggsave(pp(freightWorkDir,'/b2b-stops.png'),p,width=4,height=5,units='in')
 
 
-to_plot <- rbind(b2c_pt_stops,b2b_pt_stops)[,.(stopsPerHour=sum(cluster)),by=.(label,hour)]
-ggplot(to_plot) + geom_line(aes(hour, stopsPerHour, colour=label))
-
-to_plot <- rbind(b2b_pt,b2c_pt)
-to_plot$timeBin <- as.integer(to_plot$time/1800)
-
-
 ## FREIGHT ACTIVITY
 to_plot <- rbind(b2b_pt,b2c_pt)
 p <- to_plot[,time24:=arrivalTime%%(24*3600),][,.N,by=.(timeBin=as.POSIXct(cut(toDateTime(time24),"30 min")), label)] %>% 
@@ -208,51 +270,6 @@ p <- ggplot(to_plot, aes(x=category,y=VMT,fill=category))+
 ggsave(pp(freightWorkDir,'/freight-avg-vmt-by-category.png'),p,width=4,height=3,units='in')
 
 
-#***************************
-#FRISM
-#***************************
-freightWorkDir <- normalizePath(paste(freightDir,"/Outputs(All SF)_0322_2022_merged",sep=""))
-carriers <- readCsv(pp(freightWorkDir, "/freight-merged-carriers.csv"))
-payload <- readCsv(pp(freightWorkDir, "/freight-merged-payload-plans.csv"))
-tours <- readCsv(pp(freightWorkDir, "/freight-merged-tours.csv"))
-
-print(paste("# carriers:", length(unique(carriers$carrierId))))
-print(paste("# tours:", length(unique(carriers$tourId))))
-print(paste("# vehicleId:", length(unique(carriers$vehicleId))))
-
-plans <- rbind(plansb2b,plansb2c)[,c("payloadId","sequenceRank","tourId","payloadType","estimatedTimeOfArrivalInSec","arrivalTimeWindowInSec_lower","business")]
-tours <- rbind(toursb2b,toursb2c)[,c("tourId","departureTimeInSec","business")]
-
-p <- plans[tours, on=c("tourId","business")]
-pRank2 <- p[sequenceRank==2]
-pRank2$negativeTravelTime <- pRank2$estimatedTimeOfArrivalInSec - pRank2$departureTimeInSec - pRank2$arrivalTimeWindowInSec_lower
-p2 <- p[,diff:=estimatedTimeOfArrivalInSec-arrivalTimeWindowInSec_lower-departureTimeInSec]
-
-write.csv(
-  pRank2[negativeTravelTime<0],
-  file = pp(freightDir, "/10percent/negativeTravelTime.csv"),
-  row.names=FALSE,
-  quote=FALSE,
-  na="0")
-
-toursb2$hour <- toursb2c$departureTimeInSec/3600.0
-ggplot(toursb2b[,.N,by=hour]) + geom_line(aes(hour, N))
-
-payload$business <- "B2B"
-payload[startsWith(payloadId,"b2c")]$business <- "B2C"
-payload[,time24:=estimatedTimeOfArrivalInSec%%(24*3600),][,.N,by=.(timeBin=as.POSIXct(cut(toDateTime(time24),"30 min")), business)] %>%
-  ggplot(aes(timeBin, N, colour=business)) +
-  geom_line() +
-  scale_x_datetime("Hour",
-                   breaks=scales::date_breaks("2 hour"),
-                   labels=scales::date_format("%H", tz = dateTZ)) +
-  scale_y_continuous("Activity", breaks = scales::pretty_breaks()) +
-  scale_colour_manual("Supply Chain", values = c("#eca35b", "#20b2aa")) +
-  theme_marain() +
-  theme(legend.title = element_text(size = 10),
-        legend.text = element_text(size = 10),
-        axis.text.x = element_text(angle = 0, hjust = 1))
-
 
 ################ ***************************
 ################ validation CALTrans
@@ -269,6 +286,7 @@ counties <- data.table::data.table(
              "San Francisco", "San Mateo", "Solano", "Sonoma"),
   CNTY=c("ALA", "CC", "MRN", "NAP", "SCL", "SF", "SM", "SOL", "SON")
 )
+linkStats <- readCsv(normalizePath(paste(freightDir,"/validation/beam/0.linkstats.csv.gz",sep="")))
 
 #data.table::fwrite(network_cleaned, pp(freightDir,"/validation/network_cleaned.csv"), quote=F)
 
@@ -286,9 +304,8 @@ truck_aadtt_with_linkData <- merge(data.table::as.data.table(truck_aadtt_with_li
 
 
 ##### PREPARING BEAM/LINKSTAT DATA
-linkStats <- readCsv(normalizePath(paste(freightDir,"/validation/beam/0.linkstats.csv.gz",sep="")))
-linkstats_noFreight <- readCsv(normalizePath(paste(freightDir,"/validation/beam/0.linkstats.nofreight.csv.gz",sep="")))
-linkstats_wFreight <- readCsv(normalizePath(paste(freightDir,"/validation/beam/0.linkstats.withfreight.csv.gz",sep="")))
+#linkstats_noFreight <- readCsv(normalizePath(paste(freightDir,"/validation/beam/0.linkstats.nofreight.csv.gz",sep="")))
+#linkstats_wFreight <- readCsv(normalizePath(paste(freightDir,"/validation/beam/0.linkstats.withfreight.csv.gz",sep="")))
 #totVolume <- sum(linkstats_wFreight$volume) - sum(linkstats_noFreight$volume)
 #totAADTT <- sum(truck_aadtt_2017_bay_area$TRUCK_AADT)
 lsWFreight <- linkstats_wFreight[,.(volumeWithFreight=sum(volume),lengthInMeter=first(length)),by=.(link)]
@@ -348,5 +365,13 @@ Volume_hpms <- sum(sf_hpms$AADT_Combi+sf_hpms$AADT_Singl)
 VMT_beam <- sum(linkStats$TruckVolume * linkStats$length/1609)
 VMT_hpms <- (sum((sf_hpms$AADT_Combi+sf_hpms$AADT_Singl) * as.numeric(st_length(sf_hpms))/1609))
 
-st_length(sf_hpms[1,])/1609
-sf_hpms[1,]$Shape_Leng
+freight_pt[,.(VMT=sum(length)/1609.0),by=.(vehicleType)]
+freight_pt[,.(tourMT=sum(length)/1609.0),by=.(vehicle,vehicleType)][,.(avgTourMT=mean(tourMT)),by=.(vehicleType)]
+
+write.csv(
+  freight_pt,
+  file = pp(freightWorkDir, "/freight.pt.0.events.csv"),
+  row.names=F,
+  quote=T)
+
+

From cefa389332fcdbf1cdf1732eb7a273693a5eb0c2 Mon Sep 17 00:00:00 2001
From: Rutvik Patel <heyrutvik@gmail.com>
Date: Wed, 6 Apr 2022 09:29:09 +0530
Subject: [PATCH 40/58] corrects coord projection before snapping location

---
 .../agents/freight/input/GenericFreightReader.scala    |  3 ++-
 src/main/scala/beam/utils/SnapCoordinateUtils.scala    | 10 +++++-----
 2 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/src/main/scala/beam/agentsim/agents/freight/input/GenericFreightReader.scala b/src/main/scala/beam/agentsim/agents/freight/input/GenericFreightReader.scala
index 5164c93dd82..893b2211e3b 100644
--- a/src/main/scala/beam/agentsim/agents/freight/input/GenericFreightReader.scala
+++ b/src/main/scala/beam/agentsim/agents/freight/input/GenericFreightReader.scala
@@ -367,7 +367,8 @@ class GenericFreightReader(
       val taz = getTaz(strZone)
       (Some(taz.tazId), Left(getDistributedTazLocation(taz)))
     } else {
-      (None, Right(snapLocationHelper.computeResult(location(strX.toDouble, strY.toDouble))))
+      val wasInWgs = config.convertWgs2Utm
+      (None, Right(snapLocationHelper.computeResult(location(strX.toDouble, strY.toDouble), wasInWgs)))
     }
   }
 
diff --git a/src/main/scala/beam/utils/SnapCoordinateUtils.scala b/src/main/scala/beam/utils/SnapCoordinateUtils.scala
index 36db7dbb044..ce12bc50e55 100644
--- a/src/main/scala/beam/utils/SnapCoordinateUtils.scala
+++ b/src/main/scala/beam/utils/SnapCoordinateUtils.scala
@@ -21,12 +21,12 @@ object SnapCoordinateUtils extends LazyLogging {
   final case class SnapLocationHelper(geo: GeoUtils, streetLayer: StreetLayer, maxRadius: Double) {
     private val store: TrieMap[Coord, Option[Coord]] = TrieMap.empty
 
-    def computeResult[A](utmCoord: Coord): Result = {
-      val wgsCoord = geo.utm2Wgs(utmCoord)
-      if (streetLayer.envelope.contains(wgsCoord.getX, wgsCoord.getY)) {
+    def computeResult(planCoord: Coord, isWgs: Boolean = false): Result = {
+      val coord = if (isWgs) planCoord else geo.utm2Wgs(planCoord)
+      if (streetLayer.envelope.contains(coord.getX, coord.getY)) {
         val snapCoordOpt = store.getOrElseUpdate(
-          utmCoord,
-          Option(geo.getR5Split(streetLayer, wgsCoord, maxRadius)).map { split =>
+          planCoord,
+          Option(geo.getR5Split(streetLayer, coord, maxRadius)).map { split =>
             val updatedPlanCoord = geo.splitToCoord(split)
             geo.wgs2Utm(updatedPlanCoord)
           }

From da5392f6d8245f849cea4cdbc9c0a2ba1fe5174c Mon Sep 17 00:00:00 2001
From: Rutvik Patel <heyrutvik@gmail.com>
Date: Wed, 6 Apr 2022 11:47:58 +0530
Subject: [PATCH 41/58] create valid location within a taz

---
 .../freight/input/GenericFreightReader.scala  |  2 +-
 .../infrastructure/taz/TAZTreeMap.scala       | 39 ++++++++++++++++++-
 .../SupplementaryTripGenerator.scala          |  9 +++--
 src/main/scala/beam/sim/BeamMobsim.scala      | 10 ++++-
 4 files changed, 54 insertions(+), 6 deletions(-)

diff --git a/src/main/scala/beam/agentsim/agents/freight/input/GenericFreightReader.scala b/src/main/scala/beam/agentsim/agents/freight/input/GenericFreightReader.scala
index 893b2211e3b..0b95a8d10d0 100644
--- a/src/main/scala/beam/agentsim/agents/freight/input/GenericFreightReader.scala
+++ b/src/main/scala/beam/agentsim/agents/freight/input/GenericFreightReader.scala
@@ -360,7 +360,7 @@ class GenericFreightReader(
   }
 
   private def getDistributedTazLocation(taz: TAZ): Coord =
-    convertedLocation(TAZTreeMap.randomLocationInTAZ(taz, rnd))
+    convertedLocation(TAZTreeMap.randomLocationInTAZ(taz, rnd, Some(snapLocationHelper)))
 
   private def extractCoordOrTaz(strX: String, strY: String, strZone: String): (Option[Id[TAZ]], ClosestUTMPoint) = {
     if (isBlank(strX) || isBlank(strY)) {
diff --git a/src/main/scala/beam/agentsim/infrastructure/taz/TAZTreeMap.scala b/src/main/scala/beam/agentsim/infrastructure/taz/TAZTreeMap.scala
index 98fcdedfecb..6f6266c2660 100755
--- a/src/main/scala/beam/agentsim/infrastructure/taz/TAZTreeMap.scala
+++ b/src/main/scala/beam/agentsim/infrastructure/taz/TAZTreeMap.scala
@@ -1,5 +1,7 @@
 package beam.agentsim.infrastructure.taz
 
+import beam.utils.SnapCoordinateUtils.{Result, SnapLocationHelper}
+
 import java.io._
 import java.util
 import scala.collection.JavaConverters._
@@ -195,7 +197,7 @@ object TAZTreeMap {
     new TAZTreeMap(tazQuadTree)
   }
 
-  def randomLocationInTAZ(
+  private def createRandomLocationInTAZ(
     taz: TAZ,
     rand: scala.util.Random = new scala.util.Random(System.currentTimeMillis())
   ): Coord = {
@@ -207,6 +209,41 @@ object TAZTreeMap {
     new Coord(taz.coord.getX + x, taz.coord.getY + y)
   }
 
+  def randomLocationInTAZ(
+    taz: TAZ,
+    rand: scala.util.Random = new scala.util.Random(System.currentTimeMillis()),
+    snapLocationHelperMaybe: Option[SnapLocationHelper] = None
+  ): Coord = {
+    val tazId = taz.tazId.toString
+    snapLocationHelperMaybe match {
+      case None => createRandomLocationInTAZ(taz, rand)
+      case Some(helper) =>
+        val max = 10000
+        var counter = 0
+        var split: Coord = null
+        while (split == null && counter < max) {
+          helper.computeResult(createRandomLocationInTAZ(taz, rand)) match {
+            case Result.Succeed(splitCoord) =>
+              logger.info(s"Found valid location $splitCoord within taz $tazId in $counter attempt(s).")
+              split = splitCoord
+            case _ =>
+          }
+          counter += 1
+        }
+
+        if (split == null) {
+          // TODO what to do if we don't find valid point in "max" attempts?
+          val loc = createRandomLocationInTAZ(taz, rand)
+          logger.warn(
+            s"Could not found valid location within taz $tazId even in $max attempts. Creating one anyone $loc."
+          )
+          split = loc
+        }
+
+        split
+    }
+  }
+
   /**
     * performs a concentric ring search from the present location to find elements up to the SearchMaxRadius
     * @param quadTree tree to search
diff --git a/src/main/scala/beam/replanning/SupplementaryTripGenerator.scala b/src/main/scala/beam/replanning/SupplementaryTripGenerator.scala
index 92d1a6d03f5..195a6e16c54 100755
--- a/src/main/scala/beam/replanning/SupplementaryTripGenerator.scala
+++ b/src/main/scala/beam/replanning/SupplementaryTripGenerator.scala
@@ -8,6 +8,7 @@ import beam.router.Modes.BeamMode
 import beam.router.Modes.BeamMode.{CAR, CAV, RIDE_HAIL, RIDE_HAIL_POOLED, WALK, WALK_TRANSIT}
 import beam.sim.BeamServices
 import beam.sim.population.AttributesOfIndividual
+import beam.utils.SnapCoordinateUtils.SnapLocationHelper
 import org.matsim.api.core.v01.population.{Activity, Person, Plan}
 import org.matsim.api.core.v01.{Coord, Id}
 import org.matsim.core.population.PopulationUtils
@@ -22,7 +23,8 @@ class SupplementaryTripGenerator(
   val attributesOfIndividual: AttributesOfIndividual,
   val destinationChoiceModel: DestinationChoiceModel,
   val beamServices: BeamServices,
-  val personId: Id[Person]
+  val personId: Id[Person],
+  val snapLocationHelper: SnapLocationHelper
 ) {
   val r: Random.type = scala.util.Random
   val personSpecificSeed: Long = personId.hashCode().toLong
@@ -199,7 +201,7 @@ class SupplementaryTripGenerator(
         val newActivity =
           PopulationUtils.createActivityFromCoord(
             newActivityType,
-            TAZTreeMap.randomLocationInTAZ(chosenAlternative.taz)
+            TAZTreeMap.randomLocationInTAZ(chosenAlternative.taz, snapLocationHelperMaybe = Some(snapLocationHelper))
           )
         val activityBeforeNewActivity =
           PopulationUtils.createActivityFromCoord(prevActivity.getType, prevActivity.getCoord)
@@ -244,7 +246,8 @@ class SupplementaryTripGenerator(
         Map[SupplementaryTripAlternative, Map[SupplementaryTripAlternative, Map[DestinationParameters, Double]]]()
       } else {
         TAZs.map { taz =>
-          val destinationCoord: Coord = TAZTreeMap.randomLocationInTAZ(taz)
+          val destinationCoord: Coord =
+            TAZTreeMap.randomLocationInTAZ(taz, snapLocationHelperMaybe = Some(snapLocationHelper))
           val additionalActivity = PopulationUtils.createActivityFromCoord(newActivityType, destinationCoord)
           additionalActivity.setStartTime(startTime)
           additionalActivity.setEndTime(endTime)
diff --git a/src/main/scala/beam/sim/BeamMobsim.scala b/src/main/scala/beam/sim/BeamMobsim.scala
index f023de5c1a5..6659ff22c0a 100755
--- a/src/main/scala/beam/sim/BeamMobsim.scala
+++ b/src/main/scala/beam/sim/BeamMobsim.scala
@@ -27,6 +27,7 @@ import beam.sim.metrics.{Metrics, MetricsSupport, SimulationMetricCollector}
 import beam.sim.monitoring.ErrorListener
 import beam.sim.population.AttributesOfIndividual
 import beam.sim.vehiclesharing.Fleets
+import beam.utils.SnapCoordinateUtils.SnapLocationHelper
 import beam.utils._
 import beam.utils.csv.writers.PlansCsvWriter
 import beam.utils.logging.{LoggingMessageActor, MessageLogger}
@@ -70,6 +71,12 @@ class BeamMobsim @Inject() (
   import beamServices._
   val physsimConfig = beamConfig.beam.physsim
 
+  val snapLocationHelper = SnapLocationHelper(
+    geo,
+    beamScenario.transportNetwork.streetLayer,
+    beamConfig.beam.routing.r5.linkRadiusMeters
+  )
+
   override def run(): Unit = {
     logger.info("Starting Iteration")
     startMeasuringIteration()
@@ -224,7 +231,8 @@ class BeamMobsim @Inject() (
               person.getCustomAttributes.get("beam-attributes").asInstanceOf[AttributesOfIndividual],
               destinationChoiceModel,
               beamServices,
-              person.getId
+              person.getId,
+              snapLocationHelper
             )
           val newPlan =
             supplementaryTripGenerator.generateNewPlans(person.getSelectedPlan, destinationChoiceModel, modesAvailable)

From 0d3d5a1949139147f0dd9ad4a482137c6ef2b51c Mon Sep 17 00:00:00 2001
From: Dmitry Openkov <dimaopen@gmail.com>
Date: Wed, 6 Apr 2022 13:26:36 +0300
Subject: [PATCH 42/58] Fixed performance issues with loading payload plans: no
 map views, cached tour id -> plans map.

---
 .../agents/freight/input/GenericFreightReader.scala       | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/src/main/scala/beam/agentsim/agents/freight/input/GenericFreightReader.scala b/src/main/scala/beam/agentsim/agents/freight/input/GenericFreightReader.scala
index cf096f6278c..6987c324005 100644
--- a/src/main/scala/beam/agentsim/agents/freight/input/GenericFreightReader.scala
+++ b/src/main/scala/beam/agentsim/agents/freight/input/GenericFreightReader.scala
@@ -127,6 +127,8 @@ class GenericFreightReader(
     val plans: Map[Id[PayloadPlan], PayloadPlan] = allPlans.filter { case (_, plan) =>
       existingTours.contains(plan.tourId)
     }
+    val tourIdToPlans: Map[Id[FreightTour], IndexedSeq[PayloadPlan]] =
+      plans.values.toIndexedSeq.groupBy(_.tourId).map { case (tourId, plans) => tourId -> plans.sortBy(_.sequenceRank) }
     val tours: Map[Id[FreightTour], FreightTour] = allTours.filter { case (_, tour) =>
       existingTours.contains(tour.tourId)
     }
@@ -183,9 +185,11 @@ class GenericFreightReader(
       val carrierTourIds = tourMap.values.flatten.map(_.tourId).toSet
 
       val plansPerTour: Map[Id[FreightTour], IndexedSeq[PayloadPlan]] =
-        plans.values.groupBy(_.tourId).filterKeys(carrierTourIds).mapValues(_.toIndexedSeq.sortBy(_.sequenceRank))
+        carrierTourIds.collect {
+          case tourId if tourIdToPlans.contains(tourId) => tourId -> tourIdToPlans(tourId)
+        }.toMap
       val carrierPlanIds: Set[Id[PayloadPlan]] = plansPerTour.values.flatten.map(_.payloadId).toSet
-      val payloadMap = plans.filterKeys(carrierPlanIds)
+      val payloadMap = carrierPlanIds.map(planId => planId -> plans(planId)).toMap
 
       FreightCarrier(
         carrierId,

From fb5aefea774ed29bd3f5c522d71cae24ed56e4cf Mon Sep 17 00:00:00 2001
From: Rutvik Patel <heyrutvik@gmail.com>
Date: Wed, 6 Apr 2022 16:31:10 +0530
Subject: [PATCH 43/58] adds validate scenario method for xml input

---
 src/main/scala/beam/sim/BeamHelper.scala      |  43 +++--
 .../beam/utils/SnapCoordinateUtils.scala      |   7 +-
 .../utils/scenario/ScenarioLoaderHelper.scala | 167 +++++++++++++++++-
 3 files changed, 192 insertions(+), 25 deletions(-)

diff --git a/src/main/scala/beam/sim/BeamHelper.scala b/src/main/scala/beam/sim/BeamHelper.scala
index 3ee36ff1789..cf301301546 100755
--- a/src/main/scala/beam/sim/BeamHelper.scala
+++ b/src/main/scala/beam/sim/BeamHelper.scala
@@ -31,6 +31,7 @@ import beam.sim.modules.{BeamAgentModule, UtilsModule}
 import beam.sim.population.PopulationScaling
 import beam.sim.termination.TerminationCriterionProvider
 import beam.utils.BeamVehicleUtils.{readBeamVehicleTypeFile, readFuelTypeFile, readVehiclesFile}
+import beam.utils.SnapCoordinateUtils.SnapLocationHelper
 import beam.utils._
 import beam.utils.csv.readers
 import beam.utils.plan.sampling.AvailableModeUtils
@@ -38,7 +39,13 @@ import beam.utils.scenario.generic.GenericScenarioSource
 import beam.utils.scenario.matsim.BeamScenarioSource
 import beam.utils.scenario.urbansim.censusblock.{ScenarioAdjuster, UrbansimReaderV2}
 import beam.utils.scenario.urbansim.{CsvScenarioReader, ParquetScenarioReader, UrbanSimScenarioSource}
-import beam.utils.scenario.{BeamScenarioLoader, InputType, PreviousRunPlanMerger, UrbanSimScenarioLoader}
+import beam.utils.scenario.{
+  BeamScenarioLoader,
+  InputType,
+  PreviousRunPlanMerger,
+  ScenarioLoaderHelper,
+  UrbanSimScenarioLoader
+}
 import com.conveyal.r5.streets.StreetLayer
 import com.conveyal.r5.transit.TransportNetwork
 import com.fasterxml.jackson.databind.ObjectMapper
@@ -644,32 +651,17 @@ trait BeamHelper extends LazyLogging {
     (beamExecutionConfig, scenario, beamScenario, services, plansMerged)
   }
 
-  def fixDanglingPersons(result: MutableScenario): Unit = {
-    val peopleViaHousehold = result.getHouseholds.getHouseholds
-      .values()
-      .asScala
-      .flatMap { x =>
-        x.getMemberIds.asScala
-      }
-      .toSet
-    val danglingPeople = result.getPopulation.getPersons
-      .values()
-      .asScala
-      .filter(person => !peopleViaHousehold.contains(person.getId))
-    if (danglingPeople.nonEmpty) {
-      logger.error(s"There are ${danglingPeople.size} persons not connected to household, removing them")
-      danglingPeople.foreach { p =>
-        result.getPopulation.removePerson(p.getId)
-      }
-    }
-  }
-
   protected def buildScenarioFromMatsimConfig(
     matsimConfig: MatsimConfig,
     beamScenario: BeamScenario
   ): MutableScenario = {
+    val snapLocationHelper = SnapLocationHelper(
+      new GeoUtilsImpl(beamScenario.beamConfig),
+      beamScenario.transportNetwork.streetLayer,
+      beamScenario.beamConfig.beam.routing.r5.linkRadiusMeters
+    )
     val result = ScenarioUtils.loadScenario(matsimConfig).asInstanceOf[MutableScenario]
-    fixDanglingPersons(result)
+    ScenarioLoaderHelper.validateScenario(result, snapLocationHelper)
     result.setNetwork(beamScenario.network)
     result
   }
@@ -898,9 +890,14 @@ trait BeamHelper extends LazyLogging {
               (scenario, beamScenario, false)
             case "xml" =>
               val beamScenario = loadScenario(beamConfig, outputDirOpt)
+              val snapLocationHelper = SnapLocationHelper(
+                new GeoUtilsImpl(beamConfig),
+                beamScenario.transportNetwork.streetLayer,
+                beamScenario.beamConfig.beam.routing.r5.linkRadiusMeters
+              )
               val scenario = {
                 val result = ScenarioUtils.loadScenario(matsimConfig).asInstanceOf[MutableScenario]
-                fixDanglingPersons(result)
+                ScenarioLoaderHelper.validateScenario(result, snapLocationHelper, outputDirOpt)
                 result
               }
               (scenario, beamScenario, false)
diff --git a/src/main/scala/beam/utils/SnapCoordinateUtils.scala b/src/main/scala/beam/utils/SnapCoordinateUtils.scala
index ce12bc50e55..7e4a95825c5 100644
--- a/src/main/scala/beam/utils/SnapCoordinateUtils.scala
+++ b/src/main/scala/beam/utils/SnapCoordinateUtils.scala
@@ -21,11 +21,16 @@ object SnapCoordinateUtils extends LazyLogging {
   final case class SnapLocationHelper(geo: GeoUtils, streetLayer: StreetLayer, maxRadius: Double) {
     private val store: TrieMap[Coord, Option[Coord]] = TrieMap.empty
 
+    def find(planCoord: Coord, isWgs: Boolean = false): Option[Coord] = {
+      val coord = if (isWgs) planCoord else geo.utm2Wgs(planCoord)
+      store.get(coord).flatten
+    }
+
     def computeResult(planCoord: Coord, isWgs: Boolean = false): Result = {
       val coord = if (isWgs) planCoord else geo.utm2Wgs(planCoord)
       if (streetLayer.envelope.contains(coord.getX, coord.getY)) {
         val snapCoordOpt = store.getOrElseUpdate(
-          planCoord,
+          coord,
           Option(geo.getR5Split(streetLayer, coord, maxRadius)).map { split =>
             val updatedPlanCoord = geo.splitToCoord(split)
             geo.wgs2Utm(updatedPlanCoord)
diff --git a/src/main/scala/beam/utils/scenario/ScenarioLoaderHelper.scala b/src/main/scala/beam/utils/scenario/ScenarioLoaderHelper.scala
index 2113554ca44..d12f05f65a7 100644
--- a/src/main/scala/beam/utils/scenario/ScenarioLoaderHelper.scala
+++ b/src/main/scala/beam/utils/scenario/ScenarioLoaderHelper.scala
@@ -5,9 +5,18 @@ import beam.sim.common.GeoUtils
 import beam.utils.SnapCoordinateUtils
 import beam.utils.SnapCoordinateUtils.{Category, Error, ErrorInfo, Processed, Result, SnapLocationHelper}
 import com.typesafe.scalalogging.LazyLogging
-import org.matsim.api.core.v01.Coord
+import org.matsim.api.core.v01.{Coord, Id}
+import org.matsim.api.core.v01.population.{Activity, Leg, Person}
+import org.matsim.core.scenario.MutableScenario
+import org.matsim.households.{Household, HouseholdsFactoryImpl}
 
 import scala.collection.Iterable
+import scala.collection.mutable.ListBuffer
+import scala.jdk.CollectionConverters.{
+  collectionAsScalaIterableConverter,
+  seqAsJavaListConverter,
+  setAsJavaSetConverter
+}
 
 trait ScenarioLoaderHelper extends LazyLogging {
 
@@ -141,3 +150,159 @@ trait ScenarioLoaderHelper extends LazyLogging {
   }
 
 }
+
+object ScenarioLoaderHelper extends LazyLogging {
+
+  import org.matsim.api.core.v01.population.PlanElement
+
+  private def createHouseholdWithGivenMembers(
+    household: Household,
+    members: List[Id[Person]]
+  ): Household = {
+    val householdResult = new HouseholdsFactoryImpl().createHousehold(household.getId)
+    householdResult.setIncome(household.getIncome)
+    householdResult.setVehicleIds(household.getVehicleIds)
+    householdResult.setMemberIds(members.asJava)
+    householdResult
+  }
+
+  private def updatePlanElementCoord(
+    personId: Id[Person],
+    elements: Vector[PlanElement],
+    snapLocationHelper: SnapLocationHelper
+  ): Vector[ErrorInfo] = {
+    val errors = elements.foldLeft[Vector[ErrorInfo]](Vector.empty) { (errors, element) =>
+      element match {
+        case _: Leg => errors
+        case a: Activity =>
+          val planCoord = a.getCoord
+          snapLocationHelper.computeResult(planCoord) match {
+            case Result.Succeed(_) =>
+              // note: we don't want to update coord in-place here since we might end up removing plan from the person
+              errors
+            case Result.OutOfBoundingBoxError =>
+              errors :+ ErrorInfo(
+                personId.toString,
+                Category.ScenarioPerson,
+                Error.OutOfBoundingBox,
+                planCoord.getX,
+                planCoord.getY
+              )
+            case Result.R5SplitNullError =>
+              errors :+ ErrorInfo(
+                personId.toString,
+                Category.ScenarioPerson,
+                Error.R5SplitNull,
+                planCoord.getX,
+                planCoord.getY
+              )
+          }
+      }
+    }
+
+    if (errors.isEmpty) {
+      elements.foreach {
+        case a: Activity =>
+          val planCoord = a.getCoord
+          snapLocationHelper.find(planCoord) match {
+            case Some(coord) =>
+              a.setCoord(coord)
+            case None =>
+              logger.error("UNEXPECTED: there should be mapped split location for {} coord.", planCoord)
+          }
+        case _ =>
+      }
+    }
+
+    errors
+  }
+
+  def validateScenario(
+    scenario: MutableScenario,
+    snapLocationHelper: SnapLocationHelper,
+    outputDirMaybe: Option[String] = None
+  ): Unit = {
+    val planErrors: ListBuffer[ErrorInfo] = ListBuffer()
+    val householdErrors: ListBuffer[ErrorInfo] = ListBuffer()
+    val people: List[Person] = scenario.getPopulation.getPersons.values().asScala.toList
+    people.foreach { person =>
+      val plans = person.getPlans.asScala.toList
+      plans.foreach { plan =>
+        val elements: Vector[PlanElement] = plan.getPlanElements.asScala.toVector
+        val es: Vector[ErrorInfo] = updatePlanElementCoord(person.getId, elements, snapLocationHelper)
+        if (es.nonEmpty) {
+          planErrors.appendAll(es)
+          person.removePlan(plan)
+        }
+      }
+      val planCount = person.getPlans.size()
+      if (planCount == 0) {
+        scenario.getPopulation.removePerson(person.getId)
+      }
+    }
+
+    val validPeople: Set[Id[Person]] = scenario.getPopulation.getPersons.values().asScala.map(_.getId).toSet
+
+    val households: List[Household] = scenario.getHouseholds.getHouseholds.values().asScala.toList
+    households.foreach { household =>
+      val members = household.getMemberIds.asScala.toSet
+      val validMembers = validPeople.intersect(members)
+
+      if (validMembers.isEmpty) {
+        scenario.getHouseholds.getHouseholds.remove(household.getId)
+        scenario.getHouseholds.getHouseholdAttributes.removeAllAttributes(household.getId.toString)
+      } else {
+        val updatedHousehold = createHouseholdWithGivenMembers(household, validMembers.toList)
+        scenario.getHouseholds.getHouseholds.replace(household.getId, updatedHousehold)
+      }
+    }
+
+    val householdsWithMembers: List[Household] = scenario.getHouseholds.getHouseholds.values().asScala.toList
+    householdsWithMembers.foreach { household =>
+      val householdId = household.getId.toString
+      val attr = scenario.getHouseholds.getHouseholdAttributes
+      val locationX = attr.getAttribute(householdId, "homecoordx").asInstanceOf[Double]
+      val locationY = attr.getAttribute(householdId, "homecoordy").asInstanceOf[Double]
+      val planCoord = new Coord(locationX, locationY)
+
+      snapLocationHelper.computeResult(planCoord) match {
+        case Result.Succeed(splitCoord) =>
+          attr.putAttribute(householdId, "homecoordx", splitCoord.getX)
+          attr.putAttribute(householdId, "homecoordy", splitCoord.getY)
+        case Result.OutOfBoundingBoxError =>
+          household.getMemberIds.asScala.toList.foreach(personId => scenario.getPopulation.getPersons.remove(personId))
+          scenario.getHouseholds.getHouseholds.remove(household.getId)
+          householdErrors.append(
+            ErrorInfo(
+              householdId,
+              Category.ScenarioHousehold,
+              Error.OutOfBoundingBox,
+              planCoord.getX,
+              planCoord.getY
+            )
+          )
+        case Result.R5SplitNullError =>
+          household.getMemberIds.asScala.toList.foreach(personId => scenario.getPopulation.getPersons.remove(personId))
+          scenario.getHouseholds.getHouseholds.remove(household.getId)
+          householdErrors.append(
+            ErrorInfo(
+              householdId,
+              Category.ScenarioHousehold,
+              Error.R5SplitNull,
+              planCoord.getX,
+              planCoord.getY
+            )
+          )
+      }
+    }
+
+    outputDirMaybe.foreach { path =>
+      if (planErrors.isEmpty) logger.info("No 'snap location' error to report for scenario plans.")
+      else SnapCoordinateUtils.writeToCsv(s"$path/snapLocationPlanErrors.csv.gz", planErrors)
+
+      if (householdErrors.isEmpty) logger.info("No 'snap location' error to report for scenario households.")
+      else SnapCoordinateUtils.writeToCsv(s"$path/snapLocationHouseholdErrors.csv.gz", householdErrors)
+    }
+  }
+
+}

From 72fcac31cb623a0272b53fbeca67707f49cbbac8 Mon Sep 17 00:00:00 2001
From: Rutvik Patel <heyrutvik@gmail.com>
Date: Wed, 6 Apr 2022 16:37:55 +0530
Subject: [PATCH 44/58] changes variable name

---
 .../scala/beam/utils/scenario/ScenarioLoaderHelper.scala    | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/main/scala/beam/utils/scenario/ScenarioLoaderHelper.scala b/src/main/scala/beam/utils/scenario/ScenarioLoaderHelper.scala
index d12f05f65a7..48400e10d71 100644
--- a/src/main/scala/beam/utils/scenario/ScenarioLoaderHelper.scala
+++ b/src/main/scala/beam/utils/scenario/ScenarioLoaderHelper.scala
@@ -229,9 +229,9 @@ object ScenarioLoaderHelper extends LazyLogging {
       val plans = person.getPlans.asScala.toList
       plans.foreach { plan =>
         val elements: Vector[PlanElement] = plan.getPlanElements.asScala.toVector
-        val es: Vector[ErrorInfo] = updatePlanElementCoord(person.getId, elements, snapLocationHelper)
-        if (es.nonEmpty) {
-          planErrors.appendAll(es)
+        val errors: Vector[ErrorInfo] = updatePlanElementCoord(person.getId, elements, snapLocationHelper)
+        if (errors.nonEmpty) {
+          planErrors.appendAll(errors)
           person.removePlan(plan)
         }
       }

From a33ff8b905904cbbf2b28c7162d68710b9b07e50 Mon Sep 17 00:00:00 2001
From: Rutvik Patel <heyrutvik@gmail.com>
Date: Thu, 7 Apr 2022 08:55:56 +0530
Subject: [PATCH 45/58] adds simple test

---
 .../agents/freight/input/FreightReader.scala  |  2 +-
 .../freight/input/GenericFreightReader.scala  |  9 +--
 .../SupplementaryTripGenerator.scala          |  6 +-
 src/main/scala/beam/sim/BeamMobsim.scala      |  2 +-
 .../scenario/UrbanSimScenarioLoader.scala     |  2 +-
 .../input/GenericFreightReaderSpec.scala      |  6 +-
 .../scala/beam/utils/ModeExclusionTest.scala  |  2 +-
 .../scala/beam/utils/SnapCoordinateSpec.scala | 67 +++++++++++++++++++
 8 files changed, 82 insertions(+), 14 deletions(-)
 create mode 100644 src/test/scala/beam/utils/SnapCoordinateSpec.scala

diff --git a/src/main/scala/beam/agentsim/agents/freight/input/FreightReader.scala b/src/main/scala/beam/agentsim/agents/freight/input/FreightReader.scala
index e4981d32b8e..a2b5cae7190 100644
--- a/src/main/scala/beam/agentsim/agents/freight/input/FreightReader.scala
+++ b/src/main/scala/beam/agentsim/agents/freight/input/FreightReader.scala
@@ -203,7 +203,7 @@ object FreightReader {
           geoUtils,
           rand,
           tazMap,
-          snapLocationHelper,
+          Some(snapLocationHelper),
           outputDirMaybe
         )
       case s =>
diff --git a/src/main/scala/beam/agentsim/agents/freight/input/GenericFreightReader.scala b/src/main/scala/beam/agentsim/agents/freight/input/GenericFreightReader.scala
index 0b95a8d10d0..1e74fb21d72 100644
--- a/src/main/scala/beam/agentsim/agents/freight/input/GenericFreightReader.scala
+++ b/src/main/scala/beam/agentsim/agents/freight/input/GenericFreightReader.scala
@@ -27,8 +27,8 @@ class GenericFreightReader(
   val geoUtils: GeoUtils,
   rnd: Random,
   tazTree: TAZTreeMap,
-  val snapLocationHelper: SnapLocationHelper,
-  val outputDirMaybe: Option[String]
+  val snapLocationHelperMaybe: Option[SnapLocationHelper] = None,
+  val outputDirMaybe: Option[String] = None
 ) extends LazyLogging
     with FreightReader {
 
@@ -360,7 +360,7 @@ class GenericFreightReader(
   }
 
   private def getDistributedTazLocation(taz: TAZ): Coord =
-    convertedLocation(TAZTreeMap.randomLocationInTAZ(taz, rnd, Some(snapLocationHelper)))
+    convertedLocation(TAZTreeMap.randomLocationInTAZ(taz, rnd, snapLocationHelperMaybe))
 
   private def extractCoordOrTaz(strX: String, strY: String, strZone: String): (Option[Id[TAZ]], ClosestUTMPoint) = {
     if (isBlank(strX) || isBlank(strY)) {
@@ -368,7 +368,8 @@ class GenericFreightReader(
       (Some(taz.tazId), Left(getDistributedTazLocation(taz)))
     } else {
       val wasInWgs = config.convertWgs2Utm
-      (None, Right(snapLocationHelper.computeResult(location(strX.toDouble, strY.toDouble), wasInWgs)))
+      val loc = location(strX.toDouble, strY.toDouble)
+      (None, Right(snapLocationHelperMaybe.map(_.computeResult(loc, wasInWgs)).getOrElse(Result.Succeed(loc))))
     }
   }
 
diff --git a/src/main/scala/beam/replanning/SupplementaryTripGenerator.scala b/src/main/scala/beam/replanning/SupplementaryTripGenerator.scala
index 195a6e16c54..609f3879a2b 100755
--- a/src/main/scala/beam/replanning/SupplementaryTripGenerator.scala
+++ b/src/main/scala/beam/replanning/SupplementaryTripGenerator.scala
@@ -24,7 +24,7 @@ class SupplementaryTripGenerator(
   val destinationChoiceModel: DestinationChoiceModel,
   val beamServices: BeamServices,
   val personId: Id[Person],
-  val snapLocationHelper: SnapLocationHelper
+  val snapLocationHelperMaybe: Option[SnapLocationHelper] = None
 ) {
   val r: Random.type = scala.util.Random
   val personSpecificSeed: Long = personId.hashCode().toLong
@@ -201,7 +201,7 @@ class SupplementaryTripGenerator(
         val newActivity =
           PopulationUtils.createActivityFromCoord(
             newActivityType,
-            TAZTreeMap.randomLocationInTAZ(chosenAlternative.taz, snapLocationHelperMaybe = Some(snapLocationHelper))
+            TAZTreeMap.randomLocationInTAZ(chosenAlternative.taz, snapLocationHelperMaybe = snapLocationHelperMaybe)
           )
         val activityBeforeNewActivity =
           PopulationUtils.createActivityFromCoord(prevActivity.getType, prevActivity.getCoord)
@@ -247,7 +247,7 @@ class SupplementaryTripGenerator(
       } else {
         TAZs.map { taz =>
           val destinationCoord: Coord =
-            TAZTreeMap.randomLocationInTAZ(taz, snapLocationHelperMaybe = Some(snapLocationHelper))
+            TAZTreeMap.randomLocationInTAZ(taz, snapLocationHelperMaybe = snapLocationHelperMaybe)
           val additionalActivity = PopulationUtils.createActivityFromCoord(newActivityType, destinationCoord)
           additionalActivity.setStartTime(startTime)
           additionalActivity.setEndTime(endTime)
diff --git a/src/main/scala/beam/sim/BeamMobsim.scala b/src/main/scala/beam/sim/BeamMobsim.scala
index 6659ff22c0a..983d4a21cef 100755
--- a/src/main/scala/beam/sim/BeamMobsim.scala
+++ b/src/main/scala/beam/sim/BeamMobsim.scala
@@ -232,7 +232,7 @@ class BeamMobsim @Inject() (
               destinationChoiceModel,
               beamServices,
               person.getId,
-              snapLocationHelper
+              Some(snapLocationHelper)
             )
           val newPlan =
             supplementaryTripGenerator.generateNewPlans(person.getSelectedPlan, destinationChoiceModel, modesAvailable)
diff --git a/src/main/scala/beam/utils/scenario/UrbanSimScenarioLoader.scala b/src/main/scala/beam/utils/scenario/UrbanSimScenarioLoader.scala
index 16268ac5564..24623699c8d 100644
--- a/src/main/scala/beam/utils/scenario/UrbanSimScenarioLoader.scala
+++ b/src/main/scala/beam/utils/scenario/UrbanSimScenarioLoader.scala
@@ -31,7 +31,7 @@ class UrbanSimScenarioLoader(
   val scenarioSource: ScenarioSource,
   val geo: GeoUtils,
   val previousRunPlanMerger: Option[PreviousRunPlanMerger] = None,
-  val outputDirMaybe: Option[String]
+  val outputDirMaybe: Option[String] = None
 ) extends ScenarioLoaderHelper {
 
   private implicit val ex: ExecutionContext = scala.concurrent.ExecutionContext.Implicits.global
diff --git a/src/test/scala/beam/agentsim/agents/freight/input/GenericFreightReaderSpec.scala b/src/test/scala/beam/agentsim/agents/freight/input/GenericFreightReaderSpec.scala
index 12042d72cb8..1d9649855e4 100644
--- a/src/test/scala/beam/agentsim/agents/freight/input/GenericFreightReaderSpec.scala
+++ b/src/test/scala/beam/agentsim/agents/freight/input/GenericFreightReaderSpec.scala
@@ -45,7 +45,7 @@ class GenericFreightReaderSpec extends AnyWordSpecLike with Matchers {
 
   val rnd = new Random(2333L)
 
-  private val reader = new GenericFreightReader(freightConfig, geoUtils, rnd, tazMap, None)
+  private val reader = new GenericFreightReader(freightConfig, geoUtils, rnd, tazMap)
 
   "PayloadPlansConverter" should {
     "read Payload Plans" in {
@@ -154,12 +154,12 @@ class GenericFreightReaderSpec extends AnyWordSpecLike with Matchers {
   }
 
   private def readCarriers: IndexedSeq[FreightCarrier] = {
-    val converter = new GenericFreightReader(freightConfig, geoUtils, new Random(4324L), tazMap, None)
+    val converter = new GenericFreightReader(freightConfig, geoUtils, new Random(4324L), tazMap)
     val payloadPlans: Map[Id[PayloadPlan], PayloadPlan] = converter.readPayloadPlans()
     val tours = converter.readFreightTours()
     val vehicleTypes = BeamVehicleUtils.readBeamVehicleTypeFile("test/input/beamville/vehicleTypes.csv")
     val freightCarriers: IndexedSeq[FreightCarrier] =
-      new GenericFreightReader(freightConfig, geoUtils, new Random(73737L), tazMap, None).readFreightCarriers(
+      new GenericFreightReader(freightConfig, geoUtils, new Random(73737L), tazMap).readFreightCarriers(
         tours,
         payloadPlans,
         vehicleTypes
diff --git a/src/test/scala/beam/utils/ModeExclusionTest.scala b/src/test/scala/beam/utils/ModeExclusionTest.scala
index 17c77c19ca4..f766913391b 100644
--- a/src/test/scala/beam/utils/ModeExclusionTest.scala
+++ b/src/test/scala/beam/utils/ModeExclusionTest.scala
@@ -32,7 +32,7 @@ class ModeExclusionTest extends AnyWordSpecLike with Matchers with BeforeAndAfte
     val plans = BeamCsvScenarioReader.readPlansFile("test/input/beamville/csvInput/plans.csv")
     val personIdsWithPlanTmp = plans.map(_.personId).toSet
     val personWithPlans = persons.filter(person => personIdsWithPlanTmp.contains(person.personId))
-    val scenarioLoader = new BeamScenarioLoader(scenarioBuilder, beamScenario, scenarioSource, geoUtils)
+    val scenarioLoader = new BeamScenarioLoader(scenarioBuilder, beamScenario, scenarioSource, geoUtils, None)
     val population = scenarioLoader.buildPopulation(personWithPlans)
 
     "check for removal of single mode" in {
diff --git a/src/test/scala/beam/utils/SnapCoordinateSpec.scala b/src/test/scala/beam/utils/SnapCoordinateSpec.scala
new file mode 100644
index 00000000000..57f9c4905ea
--- /dev/null
+++ b/src/test/scala/beam/utils/SnapCoordinateSpec.scala
@@ -0,0 +1,67 @@
+package beam.utils
+
+import beam.sim.BeamHelper
+import beam.sim.common.GeoUtilsImpl
+import beam.sim.config.{BeamConfig, MatSimBeamConfigBuilder}
+import beam.utils.SnapCoordinateUtils.{Result, SnapLocationHelper}
+import beam.utils.TestConfigUtils.testConfig
+import com.typesafe.config.{ConfigFactory, Config => TypesafeConfig}
+import org.matsim.api.core.v01.Coord
+import org.matsim.api.core.v01.population.Activity
+import org.matsim.core.config.Config
+import org.scalatest.matchers.should.Matchers
+import org.scalatest.wordspec.AnyWordSpec
+
+import scala.jdk.CollectionConverters.collectionAsScalaIterableConverter
+
+class SnapCoordinateSpec extends AnyWordSpec with Matchers with BeamHelper {
+
+  "scenario plan" should {
+    "contain all valid locations" in {
+      lazy val config: TypesafeConfig = ConfigFactory
+        .parseString("""
+                       |beam.routing.r5.linkRadiusMeters = 350
+                       |""".stripMargin)
+        .withFallback(testConfig("test/input/beamville/beam.conf"))
+        .resolve()
+
+      val configBuilder = new MatSimBeamConfigBuilder(config)
+      val matsimConfig: Config = configBuilder.buildMatSimConf()
+      matsimConfig.planCalcScore().setMemorizingExperiencedPlans(true)
+      val beamConfig: BeamConfig = BeamConfig(config)
+
+      FileUtils.setConfigOutputFile(beamConfig, matsimConfig)
+
+      val (scenario, beamScenario, _) = buildBeamServicesAndScenario(
+        beamConfig,
+        matsimConfig
+      )
+
+      val snapLocationHelper: SnapLocationHelper = SnapLocationHelper(
+        new GeoUtilsImpl(beamConfig),
+        beamScenario.transportNetwork.streetLayer,
+        beamConfig.beam.routing.r5.linkRadiusMeters
+      )
+
+      val dummyCoord = new Coord()
+      val results: Seq[SnapCoordinateUtils.Result] = scenario.getPopulation.getPersons
+        .values()
+        .asScala
+        .flatMap { person =>
+          person.getPlans.asScala.flatMap { plan =>
+            plan.getPlanElements.asScala.map {
+              case e: Activity => snapLocationHelper.computeResult(e.getCoord)
+              case _           => Result.Succeed(dummyCoord)
+            }
+          }
+        }
+        .toList
+
+      results.forall {
+        case _: Result.Succeed => true
+        case _                 => false
+      } shouldBe true
+    }
+  }
+
+}

From 7bc9881387fbc7df8d5edc8d1b8b138346c44864 Mon Sep 17 00:00:00 2001
From: Rutvik Patel <heyrutvik@gmail.com>
Date: Thu, 7 Apr 2022 11:37:52 +0530
Subject: [PATCH 46/58] adds more tests

---
 .../scala/beam/utils/SnapCoordinateSpec.scala | 160 +++++++++++++++++-
 .../scenario/case1/csv/households.csv         |   3 +
 .../scenario/case1/csv/plans.csv              |   3 +
 .../input/snap-location/scenario/case1/note   |   8 +
 .../case1/xml/householdAttributes.xml         |   3 +
 .../scenario/case1/xml/households.xml         |   3 +
 .../scenario/case1/xml/population.xml         |   3 +
 .../case1/xml/populationAttributes.xml        |   3 +
 .../scenario/case2/csv/households.csv         |   3 +
 .../scenario/case2/csv/plans.csv              |   3 +
 .../input/snap-location/scenario/case2/note   |   7 +
 .../case2/xml/householdAttributes.xml         |   3 +
 .../scenario/case2/xml/households.xml         |   3 +
 .../scenario/case2/xml/population.xml         |   3 +
 .../case2/xml/populationAttributes.xml        |   3 +
 15 files changed, 209 insertions(+), 2 deletions(-)
 create mode 100644 test/test-resources/beam/input/snap-location/scenario/case1/csv/households.csv
 create mode 100644 test/test-resources/beam/input/snap-location/scenario/case1/csv/plans.csv
 create mode 100644 test/test-resources/beam/input/snap-location/scenario/case1/note
 create mode 100755 test/test-resources/beam/input/snap-location/scenario/case1/xml/householdAttributes.xml
 create mode 100755 test/test-resources/beam/input/snap-location/scenario/case1/xml/households.xml
 create mode 100755 test/test-resources/beam/input/snap-location/scenario/case1/xml/population.xml
 create mode 100755 test/test-resources/beam/input/snap-location/scenario/case1/xml/populationAttributes.xml
 create mode 100644 test/test-resources/beam/input/snap-location/scenario/case2/csv/households.csv
 create mode 100644 test/test-resources/beam/input/snap-location/scenario/case2/csv/plans.csv
 create mode 100644 test/test-resources/beam/input/snap-location/scenario/case2/note
 create mode 100755 test/test-resources/beam/input/snap-location/scenario/case2/xml/householdAttributes.xml
 create mode 100755 test/test-resources/beam/input/snap-location/scenario/case2/xml/households.xml
 create mode 100755 test/test-resources/beam/input/snap-location/scenario/case2/xml/population.xml
 create mode 100755 test/test-resources/beam/input/snap-location/scenario/case2/xml/populationAttributes.xml

diff --git a/src/test/scala/beam/utils/SnapCoordinateSpec.scala b/src/test/scala/beam/utils/SnapCoordinateSpec.scala
index 57f9c4905ea..a3ae419347a 100644
--- a/src/test/scala/beam/utils/SnapCoordinateSpec.scala
+++ b/src/test/scala/beam/utils/SnapCoordinateSpec.scala
@@ -16,6 +16,21 @@ import scala.jdk.CollectionConverters.collectionAsScalaIterableConverter
 
 class SnapCoordinateSpec extends AnyWordSpec with Matchers with BeamHelper {
 
+  /**
+    * see the notes,
+    *
+    * Scenario:
+    * case1: test/test-resources/beam/input/snap-location/scenario/case1/note
+    *   three invalid plans, update population/households accordingly
+    * case2: test/test-resources/beam/input/snap-location/scenario/case2/note
+    *   one invalid household and three invalid plans, update population/households accordingly
+    *
+    * Freight:
+    * ???
+    */
+
+  val pwd: String = System.getenv("PWD")
+
   "scenario plan" should {
     "contain all valid locations" in {
       lazy val config: TypesafeConfig = ConfigFactory
@@ -44,7 +59,7 @@ class SnapCoordinateSpec extends AnyWordSpec with Matchers with BeamHelper {
       )
 
       val dummyCoord = new Coord()
-      val results: Seq[SnapCoordinateUtils.Result] = scenario.getPopulation.getPersons
+      val population: Seq[SnapCoordinateUtils.Result] = scenario.getPopulation.getPersons
         .values()
         .asScala
         .flatMap { person =>
@@ -57,11 +72,152 @@ class SnapCoordinateSpec extends AnyWordSpec with Matchers with BeamHelper {
         }
         .toList
 
-      results.forall {
+      val households: Seq[SnapCoordinateUtils.Result] = scenario.getHouseholds.getHouseholds
+        .values()
+        .asScala
+        .map { household =>
+          val locationX = scenario.getHouseholds.getHouseholdAttributes
+            .getAttribute(household.getId.toString, "homecoordx")
+            .asInstanceOf[Double]
+          val locationY = scenario.getHouseholds.getHouseholdAttributes
+            .getAttribute(household.getId.toString, "homecoordy")
+            .asInstanceOf[Double]
+          val coord = new Coord(locationX, locationY)
+          snapLocationHelper.computeResult(coord)
+        }
+        .toList
+
+      (population ++ households).forall {
         case _: Result.Succeed => true
         case _                 => false
       } shouldBe true
     }
+
+    "remove/update invalid persons and households [case1 xml input]" in {
+      lazy val config: TypesafeConfig = ConfigFactory
+        .parseString(s"""
+                        |beam.agentsim.agents.plans {
+                        |  inputPlansFilePath = "$pwd/test/test-resources/beam/input/snap-location/scenario/case1/xml/population.xml"
+                        |  inputPersonAttributesFilePath = "$pwd/test/test-resources/beam/input/snap-location/scenario/case1/xml/populationAttributes.xml"
+                        |}
+                        |beam.agentsim.agents.households {
+                        |  inputFilePath = "$pwd/test/test-resources/beam/input/snap-location/scenario/case1/xml/households.xml"
+                        |  inputHouseholdAttributesFilePath = "$pwd/test/test-resources/beam/input/snap-location/scenario/case1/xml/householdAttributes.xml"
+                        |}
+                        |beam.routing.r5.linkRadiusMeters = 350
+                        |""".stripMargin)
+        .withFallback(testConfig("test/input/beamville/beam.conf"))
+        .resolve()
+
+      val configBuilder = new MatSimBeamConfigBuilder(config)
+      val matsimConfig: Config = configBuilder.buildMatSimConf()
+      matsimConfig.planCalcScore().setMemorizingExperiencedPlans(true)
+      val beamConfig: BeamConfig = BeamConfig(config)
+
+      FileUtils.setConfigOutputFile(beamConfig, matsimConfig)
+
+      val (scenario, _, _) = buildBeamServicesAndScenario(
+        beamConfig,
+        matsimConfig
+      )
+
+      val households = scenario.getHouseholds.getHouseholds.values().asScala.toList
+
+      scenario.getPopulation.getPersons.size() shouldBe 2
+      households.foreach { household =>
+        household.getMemberIds.size() shouldBe 1
+      }
+    }
+
+    "remove/update invalid persons and households [case2 xml input]" in {
+      lazy val config: TypesafeConfig = ConfigFactory
+        .parseString(s"""
+                        |beam.agentsim.agents.plans {
+                        |  inputPlansFilePath = "$pwd/test/test-resources/beam/input/snap-location/scenario/case2/xml/population.xml"
+                        |  inputPersonAttributesFilePath = "$pwd/test/test-resources/beam/input/snap-location/scenario/case2/xml/populationAttributes.xml"
+                        |}
+                        |beam.agentsim.agents.households {
+                        |  inputFilePath = "$pwd/test/test-resources/beam/input/snap-location/scenario/case2/xml/households.xml"
+                        |  inputHouseholdAttributesFilePath = "$pwd/test/test-resources/beam/input/snap-location/scenario/case2/xml/householdAttributes.xml"
+                        |}
+                        |beam.routing.r5.linkRadiusMeters = 350
+                        |""".stripMargin)
+        .withFallback(testConfig("test/input/beamville/beam.conf"))
+        .resolve()
+
+      val configBuilder = new MatSimBeamConfigBuilder(config)
+      val matsimConfig: Config = configBuilder.buildMatSimConf()
+      matsimConfig.planCalcScore().setMemorizingExperiencedPlans(true)
+      val beamConfig: BeamConfig = BeamConfig(config)
+
+      FileUtils.setConfigOutputFile(beamConfig, matsimConfig)
+
+      val (scenario, _, _) = buildBeamServicesAndScenario(
+        beamConfig,
+        matsimConfig
+      )
+
+      scenario.getPopulation.getPersons.size() shouldBe 1
+      scenario.getHouseholds.getHouseholds.size() shouldBe 1
+    }
+
+    "remove invalid persons and households [case1 csv input]" in {
+      lazy val config: TypesafeConfig = ConfigFactory
+        .parseString(s"""
+                        |beam.agentsim.agents.plans.inputPlansFilePath = "$pwd/test/test-resources/beam/input/snap-location/scenario/case1/csv/plans.csv"
+                        |beam.agentsim.agents.households.inputFilePath = "$pwd/test/test-resources/beam/input/snap-location/scenario/case1/csv/households.csv"
+                        |beam.routing.r5.linkRadiusMeters = 350
+                        |""".stripMargin)
+        .withFallback(testConfig("test/input/beamville/beam-csv.conf"))
+        .resolve()
+
+      val configBuilder = new MatSimBeamConfigBuilder(config)
+      val matsimConfig: Config = configBuilder.buildMatSimConf()
+      matsimConfig.planCalcScore().setMemorizingExperiencedPlans(true)
+      val beamConfig: BeamConfig = BeamConfig(config)
+
+      FileUtils.setConfigOutputFile(beamConfig, matsimConfig)
+
+      val (scenario, _, _) = buildBeamServicesAndScenario(
+        beamConfig,
+        matsimConfig
+      )
+
+      val households = scenario.getHouseholds.getHouseholds.values().asScala.toList
+
+      scenario.getPopulation.getPersons.size() shouldBe 2
+      households.foreach { household =>
+        household.getMemberIds.size() shouldBe 1
+      }
+    }
+
+    "remove invalid persons and households [case2 csv input]" in {
+      lazy val config: TypesafeConfig = ConfigFactory
+        .parseString(s"""
+                        |beam.agentsim.agents.plans.inputPlansFilePath = "$pwd/test/test-resources/beam/input/snap-location/scenario/case2/csv/plans.csv"
+                        |beam.agentsim.agents.households.inputFilePath = "$pwd/test/test-resources/beam/input/snap-location/scenario/case2/csv/households.csv"
+                        |beam.routing.r5.linkRadiusMeters = 350
+                        |""".stripMargin)
+        .withFallback(testConfig("test/input/beamville/beam-csv.conf"))
+        .resolve()
+
+      val configBuilder = new MatSimBeamConfigBuilder(config)
+      val matsimConfig: Config = configBuilder.buildMatSimConf()
+      matsimConfig.planCalcScore().setMemorizingExperiencedPlans(true)
+      val beamConfig: BeamConfig = BeamConfig(config)
+
+      FileUtils.setConfigOutputFile(beamConfig, matsimConfig)
+
+      val (scenario, _, _) = buildBeamServicesAndScenario(
+        beamConfig,
+        matsimConfig
+      )
+
+      val households = scenario.getHouseholds.getHouseholds.values().asScala.toList
+
+      scenario.getPopulation.getPersons.size() shouldBe 1
+      scenario.getHouseholds.getHouseholds.size() shouldBe 1
+    }
   }
 
 }
diff --git a/test/test-resources/beam/input/snap-location/scenario/case1/csv/households.csv b/test/test-resources/beam/input/snap-location/scenario/case1/csv/households.csv
new file mode 100644
index 00000000000..433dae022fe
--- /dev/null
+++ b/test/test-resources/beam/input/snap-location/scenario/case1/csv/households.csv
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:40a521bee13f8eccc5d13843cb4125f75e91da7425ed4f2bbe1e385f4a0b0312
+size 95
diff --git a/test/test-resources/beam/input/snap-location/scenario/case1/csv/plans.csv b/test/test-resources/beam/input/snap-location/scenario/case1/csv/plans.csv
new file mode 100644
index 00000000000..6e7de309326
--- /dev/null
+++ b/test/test-resources/beam/input/snap-location/scenario/case1/csv/plans.csv
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:10f892985fdd48bf3cdae8fe1d10f62352b4ce8032f61244fb8a9c0693672c9d
+size 1314
diff --git a/test/test-resources/beam/input/snap-location/scenario/case1/note b/test/test-resources/beam/input/snap-location/scenario/case1/note
new file mode 100644
index 00000000000..bba4e27fe6b
--- /dev/null
+++ b/test/test-resources/beam/input/snap-location/scenario/case1/note
@@ -0,0 +1,8 @@
+OutOfBoundingBox
+Person 1 x 1627138.4 y 1117.0
+
+R5SplitNull
+Person 1 x 166045.2 y 2705.4
+Person 3 x 166675.4 y 3068.543
+Person 4 x 170108.4 y 1960.535
+
diff --git a/test/test-resources/beam/input/snap-location/scenario/case1/xml/householdAttributes.xml b/test/test-resources/beam/input/snap-location/scenario/case1/xml/householdAttributes.xml
new file mode 100755
index 00000000000..15563c7198b
--- /dev/null
+++ b/test/test-resources/beam/input/snap-location/scenario/case1/xml/householdAttributes.xml
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b239a0087a0fea5d6d629dc8b90177fa818fbf9bd4eadb650bfc1c2f13f3f253
+size 705
diff --git a/test/test-resources/beam/input/snap-location/scenario/case1/xml/households.xml b/test/test-resources/beam/input/snap-location/scenario/case1/xml/households.xml
new file mode 100755
index 00000000000..cf1c3ece8e2
--- /dev/null
+++ b/test/test-resources/beam/input/snap-location/scenario/case1/xml/households.xml
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fe9c0a5dc3f9469f909434edece253fb100b89b307ef2a0ef24c88e3f3e56b92
+size 859
diff --git a/test/test-resources/beam/input/snap-location/scenario/case1/xml/population.xml b/test/test-resources/beam/input/snap-location/scenario/case1/xml/population.xml
new file mode 100755
index 00000000000..996d9d151bd
--- /dev/null
+++ b/test/test-resources/beam/input/snap-location/scenario/case1/xml/population.xml
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:53c0ae571973fe243adda66f3d68ae6dc1dce8962187d17575373bc89e03efbd
+size 2850
diff --git a/test/test-resources/beam/input/snap-location/scenario/case1/xml/populationAttributes.xml b/test/test-resources/beam/input/snap-location/scenario/case1/xml/populationAttributes.xml
new file mode 100755
index 00000000000..c2d270cac35
--- /dev/null
+++ b/test/test-resources/beam/input/snap-location/scenario/case1/xml/populationAttributes.xml
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:adb3199cd0c3d15ee9244af92c81479522908329ca44893bd7cb958554856f37
+size 998
diff --git a/test/test-resources/beam/input/snap-location/scenario/case2/csv/households.csv b/test/test-resources/beam/input/snap-location/scenario/case2/csv/households.csv
new file mode 100644
index 00000000000..829dcbf2848
--- /dev/null
+++ b/test/test-resources/beam/input/snap-location/scenario/case2/csv/households.csv
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:621b455d74f0323748f7e481f890cd5cbd3080262b681749efb4492e093a2b41
+size 96
diff --git a/test/test-resources/beam/input/snap-location/scenario/case2/csv/plans.csv b/test/test-resources/beam/input/snap-location/scenario/case2/csv/plans.csv
new file mode 100644
index 00000000000..a7aae41899f
--- /dev/null
+++ b/test/test-resources/beam/input/snap-location/scenario/case2/csv/plans.csv
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:456f1efc4fc8933ac32f832b51a0fa5b670716be7450c4cd7fb6e7d7b9244c28
+size 1313
diff --git a/test/test-resources/beam/input/snap-location/scenario/case2/note b/test/test-resources/beam/input/snap-location/scenario/case2/note
new file mode 100644
index 00000000000..41b8909e821
--- /dev/null
+++ b/test/test-resources/beam/input/snap-location/scenario/case2/note
@@ -0,0 +1,7 @@
+OutOfBoundingBox
+Household 1 x 1627138.4 y 1117.0
+
+R5SplitNull
+Person 1 x 166045.2 y 2705.4
+Person 3 x 166675.4 y 3068.543
+Person 4 x 170108.4 y 1960.535
diff --git a/test/test-resources/beam/input/snap-location/scenario/case2/xml/householdAttributes.xml b/test/test-resources/beam/input/snap-location/scenario/case2/xml/householdAttributes.xml
new file mode 100755
index 00000000000..8115f49478f
--- /dev/null
+++ b/test/test-resources/beam/input/snap-location/scenario/case2/xml/householdAttributes.xml
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:56c400740feb094463a5baabf1a66d719ef6a8059bd07ac930f390786b039f3e
+size 706
diff --git a/test/test-resources/beam/input/snap-location/scenario/case2/xml/households.xml b/test/test-resources/beam/input/snap-location/scenario/case2/xml/households.xml
new file mode 100755
index 00000000000..cf1c3ece8e2
--- /dev/null
+++ b/test/test-resources/beam/input/snap-location/scenario/case2/xml/households.xml
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fe9c0a5dc3f9469f909434edece253fb100b89b307ef2a0ef24c88e3f3e56b92
+size 859
diff --git a/test/test-resources/beam/input/snap-location/scenario/case2/xml/population.xml b/test/test-resources/beam/input/snap-location/scenario/case2/xml/population.xml
new file mode 100755
index 00000000000..e810d12ebdc
--- /dev/null
+++ b/test/test-resources/beam/input/snap-location/scenario/case2/xml/population.xml
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b9cfa90074dc01de642da79b6896271d74d3d36967731dabdea2a006f345568c
+size 2849
diff --git a/test/test-resources/beam/input/snap-location/scenario/case2/xml/populationAttributes.xml b/test/test-resources/beam/input/snap-location/scenario/case2/xml/populationAttributes.xml
new file mode 100755
index 00000000000..c2d270cac35
--- /dev/null
+++ b/test/test-resources/beam/input/snap-location/scenario/case2/xml/populationAttributes.xml
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:adb3199cd0c3d15ee9244af92c81479522908329ca44893bd7cb958554856f37
+size 998

From eab2db4e6d520569d0fc04709c43faf7c02e8dad Mon Sep 17 00:00:00 2001
From: Rutvik Patel <heyrutvik@gmail.com>
Date: Thu, 7 Apr 2022 13:19:58 +0530
Subject: [PATCH 47/58] adds test for urbansimv2 input

---
 .../scala/beam/utils/SnapCoordinateSpec.scala | 73 +++++++++++++++----
 .../scenario/case1/urbansim_v2/blocks.csv.gz  |  3 +
 .../case1/urbansim_v2/households.csv.gz       |  3 +
 .../scenario/case1/urbansim_v2/persons.csv.gz |  3 +
 .../scenario/case1/urbansim_v2/plans.csv.gz   |  3 +
 .../scenario/case2/urbansim_v2/blocks.csv.gz  |  3 +
 .../case2/urbansim_v2/households.csv.gz       |  3 +
 .../scenario/case2/urbansim_v2/persons.csv.gz |  3 +
 .../scenario/case2/urbansim_v2/plans.csv.gz   |  3 +
 9 files changed, 82 insertions(+), 15 deletions(-)
 create mode 100644 test/test-resources/beam/input/snap-location/scenario/case1/urbansim_v2/blocks.csv.gz
 create mode 100644 test/test-resources/beam/input/snap-location/scenario/case1/urbansim_v2/households.csv.gz
 create mode 100644 test/test-resources/beam/input/snap-location/scenario/case1/urbansim_v2/persons.csv.gz
 create mode 100644 test/test-resources/beam/input/snap-location/scenario/case1/urbansim_v2/plans.csv.gz
 create mode 100644 test/test-resources/beam/input/snap-location/scenario/case2/urbansim_v2/blocks.csv.gz
 create mode 100644 test/test-resources/beam/input/snap-location/scenario/case2/urbansim_v2/households.csv.gz
 create mode 100644 test/test-resources/beam/input/snap-location/scenario/case2/urbansim_v2/persons.csv.gz
 create mode 100644 test/test-resources/beam/input/snap-location/scenario/case2/urbansim_v2/plans.csv.gz

diff --git a/src/test/scala/beam/utils/SnapCoordinateSpec.scala b/src/test/scala/beam/utils/SnapCoordinateSpec.scala
index a3ae419347a..eb8540381ca 100644
--- a/src/test/scala/beam/utils/SnapCoordinateSpec.scala
+++ b/src/test/scala/beam/utils/SnapCoordinateSpec.scala
@@ -16,21 +16,11 @@ import scala.jdk.CollectionConverters.collectionAsScalaIterableConverter
 
 class SnapCoordinateSpec extends AnyWordSpec with Matchers with BeamHelper {
 
-  /**
-    * see the notes,
-    *
-    * Scenario:
-    * case1: test/test-resources/beam/input/snap-location/scenario/case1/note
-    *   three invalid plans, update population/households accordingly
-    * case2: test/test-resources/beam/input/snap-location/scenario/case2/note
-    *   one invalid household and three invalid plans, update population/households accordingly
-    *
-    * Freight:
-    * ???
-    */
-
   val pwd: String = System.getenv("PWD")
 
+  // TODO
+  // 1. compare out put csv
+
   "scenario plan" should {
     "contain all valid locations" in {
       lazy val config: TypesafeConfig = ConfigFactory
@@ -93,7 +83,7 @@ class SnapCoordinateSpec extends AnyWordSpec with Matchers with BeamHelper {
       } shouldBe true
     }
 
-    "remove/update invalid persons and households [case1 xml input]" in {
+    "remove invalid persons and households [case1 xml input]" in {
       lazy val config: TypesafeConfig = ConfigFactory
         .parseString(s"""
                         |beam.agentsim.agents.plans {
@@ -129,7 +119,7 @@ class SnapCoordinateSpec extends AnyWordSpec with Matchers with BeamHelper {
       }
     }
 
-    "remove/update invalid persons and households [case2 xml input]" in {
+    "remove invalid persons and households [case2 xml input]" in {
       lazy val config: TypesafeConfig = ConfigFactory
         .parseString(s"""
                         |beam.agentsim.agents.plans {
@@ -213,11 +203,64 @@ class SnapCoordinateSpec extends AnyWordSpec with Matchers with BeamHelper {
         matsimConfig
       )
 
+      scenario.getPopulation.getPersons.size() shouldBe 1
+      scenario.getHouseholds.getHouseholds.size() shouldBe 1
+    }
+
+    "remove invalid persons and households [case1 urbansimv2 input]" in {
+      lazy val config: TypesafeConfig = ConfigFactory
+        .parseString(s"""
+                        |beam.exchange.scenario.folder = "$pwd/test/test-resources/beam/input/snap-location/scenario/case1/urbansim_v2"
+                        |beam.routing.r5.linkRadiusMeters = 350
+                        |""".stripMargin)
+        .withFallback(testConfig("test/input/beamville/beam-urbansimv2.conf"))
+        .resolve()
+
+      val configBuilder = new MatSimBeamConfigBuilder(config)
+      val matsimConfig: Config = configBuilder.buildMatSimConf()
+      matsimConfig.planCalcScore().setMemorizingExperiencedPlans(true)
+      val beamConfig: BeamConfig = BeamConfig(config)
+
+      FileUtils.setConfigOutputFile(beamConfig, matsimConfig)
+
+      val (scenario, _, _) = buildBeamServicesAndScenario(
+        beamConfig,
+        matsimConfig
+      )
+
       val households = scenario.getHouseholds.getHouseholds.values().asScala.toList
 
+      scenario.getPopulation.getPersons.size() shouldBe 2
+      households.foreach { household =>
+        household.getMemberIds.size() shouldBe 1
+      }
+    }
+
+    "remove invalid persons and households [case2 urbansimv2 input]" in {
+      lazy val config: TypesafeConfig = ConfigFactory
+        .parseString(s"""
+                        |beam.exchange.scenario.folder = "$pwd/test/test-resources/beam/input/snap-location/scenario/case2/urbansim_v2"
+                        |beam.routing.r5.linkRadiusMeters = 350
+                        |""".stripMargin)
+        .withFallback(testConfig("test/input/beamville/beam-urbansimv2.conf"))
+        .resolve()
+
+      val configBuilder = new MatSimBeamConfigBuilder(config)
+      val matsimConfig: Config = configBuilder.buildMatSimConf()
+      matsimConfig.planCalcScore().setMemorizingExperiencedPlans(true)
+      val beamConfig: BeamConfig = BeamConfig(config)
+
+      FileUtils.setConfigOutputFile(beamConfig, matsimConfig)
+
+      val (scenario, _, _) = buildBeamServicesAndScenario(
+        beamConfig,
+        matsimConfig
+      )
+
       scenario.getPopulation.getPersons.size() shouldBe 1
       scenario.getHouseholds.getHouseholds.size() shouldBe 1
     }
+
   }
 
 }
diff --git a/test/test-resources/beam/input/snap-location/scenario/case1/urbansim_v2/blocks.csv.gz b/test/test-resources/beam/input/snap-location/scenario/case1/urbansim_v2/blocks.csv.gz
new file mode 100644
index 00000000000..8eccf4c87c2
--- /dev/null
+++ b/test/test-resources/beam/input/snap-location/scenario/case1/urbansim_v2/blocks.csv.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:567708718a96701b3977ddb90c21917b6d5dcbab79ad54d302cc8b91dfdb6cca
+size 249
diff --git a/test/test-resources/beam/input/snap-location/scenario/case1/urbansim_v2/households.csv.gz b/test/test-resources/beam/input/snap-location/scenario/case1/urbansim_v2/households.csv.gz
new file mode 100644
index 00000000000..d051b989b01
--- /dev/null
+++ b/test/test-resources/beam/input/snap-location/scenario/case1/urbansim_v2/households.csv.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fee1ebbd211c0f9d9b53f9294a053dd930e5d20edcaf84534cf5395e6c02d1b4
+size 156
diff --git a/test/test-resources/beam/input/snap-location/scenario/case1/urbansim_v2/persons.csv.gz b/test/test-resources/beam/input/snap-location/scenario/case1/urbansim_v2/persons.csv.gz
new file mode 100644
index 00000000000..a7a76074200
--- /dev/null
+++ b/test/test-resources/beam/input/snap-location/scenario/case1/urbansim_v2/persons.csv.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8a4f7fc7716d6f202f51e60baa5da09d7ac44480c9ec1e71db6a3a7805f662c1
+size 217
diff --git a/test/test-resources/beam/input/snap-location/scenario/case1/urbansim_v2/plans.csv.gz b/test/test-resources/beam/input/snap-location/scenario/case1/urbansim_v2/plans.csv.gz
new file mode 100644
index 00000000000..7673b1ab478
--- /dev/null
+++ b/test/test-resources/beam/input/snap-location/scenario/case1/urbansim_v2/plans.csv.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be7f1b757d480628e044501adbe99f1a48245467d6c767fa440a7d233971a978
+size 615
diff --git a/test/test-resources/beam/input/snap-location/scenario/case2/urbansim_v2/blocks.csv.gz b/test/test-resources/beam/input/snap-location/scenario/case2/urbansim_v2/blocks.csv.gz
new file mode 100644
index 00000000000..a77ffc2555e
--- /dev/null
+++ b/test/test-resources/beam/input/snap-location/scenario/case2/urbansim_v2/blocks.csv.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f4b46994dcb316c264c77feed485283bee13cf25fea4bf80c3b49b075e82eedd
+size 250
diff --git a/test/test-resources/beam/input/snap-location/scenario/case2/urbansim_v2/households.csv.gz b/test/test-resources/beam/input/snap-location/scenario/case2/urbansim_v2/households.csv.gz
new file mode 100644
index 00000000000..d051b989b01
--- /dev/null
+++ b/test/test-resources/beam/input/snap-location/scenario/case2/urbansim_v2/households.csv.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fee1ebbd211c0f9d9b53f9294a053dd930e5d20edcaf84534cf5395e6c02d1b4
+size 156
diff --git a/test/test-resources/beam/input/snap-location/scenario/case2/urbansim_v2/persons.csv.gz b/test/test-resources/beam/input/snap-location/scenario/case2/urbansim_v2/persons.csv.gz
new file mode 100644
index 00000000000..0427fc031f3
--- /dev/null
+++ b/test/test-resources/beam/input/snap-location/scenario/case2/urbansim_v2/persons.csv.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b8c1661871669f2c20b4f22272184285ab64a5ea213aaf248d1a81ca9ec9a1c5
+size 217
diff --git a/test/test-resources/beam/input/snap-location/scenario/case2/urbansim_v2/plans.csv.gz b/test/test-resources/beam/input/snap-location/scenario/case2/urbansim_v2/plans.csv.gz
new file mode 100644
index 00000000000..28e5c4f98c5
--- /dev/null
+++ b/test/test-resources/beam/input/snap-location/scenario/case2/urbansim_v2/plans.csv.gz
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a7d3af7a4245e2e843b0c77666a2d04a4a2f26024f4ad76b62f6b8102ffc618c
+size 613

From 50909906f4191d8af75e75be44283475736e69ef Mon Sep 17 00:00:00 2001
From: Rutvik Patel <heyrutvik@gmail.com>
Date: Thu, 7 Apr 2022 17:18:46 +0530
Subject: [PATCH 48/58] adds freight tests for snap location module

---
 .../freight/input/GenericFreightReader.scala  |  10 +-
 .../beam/utils/SnapCoordinateUtils.scala      |   8 +
 .../utils/scenario/ScenarioLoaderHelper.scala |  10 +-
 .../scala/beam/utils/SnapCoordinateSpec.scala | 159 ++++++++++++++++--
 .../freight/freight-carriers.csv              |   3 +
 .../snap-location/freight/freight-tours.csv   |   3 +
 .../snap-location/freight/payload-plans.csv   |   3 +
 7 files changed, 177 insertions(+), 19 deletions(-)
 create mode 100644 test/test-resources/beam/input/snap-location/freight/freight-carriers.csv
 create mode 100644 test/test-resources/beam/input/snap-location/freight/freight-tours.csv
 create mode 100644 test/test-resources/beam/input/snap-location/freight/payload-plans.csv

diff --git a/src/main/scala/beam/agentsim/agents/freight/input/GenericFreightReader.scala b/src/main/scala/beam/agentsim/agents/freight/input/GenericFreightReader.scala
index 1e74fb21d72..28d6ca5b9a6 100644
--- a/src/main/scala/beam/agentsim/agents/freight/input/GenericFreightReader.scala
+++ b/src/main/scala/beam/agentsim/agents/freight/input/GenericFreightReader.scala
@@ -7,7 +7,7 @@ import beam.agentsim.infrastructure.taz.{TAZ, TAZTreeMap}
 import beam.sim.common.GeoUtils
 import beam.sim.config.BeamConfig.Beam.Agentsim.Agents.Freight
 import beam.utils.SnapCoordinateUtils
-import beam.utils.SnapCoordinateUtils.{Category, Error, ErrorInfo, Result, SnapLocationHelper}
+import beam.utils.SnapCoordinateUtils.{Category, CsvFile, Error, ErrorInfo, Result, SnapLocationHelper}
 import beam.utils.csv.GenericCsvReader
 import beam.utils.matsim_conversion.MatsimPlanConversion.IdOps
 import com.typesafe.scalalogging.LazyLogging
@@ -114,7 +114,7 @@ class GenericFreightReader(
 
     outputDirMaybe.foreach { path =>
       if (errors.isEmpty) logger.info("No 'snap location' error to report for freight tours.")
-      else SnapCoordinateUtils.writeToCsv(s"$path/snapLocationFreightTourErrors.csv.gz", errors)
+      else SnapCoordinateUtils.writeToCsv(s"$path/${CsvFile.FreightTours}", errors)
     }
 
     maybeTours.flatten
@@ -202,7 +202,7 @@ class GenericFreightReader(
 
     outputDirMaybe.foreach { path =>
       if (errors.isEmpty) logger.info("No 'snap location' error to report for freight payload plans.")
-      else SnapCoordinateUtils.writeToCsv(s"$path/snapLocationFreightPayloadPlanErrors.csv.gz", errors)
+      else SnapCoordinateUtils.writeToCsv(s"$path/${CsvFile.FreightPayloadPlans}", errors)
     }
 
     maybePlans.flatten
@@ -341,11 +341,13 @@ class GenericFreightReader(
 
     outputDirMaybe.foreach { path =>
       if (errors.isEmpty) logger.info("No 'snap location' error to report for freight carriers.")
-      else SnapCoordinateUtils.writeToCsv(s"$path/snapLocationFreightCarrierErrors.csv.gz", errors)
+      else SnapCoordinateUtils.writeToCsv(s"$path/${CsvFile.FreightCarriers}", errors)
     }
 
+    val removedCarrierIds = errors.map(_.id)
     val carriersWithFleet = maybeCarrierRows.flatten
       .groupBy(_.carrierId)
+      .filterNot { case (carrierId, _) => removedCarrierIds.contains(carrierId.toString) }
       .map { case (carrierId, carrierRows) =>
         createCarrier(carrierId, carrierRows)
       }
diff --git a/src/main/scala/beam/utils/SnapCoordinateUtils.scala b/src/main/scala/beam/utils/SnapCoordinateUtils.scala
index 7e4a95825c5..c2807065d30 100644
--- a/src/main/scala/beam/utils/SnapCoordinateUtils.scala
+++ b/src/main/scala/beam/utils/SnapCoordinateUtils.scala
@@ -54,6 +54,14 @@ object SnapCoordinateUtils extends LazyLogging {
     val FreightCarrier = "Carrier"
   }
 
+  object CsvFile {
+    val Plans = "snapLocationPlanErrors.csv"
+    val Households = "snapLocationHouseholdErrors.csv"
+    val FreightTours = "snapLocationFreightTourErrors.csv"
+    val FreightPayloadPlans = "snapLocationFreightPayloadPlanErrors.csv"
+    val FreightCarriers = "snapLocationFreightCarrierErrors.csv"
+  }
+
   final case class ErrorInfo(id: String, category: String, error: String, planX: Double, planY: Double)
   final case class Processed[A](data: Seq[A] = Seq.empty, errors: Seq[ErrorInfo] = Seq.empty)
 
diff --git a/src/main/scala/beam/utils/scenario/ScenarioLoaderHelper.scala b/src/main/scala/beam/utils/scenario/ScenarioLoaderHelper.scala
index 48400e10d71..538a3a42efe 100644
--- a/src/main/scala/beam/utils/scenario/ScenarioLoaderHelper.scala
+++ b/src/main/scala/beam/utils/scenario/ScenarioLoaderHelper.scala
@@ -3,7 +3,7 @@ package beam.utils.scenario
 import beam.sim.BeamScenario
 import beam.sim.common.GeoUtils
 import beam.utils.SnapCoordinateUtils
-import beam.utils.SnapCoordinateUtils.{Category, Error, ErrorInfo, Processed, Result, SnapLocationHelper}
+import beam.utils.SnapCoordinateUtils.{Category, CsvFile, Error, ErrorInfo, Processed, Result, SnapLocationHelper}
 import com.typesafe.scalalogging.LazyLogging
 import org.matsim.api.core.v01.{Coord, Id}
 import org.matsim.api.core.v01.population.{Activity, Leg, Person}
@@ -124,7 +124,7 @@ trait ScenarioLoaderHelper extends LazyLogging {
 
     outputDirMaybe.foreach { path =>
       if (errors.isEmpty) logger.info("No 'snap location' error to report for scenario plans.")
-      else SnapCoordinateUtils.writeToCsv(s"$path/snapLocationPlanErrors.csv.gz", errors)
+      else SnapCoordinateUtils.writeToCsv(s"$path/${CsvFile.Plans}", errors)
     }
 
     val filteredCnt = plans.size - validPlans.size
@@ -139,7 +139,7 @@ trait ScenarioLoaderHelper extends LazyLogging {
 
     outputDirMaybe.foreach { path =>
       if (errors.isEmpty) logger.info("No 'snap location' error to report for scenario households.")
-      else SnapCoordinateUtils.writeToCsv(s"$path/snapLocationHouseholdErrors.csv.gz", errors)
+      else SnapCoordinateUtils.writeToCsv(s"$path/${CsvFile.Households}", errors)
     }
 
     val filteredCnt = households.size - validHouseholds.size
@@ -298,10 +298,10 @@ object ScenarioLoaderHelper extends LazyLogging {
 
     outputDirMaybe.foreach { path =>
       if (planErrors.isEmpty) logger.info("No 'snap location' error to report for scenario plans.")
-      else SnapCoordinateUtils.writeToCsv(s"$path/snapLocationPlanErrors.csv.gz", planErrors)
+      else SnapCoordinateUtils.writeToCsv(s"$path/${CsvFile.Plans}", planErrors)
 
       if (householdErrors.isEmpty) logger.info("No 'snap location' error to report for scenario households.")
-      else SnapCoordinateUtils.writeToCsv(s"$path/snapLocationHouseholdErrors.csv.gz", householdErrors)
+      else SnapCoordinateUtils.writeToCsv(s"$path/${CsvFile.Households}", householdErrors)
     }
   }
 
diff --git a/src/test/scala/beam/utils/SnapCoordinateSpec.scala b/src/test/scala/beam/utils/SnapCoordinateSpec.scala
index eb8540381ca..7fd1929df43 100644
--- a/src/test/scala/beam/utils/SnapCoordinateSpec.scala
+++ b/src/test/scala/beam/utils/SnapCoordinateSpec.scala
@@ -1,25 +1,69 @@
 package beam.utils
 
+import beam.agentsim.agents.freight.{FreightCarrier, FreightTour, PayloadPlan}
 import beam.sim.BeamHelper
 import beam.sim.common.GeoUtilsImpl
 import beam.sim.config.{BeamConfig, MatSimBeamConfigBuilder}
-import beam.utils.SnapCoordinateUtils.{Result, SnapLocationHelper}
+import beam.utils.SnapCoordinateUtils.{CsvFile, ErrorInfo, Result, SnapLocationHelper}
 import beam.utils.TestConfigUtils.testConfig
 import com.typesafe.config.{ConfigFactory, Config => TypesafeConfig}
 import org.matsim.api.core.v01.Coord
-import org.matsim.api.core.v01.population.Activity
+import org.matsim.api.core.v01.population.{Activity, Population}
 import org.matsim.core.config.Config
+import org.matsim.households.Households
 import org.scalatest.matchers.should.Matchers
 import org.scalatest.wordspec.AnyWordSpec
 
+import scala.io.Source
 import scala.jdk.CollectionConverters.collectionAsScalaIterableConverter
 
 class SnapCoordinateSpec extends AnyWordSpec with Matchers with BeamHelper {
 
   val pwd: String = System.getenv("PWD")
 
-  // TODO
-  // 1. compare out put csv
+  def readErrorCsv(path: String): List[ErrorInfo] = {
+    val src = Source.fromFile(path)
+    val errors = src
+      .getLines()
+      .drop(1)
+      .map(_.split(","))
+      .map { row =>
+        ErrorInfo(row(0), row(1), row(2), row(3).toDouble, row(4).toDouble)
+      }
+      .toList
+    src.close()
+    errors
+  }
+
+  def intersection(population: Population, path: String): Set[String] = {
+    val invalidPersonIds = readErrorCsv(path).map(_.id).toSet
+    val validPersonIds = population.getPersons.keySet().asScala.map(_.toString).toSet
+    validPersonIds.intersect(invalidPersonIds)
+  }
+
+  def intersection(households: Households, path: String): Set[String] = {
+    val invalidHouseholdIds = readErrorCsv(path).map(_.id).toSet
+    val validHouseholdIds = households.getHouseholds.keySet().asScala.map(_.toString).toSet
+    validHouseholdIds.intersect(invalidHouseholdIds)
+  }
+
+  def intersection(freightTours: Array[FreightTour], path: String): Set[String] = {
+    val invalidIds = readErrorCsv(path).map(_.id).toSet
+    val validIds = freightTours.map(_.tourId.toString).toSet
+    validIds.intersect(invalidIds)
+  }
+
+  def intersection(freightPayloadPlans: Array[PayloadPlan], path: String): Set[String] = {
+    val invalidIds = readErrorCsv(path).map(_.id).toSet
+    val validIds = freightPayloadPlans.map(_.payloadId.toString).toSet
+    validIds.intersect(invalidIds)
+  }
+
+  def intersection(freightCarriers: Array[FreightCarrier], path: String): Set[String] = {
+    val invalidIds = readErrorCsv(path).map(_.id).toSet
+    val validIds = freightCarriers.map(_.carrierId.toString).toSet
+    validIds.intersect(invalidIds)
+  }
 
   "scenario plan" should {
     "contain all valid locations" in {
@@ -104,7 +148,7 @@ class SnapCoordinateSpec extends AnyWordSpec with Matchers with BeamHelper {
       matsimConfig.planCalcScore().setMemorizingExperiencedPlans(true)
       val beamConfig: BeamConfig = BeamConfig(config)
 
-      FileUtils.setConfigOutputFile(beamConfig, matsimConfig)
+      val outputDir = FileUtils.setConfigOutputFile(beamConfig, matsimConfig)
 
       val (scenario, _, _) = buildBeamServicesAndScenario(
         beamConfig,
@@ -114,6 +158,8 @@ class SnapCoordinateSpec extends AnyWordSpec with Matchers with BeamHelper {
       val households = scenario.getHouseholds.getHouseholds.values().asScala.toList
 
       scenario.getPopulation.getPersons.size() shouldBe 2
+      intersection(scenario.getPopulation, path = s"$outputDir/${CsvFile.Plans}") shouldBe Set.empty
+
       households.foreach { household =>
         household.getMemberIds.size() shouldBe 1
       }
@@ -140,7 +186,7 @@ class SnapCoordinateSpec extends AnyWordSpec with Matchers with BeamHelper {
       matsimConfig.planCalcScore().setMemorizingExperiencedPlans(true)
       val beamConfig: BeamConfig = BeamConfig(config)
 
-      FileUtils.setConfigOutputFile(beamConfig, matsimConfig)
+      val outputDir = FileUtils.setConfigOutputFile(beamConfig, matsimConfig)
 
       val (scenario, _, _) = buildBeamServicesAndScenario(
         beamConfig,
@@ -148,7 +194,10 @@ class SnapCoordinateSpec extends AnyWordSpec with Matchers with BeamHelper {
       )
 
       scenario.getPopulation.getPersons.size() shouldBe 1
+      intersection(scenario.getPopulation, path = s"$outputDir/${CsvFile.Plans}") shouldBe Set.empty
+
       scenario.getHouseholds.getHouseholds.size() shouldBe 1
+      intersection(scenario.getHouseholds, path = s"$outputDir/${CsvFile.Households}") shouldBe Set.empty
     }
 
     "remove invalid persons and households [case1 csv input]" in {
@@ -166,7 +215,7 @@ class SnapCoordinateSpec extends AnyWordSpec with Matchers with BeamHelper {
       matsimConfig.planCalcScore().setMemorizingExperiencedPlans(true)
       val beamConfig: BeamConfig = BeamConfig(config)
 
-      FileUtils.setConfigOutputFile(beamConfig, matsimConfig)
+      val outputDir = FileUtils.setConfigOutputFile(beamConfig, matsimConfig)
 
       val (scenario, _, _) = buildBeamServicesAndScenario(
         beamConfig,
@@ -176,6 +225,8 @@ class SnapCoordinateSpec extends AnyWordSpec with Matchers with BeamHelper {
       val households = scenario.getHouseholds.getHouseholds.values().asScala.toList
 
       scenario.getPopulation.getPersons.size() shouldBe 2
+      intersection(scenario.getPopulation, path = s"$outputDir/${CsvFile.Plans}") shouldBe Set.empty
+
       households.foreach { household =>
         household.getMemberIds.size() shouldBe 1
       }
@@ -196,7 +247,7 @@ class SnapCoordinateSpec extends AnyWordSpec with Matchers with BeamHelper {
       matsimConfig.planCalcScore().setMemorizingExperiencedPlans(true)
       val beamConfig: BeamConfig = BeamConfig(config)
 
-      FileUtils.setConfigOutputFile(beamConfig, matsimConfig)
+      val outputDir = FileUtils.setConfigOutputFile(beamConfig, matsimConfig)
 
       val (scenario, _, _) = buildBeamServicesAndScenario(
         beamConfig,
@@ -204,7 +255,10 @@ class SnapCoordinateSpec extends AnyWordSpec with Matchers with BeamHelper {
       )
 
       scenario.getPopulation.getPersons.size() shouldBe 1
+      intersection(scenario.getPopulation, path = s"$outputDir/${CsvFile.Plans}") shouldBe Set.empty
+
       scenario.getHouseholds.getHouseholds.size() shouldBe 1
+      intersection(scenario.getHouseholds, path = s"$outputDir/${CsvFile.Households}") shouldBe Set.empty
     }
 
     "remove invalid persons and households [case1 urbansimv2 input]" in {
@@ -221,7 +275,7 @@ class SnapCoordinateSpec extends AnyWordSpec with Matchers with BeamHelper {
       matsimConfig.planCalcScore().setMemorizingExperiencedPlans(true)
       val beamConfig: BeamConfig = BeamConfig(config)
 
-      FileUtils.setConfigOutputFile(beamConfig, matsimConfig)
+      val outputDir = FileUtils.setConfigOutputFile(beamConfig, matsimConfig)
 
       val (scenario, _, _) = buildBeamServicesAndScenario(
         beamConfig,
@@ -231,6 +285,8 @@ class SnapCoordinateSpec extends AnyWordSpec with Matchers with BeamHelper {
       val households = scenario.getHouseholds.getHouseholds.values().asScala.toList
 
       scenario.getPopulation.getPersons.size() shouldBe 2
+      intersection(scenario.getPopulation, path = s"$outputDir/${CsvFile.Plans}") shouldBe Set.empty
+
       households.foreach { household =>
         household.getMemberIds.size() shouldBe 1
       }
@@ -250,7 +306,7 @@ class SnapCoordinateSpec extends AnyWordSpec with Matchers with BeamHelper {
       matsimConfig.planCalcScore().setMemorizingExperiencedPlans(true)
       val beamConfig: BeamConfig = BeamConfig(config)
 
-      FileUtils.setConfigOutputFile(beamConfig, matsimConfig)
+      val outputDir = FileUtils.setConfigOutputFile(beamConfig, matsimConfig)
 
       val (scenario, _, _) = buildBeamServicesAndScenario(
         beamConfig,
@@ -258,9 +314,92 @@ class SnapCoordinateSpec extends AnyWordSpec with Matchers with BeamHelper {
       )
 
       scenario.getPopulation.getPersons.size() shouldBe 1
+      intersection(scenario.getPopulation, path = s"$outputDir/${CsvFile.Plans}") shouldBe Set.empty
+
       scenario.getHouseholds.getHouseholds.size() shouldBe 1
+      intersection(scenario.getHouseholds, path = s"$outputDir/${CsvFile.Households}") shouldBe Set.empty
     }
 
+    "remove invalid freight tours" in {
+      lazy val config: TypesafeConfig = ConfigFactory
+        .parseString(s"""
+                        |beam.agentsim.agents.freight.toursFilePath = "$pwd/test/test-resources/beam/input/snap-location/freight/freight-tours.csv"
+                        |beam.routing.r5.linkRadiusMeters = 350
+                        |""".stripMargin)
+        .withFallback(testConfig("test/input/beamville/beam-freight.conf"))
+        .resolve()
+
+      val configBuilder = new MatSimBeamConfigBuilder(config)
+      val matsimConfig: Config = configBuilder.buildMatSimConf()
+      matsimConfig.planCalcScore().setMemorizingExperiencedPlans(true)
+      val beamConfig: BeamConfig = BeamConfig(config)
+
+      val outputDir = FileUtils.setConfigOutputFile(beamConfig, matsimConfig)
+
+      val (_, beamScenario, _) = buildBeamServicesAndScenario(
+        beamConfig,
+        matsimConfig
+      )
+
+      intersection(
+        beamScenario.freightCarriers.flatMap(_.tourMap.values).flatten.toArray,
+        path = s"$outputDir/${CsvFile.FreightTours}"
+      ) shouldBe Set.empty
+    }
+
+    "remove invalid freight payload plans" in {
+      lazy val config: TypesafeConfig = ConfigFactory
+        .parseString(s"""
+                        |beam.agentsim.agents.freight.plansFilePath = "$pwd/test/test-resources/beam/input/snap-location/freight/payload-plans.csv"
+                        |beam.routing.r5.linkRadiusMeters = 350
+                        |""".stripMargin)
+        .withFallback(testConfig("test/input/beamville/beam-freight.conf"))
+        .resolve()
+
+      val configBuilder = new MatSimBeamConfigBuilder(config)
+      val matsimConfig: Config = configBuilder.buildMatSimConf()
+      matsimConfig.planCalcScore().setMemorizingExperiencedPlans(true)
+      val beamConfig: BeamConfig = BeamConfig(config)
+
+      val outputDir = FileUtils.setConfigOutputFile(beamConfig, matsimConfig)
+
+      val (_, beamScenario, _) = buildBeamServicesAndScenario(
+        beamConfig,
+        matsimConfig
+      )
+
+      intersection(
+        beamScenario.freightCarriers.flatMap(_.payloadPlans.values).toArray,
+        path = s"$outputDir/${CsvFile.FreightPayloadPlans}"
+      ) shouldBe Set.empty
+    }
+
+    "remove invalid freight carriers" in {
+      lazy val config: TypesafeConfig = ConfigFactory
+        .parseString(s"""
+                        |beam.agentsim.agents.freight.carriersFilePath = "$pwd/test/test-resources/beam/input/snap-location/freight/freight-carriers.csv"
+                        |beam.routing.r5.linkRadiusMeters = 350
+                        |""".stripMargin)
+        .withFallback(testConfig("test/input/beamville/beam-freight.conf"))
+        .resolve()
+
+      val configBuilder = new MatSimBeamConfigBuilder(config)
+      val matsimConfig: Config = configBuilder.buildMatSimConf()
+      matsimConfig.planCalcScore().setMemorizingExperiencedPlans(true)
+      val beamConfig: BeamConfig = BeamConfig(config)
+
+      val outputDir = FileUtils.setConfigOutputFile(beamConfig, matsimConfig)
+
+      val (_, beamScenario, _) = buildBeamServicesAndScenario(
+        beamConfig,
+        matsimConfig
+      )
+
+      intersection(
+        beamScenario.freightCarriers.toArray,
+        path = s"$outputDir/${CsvFile.FreightCarriers}"
+      ) shouldBe Set.empty
+    }
   }
 
 }
diff --git a/test/test-resources/beam/input/snap-location/freight/freight-carriers.csv b/test/test-resources/beam/input/snap-location/freight/freight-carriers.csv
new file mode 100644
index 00000000000..d87ec9d3c7d
--- /dev/null
+++ b/test/test-resources/beam/input/snap-location/freight/freight-carriers.csv
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:403e9ccbedb831269fd57e1a5edd48e068cdf33750bf1bd02867246d7a8c058e
+size 232
diff --git a/test/test-resources/beam/input/snap-location/freight/freight-tours.csv b/test/test-resources/beam/input/snap-location/freight/freight-tours.csv
new file mode 100644
index 00000000000..3595f6b82ff
--- /dev/null
+++ b/test/test-resources/beam/input/snap-location/freight/freight-tours.csv
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a05f0f2b94046948b249cd3bf5d683b3bbb34951d40fc1be70a28ba6ef2256d0
+size 248
diff --git a/test/test-resources/beam/input/snap-location/freight/payload-plans.csv b/test/test-resources/beam/input/snap-location/freight/payload-plans.csv
new file mode 100644
index 00000000000..9c6e9b76a46
--- /dev/null
+++ b/test/test-resources/beam/input/snap-location/freight/payload-plans.csv
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4ca6c183d0add95c0d68a7b2d26bb6c62bab40d723b13271ef697c38060ca92e
+size 939

From de746a267ee214c2cf8392e606566d5a1df27942 Mon Sep 17 00:00:00 2001
From: Haitam Laarabi <10712736+htmlrb@users.noreply.github.com>
Date: Fri, 8 Apr 2022 18:38:44 -0700
Subject: [PATCH 49/58] dropping memoization

---
 build.gradle                                  |  2 +-
 src/main/R/freight/freight-processing.R       |  9 ++--
 src/main/python/freight/events_processing.py  | 27 ++++++++++
 src/main/scala/beam/sim/common/GeoUtils.scala | 52 ++++++-------------
 4 files changed, 50 insertions(+), 40 deletions(-)
 create mode 100644 src/main/python/freight/events_processing.py

diff --git a/build.gradle b/build.gradle
index e1cd992c9f2..248bf1880b3 100755
--- a/build.gradle
+++ b/build.gradle
@@ -199,7 +199,7 @@ dependencies {
     implementation "com.typesafe.scala-logging:scala-logging_${scalaBinaryVersion}:3.9.0"
     implementation "org.slf4j:log4j-over-slf4j:${slf4jVersion}"
 
-    implementation(group: 'com.github.LBNL-UCB-STI', name: 'r5', version: 'v4.5.3_unconstrainedSpatialSearch') {
+    implementation(group: 'com.github.LBNL-UCB-STI', name: 'r5', version: 'v4.5.3_linkRadiusMeters') {
         exclude group: 'ch.qos.logback', module: 'logback-classic'
         exclude group: 'org.slf4j', module: 'slf4j-simple'
     }
diff --git a/src/main/R/freight/freight-processing.R b/src/main/R/freight/freight-processing.R
index 2a98c99a86c..ba0dd4d1ede 100644
--- a/src/main/R/freight/freight-processing.R
+++ b/src/main/R/freight/freight-processing.R
@@ -23,7 +23,7 @@ freightWorkDir <- normalizePath(paste(validationDir,"/beam",sep=""))
 #   file = pp(freightWorkDir, "/filtered.0.events.csv"),
 #   row.names=F,
 #   quote=T)
-events_filtered <- readCsv(pp(freightWorkDir, "/filtered.0.events.csv"))
+events_filtered <- readCsv(pp(freightWorkDir, "/filtered.0.events.new.csv"))
 pt <- events_filtered[type=="PathTraversal"][,c("time","type","vehicleType","vehicle","secondaryFuelLevel",
                                        "primaryFuelLevel","driver","mode","seatingCapacity","startX",
                                        "startY", "endX", "endY", "capacity", "arrivalTime", "departureTime",
@@ -277,7 +277,7 @@ ggsave(pp(freightWorkDir,'/freight-avg-vmt-by-category.png'),p,width=4,height=3,
 
 ##### PREPARING NETWORK AND MATCH IT WITH POSTMILE AND TRUCK AADTT DATA
 #"primary","secondary","tertiary"
-network <- readCsv(normalizePath(paste(freightDir,"/validation/network.csv.gz",sep="")))
+network <- readCsv(normalizePath(paste(freightDir,"/validation/beam/network.csv",sep="")))
 network_cleaned <- network[
   linkModes %in% c("car;bike", "car;walk;bike") & attributeOrigType %in% c("motorway","trunk","primary", "secondary")][
     ,-c("numberOfLanes", "attributeOrigId", "fromNodeId", "toNodeId", "toLocationX", "toLocationY")]
@@ -286,7 +286,7 @@ counties <- data.table::data.table(
              "San Francisco", "San Mateo", "Solano", "Sonoma"),
   CNTY=c("ALA", "CC", "MRN", "NAP", "SCL", "SF", "SM", "SOL", "SON")
 )
-linkStats <- readCsv(normalizePath(paste(freightDir,"/validation/beam/0.linkstats.csv.gz",sep="")))
+linkStats <- readCsv(normalizePath(paste(freightDir,"/validation/beam/0.linkstats.new.csv.gz",sep="")))
 
 #data.table::fwrite(network_cleaned, pp(freightDir,"/validation/network_cleaned.csv"), quote=F)
 
@@ -365,6 +365,9 @@ Volume_hpms <- sum(sf_hpms$AADT_Combi+sf_hpms$AADT_Singl)
 VMT_beam <- sum(linkStats$TruckVolume * linkStats$length/1609)
 VMT_hpms <- (sum((sf_hpms$AADT_Combi+sf_hpms$AADT_Singl) * as.numeric(st_length(sf_hpms))/1609))
 
+Volume_beam/Volume_hpms
+VMT_beam/VMT_hpms
+
 freight_pt[,.(VMT=sum(length)/1609.0),by=.(vehicleType)]
 freight_pt[,.(tourMT=sum(length)/1609.0),by=.(vehicle,vehicleType)][,.(avgTourMT=mean(tourMT)),by=.(vehicleType)]
 
diff --git a/src/main/python/freight/events_processing.py b/src/main/python/freight/events_processing.py
new file mode 100644
index 00000000000..2cb7b09c5ce
--- /dev/null
+++ b/src/main/python/freight/events_processing.py
@@ -0,0 +1,27 @@
+import pandas as pd
+import os
+
+work_directory = '~/Data/FREIGHT/validation/beam/'
+filename = '0.events.new.csv'
+full_filename = os.path.expanduser(work_directory + filename)
+compression = None
+if full_filename.endswith(".gz"):
+    compression = 'gzip'
+data = pd.read_csv(full_filename, sep=",", index_col=None, header=0, compression=compression)
+print(data.type.unique())
+data_filtered = data.loc[data.type.isin(["PathTraversal", "actstart", "actend"])]
+print(data_filtered.type.unique())
+data_filtered = data_filtered.loc[data_filtered.vehicle.str.startswith("freight", na=True)]
+print(data_filtered.type.unique())
+data_filtered2 = data_filtered.loc[data_filtered.actType.isin(["Warehouse", "Unloading", "Loading"]) | data_filtered.actType.isnull()]
+print(data_filtered.type.unique())
+# data_filtered2 = data_filtered[
+#     ["time","type","vehicleType","vehicle","secondaryFuelLevel",
+#      "primaryFuelLevel","driver","mode","seatingCapacity","startX",
+#      "startY", "endX", "endY", "capacity", "arrivalTime", "departureTime",
+#      "secondaryFuel", "secondaryFuelType", "primaryFuelType",
+#      "numPassengers", "length", "primaryFuel", "actType", "fuel", "person",
+#      "locationY", "locationX", "duration", "chargingPointType", "parkingType", "parkingTaz"]
+# ]
+data_filtered2.to_csv(work_directory + "filtered." + filename)
+print("END")
diff --git a/src/main/scala/beam/sim/common/GeoUtils.scala b/src/main/scala/beam/sim/common/GeoUtils.scala
index a41c2a58960..ddb56a0e49c 100755
--- a/src/main/scala/beam/sim/common/GeoUtils.scala
+++ b/src/main/scala/beam/sim/common/GeoUtils.scala
@@ -17,8 +17,6 @@ import org.matsim.api.core.v01.network.Link
 import org.matsim.core.utils.geometry.transformations.GeotoolsTransformation
 import org.slf4j.LoggerFactory
 
-import scala.collection.concurrent.TrieMap
-
 case class EdgeWithCoord(edgeIndex: Int, wgsCoord: Coordinate)
 
 /**
@@ -29,12 +27,9 @@ case class EdgeWithCoord(edgeIndex: Int, wgsCoord: Coordinate)
 trait GeoUtils extends ExponentialLazyLogging {
 
   def localCRS: String
+  val defaultMaxRadiusForMapSearch = 10000
   private lazy val notExponentialLogger = Logger(LoggerFactory.getLogger(getClass.getName))
 
-  type Input = (StreetMode, Double, Coord)
-  type Func = Input => Option[Split]
-  val splitStore = TrieMap[Input, Option[Split]]()
-
   lazy val utm2Wgs: GeotoolsTransformation =
     new GeotoolsTransformation(localCRS, "EPSG:4326")
 
@@ -70,7 +65,7 @@ trait GeoUtils extends ExponentialLazyLogging {
   def getNearestR5EdgeToUTMCoord(
     streetLayer: StreetLayer,
     coordUTM: Coord,
-    maxRadius: Double
+    maxRadius: Double = defaultMaxRadiusForMapSearch
   ): Int = {
     getNearestR5Edge(streetLayer, utm2Wgs(coordUTM), maxRadius)
   }
@@ -78,7 +73,7 @@ trait GeoUtils extends ExponentialLazyLogging {
   def getNearestR5Edge(
     streetLayer: StreetLayer,
     coordWGS: Coord,
-    maxRadius: Double
+    maxRadius: Double = defaultMaxRadiusForMapSearch
   ): Int = {
     val theSplit = getR5Split(streetLayer, coordWGS, maxRadius, StreetMode.WALK)
     if (theSplit == null) {
@@ -111,7 +106,7 @@ trait GeoUtils extends ExponentialLazyLogging {
   def snapToR5Edge(
     streetLayer: StreetLayer,
     coordWGS: Coord,
-    maxRadius: Double,
+    maxRadius: Double = defaultMaxRadiusForMapSearch,
     streetMode: StreetMode = StreetMode.WALK
   ): Coord = {
     val theSplit = getR5Split(streetLayer, coordWGS, maxRadius, streetMode)
@@ -125,37 +120,22 @@ trait GeoUtils extends ExponentialLazyLogging {
   def getR5Split(
     streetLayer: StreetLayer,
     coord: Coord,
-    maxRadius: Double,
-    streetMode: StreetMode = StreetMode.WALK
-  ): Split = {
-    splitStore
-      .getOrElseUpdate((streetMode, maxRadius, coord), Option(_getR5Split(streetLayer, coord, maxRadius, streetMode)))
-      .orNull
-  }
-
-  private def _getR5Split(
-    streetLayer: StreetLayer,
-    coord: Coord,
-    maxRadius: Double,
+    maxRadius: Double = defaultMaxRadiusForMapSearch,
     streetMode: StreetMode = StreetMode.WALK
   ): Split = {
-    logger.info("Called _getR5Split with {}, {}, {}", streetMode, maxRadius, coord)
-    val isWithinBbox = streetLayer.envelope.contains(coord.getX, coord.getY)
     var radius = 10.0
     var theSplit: Split = null
-    if (isWithinBbox) {
-      while (theSplit == null && radius <= maxRadius) {
-        theSplit = streetLayer.findSplit(coord.getY, coord.getX, radius, streetMode)
-        radius = radius * 10
-      }
-      if (theSplit == null) {
-        theSplit = streetLayer.findSplit(coord.getY, coord.getX, maxRadius, streetMode)
-      }
-      if (theSplit == null) {
-        notExponentialLogger.warn(
-          s"The split is `null` for StreetLayer.BoundingBox: ${streetLayer.getEnvelope}, coord: $coord, maxRadius: $maxRadius, street mode $streetMode"
-        )
-      }
+    while (theSplit == null && radius <= maxRadius) {
+      theSplit = streetLayer.findSplit(coord.getY, coord.getX, radius, streetMode)
+      radius = radius * 10
+    }
+    if (theSplit == null) {
+      theSplit = streetLayer.findSplit(coord.getY, coord.getX, maxRadius, streetMode)
+    }
+    if (theSplit == null) {
+      notExponentialLogger.warn(
+        s"The split is `null` for StreetLayer.BoundingBox: ${streetLayer.getEnvelope}, coord: $coord, maxRadius: $maxRadius, street mode $streetMode"
+      )
     }
     theSplit
   }

From 292fede8a98b206533a912e3cc7a4062fb5c7d80 Mon Sep 17 00:00:00 2001
From: Haitam Laarabi <10712736+htmlrb@users.noreply.github.com>
Date: Fri, 8 Apr 2022 19:05:01 -0700
Subject: [PATCH 50/58] updating deploy file

---
 gradle.deploy.properties | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/gradle.deploy.properties b/gradle.deploy.properties
index 1940d898f85..0f687ec06c7 100644
--- a/gradle.deploy.properties
+++ b/gradle.deploy.properties
@@ -7,10 +7,9 @@ instanceType=r5.8xlarge
 # shutdownBehaviour = stop | terminate
 shutdownBehaviour=terminate
 s3Backup=true
-maxRAM=740g
+maxRAM=230g
 #storageSize (in GiB) =  any number between 64 and 256
-storageSize=256
-profiler_type=cpumem
+storageSize=128
 
 #r5.8xlarge (32/256)
 #c5.9xlarge (36/72)    -> 5 instances -> $1.53 per Hour

From 9ca4a78a5aa95420e8f55c29c35dfcf504afc012 Mon Sep 17 00:00:00 2001
From: Rutvik Patel <heyrutvik@gmail.com>
Date: Tue, 12 Apr 2022 12:00:02 +0530
Subject: [PATCH 51/58] validate scenario after sampling

---
 src/main/resources/beam-template.conf         |   1 +
 .../agents/freight/input/FreightReader.scala  |   1 +
 .../freight/input/GenericFreightReader.scala  |  27 ++-
 src/main/scala/beam/sim/BeamHelper.scala      |  32 ++--
 .../scala/beam/sim/config/BeamConfig.scala    |   9 +-
 .../sim/population/PopulationScaling.scala    |   9 -
 .../utils/scenario/BeamScenarioLoader.scala   |  97 +++++-----
 .../utils/scenario/ScenarioLoaderHelper.scala | 170 ++----------------
 .../scenario/UrbanSimScenarioLoader.scala     |  32 ++--
 .../input/GenericFreightReaderSpec.scala      |  19 +-
 .../scala/beam/utils/ModeExclusionTest.scala  |   2 +-
 .../scala/beam/utils/SnapCoordinateSpec.scala |  61 ++++++-
 12 files changed, 198 insertions(+), 262 deletions(-)

diff --git a/src/main/resources/beam-template.conf b/src/main/resources/beam-template.conf
index c7acb17f322..1b5b890d40d 100755
--- a/src/main/resources/beam-template.conf
+++ b/src/main/resources/beam-template.conf
@@ -31,6 +31,7 @@ beam.agentsim.lastIteration = "int | 0"
 beam.agentsim.endTime = "30:00:00"
 beam.agentsim.scheduleMonitorTask.initialDelay = 1
 beam.agentsim.scheduleMonitorTask.interval = 30
+beam.agentsim.snapLocationAndRemoveInvalidInputs = "boolean | false"
 
 beam.agentsim.agents.bodyType = "BODY-TYPE-DEFAULT"
 
diff --git a/src/main/scala/beam/agentsim/agents/freight/input/FreightReader.scala b/src/main/scala/beam/agentsim/agents/freight/input/FreightReader.scala
index a2b5cae7190..3ec0b9a1d53 100644
--- a/src/main/scala/beam/agentsim/agents/freight/input/FreightReader.scala
+++ b/src/main/scala/beam/agentsim/agents/freight/input/FreightReader.scala
@@ -203,6 +203,7 @@ object FreightReader {
           geoUtils,
           rand,
           tazMap,
+          beamConfig.beam.agentsim.snapLocationAndRemoveInvalidInputs,
           Some(snapLocationHelper),
           outputDirMaybe
         )
diff --git a/src/main/scala/beam/agentsim/agents/freight/input/GenericFreightReader.scala b/src/main/scala/beam/agentsim/agents/freight/input/GenericFreightReader.scala
index 28d6ca5b9a6..4577644f125 100644
--- a/src/main/scala/beam/agentsim/agents/freight/input/GenericFreightReader.scala
+++ b/src/main/scala/beam/agentsim/agents/freight/input/GenericFreightReader.scala
@@ -27,6 +27,7 @@ class GenericFreightReader(
   val geoUtils: GeoUtils,
   rnd: Random,
   tazTree: TAZTreeMap,
+  val snapLocationAndRemoveInvalidInputs: Boolean,
   val snapLocationHelperMaybe: Option[SnapLocationHelper] = None,
   val outputDirMaybe: Option[String] = None
 ) extends LazyLogging
@@ -68,7 +69,8 @@ class GenericFreightReader(
         extractCoordOrTaz(
           departureLocationX,
           departureLocationY,
-          row.get("departureLocationZone")
+          row.get("departureLocationZone"),
+          snapLocationAndRemoveInvalidInputs
         ) match {
           case (departureLocationZoneMaybe, Left(departureLocationZoneUTM)) =>
             Some(
@@ -167,7 +169,7 @@ class GenericFreightReader(
           operationDurationInSec
         )
 
-        extractCoordOrTaz(locationX, locationY, row.get("locationZone")) match {
+        extractCoordOrTaz(locationX, locationY, row.get("locationZone"), snapLocationAndRemoveInvalidInputs) match {
           case (locationZoneMaybe, Left(locationZoneUTM)) =>
             Some(payloadPlan.copy(locationZone = locationZoneMaybe, locationUTM = locationZoneUTM))
           case (locationZoneMaybe, Right(Result.Succeed(splitCoord))) =>
@@ -301,7 +303,12 @@ class GenericFreightReader(
         // note: placeholder to update warehouseLocationZone and warehouseLocationUTM later
         val freightCarrier = FreightCarrierRow(carrierId, tourId, vehicleId, vehicleTypeId, None, new Coord())
 
-        extractCoordOrTaz(row.get("warehouseX"), row.get("warehouseY"), row.get("warehouseZone")) match {
+        extractCoordOrTaz(
+          row.get("warehouseX"),
+          row.get("warehouseY"),
+          row.get("warehouseZone"),
+          snapLocationAndRemoveInvalidInputs
+        ) match {
           case (warehouseZoneMaybe, Left(warehouseZoneUTM)) =>
             Some(
               freightCarrier.copy(warehouseLocationZone = warehouseZoneMaybe, warehouseLocationUTM = warehouseZoneUTM)
@@ -364,14 +371,24 @@ class GenericFreightReader(
   private def getDistributedTazLocation(taz: TAZ): Coord =
     convertedLocation(TAZTreeMap.randomLocationInTAZ(taz, rnd, snapLocationHelperMaybe))
 
-  private def extractCoordOrTaz(strX: String, strY: String, strZone: String): (Option[Id[TAZ]], ClosestUTMPoint) = {
+  private def extractCoordOrTaz(
+    strX: String,
+    strY: String,
+    strZone: String,
+    snapLocationAndRemoveInvalidInputs: Boolean
+  ): (Option[Id[TAZ]], ClosestUTMPoint) = {
     if (isBlank(strX) || isBlank(strY)) {
       val taz = getTaz(strZone)
       (Some(taz.tazId), Left(getDistributedTazLocation(taz)))
     } else {
       val wasInWgs = config.convertWgs2Utm
       val loc = location(strX.toDouble, strY.toDouble)
-      (None, Right(snapLocationHelperMaybe.map(_.computeResult(loc, wasInWgs)).getOrElse(Result.Succeed(loc))))
+      val finalLoc =
+        if (snapLocationAndRemoveInvalidInputs)
+          snapLocationHelperMaybe.map(_.computeResult(loc, wasInWgs)).getOrElse(Result.Succeed(loc))
+        else Result.Succeed(loc)
+
+      (None, Right(finalLoc))
     }
   }
 
diff --git a/src/main/scala/beam/sim/BeamHelper.scala b/src/main/scala/beam/sim/BeamHelper.scala
index cf301301546..9c5eb0e7010 100755
--- a/src/main/scala/beam/sim/BeamHelper.scala
+++ b/src/main/scala/beam/sim/BeamHelper.scala
@@ -699,6 +699,18 @@ trait BeamHelper extends LazyLogging {
       PopulationScaling.samplePopulation(scenario, beamScenario, beamServices.beamConfig, beamServices, outputDir)
     }
 
+    if (beamScenario.beamConfig.beam.agentsim.snapLocationAndRemoveInvalidInputs) {
+      val snapLocationHelper = SnapLocationHelper(
+        new GeoUtilsImpl(beamScenario.beamConfig),
+        beamScenario.transportNetwork.streetLayer,
+        beamScenario.beamConfig.beam.routing.r5.linkRadiusMeters
+      )
+      ScenarioLoaderHelper.validateScenario(scenario, snapLocationHelper, Some(outputDir))
+      logger.info(s"""After snapping locations and validating scenario:
+                     |Number of households: ${scenario.getHouseholds.getHouseholds.size()}
+                     |Number of persons: ${scenario.getPopulation.getPersons.size()}""".stripMargin)
+    }
+
     // write static metrics, such as population size, vehicles fleet size, etc.
     // necessary to be called after population sampling
     BeamStaticMetricsWriter.writeSimulationParameters(
@@ -855,8 +867,7 @@ trait BeamHelper extends LazyLogging {
                 beamScenario,
                 source,
                 new GeoUtilsImpl(beamConfig),
-                Some(merger),
-                outputDirOpt
+                Some(merger)
               ).loadScenario()
             if (src == "urbansim_v2") {
               new ScenarioAdjuster(
@@ -882,24 +893,13 @@ trait BeamHelper extends LazyLogging {
                   scenarioBuilder,
                   beamScenario,
                   source,
-                  new GeoUtilsImpl(beamConfig),
-                  outputDirOpt
-                )
-                  .loadScenario()
+                  new GeoUtilsImpl(beamConfig)
+                ).loadScenario()
               }.asInstanceOf[MutableScenario]
               (scenario, beamScenario, false)
             case "xml" =>
               val beamScenario = loadScenario(beamConfig, outputDirOpt)
-              val snapLocationHelper = SnapLocationHelper(
-                new GeoUtilsImpl(beamConfig),
-                beamScenario.transportNetwork.streetLayer,
-                beamScenario.beamConfig.beam.routing.r5.linkRadiusMeters
-              )
-              val scenario = {
-                val result = ScenarioUtils.loadScenario(matsimConfig).asInstanceOf[MutableScenario]
-                ScenarioLoaderHelper.validateScenario(result, snapLocationHelper, outputDirOpt)
-                result
-              }
+              val scenario = ScenarioUtils.loadScenario(matsimConfig).asInstanceOf[MutableScenario]
               (scenario, beamScenario, false)
             case unknown =>
               throw new IllegalArgumentException(s"Beam does not support [$unknown] file type")
diff --git a/src/main/scala/beam/sim/config/BeamConfig.scala b/src/main/scala/beam/sim/config/BeamConfig.scala
index 57f17ef5f07..fe8feabd79f 100644
--- a/src/main/scala/beam/sim/config/BeamConfig.scala
+++ b/src/main/scala/beam/sim/config/BeamConfig.scala
@@ -55,7 +55,8 @@ object BeamConfig {
       thresholdForWalkingInMeters: scala.Int,
       timeBinSize: scala.Int,
       toll: BeamConfig.Beam.Agentsim.Toll,
-      tuning: BeamConfig.Beam.Agentsim.Tuning
+      tuning: BeamConfig.Beam.Agentsim.Tuning,
+      snapLocationAndRemoveInvalidInputs: scala.Boolean
     )
 
     object Agentsim {
@@ -2211,7 +2212,11 @@ object BeamConfig {
           tuning = BeamConfig.Beam.Agentsim.Tuning(
             if (c.hasPathOrNull("tuning")) c.getConfig("tuning")
             else com.typesafe.config.ConfigFactory.parseString("tuning{}")
-          )
+          ),
+          snapLocationAndRemoveInvalidInputs =
+            if (c.hasPathOrNull("snapLocationAndRemoveInvalidInputs"))
+              c.getBoolean("snapLocationAndRemoveInvalidInputs")
+            else false
         )
       }
     }
diff --git a/src/main/scala/beam/sim/population/PopulationScaling.scala b/src/main/scala/beam/sim/population/PopulationScaling.scala
index b8fa57900ec..47c72ac22bb 100644
--- a/src/main/scala/beam/sim/population/PopulationScaling.scala
+++ b/src/main/scala/beam/sim/population/PopulationScaling.scala
@@ -378,14 +378,5 @@ object PopulationScaling {
 
     val populationAdjustment = PopulationAdjustment.getPopulationAdjustment(beamServices)
     populationAdjustment.update(scenario)
-
-    // write static metrics, such as population size, vehicles fleet size, etc.
-    // necessary to be called after population sampling
-    BeamStaticMetricsWriter.writeSimulationParameters(
-      scenario,
-      beamScenario,
-      beamServices,
-      beamConfig
-    )
   }
 }
diff --git a/src/main/scala/beam/utils/scenario/BeamScenarioLoader.scala b/src/main/scala/beam/utils/scenario/BeamScenarioLoader.scala
index 76100ebf179..d7c2161e653 100644
--- a/src/main/scala/beam/utils/scenario/BeamScenarioLoader.scala
+++ b/src/main/scala/beam/utils/scenario/BeamScenarioLoader.scala
@@ -5,9 +5,9 @@ import beam.agentsim.agents.vehicles.{BeamVehicle, BeamVehicleType, VehicleManag
 import beam.router.Modes.BeamMode
 import beam.sim.BeamScenario
 import beam.sim.common.GeoUtils
+import beam.utils.logging.ExponentialLazyLogging
 import beam.utils.plan.sampling.AvailableModeUtils
 import com.google.common.annotations.VisibleForTesting
-import com.typesafe.scalalogging.LazyLogging
 import org.matsim.api.core.v01.network.Link
 import org.matsim.api.core.v01.population._
 import org.matsim.api.core.v01.{Coord, Id, Scenario}
@@ -19,30 +19,30 @@ import org.matsim.vehicles.{Vehicle, VehicleType, VehicleUtils}
 
 import java.util
 import java.util.concurrent.atomic.AtomicReference
-import scala.collection.Iterable
 import scala.collection.JavaConverters._
-import scala.concurrent.duration._
-import scala.concurrent.{Await, ExecutionContext, Future}
 import scala.util.Random
 
 class BeamScenarioLoader(
   val scenarioBuilder: ScenarioBuilder,
   var beamScenario: BeamScenario,
   val scenarioSource: ScenarioSource,
-  val geo: GeoUtils,
-  val outputDirMaybe: Option[String]
-) extends ScenarioLoaderHelper {
+  val geo: GeoUtils
+) extends ExponentialLazyLogging {
 
   import BeamScenarioLoader._
 
   type IdToAttributes = Map[String, Seq[(String, Double)]]
 
-  private implicit val ex: ExecutionContext = scala.concurrent.ExecutionContext.Implicits.global
-
   private val availableModes: Seq[String] = BeamMode.allModes.map(_.value)
 
   private val rand: Random = new Random(beamScenario.beamConfig.matsim.modules.global.randomSeed)
 
+  private lazy val plans: Iterable[PlanElement] = {
+    val r = scenarioSource.getPlans
+    logger.info(s"Read ${r.size} plans")
+    r
+  }
+
   private val scenario: MutableScenario = scenarioBuilder.build
 
   private def replaceHouseholdsAttributes(
@@ -61,35 +61,22 @@ class BeamScenarioLoader(
   def loadScenario(): Scenario = {
     logger.info("The scenario loading started...")
 
-    val plansF = Future {
-      val plans = scenarioSource.getPlans
-      logger.info(s"Read ${plans.size} plans")
-      validatePlans(plans)
-    }
-
-    val personsF = Future {
+    val personsWithPlans = {
       val persons: Iterable[PersonInfo] = scenarioSource.getPersons
-      logger.info(s"Read ${persons.size} persons")
-      persons
+      val personIdsWithPlanTmp = plans.map(_.personId).toSet
+      val result = persons.filter(person => personIdsWithPlanTmp.contains(person.personId))
+      logger.info(s"There are ${persons.size} people. ${result.size} have plans")
+      result
     }
 
-    val householdsF = Future {
-      val households = scenarioSource.getHousehold
-      logger.info(s"Read ${households.size} households")
-      validateHouseholds(households)
-    }
+    val vehicles = scenarioSource.getVehicles
 
-    val validPlans = Await.result(plansF, 1800.seconds)
-    logger.info(s"Reading plans done.")
-    val persons = Await.result(personsF, 1800.seconds)
-    logger.info(s"Reading persons done.")
-    val validHouseholds = Await.result(householdsF, 1800.seconds)
-    logger.info(s"Reading households done.")
+    val loadedHouseholds = scenarioSource.getHousehold
 
-    val vehicles = scenarioSource.getVehicles
+    val newHouseholds: Iterable[Household] =
+      buildMatsimHouseholds(loadedHouseholds, personsWithPlans, vehicles)
 
-    val personsWithPlans = getPersonsWithPlan(persons, validPlans, validHouseholds)
-    logger.info(s"There are ${personsWithPlans.size} persons with plans")
+    val households: Households = replaceHouseholds(scenario.getHouseholds, newHouseholds)
 
     beamScenario.privateVehicles.clear()
     beamScenario.privateVehicleInitialSoc.clear()
@@ -101,15 +88,14 @@ class BeamScenarioLoader(
       vehicleInfo.initialSoc.foreach(beamScenario.privateVehicleInitialSoc.put(vehicle.id, _))
     }
 
-    val newHouseholds: Iterable[Household] = buildMatsimHouseholds(validHouseholds, personsWithPlans, vehicles)
-    val matsimHouseholds: Households = replaceHouseholds(scenario.getHouseholds, newHouseholds)
-    val loadedAttributes = buildAttributesCoordinates(validHouseholds)
-    replaceHouseholdsAttributes(matsimHouseholds, loadedAttributes)
-
     val scenarioPopulation: Population = buildPopulation(personsWithPlans)
     scenario.setPopulation(scenarioPopulation)
     updateAvailableModesForPopulation(scenario)
-    replacePlansFromPopulation(scenarioPopulation, validPlans)
+
+    replacePlansFromPopulation(scenarioPopulation, plans)
+
+    val loadedAttributes = buildAttributesCoordinates(loadedHouseholds)
+    replaceHouseholdsAttributes(households, loadedAttributes)
 
     logger.info("The scenario loading is completed.")
     scenario
@@ -211,24 +197,23 @@ class BeamScenarioLoader(
       listOfElementsGroupedByPerson.groupBy(_.planIndex).foreach {
         case (_, listOfElementsGroupedByPlan) if listOfElementsGroupedByPlan.nonEmpty =>
           val person = population.getPersons.get(Id.createPersonId(personId.id))
-          if (person != null) {
-            val currentPlan = PopulationUtils.createPlan(person)
-            currentPlan.setScore(listOfElementsGroupedByPlan.head.planScore)
-            person.addPlan(currentPlan)
-
-            val personWithoutSelectedPlan = person.getSelectedPlan == null
-            val isCurrentPlanIndexSelected = listOfElementsGroupedByPlan.head.planSelected
-            val isLastPlanIteration = person.getPlans.size() == listOfElementsGroupedByPerson.size
-            if (personWithoutSelectedPlan && (isCurrentPlanIndexSelected || isLastPlanIteration)) {
-              person.setSelectedPlan(currentPlan)
-            }
 
-            listOfElementsGroupedByPlan.foreach { planElement =>
-              if (planElement.planElementType == PlanElement.Leg) {
-                buildAndAddLegToPlan(currentPlan, planElement)
-              } else if (planElement.planElementType == PlanElement.Activity) {
-                buildAndAddActivityToPlan(currentPlan, planElement)
-              }
+          val currentPlan = PopulationUtils.createPlan(person)
+          currentPlan.setScore(listOfElementsGroupedByPlan.head.planScore)
+          person.addPlan(currentPlan)
+
+          val personWithoutSelectedPlan = person.getSelectedPlan == null
+          val isCurrentPlanIndexSelected = listOfElementsGroupedByPlan.head.planSelected
+          val isLastPlanIteration = person.getPlans.size() == listOfElementsGroupedByPerson.size
+          if (personWithoutSelectedPlan && (isCurrentPlanIndexSelected || isLastPlanIteration)) {
+            person.setSelectedPlan(currentPlan)
+          }
+
+          listOfElementsGroupedByPlan.foreach { planElement =>
+            if (planElement.planElementType == PlanElement.Leg) {
+              buildAndAddLegToPlan(currentPlan, planElement)
+            } else if (planElement.planElementType == PlanElement.Activity) {
+              buildAndAddActivityToPlan(currentPlan, planElement)
             }
           }
       }
@@ -291,7 +276,7 @@ class BeamScenarioLoader(
   }
 }
 
-object BeamScenarioLoader extends LazyLogging {
+object BeamScenarioLoader extends ExponentialLazyLogging {
 
   private[utils] def buildMatsimHouseholds(
     households: Iterable[HouseholdInfo],
diff --git a/src/main/scala/beam/utils/scenario/ScenarioLoaderHelper.scala b/src/main/scala/beam/utils/scenario/ScenarioLoaderHelper.scala
index 538a3a42efe..a58902f4ab6 100644
--- a/src/main/scala/beam/utils/scenario/ScenarioLoaderHelper.scala
+++ b/src/main/scala/beam/utils/scenario/ScenarioLoaderHelper.scala
@@ -1,155 +1,15 @@
 package beam.utils.scenario
 
-import beam.sim.BeamScenario
-import beam.sim.common.GeoUtils
 import beam.utils.SnapCoordinateUtils
-import beam.utils.SnapCoordinateUtils.{Category, CsvFile, Error, ErrorInfo, Processed, Result, SnapLocationHelper}
+import beam.utils.SnapCoordinateUtils.{Category, CsvFile, Error, ErrorInfo, Result, SnapLocationHelper}
 import com.typesafe.scalalogging.LazyLogging
-import org.matsim.api.core.v01.{Coord, Id}
 import org.matsim.api.core.v01.population.{Activity, Leg, Person}
+import org.matsim.api.core.v01.{Coord, Id}
 import org.matsim.core.scenario.MutableScenario
 import org.matsim.households.{Household, HouseholdsFactoryImpl}
 
-import scala.collection.Iterable
 import scala.collection.mutable.ListBuffer
-import scala.jdk.CollectionConverters.{
-  collectionAsScalaIterableConverter,
-  seqAsJavaListConverter,
-  setAsJavaSetConverter
-}
-
-trait ScenarioLoaderHelper extends LazyLogging {
-
-  def beamScenario: BeamScenario
-  def geo: GeoUtils
-  def outputDirMaybe: Option[String]
-
-  private val snapLocationHelper: SnapLocationHelper = SnapLocationHelper(
-    geo,
-    beamScenario.transportNetwork.streetLayer,
-    beamScenario.beamConfig.beam.routing.r5.linkRadiusMeters
-  )
-
-  private def validatePersonPlans(personId: PersonId, plans: Iterable[PlanElement]): Processed[PlanElement] = {
-    plans.foldLeft[Processed[PlanElement]](Processed()) {
-      case (processed, planElement) if planElement.planElementType == PlanElement.Leg =>
-        processed.copy(data = processed.data :+ planElement)
-      case (processed, planElement) if planElement.planElementType == PlanElement.Activity =>
-        val utmCoord = new Coord(planElement.activityLocationX.get, planElement.activityLocationY.get)
-        snapLocationHelper.computeResult(utmCoord) match {
-          case Result.Succeed(splitCoord) =>
-            val updatedPlan = planElement
-              .copy(activityLocationX = Some(splitCoord.getX), activityLocationY = Some(splitCoord.getY))
-            processed.copy(data = processed.data :+ updatedPlan)
-          case Result.OutOfBoundingBoxError =>
-            processed.copy(errors =
-              processed.errors :+ ErrorInfo(
-                personId.id,
-                Category.ScenarioPerson,
-                Error.OutOfBoundingBox,
-                utmCoord.getX,
-                utmCoord.getY
-              )
-            )
-          case Result.R5SplitNullError =>
-            processed.copy(errors =
-              processed.errors :+ ErrorInfo(
-                personId.id,
-                Category.ScenarioPerson,
-                Error.R5SplitNull,
-                utmCoord.getX,
-                utmCoord.getY
-              )
-            )
-        }
-    }
-  }
-
-  private def snapLocationPlans(plans: Iterable[PlanElement]): Processed[PlanElement] = {
-    plans.groupBy(_.personId).foldLeft[Processed[PlanElement]](Processed()) {
-      case (processed, (personId, personPlans)) =>
-        val Processed(updatedPlans, errors) = validatePersonPlans(personId, personPlans)
-        if (updatedPlans.size == personPlans.size) {
-          processed.copy(data = processed.data ++ updatedPlans, errors = processed.errors ++ errors)
-        } else {
-          processed.copy(errors = processed.errors ++ errors)
-        }
-    }
-  }
-
-  private def snapLocationHouseholds(households: Iterable[HouseholdInfo]): Processed[HouseholdInfo] = {
-    households.foldLeft[Processed[HouseholdInfo]](Processed()) { case (processed, household) =>
-      val householdId = household.householdId.id
-      val utmCoord = new Coord(household.locationX, household.locationY)
-      snapLocationHelper.computeResult(utmCoord) match {
-        case Result.Succeed(splitCoord) =>
-          val updatedHousehold = household.copy(locationX = splitCoord.getX, locationY = splitCoord.getY)
-          processed.copy(data = processed.data :+ updatedHousehold)
-        case Result.OutOfBoundingBoxError =>
-          processed.copy(errors =
-            processed.errors :+ ErrorInfo(
-              householdId,
-              Category.ScenarioHousehold,
-              Error.OutOfBoundingBox,
-              utmCoord.getX,
-              utmCoord.getY
-            )
-          )
-        case Result.R5SplitNullError =>
-          processed.copy(errors =
-            processed.errors :+ ErrorInfo(
-              householdId,
-              Category.ScenarioHousehold,
-              Error.R5SplitNull,
-              utmCoord.getX,
-              utmCoord.getY
-            )
-          )
-      }
-    }
-  }
-
-  protected def getPersonsWithPlan(
-    persons: Iterable[PersonInfo],
-    plans: Iterable[PlanElement],
-    households: Iterable[HouseholdInfo]
-  ): Iterable[PersonInfo] = {
-    val personIdsWithPlan = plans.map(_.personId).toSet
-    val householdIds = households.map(_.householdId).toSet
-    persons.filter(person => personIdsWithPlan.contains(person.personId) && householdIds.contains(person.householdId))
-  }
-
-  protected def validatePlans(plans: Iterable[PlanElement]): Iterable[PlanElement] = {
-    val Processed(validPlans, errors) = snapLocationPlans(plans)
-
-    outputDirMaybe.foreach { path =>
-      if (errors.isEmpty) logger.info("No 'snap location' error to report for scenario plans.")
-      else SnapCoordinateUtils.writeToCsv(s"$path/${CsvFile.Plans}", errors)
-    }
-
-    val filteredCnt = plans.size - validPlans.size
-    if (filteredCnt > 0) {
-      logger.info(s"Filtered out $filteredCnt plans. Total number of plans: ${validPlans.size}")
-    }
-    validPlans
-  }
-
-  protected def validateHouseholds(households: Iterable[HouseholdInfo]): Iterable[HouseholdInfo] = {
-    val Processed(validHouseholds, errors) = snapLocationHouseholds(households)
-
-    outputDirMaybe.foreach { path =>
-      if (errors.isEmpty) logger.info("No 'snap location' error to report for scenario households.")
-      else SnapCoordinateUtils.writeToCsv(s"$path/${CsvFile.Households}", errors)
-    }
-
-    val filteredCnt = households.size - validHouseholds.size
-    if (filteredCnt > 0) {
-      logger.info(s"Filtered out $filteredCnt plans. Total number of plans: ${validHouseholds.size}")
-    }
-    validHouseholds
-  }
-
-}
+import scala.jdk.CollectionConverters.{collectionAsScalaIterableConverter, seqAsJavaListConverter}
 
 object ScenarioLoaderHelper extends LazyLogging {
 
@@ -222,10 +82,11 @@ object ScenarioLoaderHelper extends LazyLogging {
     snapLocationHelper: SnapLocationHelper,
     outputDirMaybe: Option[String] = None
   ): Unit = {
+
     val planErrors: ListBuffer[ErrorInfo] = ListBuffer()
-    val householdErrors: ListBuffer[ErrorInfo] = ListBuffer()
+
     val people: List[Person] = scenario.getPopulation.getPersons.values().asScala.toList
-    people.foreach { person =>
+    people.par.foreach { person =>
       val plans = person.getPlans.asScala.toList
       plans.foreach { plan =>
         val elements: Vector[PlanElement] = plan.getPlanElements.asScala.toVector
@@ -241,24 +102,31 @@ object ScenarioLoaderHelper extends LazyLogging {
       }
     }
 
+    outputDirMaybe.foreach { path =>
+      if (planErrors.isEmpty) logger.info("No 'snap location' error to report for scenario plans.")
+      else SnapCoordinateUtils.writeToCsv(s"$path/${CsvFile.Plans}", planErrors)
+    }
+
     val validPeople: Set[Id[Person]] = scenario.getPopulation.getPersons.values().asScala.map(_.getId).toSet
 
     val households: List[Household] = scenario.getHouseholds.getHouseholds.values().asScala.toList
-    households.foreach { household =>
+    households.par.foreach { household =>
       val members = household.getMemberIds.asScala.toSet
       val validMembers = validPeople.intersect(members)
 
       if (validMembers.isEmpty) {
-        scenario.getHouseholds.getHouseholds.remove(household.getId)
         scenario.getHouseholds.getHouseholdAttributes.removeAllAttributes(household.getId.toString)
-      } else {
+        scenario.getHouseholds.getHouseholds.remove(household.getId)
+      } else if (validMembers != members) {
         val updatedHousehold = createHouseholdWithGivenMembers(household, validMembers.toList)
         scenario.getHouseholds.getHouseholds.replace(household.getId, updatedHousehold)
       }
     }
 
+    val householdErrors: ListBuffer[ErrorInfo] = ListBuffer()
+
     val householdsWithMembers: List[Household] = scenario.getHouseholds.getHouseholds.values().asScala.toList
-    householdsWithMembers.foreach { household =>
+    householdsWithMembers.par.foreach { household =>
       val householdId = household.getId.toString
       val attr = scenario.getHouseholds.getHouseholdAttributes
       val locationX = attr.getAttribute(householdId, "homecoordx").asInstanceOf[Double]
@@ -297,12 +165,10 @@ object ScenarioLoaderHelper extends LazyLogging {
     }
 
     outputDirMaybe.foreach { path =>
-      if (planErrors.isEmpty) logger.info("No 'snap location' error to report for scenario plans.")
-      else SnapCoordinateUtils.writeToCsv(s"$path/${CsvFile.Plans}", planErrors)
-
       if (householdErrors.isEmpty) logger.info("No 'snap location' error to report for scenario households.")
       else SnapCoordinateUtils.writeToCsv(s"$path/${CsvFile.Households}", householdErrors)
     }
+
   }
 
 }
diff --git a/src/main/scala/beam/utils/scenario/UrbanSimScenarioLoader.scala b/src/main/scala/beam/utils/scenario/UrbanSimScenarioLoader.scala
index 24623699c8d..4f3173a3aaf 100644
--- a/src/main/scala/beam/utils/scenario/UrbanSimScenarioLoader.scala
+++ b/src/main/scala/beam/utils/scenario/UrbanSimScenarioLoader.scala
@@ -9,6 +9,7 @@ import beam.sim.vehicles.VehiclesAdjustment
 import beam.utils.SequenceUtils
 import beam.utils.plan.sampling.AvailableModeUtils
 import beam.utils.scenario.urbansim.HOVModeTransformer
+import com.typesafe.scalalogging.LazyLogging
 import org.apache.commons.math3.distribution.UniformRealDistribution
 import org.matsim.api.core.v01.population.Population
 import org.matsim.api.core.v01.{Coord, Id, Scenario}
@@ -30,9 +31,8 @@ class UrbanSimScenarioLoader(
   val beamScenario: BeamScenario,
   val scenarioSource: ScenarioSource,
   val geo: GeoUtils,
-  val previousRunPlanMerger: Option[PreviousRunPlanMerger] = None,
-  val outputDirMaybe: Option[String] = None
-) extends ScenarioLoaderHelper {
+  val previousRunPlanMerger: Option[PreviousRunPlanMerger] = None
+) extends LazyLogging {
 
   private implicit val ex: ExecutionContext = scala.concurrent.ExecutionContext.Implicits.global
 
@@ -48,7 +48,7 @@ class UrbanSimScenarioLoader(
     val plansF = Future {
       val plans = scenarioSource.getPlans
       logger.info(s"Read ${plans.size} plans")
-      validatePlans(plans)
+      plans
     }
 
     val personsF = Future {
@@ -60,30 +60,32 @@ class UrbanSimScenarioLoader(
     val householdsF = Future {
       val households = scenarioSource.getHousehold
       logger.info(s"Read ${households.size} households")
-      validateHouseholds(households)
+      households
     }
 
-    val validPlans = Await.result(plansF, 1800.seconds)
+    val inputPlans = Await.result(plansF, 1800.seconds)
     logger.info(s"Reading plans done.")
     val persons = Await.result(personsF, 1800.seconds)
     logger.info(s"Reading persons done.")
-    val validHouseholds = Await.result(householdsF, 1800.seconds)
+    val households = Await.result(householdsF, 1800.seconds)
     logger.info(s"Reading households done.")
 
-    val (mergedPlans, plansMerged) = previousRunPlanMerger.map(_.merge(validPlans)).getOrElse(validPlans -> false)
+    val (mergedPlans, plansMerged) = previousRunPlanMerger.map(_.merge(inputPlans)).getOrElse(inputPlans -> false)
 
     val plans = {
       HOVModeTransformer.reseedRandomGenerator(beamScenario.beamConfig.matsim.modules.global.randomSeed)
       HOVModeTransformer.transformHOVtoHOVCARorHOVTeleportation(mergedPlans)
     }
 
-    val personsWithPlans = getPersonsWithPlan(persons, plans, validHouseholds)
+    val householdIds = households.map(_.householdId.id).toSet
+
+    val personsWithPlans = getPersonsWithPlan(persons, plans)
+      .filter(p => householdIds.contains(p.householdId.id))
     logger.info(s"There are ${personsWithPlans.size} persons with plans")
 
     val householdIdToPersons: Map[HouseholdId, Iterable[PersonInfo]] = personsWithPlans.groupBy(_.householdId)
 
-    val householdsWithMembers =
-      validHouseholds.filter(household => householdIdToPersons.contains(household.householdId))
+    val householdsWithMembers = households.filter(household => householdIdToPersons.contains(household.householdId))
     logger.info(s"There are ${householdsWithMembers.size} non-empty households")
 
     logger.info("Applying households...")
@@ -112,6 +114,14 @@ class UrbanSimScenarioLoader(
     beamScenario.privateVehicleInitialSoc.clear()
   }
 
+  private[utils] def getPersonsWithPlan(
+    persons: Iterable[PersonInfo],
+    plans: Iterable[PlanElement]
+  ): Iterable[PersonInfo] = {
+    val personIdsWithPlan = plans.map(_.personId).toSet
+    persons.filter(person => personIdsWithPlan.contains(person.personId))
+  }
+
   private[utils] def applyHousehold(
     households: Iterable[HouseholdInfo],
     householdIdToPersons: Map[HouseholdId, Iterable[PersonInfo]],
diff --git a/src/test/scala/beam/agentsim/agents/freight/input/GenericFreightReaderSpec.scala b/src/test/scala/beam/agentsim/agents/freight/input/GenericFreightReaderSpec.scala
index 1d9649855e4..a6bfb9bb665 100644
--- a/src/test/scala/beam/agentsim/agents/freight/input/GenericFreightReaderSpec.scala
+++ b/src/test/scala/beam/agentsim/agents/freight/input/GenericFreightReaderSpec.scala
@@ -45,7 +45,8 @@ class GenericFreightReaderSpec extends AnyWordSpecLike with Matchers {
 
   val rnd = new Random(2333L)
 
-  private val reader = new GenericFreightReader(freightConfig, geoUtils, rnd, tazMap)
+  private val reader =
+    new GenericFreightReader(freightConfig, geoUtils, rnd, tazMap, snapLocationAndRemoveInvalidInputs = false)
 
   "PayloadPlansConverter" should {
     "read Payload Plans" in {
@@ -154,12 +155,24 @@ class GenericFreightReaderSpec extends AnyWordSpecLike with Matchers {
   }
 
   private def readCarriers: IndexedSeq[FreightCarrier] = {
-    val converter = new GenericFreightReader(freightConfig, geoUtils, new Random(4324L), tazMap)
+    val converter = new GenericFreightReader(
+      freightConfig,
+      geoUtils,
+      new Random(4324L),
+      tazMap,
+      snapLocationAndRemoveInvalidInputs = false
+    )
     val payloadPlans: Map[Id[PayloadPlan], PayloadPlan] = converter.readPayloadPlans()
     val tours = converter.readFreightTours()
     val vehicleTypes = BeamVehicleUtils.readBeamVehicleTypeFile("test/input/beamville/vehicleTypes.csv")
     val freightCarriers: IndexedSeq[FreightCarrier] =
-      new GenericFreightReader(freightConfig, geoUtils, new Random(73737L), tazMap).readFreightCarriers(
+      new GenericFreightReader(
+        freightConfig,
+        geoUtils,
+        new Random(73737L),
+        tazMap,
+        snapLocationAndRemoveInvalidInputs = false
+      ).readFreightCarriers(
         tours,
         payloadPlans,
         vehicleTypes
diff --git a/src/test/scala/beam/utils/ModeExclusionTest.scala b/src/test/scala/beam/utils/ModeExclusionTest.scala
index f766913391b..17c77c19ca4 100644
--- a/src/test/scala/beam/utils/ModeExclusionTest.scala
+++ b/src/test/scala/beam/utils/ModeExclusionTest.scala
@@ -32,7 +32,7 @@ class ModeExclusionTest extends AnyWordSpecLike with Matchers with BeforeAndAfte
     val plans = BeamCsvScenarioReader.readPlansFile("test/input/beamville/csvInput/plans.csv")
     val personIdsWithPlanTmp = plans.map(_.personId).toSet
     val personWithPlans = persons.filter(person => personIdsWithPlanTmp.contains(person.personId))
-    val scenarioLoader = new BeamScenarioLoader(scenarioBuilder, beamScenario, scenarioSource, geoUtils, None)
+    val scenarioLoader = new BeamScenarioLoader(scenarioBuilder, beamScenario, scenarioSource, geoUtils)
     val population = scenarioLoader.buildPopulation(personWithPlans)
 
     "check for removal of single mode" in {
diff --git a/src/test/scala/beam/utils/SnapCoordinateSpec.scala b/src/test/scala/beam/utils/SnapCoordinateSpec.scala
index 7fd1929df43..7af979c78b6 100644
--- a/src/test/scala/beam/utils/SnapCoordinateSpec.scala
+++ b/src/test/scala/beam/utils/SnapCoordinateSpec.scala
@@ -6,6 +6,7 @@ import beam.sim.common.GeoUtilsImpl
 import beam.sim.config.{BeamConfig, MatSimBeamConfigBuilder}
 import beam.utils.SnapCoordinateUtils.{CsvFile, ErrorInfo, Result, SnapLocationHelper}
 import beam.utils.TestConfigUtils.testConfig
+import beam.utils.scenario.ScenarioLoaderHelper
 import com.typesafe.config.{ConfigFactory, Config => TypesafeConfig}
 import org.matsim.api.core.v01.Coord
 import org.matsim.api.core.v01.population.{Activity, Population}
@@ -79,7 +80,7 @@ class SnapCoordinateSpec extends AnyWordSpec with Matchers with BeamHelper {
       matsimConfig.planCalcScore().setMemorizingExperiencedPlans(true)
       val beamConfig: BeamConfig = BeamConfig(config)
 
-      FileUtils.setConfigOutputFile(beamConfig, matsimConfig)
+      val outputDir = FileUtils.setConfigOutputFile(beamConfig, matsimConfig)
 
       val (scenario, beamScenario, _) = buildBeamServicesAndScenario(
         beamConfig,
@@ -91,6 +92,7 @@ class SnapCoordinateSpec extends AnyWordSpec with Matchers with BeamHelper {
         beamScenario.transportNetwork.streetLayer,
         beamConfig.beam.routing.r5.linkRadiusMeters
       )
+      ScenarioLoaderHelper.validateScenario(scenario, snapLocationHelper, Some(outputDir))
 
       val dummyCoord = new Coord()
       val population: Seq[SnapCoordinateUtils.Result] = scenario.getPopulation.getPersons
@@ -150,11 +152,18 @@ class SnapCoordinateSpec extends AnyWordSpec with Matchers with BeamHelper {
 
       val outputDir = FileUtils.setConfigOutputFile(beamConfig, matsimConfig)
 
-      val (scenario, _, _) = buildBeamServicesAndScenario(
+      val (scenario, beamScenario, _) = buildBeamServicesAndScenario(
         beamConfig,
         matsimConfig
       )
 
+      val snapLocationHelper: SnapLocationHelper = SnapLocationHelper(
+        new GeoUtilsImpl(beamConfig),
+        beamScenario.transportNetwork.streetLayer,
+        beamConfig.beam.routing.r5.linkRadiusMeters
+      )
+      ScenarioLoaderHelper.validateScenario(scenario, snapLocationHelper, Some(outputDir))
+
       val households = scenario.getHouseholds.getHouseholds.values().asScala.toList
 
       scenario.getPopulation.getPersons.size() shouldBe 2
@@ -188,11 +197,18 @@ class SnapCoordinateSpec extends AnyWordSpec with Matchers with BeamHelper {
 
       val outputDir = FileUtils.setConfigOutputFile(beamConfig, matsimConfig)
 
-      val (scenario, _, _) = buildBeamServicesAndScenario(
+      val (scenario, beamScenario, _) = buildBeamServicesAndScenario(
         beamConfig,
         matsimConfig
       )
 
+      val snapLocationHelper: SnapLocationHelper = SnapLocationHelper(
+        new GeoUtilsImpl(beamConfig),
+        beamScenario.transportNetwork.streetLayer,
+        beamConfig.beam.routing.r5.linkRadiusMeters
+      )
+      ScenarioLoaderHelper.validateScenario(scenario, snapLocationHelper, Some(outputDir))
+
       scenario.getPopulation.getPersons.size() shouldBe 1
       intersection(scenario.getPopulation, path = s"$outputDir/${CsvFile.Plans}") shouldBe Set.empty
 
@@ -217,11 +233,18 @@ class SnapCoordinateSpec extends AnyWordSpec with Matchers with BeamHelper {
 
       val outputDir = FileUtils.setConfigOutputFile(beamConfig, matsimConfig)
 
-      val (scenario, _, _) = buildBeamServicesAndScenario(
+      val (scenario, beamScenario, _) = buildBeamServicesAndScenario(
         beamConfig,
         matsimConfig
       )
 
+      val snapLocationHelper: SnapLocationHelper = SnapLocationHelper(
+        new GeoUtilsImpl(beamConfig),
+        beamScenario.transportNetwork.streetLayer,
+        beamConfig.beam.routing.r5.linkRadiusMeters
+      )
+      ScenarioLoaderHelper.validateScenario(scenario, snapLocationHelper, Some(outputDir))
+
       val households = scenario.getHouseholds.getHouseholds.values().asScala.toList
 
       scenario.getPopulation.getPersons.size() shouldBe 2
@@ -249,11 +272,18 @@ class SnapCoordinateSpec extends AnyWordSpec with Matchers with BeamHelper {
 
       val outputDir = FileUtils.setConfigOutputFile(beamConfig, matsimConfig)
 
-      val (scenario, _, _) = buildBeamServicesAndScenario(
+      val (scenario, beamScenario, _) = buildBeamServicesAndScenario(
         beamConfig,
         matsimConfig
       )
 
+      val snapLocationHelper: SnapLocationHelper = SnapLocationHelper(
+        new GeoUtilsImpl(beamConfig),
+        beamScenario.transportNetwork.streetLayer,
+        beamConfig.beam.routing.r5.linkRadiusMeters
+      )
+      ScenarioLoaderHelper.validateScenario(scenario, snapLocationHelper, Some(outputDir))
+
       scenario.getPopulation.getPersons.size() shouldBe 1
       intersection(scenario.getPopulation, path = s"$outputDir/${CsvFile.Plans}") shouldBe Set.empty
 
@@ -277,11 +307,18 @@ class SnapCoordinateSpec extends AnyWordSpec with Matchers with BeamHelper {
 
       val outputDir = FileUtils.setConfigOutputFile(beamConfig, matsimConfig)
 
-      val (scenario, _, _) = buildBeamServicesAndScenario(
+      val (scenario, beamScenario, _) = buildBeamServicesAndScenario(
         beamConfig,
         matsimConfig
       )
 
+      val snapLocationHelper: SnapLocationHelper = SnapLocationHelper(
+        new GeoUtilsImpl(beamConfig),
+        beamScenario.transportNetwork.streetLayer,
+        beamConfig.beam.routing.r5.linkRadiusMeters
+      )
+      ScenarioLoaderHelper.validateScenario(scenario, snapLocationHelper, Some(outputDir))
+
       val households = scenario.getHouseholds.getHouseholds.values().asScala.toList
 
       scenario.getPopulation.getPersons.size() shouldBe 2
@@ -308,11 +345,18 @@ class SnapCoordinateSpec extends AnyWordSpec with Matchers with BeamHelper {
 
       val outputDir = FileUtils.setConfigOutputFile(beamConfig, matsimConfig)
 
-      val (scenario, _, _) = buildBeamServicesAndScenario(
+      val (scenario, beamScenario, _) = buildBeamServicesAndScenario(
         beamConfig,
         matsimConfig
       )
 
+      val snapLocationHelper: SnapLocationHelper = SnapLocationHelper(
+        new GeoUtilsImpl(beamConfig),
+        beamScenario.transportNetwork.streetLayer,
+        beamConfig.beam.routing.r5.linkRadiusMeters
+      )
+      ScenarioLoaderHelper.validateScenario(scenario, snapLocationHelper, Some(outputDir))
+
       scenario.getPopulation.getPersons.size() shouldBe 1
       intersection(scenario.getPopulation, path = s"$outputDir/${CsvFile.Plans}") shouldBe Set.empty
 
@@ -325,6 +369,7 @@ class SnapCoordinateSpec extends AnyWordSpec with Matchers with BeamHelper {
         .parseString(s"""
                         |beam.agentsim.agents.freight.toursFilePath = "$pwd/test/test-resources/beam/input/snap-location/freight/freight-tours.csv"
                         |beam.routing.r5.linkRadiusMeters = 350
+                        |beam.agentsim.snapLocationAndRemoveInvalidInputs = true
                         |""".stripMargin)
         .withFallback(testConfig("test/input/beamville/beam-freight.conf"))
         .resolve()
@@ -352,6 +397,7 @@ class SnapCoordinateSpec extends AnyWordSpec with Matchers with BeamHelper {
         .parseString(s"""
                         |beam.agentsim.agents.freight.plansFilePath = "$pwd/test/test-resources/beam/input/snap-location/freight/payload-plans.csv"
                         |beam.routing.r5.linkRadiusMeters = 350
+                        |beam.agentsim.snapLocationAndRemoveInvalidInputs = true
                         |""".stripMargin)
         .withFallback(testConfig("test/input/beamville/beam-freight.conf"))
         .resolve()
@@ -379,6 +425,7 @@ class SnapCoordinateSpec extends AnyWordSpec with Matchers with BeamHelper {
         .parseString(s"""
                         |beam.agentsim.agents.freight.carriersFilePath = "$pwd/test/test-resources/beam/input/snap-location/freight/freight-carriers.csv"
                         |beam.routing.r5.linkRadiusMeters = 350
+                        |beam.agentsim.snapLocationAndRemoveInvalidInputs = true
                         |""".stripMargin)
         .withFallback(testConfig("test/input/beamville/beam-freight.conf"))
         .resolve()

From 64c677228c68260c2a32dd3b53e26fca28aaf656 Mon Sep 17 00:00:00 2001
From: Haitam Laarabi <10712736+htmlrb@users.noreply.github.com>
Date: Tue, 12 Apr 2022 22:28:31 -0700
Subject: [PATCH 52/58] Fix

---
 .../agentsim/infrastructure/ChargingNetwork.scala | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/src/main/scala/beam/agentsim/infrastructure/ChargingNetwork.scala b/src/main/scala/beam/agentsim/infrastructure/ChargingNetwork.scala
index bca09474253..52b876c3a04 100644
--- a/src/main/scala/beam/agentsim/infrastructure/ChargingNetwork.scala
+++ b/src/main/scala/beam/agentsim/infrastructure/ChargingNetwork.scala
@@ -26,11 +26,13 @@ import scala.util.Random
   */
 class ChargingNetwork[GEO: GeoLevel](val chargingZones: Map[Id[ParkingZoneId], ParkingZone[GEO]])
     extends ParkingNetwork[GEO](chargingZones) {
-
   import ChargingNetwork._
 
   override protected val searchFunctions: Option[InfrastructureFunctions[GEO]] = None
 
+  protected val beamVehicleIdToChargingVehicleMap: mutable.HashMap[Id[BeamVehicle], ChargingVehicle] =
+    mutable.HashMap.empty
+
   protected val chargingZoneKeyToChargingStationMap: Map[Id[ParkingZoneId], ChargingStation] =
     chargingZones.map { case (zoneId, zone) => zoneId -> ChargingStation(zone) }
 
@@ -68,7 +70,7 @@ class ChargingNetwork[GEO: GeoLevel](val chargingZones: Map[Id[ParkingZoneId], P
     * @return charging vehicle
     */
   def lookupVehicle(vehicleId: Id[BeamVehicle]): Option[ChargingVehicle] =
-    chargingZoneKeyToChargingStationMap.values.view.flatMap(_.lookupVehicle(vehicleId)).headOption
+    beamVehicleIdToChargingVehicleMap.get(vehicleId)
 
   /**
     * clear charging vehicle map
@@ -87,8 +89,8 @@ class ChargingNetwork[GEO: GeoLevel](val chargingZones: Map[Id[ParkingZoneId], P
     activityType: String,
     theSender: ActorRef
   ): Option[ChargingVehicle] = lookupStation(request.stall.parkingZoneId)
-    .map(
-      _.connect(
+    .map { chargingStation =>
+      val chargingVehicle = chargingStation.connect(
         request.tick,
         request.vehicle,
         request.stall,
@@ -98,7 +100,9 @@ class ChargingNetwork[GEO: GeoLevel](val chargingZones: Map[Id[ParkingZoneId], P
         request.shiftDuration,
         theSender
       )
-    )
+      beamVehicleIdToChargingVehicleMap.put(chargingVehicle.vehicle.id, chargingVehicle)
+      chargingVehicle
+    }
 
   /**
     * @param vehicleId vehicle to end charge
@@ -121,6 +125,7 @@ class ChargingNetwork[GEO: GeoLevel](val chargingZones: Map[Id[ParkingZoneId], P
     */
   def disconnectVehicle(vehicleId: Id[BeamVehicle], tick: Int): Option[ChargingVehicle] = {
     lookupVehicle(vehicleId) map { chargingVehicle =>
+      beamVehicleIdToChargingVehicleMap.remove(vehicleId)
       chargingVehicle.chargingStation.disconnect(chargingVehicle.vehicle.id, tick)
     } getOrElse {
       logger.debug(s"Vehicle $vehicleId is already disconnected")

From 0d742d5d7bac25d45d933330d1ac6459ec84e086 Mon Sep 17 00:00:00 2001
From: Rutvik Patel <heyrutvik@gmail.com>
Date: Wed, 13 Apr 2022 12:44:53 +0530
Subject: [PATCH 53/58] updates test xml input

---
 src/main/scala/beam/sim/BeamHelper.scala      | 23 ++++++++++++++-----
 .../scenario/case1/xml/households.xml         |  4 ++--
 .../scenario/case2/xml/households.xml         |  4 ++--
 3 files changed, 21 insertions(+), 10 deletions(-)

diff --git a/src/main/scala/beam/sim/BeamHelper.scala b/src/main/scala/beam/sim/BeamHelper.scala
index 4d4d3e6b5ed..f28e45b1cf4 100755
--- a/src/main/scala/beam/sim/BeamHelper.scala
+++ b/src/main/scala/beam/sim/BeamHelper.scala
@@ -701,18 +701,29 @@ trait BeamHelper extends LazyLogging {
 
     if (beamScenario.beamConfig.beam.agentsim.snapLocationAndRemoveInvalidInputs) {
       logger.info(s"""
-           |The parameter `beam.agentsim.snapLocationAndRemoveInvalidInputs` is enabled.
-           |This may take some time to finish based on the size of population/households.""".stripMargin)
+      |The parameter `beam.agentsim.snapLocationAndRemoveInvalidInputs` is enabled.
+      |This may take some time to finish depending on the size of population/households.""".stripMargin)
+
+      val beforeHouseholdsCount = scenario.getHouseholds.getHouseholds.size()
+      val beforePopulationCount = scenario.getPopulation.getPersons.size()
+
       val snapLocationHelper = SnapLocationHelper(
         new GeoUtilsImpl(beamScenario.beamConfig),
         beamScenario.transportNetwork.streetLayer,
         beamScenario.beamConfig.beam.routing.r5.linkRadiusMeters
       )
+
       ScenarioLoaderHelper.validateScenario(scenario, snapLocationHelper, Some(outputDir))
-      logger.info(s"""
-        |After snapping locations and validating scenario:
-        |Number of households: ${scenario.getHouseholds.getHouseholds.size()}
-        |Number of persons: ${scenario.getPopulation.getPersons.size()}""".stripMargin)
+
+      val afterHouseholdsCount = scenario.getHouseholds.getHouseholds.size()
+      val afterPopulationCount = scenario.getPopulation.getPersons.size()
+
+      logger.info(
+        s"""
+      |After snapping locations and validating scenario:
+      |Number of households: $afterHouseholdsCount. Removed: ${beforeHouseholdsCount - afterHouseholdsCount}.
+      |Number of persons: $afterPopulationCount. Removed: ${beforePopulationCount - afterPopulationCount}.""".stripMargin
+      )
     }
 
     // write static metrics, such as population size, vehicles fleet size, etc.
diff --git a/test/test-resources/beam/input/snap-location/scenario/case1/xml/households.xml b/test/test-resources/beam/input/snap-location/scenario/case1/xml/households.xml
index cf1c3ece8e2..2211510e3d0 100755
--- a/test/test-resources/beam/input/snap-location/scenario/case1/xml/households.xml
+++ b/test/test-resources/beam/input/snap-location/scenario/case1/xml/households.xml
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fe9c0a5dc3f9469f909434edece253fb100b89b307ef2a0ef24c88e3f3e56b92
-size 859
+oid sha256:1749a97d76b2735b8736104ffee1c1c887be650224cf39e4a62426ce17cf3a18
+size 875
diff --git a/test/test-resources/beam/input/snap-location/scenario/case2/xml/households.xml b/test/test-resources/beam/input/snap-location/scenario/case2/xml/households.xml
index cf1c3ece8e2..2211510e3d0 100755
--- a/test/test-resources/beam/input/snap-location/scenario/case2/xml/households.xml
+++ b/test/test-resources/beam/input/snap-location/scenario/case2/xml/households.xml
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fe9c0a5dc3f9469f909434edece253fb100b89b307ef2a0ef24c88e3f3e56b92
-size 859
+oid sha256:1749a97d76b2735b8736104ffee1c1c887be650224cf39e4a62426ce17cf3a18
+size 875

From e7a081bc1889b46a2caaa110ac3874509192ada2 Mon Sep 17 00:00:00 2001
From: Rutvik Patel <heyrutvik@gmail.com>
Date: Thu, 14 Apr 2022 10:46:58 +0530
Subject: [PATCH 54/58] simplifies code

---
 .../agents/freight/input/FreightReader.scala  |   2 +-
 .../freight/input/GenericFreightReader.scala  | 123 +++++++-----------
 .../infrastructure/taz/TAZTreeMap.scala       |  63 ++++-----
 .../SupplementaryTripGenerator.scala          |  16 +--
 src/main/scala/beam/sim/BeamMobsim.scala      |   2 +-
 .../beam/utils/SnapCoordinateUtils.scala      |  68 +++++-----
 .../utils/scenario/ScenarioLoaderHelper.scala |  22 ++--
 .../input/GenericFreightReaderSpec.scala      |  18 ++-
 .../scala/beam/utils/SnapCoordinateSpec.scala |  15 +--
 9 files changed, 152 insertions(+), 177 deletions(-)

diff --git a/src/main/scala/beam/agentsim/agents/freight/input/FreightReader.scala b/src/main/scala/beam/agentsim/agents/freight/input/FreightReader.scala
index 0f37f310268..ec96d6d07b7 100644
--- a/src/main/scala/beam/agentsim/agents/freight/input/FreightReader.scala
+++ b/src/main/scala/beam/agentsim/agents/freight/input/FreightReader.scala
@@ -201,7 +201,7 @@ object FreightReader {
           rand,
           tazMap,
           beamConfig.beam.agentsim.snapLocationAndRemoveInvalidInputs,
-          Some(snapLocationHelper),
+          snapLocationHelper,
           outputDirMaybe
         )
       case s =>
diff --git a/src/main/scala/beam/agentsim/agents/freight/input/GenericFreightReader.scala b/src/main/scala/beam/agentsim/agents/freight/input/GenericFreightReader.scala
index 9ce4f685e1d..c2870ef00ce 100644
--- a/src/main/scala/beam/agentsim/agents/freight/input/GenericFreightReader.scala
+++ b/src/main/scala/beam/agentsim/agents/freight/input/GenericFreightReader.scala
@@ -1,13 +1,12 @@
 package beam.agentsim.agents.freight.input
 
 import beam.agentsim.agents.freight._
-import beam.agentsim.agents.freight.input.GenericFreightReader.ClosestUTMPoint
 import beam.agentsim.agents.vehicles.{BeamVehicle, BeamVehicleType}
 import beam.agentsim.infrastructure.taz.{TAZ, TAZTreeMap}
 import beam.sim.common.GeoUtils
 import beam.sim.config.BeamConfig.Beam.Agentsim.Agents.Freight
 import beam.utils.SnapCoordinateUtils
-import beam.utils.SnapCoordinateUtils.{Category, CsvFile, Error, ErrorInfo, Result, SnapLocationHelper}
+import beam.utils.SnapCoordinateUtils._
 import beam.utils.csv.GenericCsvReader
 import beam.utils.matsim_conversion.MatsimPlanConversion.IdOps
 import com.typesafe.scalalogging.LazyLogging
@@ -28,7 +27,7 @@ class GenericFreightReader(
   rnd: Random,
   tazTree: TAZTreeMap,
   val snapLocationAndRemoveInvalidInputs: Boolean,
-  val snapLocationHelperMaybe: Option[SnapLocationHelper] = None,
+  val snapLocationHelper: SnapLocationHelper,
   val outputDirMaybe: Option[String] = None
 ) extends LazyLogging
     with FreightReader {
@@ -63,7 +62,7 @@ class GenericFreightReader(
           row.get("departureLocationZone"),
           snapLocationAndRemoveInvalidInputs
         ) match {
-          case (_, Left(_) | Right(Result.Succeed(_))) =>
+          case (_, Right(_)) =>
             Some(
               FreightTour(
                 tourId,
@@ -71,31 +70,28 @@ class GenericFreightReader(
                 maxTourDurationInSec
               )
             )
-          case (_, Right(Result.OutOfBoundingBoxError)) =>
+          case (_, Left(Error.OutOfBoundingBoxError)) =>
             errors.append(
               ErrorInfo(
                 tourId.toString,
                 Category.FreightTour,
-                Error.OutOfBoundingBox,
+                Error.OutOfBoundingBoxError,
                 departureLocationX.toDouble,
                 departureLocationY.toDouble
               )
             )
             None
-          case (_, Right(Result.R5SplitNullError)) =>
+          case (_, Left(Error.R5SplitNullError)) =>
             errors.append(
               ErrorInfo(
                 tourId.toString,
                 Category.FreightTour,
-                Error.R5SplitNull,
+                Error.R5SplitNullError,
                 departureLocationX.toDouble,
                 departureLocationY.toDouble
               )
             )
             None
-          case _ =>
-            logger.error(f"Following freight tour row discarded for unknown reason: $row")
-            None
         }
       }
 
@@ -137,53 +133,47 @@ class GenericFreightReader(
         val locationX = row.get("locationX")
         val locationY = row.get("locationY")
 
-        // note: placeholder to update locationZone and locationUTM later
-        val payloadPlan = PayloadPlan(
-          payloadId,
-          get("sequenceRank").toDouble.round.toInt,
-          get("tourId").createId,
-          get("payloadType").createId[PayloadType],
-          get("weightInKg").toDouble,
-          requestType,
-          activityType,
-          None,
-          new Coord(),
-          get("estimatedTimeOfArrivalInSec").toDouble.toInt,
-          get("arrivalTimeWindowInSecLower").toDouble.toInt,
-          get("arrivalTimeWindowInSecUpper").toDouble.toInt,
-          operationDurationInSec
-        )
-
         extractCoordOrTaz(locationX, locationY, row.get("locationZone"), snapLocationAndRemoveInvalidInputs) match {
-          case (locationZoneMaybe, Left(locationZoneUTM)) =>
-            Some(payloadPlan.copy(locationZone = locationZoneMaybe, locationUTM = locationZoneUTM))
-          case (locationZoneMaybe, Right(Result.Succeed(splitCoord))) =>
-            Some(payloadPlan.copy(locationZone = locationZoneMaybe, locationUTM = splitCoord))
-          case (_, Right(Result.OutOfBoundingBoxError)) =>
+          case (locationZoneMaybe, Right(coord)) =>
+            Some(
+              PayloadPlan(
+                payloadId,
+                get("sequenceRank").toDouble.round.toInt,
+                get("tourId").createId,
+                get("payloadType").createId[PayloadType],
+                get("weightInKg").toDouble,
+                requestType,
+                activityType,
+                locationZoneMaybe,
+                coord,
+                get("estimatedTimeOfArrivalInSec").toDouble.toInt,
+                get("arrivalTimeWindowInSecLower").toDouble.toInt,
+                get("arrivalTimeWindowInSecUpper").toDouble.toInt,
+                operationDurationInSec
+              )
+            )
+          case (_, Left(Error.OutOfBoundingBoxError)) =>
             errors.append(
               ErrorInfo(
                 payloadId.toString,
                 Category.FreightPayloadPlan,
-                Error.OutOfBoundingBox,
+                Error.OutOfBoundingBoxError,
                 locationX.toDouble,
                 locationY.toDouble
               )
             )
             None
-          case (_, Right(Result.R5SplitNullError)) =>
+          case (_, Left(Error.R5SplitNullError)) =>
             errors.append(
               ErrorInfo(
                 payloadId.toString,
                 Category.FreightPayloadPlan,
-                Error.R5SplitNull,
+                Error.R5SplitNullError,
                 locationX.toDouble,
                 locationY.toDouble
               )
             )
             None
-          case _ =>
-            logger.error(f"Following freight payload plan row discarded for unknown reason: $row")
-            None
         }
       }
 
@@ -203,8 +193,6 @@ class GenericFreightReader(
     allPlans: Map[Id[PayloadPlan], PayloadPlan],
     vehicleTypes: Map[Id[BeamVehicleType], BeamVehicleType]
   ): IndexedSeq[FreightCarrier] = {
-    val errors: ListBuffer[ErrorInfo] = ListBuffer()
-
     val existingTours: Set[Id[FreightTour]] = allTours.keySet.intersect(allPlans.map(_._2.tourId).toSet)
     val plans: Map[Id[PayloadPlan], PayloadPlan] = allPlans.filter { case (_, plan) =>
       existingTours.contains(plan.tourId)
@@ -284,6 +272,8 @@ class GenericFreightReader(
       )
     }
 
+    val errors: ListBuffer[ErrorInfo] = ListBuffer()
+
     val maybeCarrierRows = GenericCsvReader.readAsSeq[Option[FreightCarrierRow]](config.carriersFilePath) { row =>
       def get(key: String): String = getRowValue(config.carriersFilePath, row, key)
       //carrierId,tourId,vehicleId,vehicleTypeId,warehouseZone,warehouseX,warehouseY
@@ -298,48 +288,36 @@ class GenericFreightReader(
         val warehouseX = row.get("warehouseX")
         val warehouseY = row.get("warehouseY")
 
-        // note: placeholder to update warehouseLocationZone and warehouseLocationUTM later
-        val freightCarrier = FreightCarrierRow(carrierId, tourId, vehicleId, vehicleTypeId, None, new Coord())
-
         extractCoordOrTaz(
           row.get("warehouseX"),
           row.get("warehouseY"),
           row.get("warehouseZone"),
           snapLocationAndRemoveInvalidInputs
         ) match {
-          case (warehouseZoneMaybe, Left(warehouseZoneUTM)) =>
-            Some(
-              freightCarrier.copy(warehouseLocationZone = warehouseZoneMaybe, warehouseLocationUTM = warehouseZoneUTM)
-            )
-          case (warehouseZoneMaybe, Right(Result.Succeed(splitCoord))) =>
-            Some(
-              freightCarrier.copy(warehouseLocationZone = warehouseZoneMaybe, warehouseLocationUTM = splitCoord)
-            )
-          case (_, Right(Result.OutOfBoundingBoxError)) =>
+          case (warehouseZoneMaybe, Right(coord)) =>
+            Some(FreightCarrierRow(carrierId, tourId, vehicleId, vehicleTypeId, warehouseZoneMaybe, coord))
+          case (_, Left(Error.OutOfBoundingBoxError)) =>
             errors.append(
               ErrorInfo(
                 carrierId.toString,
                 Category.FreightCarrier,
-                Error.OutOfBoundingBox,
+                Error.OutOfBoundingBoxError,
                 warehouseX.toDouble,
                 warehouseY.toDouble
               )
             )
             None
-          case (_, Right(Result.R5SplitNullError)) =>
+          case (_, Left(Error.R5SplitNullError)) =>
             errors.append(
               ErrorInfo(
                 carrierId.toString,
                 Category.FreightCarrier,
-                Error.R5SplitNull,
+                Error.R5SplitNullError,
                 warehouseX.toDouble,
                 warehouseY.toDouble
               )
             )
             None
-          case _ =>
-            logger.error("Following freight carrier row is discarded for unknown reason: {}", row)
-            None
         }
       }
     }
@@ -366,27 +344,27 @@ class GenericFreightReader(
     case None      => throw new IllegalArgumentException(s"Cannot find taz with id $tazId")
   }
 
-  private def getDistributedTazLocation(taz: TAZ): Coord =
-    convertedLocation(TAZTreeMap.randomLocationInTAZ(taz, rnd, snapLocationHelperMaybe))
-
   private def extractCoordOrTaz(
     strX: String,
     strY: String,
     strZone: String,
     snapLocationAndRemoveInvalidInputs: Boolean
-  ): (Option[Id[TAZ]], ClosestUTMPoint) = {
+  ): (Option[Id[TAZ]], SnapCoordinateResult) = {
     if (isBlank(strX) || isBlank(strY)) {
       val taz = getTaz(strZone)
-      (Some(taz.tazId), Left(getDistributedTazLocation(taz)))
+      val coord =
+        if (snapLocationAndRemoveInvalidInputs) TAZTreeMap.randomLocationInTAZ(taz, rnd, snapLocationHelper)
+        else TAZTreeMap.randomLocationInTAZ(taz, rnd)
+
+      (Some(taz.tazId), Right(coord))
     } else {
       val wasInWgs = config.convertWgs2Utm
       val loc = location(strX.toDouble, strY.toDouble)
-      val finalLoc =
-        if (snapLocationAndRemoveInvalidInputs)
-          snapLocationHelperMaybe.map(_.computeResult(loc, wasInWgs)).getOrElse(Result.Succeed(loc))
-        else Result.Succeed(loc)
+      val coord =
+        if (snapLocationAndRemoveInvalidInputs) snapLocationHelper.computeResult(loc, wasInWgs)
+        else Right(loc)
 
-      (None, Right(finalLoc))
+      (None, coord)
     }
   }
 
@@ -404,10 +382,3 @@ class GenericFreightReader(
   }
 
 }
-
-object GenericFreightReader {
-
-  // either[tazLocation, snapLocationResult]
-  type ClosestUTMPoint = Either[Coord, Result]
-
-}
diff --git a/src/main/scala/beam/agentsim/infrastructure/taz/TAZTreeMap.scala b/src/main/scala/beam/agentsim/infrastructure/taz/TAZTreeMap.scala
index 6f6266c2660..679ea79d2eb 100755
--- a/src/main/scala/beam/agentsim/infrastructure/taz/TAZTreeMap.scala
+++ b/src/main/scala/beam/agentsim/infrastructure/taz/TAZTreeMap.scala
@@ -1,11 +1,6 @@
 package beam.agentsim.infrastructure.taz
 
-import beam.utils.SnapCoordinateUtils.{Result, SnapLocationHelper}
-
-import java.io._
-import java.util
-import scala.collection.JavaConverters._
-import scala.collection.mutable
+import beam.utils.SnapCoordinateUtils.SnapLocationHelper
 import beam.utils.matsim_conversion.ShapeUtils
 import beam.utils.matsim_conversion.ShapeUtils.{HasQuadBounds, QuadTreeBounds}
 import com.vividsolutions.jts.geom.Geometry
@@ -15,8 +10,12 @@ import org.matsim.core.utils.gis.ShapeFileReader
 import org.opengis.feature.simple.SimpleFeature
 import org.slf4j.LoggerFactory
 
+import java.io._
+import java.util
 import scala.annotation.tailrec
+import scala.collection.JavaConverters._
 import scala.collection.concurrent.TrieMap
+import scala.collection.mutable
 
 /**
   * TAZTreeMap manages a quadTree to find the closest TAZ to any coordinate.
@@ -197,9 +196,9 @@ object TAZTreeMap {
     new TAZTreeMap(tazQuadTree)
   }
 
-  private def createRandomLocationInTAZ(
+  def randomLocationInTAZ(
     taz: TAZ,
-    rand: scala.util.Random = new scala.util.Random(System.currentTimeMillis())
+    rand: scala.util.Random
   ): Coord = {
     val radius = Math.sqrt(taz.areaInSquareMeters / Math.PI) / 2
     val a = 2 * Math.PI * rand.nextDouble()
@@ -211,37 +210,31 @@ object TAZTreeMap {
 
   def randomLocationInTAZ(
     taz: TAZ,
-    rand: scala.util.Random = new scala.util.Random(System.currentTimeMillis()),
-    snapLocationHelperMaybe: Option[SnapLocationHelper] = None
+    rand: scala.util.Random,
+    snapLocationHelper: SnapLocationHelper
   ): Coord = {
     val tazId = taz.tazId.toString
-    snapLocationHelperMaybe match {
-      case None => createRandomLocationInTAZ(taz, rand)
-      case Some(helper) =>
-        val max = 10000
-        var counter = 0
-        var split: Coord = null
-        while (split == null && counter < max) {
-          helper.computeResult(createRandomLocationInTAZ(taz, rand)) match {
-            case Result.Succeed(splitCoord) =>
-              logger.info(s"Found valid location $splitCoord within taz $tazId in $counter attempt(s).")
-              split = splitCoord
-            case _ =>
-          }
-          counter += 1
-        }
-
-        if (split == null) {
-          // TODO what to do if we don't find valid point in "max" attempts?
-          val loc = createRandomLocationInTAZ(taz, rand)
-          logger.warn(
-            s"Could not found valid location within taz $tazId even in $max attempts. Creating one anyone $loc."
-          )
-          split = loc
-        }
+    val max = 10000
+    var counter = 0
+    var split: Coord = null
+    while (split == null && counter < max) {
+      snapLocationHelper.computeResult(randomLocationInTAZ(taz, rand)) match {
+        case Right(splitCoord) =>
+          split = splitCoord
+        case _ =>
+      }
+      counter += 1
+    }
 
-        split
+    if (split == null) {
+      val loc = randomLocationInTAZ(taz, rand)
+      logger.warn(
+        s"Could not found valid location within taz $tazId even in $max attempts. Creating one anyway $loc."
+      )
+      split = loc
     }
+
+    split
   }
 
   /**
diff --git a/src/main/scala/beam/replanning/SupplementaryTripGenerator.scala b/src/main/scala/beam/replanning/SupplementaryTripGenerator.scala
index 609f3879a2b..8db4efb2eef 100755
--- a/src/main/scala/beam/replanning/SupplementaryTripGenerator.scala
+++ b/src/main/scala/beam/replanning/SupplementaryTripGenerator.scala
@@ -24,9 +24,9 @@ class SupplementaryTripGenerator(
   val destinationChoiceModel: DestinationChoiceModel,
   val beamServices: BeamServices,
   val personId: Id[Person],
-  val snapLocationHelperMaybe: Option[SnapLocationHelper] = None
+  val snapLocationHelper: SnapLocationHelper
 ) {
-  val r: Random.type = scala.util.Random
+  val rnd: Random = new scala.util.Random(System.currentTimeMillis())
   val personSpecificSeed: Long = personId.hashCode().toLong
 
   val travelTimeBufferInSec: Int = 30 * 60
@@ -188,8 +188,8 @@ class SupplementaryTripGenerator(
             false -> noTrip
           )
 
-        tripMNL.sampleAlternative(tripChoice, r) match {
-          case Some(mnlSample) if mnlSample.alternativeType => destinationMNL.sampleAlternative(modeChoice, r)
+        tripMNL.sampleAlternative(tripChoice, rnd) match {
+          case Some(mnlSample) if mnlSample.alternativeType => destinationMNL.sampleAlternative(modeChoice, rnd)
           case _                                            => None
         }
     }
@@ -201,7 +201,7 @@ class SupplementaryTripGenerator(
         val newActivity =
           PopulationUtils.createActivityFromCoord(
             newActivityType,
-            TAZTreeMap.randomLocationInTAZ(chosenAlternative.taz, snapLocationHelperMaybe = snapLocationHelperMaybe)
+            TAZTreeMap.randomLocationInTAZ(chosenAlternative.taz, rnd, snapLocationHelper)
           )
         val activityBeforeNewActivity =
           PopulationUtils.createActivityFromCoord(prevActivity.getType, prevActivity.getCoord)
@@ -247,7 +247,7 @@ class SupplementaryTripGenerator(
       } else {
         TAZs.map { taz =>
           val destinationCoord: Coord =
-            TAZTreeMap.randomLocationInTAZ(taz, snapLocationHelperMaybe = snapLocationHelperMaybe)
+            TAZTreeMap.randomLocationInTAZ(taz, rnd, snapLocationHelper)
           val additionalActivity = PopulationUtils.createActivityFromCoord(newActivityType, destinationCoord)
           additionalActivity.setStartTime(startTime)
           additionalActivity.setEndTime(endTime)
@@ -382,7 +382,7 @@ class SupplementaryTripGenerator(
         } else { None }
         chosenStartIndex match {
           case Some(index) =>
-            val startTime = math.max((r.nextDouble() + index) * 3600, altStart + travelTimeBufferInSec)
+            val startTime = math.max((rnd.nextDouble() + index) * 3600, altStart + travelTimeBufferInSec)
             (
               actType,
               startTime.toInt,
@@ -415,7 +415,7 @@ class SupplementaryTripGenerator(
     keyToProb: Map[A, Double]
   ): Option[A] = {
     val totalProb = keyToProb.values.sum
-    val randomDraw = r.nextDouble()
+    val randomDraw = rnd.nextDouble()
     val probs = keyToProb.values.scanLeft(0.0)(_ + _ / totalProb).drop(1)
     keyToProb.keys.zip(probs).dropWhile { _._2 <= randomDraw }.headOption match {
       case Some(result) => Some(result._1)
diff --git a/src/main/scala/beam/sim/BeamMobsim.scala b/src/main/scala/beam/sim/BeamMobsim.scala
index 983d4a21cef..6659ff22c0a 100755
--- a/src/main/scala/beam/sim/BeamMobsim.scala
+++ b/src/main/scala/beam/sim/BeamMobsim.scala
@@ -232,7 +232,7 @@ class BeamMobsim @Inject() (
               destinationChoiceModel,
               beamServices,
               person.getId,
-              Some(snapLocationHelper)
+              snapLocationHelper
             )
           val newPlan =
             supplementaryTripGenerator.generateNewPlans(person.getSelectedPlan, destinationChoiceModel, modesAvailable)
diff --git a/src/main/scala/beam/utils/SnapCoordinateUtils.scala b/src/main/scala/beam/utils/SnapCoordinateUtils.scala
index c2807065d30..15d00410d84 100644
--- a/src/main/scala/beam/utils/SnapCoordinateUtils.scala
+++ b/src/main/scala/beam/utils/SnapCoordinateUtils.scala
@@ -4,20 +4,46 @@ import beam.sim.common.GeoUtils
 import beam.utils.csv.CsvWriter
 import com.conveyal.r5.streets.StreetLayer
 import com.typesafe.scalalogging.LazyLogging
+import enumeratum._
 import org.matsim.api.core.v01.Coord
 
 import scala.collection.concurrent.TrieMap
 
 object SnapCoordinateUtils extends LazyLogging {
 
-  trait Result
+  sealed abstract class Error(override val entryName: String) extends EnumEntry
 
-  object Result {
-    final case object OutOfBoundingBoxError extends Result
-    final case object R5SplitNullError extends Result
-    final case class Succeed(splitCoord: Coord) extends Result
+  object Error extends Enum[Error] {
+    val values = findValues
+
+    case object OutOfBoundingBoxError extends Error("OutOfBoundingBox")
+    case object R5SplitNullError extends Error("R5SplitNull")
+  }
+
+  sealed abstract class Category(override val entryName: String) extends EnumEntry
+
+  object Category extends Enum[Category] {
+    val values = findValues
+
+    case object ScenarioPerson extends Category("Person")
+    case object ScenarioHousehold extends Category("Household")
+    case object FreightTour extends Category("Tour")
+    case object FreightPayloadPlan extends Category("PayloadPlan")
+    case object FreightCarrier extends Category("Carrier")
+  }
+
+  object CsvFile {
+    val Plans = "snapLocationPlanErrors.csv"
+    val Households = "snapLocationHouseholdErrors.csv"
+    val FreightTours = "snapLocationFreightTourErrors.csv"
+    val FreightPayloadPlans = "snapLocationFreightPayloadPlanErrors.csv"
+    val FreightCarriers = "snapLocationFreightCarrierErrors.csv"
   }
 
+  final case class ErrorInfo(id: String, category: Category, error: Error, planX: Double, planY: Double)
+
+  type SnapCoordinateResult = Either[Error, Coord]
+
   final case class SnapLocationHelper(geo: GeoUtils, streetLayer: StreetLayer, maxRadius: Double) {
     private val store: TrieMap[Coord, Option[Coord]] = TrieMap.empty
 
@@ -26,7 +52,7 @@ object SnapCoordinateUtils extends LazyLogging {
       store.get(coord).flatten
     }
 
-    def computeResult(planCoord: Coord, isWgs: Boolean = false): Result = {
+    def computeResult(planCoord: Coord, isWgs: Boolean = false): SnapCoordinateResult = {
       val coord = if (isWgs) planCoord else geo.utm2Wgs(planCoord)
       if (streetLayer.envelope.contains(coord.getX, coord.getY)) {
         val snapCoordOpt = store.getOrElseUpdate(
@@ -36,39 +62,15 @@ object SnapCoordinateUtils extends LazyLogging {
             geo.wgs2Utm(updatedPlanCoord)
           }
         )
-        snapCoordOpt.fold[Result](Result.R5SplitNullError)(Result.Succeed)
-      } else Result.OutOfBoundingBoxError
+        snapCoordOpt.fold[SnapCoordinateResult](Left(Error.R5SplitNullError))(coord => Right(coord))
+      } else Left(Error.OutOfBoundingBoxError)
     }
   }
 
-  object Error {
-    val OutOfBoundingBox = "OutOfBoundingBox"
-    val R5SplitNull = "R5SplitNull"
-  }
-
-  object Category {
-    val ScenarioPerson = "Person"
-    val ScenarioHousehold = "Household"
-    val FreightTour = "Tour"
-    val FreightPayloadPlan = "PayloadPlan"
-    val FreightCarrier = "Carrier"
-  }
-
-  object CsvFile {
-    val Plans = "snapLocationPlanErrors.csv"
-    val Households = "snapLocationHouseholdErrors.csv"
-    val FreightTours = "snapLocationFreightTourErrors.csv"
-    val FreightPayloadPlans = "snapLocationFreightPayloadPlanErrors.csv"
-    val FreightCarriers = "snapLocationFreightCarrierErrors.csv"
-  }
-
-  final case class ErrorInfo(id: String, category: String, error: String, planX: Double, planY: Double)
-  final case class Processed[A](data: Seq[A] = Seq.empty, errors: Seq[ErrorInfo] = Seq.empty)
-
   def writeToCsv(path: String, errors: Seq[ErrorInfo]): Unit = {
     new CsvWriter(path, "id", "category", "error", "x", "y")
       .writeAllAndClose(
-        errors.map(error => List(error.id, error.category, error.error, error.planX, error.planY))
+        errors.map(error => List(error.id, error.category.entryName, error.error.entryName, error.planX, error.planY))
       )
     logger.info("See location error info at {}.", path)
   }
diff --git a/src/main/scala/beam/utils/scenario/ScenarioLoaderHelper.scala b/src/main/scala/beam/utils/scenario/ScenarioLoaderHelper.scala
index a58902f4ab6..a2b97c95e7e 100644
--- a/src/main/scala/beam/utils/scenario/ScenarioLoaderHelper.scala
+++ b/src/main/scala/beam/utils/scenario/ScenarioLoaderHelper.scala
@@ -1,7 +1,7 @@
 package beam.utils.scenario
 
 import beam.utils.SnapCoordinateUtils
-import beam.utils.SnapCoordinateUtils.{Category, CsvFile, Error, ErrorInfo, Result, SnapLocationHelper}
+import beam.utils.SnapCoordinateUtils.{Category, CsvFile, Error, ErrorInfo, SnapLocationHelper}
 import com.typesafe.scalalogging.LazyLogging
 import org.matsim.api.core.v01.population.{Activity, Leg, Person}
 import org.matsim.api.core.v01.{Coord, Id}
@@ -37,22 +37,22 @@ object ScenarioLoaderHelper extends LazyLogging {
         case a: Activity =>
           val planCoord = a.getCoord
           snapLocationHelper.computeResult(planCoord) match {
-            case Result.Succeed(_) =>
+            case Right(_) =>
               // note: we don't want to update coord in-place here since we might end up removing plan from the person
               errors
-            case Result.OutOfBoundingBoxError =>
+            case Left(Error.OutOfBoundingBoxError) =>
               errors :+ ErrorInfo(
                 personId.toString,
                 Category.ScenarioPerson,
-                Error.OutOfBoundingBox,
+                Error.OutOfBoundingBoxError,
                 planCoord.getX,
                 planCoord.getY
               )
-            case Result.R5SplitNullError =>
+            case Left(Error.R5SplitNullError) =>
               errors :+ ErrorInfo(
                 personId.toString,
                 Category.ScenarioPerson,
-                Error.R5SplitNull,
+                Error.R5SplitNullError,
                 planCoord.getX,
                 planCoord.getY
               )
@@ -134,29 +134,29 @@ object ScenarioLoaderHelper extends LazyLogging {
       val planCoord = new Coord(locationX, locationY)
 
       snapLocationHelper.computeResult(planCoord) match {
-        case Result.Succeed(splitCoord) =>
+        case Right(splitCoord) =>
           attr.putAttribute(householdId, "homecoordx", splitCoord.getX)
           attr.putAttribute(householdId, "homecoordy", splitCoord.getY)
-        case Result.OutOfBoundingBoxError =>
+        case Left(Error.OutOfBoundingBoxError) =>
           household.getMemberIds.asScala.toList.foreach(personId => scenario.getPopulation.getPersons.remove(personId))
           scenario.getHouseholds.getHouseholds.remove(household.getId)
           householdErrors.append(
             ErrorInfo(
               householdId,
               Category.ScenarioHousehold,
-              Error.OutOfBoundingBox,
+              Error.OutOfBoundingBoxError,
               planCoord.getX,
               planCoord.getY
             )
           )
-        case Result.R5SplitNullError =>
+        case Left(Error.R5SplitNullError) =>
           household.getMemberIds.asScala.toList.foreach(personId => scenario.getPopulation.getPersons.remove(personId))
           scenario.getHouseholds.getHouseholds.remove(household.getId)
           householdErrors.append(
             ErrorInfo(
               householdId,
               Category.ScenarioHousehold,
-              Error.R5SplitNull,
+              Error.R5SplitNullError,
               planCoord.getX,
               planCoord.getY
             )
diff --git a/src/test/scala/beam/agentsim/agents/freight/input/GenericFreightReaderSpec.scala b/src/test/scala/beam/agentsim/agents/freight/input/GenericFreightReaderSpec.scala
index 3bf55e76937..ace55ead817 100644
--- a/src/test/scala/beam/agentsim/agents/freight/input/GenericFreightReaderSpec.scala
+++ b/src/test/scala/beam/agentsim/agents/freight/input/GenericFreightReaderSpec.scala
@@ -5,6 +5,7 @@ import beam.agentsim.infrastructure.taz.TAZTreeMap
 import beam.sim.common.GeoUtils
 import beam.sim.config.BeamConfig.Beam.Agentsim.Agents.Freight
 import beam.utils.BeamVehicleUtils
+import beam.utils.SnapCoordinateUtils.SnapLocationHelper
 import beam.utils.matsim_conversion.MatsimPlanConversion.IdOps
 import org.matsim.api.core.v01.population.{Activity, Person, Plan, PopulationFactory}
 import org.matsim.api.core.v01.{Coord, Id}
@@ -45,8 +46,17 @@ class GenericFreightReaderSpec extends AnyWordSpecLike with Matchers {
 
   val rnd = new Random(2333L)
 
+  val snapLocationHelper = Mockito.mock(classOf[SnapLocationHelper])
+
   private val reader =
-    new GenericFreightReader(freightConfig, geoUtils, rnd, tazMap, snapLocationAndRemoveInvalidInputs = false)
+    new GenericFreightReader(
+      freightConfig,
+      geoUtils,
+      rnd,
+      tazMap,
+      snapLocationAndRemoveInvalidInputs = false,
+      snapLocationHelper
+    )
 
   "PayloadPlansConverter" should {
     "read Payload Plans" in {
@@ -158,7 +168,8 @@ class GenericFreightReaderSpec extends AnyWordSpecLike with Matchers {
       geoUtils,
       new Random(4324L),
       tazMap,
-      snapLocationAndRemoveInvalidInputs = false
+      snapLocationAndRemoveInvalidInputs = false,
+      snapLocationHelper
     )
     val payloadPlans: Map[Id[PayloadPlan], PayloadPlan] = converter.readPayloadPlans()
     val tours = converter.readFreightTours()
@@ -169,7 +180,8 @@ class GenericFreightReaderSpec extends AnyWordSpecLike with Matchers {
         geoUtils,
         new Random(73737L),
         tazMap,
-        snapLocationAndRemoveInvalidInputs = false
+        snapLocationAndRemoveInvalidInputs = false,
+        snapLocationHelper
       ).readFreightCarriers(
         tours,
         payloadPlans,
diff --git a/src/test/scala/beam/utils/SnapCoordinateSpec.scala b/src/test/scala/beam/utils/SnapCoordinateSpec.scala
index 7af979c78b6..c6b1c690909 100644
--- a/src/test/scala/beam/utils/SnapCoordinateSpec.scala
+++ b/src/test/scala/beam/utils/SnapCoordinateSpec.scala
@@ -4,7 +4,7 @@ import beam.agentsim.agents.freight.{FreightCarrier, FreightTour, PayloadPlan}
 import beam.sim.BeamHelper
 import beam.sim.common.GeoUtilsImpl
 import beam.sim.config.{BeamConfig, MatSimBeamConfigBuilder}
-import beam.utils.SnapCoordinateUtils.{CsvFile, ErrorInfo, Result, SnapLocationHelper}
+import beam.utils.SnapCoordinateUtils.{Category, CsvFile, Error, ErrorInfo, SnapCoordinateResult, SnapLocationHelper}
 import beam.utils.TestConfigUtils.testConfig
 import beam.utils.scenario.ScenarioLoaderHelper
 import com.typesafe.config.{ConfigFactory, Config => TypesafeConfig}
@@ -29,7 +29,7 @@ class SnapCoordinateSpec extends AnyWordSpec with Matchers with BeamHelper {
       .drop(1)
       .map(_.split(","))
       .map { row =>
-        ErrorInfo(row(0), row(1), row(2), row(3).toDouble, row(4).toDouble)
+        ErrorInfo(row(0), Category.withName(row(1)), Error.withName(row(2)), row(3).toDouble, row(4).toDouble)
       }
       .toList
     src.close()
@@ -95,20 +95,20 @@ class SnapCoordinateSpec extends AnyWordSpec with Matchers with BeamHelper {
       ScenarioLoaderHelper.validateScenario(scenario, snapLocationHelper, Some(outputDir))
 
       val dummyCoord = new Coord()
-      val population: Seq[SnapCoordinateUtils.Result] = scenario.getPopulation.getPersons
+      val population: Seq[SnapCoordinateResult] = scenario.getPopulation.getPersons
         .values()
         .asScala
         .flatMap { person =>
           person.getPlans.asScala.flatMap { plan =>
             plan.getPlanElements.asScala.map {
               case e: Activity => snapLocationHelper.computeResult(e.getCoord)
-              case _           => Result.Succeed(dummyCoord)
+              case _           => Right(dummyCoord)
             }
           }
         }
         .toList
 
-      val households: Seq[SnapCoordinateUtils.Result] = scenario.getHouseholds.getHouseholds
+      val households: Seq[SnapCoordinateResult] = scenario.getHouseholds.getHouseholds
         .values()
         .asScala
         .map { household =>
@@ -123,10 +123,7 @@ class SnapCoordinateSpec extends AnyWordSpec with Matchers with BeamHelper {
         }
         .toList
 
-      (population ++ households).forall {
-        case _: Result.Succeed => true
-        case _                 => false
-      } shouldBe true
+      (population ++ households).forall(_.isRight) shouldBe true
     }
 
     "remove invalid persons and households [case1 xml input]" in {

From 162f2f11f20e1624ff9423ac97a0d06e8b7a9d26 Mon Sep 17 00:00:00 2001
From: Rutvik Patel <heyrutvik@gmail.com>
Date: Thu, 14 Apr 2022 14:19:59 +0530
Subject: [PATCH 55/58] updates as per review comments

---
 .../freight/input/GenericFreightReader.scala  | 45 +++----------------
 .../utils/scenario/ScenarioLoaderHelper.scala | 20 +++------
 2 files changed, 12 insertions(+), 53 deletions(-)

diff --git a/src/main/scala/beam/agentsim/agents/freight/input/GenericFreightReader.scala b/src/main/scala/beam/agentsim/agents/freight/input/GenericFreightReader.scala
index c2870ef00ce..5c09fc33db1 100644
--- a/src/main/scala/beam/agentsim/agents/freight/input/GenericFreightReader.scala
+++ b/src/main/scala/beam/agentsim/agents/freight/input/GenericFreightReader.scala
@@ -70,23 +70,12 @@ class GenericFreightReader(
                 maxTourDurationInSec
               )
             )
-          case (_, Left(Error.OutOfBoundingBoxError)) =>
+          case (_, Left(error)) =>
             errors.append(
               ErrorInfo(
                 tourId.toString,
                 Category.FreightTour,
-                Error.OutOfBoundingBoxError,
-                departureLocationX.toDouble,
-                departureLocationY.toDouble
-              )
-            )
-            None
-          case (_, Left(Error.R5SplitNullError)) =>
-            errors.append(
-              ErrorInfo(
-                tourId.toString,
-                Category.FreightTour,
-                Error.R5SplitNullError,
+                error,
                 departureLocationX.toDouble,
                 departureLocationY.toDouble
               )
@@ -152,23 +141,12 @@ class GenericFreightReader(
                 operationDurationInSec
               )
             )
-          case (_, Left(Error.OutOfBoundingBoxError)) =>
+          case (_, Left(error)) =>
             errors.append(
               ErrorInfo(
                 payloadId.toString,
                 Category.FreightPayloadPlan,
-                Error.OutOfBoundingBoxError,
-                locationX.toDouble,
-                locationY.toDouble
-              )
-            )
-            None
-          case (_, Left(Error.R5SplitNullError)) =>
-            errors.append(
-              ErrorInfo(
-                payloadId.toString,
-                Category.FreightPayloadPlan,
-                Error.R5SplitNullError,
+                error,
                 locationX.toDouble,
                 locationY.toDouble
               )
@@ -296,23 +274,12 @@ class GenericFreightReader(
         ) match {
           case (warehouseZoneMaybe, Right(coord)) =>
             Some(FreightCarrierRow(carrierId, tourId, vehicleId, vehicleTypeId, warehouseZoneMaybe, coord))
-          case (_, Left(Error.OutOfBoundingBoxError)) =>
-            errors.append(
-              ErrorInfo(
-                carrierId.toString,
-                Category.FreightCarrier,
-                Error.OutOfBoundingBoxError,
-                warehouseX.toDouble,
-                warehouseY.toDouble
-              )
-            )
-            None
-          case (_, Left(Error.R5SplitNullError)) =>
+          case (_, Left(error)) =>
             errors.append(
               ErrorInfo(
                 carrierId.toString,
                 Category.FreightCarrier,
-                Error.R5SplitNullError,
+                error,
                 warehouseX.toDouble,
                 warehouseY.toDouble
               )
diff --git a/src/main/scala/beam/utils/scenario/ScenarioLoaderHelper.scala b/src/main/scala/beam/utils/scenario/ScenarioLoaderHelper.scala
index a2b97c95e7e..2a3f108a1bc 100644
--- a/src/main/scala/beam/utils/scenario/ScenarioLoaderHelper.scala
+++ b/src/main/scala/beam/utils/scenario/ScenarioLoaderHelper.scala
@@ -40,19 +40,11 @@ object ScenarioLoaderHelper extends LazyLogging {
             case Right(_) =>
               // note: we don't want to update coord in-place here since we might end up removing plan from the person
               errors
-            case Left(Error.OutOfBoundingBoxError) =>
+            case Left(error) =>
               errors :+ ErrorInfo(
                 personId.toString,
                 Category.ScenarioPerson,
-                Error.OutOfBoundingBoxError,
-                planCoord.getX,
-                planCoord.getY
-              )
-            case Left(Error.R5SplitNullError) =>
-              errors :+ ErrorInfo(
-                personId.toString,
-                Category.ScenarioPerson,
-                Error.R5SplitNullError,
+                error,
                 planCoord.getX,
                 planCoord.getY
               )
@@ -137,26 +129,26 @@ object ScenarioLoaderHelper extends LazyLogging {
         case Right(splitCoord) =>
           attr.putAttribute(householdId, "homecoordx", splitCoord.getX)
           attr.putAttribute(householdId, "homecoordy", splitCoord.getY)
-        case Left(Error.OutOfBoundingBoxError) =>
+        case Left(error @ Error.OutOfBoundingBoxError) =>
           household.getMemberIds.asScala.toList.foreach(personId => scenario.getPopulation.getPersons.remove(personId))
           scenario.getHouseholds.getHouseholds.remove(household.getId)
           householdErrors.append(
             ErrorInfo(
               householdId,
               Category.ScenarioHousehold,
-              Error.OutOfBoundingBoxError,
+              error,
               planCoord.getX,
               planCoord.getY
             )
           )
-        case Left(Error.R5SplitNullError) =>
+        case Left(error @ Error.R5SplitNullError) =>
           household.getMemberIds.asScala.toList.foreach(personId => scenario.getPopulation.getPersons.remove(personId))
           scenario.getHouseholds.getHouseholds.remove(household.getId)
           householdErrors.append(
             ErrorInfo(
               householdId,
               Category.ScenarioHousehold,
-              Error.R5SplitNullError,
+              error,
               planCoord.getX,
               planCoord.getY
             )

From 3ae745af28af6d618f320e289c925ef68e1b57ed Mon Sep 17 00:00:00 2001
From: Rutvik Patel <heyrutvik@gmail.com>
Date: Thu, 14 Apr 2022 14:56:21 +0530
Subject: [PATCH 56/58] simplifies error match case

---
 .../beam/utils/scenario/ScenarioLoaderHelper.scala | 14 +-------------
 1 file changed, 1 insertion(+), 13 deletions(-)

diff --git a/src/main/scala/beam/utils/scenario/ScenarioLoaderHelper.scala b/src/main/scala/beam/utils/scenario/ScenarioLoaderHelper.scala
index 2a3f108a1bc..3399aad497f 100644
--- a/src/main/scala/beam/utils/scenario/ScenarioLoaderHelper.scala
+++ b/src/main/scala/beam/utils/scenario/ScenarioLoaderHelper.scala
@@ -129,19 +129,7 @@ object ScenarioLoaderHelper extends LazyLogging {
         case Right(splitCoord) =>
           attr.putAttribute(householdId, "homecoordx", splitCoord.getX)
           attr.putAttribute(householdId, "homecoordy", splitCoord.getY)
-        case Left(error @ Error.OutOfBoundingBoxError) =>
-          household.getMemberIds.asScala.toList.foreach(personId => scenario.getPopulation.getPersons.remove(personId))
-          scenario.getHouseholds.getHouseholds.remove(household.getId)
-          householdErrors.append(
-            ErrorInfo(
-              householdId,
-              Category.ScenarioHousehold,
-              error,
-              planCoord.getX,
-              planCoord.getY
-            )
-          )
-        case Left(error @ Error.R5SplitNullError) =>
+        case Left(error) =>
           household.getMemberIds.asScala.toList.foreach(personId => scenario.getPopulation.getPersons.remove(personId))
           scenario.getHouseholds.getHouseholds.remove(household.getId)
           householdErrors.append(

From 9f4943933872d3fed2fbae2a02dc9a7833f0721e Mon Sep 17 00:00:00 2001
From: Haitam Laarabi <10712736+haitamlaarabi@users.noreply.github.com>
Date: Sun, 17 Apr 2022 17:50:50 -0700
Subject: [PATCH 57/58] addressing @JustinPihony 's comments (#3515)

Co-authored-by: Haitam Laarabi <10712736+htmlrb@users.noreply.github.com>
---
 .../python/gemini/{debugging.py => debug_parking_mnl.py}  | 0
 src/main/scala/beam/router/r5/CarWeightCalculator.scala   | 8 ++++----
 2 files changed, 4 insertions(+), 4 deletions(-)
 rename src/main/python/gemini/{debugging.py => debug_parking_mnl.py} (100%)

diff --git a/src/main/python/gemini/debugging.py b/src/main/python/gemini/debug_parking_mnl.py
similarity index 100%
rename from src/main/python/gemini/debugging.py
rename to src/main/python/gemini/debug_parking_mnl.py
diff --git a/src/main/scala/beam/router/r5/CarWeightCalculator.scala b/src/main/scala/beam/router/r5/CarWeightCalculator.scala
index 6d1c48f0a26..e786ff31571 100644
--- a/src/main/scala/beam/router/r5/CarWeightCalculator.scala
+++ b/src/main/scala/beam/router/r5/CarWeightCalculator.scala
@@ -34,7 +34,7 @@ class CarWeightCalculator(workerParams: R5Parameters, travelTimeNoiseFraction: D
     vehicleType: Option[BeamVehicleType],
     time: Double,
     shouldAddNoise: Boolean,
-    hgv: Boolean = false
+    heavyGoodsVehicle: Boolean = false
   ): Double = {
     val link = networkHelper.getLinkUnsafe(linkId)
     assert(link != null)
@@ -58,9 +58,9 @@ class CarWeightCalculator(workerParams: R5Parameters, travelTimeNoiseFraction: D
     val linkTravelTime = Math.max(physSimTravelTimeWithNoise, minTravelTime)
     val result = Math.min(linkTravelTime, maxTravelTime)
 
-    // TODO this is only prototype
-    val isLinkHgv = Try(link.getAttributes.getAttribute("hgv")).map(_.asInstanceOf[Boolean]).getOrElse(false)
-    if (hgv) {
+    if (heavyGoodsVehicle) {
+      // TODO this is only prototype
+      val isLinkHgv = Try(link.getAttributes.getAttribute("hgv")).map(_.asInstanceOf[Boolean]).getOrElse(false)
       if (isLinkHgv) result / 10 else result * 10
     } else
       result

From c76f87bfa878f0f232ecaf735051cc9b0a476802 Mon Sep 17 00:00:00 2001
From: Justin Pihony <justin.pihony@gmail.com>
Date: Mon, 18 Apr 2022 02:27:41 -0400
Subject: [PATCH 58/58] Sync - and add git upgrade (#3517)

---
 .../main/python/updateDependencies/lambda_function.py    | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/aws/src/main/python/updateDependencies/lambda_function.py b/aws/src/main/python/updateDependencies/lambda_function.py
index 4a29a541d14..a8fc4b568f0 100644
--- a/aws/src/main/python/updateDependencies/lambda_function.py
+++ b/aws/src/main/python/updateDependencies/lambda_function.py
@@ -37,15 +37,19 @@
   - sudo apt-get install gcc-8 g++-8 -y
   - sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-8 60 --slave /usr/bin/g++ g++ /usr/bin/g++-8
   - sudo apt install jq -y
+  - sudo add-apt-repository ppa:git-core/ppa -y
+  - sudo apt-get update
+  - sudo apt-get install git -y
   - echo "-------------------Finished updating Beam dependencies----------------------"
   - cd /home/ubuntu/git/beam
   - echo "send notification ..."
   - /home/ubuntu/git/glip.sh -i "http://icons.iconarchive.com/icons/uiconstock/socialmedia/32/AWS-icon.png" -a "Updating Dependencies" -b "Beam automated deployment image update started on $(ec2metadata --instance-id)."
   - echo "git checkout ..."
-  - sudo git reset origin/HEAD
+  - sudo git reset --hard origin/HEAD
   - sudo git checkout -- .
   - sudo git clean -df
   - sudo git checkout develop
+  - sudo git reset --hard origin/develop
   - sudo git pull
   - sudo git fetch
   - sudo git fetch --prune
@@ -101,6 +105,7 @@ def init_ec2(region):
 
 def deploy(script, instance_type, region_prefix, shutdown_behaviour, instance_name, en_vars):
     res = ec2.run_instances(ImageId=en_vars[region_prefix + 'IMAGE_ID'],
+
                             InstanceType=instance_type,
                             UserData=script,
                             KeyName=en_vars[region_prefix + 'KEY_NAME'],
@@ -114,7 +119,7 @@ def deploy(script, instance_type, region_prefix, shutdown_behaviour, instance_na
                                 'Tags': [ {
                                     'Key': 'Name',
                                     'Value': instance_name
-                                } ]
+                                 }]
                             } ])
     return res['Instances'][0]['InstanceId']