From 6164a281421f4c617441aca63ab61c669ec4c1b0 Mon Sep 17 00:00:00 2001 From: Sam Greenbury Date: Tue, 7 Jan 2025 21:41:17 +0000 Subject: [PATCH 01/19] Adjust matching to account for common travel day --- config/base.toml | 2 +- scripts/2_match_households_and_individuals.py | 13 ++++++++- scripts/3.1_assign_primary_feasible_zones.py | 2 +- scripts/3.2.1_assign_primary_zone_edu.py | 2 +- scripts/3.2.2_assign_primary_zone_work.py | 2 +- scripts/3.2.3_assign_secondary_zone.py | 2 +- scripts/4_validation.py | 2 +- src/acbm/assigning/feasible_zones_primary.py | 2 +- src/acbm/config.py | 2 +- src/acbm/utils.py | 27 +++++++++++++++++++ tests/test_utils.py | 21 +++++++++++++++ 11 files changed, 68 insertions(+), 9 deletions(-) create mode 100644 tests/test_utils.py diff --git a/config/base.toml b/config/base.toml index 692c9b5..a8b8357 100644 --- a/config/base.toml +++ b/config/base.toml @@ -20,7 +20,7 @@ nts_regions = [ ] # nts day of the week to use # 1: Monday, 2: Tuesday, 3: Wednesday, 4: Thursday, 5: Friday, 6: Saturday, 7: Sunday -nts_day_of_week = 3 +nts_days_of_week = [3] # what crs do we want the output to be in? (just add the number, e.g. 3857) output_crs = 3857 diff --git a/scripts/2_match_households_and_individuals.py b/scripts/2_match_households_and_individuals.py index b86ca93..a3b7496 100644 --- a/scripts/2_match_households_and_individuals.py +++ b/scripts/2_match_households_and_individuals.py @@ -16,6 +16,7 @@ transform_by_group, truncate_values, ) +from acbm.utils import households_with_common_travel_days @acbm_cli @@ -237,8 +238,18 @@ def get_interim_path( nts_households = nts_filter_by_region(nts_households, psu, regions) nts_trips = nts_filter_by_region(nts_trips, psu, regions) - # Create dictionaries of key value pairs + # Ensure that the households have at least one day in `nts_days_of_week` that + # all household members have trips for + hids = households_with_common_travel_days( + nts_trips, config.parameters.nts_days_of_week + ) + # Subset individuals and households given filtering of trips + nts_trips = nts_trips[nts_trips["HouseholdID"].isin(hids)] + nts_individuals = nts_individuals[nts_individuals["HouseholdID"].isin(hids)] + nts_households = nts_households[nts_households["HouseholdID"].isin(hids)] + + # Create dictionaries of key value pairs """ guide to the dictionaries: diff --git a/scripts/3.1_assign_primary_feasible_zones.py b/scripts/3.1_assign_primary_feasible_zones.py index 392be8a..4da27c7 100644 --- a/scripts/3.1_assign_primary_feasible_zones.py +++ b/scripts/3.1_assign_primary_feasible_zones.py @@ -31,7 +31,7 @@ def main(config_file): # Filter to a specific day of the week logger.info("Filtering activity chains to a specific day of the week") activity_chains = activity_chains[ - activity_chains["TravDay"] == config.parameters.nts_day_of_week + activity_chains["TravDay"].isin(config.parameters.nts_days_of_week) ] # --- Study area boundaries diff --git a/scripts/3.2.1_assign_primary_zone_edu.py b/scripts/3.2.1_assign_primary_zone_edu.py index 183a46f..58a3822 100644 --- a/scripts/3.2.1_assign_primary_zone_edu.py +++ b/scripts/3.2.1_assign_primary_zone_edu.py @@ -54,7 +54,7 @@ def main(config_file): config, columns=cols_for_assignment_edu() ) activity_chains = activity_chains[ - activity_chains["TravDay"] == config.parameters.nts_day_of_week + activity_chains["TravDay"].isin(config.parameters.nts_days_of_week) ] logger.info("Filtering activity chains for trip purpose: education") diff --git a/scripts/3.2.2_assign_primary_zone_work.py b/scripts/3.2.2_assign_primary_zone_work.py index 1c5e98b..98256d1 100644 --- a/scripts/3.2.2_assign_primary_zone_work.py +++ b/scripts/3.2.2_assign_primary_zone_work.py @@ -45,7 +45,7 @@ def main(config_file): activity_chains = activity_chains_for_assignment(config, cols_for_assignment_work()) activity_chains = activity_chains[ - activity_chains["TravDay"] == config.parameters.nts_day_of_week + activity_chains["TravDay"].isin(config.parameters.nts_days_of_week) ] logger.info("Filtering activity chains for trip purpose: work") diff --git a/scripts/3.2.3_assign_secondary_zone.py b/scripts/3.2.3_assign_secondary_zone.py index a87ea10..66f1145 100644 --- a/scripts/3.2.3_assign_secondary_zone.py +++ b/scripts/3.2.3_assign_secondary_zone.py @@ -40,7 +40,7 @@ def main(config_file): activity_chains = activity_chains_for_assignment(config) activity_chains = activity_chains[ - activity_chains["TravDay"] == config.parameters.nts_day_of_week + activity_chains["TravDay"].isin(config.parameters.nts_days_of_week) ] # TODO: remove obsolete comment diff --git a/scripts/4_validation.py b/scripts/4_validation.py index ba246ce..ab64025 100644 --- a/scripts/4_validation.py +++ b/scripts/4_validation.py @@ -29,7 +29,7 @@ def main(config_file): # NTS data legs_nts = pd.read_parquet(config.output_path / "nts_trips.parquet") - legs_nts = legs_nts[legs_nts["TravDay"] == config.parameters.nts_day_of_week] + legs_nts = legs_nts[legs_nts["TravDay"].isin(config.parameters.nts_days_of_week)] # Model outputs legs_acbm = pd.read_csv(config.output_path / "legs.csv") diff --git a/src/acbm/assigning/feasible_zones_primary.py b/src/acbm/assigning/feasible_zones_primary.py index 9ef9f26..9c3eb85 100644 --- a/src/acbm/assigning/feasible_zones_primary.py +++ b/src/acbm/assigning/feasible_zones_primary.py @@ -25,7 +25,7 @@ activity_chains_schema = DataFrameSchema( { "mode": Column(str), - "TravDay": Column(pa.Float, Check.isin([1, 2, 3, 4, 5, 6, 7]), nullable=True), + # "TravDay": Column(pa.Float, Check.isin([1, 2, 3, 4, 5, 6, 7]), nullable=True), "tst": Column(pa.Float, Check.less_than_or_equal_to(1440), nullable=True), "TripTotalTime": Column(pa.Float, nullable=True), # TODO: add more columns ... diff --git a/src/acbm/config.py b/src/acbm/config.py index 477add7..4495700 100644 --- a/src/acbm/config.py +++ b/src/acbm/config.py @@ -26,7 +26,7 @@ class Parameters(BaseModel): boundary_geography: str nts_years: list[int] nts_regions: list[str] - nts_day_of_week: int + nts_days_of_week: list[int] output_crs: int tolerance_work: float | None = None tolerance_edu: float | None = None diff --git a/src/acbm/utils.py b/src/acbm/utils.py index 41fbed7..df8bdfc 100644 --- a/src/acbm/utils.py +++ b/src/acbm/utils.py @@ -39,3 +39,30 @@ def get_travel_times(config: Config, use_estimates: bool = False) -> pd.DataFram if config.parameters.travel_times and not use_estimates: return pd.read_parquet(config.travel_times_filepath) return pd.read_parquet(config.travel_times_estimates_filepath) + + +def households_with_common_travel_days( + nts_trips: pd.DataFrame, days: list[int] +) -> list[int]: + return ( + nts_trips.groupby(["HouseholdID", "IndividualID"])["TravDay"] + .apply(list) + .map(set) + .to_frame() + .groupby(["HouseholdID"])["TravDay"] + .apply( + lambda sets_of_days: set.intersection(*sets_of_days) + if set.intersection(*sets_of_days) + else None + ) + .to_frame()["TravDay"] + .apply( + lambda common_days: [day for day in common_days if day in days] + if common_days is not None + else [] + ) + .apply(lambda common_days: common_days if common_days else pd.NA) + .dropna() + .reset_index()["HouseholdID"] + .to_list() + ) diff --git a/tests/test_utils.py b/tests/test_utils.py new file mode 100644 index 0000000..0a90147 --- /dev/null +++ b/tests/test_utils.py @@ -0,0 +1,21 @@ +import pandas as pd +import pytest + +from acbm.utils import households_with_common_travel_days + + +@pytest.fixture +def nts_trips(): + return pd.DataFrame.from_dict( + { + "IndividualID": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10], + "HouseholdID": [1, 1, 1, 2, 2, 2, 3, 3, 3, 3], + "TravDay": [1, 1, 1, 2, 3, 2, 3, 3, 3, 3], + } + ) + + +def test_households_with_common_travel_days(nts_trips): + assert households_with_common_travel_days(nts_trips, [1]) == [1] + assert households_with_common_travel_days(nts_trips, [1, 2]) == [1] + assert households_with_common_travel_days(nts_trips, [1, 3]) == [1, 3] From 0cea61ffc671df25c628d747c0c115446ed7311b Mon Sep 17 00:00:00 2001 From: Sam Greenbury Date: Wed, 8 Jan 2025 16:22:50 +0000 Subject: [PATCH 02/19] Add 'DayID' to outputs --- src/acbm/assigning/utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/acbm/assigning/utils.py b/src/acbm/assigning/utils.py index d866d9e..efbd21e 100644 --- a/src/acbm/assigning/utils.py +++ b/src/acbm/assigning/utils.py @@ -18,6 +18,7 @@ def cols_for_assignment_all() -> list[str]: "age_years", "TripDisIncSW", "tet", + "DayID", ] From 88f4911febefc80e4d562f9461d9ad975afdd510 Mon Sep 17 00:00:00 2001 From: Sam Greenbury Date: Wed, 8 Jan 2025 17:43:24 +0000 Subject: [PATCH 03/19] Fix missing subset of nts_trips --- scripts/2_match_households_and_individuals.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scripts/2_match_households_and_individuals.py b/scripts/2_match_households_and_individuals.py index a3b7496..ab4f8fb 100644 --- a/scripts/2_match_households_and_individuals.py +++ b/scripts/2_match_households_and_individuals.py @@ -245,7 +245,10 @@ def get_interim_path( ) # Subset individuals and households given filtering of trips - nts_trips = nts_trips[nts_trips["HouseholdID"].isin(hids)] + nts_trips = nts_trips[ + nts_trips["HouseholdID"].isin(hids) + & nts_trips["TravDay"].isin(config.parameters.nts_days_of_week) + ] nts_individuals = nts_individuals[nts_individuals["HouseholdID"].isin(hids)] nts_households = nts_households[nts_households["HouseholdID"].isin(hids)] From e0fc2f9135a398b114f5b3b82acea33086d16ef4 Mon Sep 17 00:00:00 2001 From: Sam Greenbury Date: Wed, 8 Jan 2025 17:46:37 +0000 Subject: [PATCH 04/19] Add interim output with chosen random day for households --- scripts/3.1_assign_primary_feasible_zones.py | 21 ++++++++++++++++---- scripts/3.2.1_assign_primary_zone_edu.py | 5 +---- scripts/3.2.2_assign_primary_zone_work.py | 8 +++----- scripts/3.2.3_assign_secondary_zone.py | 5 +---- src/acbm/assigning/utils.py | 18 +++++++++++++---- 5 files changed, 36 insertions(+), 21 deletions(-) diff --git a/scripts/3.1_assign_primary_feasible_zones.py b/scripts/3.1_assign_primary_feasible_zones.py index 4da27c7..809c1b3 100644 --- a/scripts/3.1_assign_primary_feasible_zones.py +++ b/scripts/3.1_assign_primary_feasible_zones.py @@ -1,6 +1,7 @@ import pickle as pkl import geopandas as gpd +import numpy as np import pandas as pd from acbm.assigning.feasible_zones_primary import get_possible_zones @@ -28,11 +29,23 @@ def main(config_file): activity_chains = activity_chains_for_assignment(config) logger.info("Activity chains loaded") - # Filter to a specific day of the week logger.info("Filtering activity chains to a specific day of the week") - activity_chains = activity_chains[ - activity_chains["TravDay"].isin(config.parameters.nts_days_of_week) - ] + + # Generate random sample of days by household + activity_chains.merge( + activity_chains.groupby(["household"])["TravDay"] + .apply(np.unique) + .apply(np.random.choice) + .rename("ChosenTravDay"), + left_on=["household", "TravDay"], + right_on=["household", "ChosenTravDay"], + how="inner", + )[["id", "household", "TravDay"]].drop_duplicates().to_parquet( + config.output_path / "interim" / "assigning" / "chosen_trav_day.parquet" + ) + + # Filter to chosen day + activity_chains = activity_chains_for_assignment(config, subset_to_chosen_day=True) # --- Study area boundaries diff --git a/scripts/3.2.1_assign_primary_zone_edu.py b/scripts/3.2.1_assign_primary_zone_edu.py index 58a3822..d540eec 100644 --- a/scripts/3.2.1_assign_primary_zone_edu.py +++ b/scripts/3.2.1_assign_primary_zone_edu.py @@ -51,11 +51,8 @@ def main(config_file): logger.info("Loading activity chains") activity_chains = activity_chains_for_assignment( - config, columns=cols_for_assignment_edu() + config, columns=cols_for_assignment_edu(), subset_to_chosen_day=True ) - activity_chains = activity_chains[ - activity_chains["TravDay"].isin(config.parameters.nts_days_of_week) - ] logger.info("Filtering activity chains for trip purpose: education") activity_chains_edu = activity_chains[activity_chains["dact"] == "education"] diff --git a/scripts/3.2.2_assign_primary_zone_work.py b/scripts/3.2.2_assign_primary_zone_work.py index 98256d1..2cd561b 100644 --- a/scripts/3.2.2_assign_primary_zone_work.py +++ b/scripts/3.2.2_assign_primary_zone_work.py @@ -42,11 +42,9 @@ def main(config_file): # --- Activity chains logger.info("Loading activity chains") - - activity_chains = activity_chains_for_assignment(config, cols_for_assignment_work()) - activity_chains = activity_chains[ - activity_chains["TravDay"].isin(config.parameters.nts_days_of_week) - ] + activity_chains = activity_chains_for_assignment( + config, cols_for_assignment_work(), subset_to_chosen_day=True + ) logger.info("Filtering activity chains for trip purpose: work") activity_chains_work = activity_chains[activity_chains["dact"] == "work"] diff --git a/scripts/3.2.3_assign_secondary_zone.py b/scripts/3.2.3_assign_secondary_zone.py index 66f1145..0f8b045 100644 --- a/scripts/3.2.3_assign_secondary_zone.py +++ b/scripts/3.2.3_assign_secondary_zone.py @@ -38,10 +38,7 @@ def main(config_file): # --- Load in the data logger.info("Loading: activity chains") - activity_chains = activity_chains_for_assignment(config) - activity_chains = activity_chains[ - activity_chains["TravDay"].isin(config.parameters.nts_days_of_week) - ] + activity_chains = activity_chains_for_assignment(config, subset_to_chosen_day=True) # TODO: remove obsolete comment # --- Add OA21CD to the data diff --git a/src/acbm/assigning/utils.py b/src/acbm/assigning/utils.py index efbd21e..53ce213 100644 --- a/src/acbm/assigning/utils.py +++ b/src/acbm/assigning/utils.py @@ -11,7 +11,6 @@ def cols_for_assignment_all() -> list[str]: """Gets activity chains with subset of columns required for assignment.""" return [ *cols_for_assignment_edu(), - "household", "oact", "nts_ind_id", "nts_hh_id", @@ -25,12 +24,13 @@ def cols_for_assignment_all() -> list[str]: def cols_for_assignment_edu() -> list[str]: """Gets activity chains with subset of columns required for assignment.""" return [ + "id", + "household", "TravDay", "OA11CD", "dact", "mode", "tst", - "id", "seq", "TripTotalTime", "education_type", @@ -43,16 +43,26 @@ def cols_for_assignment_work() -> list[str]: def activity_chains_for_assignment( - config: Config, columns: list[str] | None = None + config: Config, columns: list[str] | None = None, subset_to_chosen_day: bool = False ) -> pd.DataFrame: """Gets activity chains with subset of columns required for assignment.""" if columns is None: columns = cols_for_assignment_all() - return pd.read_parquet( + activity_chains = pd.read_parquet( config.spc_with_nts_trips_filepath, columns=columns, ) + if not subset_to_chosen_day: + return activity_chains + + return activity_chains.merge( + pd.read_parquet( + config.output_path / "interim" / "assigning" / "chosen_trav_day.parquet" + ), + on=["id", "household", "TravDay"], + how="inner", + ) def _map_time_to_day_part( From ed45b922b3de2294b9971b4d10ae0a2fb3e32058 Mon Sep 17 00:00:00 2001 From: Sam Greenbury Date: Thu, 9 Jan 2025 18:40:49 +0000 Subject: [PATCH 05/19] Add todo --- scripts/2_match_households_and_individuals.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/2_match_households_and_individuals.py b/scripts/2_match_households_and_individuals.py index ab4f8fb..09b2eb2 100644 --- a/scripts/2_match_households_and_individuals.py +++ b/scripts/2_match_households_and_individuals.py @@ -234,6 +234,7 @@ def get_interim_path( regions = config.parameters.nts_regions + # TODO: Currently this only seems to work for 2019, check this nts_individuals = nts_filter_by_region(nts_individuals, psu, regions) nts_households = nts_filter_by_region(nts_households, psu, regions) nts_trips = nts_filter_by_region(nts_trips, psu, regions) From ce5a261d320f96a81dd6f3cacc8c99167929ce4a Mon Sep 17 00:00:00 2001 From: Sam Greenbury Date: Tue, 14 Jan 2025 18:13:34 +0000 Subject: [PATCH 06/19] Handle missing data and revise test for common trav days --- src/acbm/utils.py | 2 ++ tests/test_utils.py | 24 +++++++++++++++++++++--- 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/src/acbm/utils.py b/src/acbm/utils.py index df8bdfc..91469e1 100644 --- a/src/acbm/utils.py +++ b/src/acbm/utils.py @@ -59,6 +59,8 @@ def households_with_common_travel_days( .apply( lambda common_days: [day for day in common_days if day in days] if common_days is not None + and common_days != {pd.NA} + and common_days != {np.nan} else [] ) .apply(lambda common_days: common_days if common_days else pd.NA) diff --git a/tests/test_utils.py b/tests/test_utils.py index 0a90147..565ce31 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -8,9 +8,26 @@ def nts_trips(): return pd.DataFrame.from_dict( { - "IndividualID": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10], - "HouseholdID": [1, 1, 1, 2, 2, 2, 3, 3, 3, 3], - "TravDay": [1, 1, 1, 2, 3, 2, 3, 3, 3, 3], + "IndividualID": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16], + "HouseholdID": [1, 1, 1, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 5, 5, 5], + "TravDay": [ + 1, + 1, + 1, + 2, + 3, + 2, + 3, + 3, + 3, + 3, + pd.NA, + pd.NA, + pd.NA, + pd.NA, + pd.NA, + 4, + ], } ) @@ -19,3 +36,4 @@ def test_households_with_common_travel_days(nts_trips): assert households_with_common_travel_days(nts_trips, [1]) == [1] assert households_with_common_travel_days(nts_trips, [1, 2]) == [1] assert households_with_common_travel_days(nts_trips, [1, 3]) == [1, 3] + assert households_with_common_travel_days(nts_trips, [1, 3, 4]) == [1, 3] From ced648d280316467d465308703ad3847f05987fb Mon Sep 17 00:00:00 2001 From: Sam Greenbury Date: Tue, 14 Jan 2025 18:42:12 +0000 Subject: [PATCH 07/19] Fix test --- tests/test_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_config.py b/tests/test_config.py index 759816e..0d2372e 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -11,4 +11,4 @@ def config(): def test_id(config): - assert config.id == "0ebb8c3ee7" + assert config.id == "464741c848" From 5f17c5656760be59aec800523a9052165fa14a5b Mon Sep 17 00:00:00 2001 From: Sam Greenbury Date: Wed, 15 Jan 2025 17:07:30 +0000 Subject: [PATCH 08/19] Add chosen travel day modelling using pwkstat --- src/acbm/assigning/utils.py | 90 +++++++++++++++++++++++++++++++++++++ src/acbm/config.py | 5 +++ src/acbm/utils.py | 8 ++-- 3 files changed, 99 insertions(+), 4 deletions(-) diff --git a/src/acbm/assigning/utils.py b/src/acbm/assigning/utils.py index 53ce213..b099463 100644 --- a/src/acbm/assigning/utils.py +++ b/src/acbm/assigning/utils.py @@ -3,6 +3,7 @@ import geopandas as gpd import numpy as np import pandas as pd +import polars as pl from acbm.config import Config @@ -573,3 +574,92 @@ def replace_intrazonal_travel_time( # Return the modified DataFrame return travel_times_copy + + +def get_chosen_day(config: Config) -> pd.DataFrame: + """Gets the chosen day for population given config.""" + acs = pl.DataFrame(activity_chains_for_assignment(config)) + work_days = ( + acs.filter(pl.col("dact").eq("work")) + .group_by("id") + .agg(pl.col("TravDay").unique()) + .select(["id", pl.col("TravDay").list.drop_nulls().list.sample(n=1)]) + .explode("TravDay") + .rename({"TravDay": "TravDayWork"}) + ) + non_work_days = ( + acs.filter(~pl.col("dact").eq("work")) + .group_by("id") + .agg(pl.col("TravDay").unique()) + .select(["id", pl.col("TravDay").list.drop_nulls().list.sample(n=1)]) + .explode("TravDay") + .rename({"TravDay": "TravDayNonWork"}) + ) + + any_days = ( + acs.group_by("id") + .agg(pl.col("TravDay").unique()) + .select(["id", pl.col("TravDay").list.drop_nulls()]) + .select( + [ + "id", + pl.when(pl.col("TravDay").list.len() > 0) + # Note: this has to be set to with_replacement despite non-empty check + .then(pl.col("TravDay").list.sample(n=1, with_replacement=True)) + .otherwise(None), + ] + ) + .explode("TravDay") + .rename({"TravDay": "TravDayAny"}) + ).sort("id") + + # Combine day choices for different conditions + acs_combine = ( + acs.join(work_days, on="id", how="left", coalesce=True) + .join(non_work_days, on="id", how="left", coalesce=True) + .join(any_days, on="id", how="left", coalesce=True) + .join( + pl.scan_parquet(config.spc_combined_filepath) + .select(["id", "pwkstat"]) + .collect(), + on="id", + ) + ) + + # Choose a day given pwkstat + acs_combine = acs_combine.with_columns( + [ + # If pwkstat = 1 (full time) + # and a work travel day is available + pl.when(pl.col("pwkstat").eq(1) & pl.col("TravDayWork").is_not_null()) + .then(pl.col("TravDayWork")) + .otherwise( + # If pwkstat = 1 (full time) + # and a work travel day is NOT available + pl.when(pl.col("pwkstat").eq(1) & pl.col("TravDayWork").is_null()) + .then(pl.col("TravDayAny")) + .otherwise( + # If pwkstat = 2 (part time) + # and a work travel day is available + # and a non-work travel day is available + pl.when( + pl.col("pwkstat").eq(2) + & pl.col("TravDayWork").is_not_null() + & pl.col("TravDayNonWork").is_not_null() + ) + .then( + # Sample either TravDayWork or TravDayNonWork + # stochastically given config + pl.col("TravDayWork") + # TODO: update from config + if np.random.random() < 1 + else pl.col("TravDayNonWork") + ) + .otherwise(pl.col("TravDayAny")) + ) + ) + .alias("ChosenTravDay") + ] + ) + + return acs_combine.select(["id", "ChosenTravDay"]).unique().sort("id").to_pandas() diff --git a/src/acbm/config.py b/src/acbm/config.py index 4495700..be157be 100644 --- a/src/acbm/config.py +++ b/src/acbm/config.py @@ -9,6 +9,7 @@ import geopandas as gpd import jcs import numpy as np +import polars as pl import tomlkit from pydantic import BaseModel, Field, field_serializer, field_validator @@ -30,6 +31,8 @@ class Parameters(BaseModel): output_crs: int tolerance_work: float | None = None tolerance_edu: float | None = None + common_household_day: bool = False + part_time_work_prob: float = 0.7 @dataclass(frozen=True) @@ -359,6 +362,8 @@ def init_rng(self): try: np.random.seed(self.seed) random.seed(self.seed) + pl.set_random_seed(self.seed) + except Exception as err: msg = f"config does not provide a rng seed with err: {err}" raise ValueError(msg) from err diff --git a/src/acbm/utils.py b/src/acbm/utils.py index 91469e1..42a328a 100644 --- a/src/acbm/utils.py +++ b/src/acbm/utils.py @@ -42,14 +42,14 @@ def get_travel_times(config: Config, use_estimates: bool = False) -> pd.DataFram def households_with_common_travel_days( - nts_trips: pd.DataFrame, days: list[int] + nts_trips: pd.DataFrame, days: list[int], hid="HouseholdID", pid="IndividualID" ) -> list[int]: return ( - nts_trips.groupby(["HouseholdID", "IndividualID"])["TravDay"] + nts_trips.groupby([hid, pid])["TravDay"] .apply(list) .map(set) .to_frame() - .groupby(["HouseholdID"])["TravDay"] + .groupby([hid])["TravDay"] .apply( lambda sets_of_days: set.intersection(*sets_of_days) if set.intersection(*sets_of_days) @@ -65,6 +65,6 @@ def households_with_common_travel_days( ) .apply(lambda common_days: common_days if common_days else pd.NA) .dropna() - .reset_index()["HouseholdID"] + .reset_index()[hid] .to_list() ) From 4b6db8c4f619ad7aad41e667bf513d76eb41ba1d Mon Sep 17 00:00:00 2001 From: Sam Greenbury Date: Wed, 15 Jan 2025 18:09:47 +0000 Subject: [PATCH 09/19] Revise subsetting of households given trav day config --- scripts/2_match_households_and_individuals.py | 16 +++++-- scripts/3.1_assign_primary_feasible_zones.py | 12 +---- src/acbm/assigning/utils.py | 25 +++++++++- src/acbm/config.py | 2 +- src/acbm/utils.py | 34 ++++++++++++++ tests/test_config.py | 2 +- tests/test_utils.py | 46 +++++++++++++++++-- 7 files changed, 117 insertions(+), 20 deletions(-) diff --git a/scripts/2_match_households_and_individuals.py b/scripts/2_match_households_and_individuals.py index 09b2eb2..fd7d37e 100644 --- a/scripts/2_match_households_and_individuals.py +++ b/scripts/2_match_households_and_individuals.py @@ -16,7 +16,10 @@ transform_by_group, truncate_values, ) -from acbm.utils import households_with_common_travel_days +from acbm.utils import ( + households_with_common_travel_days, + households_with_travel_days_in_nts_weeks, +) @acbm_cli @@ -241,9 +244,14 @@ def get_interim_path( # Ensure that the households have at least one day in `nts_days_of_week` that # all household members have trips for - hids = households_with_common_travel_days( - nts_trips, config.parameters.nts_days_of_week - ) + if config.parameters.common_household_day: + hids = households_with_common_travel_days( + nts_trips, config.parameters.nts_days_of_week + ) + else: + hids = households_with_travel_days_in_nts_weeks( + nts_trips, config.parameters.nts_days_of_week + ) # Subset individuals and households given filtering of trips nts_trips = nts_trips[ diff --git a/scripts/3.1_assign_primary_feasible_zones.py b/scripts/3.1_assign_primary_feasible_zones.py index 809c1b3..6da9d26 100644 --- a/scripts/3.1_assign_primary_feasible_zones.py +++ b/scripts/3.1_assign_primary_feasible_zones.py @@ -1,13 +1,13 @@ import pickle as pkl import geopandas as gpd -import numpy as np import pandas as pd from acbm.assigning.feasible_zones_primary import get_possible_zones from acbm.assigning.utils import ( activity_chains_for_assignment, get_activities_per_zone, + get_chosen_day, intrazone_time, replace_intrazonal_travel_time, zones_to_time_matrix, @@ -32,15 +32,7 @@ def main(config_file): logger.info("Filtering activity chains to a specific day of the week") # Generate random sample of days by household - activity_chains.merge( - activity_chains.groupby(["household"])["TravDay"] - .apply(np.unique) - .apply(np.random.choice) - .rename("ChosenTravDay"), - left_on=["household", "TravDay"], - right_on=["household", "ChosenTravDay"], - how="inner", - )[["id", "household", "TravDay"]].drop_duplicates().to_parquet( + get_chosen_day(config).to_parquet( config.output_path / "interim" / "assigning" / "chosen_trav_day.parquet" ) diff --git a/src/acbm/assigning/utils.py b/src/acbm/assigning/utils.py index b099463..020a5c4 100644 --- a/src/acbm/assigning/utils.py +++ b/src/acbm/assigning/utils.py @@ -579,6 +579,23 @@ def replace_intrazonal_travel_time( def get_chosen_day(config: Config) -> pd.DataFrame: """Gets the chosen day for population given config.""" acs = pl.DataFrame(activity_chains_for_assignment(config)) + + if config.parameters.common_household_day: + return ( + acs.join( + acs.group_by("household") + .agg(pl.col("TravDay").unique().sample(1, with_replacement=True)) + .explode("TravDay"), + on=["household", "TravDay"], + how="inner", + ) + .select(["id", "TravDay"]) + .unique() + .sort("id") + .to_pandas() + ) + + # For any TravDay and modelling increased households work_days = ( acs.filter(pl.col("dact").eq("work")) .group_by("id") @@ -662,4 +679,10 @@ def get_chosen_day(config: Config) -> pd.DataFrame: ] ) - return acs_combine.select(["id", "ChosenTravDay"]).unique().sort("id").to_pandas() + return ( + acs_combine.select(["id", "ChosenTravDay"]) + .unique() + .rename({"ChosenTravDay": "TravDay"}) + .sort("id") + .to_pandas() + ) diff --git a/src/acbm/config.py b/src/acbm/config.py index be157be..ed3daf4 100644 --- a/src/acbm/config.py +++ b/src/acbm/config.py @@ -31,7 +31,7 @@ class Parameters(BaseModel): output_crs: int tolerance_work: float | None = None tolerance_edu: float | None = None - common_household_day: bool = False + common_household_day: bool = True part_time_work_prob: float = 0.7 diff --git a/src/acbm/utils.py b/src/acbm/utils.py index 42a328a..5703840 100644 --- a/src/acbm/utils.py +++ b/src/acbm/utils.py @@ -1,5 +1,6 @@ import numpy as np import pandas as pd +import polars as pl from sklearn.metrics import mean_squared_error from acbm.config import Config @@ -68,3 +69,36 @@ def households_with_common_travel_days( .reset_index()[hid] .to_list() ) + + +def households_with_travel_days_in_nts_weeks( + nts_trips: pd.DataFrame, days: list[int], hid="HouseholdID", pid="IndividualID" +) -> list[int]: + return ( + pl.DataFrame(nts_trips) + .group_by([hid, pid]) + .agg(pl.col("TravDay").unique()) + .select( + [ + hid, + pid, + pl.col("TravDay").list.drop_nulls().list.set_intersection(pl.lit(days)), + ] + ) + .select( + [ + hid, + pid, + pl.when(pl.col("TravDay").list.len().eq(0)) + .then(None) + .otherwise(pl.col("TravDay")) + .alias("TravDay"), + ] + ) + .group_by(hid) + .agg(pl.col("TravDay").list.len().ne(0).all()) + .filter(pl.col("TravDay").eq(True)) + .get_column(hid) + .sort() + .to_list() + ) diff --git a/tests/test_config.py b/tests/test_config.py index 0d2372e..878afd3 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -11,4 +11,4 @@ def config(): def test_id(config): - assert config.id == "464741c848" + assert config.id == "5709d35b6f" diff --git a/tests/test_utils.py b/tests/test_utils.py index 565ce31..5da7ef9 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,15 +1,39 @@ import pandas as pd import pytest -from acbm.utils import households_with_common_travel_days +from acbm.utils import ( + households_with_common_travel_days, + households_with_travel_days_in_nts_weeks, +) @pytest.fixture def nts_trips(): return pd.DataFrame.from_dict( { - "IndividualID": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16], - "HouseholdID": [1, 1, 1, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 5, 5, 5], + "IndividualID": [ + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + ], + "HouseholdID": [1, 1, 1, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 5, 5, 5, 6, 6, 7, 7], "TravDay": [ 1, 1, @@ -27,6 +51,10 @@ def nts_trips(): pd.NA, pd.NA, 4, + 4, + 5, + 5, + pd.NA, ], } ) @@ -37,3 +65,15 @@ def test_households_with_common_travel_days(nts_trips): assert households_with_common_travel_days(nts_trips, [1, 2]) == [1] assert households_with_common_travel_days(nts_trips, [1, 3]) == [1, 3] assert households_with_common_travel_days(nts_trips, [1, 3, 4]) == [1, 3] + + +def test_households_with_travel_days_in_nts_weeks(nts_trips): + assert households_with_travel_days_in_nts_weeks(nts_trips, [1]) == [1] + assert households_with_travel_days_in_nts_weeks(nts_trips, [1, 2]) == [1] + assert households_with_travel_days_in_nts_weeks(nts_trips, [1, 3]) == [1, 3] + assert households_with_travel_days_in_nts_weeks(nts_trips, [1, 3, 4]) == [1, 3] + assert households_with_travel_days_in_nts_weeks(nts_trips, [1, 3, 4, 5]) == [ + 1, + 3, + 6, + ] From 1f5e0b94604b2fbca13b42123695e3a7e395f8e6 Mon Sep 17 00:00:00 2001 From: Sam Greenbury Date: Wed, 15 Jan 2025 18:19:08 +0000 Subject: [PATCH 10/19] Fix test --- tests/test_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_config.py b/tests/test_config.py index 878afd3..7f12069 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -11,4 +11,4 @@ def config(): def test_id(config): - assert config.id == "5709d35b6f" + assert config.id == "21e42c9d68" From 4ce4b1a3b4cbe394bfda7657850c017c81d27349 Mon Sep 17 00:00:00 2001 From: Sam Greenbury Date: Wed, 15 Jan 2025 18:35:59 +0000 Subject: [PATCH 11/19] Add logging for NTS filtering --- scripts/2_match_households_and_individuals.py | 10 +++++-- src/acbm/preprocessing.py | 27 +++++++++++++++++-- 2 files changed, 33 insertions(+), 4 deletions(-) diff --git a/scripts/2_match_households_and_individuals.py b/scripts/2_match_households_and_individuals.py index fd7d37e..4025cbb 100644 --- a/scripts/2_match_households_and_individuals.py +++ b/scripts/2_match_households_and_individuals.py @@ -226,22 +226,28 @@ def get_interim_path( logger.info("Filtering NTS data by specified year(s)") + logger.info(f"Total NTS households: {nts_households.shape[0]:,.0f}") years = config.parameters.nts_years nts_individuals = nts_filter_by_year(nts_individuals, psu, years) nts_households = nts_filter_by_year(nts_households, psu, years) nts_trips = nts_filter_by_year(nts_trips, psu, years) + logger.info( + f"Total NTS households (after year filtering): {nts_households.shape[0]:,.0f}" + ) # #### Filter by geography - # regions = config.parameters.nts_regions - # TODO: Currently this only seems to work for 2019, check this nts_individuals = nts_filter_by_region(nts_individuals, psu, regions) nts_households = nts_filter_by_region(nts_households, psu, regions) nts_trips = nts_filter_by_region(nts_trips, psu, regions) + logger.info( + f"Total NTS households (after region filtering): {nts_households.shape[0]:,.0f}" + ) + # Ensure that the households have at least one day in `nts_days_of_week` that # all household members have trips for if config.parameters.common_household_day: diff --git a/src/acbm/preprocessing.py b/src/acbm/preprocessing.py index 7601659..d6d0bdb 100644 --- a/src/acbm/preprocessing.py +++ b/src/acbm/preprocessing.py @@ -70,11 +70,10 @@ def nts_filter_by_year( data: pandas DataFrame The NTS data to be filtered - psu: pandas DataFrame - The Primary Sampling Unit table in the NTS. It has the year years: list The chosen year(s) """ + # return data.loc[data["SurveyYear"].isin(years)] # Check that all values of 'years' exist in the 'SurveyYear' column of 'psu' # Get the unique years in the 'SurveyYear' column of 'psu' @@ -111,6 +110,7 @@ def nts_filter_by_region( # 1. Create a column in the PSU table with the region names # Dictionary of the regions in the NTS and how they are coded + # PSUGOR_B02ID but does not have values for 2021 and 2022 region_dict = { -10.0: "DEAD", -9.0: "DNA", @@ -127,8 +127,31 @@ def nts_filter_by_region( 10.0: "Wales", 11.0: "Scotland", } + # PSUStatsReg_B01ID but does not have values for 2021 and 2022 + # region_dict = { + # -10.0: "DEAD", + # -9.0: "DNA", + # -8.0: "NA", + # 1.0: "Northern, Metropolitan", + # 2.0: "Northern, Non-metropolitan", + # 3.0: "Yorkshire / Humberside, Metropolitan", + # 4.0: "Yorkshire / Humberside, Non-metropolitan", + # 5.0: "East Midlands", + # 6.0: "East Anglia", + # 7.0: "South East (excluding London Boroughs)", + # 8.0: "London Boroughs", + # 9.0: "South West", + # 10.0: "West Midlands, Metropolitan", + # 11.0: "West Midlands, Non-metropolitan", + # 12.0: "North West, Metropolitan", + # 13.0: "North West, Non-metropolitan", + # 14.0: "Wales", + # 15.0: "Scotland", + # } + # In the PSU table, create a column with the region names psu["region_name"] = psu["PSUGOR_B02ID"].map(region_dict) + # psu["region_name"] = psu["PSUStatsReg_B01ID"].map(region_dict) # 2. Check that all values of 'years' exist in the 'SurveyYear' column of 'psu' From 46946d17d335b218ee82db13b726a3696a24ca8e Mon Sep 17 00:00:00 2001 From: Sam Greenbury Date: Wed, 15 Jan 2025 18:36:52 +0000 Subject: [PATCH 12/19] Fix merge columns --- src/acbm/assigning/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/acbm/assigning/utils.py b/src/acbm/assigning/utils.py index 020a5c4..614612e 100644 --- a/src/acbm/assigning/utils.py +++ b/src/acbm/assigning/utils.py @@ -61,7 +61,7 @@ def activity_chains_for_assignment( pd.read_parquet( config.output_path / "interim" / "assigning" / "chosen_trav_day.parquet" ), - on=["id", "household", "TravDay"], + on=["id", "TravDay"], how="inner", ) From 43014932a25a55a05c94d6139da045b6a378514e Mon Sep 17 00:00:00 2001 From: Sam Greenbury Date: Wed, 15 Jan 2025 21:54:06 +0000 Subject: [PATCH 13/19] Add matching for remaining individuals --- scripts/2_match_households_and_individuals.py | 15 ++++- src/acbm/matching.py | 59 +++++++++++++++++++ 2 files changed, 73 insertions(+), 1 deletion(-) diff --git a/scripts/2_match_households_and_individuals.py b/scripts/2_match_households_and_individuals.py index 4025cbb..546977d 100644 --- a/scripts/2_match_households_and_individuals.py +++ b/scripts/2_match_households_and_individuals.py @@ -7,7 +7,7 @@ from acbm.assigning.utils import cols_for_assignment_all from acbm.cli import acbm_cli from acbm.config import load_and_setup_config -from acbm.matching import MatcherExact, match_individuals +from acbm.matching import MatcherExact, match_individuals, match_remaining_individuals from acbm.preprocessing import ( count_per_group, nts_filter_by_region, @@ -953,6 +953,19 @@ def get_interim_path( show_progress=True, ) + # match remaining individuals + remaining_ids = spc_edited.loc[ + ~spc_edited.index.isin(matches_ind.keys()), "id" + ].to_list() + matches_remaining_ind = match_remaining_individuals( + df1=spc_edited, + df2=nts_individuals, + matching_columns=["age_group", "sex"], + remaining_ids=remaining_ids, + show_progress=True, + ) + matches_ind.update(matches_remaining_ind) + # save random sample with open( get_interim_path("matches_ind_level_categorical_random_sample.pkl"), "wb" diff --git a/src/acbm/matching.py b/src/acbm/matching.py index 4c0a5b1..50b05e2 100644 --- a/src/acbm/matching.py +++ b/src/acbm/matching.py @@ -294,3 +294,62 @@ def match_individuals( matches.update(match) return matches + + +def match_remaining_individuals( + df1: pd.DataFrame, + df2: pd.DataFrame, + matching_columns: list, + remaining_ids: list[int], + show_progress: bool = False, +) -> dict: + """ + Apply a matching function iteratively to members of each household. + In each iteration, filter df1 and df2 to the household ids of item i + in matches_hh, and then apply the matching function to the filtered DataFrames. + + Parameters + ---------- + df1: pandas DataFrame + The first DataFrame to be matched on + df2: pandas DataFrame + The second DataFrame to be matched with + matching_columns: list + The columns to be used for the matching + matches_hh: dict + A dictionary with the matched household ids {df1_id: df2_id} + show_progress: bool + Whether to print the progress of the matching to the console + + Returns + ------- + matches: dict + A dictionary with the matched row indeces from the two DataFrames {df1: df2} + + """ + # Initialize an empty dic to store the matches + matches = {} + + # loop over all groups of df1_id + # note: for large populations looping through the groups (keys) of the + # large dataframe (assumed to be df1) is more efficient than looping + # over keys and subsetting on a key in each iteration. + df1_remaining = df1.loc[df1["id"].isin(remaining_ids)] + chunk_size = 1000 + for i, rows_df1 in df1_remaining.groupby( + np.arange(len(df1_remaining)) // chunk_size + ): + rows_df2 = df2 + if show_progress: + # Print the iteration number and the number of keys in the dict + print( + f"Matching remaining individuals, {i * chunk_size} out of: {len(remaining_ids)}" + ) + + # apply the matching + match = match_psm(rows_df1, rows_df2, matching_columns) + + # append the results to the main dict + matches.update(match) + + return matches From be2eb6b8f5176079fea5d9b49de8393a6c48b87e Mon Sep 17 00:00:00 2001 From: Sam Greenbury Date: Wed, 15 Jan 2025 21:55:25 +0000 Subject: [PATCH 14/19] Revise region variable to 'PSUStatsReg_B01ID' --- src/acbm/preprocessing.py | 67 ++++++++++++++++++++------------------- 1 file changed, 34 insertions(+), 33 deletions(-) diff --git a/src/acbm/preprocessing.py b/src/acbm/preprocessing.py index d6d0bdb..4d8aa7e 100644 --- a/src/acbm/preprocessing.py +++ b/src/acbm/preprocessing.py @@ -111,47 +111,48 @@ def nts_filter_by_region( # Dictionary of the regions in the NTS and how they are coded # PSUGOR_B02ID but does not have values for 2021 and 2022 - region_dict = { - -10.0: "DEAD", - -9.0: "DNA", - -8.0: "NA", - 1.0: "North East", - 2.0: "North West", - 3.0: "Yorkshire and the Humber", - 4.0: "East Midlands", - 5.0: "West Midlands", - 6.0: "East of England", - 7.0: "London", - 8.0: "South East", - 9.0: "South West", - 10.0: "Wales", - 11.0: "Scotland", - } - # PSUStatsReg_B01ID but does not have values for 2021 and 2022 # region_dict = { # -10.0: "DEAD", # -9.0: "DNA", # -8.0: "NA", - # 1.0: "Northern, Metropolitan", - # 2.0: "Northern, Non-metropolitan", - # 3.0: "Yorkshire / Humberside, Metropolitan", - # 4.0: "Yorkshire / Humberside, Non-metropolitan", - # 5.0: "East Midlands", - # 6.0: "East Anglia", - # 7.0: "South East (excluding London Boroughs)", - # 8.0: "London Boroughs", + # 1.0: "North East", + # 2.0: "North West", + # 3.0: "Yorkshire and the Humber", + # 4.0: "East Midlands", + # 5.0: "West Midlands", + # 6.0: "East of England", + # 7.0: "London", + # 8.0: "South East", # 9.0: "South West", - # 10.0: "West Midlands, Metropolitan", - # 11.0: "West Midlands, Non-metropolitan", - # 12.0: "North West, Metropolitan", - # 13.0: "North West, Non-metropolitan", - # 14.0: "Wales", - # 15.0: "Scotland", + # 10.0: "Wales", + # 11.0: "Scotland", # } + # PSUStatsReg_B01ID but does not have values for 2021 and 2022 + region_dict = { + -10.0: "DEAD", + -9.0: "DNA", + -8.0: "NA", + 1.0: "Northern, Metropolitan", + 2.0: "Northern, Non-metropolitan", + 3.0: "Yorkshire / Humberside, Metropolitan", + 4.0: "Yorkshire / Humberside, Non-metropolitan", + 5.0: "East Midlands", + 6.0: "East Anglia", + 7.0: "South East (excluding London Boroughs)", + 8.0: "London Boroughs", + 9.0: "South West", + 10.0: "West Midlands, Metropolitan", + 11.0: "West Midlands, Non-metropolitan", + 12.0: "North West, Metropolitan", + 13.0: "North West, Non-metropolitan", + 14.0: "Wales", + 15.0: "Scotland", + } + # In the PSU table, create a column with the region names - psu["region_name"] = psu["PSUGOR_B02ID"].map(region_dict) - # psu["region_name"] = psu["PSUStatsReg_B01ID"].map(region_dict) + # psu["region_name"] = psu["PSUGOR_B02ID"].map(region_dict) + psu["region_name"] = psu["PSUStatsReg_B01ID"].map(region_dict) # 2. Check that all values of 'years' exist in the 'SurveyYear' column of 'psu' From 748260a1d0231e47112b3efb2b4eefe4599e8177 Mon Sep 17 00:00:00 2001 From: Sam Greenbury Date: Wed, 15 Jan 2025 22:23:04 +0000 Subject: [PATCH 15/19] Fix logging --- src/acbm/matching.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/acbm/matching.py b/src/acbm/matching.py index 50b05e2..595fda6 100644 --- a/src/acbm/matching.py +++ b/src/acbm/matching.py @@ -342,7 +342,7 @@ def match_remaining_individuals( rows_df2 = df2 if show_progress: # Print the iteration number and the number of keys in the dict - print( + logger.info( f"Matching remaining individuals, {i * chunk_size} out of: {len(remaining_ids)}" ) From 5105dab073ea2108d42854de46b2bd1bcee7a41c Mon Sep 17 00:00:00 2001 From: Sam Greenbury <50113363+sgreenbury@users.noreply.github.com> Date: Thu, 16 Jan 2025 18:04:37 +0000 Subject: [PATCH 16/19] Remove obsolete comment --- src/acbm/preprocessing.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/acbm/preprocessing.py b/src/acbm/preprocessing.py index 4d8aa7e..6b0a0ab 100644 --- a/src/acbm/preprocessing.py +++ b/src/acbm/preprocessing.py @@ -73,7 +73,6 @@ def nts_filter_by_year( years: list The chosen year(s) """ - # return data.loc[data["SurveyYear"].isin(years)] # Check that all values of 'years' exist in the 'SurveyYear' column of 'psu' # Get the unique years in the 'SurveyYear' column of 'psu' From 32dfe63e59a0a0a956ea4c3e2c99d7d091ddac2e Mon Sep 17 00:00:00 2001 From: Sam Greenbury Date: Thu, 16 Jan 2025 18:23:17 +0000 Subject: [PATCH 17/19] Uncomment code --- src/acbm/assigning/feasible_zones_primary.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/acbm/assigning/feasible_zones_primary.py b/src/acbm/assigning/feasible_zones_primary.py index 9c3eb85..9ef9f26 100644 --- a/src/acbm/assigning/feasible_zones_primary.py +++ b/src/acbm/assigning/feasible_zones_primary.py @@ -25,7 +25,7 @@ activity_chains_schema = DataFrameSchema( { "mode": Column(str), - # "TravDay": Column(pa.Float, Check.isin([1, 2, 3, 4, 5, 6, 7]), nullable=True), + "TravDay": Column(pa.Float, Check.isin([1, 2, 3, 4, 5, 6, 7]), nullable=True), "tst": Column(pa.Float, Check.less_than_or_equal_to(1440), nullable=True), "TripTotalTime": Column(pa.Float, nullable=True), # TODO: add more columns ... From 6ebd5be1a4370503a6c339b26addbc4bb42e30ca Mon Sep 17 00:00:00 2001 From: Sam Greenbury Date: Thu, 16 Jan 2025 20:37:30 +0000 Subject: [PATCH 18/19] Limit the number of processes to proportion of cpu_count() --- src/acbm/assigning/select_facility.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/acbm/assigning/select_facility.py b/src/acbm/assigning/select_facility.py index 28a9e0d..485544e 100644 --- a/src/acbm/assigning/select_facility.py +++ b/src/acbm/assigning/select_facility.py @@ -1,5 +1,5 @@ import logging -from multiprocessing import Pool +from multiprocessing import Pool, cpu_count from typing import Optional, Tuple import geopandas as gpd @@ -204,9 +204,9 @@ def select_facility( dict[str, Tuple[str, Point ] | Tuple[float, float]]: Unique ID column as keys with selected facility ID and facility ID's geometry, or (np.nan, np.nan) """ - # TODO: update this to be configurable, `None` is os.process_cpu_count() # TODO: check if this is deterministic for a given seed (or pass seed to pool) - with Pool(None) as p: + # TODO: update to be configurable + with Pool(max(int(cpu_count() * 0.75), 1)) as p: # Set to a large enough chunk size so that each process # has a sufficiently large amount of processing to do. chunk_size = 16_000 From 1dd281b5936cb2f6b3624947723054cd8b594f2f Mon Sep 17 00:00:00 2001 From: Sam Greenbury Date: Mon, 20 Jan 2025 22:06:48 +0000 Subject: [PATCH 19/19] Only consider working not from home --- scripts/3.2.2_assign_primary_zone_work.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/3.2.2_assign_primary_zone_work.py b/scripts/3.2.2_assign_primary_zone_work.py index 2cd561b..30ae0b9 100644 --- a/scripts/3.2.2_assign_primary_zone_work.py +++ b/scripts/3.2.2_assign_primary_zone_work.py @@ -98,7 +98,7 @@ def main(config_file): logger.info("Step 4: Filtering rows and dropping unnecessary columns") travel_demand_clipped = travel_demand[ - travel_demand["Place of work indicator (4 categories) code"].isin([1, 3]) + travel_demand["Place of work indicator (4 categories) code"].isin([3]) ] travel_demand_clipped = travel_demand_clipped.drop( columns=[ @@ -139,7 +139,7 @@ def main(config_file): logger.info("Step 2: Filtering rows and dropping unnecessary columns") travel_demand_clipped = travel_demand[ - travel_demand["Place of work indicator (4 categories) code"].isin([1, 3]) + travel_demand["Place of work indicator (4 categories) code"].isin([3]) ] travel_demand_clipped = travel_demand_clipped.drop( columns=[