From c6cb179d10b9959d9d20dd09979c4e230c1337bc Mon Sep 17 00:00:00 2001 From: Creatune Date: Mon, 27 Jan 2020 17:01:38 +0530 Subject: [PATCH] Port To Python 3 --- baus.py | 20 ++--- baus/datasources.py | 40 +++++----- baus/earthquake.py | 18 ++--- baus/models.py | 124 ++++++++++++++--------------- baus/preprocessing.py | 20 ++--- baus/slr.py | 14 ++-- baus/subsidies.py | 103 ++++++++++++------------ baus/summaries.py | 6 +- baus/ual.py | 62 +++++++-------- baus/utils.py | 22 ++--- baus/validation.py | 14 ++-- baus/variables.py | 4 +- scripts/average_runs.py | 11 +-- scripts/backout_zoning.py | 6 +- scripts/check_data.py | 22 ++--- scripts/check_emp_totals.py | 2 +- scripts/check_hh_totals.py | 2 +- scripts/check_rhna.py | 6 +- scripts/county_summaries.py | 2 +- scripts/export.py | 2 +- scripts/filter_large_projects.py | 2 +- scripts/fix_zoning_missing_id.py | 22 ++--- scripts/import_mongo.py | 4 +- scripts/make_net_from_shapefile.py | 8 +- scripts/make_shapefile.py | 2 +- scripts/match_city_totals.py | 28 +++---- scripts/output_csv_utils.py | 6 +- scripts/parcel_output_diff.py | 8 +- scripts/parcel_tract_assignment.py | 6 +- scripts/serve_json.py | 8 +- scripts/vmt_compare.py | 6 +- 31 files changed, 301 insertions(+), 299 deletions(-) diff --git a/baus.py b/baus.py index bf956b172..bbaee716c 100644 --- a/baus.py +++ b/baus.py @@ -105,8 +105,8 @@ run_num = orca.get_injectable("run_number") if LOGS: - print '***The Standard stream is being written to /runs/run{0}.log***'\ - .format(run_num) + print('***The Standard stream is being written to /runs/run{0}.log***'\ + .format(run_num)) sys.stdout = sys.stderr = open("runs/run%d.log" % run_num, 'w') if SLACK: @@ -304,8 +304,8 @@ def run_models(MODE, SCENARIO): # start the simulation in the next round - only the models above run # for the IN_YEAR - years_to_run = range(IN_YEAR+EVERY_NTH_YEAR, OUT_YEAR+1, - EVERY_NTH_YEAR) + years_to_run = list(range(IN_YEAR+EVERY_NTH_YEAR, OUT_YEAR+1, + EVERY_NTH_YEAR)) models = get_simulation_models(SCENARIO) orca.run(models, iter_vars=years_to_run) @@ -364,10 +364,10 @@ def run_models(MODE, SCENARIO): raise "Invalid mode" -print "Started", time.ctime() -print "Current Branch : ", BRANCH.rstrip() -print "Current Commit : ", CURRENT_COMMIT.rstrip() -print "Current Scenario : ", orca.get_injectable('scenario').rstrip() +print("Started", time.ctime()) +print("Current Branch : ", BRANCH.rstrip()) +print("Current Commit : ", CURRENT_COMMIT.rstrip()) +print("Current Scenario : ", orca.get_injectable('scenario').rstrip()) if SLACK: @@ -381,7 +381,7 @@ def run_models(MODE, SCENARIO): run_models(MODE, SCENARIO) except Exception as e: - print traceback.print_exc() + print(traceback.print_exc()) if SLACK: slack.chat.post_message( '#sim_updates', @@ -391,7 +391,7 @@ def run_models(MODE, SCENARIO): raise e sys.exit(0) -print "Finished", time.ctime() +print("Finished", time.ctime()) if MAPS: diff --git a/baus/datasources.py b/baus/datasources.py index 4340847bf..1317b8ee1 100644 --- a/baus/datasources.py +++ b/baus/datasources.py @@ -5,9 +5,9 @@ from urbansim_defaults import utils from urbansim.utils import misc import orca -import preprocessing -from utils import geom_id_to_parcel_id, parcel_id_to_geom_id -from utils import nearest_neighbor +from . import preprocessing +from .utils import geom_id_to_parcel_id, parcel_id_to_geom_id +from .utils import nearest_neighbor ##################### @@ -49,19 +49,19 @@ def limits_settings(settings, scenario): d = settings['development_limits'] - if scenario in d.keys(): - print "Using limits for scenario: %s" % scenario + if scenario in list(d.keys()): + print("Using limits for scenario: %s" % scenario) assert "default" in d d_scen = d[scenario] d = d["default"] - for key, value in d_scen.iteritems(): + for key, value in d_scen.items(): d.setdefault(key, {}) d[key].update(value) return d - print "Using default limits" + print("Using default limits") return d["default"] @@ -72,20 +72,20 @@ def inclusionary_housing_settings(settings, scenario): s = settings['inclusionary_housing_settings'] - if scenario in s.keys(): - print "Using inclusionary settings for scenario: %s" % scenario + if scenario in list(s.keys()): + print("Using inclusionary settings for scenario: %s" % scenario) s = s[scenario] - elif "default" in s.keys(): - print "Using default inclusionary settings" + elif "default" in list(s.keys()): + print("Using default inclusionary settings") s = s["default"] d = {} for item in s: # this is a list of cities with an inclusionary rate that is the # same for all the cities in the list - print "Setting inclusionary rates for %d cities to %.2f" %\ - (len(item["values"]), item["amount"]) + print("Setting inclusionary rates for %d cities to %.2f" %\ + (len(item["values"]), item["amount"])) # this is a list of inclusionary rates and the cities they apply # to - need tro turn it in a map of city names to rates for juris in item["values"]: @@ -176,7 +176,7 @@ def fetch_from_s3(settings): file = os.path.join("data", file) if os.path.exists(file): continue - print "Downloading " + file + print("Downloading " + file) key = bucket.get_key(file, validate=False) key.get_contents_to_filename(file) @@ -353,11 +353,11 @@ def zoning_scenario(parcels_geography, scenario, settings): scenario_zoning = pd.read_csv( os.path.join(misc.data_dir(), 'zoning_mods_%s.csv' % scenario)) - for k in settings["building_type_map"].keys(): + for k in list(settings["building_type_map"].keys()): scenario_zoning[k] = np.nan def add_drop_helper(col, val): - for ind, item in scenario_zoning[col].iteritems(): + for ind, item in scenario_zoning[col].items(): if not isinstance(item, str): continue for btype in item.split(): @@ -556,7 +556,7 @@ def get_dev_projects_table(scenario, parcels): cnts = df.geom_id.isin(parcels.geom_id).value_counts() if False in cnts.index: - print "%d MISSING GEOMIDS!" % cnts.loc[False] + print("%d MISSING GEOMIDS!" % cnts.loc[False]) df = df[df.geom_id.isin(parcels.geom_id)] @@ -595,7 +595,7 @@ def development_projects(parcels, settings, scenario): df["building_type"] = df.building_type.replace("GV", "OF") df["building_type"] = df.building_type.replace("SC", "OF") - building_types = settings["building_type_map"].keys() + building_types = list(settings["building_type_map"].keys()) # only deal with building types we recorgnize # otherwise hedonics break df = df[df.building_type.isin(building_types)] @@ -607,10 +607,10 @@ def development_projects(parcels, settings, scenario): df = df.dropna(subset=["year_built"]) df = df[df.action.isin(["add", "build"])] - print "Describe of development projects" + print("Describe of development projects") # this makes sure dev projects has all the same columns as buildings # which is the point of this method - print df[orca.get_table('buildings').local_columns].describe() + print(df[orca.get_table('buildings').local_columns].describe()) return df diff --git a/baus/earthquake.py b/baus/earthquake.py index 5a763f636..eb5a786e6 100644 --- a/baus/earthquake.py +++ b/baus/earthquake.py @@ -2,9 +2,9 @@ import numpy as np import pandas as pd from urbansim_defaults import utils -import datasources -import variables -import summaries +from . import datasources +from . import variables +from . import summaries from operator import itemgetter import itertools @@ -250,7 +250,7 @@ def earthquake_demolish(parcels, parcels_tract, tracts_earthquake, buildings, # using the lookup table created with "parcel_tract_assignment.ipynb" census_tract = pd.Series(parcels_tract['census_tract'], parcels_tract.index) - print "Number of parcels with census tracts is: %d" % len(census_tract) + print("Number of parcels with census tracts is: %d" % len(census_tract)) orca.add_column('parcels', 'tract', census_tract) # group parcels by their census tract @@ -265,7 +265,7 @@ def earthquake_demolish(parcels, parcels_tract, tracts_earthquake, buildings, key=itemgetter(0)): tract_parcels_grp.append(list(parcels)) tracts.append(tract) - print "Number of census tract groups is: %d" % len(tract_parcels_grp) + print("Number of census tract groups is: %d" % len(tract_parcels_grp)) # for the parcels in each tract, destroy X% of parcels in that tract tracts_earthquake = tracts_earthquake.to_frame() @@ -340,8 +340,8 @@ def earthquake_demolish(parcels, parcels_tract, tracts_earthquake, buildings, fire_buildings.extend(buildings_fire) eq_buildings.extend(buildings_fire) - print "Total number of buildings being destroyed is: %d" \ - % len(eq_buildings) + print("Total number of buildings being destroyed is: %d" \ + % len(eq_buildings)) orca.add_injectable("eq_buildings", eq_buildings) orca.add_injectable("existing_buildings", existing_buildings) @@ -353,7 +353,7 @@ def earthquake_demolish(parcels, parcels_tract, tracts_earthquake, buildings, eq_demolish = buildings.local[buildings.index.isin (eq_buildings)] orca.add_table("eq_demolish", eq_demolish) - print "Demolishing %d buildings" % len(eq_demolish) + print("Demolishing %d buildings" % len(eq_demolish)) households = households.to_frame() hh_unplaced = households[households["building_id"] == -1] @@ -383,4 +383,4 @@ def earthquake_demolish(parcels, parcels_tract, tracts_earthquake, buildings, orca.add_table("buildings", buildings) buildings = orca.get_table("buildings") - print "Demolished %d buildings" % (l1 - len(buildings)) + print("Demolished %d buildings" % (l1 - len(buildings))) diff --git a/baus/models.py b/baus/models.py index ab5ae13dc..d43d39d9e 100644 --- a/baus/models.py +++ b/baus/models.py @@ -3,18 +3,18 @@ import sys import orca import yaml -import datasources -import variables -from utils import parcel_id_to_geom_id, geom_id_to_parcel_id, add_buildings -from utils import round_series_match_target, groupby_random_choice +from . import datasources +from . import variables +from .utils import parcel_id_to_geom_id, geom_id_to_parcel_id, add_buildings +from .utils import round_series_match_target, groupby_random_choice from urbansim.utils import networks import pandana.network as pdna from urbansim_defaults import models from urbansim_defaults import utils from urbansim.developer import sqftproforma, developer from urbansim.developer.developer import Developer as dev -import subsidies -import summaries +from . import subsidies +from . import summaries import numpy as np import pandas as pd @@ -31,14 +31,14 @@ def elcm_simulate(jobs, buildings, aggregations, elcm_config): @orca.step() def households_transition(households, household_controls, year, settings): s = orca.get_table('households').base_income_quartile.value_counts() - print "Distribution by income before:\n", (s/s.sum()) + print("Distribution by income before:\n", (s/s.sum())) ret = utils.full_transition(households, household_controls, year, settings['households_transition'], "building_id") s = orca.get_table('households').base_income_quartile.value_counts() - print "Distribution by income after:\n", (s/s.sum()) + print("Distribution by income after:\n", (s/s.sum())) return ret @@ -84,8 +84,8 @@ def _proportional_jobs_model( available_jobs = \ jobs_df.query("empsix == '%s' and building_id == -1" % sector) - print "Need more jobs total: %d" % need_more_jobs_total - print "Available jobs: %d" % len(available_jobs) + print("Need more jobs total: %d" % need_more_jobs_total) + print("Available jobs: %d" % len(available_jobs)) if len(available_jobs) == 0: # corner case @@ -109,16 +109,16 @@ def _proportional_jobs_model( if need_more_jobs_total <= 0: return pd.Series() - print "Need more jobs\n", need_more_jobs + print("Need more jobs\n", need_more_jobs) excess = need_more_jobs.sub(locations_series.value_counts(), fill_value=0) - print "Excess demand\n", excess[excess > 0] + print("Excess demand\n", excess[excess > 0]) # there's an issue with groupby_random_choice where it can't choose from # a set of locations that don't exist - e.g. we have 2 jobs in a certain # city but not locations to put them in. we need to drop this demand drop = need_more_jobs.index.difference(locations_series.unique()) - print "We don't have any locations for these locations:\n", drop + print("We don't have any locations for these locations:\n", drop) need_more_jobs = need_more_jobs.drop(drop) # choose random locations within jurises to match need_more_jobs totals @@ -186,7 +186,7 @@ def proportional_elcm(jobs, households, buildings, parcels, location_options = building_subset.juris.repeat( building_subset.vacant_job_spaces.clip(0)) - print "Running proportional jobs model for retail" + print("Running proportional jobs model for retail") s = _proportional_jobs_model( # we now take the ratio of retail jobs to households as an input @@ -242,7 +242,7 @@ def proportional_elcm(jobs, households, buildings, parcels, target_jobs = target_jobs.astype('int') - print "Running proportional jobs model for gov/edu" + print("Running proportional jobs model for gov/edu") # location options are vacant job spaces in retail buildings - this will # overfill certain location because we don't have enough space @@ -309,7 +309,7 @@ def scheduled_development_events(buildings, development_projects, # first demolish demolish = demolish_events.to_frame().\ query("%d <= year_built < %d" % (year, year + years_per_iter)) - print "Demolishing/building %d buildings" % len(demolish) + print("Demolishing/building %d buildings" % len(demolish)) l1 = len(buildings) buildings = utils._remove_developed_buildings( buildings.to_frame(buildings.local_columns), @@ -317,8 +317,8 @@ def scheduled_development_events(buildings, development_projects, unplace_agents=["households", "jobs"]) orca.add_table("buildings", buildings) buildings = orca.get_table("buildings") - print "Demolished %d buildings" % (l1 - len(buildings)) - print " (this number is smaller when parcel has no existing buildings)" + print("Demolished %d buildings" % (l1 - len(buildings))) + print(" (this number is smaller when parcel has no existing buildings)") # then build dps = development_projects.to_frame().\ @@ -356,7 +356,7 @@ def scheduled_development_events(buildings, development_projects, def supply_and_demand_multiplier_func(demand, supply): s = demand / supply settings = orca.get_injectable('settings') - print "Number of submarkets where demand exceeds supply:", len(s[s > 1.0]) + print("Number of submarkets where demand exceeds supply:", len(s[s > 1.0])) # print "Raw relationship of supply and demand\n", s.describe() supply_correction = settings["price_equilibration"] clip_change_high = supply_correction["kwargs"]["clip_change_high"] @@ -393,8 +393,8 @@ def add_extra_columns_func(df): if "deed_restricted_units" not in df.columns: df["deed_restricted_units"] = 0 else: - print "Number of deed restricted units built = %d" %\ - df.deed_restricted_units.sum() + print("Number of deed restricted units built = %d" %\ + df.deed_restricted_units.sum()) df["redfin_sale_year"] = 2012 df["redfin_sale_price"] = np.nan @@ -468,8 +468,8 @@ def residential_developer(feasibility, households, buildings, parcels, year, juris_name = parcels_geography.juris_name.\ reindex(parcels.index).fillna('Other') - juris_list = limits_settings[typ].keys() - for juris, limit in limits_settings[typ].items(): + juris_list = list(limits_settings[typ].keys()) + for juris, limit in list(limits_settings[typ].items()): # the actual target is the limit times the number of years run # so far in the simulation (plus this year), minus the amount @@ -503,8 +503,8 @@ def residential_developer(feasibility, households, buildings, parcels, year, for parcel_mask, target, final_target, juris in targets: - print "Running developer for %s with target of %d" % \ - (str(juris), target) + print("Running developer for %s with target of %d" % \ + (str(juris), target)) # this was a fairly heinous bug - have to get the building wrapper # again because the buildings df gets modified by the run_developer @@ -591,7 +591,7 @@ def retail_developer(jobs, buildings, parcels, nodes, feasibility, p = f1 * 1.5 + f2 p = p.clip(lower=1.0/len(p)/10) - print "Attempting to build {:,} retail sqft".format(target) + print("Attempting to build {:,} retail sqft".format(target)) # order by weighted random sample feasibility = feasibility.sample(frac=1.0, weights=p) @@ -624,10 +624,10 @@ def retail_developer(jobs, buildings, parcels, nodes, feasibility, # add the buidings and demolish old buildings, and add to debug output devs = pd.DataFrame(devs, columns=feasibility.columns) - print "Building {:,} retail sqft in {:,} projects".format( - devs.non_residential_sqft.sum(), len(devs)) + print("Building {:,} retail sqft in {:,} projects".format( + devs.non_residential_sqft.sum(), len(devs))) if target > 0: - print " WARNING: retail target not met" + print(" WARNING: retail target not met") devs["form"] = "retail" devs = add_extra_columns_func(devs) @@ -658,13 +658,13 @@ def office_developer(feasibility, jobs, buildings, parcels, year, buildings.job_spaces.sum(), dev_settings['kwargs']['target_vacancy']) - print "Total units to build = %d" % all_units + print("Total units to build = %d" % all_units) if all_units <= 0: return for typ in ["Office"]: - print "\nRunning for type: ", typ + print("\nRunning for type: ", typ) num_units = all_units * float(dev_settings['type_splits'][typ]) @@ -677,8 +677,8 @@ def office_developer(feasibility, jobs, buildings, parcels, year, juris_name = parcels_geography.juris_name.\ reindex(parcels.index).fillna('Other') - juris_list = limits_settings[typ].keys() - for juris, limit in limits_settings[typ].items(): + juris_list = list(limits_settings[typ].keys()) + for juris, limit in list(limits_settings[typ].items()): # the actual target is the limit times the number of years run # so far in the simulation (plus this year), minus the amount @@ -692,9 +692,9 @@ def office_developer(feasibility, jobs, buildings, parcels, year, target = (year - 2015 + 1) * limit - current_total if target <= 0: - print "Already met target for juris = %s" % juris - print " target = %d, current_total = %d" %\ - (target, current_total) + print("Already met target for juris = %s" % juris) + print(" target = %d, current_total = %d" %\ + (target, current_total)) continue targets.append((juris_name == juris, target, juris)) @@ -709,9 +709,9 @@ def office_developer(feasibility, jobs, buildings, parcels, year, for parcel_mask, target, juris in targets: - print "Running developer for %s with target of %d" % \ - (str(juris), target) - print "Parcels in play:\n", pd.Series(parcel_mask).value_counts() + print("Running developer for %s with target of %d" % \ + (str(juris), target)) + print("Parcels in play:\n", pd.Series(parcel_mask).value_counts()) # this was a fairly heinous bug - have to get the building wrapper # again because the buildings df gets modified by the run_developer @@ -758,11 +758,11 @@ def developer_reprocess(buildings, year, years_per_iter, jobs, to_add = res_units * .05 - job_spaces if to_add > 0: - print "Adding %d job_spaces" % to_add + print("Adding %d job_spaces" % to_add) res_units = buildings.residential_units[s] # bias selection of places to put job spaces based on res units - print res_units.describe() - print res_units[res_units < 0] + print(res_units.describe()) + print(res_units[res_units < 0]) add_indexes = np.random.choice(res_units.index.values, size=to_add, replace=True, p=(res_units/res_units.sum())) @@ -770,12 +770,12 @@ def developer_reprocess(buildings, year, years_per_iter, jobs, add_indexes = pd.Series(add_indexes).value_counts() # this is sqft per job for residential bldgs add_sizes = add_indexes * 400 - print "Job spaces in res before adjustment: ", \ - buildings.job_spaces[s].sum() + print("Job spaces in res before adjustment: ", \ + buildings.job_spaces[s].sum()) buildings.local.loc[add_sizes.index, "non_residential_sqft"] += add_sizes.values - print "Job spaces in res after adjustment: ",\ - buildings.job_spaces[s].sum() + print("Job spaces in res after adjustment: ",\ + buildings.job_spaces[s].sum()) # the second step here is to add retail to buildings that are greater than # X stories tall - presumably this is a ground floor retail policy @@ -783,12 +783,12 @@ def developer_reprocess(buildings, year, years_per_iter, jobs, new_buildings = old_buildings.query( '%d == year_built and stories >= 4' % year) - print "Attempting to add ground floor retail to %d devs" % \ - len(new_buildings) + print("Attempting to add ground floor retail to %d devs" % \ + len(new_buildings)) retail = parcel_is_allowed_func("retail") new_buildings = new_buildings[retail.loc[new_buildings.parcel_id].values] - print "Disallowing dev on these parcels:" - print " %d devs left after retail disallowed" % len(new_buildings) + print("Disallowing dev on these parcels:") + print(" %d devs left after retail disallowed" % len(new_buildings)) # this is the key point - make these new buildings' nonres sqft equal # to one story of the new buildings @@ -809,8 +809,8 @@ def developer_reprocess(buildings, year, years_per_iter, jobs, ratio = parcels.retail_ratio.loc[new_buildings.parcel_id] new_buildings = new_buildings[ratio.values > ratio.median()] - print "Adding %d sqft of ground floor retail in %d locations" % \ - (new_buildings.non_residential_sqft.sum(), len(new_buildings)) + print("Adding %d sqft of ground floor retail in %d locations" % \ + (new_buildings.non_residential_sqft.sum(), len(new_buildings))) all_buildings = dev.merge(old_buildings, new_buildings) orca.add_table("buildings", all_buildings) @@ -829,8 +829,8 @@ def developer_reprocess(buildings, year, years_per_iter, jobs, ['year_built', 'building_sqft', 'general_type']) sqft_by_gtype = buildings_df.query('year_built >= %d' % year).\ groupby('general_type').building_sqft.sum() - print "New square feet by general type in millions:\n",\ - sqft_by_gtype / 1000000.0 + print("New square feet by general type in millions:\n",\ + sqft_by_gtype / 1000000.0) def proportional_job_allocation(parcel_id): @@ -868,9 +868,9 @@ def proportional_job_allocation(parcel_id): # make sure index is incrementing new_jobs.index = new_jobs.index + 1 + np.max(all_jobs.index.values) - print "Adding {} new jobs to parcel {} with proportional model".format( - num_new_jobs, parcel_id) - print new_jobs.head() + print("Adding {} new jobs to parcel {} with proportional model".format( + num_new_jobs, parcel_id)) + print(new_jobs.head()) all_jobs = all_jobs.append(new_jobs) orca.add_table("jobs", all_jobs) @@ -906,7 +906,7 @@ def net(settings): # yeah, starting to hardcode stuff, not great, but can only # do nearest queries on the first graph I initialize due to crummy # limitation in pandana - for key in settings["build_networks"].keys(): + for key in list(settings["build_networks"].keys()): nets[key] = make_network_from_settings( settings['build_networks'][key] ) @@ -951,7 +951,7 @@ def neighborhood_vars(net): nodes = nodes.replace(np.inf, np.nan) nodes = nodes.fillna(0) - print nodes.describe() + print(nodes.describe()) orca.add_table("nodes", nodes) @@ -964,7 +964,7 @@ def regional_vars(net): index_col="tmnode_id") nodes = pd.concat([nodes, nodes2], axis=1) - print nodes.describe() + print(nodes.describe()) orca.add_table("tmnodes", nodes) @@ -989,7 +989,7 @@ def regional_pois(settings, landmarks): cols[locname] = n.nearest_pois(75, "tmp", num_pois=1)[1] df = pd.DataFrame(cols) - print df.describe() + print(df.describe()) df.index.name = "tmnode_id" df.to_csv('regional_poi_distances.csv') @@ -998,7 +998,7 @@ def regional_pois(settings, landmarks): def price_vars(net): nodes2 = networks.from_yaml(net["walk"], "price_vars.yaml") nodes2 = nodes2.fillna(0) - print nodes2.describe() + print(nodes2.describe()) nodes = orca.get_table('nodes') nodes = nodes.to_frame().join(nodes2) orca.add_table("nodes", nodes) diff --git a/baus/preprocessing.py b/baus/preprocessing.py index a81346de8..1db5140df 100644 --- a/baus/preprocessing.py +++ b/baus/preprocessing.py @@ -1,7 +1,7 @@ import orca import pandas as pd from urbansim.utils import misc -from validation import assert_series_equal +from .validation import assert_series_equal # the way this works is there is an orca step to do jobs allocation, which @@ -18,7 +18,7 @@ def allocate_jobs(baseyear_taz_controls, settings, buildings, parcels): sector_map = settings["naics_to_empsix"] jobs = [] for taz, row in baseyear_taz_controls.local.iterrows(): - for sector_col, num in row.iteritems(): + for sector_col, num in row.items(): # not a sector total if not sector_col.startswith("emp_sec"): @@ -38,7 +38,7 @@ def allocate_jobs(baseyear_taz_controls, settings, buildings, parcels): # just do random assignment weighted by job spaces - we'll then # fill in the job_spaces if overfilled in the next step (code # has existed in urbansim for a while) - for taz, cnt in df.groupby('taz').size().iteritems(): + for taz, cnt in df.groupby('taz').size().items(): potential_add_locations = buildings.non_residential_sqft[ (zone_id == taz) & @@ -158,14 +158,14 @@ def assign_deed_restricted_units(df, parcels): units = pd.Series(buildings_ids.index.values).value_counts() df.loc[units.index, "deed_restricted_units"] += units.values - print "Total deed restricted units after random selection: %d" % \ - df.deed_restricted_units.sum() + print("Total deed restricted units after random selection: %d" % \ + df.deed_restricted_units.sum()) df["deed_restricted_units"] = \ df[["deed_restricted_units", "residential_units"]].min(axis=1) - print "Total deed restricted units after truncating to res units: %d" % \ - df.deed_restricted_units.sum() + print("Total deed restricted units after truncating to res units: %d" % \ + df.deed_restricted_units.sum()) return df @@ -250,15 +250,15 @@ def correct_baseyear_vacancies(buildings, parcels, jobs, store): jobs_county = misc.reindex(buildings_county, jobs.building_id) - print "Vacancy rate by county:\n", \ + print("Vacancy rate by county:\n", \ buildings.job_spaces.groupby(buildings_county).sum() / \ - jobs_county.value_counts() - 1.0 + jobs_county.value_counts() - 1.0) jobs_juris = misc.reindex(buildings_juris, jobs.building_id) s = buildings.job_spaces.groupby(buildings_juris).sum() / \ jobs_juris.value_counts() - 1.0 - print "Vacancy rate by juris:\n", s.to_string() + print("Vacancy rate by juris:\n", s.to_string()) return buildings diff --git a/baus/slr.py b/baus/slr.py index d9c6ce485..04283b864 100644 --- a/baus/slr.py +++ b/baus/slr.py @@ -2,9 +2,9 @@ import numpy as np import pandas as pd from urbansim_defaults import utils -import datasources -import variables -import summaries +from . import datasources +from . import variables +from . import summaries # select and tag parcels that are indundated in the current year: @@ -22,12 +22,12 @@ def slr_inundate(scenario, parcels, slr_progression_C, slr_progression_R, else: slr_progression = slr_progression_C.to_frame() # placeholder inundation_yr = slr_progression.query('year==@year')['inundated'].item() - print "Inundation in model year is %d inches" % inundation_yr + print("Inundation in model year is %d inches" % inundation_yr) slr_parcel_inundation = slr_parcel_inundation.to_frame() destroy_parcels = slr_parcel_inundation.\ query('inundation<=@inundation_yr').astype('bool') orca.add_table('destroy_parcels', destroy_parcels) - print "Number of parcels destroyed: %d" % len(destroy_parcels) + print("Number of parcels destroyed: %d" % len(destroy_parcels)) slr_nodev = pd.Series(False, parcels.index) destroy = pd.Series(destroy_parcels['inundation']) @@ -46,7 +46,7 @@ def slr_remove_dev(buildings, destroy_parcels, year, parcels, (destroy_parcels.index)] orca.add_table("slr_demolish", slr_demolish) - print "Demolishing %d buildings" % len(slr_demolish) + print("Demolishing %d buildings" % len(slr_demolish)) households = households.to_frame() hh_unplaced = households[households["building_id"] == -1] jobs = jobs.to_frame() @@ -70,4 +70,4 @@ def slr_remove_dev(buildings, destroy_parcels, year, parcels, orca.add_injectable("jobs_unplaced_slr", jobs_unplaced_slr) orca.add_table("buildings", buildings) buildings = orca.get_table("buildings") - print "Demolished %d buildings" % (l1 - len(buildings)) + print("Demolished %d buildings" % (l1 - len(buildings))) diff --git a/baus/subsidies.py b/baus/subsidies.py index 557ca9a5c..7dbc54caa 100644 --- a/baus/subsidies.py +++ b/baus/subsidies.py @@ -5,10 +5,11 @@ import numpy as np from urbansim import accounts from urbansim_defaults import utils -from cStringIO import StringIO +from io import StringIO from urbansim.utils import misc -from utils import add_buildings +from .utils import add_buildings from urbansim.developer import sqftproforma +from functools import reduce # this method is a custom profit to probability function where we test the @@ -39,7 +40,7 @@ def coffer(settings): "vmt_com_acct": accounts.Account("vmt_com_acct") } - for key, acct in settings["acct_settings"]["lump_sum_accounts"].items(): + for key, acct in list(settings["acct_settings"]["lump_sum_accounts"].items()): d[acct["name"]] = accounts.Account(acct["name"]) return d @@ -56,7 +57,7 @@ def lump_sum_accounts(settings, year, buildings, coffer, s = settings["acct_settings"]["lump_sum_accounts"] - for key, acct in s.items(): + for key, acct in list(s.items()): if scenario not in acct["enable_in_scenarios"]: continue @@ -80,7 +81,7 @@ def lump_sum_accounts(settings, year, buildings, coffer, # comments alongside the code def inclusionary_housing_revenue_reduction(feasibility, units): - print "Computing adjustments due to inclusionary housing" + print("Computing adjustments due to inclusionary housing") # AMI by jurisdiction # @@ -112,7 +113,7 @@ def value_can_afford(monthly_payment): return np.npv(ten_year_average_interest/12, [monthly_payment]*30*12) value_can_afford = {k: value_can_afford(v) for k, v in - monthly_affordable_payment.to_dict().items()} + list(monthly_affordable_payment.to_dict().items())} value_can_afford = pd.Series(value_can_afford) # account for interest and property taxes @@ -134,14 +135,14 @@ def value_can_afford(monthly_payment): feasibility[('residential', 'building_revenue')] / units revenue_diff_per_unit = (ave_price_per_unit - value_can_afford).fillna(0) - print "Revenue difference per unit (not zero values)" - print revenue_diff_per_unit[revenue_diff_per_unit > 0].describe() + print("Revenue difference per unit (not zero values)") + print(revenue_diff_per_unit[revenue_diff_per_unit > 0].describe()) revenue_reduction = revenue_diff_per_unit * num_affordable_units s = num_affordable_units.groupby(parcels_geography.juris_name).sum() - print "Feasibile affordable units by jurisdiction" - print s[s > 0].sort_values() + print("Feasibile affordable units by jurisdiction") + print(s[s > 0].sort_values()) return revenue_reduction, num_affordable_units @@ -151,14 +152,14 @@ def value_can_afford(monthly_payment): # is done here as well def policy_modifications_of_profit(feasibility, parcels): - print "Making policy modifications to profitability" + print("Making policy modifications to profitability") # this first section adds parcel unit-based fees units = feasibility[('residential', 'residential_sqft')] / \ parcels.ave_sqft_per_unit fees = (units * parcels.fees_per_unit).fillna(0) - print "Sum of residential fees: ", fees.sum() + print("Sum of residential fees: ", fees.sum()) feasibility[("residential", "fees")] = fees feasibility[("residential", "max_profit")] -= fees @@ -171,7 +172,7 @@ def policy_modifications_of_profit(feasibility, parcels): sqft = feasibility[(use, 'non_residential_sqft')] fees = (sqft * parcels.fees_per_sqft).fillna(0) - print "Sum of non-residential fees (%s): %.0f" % (use, fees.sum()) + print("Sum of non-residential fees (%s): %.0f" % (use, fees.sum())) feasibility[(use, "fees")] = fees feasibility[(use, "max_profit")] -= fees @@ -182,11 +183,11 @@ def policy_modifications_of_profit(feasibility, parcels): assert np.all(num_affordable_units <= units.fillna(0)) - print "Describe of inclusionary revenue reduction:\n", \ - revenue_reduction[revenue_reduction > 0].describe() + print("Describe of inclusionary revenue reduction:\n", \ + revenue_reduction[revenue_reduction > 0].describe()) - print "Describe of number of affordable units:\n", \ - num_affordable_units[num_affordable_units > 0].describe() + print("Describe of number of affordable units:\n", \ + num_affordable_units[num_affordable_units > 0].describe()) feasibility[("residential", "policy_based_revenue_reduction")] = \ revenue_reduction @@ -207,7 +208,7 @@ def policy_modifications_of_profit(feasibility, parcels): pct_modifications = feasibility[("residential", "vmt_res_cat")].\ map(sb743_settings["sb743_pcts"]) + 1 - print "Modifying profit for SB743:\n", pct_modifications.describe() + print("Modifying profit for SB743:\n", pct_modifications.describe()) feasibility[("residential", "max_profit")] *= pct_modifications @@ -231,16 +232,16 @@ def policy_modifications_of_profit(feasibility, parcels): pct_modifications = \ pct_modifications.reindex(pzc.index).fillna(1.0) - print "Modifying profit for Land Value Tax:\n", \ - pct_modifications.describe() + print("Modifying profit for Land Value Tax:\n", \ + pct_modifications.describe()) feasibility[("residential", "max_profit")] *= pct_modifications if "profitability_adjustment_policies" in settings["acct_settings"]: for key, policy in \ - settings["acct_settings"][ - "profitability_adjustment_policies"].items(): + list(settings["acct_settings"][ + "profitability_adjustment_policies"].items()): if orca.get_injectable("scenario") in \ policy["enable_in_scenarios"]: @@ -252,13 +253,13 @@ def policy_modifications_of_profit(feasibility, parcels): policy["profitability_adjustment_formula"]) pct_modifications += 1.0 - print "Modifying profit for %s:\n" % policy["name"], \ - pct_modifications.describe() + print("Modifying profit for %s:\n" % policy["name"], \ + pct_modifications.describe()) feasibility[("residential", "max_profit")] *= pct_modifications - print "There are %d affordable units if all feasible projects are built" %\ - feasibility[("residential", "deed_restricted_units")].sum() + print("There are %d affordable units if all feasible projects are built" %\ + feasibility[("residential", "deed_restricted_units")].sum()) return feasibility @@ -278,7 +279,7 @@ def calculate_vmt_fees(settings, year, buildings, vmt_fee_categories, coffer, if not len(df): return - print "%d projects pass the vmt filter" % len(df) + print("%d projects pass the vmt filter" % len(df)) total_fees = 0 @@ -287,17 +288,17 @@ def calculate_vmt_fees(settings, year, buildings, vmt_fee_categories, coffer, df["res_for_res_fees"] = df.vmt_res_cat.map( vmt_settings["res_for_res_fee_amounts"]) total_fees += (df.res_for_res_fees * df.residential_units).sum() - print "Applying vmt fees to %d units" % df.residential_units.sum() + print("Applying vmt fees to %d units" % df.residential_units.sum()) if scenario in vmt_settings["com_for_res_scenarios"]: df["com_for_res_fees"] = df.vmt_res_cat.map( vmt_settings["com_for_res_fee_amounts"]) total_fees += (df.com_for_res_fees * df.non_residential_sqft).sum() - print "Applying vmt fees to %d commerical sqft" % \ - df.non_residential_sqft.sum() + print("Applying vmt fees to %d commerical sqft" % \ + df.non_residential_sqft.sum()) - print "Adding total vmt fees for res amount of $%.2f" % total_fees + print("Adding total vmt fees for res amount of $%.2f" % total_fees) metadata = { "description": "VMT development fees", @@ -315,10 +316,10 @@ def calculate_vmt_fees(settings, year, buildings, vmt_fee_categories, coffer, df["com_for_com_fees"] = df.vmt_res_cat.map( vmt_settings["com_for_com_fee_amounts"]) total_fees += (df.com_for_com_fees * df.non_residential_sqft).sum() - print "Applying vmt fees to %d commerical sqft" % \ - df.non_residential_sqft.sum() + print("Applying vmt fees to %d commerical sqft" % \ + df.non_residential_sqft.sum()) - print "Adding total vmt fees for com amount of $%.2f" % total_fees + print("Adding total vmt fees for com amount of $%.2f" % total_fees) coffer["vmt_com_acct"].add_transaction(total_fees, subaccount="regional", metadata=metadata) @@ -355,8 +356,8 @@ def subsidized_office_developer(feasibility, coffer, acct_settings, year, # make parcel_id available feasibility = feasibility.reset_index() - print "%.0f subsidy with %d developments to choose from" % ( - total_subsidy, len(feasibility)) + print("%.0f subsidy with %d developments to choose from" % ( + total_subsidy, len(feasibility))) devs = [] @@ -403,8 +404,8 @@ def subsidized_office_developer(feasibility, coffer, acct_settings, year, # add the buidings and demolish old buildings, and add to debug output devs = pd.DataFrame(devs, columns=feasibility.columns) - print "Building {:,} subsidized office sqft in {:,} projects".format( - devs.non_residential_sqft.sum(), len(devs)) + print("Building {:,} subsidized office sqft in {:,} projects".format( + devs.non_residential_sqft.sum(), len(devs))) devs["form"] = "office" devs = add_extra_columns_func(devs) @@ -532,10 +533,10 @@ def run_subsidized_developer(feasibility, parcels, buildings, households, feasibility["subaccount"] = feasibility.eval(sending_bldgs) # step 6 for subacct, amount in account.iter_subaccounts(): - print "Subaccount: ", subacct + print("Subaccount: ", subacct) df = feasibility[feasibility.subaccount == subacct] - print "Number of feasible projects in receiving zone:", len(df) + print("Number of feasible projects in receiving zone:", len(df)) if len(df) == 0: continue @@ -546,7 +547,7 @@ def run_subsidized_developer(feasibility, parcels, buildings, households, # (orca.get_injectable("year"), account.name, subacct)) # step 8 - print "Amount in subaccount: ${:,.2f}".format(amount) + print("Amount in subaccount: ${:,.2f}".format(amount)) num_bldgs = int((-1*df.max_profit).cumsum().searchsorted(amount)) if num_bldgs == 0: @@ -629,22 +630,22 @@ def run_subsidized_developer(feasibility, parcels, buildings, households, assert np.all(buildings.local.deed_restricted_units.fillna(0) <= buildings.local.residential_units.fillna(0)) - print "Amount left after subsidy: ${:,.2f}".\ - format(account.total_transactions_by_subacct(subacct)) + print("Amount left after subsidy: ${:,.2f}".\ + format(account.total_transactions_by_subacct(subacct))) new_buildings_list.append(new_buildings) total_len = reduce(lambda x, y: x+len(y), new_buildings_list, 0) if total_len == 0: - print "No subsidized buildings" + print("No subsidized buildings") return new_buildings = pd.concat(new_buildings_list) - print "Built {} total subsidized buildings".format(len(new_buildings)) - print " Total subsidy: ${:,.2f}".format( - -1*new_buildings.max_profit.sum()) - print " Total subsidized units: {:.0f}".\ - format(new_buildings.residential_units.sum()) + print("Built {} total subsidized buildings".format(len(new_buildings))) + print(" Total subsidy: ${:,.2f}".format( + -1*new_buildings.max_profit.sum())) + print(" Total subsidized units: {:.0f}".\ + format(new_buildings.residential_units.sum())) new_buildings["subsidized"] = True new_buildings["policy_name"] = policy_name @@ -726,13 +727,13 @@ def subsidized_residential_developer_lump_sum_accts( settings, summary, coffer, form_to_btype_func, scenario): - for key, acct in settings["acct_settings"]["lump_sum_accounts"].items(): + for key, acct in list(settings["acct_settings"]["lump_sum_accounts"].items()): # quick return in order to save performance time if scenario not in acct["enable_in_scenarios"]: continue - print "Running the subsidized developer for acct: %s" % acct["name"] + print("Running the subsidized developer for acct: %s" % acct["name"]) # need to rerun the subsidized feasibility every time and get new # results - this is not ideal and is a story to fix in pivotal, but the diff --git a/baus/summaries.py b/baus/summaries.py index 3396e1976..9c5afbfc4 100644 --- a/baus/summaries.py +++ b/baus/summaries.py @@ -4,7 +4,7 @@ import pandas as pd from pandas.util import testing as pdt import numpy as np -from utils import random_indexes, round_series_match_target,\ +from .utils import random_indexes, round_series_match_target,\ scale_by_target, simple_ipf from urbansim.utils import misc from scripts.output_csv_utils import format_df @@ -371,7 +371,7 @@ def diagnostic_output(households, buildings, parcels, taz, jobs, settings, # save the dropped buildings to a csv if "dropped_buildings" in orca.orca._TABLES: df = orca.get_table("dropped_buildings").to_frame() - print "Dropped buildings", df.describe() + print("Dropped buildings", df.describe()) df.to_csv( "runs/run{}_dropped_buildings.csv".format(run_number) ) @@ -527,7 +527,7 @@ def geographic_summary(parcels, households, jobs, buildings, taz_geography, # Write Summary of Accounts if year == final_year: - for acct_name, acct in orca.get_injectable("coffer").iteritems(): + for acct_name, acct in orca.get_injectable("coffer").items(): fname = "runs/run{}_acctlog_{}_{}.csv".\ format(run_number, acct_name, year) acct.to_frame().to_csv(fname) diff --git a/baus/ual.py b/baus/ual.py index 3ddb60966..439564556 100644 --- a/baus/ual.py +++ b/baus/ual.py @@ -158,10 +158,10 @@ def assign_tenure_to_units(residential_units, households): units.loc[own, 'tenure'] = 'own' units.loc[rent, 'tenure'] = 'rent' - print "Init unit tenure assignment: %d%% owner occupied, %d%% unfilled" % \ + print("Init unit tenure assignment: %d%% owner occupied, %d%% unfilled" % \ (round(len(units[units.tenure == 'own'])*100 / len(units[units.tenure.notnull()])), - round(len(units[units.tenure.isnull()])*100 / len(units))) + round(len(units[units.tenure.isnull()])*100 / len(units)))) # Fill remaining units with random tenure assignment # TO DO: Make this weighted by existing allocation, rather than 50/50 @@ -301,7 +301,7 @@ def reconcile_placed_households(households, residential_units): hh = households.to_frame(['unit_id', 'building_id']) hh.index.rename('household_id', inplace=True) hh = hh.reset_index() - print "hh columns: %s" % hh.columns + print("hh columns: %s" % hh.columns) # hh.index.name='household_id' units = residential_units.to_frame(['building_id']).reset_index() @@ -314,9 +314,9 @@ def reconcile_placed_households(households, residential_units): hh = hh.drop('building_id', axis=1) hh = pd.merge(hh, units, on='unit_id', how='left').\ set_index('household_id') - print "hh index.names: %s" % hh.index.names + print("hh index.names: %s" % hh.index.names) - print "%d movers updated" % len(hh) + print("%d movers updated" % len(hh)) households.update_col_from_series('building_id', hh.building_id, cast=True) # Verify final data characteristics @@ -366,17 +366,17 @@ def reconcile_unplaced_households(households): ''' def _print_status(): - print "Households not in a unit: %d" % \ - (households.unit_id == -1).sum() - print "Househing missing a unit: %d" % \ - households.unit_id.isnull().sum() - print "Households not in a building: %d" % \ - (households.building_id == -1).sum() - print "Househing missing a building: %d" % \ - households.building_id.isnull().sum() + print("Households not in a unit: %d" % \ + (households.unit_id == -1).sum()) + print("Househing missing a unit: %d" % \ + households.unit_id.isnull().sum()) + print("Households not in a building: %d" % \ + (households.building_id == -1).sum()) + print("Househing missing a building: %d" % \ + households.building_id.isnull().sum()) _print_status() - print "Reconciling unplaced households..." + print("Reconciling unplaced households...") hh = households.to_frame(['building_id', 'unit_id']) # Get indexes of households unplaced in buildings or in units @@ -437,10 +437,10 @@ def remove_old_units(buildings, residential_units): units = residential_units.to_frame(residential_units.local_columns) current_units = units[units.building_id.isin(buildings.index)] - print "Removing %d units from %d buildings that no longer exist" % \ + print("Removing %d units from %d buildings that no longer exist" % \ ((len(units) - len(current_units)), (len(units.groupby('building_id')) - - len(current_units.groupby('building_id')))) + len(current_units.groupby('building_id'))))) orca.add_table('residential_units', current_units) @@ -503,8 +503,8 @@ def initialize_new_units(buildings, residential_units): all_units = dev.merge(old_units, new_units) all_units.index.name = 'unit_id' - print "Creating %d residential units for %d new buildings" % \ - (len(new_units), len(new_bldgs)) + print("Creating %d residential units for %d new buildings" % \ + (len(new_units), len(new_bldgs))) orca.add_table('residential_units', all_units) @@ -568,8 +568,8 @@ def assign_tenure_to_new_units(residential_units, settings): units.loc[~rental_units, 'tenure'] = 'own' units.loc[rental_units, 'tenure'] = 'rent' - print "Adding tenure assignment to %d new residential units" % len(units) - print units.describe() + print("Adding tenure assignment to %d new residential units" % len(units)) + print(units.describe()) residential_units.update_col_from_series( 'tenure', units.tenure, cast=True) @@ -630,7 +630,7 @@ def _mtc_clip(table, col_name, settings, price_scale=1): low = float(settings["rsh_simulate"]["low"]) * price_scale high = float(settings["rsh_simulate"]["high"]) * price_scale table.update_col(col_name, table[col_name].clip(low, high)) - print "Clipping produces\n", table[col_name].describe() + print("Clipping produces\n", table[col_name].describe()) @orca.step() @@ -708,9 +708,9 @@ def households_relocation(households, settings): rates = pd.DataFrame.from_dict(settings['relocation_rates']) - print "Total agents: %d" % len(households) - print "Total currently unplaced: %d" % (households.unit_id == -1).sum() - print "Assigning for relocation..." + print("Total agents: %d" % len(households)) + print("Total currently unplaced: %d" % (households.unit_id == -1).sum()) + print("Assigning for relocation...") # Initialize model, choose movers, and un-place them from buildings # and units @@ -721,7 +721,7 @@ def households_relocation(households, settings): households.update_col_from_series( 'unit_id', pd.Series(-1, index=mover_ids), cast=True) - print "Total currently unplaced: %d" % (households.unit_id == -1).sum() + print("Total currently unplaced: %d" % (households.unit_id == -1).sum()) return @@ -860,12 +860,12 @@ def balance_rental_and_ownership_hedonics(households, settings, owner_utilization = hh_rent_own.own / float(unit_rent_own.own) renter_utilization = hh_rent_own.rent / float(unit_rent_own.rent) - print "Owner utilization = %.3f" % owner_utilization - print "Renter utilization = %.3f" % renter_utilization + print("Owner utilization = %.3f" % owner_utilization) + print("Renter utilization = %.3f" % renter_utilization) utilization_ratio = renter_utilization / owner_utilization - print "Ratio of renter utilization to owner utilization = %.3f" %\ - utilization_ratio + print("Ratio of renter utilization to owner utilization = %.3f" %\ + utilization_ratio) if "original_cap_rate" not in settings: settings["original_cap_rate"] = settings["cap_rate"] @@ -876,7 +876,7 @@ def balance_rental_and_ownership_hedonics(households, settings, utilization_ratio /= factor elif utilization_ratio > 1.0: utilization_ratio *= factor - print "Modified ratio = %.3f" % utilization_ratio + print("Modified ratio = %.3f" % utilization_ratio) # adjust the cap rate based on utilization ratio - higher ratio # here means renter utilization is higher than owner utilization @@ -885,4 +885,4 @@ def balance_rental_and_ownership_hedonics(households, settings, settings["cap_rate"] = settings["original_cap_rate"] /\ utilization_ratio - print "New cap rate = %.2f" % settings["cap_rate"] + print("New cap rate = %.2f" % settings["cap_rate"]) diff --git a/baus/utils.py b/baus/utils.py index aae97ea3d..0b9beda8e 100644 --- a/baus/utils.py +++ b/baus/utils.py @@ -37,19 +37,19 @@ def save_and_restore_state(in_d, outhdf="save_state.h5"): store = pd.HDFStore(outhdf) out_d = {} for table_name in store: - print "Restoring", table_name + print("Restoring", table_name) out_d[table_name[1:]] = store[table_name] return out_d # the state doesn't exist and thus needs to be saved store = pd.HDFStore(outhdf, "w") - for table_name, table in in_d.items(): + for table_name, table in list(in_d.items()): try: table = table.local except: # not a dataframe wrapper continue - print "Saving", table_name + print("Saving", table_name) store[table_name] = table store.close() sys.exit(0) @@ -117,7 +117,7 @@ def groupby_random_choice(s, counts, replace=True): return pd.concat([ s[s == grp].sample(cnt, replace=replace) - for grp, cnt in counts[counts > 0].iteritems() + for grp, cnt in counts[counts > 0].items() ]) @@ -184,8 +184,8 @@ def constrained_normalization(marginals, constraint, total): num_constrained = len(constrained[constrained is True]) num_exceeds = len(exceeds[exceeds is True]) - print "Len constrained = %d, exceeds = %d" %\ - (num_constrained, num_exceeds) + print("Len constrained = %d, exceeds = %d" %\ + (num_constrained, num_exceeds)) if num_exceeds == 0: return marginals @@ -212,7 +212,7 @@ def simple_ipf(seed_matrix, col_marginals, row_marginals, tolerance=1, cnt=0): seed_matrix *= ratios closeness = np.absolute(row_marginals - seed_matrix.sum(axis=1)).sum() assert np.absolute(col_marginals - seed_matrix.sum(axis=0)).sum() < .01 - print "row closeness", closeness + print("row closeness", closeness) if closeness < tolerance: return seed_matrix @@ -222,7 +222,7 @@ def simple_ipf(seed_matrix, col_marginals, row_marginals, tolerance=1, cnt=0): seed_matrix = seed_matrix * ratios.reshape((ratios.size, 1)) assert np.absolute(row_marginals - seed_matrix.sum(axis=1)).sum() < .01 closeness = np.absolute(col_marginals - seed_matrix.sum(axis=0)).sum() - print "col closeness", closeness + print("col closeness", closeness) if closeness < tolerance: return seed_matrix @@ -255,7 +255,7 @@ def compare_dfs(df1, df2): rowcomp = df2.loc[label] # for each value - for col, val in row.iteritems(): + for col, val in row.items(): val2 = rowcomp[col] @@ -313,7 +313,7 @@ def apply_format(worksheet, df, format, filter): s = df.stack() - for (lab, col), val in filter(s).iteritems(): + for (lab, col), val in filter(s).items(): rowind = df.index.get_loc(lab)+2 colind = df.columns.get_loc(col)+1 @@ -344,7 +344,7 @@ def compare_summary(df1, df2, index_names=None, pctdiff=10, s = df3[cols].stack() buf = "" - for (lab, col), val in s[s > 10].iteritems(): + for (lab, col), val in s[s > 10].items(): lab = index_names.loc[lab] buf += "%s '%s' is %d%% off in column '%s'\n" % \ (geog_name, lab, val, col) diff --git a/baus/validation.py b/baus/validation.py index a90271876..99b4a687a 100644 --- a/baus/validation.py +++ b/baus/validation.py @@ -3,7 +3,7 @@ import orca import pandas as pd from pandas.util import testing as pdt -from utils import save_and_restore_state +from .utils import save_and_restore_state from urbansim.utils import misc @@ -20,7 +20,7 @@ def assert_series_equal(s1, s2, head=None): # make sure the household controls are currently being matched def check_household_controls(households, household_controls, year): - print "Check household controls" + print("Check household controls") current_household_controls = household_controls.local.loc[year] current_household_controls = current_household_controls.\ set_index("base_income_quartile").total_number_of_households @@ -33,7 +33,7 @@ def check_household_controls(households, household_controls, year): # make sure the employment controls are currently being matched def check_job_controls(jobs, employment_controls, year, settings): - print "Check job controls" + print("Check job controls") current_employment_controls = employment_controls.local.loc[year] current_employment_controls = current_employment_controls.\ set_index("empsix_id").number_of_jobs @@ -48,7 +48,7 @@ def check_job_controls(jobs, employment_controls, year, settings): def check_residential_units(residential_units, buildings): - print "Check residential units" + print("Check residential units") # assert we fanned out the residential units correctly assert len(residential_units) == buildings.residential_units.sum() @@ -71,7 +71,7 @@ def check_residential_units(residential_units, buildings): # make sure everyone gets a house - this might not exist in the real world, # but due to the nature of control totals it exists here def check_no_unplaced_households(households, year): - print "Check no unplaced households" + print("Check no unplaced households") if year <= 2030: # for some reason, since we added renter/owner models, we do have # unplaced households in the first couple of years, which eventually @@ -88,7 +88,7 @@ def check_no_unplaced_households(households, year): # check not more households than units or jobs than job spaces def check_no_overfull_buildings(households, buildings): - print "Check no overfull buildings" + print("Check no overfull buildings") assert True not in (buildings.vacant_res_units < 0).value_counts() # there are overfull job spaces based on the assignment and also # proportional job model @@ -97,7 +97,7 @@ def check_no_overfull_buildings(households, buildings): # households have both unit ids and building ids - make sure they're in sync def check_unit_ids_match_building_ids(households, residential_units): - print "Check unit ids and building ids match" + print("Check unit ids and building ids match") building_ids = misc.reindex( residential_units.building_id, households.unit_id) assert_series_equal(building_ids, households.building_id, 25000) diff --git a/baus/variables.py b/baus/variables.py index fc993bbc9..5db76e519 100644 --- a/baus/variables.py +++ b/baus/variables.py @@ -2,8 +2,8 @@ import pandas as pd from urbansim.utils import misc import orca -import datasources -from utils import nearest_neighbor, groupby_random_choice +from . import datasources +from .utils import nearest_neighbor, groupby_random_choice from urbansim_defaults import utils from urbansim_defaults import variables diff --git a/scripts/average_runs.py b/scripts/average_runs.py index 3f0596f28..72e27bfc1 100644 --- a/scripts/average_runs.py +++ b/scripts/average_runs.py @@ -1,13 +1,14 @@ import pandas as pd import os import random +from functools import reduce # this script takes multiple runs and averages them in order to reduce the # impacts of stochasticity - this also runs at different sample sizes to # find out how many runs we need to do in order to reduce stochasticity -run_nums = range(334, 550) -SAMPLE_SIZES = range(1, 45, 5) +run_nums = list(range(334, 550)) +SAMPLE_SIZES = list(range(1, 45, 5)) NUM_SAMPLES = 2 flds = ("TOTHH,TOTEMP").split(',') @@ -49,9 +50,9 @@ def variability_measure(dfs, N, M): return pct_diff.unstack().fillna(0).quantile(.95) dfs = get_2040_taz_summaries(run_nums) -print "Total sims = {}".format(len(dfs)) +print("Total sims = {}".format(len(dfs))) # need to test different sample sizes for sample_size in SAMPLE_SIZES: - print sample_size - print variability_measure(dfs, NUM_SAMPLES, sample_size) + print(sample_size) + print(variability_measure(dfs, NUM_SAMPLES, sample_size)) diff --git a/scripts/backout_zoning.py b/scripts/backout_zoning.py index f2889525b..d0757a541 100644 --- a/scripts/backout_zoning.py +++ b/scripts/backout_zoning.py @@ -14,7 +14,7 @@ if "Current Scenario : 4" not in log: continue - print runnum + print(runnum) # now we know it's scenario 4 df = pd.read_csv("runs/run%d_parcel_output.csv" % runnum).\ @@ -42,8 +42,8 @@ else: max_far = pd.concat([max_far, far], axis=1).max(axis=1) -print max_dua.describe() -print max_far.describe() +print(max_dua.describe()) +print(max_far.describe()) pd.DataFrame({ "max_built_dua": max_dua, diff --git a/scripts/check_data.py b/scripts/check_data.py index 5a14b79f9..dde343b6b 100644 --- a/scripts/check_data.py +++ b/scripts/check_data.py @@ -9,24 +9,24 @@ s = buildings.parcel_id.isin(parcels.index) -print "Building's parcel id in parcel index\n", s.value_counts() +print("Building's parcel id in parcel index\n", s.value_counts()) s = households.building_id.isin(buildings.index) -print "Households with no building assigned: \n", \ - (households.building_id == -1).value_counts() -print "Household's building id in building index\n", s.value_counts() +print("Households with no building assigned: \n", \ + (households.building_id == -1).value_counts()) +print("Household's building id in building index\n", s.value_counts()) s = jobs.building_id.isin(buildings.index) -print "Jobs with no building assigned: \n", \ - (jobs.building_id == -1).value_counts() -print "Job's building id in building index\n", s.value_counts() +print("Jobs with no building assigned: \n", \ + (jobs.building_id == -1).value_counts()) +print("Job's building id in building index\n", s.value_counts()) -print "Len jobs" +print("Len jobs") -print len(jobs) +print(len(jobs)) -print "Num job spaces" +print("Num job spaces") -print buildings.job_spaces.sum() +print(buildings.job_spaces.sum()) diff --git a/scripts/check_emp_totals.py b/scripts/check_emp_totals.py index f5a1f24af..c5019d81b 100644 --- a/scripts/check_emp_totals.py +++ b/scripts/check_emp_totals.py @@ -9,7 +9,7 @@ for year in range(2010, 2045, 5): - print year + print(year) taz_df = pd.read_csv("runs/run{}_taz_summaries_{}.csv". format(runnum, year)) juris_df = pd.read_csv("runs/run{}_juris_summaries_{}.csv". diff --git a/scripts/check_hh_totals.py b/scripts/check_hh_totals.py index 827db37f8..d517f4c85 100644 --- a/scripts/check_hh_totals.py +++ b/scripts/check_hh_totals.py @@ -10,7 +10,7 @@ for year in range(2010, 2045, 5): - print year + print(year) taz_df = pd.read_csv("runs/run{}_taz_summaries_{}.csv". format(runnum, year)) juris_df = pd.read_csv("runs/run{}_juris_summaries_{}.csv". diff --git a/scripts/check_rhna.py b/scripts/check_rhna.py index 2b9027301..d7ff1f7b8 100644 --- a/scripts/check_rhna.py +++ b/scripts/check_rhna.py @@ -4,8 +4,8 @@ args = sys.argv[1:] if len(args) == 0: - print "Checks growth exceeds RHNA targets for given out year" - print "usage: " + print("Checks growth exceeds RHNA targets for given out year") + print("usage: ") sys.exit(0) runnum = int(args[0]) @@ -22,4 +22,4 @@ s = juris_df2.tothh - juris_df.tothh - controls_df.rhna14to22 -print s[s < 0] +print(s[s < 0]) diff --git a/scripts/county_summaries.py b/scripts/county_summaries.py index 091237df8..76750ce46 100644 --- a/scripts/county_summaries.py +++ b/scripts/county_summaries.py @@ -31,7 +31,7 @@ growthnotinpdas = df[(df.building_type_id <= 3) & (df.pda.isnull())].\ groupby("county").net_units.sum() pctgrowthinpdas = growthinpdas / (growthnotinpdas+growthinpdas) - print pctgrowthinpdas + print(pctgrowthinpdas) baseyear = pd.read_csv("output/baseyear_taz_summaries_2010.csv") baseyear["county"] = baseyear.zone_id.map(counties_map) diff --git a/scripts/export.py b/scripts/export.py index 256073f36..356e6dc2a 100644 --- a/scripts/export.py +++ b/scripts/export.py @@ -59,7 +59,7 @@ # get building types df["building_type"] = \ df.first_building_type_id.map({v: k for k, v in - settings["building_type_map2"].items()}) + list(settings["building_type_map2"].items())}) df["oldest_building"][df.oldest_building > 2200] = np.nan diff --git a/scripts/filter_large_projects.py b/scripts/filter_large_projects.py index 530525668..3cd1fed56 100644 --- a/scripts/filter_large_projects.py +++ b/scripts/filter_large_projects.py @@ -21,6 +21,6 @@ def readfile(year): df = grps.first() df["occurences"] = grps.size() -print len(df), df.occurences.describe() +print(len(df), df.occurences.describe()) df.to_csv("large_projects.csv") diff --git a/scripts/fix_zoning_missing_id.py b/scripts/fix_zoning_missing_id.py index f9c060b0f..944f2c64a 100644 --- a/scripts/fix_zoning_missing_id.py +++ b/scripts/fix_zoning_missing_id.py @@ -21,15 +21,15 @@ def zcsv(): zdf = z.to_frame() null_df = zdf.loc[zdf.zoning_id.isnull(), :] -print "there are " + str(len(null_df.index)) + " empty zoning ids" -print "number of parcels with null values by city:" -print null_df.tablename.value_counts() +print("there are " + str(len(null_df.index)) + " empty zoning ids") +print("number of parcels with null values by city:") +print(null_df.tablename.value_counts()) -print "number of parcels with null values by source zoning code by city:" -for ix, val in null_df.tablename.value_counts().iteritems(): +print("number of parcels with null values by source zoning code by city:") +for ix, val in null_df.tablename.value_counts().items(): if val > 5: - print ix - print null_df[null_df.tablename == ix].zoning.value_counts() + print(ix) + print(null_df[null_df.tablename == ix].zoning.value_counts()) zl_df = zl.to_frame() @@ -45,14 +45,14 @@ def zcsv(): right_on=['name', 'tablename'], left_on=['zoning', 'tablename']) mdf = mdf.set_index(mdf.geom_id) -print "replaced " + str(len(mdf.index)) + " empty zoning ids" +print("replaced " + str(len(mdf.index)) + " empty zoning ids") zdf.loc[mdf.index, 'zoning_id'] = mdf['zoning_lookup_table_id'] null_df = zdf.loc[zdf.zoning_id.isnull(), :] -print "there are " + str(len(null_df.index)) + " empty zoning ids" +print("there are " + str(len(null_df.index)) + " empty zoning ids") -print "number of parcels with null values by city:" -print null_df.tablename.value_counts() +print("number of parcels with null values by city:") +print(null_df.tablename.value_counts()) x = datetime.date.today() csvname = 'data/' + str(x.year) + '_' + str(x.month) + '_' + \ diff --git a/scripts/import_mongo.py b/scripts/import_mongo.py index d01acf971..b471a9497 100644 --- a/scripts/import_mongo.py +++ b/scripts/import_mongo.py @@ -26,5 +26,5 @@ df = np.matrix(numpy_arrays).transpose() df = pd.DataFrame(df, columns=columns) -print time.time()-t1 -print df.describe() +print(time.time()-t1) +print(df.describe()) diff --git a/scripts/make_net_from_shapefile.py b/scripts/make_net_from_shapefile.py index 71194443f..eac027e39 100644 --- a/scripts/make_net_from_shapefile.py +++ b/scripts/make_net_from_shapefile.py @@ -40,9 +40,9 @@ nodes[edge["from"]] = {"x": p1[0], "y": p1[1]} nodes[edge["to"]] = {"x": p2[0], "y": p2[1]} -store["nodes"] = pd.DataFrame(nodes.values(), index=nodes.keys()) +store["nodes"] = pd.DataFrame(list(nodes.values()), index=list(nodes.keys())) store["edges"] = pd.DataFrame(edges) -print store["nodes"].describe() -print store["nodes"].index -print store["edges"].describe() +print(store["nodes"].describe()) +print(store["nodes"].index) +print(store["edges"].describe()) diff --git a/scripts/make_shapefile.py b/scripts/make_shapefile.py index e3d856684..03bef6275 100644 --- a/scripts/make_shapefile.py +++ b/scripts/make_shapefile.py @@ -14,7 +14,7 @@ gdf = gdf[gdf.zoned_du > 0] -print len(gdf) +print(len(gdf)) gdf = gdf.reset_index() gdf["GEOM_ID"] = gdf.GEOM_ID.astype('int') diff --git a/scripts/match_city_totals.py b/scripts/match_city_totals.py index 360881861..6c7b515f7 100644 --- a/scripts/match_city_totals.py +++ b/scripts/match_city_totals.py @@ -10,7 +10,7 @@ targets = pd.read_csv("data/juris_controls.csv", index_col="Jurisdiction").Households -print "Sum of cities", targets.sum() +print("Sum of cities", targets.sum()) households = orca.get_table("households") buildings = orca.get_table("buildings") @@ -37,15 +37,15 @@ new_households.index += len(households_df) households_df = households_df.append(new_households) -print "len households", len(households_df) +print("len households", len(households_df)) actual = households_df.juris.value_counts() # now compare the two diff = (actual - targets).sort_values(ascending=False) -print diff.describe() -print diff[diff > 0].sum() -print diff[diff < 0].sum() +print(diff.describe()) +print(diff[diff > 0].sum()) +print(diff[diff < 0].sum()) overfull = diff[diff > 0] underfull = diff[diff < 0] * -1 @@ -53,9 +53,9 @@ hh_ids = np.concatenate([ np.random.choice(household_ids_by_juris.loc[juris], size=value, replace=False) - for juris, value in overfull.iteritems()]) + for juris, value in overfull.items()]) -print (diff - households_df.loc[hh_ids].juris.value_counts()).describe() +print((diff - households_df.loc[hh_ids].juris.value_counts()).describe()) # unplaced movers = households_df[households_df.building_id == -1].index @@ -79,7 +79,7 @@ def make_city_list(overfull): np.random.choice(buildings_ids_by_juris.loc[d]) for d in juris_destinations] -print len(building_destinations) +print(len(building_destinations)) # putting the unplaced in advantageous positions s = pd.Series(building_destinations, index=movers) @@ -88,7 +88,7 @@ def make_city_list(overfull): pd.DataFrame({"building_id": s}).\ to_csv("household_building_id_overrides.csv") -print s.describe() +print(s.describe()) orig_zone_id_counts = households_df.zone_id.value_counts() @@ -105,9 +105,9 @@ def make_city_list(overfull): households_df.parcel_id) diff = households_df.juris.value_counts() - targets -print diff.describe() -print diff[diff > 0].sum() -print diff[diff < 0].sum() +print(diff.describe()) +print(diff[diff > 0].sum()) +print(diff[diff < 0].sum()) -print (households_df.zone_id.value_counts() - - orig_zone_id_counts).describe() +print((households_df.zone_id.value_counts() - + orig_zone_id_counts).describe()) diff --git a/scripts/output_csv_utils.py b/scripts/output_csv_utils.py index 84a36e79b..f85cbcb17 100644 --- a/scripts/output_csv_utils.py +++ b/scripts/output_csv_utils.py @@ -10,13 +10,13 @@ def format_df(df, formatters=None, **kwargs): formatting_columns = list(set(formatters.keys()).intersection(df.columns)) df_copy = df[formatting_columns].copy() na_rep = kwargs.get('na_rep') or '' - for col, formatter in formatters.items(): + for col, formatter in list(formatters.items()): try: df[col] = df[col].apply(lambda x: na_rep if pd.isnull(x) else formatter.format(x)) except KeyError: - print('{} does not exist in the dataframe.'.format(col)) +\ - 'Ignoring the formatting specifier' + print(('{} does not exist in the dataframe.'.format(col)) +\ + 'Ignoring the formatting specifier') return df diff --git a/scripts/parcel_output_diff.py b/scripts/parcel_output_diff.py index d87931b4e..166622fb8 100644 --- a/scripts/parcel_output_diff.py +++ b/scripts/parcel_output_diff.py @@ -4,8 +4,8 @@ args = sys.argv[1:] if len(args) != 2: - print "Identify buildings which are not in the baserun" - print "Usage " + print("Identify buildings which are not in the baserun") + print("Usage ") sys.exit() scen = pd.read_csv("runs/run%d_parcel_output.csv" % int(args[0]), @@ -46,8 +46,8 @@ def close(v1, v2, tol): else: cnt3 += 1 -print "%d new developments and %d changed developments (%d unchanged)" % \ - (cnt1, cnt2, cnt3) +print("%d new developments and %d changed developments (%d unchanged)" % \ + (cnt1, cnt2, cnt3)) base = pd.DataFrame(rows).\ to_csv("runs/run%d_parcel_output_diff.csv" % int(args[0])) diff --git a/scripts/parcel_tract_assignment.py b/scripts/parcel_tract_assignment.py index 3e71eca5f..6f421c547 100644 --- a/scripts/parcel_tract_assignment.py +++ b/scripts/parcel_tract_assignment.py @@ -1,7 +1,7 @@ import pandas as pd parcels = pd.read_hdf('./bayarea_urbansim/data/2015_09_01_bayarea_v3.h5', - key=u'parcels', mode='r') + key='parcels', mode='r') parcels.head() tract_zone_xwalk = pd.read_csv('./tract2000_zone_sd.csv') @@ -20,5 +20,5 @@ parcels_clp.to_csv('parcel_tract_xwalk.csv') # checks -print(len(parcels_clp['census_tract'].unique())) -print(len(parcels_clp.index)) +print((len(parcels_clp['census_tract'].unique()))) +print((len(parcels_clp.index))) diff --git a/scripts/serve_json.py b/scripts/serve_json.py index 5586c70b0..6aa9d1f63 100644 --- a/scripts/serve_json.py +++ b/scripts/serve_json.py @@ -12,7 +12,7 @@ MAX_PARCELS_RETURNED = 5000 -print "Loading" +print("Loading") store = pd.HDFStore('data/bayarea_v3.h5') @@ -28,20 +28,20 @@ 'type2', 'type3', 'type4'] zoning = orca.get_table('zoning_baseline').to_frame(flds) -print "Ready" +print("Ready") @app.route('/extract/') def get_data(query): - print "Got query:", query + print("Got query:", query) # global parcels, buildings, households, jobs, zoning_baseline global parcels, buildings, zoning_baseline p = parcels.reset_index().query(query).set_index('parcel_id') - print "Len parcels:", len(p) + print("Len parcels:", len(p)) if len(p) > MAX_PARCELS_RETURNED: return jsonify(**{ diff --git a/scripts/vmt_compare.py b/scripts/vmt_compare.py index 7c816d31d..d1760ae23 100644 --- a/scripts/vmt_compare.py +++ b/scripts/vmt_compare.py @@ -9,7 +9,7 @@ vmt_order = ["S", "M", "MH", "H", "VH"] for runnum in [RUN1, RUN2]: - print "VMT summary for run =", runnum + print("VMT summary for run =", runnum) df = pd.read_csv('runs/run%d_parcel_output.csv' % runnum, low_memory=False) @@ -17,8 +17,8 @@ df = df[df.form == "residential"] total_units = df.net_units.sum() - print "Total units =", total_units + print("Total units =", total_units) s = df.groupby("vmt_res_cat").net_units.sum().loc[vmt_order] - print s / total_units + print(s / total_units)