use all TAZs when using sharrow

activitysim/abm/models/trip_destination.py

@@ -30,12 +30,12 @@
 )
 from activitysim.core.configuration.base import PreprocessorSettings
 from activitysim.core.configuration.logit import LocationComponentSettings
+from activitysim.core.exceptions import DuplicateWorkflowTableError, InvalidTravelError
 from activitysim.core.interaction_sample import interaction_sample
 from activitysim.core.interaction_sample_simulate import interaction_sample_simulate
 from activitysim.core.skim_dictionary import DataFrameMatrix
 from activitysim.core.tracing import print_elapsed_time
 from activitysim.core.util import assign_in_place, reindex
-from activitysim.core.exceptions import InvalidTravelError, DuplicateWorkflowTableError
 
 logger = logging.getLogger(__name__)
 
@@ -273,12 +273,16 @@ def destination_sample(
     return choices
 
 
-def aggregate_size_term_matrix(maz_size_term_matrix, network_los):
+def aggregate_size_term_matrix(maz_size_term_matrix, network_los, all_tazs=None):
     df = maz_size_term_matrix.df
     assert ALT_DEST_TAZ not in df
 
     dest_taz = network_los.map_maz_to_taz(df.index)
     taz_size_term_matrix = df.groupby(dest_taz).sum()
+    if all_tazs is not None:
+        taz_size_term_matrix = taz_size_term_matrix.reindex(
+            all_tazs, fill_value=0
+        ).rename_axis(taz_size_term_matrix.index.name, axis=0)
 
     taz_size_term_matrix = DataFrameMatrix(taz_size_term_matrix)
 
@@ -612,7 +616,16 @@ def destination_presample(
 
     alt_dest_col_name = model_settings.ALT_DEST_COL_NAME
 
-    TAZ_size_term_matrix = aggregate_size_term_matrix(size_term_matrix, network_los)
+    if state.settings.sharrow:
+        # when using sharrow, we use the skim_dataset structure, and need to ensure
+        # that all TAZs are represented in the size_term_matrix, even those with no MAZs
+        all_tazs = state.get_dataframe("land_use_taz").index
+    else:
+        all_tazs = None
+
+    TAZ_size_term_matrix = aggregate_size_term_matrix(
+        size_term_matrix, network_los, all_tazs
+    )
 
     TRIP_ORIGIN = model_settings.TRIP_ORIGIN
     PRIMARY_DEST = model_settings.PRIMARY_DEST
@@ -627,6 +640,17 @@ def destination_presample(
         network_los.map_maz_to_taz(alternatives.index)
     ).sum()
 
+    # We now have aggregated alternatives indexed by TAZ instead of MAZ.
+    # For sharrow, we need the TAZ indexing to be "complete", i.e. include all TAZ ids,
+    # even those that had no MAZs (and so were missing from the aggregation result).
+    # this is needed because we are going to taking the entire set of TAZ alternatives
+    # as a vector which will need to align with the TAZ skims.
+    if state.settings.sharrow:
+        all_tazs = state.get_dataframe("land_use_taz").index
+        alternatives = alternatives.reindex(all_tazs, fill_value=0).rename_axis(
+            alternatives.index.name, axis=0
+        )
+
     # # i did this but after changing alt_dest_col_name to 'trip_dest' it
     # # shouldn't be needed anymore
     # alternatives.index.name = ALT_DEST_TAZ
@@ -1520,13 +1544,13 @@ def run_trip_destination(
                             """
 
                         When using the trip destination model with sharrow, it is necessary
-                        to set a value for `purpose_index_num` in the trip destination 
-                        annotate trips preprocessor.  This allows for an optimized compiled 
+                        to set a value for `purpose_index_num` in the trip destination
+                        annotate trips preprocessor.  This allows for an optimized compiled
                         lookup of the size term from the array of size terms.  The value of
-                        `purpose_index_num` should be the integer column position in the size 
-                        matrix, with usual zero-based numpy indexing semantics (i.e. the first 
+                        `purpose_index_num` should be the integer column position in the size
+                        matrix, with usual zero-based numpy indexing semantics (i.e. the first
                         column is zero).  The preprocessor expression most likely needs to be
-                        "size_terms.get_cols(df.purpose)" unless some unusual transform of 
+                        "size_terms.get_cols(df.purpose)" unless some unusual transform of
                         size terms has been employed.
 
                         """

activitysim/abm/test/trip_dest/configs/_dummy_coefficients.csv

@@ -0,0 +1,2 @@
+coefficient_name,value,constrain
+coef_one,1,T

activitysim/abm/test/trip_dest/configs/constants.yaml

@@ -0,0 +1,3 @@
+max_walk_distance: 3
+walkSpeed: 3.00
+walkThresh: 1.50

activitysim/abm/test/trip_dest/configs/destination_choice_size_terms.csv

@@ -0,0 +1,24 @@
+segment,model_selector,Acres,Population
+work_low,workplace,0,0.602938343
+work_med,workplace,0,0.487310988
+work_high,workplace,0,0.359519129
+work_veryhigh,workplace,0,0.169128839
+university,school,1,0
+preschool,school,0.07552,1
+gradeschool,school,1,0
+highschool,school,1,0
+escort,non_mandatory,0.465,0.144
+shopping,non_mandatory,1,0
+eatout,non_mandatory,0.4,0.6
+othmaint,non_mandatory,0,0.518
+social,non_mandatory,0.2,0.478
+othdiscr,non_mandatory,0.252252252,0.272272272
+atwork,atwork,0,0.258
+work,trip,0.2,0.166666667
+escort,trip,0.655,0.144
+shopping,trip,0.86,0
+eatout,trip,0.2,1
+othmaint,trip,0.001,0.518
+social,trip,0.3,0.478
+othdiscr,trip,0.252252252,0.272272272
+univ,trip,0.999001,0

activitysim/abm/test/trip_dest/configs/network_los.yaml

@@ -0,0 +1,14 @@
+read_skim_cache: False
+write_skim_cache: False
+zone_system: 2
+taz_skims: 
+  omx: min*.omx
+maz: maz.csv
+maz_to_maz:
+  tables:
+   - maz_to_maz_walk.csv
+skim_time_periods:
+    time_window: 1440
+    period_minutes: 60
+    periods: [0, 5, 9, 15, 18, 20, 24]
+    labels: ['EA', 'AM', 'MD', 'PM', 'EV', 'EA']

activitysim/abm/test/trip_dest/configs/settings.yaml

@@ -0,0 +1,56 @@
+sharrow: test
+recode_pipeline_columns: true
+
+expression_profile: False
+households_sample_size: 0
+sharrow_cache_dir: sharrow_cache
+chunk_training_mode: explicit
+check_for_variability: False
+use_shadow_pricing: False
+want_dest_choice_sample_tables: False
+trace_hh_id: 
+cleanup_trace_files_on_resume: True
+
+input_table_list:
+  - tablename: households
+    filename: households.csv
+    index_col: household_id
+    rename_columns:
+      MAZ: home_zone_id
+      hhno: household_id
+    recode_columns:
+      home_zone_id: land_use.zone_id
+    keep_columns:
+      - home_zone_id
+      - income
+  - tablename: persons
+    filename: persons.csv
+    index_col: person_id
+    rename_columns:
+      PERID: person_id
+    keep_columns:
+      - household_id
+      - age
+      - PNUM
+  - tablename: land_use # MAZ,TAZ,Acres,Population
+    filename: land_use.csv
+    index_col: zone_id
+    rename_columns:
+      MAZ: zone_id
+    recode_columns:
+      zone_id: zero-based
+      TAZ: land_use_taz.TAZ
+    keep_columns:
+      - TAZ
+      - Acres
+      - Population
+  - tablename: land_use_taz
+    filename: taz.csv
+    index_col: TAZ
+    recode_columns:
+      TAZ: zero-based
+
+resume_after:
+
+models:
+  - trip_destination

activitysim/abm/test/trip_dest/configs/trip_destination.csv

@@ -0,0 +1,10 @@
+Description,Expression,work,univ,school,escort,shopping,eatout,othmaint,social,othdiscr,atwork
+size term,"@np.log1p(size_terms.get(df.alt_dest, df.purpose)) # sharrow: np.log1p(size_terms['sizearray'])",1,1,1,1,1,1,1,1,1,1
+no attractions,"@size_terms.get(df.alt_dest, df.purpose) == 0 # sharrow: size_terms['sizearray'] == 0",-999,-999,-999,-999,-999,-999,-999,-999,-999,-999
+stop proximity to home (outbound),@df.outbound * od_skims['distwalk'],-0.3800,0,0,0,0,0,0,0,0,0
+stop proximity to home (inbound),@~df.outbound * dp_skims['distwalk'],-0.1500,0,0,0,0,0,0,0,0,0
+stop proximity to main destination (outbound),@df.outbound * dp_skims['distwalk'],-0.26,,,,,,,,,
+"Sample of alternatives correction factor","@np.minimum(np.log(df.pick_count/df.prob), 60)",1,1,1,1,1,1,1,1,1,1
+Mode choice logsum from origin to stop,od_logsum,1.821,1.821,1.821,1.821,1.821,1.821,1.821,1.821,1.821,1.821
+Mode choice logsum from stop to destination,dp_logsum,1.821,1.821,1.821,1.821,1.821,1.821,1.821,1.821,1.821,1.821
+meaty,@odt_skims['meat'],-0.01,-0.01,-0.01,-0.01,-0.01,-0.01,-0.01,-0.01,-0.01,-0.01

activitysim/abm/test/trip_dest/configs/trip_destination.yaml

@@ -0,0 +1,35 @@
+SAMPLE_SIZE: 30
+
+DESTINATION_SAMPLE_SPEC: trip_destination_sample.csv
+DESTINATION_SPEC: trip_destination.csv
+COEFFICIENTS: _dummy_coefficients.csv
+
+LOGSUM_SETTINGS: trip_mode_choice.yaml
+
+# optional (comment out if not desired)
+DEST_CHOICE_LOGSUM_COLUMN_NAME: destination_logsum
+
+# comment out DEST_CHOICE_LOGSUM_COLUMN_NAME if saved alt logsum table
+DEST_CHOICE_SAMPLE_TABLE_NAME: trip_destination_sample
+
+# model-specific logsum-related settings
+TRIP_ORIGIN: origin
+ALT_DEST_COL_NAME: alt_dest
+PRIMARY_DEST: destination
+
+REDUNDANT_TOURS_MERGED_CHOOSER_COLUMNS:
+  - tour_mode
+
+preprocessor:
+  SPEC: trip_destination_annotate_trips_preprocessor
+  DF: trips
+  TABLES:
+    - tours
+    - persons
+
+# drop failed trips and cleanup failed trip leg_mates for consistency
+# (i.e. adjust trip_count, trip_num, first for missing failed trips)
+CLEANUP: False
+
+# this setting is used by testing code to force failed trip_destination
+# fail_some_trips_for_testing: False

activitysim/abm/test/trip_dest/configs/trip_destination_annotate_trips_preprocessor.csv

@@ -0,0 +1,9 @@
+Description,Target,Expression
+,tour_mode,"reindex(tours.tour_mode, df.tour_id)"
+,_tod,"np.where(df.outbound,reindex_i(tours.start, df.tour_id),reindex_i(tours.end, df.tour_id))"
+,trip_period,network_los.skim_time_period_label(_tod)
+,is_joint,"reindex(tours.tour_category, df.tour_id)=='joint'"
+,purpose_index_num,"size_terms.get_cols(df.purpose)"
+,tour_mode_is_walk,"reindex(tours.tour_mode, df.tour_id)=='WALK'"
+,tour_mode_is_bike,"reindex(tours.tour_mode, df.tour_id)=='BIKE'"
+,tour_leg_dest,"np.where(df.outbound,reindex(tours.destination, df.tour_id), reindex(tours.origin, df.tour_id)).astype(int)"

activitysim/abm/test/trip_dest/configs/trip_destination_sample.csv

@@ -0,0 +1,8 @@
+Description,Expression,work,univ,school,escort,shopping,eatout,othmaint,social,othdiscr,atwork
+,_od_DIST@od_skims['distwalk'],1,1,1,1,1,1,1,1,1,1
+,_dp_DIST@dp_skims['distwalk'],1,1,1,1,1,1,1,1,1,1
+size term,"@np.log1p(size_terms.get(df.alt_dest, df.purpose)) # sharrow: np.log1p(size_terms['sizearray'])",1,1,1,1,1,1,1,1,1,1
+no attractions,"@size_terms.get(df.alt_dest, df.purpose) == 0 # sharrow: size_terms['sizearray'] == 0",-999,-999,-999,-999,-999,-999,-999,-999,-999,-999
+stop proximity to home (outbound),@df.outbound * _od_DIST,-0.3800,0,0,0,0,0,0,0,0,0
+stop proximity to home (inbound),@~df.outbound * _od_DIST,-0.1500,0,0,0,0,0,0,0,0,0
+stop proximity to main destination (outbound),@df.outbound * _dp_DIST,-0.26,,,,,,,,,

activitysim/abm/test/trip_dest/configs/trip_mode_choice.csv

@@ -0,0 +1,6 @@
+Label,Description,Expression,DRIVEALONEFREE,SHAREDRIDE,WALK
+#,Drive alone ,,,,
+util_DRIVEALONEFREE_Unavailable_for_persons_less_than_16,DRIVEALONEFREE - Unavailable for persons less than 16,age < 16,-999,,
+util_DRIVEALONEFREE_Person_is_between_16_and_19_years_old,DRIVEALONEFREE - Person is between 16 and 19 years old,@(df.age >= 16) & (df.age <= 19),coef_age1619_da,,
+util_SHARED2FREE_In_vehicle_time,SHARED2FREE - In-vehicle time,@odt_skims['meat'],,coef_ivt,
+util_WALK_Time_short,WALK - Time up to 1.5 miles,@od_skims['distwalk'],,,coef_walktime_short

activitysim/abm/test/trip_dest/configs/trip_mode_choice.yaml

@@ -0,0 +1,52 @@
+# trip_mode_choice.yaml
+
+compute_settings:
+  fastmath: false  # use of isnan in utility functions requires fastmath=False
+
+SPEC: trip_mode_choice.csv
+COEFFICIENTS: trip_mode_choice_coefficients.csv
+COEFFICIENT_TEMPLATE: trip_mode_choice_coefficients_template.csv
+
+LOGIT_TYPE: NL
+
+NESTS:
+  name: root
+  coefficient: coef_nest_root
+  alternatives:
+      - name: AUTO
+        coefficient: coef_nest_AUTO
+        alternatives:
+          - DRIVEALONEFREE
+          - SHAREDRIDE
+      - name: NONMOTORIZED
+        coefficient: coef_nest_NONMOTORIZED
+        alternatives:
+          - WALK
+
+CONSTANTS:
+  orig_col_name: origin
+  dest_col_name: destination
+  I_MODE_MAP:
+    DRIVEALONEFREE: 1
+    SHAREDRIDE: 2
+    WALK: 3
+  I_SOV_MODES: 1
+  I_AUTO_MODES: [1,2]
+  I_WALK_MODE: 3
+
+# so far, we can use the same spec as for non-joint tours
+preprocessor:
+  SPEC: trip_mode_choice_annotate_trips_preprocessor
+  DF: df
+  TABLES:
+    - land_use
+    - tours
+
+MODE_CHOICE_LOGSUM_COLUMN_NAME: mode_choice_logsum
+
+TOURS_MERGED_CHOOSER_COLUMNS:
+  - tour_category
+  - tour_mode
+  - start
+  - end
+  - age

activitysim/abm/test/trip_dest/configs/trip_mode_choice_annotate_trips_preprocessor.csv

@@ -0,0 +1,8 @@
+Description,Target,Expression
+,i_tour_mode,df.tour_mode.map(I_MODE_MAP)
+,tour_mode_is_SOV,i_tour_mode == I_SOV_MODES
+,tour_mode_is_auto,i_tour_mode.isin(I_AUTO_MODES)
+,tour_mode_is_walk,i_tour_mode == I_WALK_MODE
+,inbound,~df.outbound
+,first_trip,df.trip_num == 1
+,last_trip,df.trip_num == df.trip_count

activitysim/abm/test/trip_dest/configs/trip_mode_choice_coefficients.csv

@@ -0,0 +1,32 @@
+coefficient_name,value,constrain
+coef_one,1,T
+coef_zero,0,T
+coef_nest_root,1,T
+coef_nest_AUTO,0.72,T
+coef_nest_AUTO_DRIVEALONE,0.35,T
+coef_nest_AUTO_SHAREDRIDE2,0.35,T
+coef_nest_AUTO_SHAREDRIDE3,0.35,T
+coef_nest_NONMOTORIZED,0.72,T
+coef_nest_TRANSIT,0.72,T
+coef_nest_TRANSIT_WALKACCESS,0.5,T
+coef_nest_TRANSIT_DRIVEACCESS,0.5,T
+coef_nest_RIDEHAIL,0.36,T
+coef_nest_SCHOOL_BUS,0.5,T
+coef_ivt_disc,-0.03,F
+coef_ivt_maint,-0.034,F
+coef_ivt_school,-0.02,F
+coef_ivt_univ,-0.032,F
+coef_ivt_work,-0.01,F
+coef_ivt_atwork,-0.064,F
+coef_walktime_short_disc,-0.0558,F
+coef_walktime_short_maint,-0.0558,F
+coef_walktime_short_school,-0.0542,F
+coef_walktime_short_univ,-0.0542,F
+coef_walktime_short_work,-0.1,F
+coef_walktime_short_atwork,-0.0558,F
+coef_age1619_da_disc,0,F
+coef_age1619_da_maint,0,F
+coef_age1619_da_school,0,F
+coef_age1619_da_univ,0,F
+coef_age1619_da_work,0,F
+coef_age1619_da_atwork,0,F

activitysim/abm/test/trip_dest/configs/trip_mode_choice_coefficients_template.csv

@@ -0,0 +1,17 @@
+coefficient_name,eatout,escort,othdiscr,othmaint,school,shopping,social,univ,work,atwork
+coef_one,,,,,,,,,,
+coef_nest_root,,,,,,,,,,
+coef_nest_AUTO,,,,,,,,,,
+coef_nest_AUTO_DRIVEALONE,,,,,,,,,,
+coef_nest_AUTO_SHAREDRIDE2,,,,,,,,,,
+coef_nest_AUTO_SHAREDRIDE3,,,,,,,,,,
+coef_nest_NONMOTORIZED,,,,,,,,,,
+coef_nest_TRANSIT,,,,,,,,,,
+coef_nest_TRANSIT_WALKACCESS,,,,,,,,,,
+coef_nest_TRANSIT_DRIVEACCESS,,,,,,,,,,
+coef_nest_RIDEHAIL,,,,,,,,,,
+coef_nest_SCHOOL_BUS,,,,,,,,,,
+#,,,,,,,,,,
+coef_ivt,coef_ivt_disc,coef_ivt_maint,coef_ivt_disc,coef_ivt_maint,coef_ivt_school,coef_ivt_maint,coef_ivt_disc,coef_ivt_univ,coef_ivt_work,coef_ivt_atwork
+coef_walktime_short,coef_walktime_short_disc,coef_walktime_short_maint,coef_walktime_short_disc,coef_walktime_short_maint,coef_walktime_short_school,coef_walktime_short_maint,coef_walktime_short_disc,coef_walktime_short_univ,coef_walktime_short_work,coef_walktime_short_atwork
+coef_age1619_da,coef_age1619_da_disc,coef_age1619_da_maint,coef_age1619_da_disc,coef_age1619_da_maint,coef_age1619_da_school,coef_age1619_da_maint,coef_age1619_da_disc,coef_age1619_da_univ,coef_age1619_da_work,coef_age1619_da_atwork

activitysim/abm/test/trip_dest/data/households.csv

@@ -0,0 +1,11 @@
+income,hhno,MAZ
+83800,1,101
+83800,2,102
+54650,3,103
+54650,4,104
+54650,5,108
+80000,6,109
+40000,7,111
+65000,8,121
+45800,9,122
+9600,10,101