Allow for geographies to exist between meta and seed geographies

.travis.yml

@@ -3,8 +3,10 @@ language: python
 sudo: false
 
 python:
-- '3.7'
-- '3.8'
+- '3.9'
+- '3.10'
+- '3.11'
+- '3.12'
 
 install:
 - wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh

examples/example_test/configs_intermediate/controls.csv

@@ -0,0 +1,11 @@
+target,geography,seed_table,importance,control_field,expression
+num_hh,TAZ,households,1000000000,HHBASE,(households.WGTP > 0) & (households.WGTP < np.inf)
+hh_size_1,TAZ,households,1000,HHSIZE1,households.NP == 1
+hh_size_2,TAZ,households,1000,HHSIZE2,households.NP == 2
+hh_size_3,TAZ,households,1000,HHSIZE3,households.NP == 3
+hh_size_4_plus,TAZ,households,1000,HHSIZE4,households.NP >= 4
+students_by_housing_type,TAZ,persons,1000,OSUFAM,persons.OSUTAG == 1
+hh_by_type,TRACT,households,100,SF,households.HTYPE == 1
+persons_occ_1,DISTRICT,persons,1000,OCCP1,persons.OCCP == 1
+persons_occ_2,REGION,persons,100,OCCP2,persons.OCCP == 2
+persons_occ_3,REGION,persons,100,OCCP3,persons.OCCP == 3

examples/example_test/configs_intermediate/logging.yaml

@@ -0,0 +1,54 @@
+# Config for logging
+# ------------------
+# See http://docs.python.org/2.7/library/logging.config.html#configuration-dictionary-schema
+
+logging:
+  version: 1
+  disable_existing_loggers: true
+
+
+  # Configuring the default (root) logger is highly recommended
+  root:
+    level: NOTSET
+    handlers: [console]
+
+  loggers:
+
+    populationsim:
+      level: DEBUG
+      handlers: [console, logfile]
+      propagate: false
+
+    orca:
+      level: WARN
+      handlers: [console, logfile]
+      propagate: false
+
+  handlers:
+
+    logfile:
+      class: logging.FileHandler
+      filename: !!python/object/apply:populationsim.core.config.log_file_path ['populationsim.log']
+      mode: w
+      formatter: fileFormatter
+      level: NOTSET
+
+    console:
+      class: logging.StreamHandler
+      stream: ext://sys.stdout
+      formatter: simpleFormatter
+      level: NOTSET
+      #level: WARN
+
+  formatters:
+
+    simpleFormatter:
+      class: logging.Formatter
+      # format: '%(levelname)s - %(name)s - %(message)s'
+      format: '%(levelname)s - %(message)s'
+      datefmt: '%d/%m/%Y %H:%M:%S'
+
+    fileFormatter:
+      class: logging.Formatter
+      format: '%(asctime)s - %(levelname)s - %(name)s - %(message)s'
+      datefmt: '%d/%m/%Y %H:%M:%S'

examples/example_test/configs_intermediate/repop_controls.csv

@@ -0,0 +1,3 @@
+target,geography,seed_table,importance,control_field,expression
+num_hh,TAZ,households,1000000000,HHBASE,(households.WGTP > 0) & (households.WGTP < np.inf)
+muggle,TAZ,households,1000,MUGGLE,households.wizard == 0

examples/example_test/configs_intermediate/settings.yaml

@@ -0,0 +1,133 @@
+# This flag turns off integerization entirely (default False)
+NO_INTEGERIZATION_EVER: True
+
+# These settings are ignored if not integerizing
+INTEGERIZE_WITH_BACKSTOPPED_CONTROLS: True
+SUB_BALANCE_WITH_FLOAT_SEED_WEIGHTS: False
+GROUP_BY_INCIDENCE_SIGNATURE: True
+INTEGERIZER_TIMEOUT: 300 # seconds
+USE_SIMUL_INTEGERIZER: True
+
+geographies: [REGION, DISTRICT, PUMA, TRACT, TAZ]
+seed_geography: PUMA
+
+# Tracing
+# ------------------------------------------------------------------
+trace_geography:
+  TAZ: 100
+  TRACT: 1
+
+
+household_weight_col: WGTP
+household_id_col: hh_id
+total_hh_control: num_hh
+max_expansion_factor: 5
+
+min_expansion_factor: 0.2
+
+#control_file_name: controls.csv
+#data_dir: data
+
+# input_pre_processor input_table_list
+input_table_list:
+  - tablename: households
+    filename : seed_households.csv
+    index_col: hh_id
+    rename_columns:
+      SERIALNO: hh_id
+  - tablename: persons
+    # expression_filename: seed_persons_expressions.csv
+    filename : seed_persons.csv
+    rename_columns:
+      SERIALNO: hh_id
+      SPORDER: per_num
+  - tablename: geo_cross_walk
+    filename : geo_cross_walk.csv
+    rename_columns:
+      TRACTCE: TRACT
+  - tablename: TAZ_control_data
+    filename : taz_controls.csv
+  - tablename: TRACT_control_data
+    filename : tract_controls.csv
+  - tablename: DISTRICT_control_data
+    filename : district_controls.csv
+  - tablename: REGION_control_data
+    filename:  region_controls.csv
+
+
+run_list:
+  steps:
+    - input_pre_processor
+    - setup_data_structures
+    - initial_seed_balancing
+    - meta_control_factoring
+    - final_seed_balancing
+    - integerize_final_seed_weights
+    - sub_balancing.geography=TRACT
+    - sub_balancing.geography=TAZ
+    - expand_households
+    - summarize
+    - write_tables
+    - write_synthetic_population
+
+  #resume_after: expand_households
+
+# "-m repop" command line option allows specification of repop run_list`
+# expand_households options are append or replace
+repop:
+  steps:
+    - input_pre_processor.table_list=repop_input_table_list
+    - repop_setup_data_structures
+    - initial_seed_balancing.final=true
+    - integerize_final_seed_weights.repop
+    - repop_balancing
+    - expand_households.repop;replace
+    - write_tables.repop
+    - write_synthetic_population.repop
+
+  resume_after: summarize
+
+repop_control_file_name: repop_controls.csv
+
+repop_input_table_list:
+  - filename : repop_taz_controls.csv
+    tablename: TAZ_control_data
+
+
+
+# output_tables can specify either a list of output tables to include or to skip
+# if neither is specified, then no checkpointed tables will be written
+
+output_tables:
+  action: include
+  tables:
+    - PUMA_weights
+    - TAZ_weights
+    - TRACT_weights
+    - trace_TAZ_weights
+    - trace_TRACT_weights
+    - summary_hh_weights
+    - summary_TAZ
+
+#    - expanded_household_ids
+
+#
+#output_tables:
+#  action: include
+#  tables:
+#    - expanded_household_ids
+
+output_synthetic_population:
+  household_id: household_id
+  households:
+    filename: synthetic_households.csv
+    columns:
+      - NP
+      - WGTP
+      - HTYPE
+  persons:
+    filename: synthetic_persons.csv
+    columns:
+      - per_num
+      - OSUTAG
+      - OCCP

examples/example_test/data_intermediate/district_controls.csv

@@ -0,0 +1,4 @@
+DISTRICT,OCCP1,HHBASE,HHSIZE1,HHSIZE2,HHSIZE3,HHSIZE4,OSUFAM
+1,720,450,90,144,144,72,318
+2,720,450,90,144,144,72,318
+3,960,600,120,192,192,96,424

examples/example_test/data_intermediate/geo_cross_walk.csv

@@ -0,0 +1,39 @@
+TAZ,TRACTCE,PUMA,DISTRICT,REGION
+100,1,600,1,1
+101,1,600,1,1
+102,1,600,1,1
+103,2,600,1,1
+104,2,600,1,1
+105,2,600,1,1
+200,21,601,1,1
+201,21,601,1,1
+202,21,601,1,1
+203,22,601,1,1
+204,22,601,1,1
+205,22,601,1,1
+300,31,602,2,1
+301,31,602,2,1
+302,31,602,2,1
+303,32,602,2,1
+304,32,602,2,1
+305,32,602,2,1
+400,41,603,2,1
+401,41,603,2,1
+402,41,603,2,1
+403,42,603,2,1
+404,42,603,2,1
+405,42,603,2,1
+500,51,604,3,1
+501,51,604,3,1
+502,51,604,3,1
+503,52,604,3,1
+504,52,604,3,1
+505,52,604,3,1
+600,61,605,3,1
+601,61,605,3,1
+602,61,605,3,1
+603,62,605,3,1
+604,62,605,3,1
+605,62,605,3,1
+#,,,,
+700,72,606,1,1

examples/example_test/data_intermediate/region_controls.csv

@@ -0,0 +1,2 @@
+REGION,OCCP2,OCCP3
+1,300,1560

examples/example_test/data_intermediate/repop_taz_controls.csv

@@ -0,0 +1,5 @@
+TAZ,TRACT,PUMA,DISTRICT,HHBASE,HHSIZE1,HHSIZE2,HHSIZE3,HHSIZE4,OSUFAM,SF,MUGGLE
+100,1,600,1,26,6,8,8,4,53,26,20
+101,1,600,1,26,6,8,8,4,53,26,21
+205,22,601,1,51,11,16,16,8,0,51,40
+301,31,602,2,26,6,8,8,4,53,26,19

examples/example_test/data_intermediate/seed_households.csv

@@ -0,0 +1,63 @@
+SERIALNO,NP,WGTP,HTYPE,PUMA,wizard
+1,1,30,1,600,1
+2,2,48,1,600,1
+3,3,48,1,600,0
+4,4,14,1,600,0
+5,5,10,1,600,0
+6,10,5,0,600,0
+101,1,30,1,601,1
+102,2,48,1,601,0
+103,3,48,1,601,0
+104,4,14,1,601,0
+105,5,10,1,601,0
+106,10,5,0,601,0
+111,1,30,1,601,0
+112,2,48,1,601,0
+113,3,48,1,601,0
+114,4,14,1,601,0
+115,5,10,1,601,0
+116,10,5,0,601,0
+201,1,30,1,602,1
+202,2,48,1,602,0
+203,3,48,1,602,0
+204,4,14,1,602,0
+205,5,10,1,602,0
+206,10,5,0,602,0
+301,1,30,1,603,1
+302,2,48,1,603,0
+303,3,48,1,603,0
+304,4,14,1,603,0
+305,5,10,1,603,0
+306,10,5,0,603,0
+311,1,30,1,603,0
+312,2,48,1,603,0
+313,3,48,1,603,0
+314,4,14,1,603,0
+315,5,10,1,603,0
+316,10,5,0,603,0
+401,1,30,1,604,1
+402,2,48,1,604,0
+403,3,48,1,604,0
+404,4,14,1,604,0
+405,5,10,1,604,0
+406,10,5,0,604,0
+411,1,30,1,604,0
+412,2,48,1,604,0
+413,3,48,1,604,0
+414,4,14,1,604,0
+415,5,10,1,604,0
+416,10,5,0,604,0
+501,1,30,1,605,1
+502,2,48,1,605,0
+503,3,48,1,605,0
+504,4,14,1,605,0
+505,5,10,1,605,0
+506,10,5,0,605,0
+511,1,30,1,605,0
+512,2,48,1,605,0
+513,3,48,1,605,0
+514,4,14,1,605,0
+515,5,10,1,605,0
+516,10,5,0,605,0
+# this hh outside of seed zones should get dropped
+600,10,5,0,606,0