diff --git a/scripts/jobs/parking/parking_cedar_fulling_total_summary.py b/scripts/jobs/parking/parking_cedar_fulling_total_summary.py index 022bda409..a2a26058b 100644 --- a/scripts/jobs/parking/parking_cedar_fulling_total_summary.py +++ b/scripts/jobs/parking/parking_cedar_fulling_total_summary.py @@ -9,6 +9,7 @@ from scripts.helpers.helpers import create_pushdown_predicate, get_glue_env_var + environment = get_glue_env_var("environment") @@ -31,8 +32,7 @@ def sparkSqlQuery(glueContext, query, mapping, transformation_ctx) -> DynamicFra database="dataplatform-" + environment + "-liberator-refined-zone", table_name="parking_cedar_payments", transformation_ctx="AmazonS3_node1625732038443", - # teporarily removed while table partitions are fixed - # push_down_predicate=create_pushdown_predicate("import_date", 7), + push_down_predicate=create_pushdown_predicate("import_date", 7), ) # Script generated for node Amazon S3 diff --git a/scripts/jobs/parking/parking_ceo_average_on_street_hrs_mins_secs.py b/scripts/jobs/parking/parking_ceo_average_on_street_hrs_mins_secs.py index 05315dfc5..ae7d06efc 100644 --- a/scripts/jobs/parking/parking_ceo_average_on_street_hrs_mins_secs.py +++ b/scripts/jobs/parking/parking_ceo_average_on_street_hrs_mins_secs.py @@ -9,6 +9,7 @@ from scripts.helpers.helpers import create_pushdown_predicate, get_glue_env_var + environment = get_glue_env_var("environment") @@ -31,8 +32,7 @@ def sparkSqlQuery(glueContext, query, mapping, transformation_ctx) -> DynamicFra database="dataplatform-" + environment + "-liberator-refined-zone", table_name="parking_ceo_on_street", transformation_ctx="AmazonS3_node1628173244776", - # teporarily removed while table partitions are fixed - # push_down_predicate=create_pushdown_predicate("import_date", 7), + push_down_predicate=create_pushdown_predicate("import_date", 7), ) # Script generated for node Amazon S3 @@ -40,8 +40,7 @@ def sparkSqlQuery(glueContext, query, mapping, transformation_ctx) -> DynamicFra database="dataplatform-" + environment + "-liberator-refined-zone", table_name="parking_ceo_summary", transformation_ctx="AmazonS3_node1638273151502", - # teporarily removed while table partitions are fixed - # push_down_predicate=create_pushdown_predicate("import_date", 7), + push_down_predicate=create_pushdown_predicate("import_date", 7), ) # Script generated for node ApplyMapping diff --git a/scripts/jobs/parking/parking_deployment_target_details.py b/scripts/jobs/parking/parking_deployment_target_details.py index 57b2afeb3..2a78def29 100644 --- a/scripts/jobs/parking/parking_deployment_target_details.py +++ b/scripts/jobs/parking/parking_deployment_target_details.py @@ -9,6 +9,7 @@ from scripts.helpers.helpers import create_pushdown_predicate, get_glue_env_var + environment = get_glue_env_var("environment") @@ -45,8 +46,7 @@ def sparkSqlQuery(glueContext, query, mapping, transformation_ctx) -> DynamicFra database="dataplatform-" + environment + "-liberator-refined-zone", table_name="parking_ceo_on_street", transformation_ctx="AmazonS3_node1633593851886", - # teporarily removed while table partitions are fixed - # push_down_predicate=create_pushdown_predicate("import_date", 7), + push_down_predicate=create_pushdown_predicate("import_date", 7), ) # Script generated for node Amazon S3 diff --git a/scripts/jobs/parking/parking_permit_street_stress.py b/scripts/jobs/parking/parking_permit_street_stress.py index f6cab093a..336a182e1 100644 --- a/scripts/jobs/parking/parking_permit_street_stress.py +++ b/scripts/jobs/parking/parking_permit_street_stress.py @@ -53,8 +53,7 @@ def sparkSqlQuery(glueContext, query, mapping, transformation_ctx) -> DynamicFra database="dataplatform-" + environment + "-liberator-refined-zone", table_name="parking_permit_denormalised_data", transformation_ctx="AmazonS3_node1681807784480", - # teporarily removed while table partitions are fixed - # push_down_predicate=create_pushdown_predicate("import_date", 7), + push_down_predicate=create_pushdown_predicate("import_date", 7), ) # Script generated for node SQL diff --git a/scripts/jobs/parking/parking_permit_street_stress_with_cpz.py b/scripts/jobs/parking/parking_permit_street_stress_with_cpz.py index 30a0f28f6..04dc31326 100644 --- a/scripts/jobs/parking/parking_permit_street_stress_with_cpz.py +++ b/scripts/jobs/parking/parking_permit_street_stress_with_cpz.py @@ -53,8 +53,7 @@ def sparkSqlQuery(glueContext, query, mapping, transformation_ctx) -> DynamicFra database="dataplatform-" + environment + "-liberator-refined-zone", table_name="parking_permit_denormalised_data", transformation_ctx="AmazonS3_node1681807784480", - # teporarily removed while table partitions are fixed - # push_down_predicate=create_pushdown_predicate("import_date", 7), + push_down_predicate=create_pushdown_predicate("import_date", 7), ) # Script generated for node SQL diff --git a/scripts/jobs/parking/parking_suspensions_processed.py b/scripts/jobs/parking/parking_suspensions_processed.py index f702a687f..feedd8337 100644 --- a/scripts/jobs/parking/parking_suspensions_processed.py +++ b/scripts/jobs/parking/parking_suspensions_processed.py @@ -45,8 +45,7 @@ def sparkSqlQuery(glueContext, query, mapping, transformation_ctx) -> DynamicFra database="dataplatform-" + environment + "-liberator-refined-zone", table_name="parking_suspension_denormalised_data", transformation_ctx="AmazonS3_node1661350417347", - # teporarily removed while table partitions are fixed - # push_down_predicate=create_pushdown_predicate("import_date", 7), + push_down_predicate=create_pushdown_predicate("import_date", 7), ) # Script generated for node SQL diff --git a/scripts/jobs/parking/parking_suspensions_processed_with_finyear.py b/scripts/jobs/parking/parking_suspensions_processed_with_finyear.py index 4e74cd9d9..f8aa67192 100644 --- a/scripts/jobs/parking/parking_suspensions_processed_with_finyear.py +++ b/scripts/jobs/parking/parking_suspensions_processed_with_finyear.py @@ -45,8 +45,7 @@ def sparkSqlQuery(glueContext, query, mapping, transformation_ctx) -> DynamicFra database="dataplatform-" + environment + "-liberator-refined-zone", table_name="parking_suspension_denormalised_data", transformation_ctx="AmazonS3_node1661350417347", - # teporarily removed while table partitions are fixed - # push_down_predicate=create_pushdown_predicate("import_date", 7), + push_down_predicate=create_pushdown_predicate("import_date", 7), ) # Script generated for node Amazon S3