From 112010d9578632d5112797f1743ecc421f5ace5f Mon Sep 17 00:00:00 2001 From: daroclark Date: Fri, 5 Sep 2025 12:42:35 +0100 Subject: [PATCH 1/5] Update rentsense_former_tenants_to_refined.py Adding a bit to only get the payment ref that has ended most recently. --- .../rentsense_former_tenants_to_refined.py | 23 +++++++++---------- 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/scripts/jobs/rentsense_former_tenants_to_refined.py b/scripts/jobs/rentsense_former_tenants_to_refined.py index ac258ba8e..ee9aa5175 100644 --- a/scripts/jobs/rentsense_former_tenants_to_refined.py +++ b/scripts/jobs/rentsense_former_tenants_to_refined.py @@ -23,6 +23,7 @@ datediff, when, ) +from pyspark.sql.window import Window from scripts.helpers.helpers import ( PARTITION_KEYS, add_import_time_columns, @@ -980,23 +981,21 @@ def export_dynamic_frame_as_xml_gzip( case_priorities, accounts4.tenancy_ref == case_priorities.tenancy_ref2, "left" ) - accounts6 = accounts5.selectExpr( - "AccountReference as AccountReference", - # "TenureType", - # "TenureTypeCode", - # "max_date as TenancyStartDate", + w = Window.partitionBy("AccountReference").orderBy(F.col("TenancyEndDate").desc()) + + accounts6 = ( + accounts5 + .withColumn("rn", F.row_number().over(w)) + .filter(F.col("rn") == 1) + .selectExpr( + "AccountReference", "TenancyEndDate", "LocalAuthority", - # "HousingOfficerName", "Patch", "'Hackney' as Region", - # "import_date as import_date", - # "tenancy_ref as TenReference", - # "is_paused_until as BreathingSpaceEndDate", - "Case when Deceased=1 then 'Y' else 'N' end as Deceased" - # "previousweekbalance" + "case when Deceased=1 then 'Y' else 'N' end as Deceased" ) - +) accounts7 = accounts6.filter("AccountReference is not null") accounts8 = accounts7.distinct() From 8bd80c0b35f29d4038f0343d9e4a24a8e9daf987 Mon Sep 17 00:00:00 2001 From: daroclark Date: Fri, 5 Sep 2025 12:51:37 +0100 Subject: [PATCH 2/5] Update rentsense_former_tenants_to_refined.py --- scripts/jobs/rentsense_former_tenants_to_refined.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/jobs/rentsense_former_tenants_to_refined.py b/scripts/jobs/rentsense_former_tenants_to_refined.py index ee9aa5175..793d728e5 100644 --- a/scripts/jobs/rentsense_former_tenants_to_refined.py +++ b/scripts/jobs/rentsense_former_tenants_to_refined.py @@ -995,7 +995,7 @@ def export_dynamic_frame_as_xml_gzip( "'Hackney' as Region", "case when Deceased=1 then 'Y' else 'N' end as Deceased" ) -) + ) accounts7 = accounts6.filter("AccountReference is not null") accounts8 = accounts7.distinct() From 81436eaef296f4a771d5e9894f6868d7a5d7b03e Mon Sep 17 00:00:00 2001 From: daroclark Date: Fri, 5 Sep 2025 12:52:44 +0100 Subject: [PATCH 3/5] Update rentsense_former_tenants_to_refined.py From d56986af67318aade9f4ab3ae27e98c6d278d39a Mon Sep 17 00:00:00 2001 From: daroclark Date: Fri, 5 Sep 2025 12:56:01 +0100 Subject: [PATCH 4/5] Update rentsense_former_tenants_to_refined.py --- .../jobs/rentsense_former_tenants_to_refined.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/scripts/jobs/rentsense_former_tenants_to_refined.py b/scripts/jobs/rentsense_former_tenants_to_refined.py index 793d728e5..f4a0ce8d3 100644 --- a/scripts/jobs/rentsense_former_tenants_to_refined.py +++ b/scripts/jobs/rentsense_former_tenants_to_refined.py @@ -987,13 +987,13 @@ def export_dynamic_frame_as_xml_gzip( accounts5 .withColumn("rn", F.row_number().over(w)) .filter(F.col("rn") == 1) - .selectExpr( - "AccountReference", - "TenancyEndDate", - "LocalAuthority", - "Patch", - "'Hackney' as Region", - "case when Deceased=1 then 'Y' else 'N' end as Deceased" + .selectExpr( + "AccountReference", + "TenancyEndDate", + "LocalAuthority", + "Patch", + "'Hackney' as Region", + "case when Deceased=1 then 'Y' else 'N' end as Deceased" ) ) accounts7 = accounts6.filter("AccountReference is not null") From 6500ab1d538be9fbbe6d4317835d0df59e087061 Mon Sep 17 00:00:00 2001 From: daroclark Date: Fri, 5 Sep 2025 14:11:25 +0100 Subject: [PATCH 5/5] Update rentsense_former_tenants_to_refined.py --- scripts/jobs/rentsense_former_tenants_to_refined.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/scripts/jobs/rentsense_former_tenants_to_refined.py b/scripts/jobs/rentsense_former_tenants_to_refined.py index f4a0ce8d3..1964e5a19 100644 --- a/scripts/jobs/rentsense_former_tenants_to_refined.py +++ b/scripts/jobs/rentsense_former_tenants_to_refined.py @@ -982,19 +982,18 @@ def export_dynamic_frame_as_xml_gzip( ) w = Window.partitionBy("AccountReference").orderBy(F.col("TenancyEndDate").desc()) - + accounts6 = ( - accounts5 - .withColumn("rn", F.row_number().over(w)) - .filter(F.col("rn") == 1) + accounts5.withColumn("rn", F.row_number().over(w)) + .filter(F.col("rn") == 1) .selectExpr( "AccountReference", "TenancyEndDate", "LocalAuthority", "Patch", "'Hackney' as Region", - "case when Deceased=1 then 'Y' else 'N' end as Deceased" - ) + "case when Deceased=1 then 'Y' else 'N' end as Deceased", + ) ) accounts7 = accounts6.filter("AccountReference is not null")