Skip to content

Fix permit diesel trends using single digit day and month #2253

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Tian-2017 merged 1 commit into from
May 19, 2025
Merged

Fix permit diesel trends using single digit day and month #2253

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
107 changes: 53 additions & 54 deletions scripts/jobs/parking/parking_permit_diesel_trends_bought_in_month.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,24 +64,24 @@ def sparkSqlQuery(glueContext, query, mapping, transformation_ctx) -> DynamicFra
/*** Create the Calendar formatted data ***/
With Calendar_Data as (
SELECT
date as Calendar_date, workingday, dow, holiday,fin_year,
date as Calendar_date, workingday, dow, holiday,fin_year,
cast(substr(cast(date as string),1, 8)||'01' as date) as MonthStartDate,
CASE
When cast(substr(date, 6,2) as int) = 1 Then 'Q4' -- Jan
When cast(substr(date, 6,2) as int) = 2 Then 'Q4' -- Feb
When cast(substr(date, 6,2) as int) = 2 Then 'Q4' -- Feb
When cast(substr(date, 6,2) as int) = 3 Then 'Q4' -- March
When cast(substr(date, 6,2) as int) = 4 Then 'Q1' -- Apr
When cast(substr(date, 6,2) as int) = 5 Then 'Q1' -- May
When cast(substr(date, 6,2) as int) = 6 Then 'Q1' -- June
When cast(substr(date, 6,2) as int) = 7 Then 'Q2' -- Jul
When cast(substr(date, 6,2) as int) = 8 Then 'Q2' -- Aug
When cast(substr(date, 6,2) as int) = 4 Then 'Q1' -- Apr
When cast(substr(date, 6,2) as int) = 5 Then 'Q1' -- May
When cast(substr(date, 6,2) as int) = 6 Then 'Q1' -- June
When cast(substr(date, 6,2) as int) = 7 Then 'Q2' -- Jul
When cast(substr(date, 6,2) as int) = 8 Then 'Q2' -- Aug
When cast(substr(date, 6,2) as int) = 9 Then 'Q2' -- Sept
When cast(substr(date, 6,2) as int) = 10 Then 'Q3' -- Oct
When cast(substr(date, 6,2) as int) = 11 Then 'Q3' -- Nov
When cast(substr(date, 6,2) as int) = 10 Then 'Q3' -- Oct
When cast(substr(date, 6,2) as int) = 11 Then 'Q3' -- Nov
When cast(substr(date, 6,2) as int) = 12 Then 'Q3' -- Dec
Else ''
END as QTR,

ROW_NUMBER() OVER ( PARTITION BY date ORDER BY date, import_date DESC) row_num
FROM calendar),

Expand All @@ -94,17 +94,17 @@ def sparkSqlQuery(glueContext, query, mapping, transformation_ctx) -> DynamicFra
END as date) as Format_date, fin_year, QTR
FROM Calendar_Data
WHERE row_num = 1),

Latest_Report_Year as (
SELECT
/**** Find the Latest Financial Year ***/
CASE
/** Q1 **/
When cast(substr(cast(Format_date as string),6,2) as int) = 4 Then
When cast(substr(cast(Format_date as string),6,2) as int) = 4 Then
cast(cast(fin_year as int) - 1 as string)
When cast(substr(cast(Format_date as string),6,2) as int) = 5 Then
When cast(substr(cast(Format_date as string),6,2) as int) = 5 Then
cast(cast(fin_year as int) - 1 as string)
When cast(substr(cast(Format_date as string),6,2) as int) = 6 Then
When cast(substr(cast(Format_date as string),6,2) as int) = 6 Then
cast(cast(fin_year as int) - 1 as string)
/** Q2 **/
When cast(substr(cast(Format_date as string),6,2) as int) = 7 Then fin_year
Expand Down Expand Up @@ -140,29 +140,29 @@ def sparkSqlQuery(glueContext, query, mapping, transformation_ctx) -> DynamicFra
END as LatestQTR,

CASE
When cast(substr(cast(Format_date as string),6,2) as int) = 4 Then
cast(substr(cast(Format_date as string),1,5)||'03-01' as date)
When cast(substr(cast(Format_date as string),6,2) as int) = 5 Then
cast(substr(cast(Format_date as string),1,5)||'03-01' as date)
When cast(substr(cast(Format_date as string),6,2) as int) = 6 Then
When cast(substr(cast(Format_date as string),6,2) as int) = 4 Then
cast(substr(cast(Format_date as string),1,5)||'03-01' as date)
When cast(substr(cast(Format_date as string),6,2) as int) = 5 Then
cast(substr(cast(Format_date as string),1,5)||'03-01' as date)
When cast(substr(cast(Format_date as string),6,2) as int) = 6 Then
cast(substr(cast(Format_date as string),1,5)||'03-01' as date)
When cast(substr(cast(Format_date as string),6,2) as int) = 7 Then
cast(substr(cast(Format_date as string),1,5)||'06-01' as date)
When cast(substr(cast(Format_date as string),6,2) as int) = 8 Then
cast(substr(cast(Format_date as string),1,5)||'06-01' as date)
When cast(substr(cast(Format_date as string),6,2) as int) = 9 Then
cast(substr(cast(Format_date as string),1,5)||'06-01' as date)
When cast(substr(cast(Format_date as string),6,2) as int) = 10 Then
cast(substr(cast(Format_date as string),1,5)||'09-01' as date)
When cast(substr(cast(Format_date as string),6,2) as int) = 11 Then
cast(substr(cast(Format_date as string),1,5)||'09-01' as date)
When cast(substr(cast(Format_date as string),6,2) as int) = 12 Then
cast(substr(cast(Format_date as string),1,5)||'09-01' as date)
When cast(substr(cast(Format_date as string),6,2) as int) = 1 Then
cast(substr(cast(Format_date as string),1,5)||'12-01' as date)
When cast(substr(cast(Format_date as string),6,2) as int) = 2 Then
cast(substr(cast(Format_date as string),1,5)||'12-01' as date)
When cast(substr(cast(Format_date as string),6,2) as int) = 3 Then
When cast(substr(cast(Format_date as string),6,2) as int) = 7 Then
cast(substr(cast(Format_date as string),1,5)||'06-01' as date)
When cast(substr(cast(Format_date as string),6,2) as int) = 8 Then
cast(substr(cast(Format_date as string),1,5)||'06-01' as date)
When cast(substr(cast(Format_date as string),6,2) as int) = 9 Then
cast(substr(cast(Format_date as string),1,5)||'06-01' as date)
When cast(substr(cast(Format_date as string),6,2) as int) = 10 Then
cast(substr(cast(Format_date as string),1,5)||'09-01' as date)
When cast(substr(cast(Format_date as string),6,2) as int) = 11 Then
cast(substr(cast(Format_date as string),1,5)||'09-01' as date)
When cast(substr(cast(Format_date as string),6,2) as int) = 12 Then
cast(substr(cast(Format_date as string),1,5)||'09-01' as date)
When cast(substr(cast(Format_date as string),6,2) as int) = 1 Then
cast(substr(cast(Format_date as string),1,5)||'12-01' as date)
When cast(substr(cast(Format_date as string),6,2) as int) = 2 Then
cast(substr(cast(Format_date as string),1,5)||'12-01' as date)
When cast(substr(cast(Format_date as string),6,2) as int) = 3 Then
cast(substr(cast(Format_date as string),1,5) ||'12-01' as date)
END as Adj_Date
From CalendarFormat
Expand All @@ -172,16 +172,16 @@ def sparkSqlQuery(glueContext, query, mapping, transformation_ctx) -> DynamicFra
SELECT distinct
MonthStartDate
FROM Calendar_Data as A
WHERE MonthStartDate >= cast('2021-04-01' as date) and
WHERE MonthStartDate >= cast('2021-04-01' as date) and
MonthStartDate <= current_date),
/***********************************************************************
Collect the Permit 2019 data as benchmark
************************************************************************/
Permit_VRMS as (
Select distinct new_vrm, new_make,new_model,new_fuel,new_engine_capactiy,
Select distinct new_vrm, new_make,new_model,new_fuel,new_engine_capactiy,
new_co2_emission
From liberator_permit_vrm_update
WHERE import_date = (Select MAX(import_date) from
From liberator_permit_vrm_update
WHERE import_date = (Select MAX(import_date) from
liberator_permit_vrm_update)),
PERMIT_2019 as (
SELECT
Expand All @@ -201,9 +201,9 @@ def sparkSqlQuery(glueContext, query, mapping, transformation_ctx) -> DynamicFra
END as co2_emission
From parking_permit_denormalised_data as A
LEFT JOIN Permit_VRMS as B ON A.vrm = B.new_vrm
WHERE ImportDateTime = (Select MAX(ImportDateTime) from
WHERE ImportDateTime = (Select MAX(ImportDateTime) from
parking_permit_denormalised_data)
and cast(substr(cast(application_date as string), 1, 10) as date)
and cast(substr(cast(application_date as string), 1, 10) as date)
between cast('2020-03-01' as date) and cast('2020-03-31' as date)
and permit_type != 'Dispensation' AND latest_permit_status not IN ('Cancelled','Rejected','RENEW_REJECTED')
AND cpz_name IN ('Zone A', 'Zone D','Zone F','Zone G','Zone G2','Zone H','Zone K','Zone L',
Expand All @@ -215,7 +215,7 @@ def sparkSqlQuery(glueContext, query, mapping, transformation_ctx) -> DynamicFra
count(*) as TotalPermits,
SUM(CASE When Fuel = 'DIESEL' Then 1 Else 0 END) as DieselPermitTotal,
SUM(CASE When Fuel = 'ELECTRIC' Then 1 Else 0 END) as ELECTRICPermitTotal,

/*** Calc the percantage ***/
(cast(count(*) as decimal(10,4)) - SUM(CASE When Fuel = 'DIESEL' Then 1 Else 0 END))
/
Expand All @@ -235,18 +235,18 @@ def sparkSqlQuery(glueContext, query, mapping, transformation_ctx) -> DynamicFra
Collect the 'current' permit data, from 1st April 2021 (after COVID)
*********************************************************************************************************************/
Current_Permit as (
SELECT *
SELECT *
FROM parking_permit_denormalised_data as A
LEFT JOIN Permit_VRMS as B ON A.vrm = B.new_vrm
WHERE ImportDateTime = (Select MAX(ImportDateTime) from
WHERE ImportDateTime = (Select MAX(ImportDateTime) from
parking_permit_denormalised_data)
AND permit_type != 'Dispensation' AND latest_permit_status not IN ('Cancelled','Rejected','RENEW_REJECTED')
AND cpz_name IN ('Zone A', 'Zone D','Zone F','Zone G','Zone G2','Zone H','Zone K','Zone L',
'Zone N','Zone P','Zone Q','Zone S','Zone U','Zone T')),

Permit_Summary as (
SELECT
permit_reference, application_date, vrm, start_date, end_date,
permit_reference, application_date, vrm, start_date, end_date,
cast(substr(cast(application_date as string), 1, 8)||'01' as date) as MonthDate,
CASE
When new_vrm is NULL Then fuel
Expand All @@ -261,7 +261,7 @@ def sparkSqlQuery(glueContext, query, mapping, transformation_ctx) -> DynamicFra
ELSE new_co2_emission
END as co2_emission
FROM Current_Permit as A
WHERE cast(substr(cast(application_date as string), 1, 10) as date) >=
WHERE cast(substr(cast(application_date as string), 1, 10) as date) >=
cast('2021-04-01' as date)),

/*** Total the number of 'open' Permits & Diesel, etc Permits annd the 2019 data ***/
Expand All @@ -270,8 +270,8 @@ def sparkSqlQuery(glueContext, query, mapping, transformation_ctx) -> DynamicFra
MonthDate,
count(*) as TotalPermits,
SUM(CASE When Fuel = 'DIESEL' Then 1 Else 0 END) as DieselPermitTotal,
SUM(CASE When Fuel = 'ELECTRIC' Then 1 Else 0 END) as ELECTRICPermitTotal,
SUM(CASE When Fuel = 'ELECTRIC' Then 1 Else 0 END) as ELECTRICPermitTotal,

/*** Calc the percantage ***/
(cast(count(*) as decimal(10,4)) - SUM(CASE When Fuel = 'DIESEL' Then 1 Else 0 END))
/
Expand All @@ -294,11 +294,10 @@ def sparkSqlQuery(glueContext, query, mapping, transformation_ctx) -> DynamicFra
SELECT
*,
current_timestamp() as ImportDateTime,
replace(cast(current_date() as string),'-','') as import_date,

cast(Year(current_date) as string) as import_year,
cast(month(current_date) as string) as import_month,
cast(day(current_date) as string) as import_day
format_datetime(current_date, 'yyyy') AS import_year,
format_datetime(current_date, 'MM') AS import_month,
format_datetime(current_date, 'dd') AS import_day,
format_datetime(current_date, 'yyyyMMdd') AS import_date
FROM Permit_Report_ALL
"""
SQL_node1658765472050 = sparkSqlQuery(
Expand Down