diff --git a/macros/end_date_utils.sql b/macros/end_date_utils.sql new file mode 100644 index 00000000..25f1d05e --- /dev/null +++ b/macros/end_date_utils.sql @@ -0,0 +1,61 @@ +{% macro date_within_end_date(date_col, end_date, inclusive=true) %} +{#- + Produces sql to determine if a date is within an end date or not. + + The inclusivity of the end_date is what matters here. If end_date + is inclusive and date_col is less than or equal to the end_date, + then it is within the end_date. But if the end date is exclusive and + date_col equals end_date then it is not within the end date. + + Inclusivity is assumed. + + Edfi makes no mention of whether end_dates should be considered + inclusive or exclusive. When asked they will say to do what makes + sense of your own implementation of Edfi. + + If end_date is null, then date_col is within the end date. + + Args: + date_col: the date to be compared against the end_date + end_date: the end date of some date range. + inclusive: should the end_date be considered inclusive or not + -#} + {% if inclusive %} + ({{ end_date }} is null or {{ date_col }} <= {{ end_date }}) + {% else %} + ({{ end_date }} is null or {{ date_col }} < {{ end_date }}) + {% endif %} +{% endmacro %} + +{% macro day_count_in_range(begin_date, end_date, inclusive=true) %} +{#- + Inclusivity for an end date (or begin date, for that matter) matters + when trying to determine the dates within a date range. If the end date + is exclusive, then that day does not count towards the day count. + + If the end date is null, then we will be calculating the day count as of + today AND today COUNTS as a day in the range, regardless of inclusivity. + + Args: + begin_date: the start of the date range + end_date: the end of the date range + inclusive: should the end_date be considered inclusive or not + -#} + {{ return(adapter.dispatch('day_count_in_range', 'edu_wh')(begin_date, end_date, inclusive=true)) }} +{% endmacro %} + +{% macro snowflake__day_count_in_range(begin_date, end_date, inclusive) -%} + {% if inclusive %} + date(coalesce({{ end_date }}, getdate())) - {{ begin_date }} + 1 + {% else %} + date(coalesce({{ end_date }}, getdate())) - {{ begin_date }} + {% endif %} +{%- endmacro %} + +{% macro databricks__day_count_in_range(begin_date, end_date, inclusive) -%} + {% if inclusive %} + date_diff(dateadd(day, 1, coalesce({{ end_date }}, CURRENT_DATE())), {{ begin_date }}) + {% else %} + date_diff(coalesce({{ end_date }}, dateadd(day, 1, CURRENT_DATE())), {{ begin_date }}) + {% endif %} +{%- endmacro %} \ No newline at end of file diff --git a/models/core_warehouse/fct_student_daily_attendance.sql b/models/core_warehouse/fct_student_daily_attendance.sql index 479e2445..e66ff2bb 100644 --- a/models/core_warehouse/fct_student_daily_attendance.sql +++ b/models/core_warehouse/fct_student_daily_attendance.sql @@ -125,8 +125,7 @@ fill_positive_attendance as ( fct_student_school_att.attendance_event_reason, -- set enrollment flag: 1 during enrollment, 0 after, no row prior case - when stu_enr_att_cal.calendar_date <= stu_enr_att_cal.exit_withdraw_date - or stu_enr_att_cal.exit_withdraw_date is null + when {{ date_within_end_date('stu_enr_att_cal.calendar_date', 'stu_enr_att_cal.exit_withdraw_date', var('edu:enroll:exit_withdraw_date_inclusive', True)) }} then 1.0 else 0.0 end is_enrolled, diff --git a/models/core_warehouse/fct_student_school_association.sql b/models/core_warehouse/fct_student_school_association.sql index c47f3c99..abc55efc 100644 --- a/models/core_warehouse/fct_student_school_association.sql +++ b/models/core_warehouse/fct_student_school_association.sql @@ -66,8 +66,7 @@ formatted as ( stg_stu_school.school_year = max(stg_stu_school.school_year) over(partition by stg_stu_school.tenant_code) -- not yet exited - and (exit_withdraw_date is null - or exit_withdraw_date >= current_date()) + and {{ date_within_end_date('current_date()', 'exit_withdraw_date', var('edu:enroll:exit_withdraw_date_inclusive', True)) }} -- enrollment has begun and entry_date <= current_date(), true, false @@ -113,14 +112,12 @@ formatted as ( where true {% if var('edu:enroll:exclude_exit_before_first_day', True) -%} -- exclude students who exited before the first school day - and (exit_withdraw_date >= bld_school_calendar_windows.first_school_day - or exit_withdraw_date is null + and ({{ date_within_end_date('bld_school_calendar_windows.first_school_day', 'exit_withdraw_date', var('edu:enroll:exit_withdraw_date_inclusive', True)) }} or bld_school_calendar_windows.first_school_day is null ) {% endif %} -- exclude students whose exit day is before their entry day - and (exit_withdraw_date >= entry_date - or exit_withdraw_date is null) + and ( {{ date_within_end_date('entry_date', 'exit_withdraw_date', var('edu:enroll:exit_withdraw_date_inclusive', True)) }} ) -- exclude students who never actually enrolled {% set excl_withdraw_codes = var('edu:enroll:exclude_withdraw_codes') %} {% if excl_withdraw_codes | length -%} diff --git a/models/core_warehouse/fct_student_school_attendance_event.sql b/models/core_warehouse/fct_student_school_attendance_event.sql index c12ad749..54ab4d6f 100644 --- a/models/core_warehouse/fct_student_school_attendance_event.sql +++ b/models/core_warehouse/fct_student_school_attendance_event.sql @@ -41,7 +41,7 @@ fct_student_school_assoc as ( */ select *, - date(coalesce(exit_withdraw_date,getdate())) - entry_date as enrollment_length + {{ day_count_in_range('entry_date', 'exit_withdraw_date', var('edu:enroll:exit_withdraw_date_inclusive', True)) }} as enrollment_length from {{ ref('fct_student_school_association') }} ), xwalk_att_events as ( @@ -79,7 +79,8 @@ joined as ( join dim_calendar_date on fct_student_school_assoc.k_school_calendar = dim_calendar_date.k_school_calendar and stg_stu_sch_attend.attendance_event_date = dim_calendar_date.calendar_date - and dim_calendar_date.calendar_date between fct_student_school_assoc.entry_date and coalesce(fct_student_school_assoc.exit_withdraw_date,current_date()) + and dim_calendar_date.calendar_date >= fct_student_school_assoc.entry_date + and {{ date_within_end_date('dim_calendar_date.calendar_date', 'fct_student_school_assoc.exit_withdraw_date', var('edu:enroll:exit_withdraw_date_inclusive', True)) }} join xwalk_att_events on stg_stu_sch_attend.attendance_event_category = xwalk_att_events.attendance_event_descriptor ), diff --git a/models/qc/sections_per_enrollment.sql b/models/qc/sections_per_enrollment.sql index 4acdc368..47efe6aa 100644 --- a/models/qc/sections_per_enrollment.sql +++ b/models/qc/sections_per_enrollment.sql @@ -19,16 +19,12 @@ left join fct_student_section_association and fct_student_school_association.k_school = fct_student_section_association.k_school and ( ( - fct_student_section_association.begin_date - between fct_student_school_association.entry_date and coalesce( - fct_student_school_association.exit_withdraw_date, current_date() - ) + fct_student_section_association.begin_date >= fct_student_school_association.entry_date + and {{ date_within_end_date('fct_student_section_association.begin_date', 'fct_student_school_association.exit_withdraw_date', var('edu:enroll:exit_withdraw_date_inclusive', True)) }} ) or ( - fct_student_section_association.end_date - between fct_student_school_association.entry_date and coalesce( - fct_student_school_association.exit_withdraw_date, current_date() - ) + fct_student_section_association.end_date >= fct_student_school_association.entry_date + and {{ date_within_end_date('fct_student_section_association.end_date', 'fct_student_school_association.exit_withdraw_date', var('edu:enroll:exit_withdraw_date_inclusive', True)) }} ) ) group by 1, 2, 3, 4, 5 diff --git a/tests/integrity_tests/all_student_school_associations_have_calendars.sql b/tests/integrity_tests/all_student_school_associations_have_calendars.sql index 64bf39b3..e427ba7b 100644 --- a/tests/integrity_tests/all_student_school_associations_have_calendars.sql +++ b/tests/integrity_tests/all_student_school_associations_have_calendars.sql @@ -55,8 +55,6 @@ join first_school_day and first_school_day.k_school_calendar is null where true -- exclude students who exited before first day of school -and (fct_stu_school_assoc.exit_withdraw_date > first_school_day.first_school_day -or fct_stu_school_assoc.exit_withdraw_date is null) - +and {{ date_within_end_date('first_school_day.first_school_day', 'fct_stu_school_assoc.exit_withdraw_date', var('edu:enroll:exit_withdraw_date_inclusive', True)) }} group by 1,2,3 having n_students_missing_calendar != 0 \ No newline at end of file diff --git a/tests/value_tests/invalid_enrollments.sql b/tests/value_tests/invalid_enrollments.sql index 11bf2bd9..e55a865a 100644 --- a/tests/value_tests/invalid_enrollments.sql +++ b/tests/value_tests/invalid_enrollments.sql @@ -31,8 +31,8 @@ first_school_day as ( group by 1 ) select - exit_withdraw_date < first_school_day.calendar_date as exit_before_first_day, - exit_withdraw_date < entry_date as exit_before_entry, + not ( {{ date_within_end_date('first_school_day.calendar_date', 'exit_withdraw_date', var('edu:enroll:exit_withdraw_date_inclusive', True)) }} ) as exit_before_first_day, + not ( {{ date_within_end_date('entry_date', 'exit_withdraw_date', var('edu:enroll:exit_withdraw_date_inclusive', True)) }} ) as exit_before_entry, {% set excl_withdraw_codes = var('edu:enroll:exclude_withdraw_codes') %} {% if excl_withdraw_codes | length -%} {% if excl_withdraw_codes is string -%} diff --git a/tests/value_tests/overlapping_enrollments.sql b/tests/value_tests/overlapping_enrollments.sql index bc20cc2c..61ccdf7d 100644 --- a/tests/value_tests/overlapping_enrollments.sql +++ b/tests/value_tests/overlapping_enrollments.sql @@ -38,9 +38,8 @@ counted as ( from dim_calendar_date join fct_student_school_assoc on dim_calendar_date.k_school_calendar = fct_student_school_assoc.k_school_calendar - and dim_calendar_date.calendar_date between - fct_student_school_assoc.entry_date and - coalesce(fct_student_school_assoc.exit_withdraw_date, current_date()) + and dim_calendar_date.calendar_date >= fct_student_school_assoc.entry_date + and {{ date_within_end_date('dim_calendar_date.calendar_date', 'fct_student_school_assoc.exit_withdraw_date', var('edu:enroll:exit_withdraw_date_inclusive', True)) }} group by 1,2,3 having duplicate_days > 1 ),