diff --git a/dbt_project.yml b/dbt_project.yml index ef064d31..75c21c99 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -40,6 +40,11 @@ models: +materialized: view stage: +materialized: table + sedm: + base: + +materialized: view + stage: + +materialized: table vars: extensions: null diff --git a/macros/gen_skey.sql b/macros/gen_skey.sql index 4fd8dd1a..e052ad7f 100644 --- a/macros/gen_skey.sql +++ b/macros/gen_skey.sql @@ -337,6 +337,28 @@ 'col_list': ['post_secondary_institution_id'], 'annualize': False }, + 'k_student_iep': { + 'reference_name': 'student_iep_reference', + 'col_list': [ + 'iep_finalized_date', + 'iep_servicing_ed_org_id', + 'student_iep_association_id', + 'student_unique_id'], + 'annualize': True + }, + 'k_student_iep_service_prescription': { + 'reference_name': 'student_iep_reference', + 'col_list': [ + 'iep_service_id', + 'iep_servicing_ed_org_id', + 'service_prescription', + 'service_prescription_date', + 'student_iep_association_id', + 'student_unique_id' + ], + 'annualize': False + }, + 'k_template': { 'reference_name': '', 'col_list': [], diff --git a/models/staging/sedm/base/_sedm__base.yml b/models/staging/sedm/base/_sedm__base.yml new file mode 100644 index 00000000..82236e2b --- /dev/null +++ b/models/staging/sedm/base/_sedm__base.yml @@ -0,0 +1,32 @@ +version: 2 + +models: + - name: base_sedm__student_iep_associations + config: + tags: ['sedm'] + enabled: "{{ var('src:domain:sedm:enabled', False) }}" + - name: base_sedm__ideaevents + config: + tags: ['sedm'] + enabled: "{{ var('src:domain:sedm:enabled', False) }}" + - name: base_sedm__student_iep_goals + config: + tags: ['sedm'] + enabled: "{{ var('src:domain:sedm:enabled', False) }}" + - name: base_sedm__student_iep_service_prescriptions + config: + tags: ['sedm'] + enabled: "{{ var('src:domain:sedm:enabled', False) }}" + - name: base_sedm__student_iep_service_deliveries + config: + tags: ['sedm'] + enabled: "{{ var('src:domain:sedm:enabled', False) }}" + - name: base_sedm__student_iep_disability_collections + config: + tags: ['sedm'] + enabled: "{{ var('src:domain:sedm:enabled', False) }}" + - name: base_sedm__student_iep_accommodation_collections + config: + tags: ['sedm'] + enabled: "{{ var('src:domain:sedm:enabled', False) }}" + diff --git a/models/staging/sedm/base/base_sedm__idea_events.sql b/models/staging/sedm/base/base_sedm__idea_events.sql new file mode 100644 index 00000000..d2579e28 --- /dev/null +++ b/models/staging/sedm/base/base_sedm__idea_events.sql @@ -0,0 +1,40 @@ +with idea_events as ( + {{ source_edfi3('idea_events') }} +), + +renamed as ( + select + tenant_code, + api_year, + pull_timestamp, + last_modified_timestamp, + file_row_number, + filename, + is_deleted, + v:id::string as record_guid, + ods_version, + data_model_version, + -- key columns + v:ideaEventID::string as idea_event_id, + v:studentReference:studentUniqueId::string as student_unique_id, + v:educationOrganizationReference:educationOrganizationId::int as ed_org_id, + v:educationOrganizationReference:link:rel::string as ed_org_type, + {{ extract_descriptor('v:ideaEventDescriptor::string') }} as idea_event, + -- value columns + v:eventBeginDate::date as event_begin_date, + v:eventEndDate::date as event_end_date, + v:eventNarrative::string as event_narrative, + -- descriptors + {{ extract_descriptor('v:eventReasonDescriptor::string') }} as event_reason, + {{ extract_descriptor('v:eventComplianceDescriptor::string') }} as event_compliance, + -- references + v:educationOrganizationReference as education_organization_reference, + v:studentReference as student_reference, + + -- edfi extensions + v:_ext as v_ext + + from idea_events +) + +select * from renamed diff --git a/models/staging/sedm/base/base_sedm__student_iep_accommodation_collections.sql b/models/staging/sedm/base/base_sedm__student_iep_accommodation_collections.sql new file mode 100644 index 00000000..83c78f99 --- /dev/null +++ b/models/staging/sedm/base/base_sedm__student_iep_accommodation_collections.sql @@ -0,0 +1,34 @@ +with student_iep_accommodation_collections as ( + {{ source_edfi3('student_iep_accommodation_collections') }} +), + +renamed as ( + select + tenant_code, + api_year, + pull_timestamp, + last_modified_timestamp, + file_row_number, + filename, + is_deleted, + v:id::string as record_guid, + ods_version, + data_model_version, + -- key columns + v:studentIEPReference:studentIEPAssociationID::string as student_iep_association_id, + v:studentReference:studentUniqueId::string as student_unique_id, + v:studentIEPReference:educationOrganizationReference:educationOrganizationId::int as iep_servicing_ed_org_id, + -- references + v:studentReference as student_reference, + v:studentIEPReference as student_iep_reference, + -- lists + v:accommodations as v_accommodations, + + -- edfi extensions + v:_ext as v_ext + + from student_iep_accommodation_collections +) + +select * from renamed + diff --git a/models/staging/sedm/base/base_sedm__student_iep_disability_collections.sql b/models/staging/sedm/base/base_sedm__student_iep_disability_collections.sql new file mode 100644 index 00000000..2b8dd1f5 --- /dev/null +++ b/models/staging/sedm/base/base_sedm__student_iep_disability_collections.sql @@ -0,0 +1,33 @@ +with student_iep_disability_collections as ( + {{ source_edfi3('student_iep_disability_collections') }} +), + +renamed as ( + select + tenant_code, + api_year, + pull_timestamp, + last_modified_timestamp, + file_row_number, + filename, + is_deleted, + v:id::string as record_guid, + ods_version, + data_model_version, + -- key columns + v:studentIEPReference:studentIEPAssociationID::string as student_iep_association_id, + v:studentReference:studentUniqueId::string as student_unique_id, + v:studentIEPReference:educationOrganizationReference:educationOrganizationId::int as iep_servicing_ed_org_id, + -- references + v:studentReference as student_reference, + v:studentIEPReference as student_iep_reference, + -- lists + v:disabilities as v_disabilities, + + -- edfi extensions + v:_ext as v_ext + + from student_iep_disability_collections +) + +select * from renamed diff --git a/models/staging/sedm/base/base_sedm__student_iep_goals.sql b/models/staging/sedm/base/base_sedm__student_iep_goals.sql new file mode 100644 index 00000000..8815fe40 --- /dev/null +++ b/models/staging/sedm/base/base_sedm__student_iep_goals.sql @@ -0,0 +1,38 @@ +with student_iep_goals as ( + {{ source_edfi3('student_iep_goals') }} +), + +renamed as ( + select + tenant_code, + api_year, + pull_timestamp, + last_modified_timestamp, + file_row_number, + filename, + is_deleted, + v:id::string as record_guid, + ods_version, + data_model_version, + -- key columns + v:iepGoalID::string as iep_goal_id, + v:studentReference:studentUniqueId::string as student_unique_id, + v:studentIEPReference:studentIEPAssociationID::string as student_iep_association_id, + -- value columns + v:iepGoalDetails::string as iep_goal_details, + v:goalAchievementPeriodBeginDate::date as goal_achievement_period_begin_date, + v:goalAchievementPeriodEndDate::date as goal_achievement_period_end_date, + -- descriptors + {{ extract_descriptor('v:iepGoalDescriptor::string') }} as iep_goal, + -- references + v:studentReference as student_reference, + v:studentIEPReference as student_iep_reference, + v:ideaeventReferences as ideaevent_references, + + -- edfi extensions + v:_ext as v_ext + + from student_iep_goals +) + +select * from renamed diff --git a/models/staging/sedm/base/base_sedm__student_iep_service_deliveries.sql b/models/staging/sedm/base/base_sedm__student_iep_service_deliveries.sql new file mode 100644 index 00000000..fb7396ed --- /dev/null +++ b/models/staging/sedm/base/base_sedm__student_iep_service_deliveries.sql @@ -0,0 +1,46 @@ +with student_iep_service_deliveries as ( + {{ source_edfi3('student_iep_service_deliveries') }} +), + +renamed as ( + select + -- generic columns + tenant_code, + api_year, + pull_timestamp, + last_modified_timestamp, + file_row_number, + filename, + is_deleted, + v:id::string as record_guid, + ods_version, + data_model_version, + -- key columns + v:iepServiceDeliveryID::string as iep_service_delivery_id, + v:studentReference:studentUniqueId::string as student_unique_id, + v:studentIEPReference:studentIEPAssociationID::string as student_iep_association_id, + {{ extract_descriptor('v:serviceDeliveryDksescriptor::string') }} as service_delivery, + v:serviceDeliveryDate::date as service_delivery_date, + -- value columns + v:studentIEPServicePrescriptionReference:iepServiceID::string as iep_service_prescription_id, + v:studentIEPServicePrescriptionReference:servicePrescriptionDate::date as service_prescription_date, + v:serviceProviderStaffReference:staffUniqueId::string as service_provider_staff_unique_id, + -- descriptors + {{ extract_descriptor('v:serviceProviderDescriptor::string') }} as service_provider_type, + -- references + v:studentReference as student_reference, + v:studentIEPReference as student_iep_reference, + v:studentIEPServicePrescriptionReference as student_iep_service_prescription_reference, + v:serviceProviderStaffReference as service_provider_staff_reference, + v:ideaeventReferences as ideaevent_references, + -- lists + v:serviceProviders as v_service_providers, + v:externalServiceProviders as v_external_service_providers, + + -- edfi extensions + v:_ext as v_ext + + from student_iep_service_deliveries +) + +select * from renamed diff --git a/models/staging/sedm/base/base_sedm__student_iep_service_prescriptions.sql b/models/staging/sedm/base/base_sedm__student_iep_service_prescriptions.sql new file mode 100644 index 00000000..bb3c9d8f --- /dev/null +++ b/models/staging/sedm/base/base_sedm__student_iep_service_prescriptions.sql @@ -0,0 +1,48 @@ +with student_iep_service_prescriptions as ( + {{ source_edfi3('student_iep_service_prescriptions') }} +), + +renamed as ( + select + tenant_code, + api_year, + pull_timestamp, + last_modified_timestamp, + file_row_number, + filename, + is_deleted, + v:id::string as record_guid, + ods_version, + data_model_version, + -- key columns + v:iepServiceID::string as iep_service_id, + v:studentReference:studentUniqueId::string as student_unique_id, + v:studentIEPReference:studentIEPAssociationID::string as student_iep_association_id, + v:studentIEPReference:educationOrganizationReference:educationOrganizationId::int as iep_servicing_ed_org_id, + v:studentIEPReference:iepFinalizedDate::date as iep_finalized_date, + {{ extract_descriptor('v:servicePrescriptionDescriptor::string') }} as service_prescription, + v:servicePrescriptionDate::date as service_prescription_date, + -- value columns + v:beginDate::date as begin_date, + v:endDate::date as end_date, + v:durationMinutes::float as duration_minutes, + v:frequencyValue::float as frequency_value, + v:serviceEducationOrganizationID::int as service_ed_org_id, + v:serviceProviderStaffID::string as service_provider_staff_id, + -- descriptors + {{ extract_descriptor('v:durationPeriodDescriptor::string') }} as duration_period, + {{ extract_descriptor('v:frequencyPeriodDescriptor::string') }} as frequency_period, + {{ extract_descriptor('v:serviceLocationTypeDescriptor::string') }} as service_location_type, + -- references + v:studentReference as student_reference, + v:studentIEPReference as student_iep_reference, + v:serviceProviderStaffReference as service_provider_staff_reference, + v:ideaeventReferences as ideaevent_references, + + -- edfi extensions + v:_ext as v_ext + + from student_iep_service_prescriptions +) + +select * from renamed diff --git a/models/staging/sedm/base/base_sedm__student_ieps.sql b/models/staging/sedm/base/base_sedm__student_ieps.sql new file mode 100644 index 00000000..b8b83c99 --- /dev/null +++ b/models/staging/sedm/base/base_sedm__student_ieps.sql @@ -0,0 +1,50 @@ +with student_ieps as ( + {{ source_edfi3('student_ieps') }} +), + +renamed as ( + select + -- generic columns + tenant_code, + api_year, + pull_timestamp, + last_modified_timestamp, + file_row_number, + filename, + is_deleted, + v:id::string as record_guid, + ods_version, + data_model_version, + -- key columns + v:studentIEPAssociationID::string as student_iep_association_id, + v:educationOrganizationReference:educationOrganizationId::int as iep_servicing_ed_org_id, + v:educationOrganizationReference:link:rel::string as iep_servicing_ed_org_type, + v:iepFinalizedDate::date as iep_finalized_date, + v:studentReference:studentUniqueId::string as student_unique_id, + -- value columns + v:iepBeginDate::date as iep_begin_date, + v:iepEndDate::date as iep_end_date, + v:iepAmendedDate::date as iep_amended_date, + v:medicallyFragile::boolean as is_medically_fragile, + v:multiplyDisabled::boolean as is_multiply_disabled, + v:schoolHoursPerWeek::float as school_hours_per_week, + v:specialEducationHoursPerWeek::float as spec_ed_hours_per_week, + -- descriptors + {{ extract_descriptor('v:iepStatusDescriptor::string') }} as iep_status, + {{ extract_descriptor('v:reasonExitedDescriptor::string') }} as reason_exited, + {{ extract_descriptor('v:specialEducationSettingDescriptor::string') }} as special_education_setting, + -- references + v:educationOrganizationReference as education_organization_reference, + v:studentReference as student_reference, + v:studentSpecialEducationProgramAssociationReference as student_spec_ed_program_association_reference, + v:studentSpecialEducationProgramEligibilityAssociationReference as student_spec_ed_program_eligibility_association_reference, + -- lists + v:ideaEvents as v_idea_events, + -- edfi extensions + v:_ext as v_ext + + from student_ieps +) + +select * from renamed + diff --git a/models/staging/sedm/stage/stg_sedm__idea_events.sql b/models/staging/sedm/stage/stg_sedm__idea_events.sql new file mode 100644 index 00000000..0483498a --- /dev/null +++ b/models/staging/sedm/stage/stg_sedm__idea_events.sql @@ -0,0 +1,33 @@ +with base_idea_events as ( + select * from {{ ref('base_sedm__idea_events') }} +), + +keyed as ( + select + {{ dbt_utils.generate_surrogate_key( + [ + 'tenant_code', + 'api_year', + 'idea_event_id', + 'student_unique_id' + 'idea_event' + ] + ) }} as k_idea_event, + {{ gen_skey('k_student') }}, + {{ gen_skey('k_student_xyear') }}, + base_idea_events.* + {{ extract_extension(model_name=this.name, flatten=True) }} + + from base_idea_events +), + +deduped as ( + {{ dbt_utils.deduplicate( + relation='keyed', + partition_by='k_idea_event', + order_by='last_modified_timestamp desc, pull_timestamp desc' + ) }} +) + +select * from deduped +where not is_deleted diff --git a/models/staging/sedm/stage/stg_sedm__student_iep_accommodation_collections.sql b/models/staging/sedm/stage/stg_sedm__student_iep_accommodation_collections.sql new file mode 100644 index 00000000..f48407f0 --- /dev/null +++ b/models/staging/sedm/stage/stg_sedm__student_iep_accommodation_collections.sql @@ -0,0 +1,34 @@ +with base_student_iep_accommodation_collections as ( + select * from {{ ref('base_sedm__student_iep_accommodation_collections') }} +), + +keyed as ( + select + {{ dbt_utils.generate_surrogate_key( + [ + 'tenant_code', + 'api_year', + 'lower(student_iep_association_id)', + 'lower(student_unique_id)', + 'iep_servicing_ed_org_id' + ] + ) }} as k_student_iep_accommodation_collection, + {{ gen_skey('k_student') }}, + {{ gen_skey('k_student_xyear') }}, + {{ gen_skey('k_student_iep') }}, + base_student_iep_accommodation_collections.* + {{ extract_extension(model_name=this.name, flatten=True) }} + + from base_student_iep_accommodation_collections +), + +deduped as ( + {{ dbt_utils.deduplicate( + relation='keyed', + partition_by='k_student_iep_accommodation_collection', + order_by='last_modified_timestamp desc, pull_timestamp desc' + ) }} +) + +select * from deduped +where not is_deleted diff --git a/models/staging/sedm/stage/stg_sedm__student_iep_disability_collections.sql b/models/staging/sedm/stage/stg_sedm__student_iep_disability_collections.sql new file mode 100644 index 00000000..34d615c0 --- /dev/null +++ b/models/staging/sedm/stage/stg_sedm__student_iep_disability_collections.sql @@ -0,0 +1,35 @@ +with base_student_iep_disability_collections as ( + select * from {{ ref('base_sedm__student_iep_disability_collections') }} +), + +keyed as ( + select + {{ dbt_utils.generate_surrogate_key( + [ + 'tenant_code', + 'api_year', + 'lower(student_iep_association_id)', + 'lower(student_unique_id)', + 'iep_servicing_ed_org_id' + ] + ) }} as k_student_iep_disability_collection, + {{ gen_skey('k_student') }}, + {{ gen_skey('k_student_xyear') }}, + {{ gen_skey('k_student_iep') }}, + api_year as school_year, + base_student_iep_disability_collections.* + {{ extract_extension(model_name=this.name, flatten=True) }} + + from base_student_iep_disability_collections +), + +deduped as ( + {{ dbt_utils.deduplicate( + relation='keyed', + partition_by='k_student_iep_disability_collection', + order_by='last_modified_timestamp desc, pull_timestamp desc' + ) }} +) + +select * from deduped +where not is_deleted diff --git a/models/staging/sedm/stage/stg_sedm__student_iep_goals.sql b/models/staging/sedm/stage/stg_sedm__student_iep_goals.sql new file mode 100644 index 00000000..ee27808b --- /dev/null +++ b/models/staging/sedm/stage/stg_sedm__student_iep_goals.sql @@ -0,0 +1,35 @@ +with base_student_iep_goals as ( + select * from {{ ref('base_sedm__student_iep_goals') }} +), + +keyed as ( + select + {{ dbt_utils.generate_surrogate_key( + [ + 'tenant_code', + 'api_year', + 'lower(iep_goal_id)', + 'lower(student_unique_id)', + 'lower(student_iep_association_id)' + ] + ) }} as k_student_iep_goal, + {{ gen_skey('k_student') }}, + {{ gen_skey('k_student_xyear') }}, + {{ gen_skey('k_student_iep') }}, + api_year as school_year, + base_student_iep_goals.* + {{ extract_extension(model_name=this.name, flatten=True) }} + + from base_student_iep_goals +), + +deduped as ( + {{ dbt_utils.deduplicate( + relation='keyed', + partition_by='k_student_iep_goal', + order_by='last_modified_timestamp desc, pull_timestamp desc' + ) }} +) + +select * from deduped +where not is_deleted diff --git a/models/staging/sedm/stage/stg_sedm__student_iep_service_deliveries.sql b/models/staging/sedm/stage/stg_sedm__student_iep_service_deliveries.sql new file mode 100644 index 00000000..77371a35 --- /dev/null +++ b/models/staging/sedm/stage/stg_sedm__student_iep_service_deliveries.sql @@ -0,0 +1,39 @@ +with base_student_iep_service_deliveries as ( + select * from {{ ref('base_sedm__student_iep_service_deliveries') }} +), + +keyed as ( + select + {{ dbt_utils.generate_surrogate_key( + [ + 'tenant_code', + 'api_year', + 'lower(iep_service_delivery_id)', + 'lower(student_unique_id)', + 'lower(student_iep_association_id)', + 'lower(service_delivery)', + 'service_delivery_date' + ] + ) }} as k_student_iep_service_delivery, + {{ gen_skey('k_student') }}, + {{ gen_skey('k_student_xyear') }}, + {{ gen_skey('k_student_iep') }}, + {{ gen_skey('k_student_iep_service_prescription') }}, + {{ gen_skey('k_staff', alt_ref='service_provider_staff_reference') }}, + api_year as school_year, + base_student_iep_service_deliveries.* + {{ extract_extension(model_name=this.name, flatten=True) }} + + from base_student_iep_service_deliveries +), + +deduped as ( + {{ dbt_utils.deduplicate( + relation='keyed', + partition_by='k_student_iep_service_delivery', + order_by='last_modified_timestamp desc, pull_timestamp desc' + ) }} +) + +select * from deduped +where not is_deleted diff --git a/models/staging/sedm/stage/stg_sedm__student_iep_service_prescriptions.sql b/models/staging/sedm/stage/stg_sedm__student_iep_service_prescriptions.sql new file mode 100644 index 00000000..c050bb35 --- /dev/null +++ b/models/staging/sedm/stage/stg_sedm__student_iep_service_prescriptions.sql @@ -0,0 +1,39 @@ +with base_student_iep_service_prescriptions as ( + select * from {{ ref('base_sedm__student_iep_service_prescriptions') }} +), + +keyed as ( + select + {{ dbt_utils.generate_surrogate_key( + [ + 'tenant_code', + 'api_year', + 'lower(iep_service_id)', + 'lower(iep_servicing_ed_org_id)', + 'lower(service_prescription)', + 'service_prescription_date', + 'lower(student_iep_association_id)', + 'lower(student_unique_id)' + ] + ) }} as k_student_iep_service_prescription, + {{ gen_skey('k_student') }}, + {{ gen_skey('k_student_xyear') }}, + {{ gen_skey('k_student_iep') }}, + {{ gen_skey('k_staff', alt_ref='service_provider_staff_reference') }}, + api_year as school_year, + base_student_iep_service_prescriptions.* + {{ extract_extension(model_name=this.name, flatten=True) }} + + from base_student_iep_service_prescriptions +), + +deduped as ( + {{ dbt_utils.deduplicate( + relation='keyed', + partition_by='k_student_iep_service_prescription', + order_by='last_modified_timestamp desc, pull_timestamp desc' + ) }} +) + +select * from deduped +where not is_deleted diff --git a/models/staging/sedm/stage/stg_sedm__student_ieps.sql b/models/staging/sedm/stage/stg_sedm__student_ieps.sql new file mode 100644 index 00000000..b29b1d32 --- /dev/null +++ b/models/staging/sedm/stage/stg_sedm__student_ieps.sql @@ -0,0 +1,36 @@ +with base_student_ieps as ( + select * from {{ ref('base_sedm__student_ieps') }} +), + +keyed as ( + select + {{ dbt_utils.generate_surrogate_key( + [ + 'tenant_code', + 'api_year', + 'lower(student_iep_association_id)', + 'iep_servicing_ed_org_id', + 'iep_finalized_date', + 'lower(student_unique_id)' + ] + ) }} as k_student_iep, + {{ gen_skey('k_student') }}, + {{ gen_skey('k_student_xyear') }}, + {{ edorg_ref(annualize=False) }}, + api_year as school_year, + base_student_ieps.* + {{ extract_extension(model_name=this.name, flatten=True) }} + + from base_student_ieps +), + +deduped as ( + {{ dbt_utils.deduplicate( + relation='keyed', + partition_by='k_student_iep', + order_by='last_modified_timestamp desc, pull_timestamp desc' + ) }} +) + +select * from deduped +where not is_deleted