diff --git a/models/staging/edfi_3/base/_edfi_3__base.yml b/models/staging/edfi_3/base/_edfi_3__base.yml index 518e717..d1efbca 100644 --- a/models/staging/edfi_3/base/_edfi_3__base.yml +++ b/models/staging/edfi_3/base/_edfi_3__base.yml @@ -79,9 +79,6 @@ models: - name: base_ef3__parents config: tags: ['core'] - - name: base_ef3__persons - config: - tags: ['core'] - name: base_ef3__programs config: tags: ['core'] diff --git a/models/staging/edfi_3/stage/_edfi_3__stage.yml b/models/staging/edfi_3/stage/_edfi_3__stage.yml index 617e0f5..edbdad3 100644 --- a/models/staging/edfi_3/stage/_edfi_3__stage.yml +++ b/models/staging/edfi_3/stage/_edfi_3__stage.yml @@ -401,6 +401,10 @@ models: config: tags: ['core'] + - name: stg_ef3__school_year_types + config: + tags: ['core'] + - name: stg_ef3__schools config: tags: ['core'] @@ -460,6 +464,10 @@ models: config: tags: ['core'] + - name: stg_ef3__staff_absence_events + config: + tags: ['core'] + - name: stg_ef3__staff_education_organization_assignment_associations config: tags: ['core'] diff --git a/models/staging/edfi_3/stage/stg_ef3__school_year_types.sql b/models/staging/edfi_3/stage/stg_ef3__school_year_types.sql new file mode 100644 index 0000000..29d023a --- /dev/null +++ b/models/staging/edfi_3/stage/stg_ef3__school_year_types.sql @@ -0,0 +1,28 @@ +with school_year_types as ( + select * from {{ ref('base_ef3__school_year_types') }} + where not is_deleted +), +keyed as ( + select + {{ dbt_utils.surrogate_key( + [ + 'tenant_code', + 'api_year', + 'lower(school_year)', + 'lower(current_school_year)' + ] + )}} as k_school_year_types, + api_year as school_year, + school_year_types.* + {{ extract_extension(model_name=this.name, flatten=True) }} + from school_year_types +), +deduped as ( + {{ + dbt_utils.deduplicate( + relation='keyed', + partition_by='k_school_year_types', + order_by='pull_timestamp desc' + ) + }} +) diff --git a/models/staging/edfi_3/stage/stg_ef3__staff_absence_events.sql b/models/staging/edfi_3/stage/stg_ef3__staff_absence_events.sql new file mode 100644 index 0000000..85a16bc --- /dev/null +++ b/models/staging/edfi_3/stage/stg_ef3__staff_absence_events.sql @@ -0,0 +1,28 @@ +with staff_absence_events as ( + select * from {{ ref('base_ef3__staff_absence_events') }} +), +keyed as ( + select + {{ dbt_utils.surrogate_key( + [ + 'tenant_code', + 'lower(absence_event_category)', + 'lower(survey_identifier)', + 'lower(namespace)', + 'lower(course_code)' + ] + ) }} as k_staff_absence_events, + staff_absence_events.* + {{ extract_extension(model_name=this.name, flatten=True) }} + from staff_absence_events +), +deduped as ( + {{ + dbt_utils.deduplicate( + relation='keyed', + partition_by='k_staff_absence_events', + order_by='api_year desc, pull_timestamp desc' + ) + }} +) +select * from deduped diff --git a/models/staging/edfi_3/stage/stg_ef3__survey_course_associations.sql b/models/staging/edfi_3/stage/stg_ef3__survey_course_associations.sql new file mode 100644 index 0000000..b9bdac3 --- /dev/null +++ b/models/staging/edfi_3/stage/stg_ef3__survey_course_associations.sql @@ -0,0 +1,29 @@ +with base_survey_course_associations as ( + select * from {{ ref('base_ef3__survey_course_associations') }} + where not is_deleted +), +keyed as ( + select + {{ gen_skey('k_survey') }}, + {{ gen_skey('k_course') }}, + {{ dbt_utils.surrogate_key( + [ + 'tenant_code', + 'lower(education_organization_id)', + 'lower(event_date)' + ] + ) }} as k_survey_course_associations, + base_survey_course_associations.* + {{ extract_extension(model_name=this.name, flatten=True) }} + from base_survey_course_associations +), +deduped as ( + {{ + dbt_utils.deduplicate( + relation='keyed', + partition_by='k_survey, k_course, entry_date, k_survey_course_associations', + order_by='pull_timestamp desc' + ) + }} +) +select * from deduped