Skip to content

Commit

Permalink
Use new NTD data sources for Staging and Mart monthly tables
Browse files Browse the repository at this point in the history 
- Also Format SQL files and reorder columns

[#3519]
  • Loading branch information
@erikamov
erikamov committed Nov 13, 2024
1 parent 4a5f13f commit 7e8bce2
Show file tree
Hide file tree
Showing 7 changed files with 312 additions and 319 deletions.
98 changes: 50 additions & 48 deletions warehouse/models/intermediate/ntd/int_ntd__monthly_ridership_with_adjustments_joined.sql
View file
Original file line number Diff line number Diff line change
@@ -1,60 +1,62 @@
{{ config(materialized="table") }}

with
voms as (
select * from {{ ref("int_ntd__monthly_ridership_with_adjustments_voms") }}
WITH
voms AS (
SELECT * FROM {{ ref("int_ntd__monthly_ridership_with_adjustments_voms") }}
),

vrh as (select * from {{ ref("int_ntd__monthly_ridership_with_adjustments_vrh") }}),
vrm as (select * from {{ ref("int_ntd__monthly_ridership_with_adjustments_vrm") }}),
upt as (select * from {{ ref("int_ntd__monthly_ridership_with_adjustments_upt") }}),

int_ntd__monthly_ridership_with_adjustments_joined as (
select voms.*, upt.upt, vrm.vrm, vrh.vrh
from voms
vrh AS (
SELECT * FROM {{ ref("int_ntd__monthly_ridership_with_adjustments_vrh") }}
),

full outer join
upt
on voms.ntd_id = upt.ntd_id
and voms.year = upt.year
and voms.mode = upt.mode
and voms.reporter_type = upt.reporter_type
and voms.agency = upt.agency
and voms._3_mode = upt._3_mode
and voms.period_month = upt.period_month
and voms.period_year = upt.period_year
and voms.tos = upt.tos
and voms.mode_type_of_service_status = upt.mode_type_of_service_status
vrm AS (
SELECT * FROM {{ ref("int_ntd__monthly_ridership_with_adjustments_vrm") }}
),

full outer join
vrm
on voms.ntd_id = vrm.ntd_id
and voms.year = vrm.year
and voms.mode = vrm.mode
and voms.reporter_type = vrm.reporter_type
and voms.agency = vrm.agency
and voms._3_mode = vrm._3_mode
and voms.period_month = vrm.period_month
and voms.period_year = vrm.period_year
and voms.tos = vrm.tos
and voms.mode_type_of_service_status = vrm.mode_type_of_service_status
upt AS (
SELECT * FROM {{ ref("int_ntd__monthly_ridership_with_adjustments_upt") }}
),

full outer join
vrh
on voms.ntd_id = vrh.ntd_id
and voms.year = vrh.year
and voms.mode = vrh.mode
and voms.reporter_type = vrh.reporter_type
and voms.agency = vrh.agency
and voms._3_mode = vrh._3_mode
and voms.period_month = vrh.period_month
and voms.period_year = vrh.period_year
and voms.tos = vrh.tos
and voms.mode_type_of_service_status = vrh.mode_type_of_service_status
int_ntd__monthly_ridership_with_adjustments_joined AS (
SELECT voms.*,
upt.upt,
vrm.vrm,
vrh.vrh
FROM voms
FULL OUTER JOIN upt
ON voms.ntd_id = upt.ntd_id
AND voms.mode = upt.mode
AND voms.reporter_type = upt.reporter_type
AND voms.agency = upt.agency
AND voms._3_mode = upt._3_mode
AND voms.period_month = upt.period_month
AND voms.period_year = upt.period_year
AND voms.tos = upt.tos
AND voms.mode_type_of_service_status = upt.mode_type_of_service_status
FULL OUTER JOIN vrm
ON voms.ntd_id = vrm.ntd_id
AND voms.mode = vrm.mode
AND voms.reporter_type = vrm.reporter_type
AND voms.agency = vrm.agency
AND voms._3_mode = vrm._3_mode
AND voms.period_month = vrm.period_month
AND voms.period_year = vrm.period_year
AND voms.tos = vrm.tos
AND voms.mode_type_of_service_status = vrm.mode_type_of_service_status
FULL OUTER JOIN vrh
ON voms.ntd_id = vrh.ntd_id
AND voms.mode = vrh.mode
AND voms.reporter_type = vrh.reporter_type
AND voms.agency = vrh.agency
AND voms._3_mode = vrh._3_mode
AND voms.period_month = vrh.period_month
AND voms.period_year = vrh.period_year
AND voms.tos = vrh.tos
AND voms.mode_type_of_service_status = vrh.mode_type_of_service_status
-- where voms.ntd_id not in ("10089", "20170", "30069", "90178", "90179")
-- These agencies have null for uace_cd and uza_name and perhaps are not good to
-- have in the dataset.
-- If you don't want them then add that where clause back in.
)
select *
from int_ntd__monthly_ridership_with_adjustments_joined

SELECT * FROM int_ntd__monthly_ridership_with_adjustments_joined
112 changes: 54 additions & 58 deletions warehouse/models/intermediate/ntd/int_ntd__monthly_ridership_with_adjustments_upt.sql
View file
Original file line number Diff line number Diff line change
@@ -1,72 +1,68 @@
with
source_pivoted as (
WITH
source_pivoted AS (
{{
dbt_utils.unpivot(
cast_to="int",
relation=ref("stg_ntd__monthly_ridership_with_adjustments_upt"),
relation=ref("stg_ntd_ridership_historical__complete_monthly_ridership_with_adjustments_and_estimates__upt"),
exclude=[
"uza_name",
"uace_cd",
"dt",
"ts",
"year",
"ntd_id",
"reporter_type",
"_3_mode",
"agency",
"mode_type_of_service_status",
"legacy_ntd_id",
"mode",
"_3_mode",
"mode_type_of_service_status",
"ntd_id",
"reporter_type",
"tos",
"legacy_ntd_id",
"uace_cd",
"uza_name",
"dt",
"execution_ts"
],
field_name="period",
value_name="upt",
)
}}
),
int_ntd__monthly_ridership_with_adjustments_upt as (
select
uza_name,
format("%05d", cast(uace_cd as int64)) as uace_cd,
dt as _dt,
ts,
year,
format("%05d", cast(ntd_id as int64)) as ntd_id,
legacy_ntd_id,
reporter_type,
agency,
mode_type_of_service_status,
mode,
_3_mode,
tos,
split(period, '_')[offset(2)] as period_year,
split(period, '_')[offset(1)] as period_month,
upt
from source_pivoted
where
mode in (
"DR",
"FB",
"LR",
"MB",
"SR",
"TB",
"VP",
"CB",
"RB",
"CR",
"YR",
"MG",
"MO",
"AR",
"TR",
"HR",
"OR",
"IP",
"AG",
"PB",
"CC"
)

int_ntd__monthly_ridership_with_adjustments_upt AS (
SELECT format("%05d", cast(cast(ntd_id AS NUMERIC) AS INT64)) AS ntd_id,
legacy_ntd_id,
agency,
reporter_type,
split(period, '_')[offset(2)] AS period_year,
split(period, '_')[offset(1)] AS period_month,
uza_name,
format("%05d", cast(uace_cd AS INT64)) AS uace_cd,
mode,
mode_type_of_service_status,
_3_mode,
tos,
upt,
dt AS _dt,
execution_ts
FROM source_pivoted
WHERE ntd_id IS NOT NULL
AND mode IN ("AG",
"AR",
"CB",
"CC",
"CR",
"DR",
"FB",
"HR",
"IP",
"LR",
"MB",
"MG",
"MO",
"OR",
"PB",
"RB",
"SR",
"TB",
"TR",
"VP",
"YR")
)
select *
from int_ntd__monthly_ridership_with_adjustments_upt

SELECT * FROM int_ntd__monthly_ridership_with_adjustments_upt
112 changes: 54 additions & 58 deletions warehouse/models/intermediate/ntd/int_ntd__monthly_ridership_with_adjustments_voms.sql
View file
Original file line number Diff line number Diff line change
@@ -1,72 +1,68 @@
with
source_pivoted as (
WITH
source_pivoted AS (
{{
dbt_utils.unpivot(
cast_to="int",
relation=ref("stg_ntd__monthly_ridership_with_adjustments_voms"),
relation=ref("stg_ntd_ridership_historical__complete_monthly_ridership_with_adjustments_and_estimates__voms"),
exclude=[
"uza_name",
"uace_cd",
"dt",
"ts",
"year",
"ntd_id",
"reporter_type",
"_3_mode",
"agency",
"mode_type_of_service_status",
"legacy_ntd_id",
"mode",
"_3_mode",
"mode_type_of_service_status",
"ntd_id",
"reporter_type",
"tos",
"legacy_ntd_id",
"uace_cd",
"uza_name",
"dt",
"execution_ts"
],
field_name="period",
value_name="voms",
)
}}
),
int_ntd__monthly_ridership_with_adjustments_voms as (
select
uza_name,
format("%05d", cast(uace_cd as int64)) as uace_cd,
dt as _dt,
ts,
year,
format("%05d", cast(ntd_id as int64)) as ntd_id,
legacy_ntd_id,
reporter_type,
agency,
mode_type_of_service_status,
mode,
_3_mode,
tos,
split(period, '_')[offset(2)] as period_year,
split(period, '_')[offset(1)] as period_month,
voms
from source_pivoted
where
mode in (
"DR",
"FB",
"LR",
"MB",
"SR",
"TB",
"VP",
"CB",
"RB",
"CR",
"YR",
"MG",
"MO",
"AR",
"TR",
"HR",
"OR",
"IP",
"AG",
"PB",
"CC"
)

int_ntd__monthly_ridership_with_adjustments_voms AS (
SELECT format("%05d", cast(cast(ntd_id AS NUMERIC) AS INT64)) AS ntd_id,
legacy_ntd_id,
agency,
reporter_type,
split(period, '_')[offset(2)] AS period_year,
split(period, '_')[offset(1)] AS period_month,
uza_name,
format("%05d", cast(uace_cd AS INT64)) AS uace_cd,
mode,
mode_type_of_service_status,
_3_mode,
tos,
voms,
dt AS _dt,
execution_ts
FROM source_pivoted
WHERE ntd_id IS NOT NULL
AND mode IN ("AG",
"AR",
"CB",
"CC",
"CR",
"DR",
"FB",
"HR",
"IP",
"LR",
"MB",
"MG",
"MO",
"OR",
"PB",
"RB",
"SR",
"TB",
"TR",
"VP",
"YR")
)
select *
from int_ntd__monthly_ridership_with_adjustments_voms

SELECT * FROM int_ntd__monthly_ridership_with_adjustments_voms
Loading

0 comments on commit 7e8bce2

Please sign in to comment.