From ae441b5993d0b793ed29ba071e498abf093871c7 Mon Sep 17 00:00:00 2001 From: Victor Miguel Rocha Date: Tue, 21 Jan 2025 15:38:16 -0300 Subject: [PATCH] =?UTF-8?q?Corrige=20a=20materializa=C3=A7=C3=A3o=20das=20?= =?UTF-8?q?tabelas=20historicas=20do=20subs=C3=ADdio=20|=20corrige=20sourc?= =?UTF-8?q?e=20de=20sppo=5Fveiculo=5Fdia=20(#391)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * corrige materialização das tabelas historicas do subsídio * corrige source de infracao_staging e licenciamento_stu_staging e corrige referencia de sppo_veiculo_dia * atualiza changelog * corrige label --- .../migration/projeto_subsidio_sppo/CHANGELOG.md | 6 ++++++ .../migration/projeto_subsidio_sppo/flows.py | 15 ++++++++++----- pipelines/migration/veiculo/flows.py | 2 ++ queries/models/sources.yml | 4 ++-- queries/models/veiculo/CHANGELOG.md | 5 +++++ queries/models/veiculo/sppo_veiculo_dia.sql | 4 ++-- .../models/veiculo/staging/infracao_staging.sql | 2 +- .../veiculo/staging/licenciamento_stu_staging.sql | 2 +- 8 files changed, 29 insertions(+), 11 deletions(-) diff --git a/pipelines/migration/projeto_subsidio_sppo/CHANGELOG.md b/pipelines/migration/projeto_subsidio_sppo/CHANGELOG.md index df4606147..f400247a9 100644 --- a/pipelines/migration/projeto_subsidio_sppo/CHANGELOG.md +++ b/pipelines/migration/projeto_subsidio_sppo/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog - projeto_subsidio_sppo +## [1.0.8] - 2025-01-03 + +### Corrigido + +- corrigida a materialização dos modelos do dataset `monitoramento` no flow do subsídio (https://github.com/prefeitura-rio/pipelines_rj_smtr/pull/391) + ## [1.0.7] - 2025-01-03 ### Adicionado diff --git a/pipelines/migration/projeto_subsidio_sppo/flows.py b/pipelines/migration/projeto_subsidio_sppo/flows.py index 7fbb820e2..4b19a09b8 100644 --- a/pipelines/migration/projeto_subsidio_sppo/flows.py +++ b/pipelines/migration/projeto_subsidio_sppo/flows.py @@ -276,9 +276,8 @@ upstream_tasks=[dbt_vars_2], ) dbt_vars_3 = get_join_dict( - dict_list=[dbt_vars_1], - new_dict=date_intervals["second_range"] - | {"tipo_materializacao": "monitoramento"}, + dict_list=[dbt_vars_2], + new_dict={"tipo_materializacao": "monitoramento"}, upstream_tasks=[SUBSIDIO_SPPO_APURACAO_RUN_2], )[0] @@ -335,10 +334,16 @@ _vars=_vars, ) + _vars_2 = get_join_dict( + dict_list=[_vars], + new_dict={"tipo_materializacao": "monitoramento"}, + upstream_tasks=[SUBSIDIO_SPPO_APURACAO_RUN], + )[0] + SUBSIDIO_SPPO_APURACAO_RUN_2 = run_dbt_selector( selector_name="monitoramento_subsidio", - _vars=_vars | {"tipo_materializacao": "monitoramento"}, - upstream_tasks=[SUBSIDIO_SPPO_APURACAO_RUN], + _vars=_vars_2, + upstream_tasks=[_vars_2], ) # POST-DATA QUALITY CHECK # SUBSIDIO_SPPO_DATA_QUALITY_POS = run_dbt_tests( diff --git a/pipelines/migration/veiculo/flows.py b/pipelines/migration/veiculo/flows.py index 1ab59e340..d44995e8d 100644 --- a/pipelines/migration/veiculo/flows.py +++ b/pipelines/migration/veiculo/flows.py @@ -2,6 +2,8 @@ # pylint: disable=W0511 """ Flows for veiculos + +DBT: 2025-01-21 """ from copy import deepcopy diff --git a/queries/models/sources.yml b/queries/models/sources.yml index 1047cb75b..5ada71330 100644 --- a/queries/models/sources.yml +++ b/queries/models/sources.yml @@ -110,10 +110,10 @@ sources: - name: sppo_vistoria_tr_subtt_cglf_2024 - name: sppo_vistoria_tr_subtt_cglf_pendentes_2024 - name: sppo_registro_agente_verao - - name: sppo_infracao - name: sppo_licenciamento_solicitacao - name: sppo_licenciamento - - name: sppo_licenciamento_stu + - name: infracao + - name: licenciamento_stu - name: dashboard_subsidio_sppo_staging database: rj-smtr-staging diff --git a/queries/models/veiculo/CHANGELOG.md b/queries/models/veiculo/CHANGELOG.md index ae57816c9..de0f662c0 100644 --- a/queries/models/veiculo/CHANGELOG.md +++ b/queries/models/veiculo/CHANGELOG.md @@ -1,5 +1,10 @@ # Changelog - veiculo +## [2.0.1] - 2025-01-21 + +### Corrigido +- Corrigido os sources dos modelos `infracao.sql` e `licenciamento.sql` (https://github.com/prefeitura-rio/pipelines_rj_smtr/pull/391) + ## [2.0.0] - 2025-01-16 ### Adicionado diff --git a/queries/models/veiculo/sppo_veiculo_dia.sql b/queries/models/veiculo/sppo_veiculo_dia.sql index 8d79a5117..a4a1830f8 100644 --- a/queries/models/veiculo/sppo_veiculo_dia.sql +++ b/queries/models/veiculo/sppo_veiculo_dia.sql @@ -39,8 +39,8 @@ gps AS ( DISTINCT data, id_veiculo FROM - -- {{ ref("gps_sppo") }} - `rj-smtr.br_rj_riodejaneiro_veiculos.gps_sppo` + {{ ref("gps_sppo") }} + -- `rj-smtr.br_rj_riodejaneiro_veiculos.gps_sppo` WHERE data = DATE("{{ var('run_date') }}") ), autuacoes AS ( diff --git a/queries/models/veiculo/staging/infracao_staging.sql b/queries/models/veiculo/staging/infracao_staging.sql index a84054fcd..8ff42ab9c 100644 --- a/queries/models/veiculo/staging/infracao_staging.sql +++ b/queries/models/veiculo/staging/infracao_staging.sql @@ -21,5 +21,5 @@ SELECT IF(JSON_VALUE(content,'$.data_pagamento') = "", NULL, PARSE_DATE("%d/%m/%Y", JSON_VALUE(content,'$.data_pagamento'))) data_pagamento, SAFE_CAST(DATETIME(TIMESTAMP_TRUNC(TIMESTAMP(timestamp_captura), SECOND), "America/Sao_Paulo" ) AS DATETIME) timestamp_captura FROM - {{ source('veiculo_staging','sppo_infracao') }} as t + {{ source('veiculo_staging','infracao') }} as t diff --git a/queries/models/veiculo/staging/licenciamento_stu_staging.sql b/queries/models/veiculo/staging/licenciamento_stu_staging.sql index 72526166b..f7336564a 100644 --- a/queries/models/veiculo/staging/licenciamento_stu_staging.sql +++ b/queries/models/veiculo/staging/licenciamento_stu_staging.sql @@ -43,4 +43,4 @@ SELECT SAFE_CAST(PARSE_DATETIME("%d/%m/%Y", JSON_VALUE(content,"$.data_inicio_vinculo")) AS DATE) END AS data_inicio_vinculo, FROM - {{ source('veiculo_staging','sppo_licenciamento_stu') }} as t \ No newline at end of file + {{ source('veiculo_staging','licenciamento_stu') }} as t \ No newline at end of file