Skip to content

Commit

Permalink
Merge branch 'main' into view-viagem-transacao-monitoramento
Browse files Browse the repository at this point in the history
  • Loading branch information
@mergify
mergify[bot] authored Dec 27, 2024
2 parents e825778 + 3081fd0 commit c4a1a8c
Show file tree
Hide file tree
Showing 172 changed files with 10,151 additions and 4,749 deletions.
4 changes: 4 additions & 0 deletions .github/workflows/cd-docs.yaml
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,9 @@
name: CD-docs

on:
pull_request:
branches:
- main
push:
branches:
- main
Expand All @@ -11,6 +14,7 @@ env:
GKE_CLUSTER: ${{ secrets.GKE_APP_CLUSTER_NAME }}
GKE_ZONE: ${{ secrets.GKE_CLUSTER_ZONE }}
IMAGE_NAME: gcr.io/rj-smtr/dbt-docs
DBT_USER: ${{ github.actor }}

jobs:
build-container:
Expand Down
6 changes: 6 additions & 0 deletions pipelines/CHANGELOG.md
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,11 @@
# Changelog - pipelines

## [1.0.1] - 2024-12-13

### Adicionado

- Adiciona task `check_fail` (https://github.com/prefeitura-rio/pipelines_rj_smtr/pull/359)

## [1.0.0] - 2024-09-09

### Alterado
Expand Down
18 changes: 18 additions & 0 deletions pipelines/capture/jae/CHANGELOG.md
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
# Changelog - source_jae

## [1.1.0] - 2024-12-19

### Adicionado

- Cria flow `verificacao_ip` para alertar sobre falhas de conexão com o banco de dados da Jaé (https://github.com/prefeitura-rio/pipelines_rj_smtr/pull/366)

## [1.0.1] - 2024-12-16

### Alterado
- Altera IP do banco de tracking da Jaé (https://github.com/prefeitura-rio/pipelines_rj_smtr/pull/362)

## [1.0.0] - 2024-11-25

### Adicionado

- Cria flow de captura da relação entre transação e ordem de pagamento
280 changes: 41 additions & 239 deletions pipelines/capture/jae/constants.py
View file
Original file line number Diff line number Diff line change
@@ -1,27 +1,26 @@
# -*- coding: utf-8 -*-
"""
Valores constantes para captura de dados da JAE
Valores constantes para captura de dados da Jaé
"""

from datetime import datetime
from enum import Enum

from pipelines.utils.incremental_capture_strategy import (
DatetimeIncremental,
IDIncremental,
)
from pipelines.schedules import create_daily_cron
from pipelines.utils.gcp.bigquery import SourceTable


class constants(Enum):
class constants(Enum): # pylint: disable=c0103
"""
Valores constantes para captura de dados da JAE
Valores constantes para captura de dados da Jaé
"""

JAE_SOURCE_NAME = "jae"

JAE_DATABASES = {
JAE_DATABASE_SETTINGS = {
"principal_db": {
"engine": "mysql",
"host": "10.5.114.121",
"host": "10.5.114.227",
},
"tarifa_db": {
"engine": "postgresql",
Expand All @@ -33,255 +32,58 @@ class constants(Enum):
},
"tracking_db": {
"engine": "postgresql",
"host": "10.5.15.25",
"host": "10.5.12.67",
},
"ressarcimento_db": {
"engine": "postgresql",
"host": "10.5.15.127",
"host": "10.5.12.50",
},
"gratuidade_db": {
"engine": "postgresql",
"host": "10.5.12.107",
},
"fiscalizacao_db": {
"engine": "postgresql",
"host": "10.5.115.29",
},
}

JAE_PRIVATE_BUCKET = {"dev": "br-rj-smtr-jae-private-dev", "prod": "br-rj-smtr-jae-private-dev"}
JAE_SECRET_PATH = "smtr_jae_access_data"

JAE_RAW_FILETYPE = "json"
TRANSACAO_ORDEM_TABLE_ID = "transacao_ordem"

TRANSACAO_DEFAULT_PARAMS = {
"table_id": "transacao",
"raw_filetype": JAE_RAW_FILETYPE,
"incremental_capture_strategy": DatetimeIncremental(
max_incremental_window={"hours": 3}, first_value="2024-02-15 00:00:00"
).to_dict(),
"data_extractor_params": {
"database": "transacao_db",
JAE_TABLE_CAPTURE_PARAMS = {
TRANSACAO_ORDEM_TABLE_ID: {
"query": """
SELECT
*
id,
id_ordem_ressarcimento,
data_processamento,
data_transacao
FROM
transacao
WHERE
data_processamento > '{{ start }}'
AND data_processamento <= '{{ end }}'
DATE(data_processamento) >= DATE('{start}')
AND DATE(data_processamento) <= DATE('{end}')
AND id_ordem_ressarcimento IS NOT NULL
""",
},
"primary_keys": ["id"],
}

TRANSACAO_RIOCARD_DEFAULT_PARAMS = {
"table_id": "transacao_riocard",
"raw_filetype": JAE_RAW_FILETYPE,
"incremental_capture_strategy": DatetimeIncremental(
max_incremental_window={"hours": 1}, first_value="2024-04-03T01:00:00+00:00"
).to_dict(),
"data_extractor_params": {
"database": "transacao_db",
"query": """
SELECT
*
FROM
transacao_riocard
WHERE
data_processamento > '{{ start }}'
AND data_processamento <= '{{ end }}'
""",
},
"primary_keys": ["id"],
"save_bucket_names": {"dev": "rj-smtr-dev", "prod": "rj-smtr-dev"},
}
}

GPS_VALIDADOR_CAPTURE_PARAMS = {
"table_id": "gps_validador",
"raw_filetype": JAE_RAW_FILETYPE,
"incremental_capture_strategy": IDIncremental(
max_incremental_window=100_000,
id_column_name="id",
first_value=406_064_585,
).to_dict(),
"data_extractor_params": {
"database": "tracking_db",
"query": """
SELECT
*
FROM
tracking_detalhe
WHERE
id > {{ start }} AND id <= {{ end }}
""",
"page_size": 1000,
},
"primary_keys": ["id"],
}
TRANSACAO_ORDEM_SOURCE = SourceTable(
source_name=JAE_SOURCE_NAME,
table_id=TRANSACAO_ORDEM_TABLE_ID,
first_timestamp=datetime(2024, 11, 21, 0, 0, 0),
schedule_cron=create_daily_cron(hour=6),
partition_date_only=True,
max_recaptures=5,
primary_keys=[
"id",
"id_ordem_ressarcimento",
"data_processamento",
"data_transacao",
],
)

AUXILIAR_GENERAL_CAPTURE_PARAMS = {
"incremental_capture_strategy": DatetimeIncremental(
max_incremental_window={"hours": 5}
).to_dict(),
"raw_filetype": JAE_RAW_FILETYPE,
}

AUXILIAR_TABLE_CAPTURE_PARAMS = [
{
"table_id": "linha",
"data_extractor_params": {
"database": "principal_db",
"query": """
SELECT
*
FROM
LINHA
{% if is_incremental() %}
WHERE
DT_INCLUSAO BETWEEN '{{ start }}'
AND '{{ end }}'
{% endif %}
""",
},
"primary_keys": ["CD_LINHA"],
},
{
"table_id": "operadora_transporte",
"data_extractor_params": {
"database": "principal_db",
"query": """
SELECT
*
FROM
OPERADORA_TRANSPORTE
{% if is_incremental() %}
WHERE
DT_INCLUSAO BETWEEN '{{ start }}'
AND '{{ end }}'
{% endif %}
""",
},
"primary_keys": ["CD_OPERADORA_TRANSPORTE"],
},
{
"table_id": "cliente",
"data_extractor_params": {
"database": "principal_db",
"query": """
SELECT
c.*
FROM
CLIENTE c
{% if is_incremental() %}
WHERE
DT_CADASTRO BETWEEN '{{ start }}'
AND '{{ end }}'
{% endif %}
""",
},
"primary_keys": ["CD_CLIENTE"],
"save_bucket_names": JAE_PRIVATE_BUCKET,
},
{
"table_id": "pessoa_fisica",
"data_extractor_params": {
"database": "principal_db",
"query": """
SELECT
p.*,
c.DT_CADASTRO
FROM
PESSOA_FISICA p
JOIN
CLIENTE c
ON
p.CD_CLIENTE = c.CD_CLIENTE
{% if is_incremental() %}
WHERE
c.DT_CADASTRO BETWEEN '{{ start }}'
AND '{{ end }}'
{% endif %}
""",
},
"primary_keys": ["CD_CLIENTE"],
"save_bucket_names": JAE_PRIVATE_BUCKET,
},
{
"table_id": "gratuidade",
"data_extractor_params": {
"database": "gratuidade_db",
"query": """
SELECT
g.*,
t.descricao AS tipo_gratuidade
FROM
gratuidade g
LEFT JOIN
tipo_gratuidade t
ON
g.id_tipo_gratuidade = t.id
{% if is_incremental() %}
WHERE
g.data_inclusao BETWEEN '{{ start }}'
AND '{{ end }}'
{% endif %}
""",
},
"primary_keys": ["id"],
"save_bucket_names": JAE_PRIVATE_BUCKET,
},
{
"table_id": "consorcio",
"data_extractor_params": {
"database": "principal_db",
"query": """
SELECT
*
FROM
CONSORCIO
{% if is_incremental() %}
WHERE
DT_INCLUSAO BETWEEN '{{ start }}'
AND '{{ end }}'
{% endif %}
""",
},
"primary_keys": ["CD_CONSORCIO"],
},
{
"table_id": "percentual_rateio_integracao",
"data_extractor_params": {
"database": "ressarcimento_db",
"query": """
SELECT
*
FROM
percentual_rateio_integracao
{% if is_incremental() %}
WHERE
dt_inclusao BETWEEN '{{ start }}'
AND '{{ end }}'
{% endif %}
""",
},
"primary_keys": ["id"],
},
{
"table_id": "contato_pessoa_juridica",
"data_extractor_params": {
"database": "principal_db",
"query": """
SELECT
*
FROM
CONTATO_PESSOA_JURIDICA
{% if is_incremental() %}
WHERE
DT_INCLUSAO BETWEEN '{{ start }}'
AND '{{ end }}'
{% endif %}
""",
},
"primary_keys": [
"NR_SEQ_CONTATO",
"CD_CLIENTE",
],
"save_bucket_names": JAE_PRIVATE_BUCKET,
},
]
ALERT_WEBHOOK = "alertas_bilhetagem"
Loading

0 comments on commit c4a1a8c

Please sign in to comment.