Merge branch 'main' into add_format_sql

prefeitura-rio · Jan 6, 2025 · 8badafb · 8badafb
2 parents 320579a + 753f4d5
commit 8badafb
Show file tree

Hide file tree

Showing 4 changed files with 37 additions and 19 deletions.
diff --git a/pipelines/migration/br_rj_riodejaneiro_gtfs/CHANGELOG.md b/pipelines/migration/br_rj_riodejaneiro_gtfs/CHANGELOG.md
@@ -1,5 +1,15 @@
 # Changelog - gtfs
 
+## [1.2.0] - 2025-01-03
+
+### Adicionado
+- Adicionado schedule de 5 minutos do flow de captura do gtfs (https://github.com/prefeitura-rio/pipelines_rj_smtr/pull/379)
+
+- Adicionado parâmetros personalizados de execução no arquivo `flows.py` (https://github.com/prefeitura-rio/pipelines_rj_smtr/pull/379)
+
+### Removido
+- Removido o teste de quantidade de abas na planilha da Ordem de Serviço (https://github.com/prefeitura-rio/pipelines_rj_smtr/pull/379)
+
 ## [1.1.9] - 2025-01-02
 
 ### Alterado

diff --git a/pipelines/migration/br_rj_riodejaneiro_gtfs/flows.py b/pipelines/migration/br_rj_riodejaneiro_gtfs/flows.py
@@ -45,8 +45,7 @@
     upload_raw_data_to_gcs,
     upload_staging_data_to_gcs,
 )
-
-# from pipelines.schedules import every_5_minutes
+from pipelines.schedules import every_5_minutes
 from pipelines.tasks import (
     check_fail,
     get_scheduled_timestamp,
@@ -287,13 +286,17 @@
 gtfs_captura_nova.run_config = KubernetesRun(
     image=constants.DOCKER_IMAGE.value,
     labels=[constants.RJ_SMTR_AGENT_LABEL.value],
+    cpu_limit="1000m",
+    memory_limit="4600Mi",
+    cpu_request="500m",
+    memory_request="1000Mi",
 )
 gtfs_captura_nova.state_handlers = [
     handler_inject_bd_credentials,
     handler_initialize_sentry,
     handler_skip_if_running,
 ]
-# gtfs_captura_nova.schedule = every_5_minutes
+gtfs_captura_nova.schedule = every_5_minutes
 
 
 # with Flow(

diff --git a/pipelines/migration/br_rj_riodejaneiro_gtfs/utils.py b/pipelines/migration/br_rj_riodejaneiro_gtfs/utils.py
@@ -204,8 +204,13 @@ def processa_ordem_servico(
         None
     """
 
-    if len(sheetnames) != 3 and regular_sheet_index is None:
-        raise Exception("More than 3 tabs in the file. Please specify the regular sheet index.")
+    if (
+        len([sheet for sheet in sheetnames if "ANEXO I:" in sheet]) != 1
+        and regular_sheet_index is None
+    ):
+        raise Exception(
+            "More than 1 regular sheet in the file. Please specify the regular sheet index."
+        )
 
     if regular_sheet_index is None:
         regular_sheet_index = next(

diff --git a/queries/models/planejamento/schema.yml b/queries/models/planejamento/schema.yml
@@ -25,22 +25,22 @@ models:
         description: "{{ doc('faixa_horaria_fim') }}"
       - name: partidas
         description: "{{ doc('partidas') }}"
-        tests:
-          - dbt_expectations.expect_column_sum_to_be_between:
-              name: dbt_expectations.expect_column_sum_to_be_between__partidas__ordem_servico_faixa_horaria
-              min_value: 0
-              group_by: [ feed_start_date, tipo_os, tipo_dia, servico ]
-              strictly: true
-              where: "feed_start_date = '{{ var('data_versao_gtfs') }}'"
+        # tests:
+        #   - dbt_expectations.expect_column_sum_to_be_between:
+        #       name: dbt_expectations.expect_column_sum_to_be_between__partidas__ordem_servico_faixa_horaria
+        #       min_value: 0
+        #       group_by: [ feed_start_date, tipo_os, tipo_dia, servico ]
+        #       strictly: true
+        #       where: "feed_start_date = '{{ var('data_versao_gtfs') }}'"
       - name: quilometragem
         description: "{{ doc('quilometragem') }}"
-        tests:
-          - dbt_expectations.expect_column_sum_to_be_between:
-              name: dbt_expectations.expect_column_sum_to_be_between__quilometragem__ordem_servico_faixa_horaria
-              min_value: 0
-              group_by: [ feed_start_date, tipo_os, tipo_dia, servico ]
-              strictly: true
-              where: "feed_start_date = '{{ var('data_versao_gtfs') }}'"
+        # tests:
+        #   - dbt_expectations.expect_column_sum_to_be_between:
+        #       name: dbt_expectations.expect_column_sum_to_be_between__quilometragem__ordem_servico_faixa_horaria
+        #       min_value: 0
+        #       group_by: [ feed_start_date, tipo_os, tipo_dia, servico ]
+        #       strictly: true
+        #       where: "feed_start_date = '{{ var('data_versao_gtfs') }}'"
   - name: segmento_shape
     description: Tabela contendo os shapes segmentados usados na validação de viagens.
     columns: