Merge branch 'master' into staging/smtr-subisio-sppo

pipelines/constants.py

@@ -39,6 +39,8 @@ class constants(Enum):  # pylint: disable=c0103
 
     RJ_SMFP_AGENT_LABEL = "rj-smfp"
 
+    RJ_SMI_AGENT_LABEL = "rj-smi"
+
     RJ_SECONSERVA_AGENT_LABEL = "rj-seconserva"
 
     ######################################

pipelines/flows.py

@@ -12,5 +12,6 @@
 from pipelines.rj_segovi import *
 from pipelines.rj_sme import *
 from pipelines.rj_smfp import *
+from pipelines.rj_smi import *
 from pipelines.rj_smtr import *
 from pipelines.utils import *

pipelines/rj_smi/__init__.py

@@ -0,0 +1,6 @@
+# -*- coding: utf-8 -*-
+"""
+Prefect flows for rj_smi project
+"""
+
+from pipelines.rj_smi.dump_db_siscob.flows import *

pipelines/rj_smi/dump_db_siscob/flows.py

@@ -0,0 +1,41 @@
+# -*- coding: utf-8 -*-
+"""
+Database dumping flows for segovi project (SISCOB)
+"""
+
+from copy import deepcopy
+
+from prefect.run_configs import KubernetesRun
+from prefect.storage import GCS
+
+from pipelines.constants import constants
+from pipelines.rj_smi.dump_db_siscob.schedules import (
+    siscob_update_schedule,
+)
+from pipelines.utils.dump_db.flows import dump_sql_flow
+from pipelines.utils.utils import set_default_parameters
+
+
+dump_siscob_flow = deepcopy(dump_sql_flow)
+dump_siscob_flow.name = "SMI: SISCOB - Ingerir tabelas de banco SQL"
+dump_siscob_flow.storage = GCS(constants.GCS_FLOWS_BUCKET.value)
+dump_siscob_flow.run_config = KubernetesRun(
+    image=constants.DOCKER_IMAGE.value,
+    labels=[
+        constants.RJ_SMI_AGENT_LABEL.value,
+    ],
+)
+
+siscob_default_parameters = {
+    "db_database": "SISCOB200",
+    "db_host": "10.70.1.34",
+    "db_port": "1433",
+    "db_type": "sql_server",
+    "vault_secret_path": "siscob",
+    "dataset_id": "infraestrutura_siscob_obras",
+}
+dump_siscob_flow = set_default_parameters(
+    dump_siscob_flow, default_parameters=siscob_default_parameters
+)
+
+dump_siscob_flow.schedule = siscob_update_schedule

pipelines/rj_smi/dump_db_siscob/schedules.py

@@ -0,0 +1,164 @@
+# -*- coding: utf-8 -*-
+"""
+Schedules for the database dump pipeline
+"""
+
+from datetime import timedelta, datetime
+
+from prefect.schedules import Schedule
+import pytz
+
+from pipelines.constants import constants
+from pipelines.utils.dump_db.utils import generate_dump_db_schedules
+from pipelines.utils.utils import untuple_clocks as untuple
+
+
+#####################################
+#
+# SISCOB Schedules
+#
+#####################################
+
+siscob_queries = {
+    "obra": {
+        "materialize_after_dump": True,
+        "materialization_mode": "prod",
+        "dump_mode": "overwrite",
+        "execute_query": """
+            Select
+                CD_OBRA,
+                DS_TITULO,
+                ORGAO_CONTRATANTE,
+                ORGAO_EXECUTOR,
+                NR_PROCESSO,
+                OBJETO,
+                NM_FAVORECIDO,
+                CNPJ,
+                NR_LICITACAO,
+                MODALIDADE,
+                DT_ASS_CONTRATO,
+                DT_INICIO_OBRA,
+                DT_TERMINO_PREVISTO,
+                DT_TERMINO_ATUAL,
+                NR_CONTRATO,
+                AA_EXERCICIO,
+                SITUACAO,
+                VL_ORCADO_C_BDI,
+                VL_CONTRATADO,
+                VL_VIGENTE,
+                PC_MEDIDO,
+                PRAZO_INICIAL
+            from dbo.fuSEGOVI_Dados_da_Obra();
+            """,
+    },
+    "medicao": {
+        "materialize_after_dump": True,
+        "materialization_mode": "prod",
+        "dump_mode": "overwrite",
+        "execute_query": """
+            Select
+                CD_OBRA,
+                NR_MEDICAO,
+                CD_ETAPA,
+                TP_MEDICAO_D,
+                DT_INI_MEDICAO,
+                DT_FIM_MEDICAO,
+                VL_FINAL
+            from dbo.fuSEGOVI_Medicoes();
+            """,
+    },
+    "termo_aditivo": {
+        "materialize_after_dump": True,
+        "materialization_mode": "prod",
+        "dump_mode": "overwrite",
+        "execute_query": """
+            Select
+                CD_OBRA,
+                NR_DO_TERMO,
+                TP_ACERTO,
+                DT_DO,
+                DT_AUTORIZACAO,
+                VL_ACERTO
+            from dbo.fuSEGOVI_Termos_Aditivos();
+            """,
+    },
+    "cronograma_financeiro": {
+        "materialize_after_dump": True,
+        "materialization_mode": "prod",
+        "dump_mode": "overwrite",
+        "execute_query": """
+            Select
+                CD_OBRA,
+                ETAPA,
+                DT_INICIO_ETAPA,
+                DT_FIM_ETAPA,
+                PC_PERCENTUAL,
+                VL_ESTIMADO
+            from dbo.fuSEGOVI_Cronograma_Financeiro();
+            """,
+    },
+    "localizacao": {
+        "materialize_after_dump": True,
+        "materialization_mode": "prod",
+        "dump_mode": "overwrite",
+        "execute_query": """
+            Select
+                CD_OBRA,
+                ENDERECO,
+                NM_BAIRRO,
+                NM_RA,
+                NM_AP
+            from dbo.fuSEGOVI_Localizacoes_obra();
+            """,
+    },
+    "cronograma_alteracao": {
+        "materialize_after_dump": True,
+        "materialization_mode": "prod",
+        "dump_mode": "overwrite",
+        "execute_query": """
+            Select
+                CD_OBRA,
+                NR_PROCESSO,
+                TP_ALTERACAO,
+                DT_PUBL_DO,
+                CD_ETAPA,
+                NR_PRAZO,
+                DT_VALIDADE,
+                DS_OBSERVACAO
+            from dbo.fuSEGOVI_Alteração_de_Cronograma();
+            """,
+    },
+    "programa_fonte": {
+        "materialize_after_dump": True,
+        "materialization_mode": "prod",
+        "dump_mode": "overwrite",
+        "execute_query": """
+            Select
+                CD_OBRA,
+                CD_PRG_TRAB,
+                PROGRAMA_TRABALHO,
+                CD_FONTE_RECURSO,
+                FONTE_RECURSO,
+                CD_NATUREZA_DSP,
+                NATUREZA_DESPESA
+            from dbo.fuSEGOVI_Programa_Fonte();
+            """,
+    },
+}
+
+siscob_clocks = generate_dump_db_schedules(
+    interval=timedelta(days=7),
+    start_date=datetime(2022, 10, 2, 0, 0, tzinfo=pytz.timezone("America/Sao_Paulo")),
+    labels=[
+        constants.RJ_SMI_AGENT_LABEL.value,
+    ],
+    db_database="SISCOB200",
+    db_host="10.70.1.34",
+    db_port="1433",
+    db_type="sql_server",
+    dataset_id="infraestrutura_siscob_obras",
+    vault_secret_path="siscob",
+    table_parameters=siscob_queries,
+)
+
+siscob_update_schedule = Schedule(clocks=untuple(siscob_clocks))