From 1ff72df5d13d68bd6592ea6d817c9289d162029c Mon Sep 17 00:00:00 2001 From: d116626 Date: Tue, 19 Sep 2023 12:40:31 -0300 Subject: [PATCH] chore: add biglake to ergon tables --- pipelines/rj_smfp/dump_db_ergon/schedules.py | 41 ++++++++++++++++++- .../dump_db_ergon_comlurb/schedules.py | 15 ++++++- 2 files changed, 54 insertions(+), 2 deletions(-) diff --git a/pipelines/rj_smfp/dump_db_ergon/schedules.py b/pipelines/rj_smfp/dump_db_ergon/schedules.py index 605d7dfb19..d709e913b3 100644 --- a/pipelines/rj_smfp/dump_db_ergon/schedules.py +++ b/pipelines/rj_smfp/dump_db_ergon/schedules.py @@ -22,125 +22,148 @@ ergon_queries = { "cargo": { "materialize_after_dump": True, + "biglake_table": True, "materialization_mode": "prod", "dump_mode": "overwrite", "execute_query": "SELECT * FROM C_ERGON.VW_DLK_ERG_CARGOS_", }, "categoria": { "materialize_after_dump": True, + "biglake_table": True, "materialization_mode": "prod", "dump_mode": "overwrite", "execute_query": "SELECT * FROM C_ERGON.VW_DLK_ERG_CATEGORIAS_", }, "empresa": { "materialize_after_dump": True, + "biglake_table": True, "materialization_mode": "prod", "dump_mode": "overwrite", "execute_query": "SELECT * FROM C_ERGON.VW_DLK_ERG_EMPRESAS", }, "matricula": { "materialize_after_dump": True, + "biglake_table": True, "materialization_mode": "prod", "dump_mode": "overwrite", "execute_query": "SELECT * FROM C_ERGON.VW_DLK_ERG_ERG_MATRICULAS", }, "fita_banco": { "materialize_after_dump": True, + "biglake_table": True, "materialization_mode": "prod", "partition_columns": "MES_ANO", "dump_mode": "append", + "lower_bound_date": "current_month", "execute_query": "SELECT * FROM C_ERGON.VW_DLK_ERG_FITA_BANCO", }, "folha_empresa": { "materialize_after_dump": True, + "biglake_table": True, "materialization_mode": "prod", "partition_columns": "MES_ANO", "dump_mode": "append", + "lower_bound_date": "current_month", "execute_query": "SELECT * FROM C_ERGON.VW_DLK_ERG_FOLHAS_EMP", }, "forma_provimento": { "materialize_after_dump": True, + "biglake_table": True, "materialization_mode": "prod", "dump_mode": "overwrite", "execute_query": "SELECT * FROM C_ERGON.VW_DLK_ERG_FORMAS_PROV_", }, "funcionario": { "materialize_after_dump": True, + "biglake_table": True, "materialization_mode": "prod", "dump_mode": "overwrite", "execute_query": "SELECT * FROM C_ERGON.VW_DLK_ERG_FUNCIONARIOS", }, "horario_trabalho": { "materialize_after_dump": True, + "biglake_table": True, "materialization_mode": "prod", "dump_mode": "overwrite", "execute_query": "SELECT * FROM C_ERGON.VW_DLK_ERG_HORARIO_TRAB_", }, "setor": { "materialize_after_dump": True, + "biglake_table": True, "materialization_mode": "prod", "dump_mode": "overwrite", "execute_query": "SELECT * FROM C_ERGON.VW_DLK_ERG_HSETOR_", }, "jornada": { "materialize_after_dump": True, + "biglake_table": True, "materialization_mode": "prod", "dump_mode": "overwrite", "execute_query": "SELECT * FROM C_ERGON.VW_DLK_ERG_JORNADAS_", }, "orgaos_externos": { "materialize_after_dump": True, + "biglake_table": True, "materialization_mode": "prod", "dump_mode": "overwrite", "execute_query": "SELECT * FROM C_ERGON.VW_DLK_ERG_ORGAOS_EXTERNOS", }, "orgaos_regime_juridico": { "materialize_after_dump": True, + "biglake_table": True, "materialization_mode": "prod", "dump_mode": "overwrite", "execute_query": "SELECT * FROM C_ERGON.VW_DLK_ERG_ORGAOS_REGIMES_JUR_", }, "provimento": { "materialize_after_dump": True, + "biglake_table": True, "materialization_mode": "prod", "dump_mode": "overwrite", "execute_query": "SELECT * FROM C_ERGON.VW_DLK_ERG_PROVIMENTOS_EV", }, "regime_juridico": { "materialize_after_dump": True, + "biglake_table": True, "materialization_mode": "prod", "dump_mode": "overwrite", "execute_query": "SELECT * FROM C_ERGON.VW_DLK_ERG_REGIMES_JUR_", }, "tipo_folha": { "materialize_after_dump": True, + "biglake_table": True, "materialization_mode": "prod", "dump_mode": "overwrite", "execute_query": "SELECT * FROM C_ERGON.VW_DLK_ERG_TIPO_FOLHA", }, "tipo_orgao": { "materialize_after_dump": True, + "biglake_table": True, "materialization_mode": "prod", "dump_mode": "overwrite", "execute_query": "SELECT * FROM C_ERGON.VW_DLK_ERG_TIPO_ORGAO", }, "tipo_vinculo": { "materialize_after_dump": True, + "biglake_table": True, "materialization_mode": "prod", "dump_mode": "overwrite", "execute_query": "SELECT * FROM C_ERGON.VW_DLK_ERG_TIPO_VINC_", }, "vinculo": { "materialize_after_dump": True, + "biglake_table": True, "materialization_mode": "prod", "dump_mode": "overwrite", "execute_query": "SELECT * FROM C_ERGON.VW_DLK_ERG_VINCULOS", }, "licenca_afastamento": { "materialize_after_dump": True, + "biglake_table": True, "materialization_mode": "prod", "partition_columns": "DTINI", "dump_mode": "append", + "lower_bound_date": "current_month", "execute_query": """ SELECT NUMFUNC,NUMVINC,DTINI,DTFIM,TIPOFREQ,CODFREQ,MOTIVO,DTPREVFIM,FLEX_CAMPO_01, FLEX_CAMPO_02,EMP_CODIGO,FLEX_CAMPO_07 @@ -149,9 +172,11 @@ }, "frequencia": { "materialize_after_dump": True, + "biglake_table": True, "materialization_mode": "prod", "partition_columns": "DTINI", "dump_mode": "append", + "lower_bound_date": "current_month", "execute_query": """ SELECT NUMFUNC,NUMVINC,DTINI,DTFIM,TIPOFREQ,CODFREQ,OBS,EMP_CODIGO FROM ERGON.FREQUENCIAS @@ -159,9 +184,11 @@ }, "vantagens": { "materialize_after_dump": True, + "biglake_table": True, "materialization_mode": "prod", "partition_columns": "DTINI", "dump_mode": "append", + "lower_bound_date": "current_month", "execute_query": """ SELECT NUMFUNC,NUMVINC,VANTAGEM,DTINI,DTFIM,VALOR,INFO,TIPO_INCORPORACAO,PERC_INC_FUNCAO, INC_TABELAVENC,INC_REFERENCIA,OBS,VALOR2,INFO2,VALOR3,INFO3,VALOR4,INFO4,VALOR5,INFO5, @@ -171,6 +198,7 @@ }, "total_contagem": { "materialize_after_dump": True, + "biglake_table": True, "materialization_mode": "prod", "dump_mode": "overwrite", "execute_query": """ @@ -181,6 +209,7 @@ }, "pre_contagem": { "materialize_after_dump": True, + "biglake_table": True, "materialization_mode": "prod", "dump_mode": "overwrite", "execute_query": """ @@ -190,6 +219,7 @@ }, "averbacoes": { "materialize_after_dump": True, + "biglake_table": True, "materialization_mode": "prod", "dump_mode": "overwrite", "execute_query": """ @@ -200,6 +230,7 @@ }, "averbacoes_contagem": { "materialize_after_dump": True, + "biglake_table": True, "materialization_mode": "prod", "dump_mode": "overwrite", "execute_query": """ @@ -209,6 +240,7 @@ }, "frequencia_antigo": { "materialize_after_dump": True, + "biglake_table": True, "materialization_mode": "prod", "dump_mode": "overwrite", "execute_query": """ @@ -218,6 +250,7 @@ }, "afastamento_antigo": { "materialize_after_dump": True, + "biglake_table": True, "materialization_mode": "prod", "dump_mode": "overwrite", "execute_query": """ @@ -227,6 +260,7 @@ }, "afastamento_antigo_nomes": { "materialize_after_dump": True, + "biglake_table": True, "materialization_mode": "prod", "dump_mode": "overwrite", "execute_query": """ @@ -237,6 +271,7 @@ }, "tipo_tempo": { "materialize_after_dump": True, + "biglake_table": True, "materialization_mode": "prod", "dump_mode": "overwrite", "execute_query": """ @@ -247,8 +282,10 @@ }, "ficha_financeira": { "materialize_after_dump": True, + "biglake_table": True, "materialization_mode": "prod", "dump_mode": "append", + "lower_bound_date": "current_month", "partition_columns": "MES_ANO_FOLHA", "execute_query": """ SELECT MES_ANO_FOLHA,NUM_FOLHA,LANCAMENTO,NUMFUNC,NUMVINC,NUMPENS,MES_ANO_DIREITO, @@ -258,8 +295,10 @@ }, "ficha_financeira_contabil": { "materialize_after_dump": True, + "biglake_table": True, "materialization_mode": "prod", "dump_mode": "append", + "lower_bound_date": "current_month", "partition_columns": "MES_ANO_FOLHA", "execute_query": """ SELECT MES_ANO_FOLHA,NUM_FOLHA,NUMFUNC,NUMVINC,NUMPENS,SETOR,SECRETARIA,TIPO_FUNC, @@ -272,7 +311,7 @@ ergon_clocks = generate_dump_db_schedules( interval=timedelta(days=1), - start_date=datetime(2022, 11, 9, 10, 30, tzinfo=pytz.timezone("America/Sao_Paulo")), + start_date=datetime(2022, 11, 9, 22, 30, tzinfo=pytz.timezone("America/Sao_Paulo")), labels=[ constants.RJ_SMFP_AGENT_LABEL.value, ], diff --git a/pipelines/rj_smfp/dump_db_ergon_comlurb/schedules.py b/pipelines/rj_smfp/dump_db_ergon_comlurb/schedules.py index 8a8ca52977..361b7b077e 100644 --- a/pipelines/rj_smfp/dump_db_ergon_comlurb/schedules.py +++ b/pipelines/rj_smfp/dump_db_ergon_comlurb/schedules.py @@ -23,6 +23,7 @@ ergon_queries = { "cargo": { "materialize_after_dump": True, + "biglake_table": True, "materialization_mode": "prod", "dump_mode": "overwrite", "dbt_alias": True, @@ -105,6 +106,7 @@ }, "dependente": { "materialize_after_dump": True, + "biglake_table": True, "materialization_mode": "prod", "dump_mode": "overwrite", "dbt_alias": True, @@ -216,6 +218,7 @@ }, "funcionario_evento": { "materialize_after_dump": True, + "biglake_table": True, "materialization_mode": "prod", "dump_mode": "overwrite", "dbt_alias": True, @@ -333,6 +336,7 @@ }, "ficha_financeira": { "materialize_after_dump": True, + "biglake_table": True, "materialization_mode": "prod", "dump_mode": "overwrite", "dbt_alias": True, @@ -359,8 +363,10 @@ }, "fita_banco": { "materialize_after_dump": True, + "biglake_table": True, "materialization_mode": "prod", "dump_mode": "append", + "lower_bound_date": "current_month", "partition_columns": "MES_ANO", "dbt_alias": True, "execute_query": """ @@ -449,6 +455,7 @@ }, "frequencia": { "materialize_after_dump": True, + "biglake_table": True, "materialization_mode": "prod", "dump_mode": "overwrite", "dbt_alias": True, @@ -524,6 +531,7 @@ }, "funcionario": { "materialize_after_dump": True, + "biglake_table": True, "materialization_mode": "prod", "dump_mode": "overwrite", "dbt_alias": True, @@ -729,6 +737,7 @@ }, "setor_h": { "materialize_after_dump": True, + "biglake_table": True, "materialization_mode": "prod", "dump_mode": "overwrite", "dbt_alias": True, @@ -739,6 +748,7 @@ }, "licenca_afastamento": { "materialize_after_dump": True, + "biglake_table": True, "materialization_mode": "prod", "dump_mode": "overwrite", "dbt_alias": True, @@ -829,6 +839,7 @@ }, "setor": { "materialize_after_dump": True, + "biglake_table": True, "materialization_mode": "prod", "dump_mode": "overwrite", "dbt_alias": True, @@ -882,6 +893,7 @@ }, "vantagens": { "materialize_after_dump": True, + "biglake_table": True, "materialization_mode": "prod", "dump_mode": "overwrite", "dbt_alias": True, @@ -958,6 +970,7 @@ }, "vinculo": { "materialize_after_dump": True, + "biglake_table": True, "materialization_mode": "prod", "dump_mode": "overwrite", "dbt_alias": True, @@ -1121,7 +1134,7 @@ ergon_clocks = generate_dump_db_schedules( interval=timedelta(days=1), start_date=datetime( - 2022, 10, 25, 18, 30, tzinfo=pytz.timezone("America/Sao_Paulo") + 2022, 10, 25, 23, 30, tzinfo=pytz.timezone("America/Sao_Paulo") ), labels=[ constants.RJ_SMFP_AGENT_LABEL.value,