Skip to content

Commit

Permalink
Merge pull request #173 from basedosdados/JoaoCarabetta/issue172
Browse files Browse the repository at this point in the history
Fix BUG: Table publish throwing error 500 #172
  • Loading branch information
JoaoCarabetta authored Nov 17, 2020
2 parents 32a73cc + db06e28 commit 5a71b99
Show file tree
Hide file tree
Showing 52 changed files with 116 additions and 44 deletions.
4 changes: 4 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
# OLD repo name
.bases
.mais
bases/pytest/*
bases/test/

Expand All @@ -15,6 +16,7 @@ secrets
*.ipynb
*.csv
docs/site
TODO

tests/tmp_bases/
# Byte-compiled / optimized / DLL files
Expand Down Expand Up @@ -159,3 +161,5 @@ dmypy.json
# Cython debug symbols
cython_debug/
.vscode/settings.json


2 changes: 2 additions & 0 deletions basedosdados/configs/templates/table/publish.sql
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@ TIPOS:
- Mais detalhes: https://cloud.google.com/bigquery/docs/reference/standard-sql/data-types
*/
{% set project = project_id.split('-') %}
CREATE VIEW {{ project[0] }}.{{ dataset_id }}.{{ table_id }} AS
SELECT
{% for column in columns|list + partition_columns|list -%}
{%- if not loop.last -%}
Expand Down
12 changes: 3 additions & 9 deletions basedosdados/table.py
Original file line number Diff line number Diff line change
Expand Up @@ -354,18 +354,12 @@ def publish(self, if_exists="raise"):
* Check if all required fields are filled
"""

view = bigquery.Table(self.table_full_name["prod"])

view.view_query = (self.table_folder / "publish.sql").open("r").read()

view.description = self._render_template(
"table/table_description.txt", self.table_config
)

if if_exists == "replace":
self.delete(mode="prod")

self.client["bigquery_prod"].create_table(view)
self.client["bigquery_prod"].query(
(self.table_folder / "publish.sql").open("r").read()
)

self.update("prod")

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ TIPOS:
- Mais detalhes: https://cloud.google.com/bigquery/docs/reference/standard-sql/data-types
*/
CREATE VIEW basedosdados.br_basedosdados_diretorios_brasil.estados AS
SELECT
SAFE_CAST(id_estado AS INT64) id_estado,
SAFE_CAST(estado_abrev AS STRING) estado_abrev,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ TIPOS:
- Mais detalhes: https://cloud.google.com/bigquery/docs/reference/standard-sql/data-types
*/
CREATE VIEW basedosdados.br_basedosdados_diretorios_brasil.municipios AS
SELECT
SAFE_CAST(id_municipio AS INT64) id_municipio,
SAFE_CAST(id_municipio_6 AS INT64) id_municipio_6,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ TIPOS:
- Mais detalhes: https://cloud.google.com/bigquery/docs/reference/standard-sql/data-types
*/
CREATE VIEW basedosdados.br_camara_atividade_legislativa.deputados AS
SELECT
SAFE_CAST(id AS NUMERIC) id_deputado,
SAFE_CAST(cpf AS NUMERIC) cpf,
Expand Down
1 change: 1 addition & 0 deletions bases/br_camara_atividade_legislativa/orgaos/publish.sql
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ TIPOS:
- Mais detalhes: https://cloud.google.com/bigquery/docs/reference/standard-sql/data-types
*/
CREATE VIEW basedosdados.br_camara_atividade_legislativa.orgaos AS
SELECT
SAFE_CAST(id_orgao AS STRING) id_orgao,
SAFE_CAST(sigla AS STRING) sigla,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -65,3 +65,4 @@ SELECT
SAFE_CAST(uf_exercicio AS STRING) sigla_uf_exercicio,
SAFE_CAST(data AS DATE) data
FROM `gabinete-compartilhado.views_publicos.servidores_executivo_federal_civis_cadastro`

1 change: 1 addition & 0 deletions bases/br_geobr_mapas/biomas/publish.sql
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ TIPOS:
- Mais detalhes: https://cloud.google.com/bigquery/docs/reference/standard-sql/data-types
*/
CREATE VIEW basedosdados.br_geobr_mapas.biomas AS
SELECT
SAFE_CAST(id_bioma AS STRING) id_bioma,
SAFE_CAST(nome_bioma AS STRING) nome_bioma,
Expand Down
1 change: 1 addition & 0 deletions bases/br_geobr_mapas/setor_censitario_2010/publish.sql
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ TIPOS:
- Mais detalhes: https://cloud.google.com/bigquery/docs/reference/standard-sql/data-types
*/
CREATE VIEW basedosdados.br_geobr_mapas.setor_censitario_2010 AS
SELECT
SAFE_CAST(code_tract AS NUMERIC) id_setor_censitario,
SAFE_CAST(zone AS STRING) zona,
Expand Down
1 change: 1 addition & 0 deletions bases/br_ibge_amc/municipio_de_para/publish.sql
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ TIPOS:
- Mais detalhes: https://cloud.google.com/bigquery/docs/reference/standard-sql/data-types
*/
CREATE VIEW basedosdados.br_ibge_amc.municipio_de_para AS
SELECT
SAFE_CAST(id_municipio AS INT64) id_municipio,
SAFE_CAST(ano_de AS INT64) ano_de,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ TIPOS:
- Mais detalhes: https://cloud.google.com/bigquery/docs/reference/standard-sql/data-types
*/
CREATE VIEW basedosdados.br_ibge_nomes_brasil.quantidade_municipio_nome_2010 AS
SELECT
SAFE_CAST(id_municipio AS INT64) id_municipio,
SAFE_CAST(nome AS STRING) nome,
Expand Down
1 change: 1 addition & 0 deletions bases/br_ibge_pib/municipios/publish.sql
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ TIPOS:
- Mais detalhes: https://cloud.google.com/bigquery/docs/reference/standard-sql/data-types
*/
CREATE VIEW basedosdados.br_ibge_pib.municipios AS
SELECT
SAFE_CAST(id_municipio AS INT64) id_municipio,
SAFE_CAST(ano AS INT64) ano,
Expand Down
1 change: 1 addition & 0 deletions bases/br_ibge_pnad_covid/microdados/publish.sql
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ TIPOS:
- Mais detalhes: https://cloud.google.com/bigquery/docs/reference/standard-sql/data-types
*/
CREATE VIEW basedosdados.br_ibge_pnad_covid.microdados AS
SELECT
SAFE_CAST(ano AS INT64) ano,
SAFE_CAST(mes AS INT64) mes,
Expand Down
1 change: 1 addition & 0 deletions bases/br_ibge_populacao/municipios/publish.sql
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ TIPOS:
- Mais detalhes: https://cloud.google.com/bigquery/docs/reference/standard-sql/data-types
*/
CREATE VIEW basedosdados.br_ibge_populacao.municipios AS
SELECT
SAFE_CAST(id_municipio AS INT64) id_municipio,
SAFE_CAST(ano AS INT64) ano,
Expand Down
1 change: 1 addition & 0 deletions bases/br_inpe_prodes/desmatamento_municipios/publish.sql
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ TIPOS:
- Mais detalhes: https://cloud.google.com/bigquery/docs/reference/standard-sql/data-types
*/
CREATE VIEW basedosdados.br_inpe_prodes.desmatamento_municipios AS
SELECT
SAFE_CAST(id_municipio AS INT64) id_municipio,
SAFE_CAST(ano AS INT64) ano,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ TIPOS:
- Mais detalhes: https://cloud.google.com/bigquery/docs/reference/standard-sql/data-types
*/
CREATE VIEW basedosdados.br_mapbiomas_estatisticas.cobertura_estado_bioma_classe AS
SELECT
SAFE_CAST(ano AS INT64) ano,
SAFE_CAST(estado_abrev AS STRING) estado_abrev,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ TIPOS:
- Mais detalhes: https://cloud.google.com/bigquery/docs/reference/standard-sql/data-types
*/
CREATE VIEW basedosdados.br_mapbiomas_estatisticas.cobertura_municipio_classe AS
SELECT
SAFE_CAST(ano AS INT64) ano,
SAFE_CAST(id_municipio AS INT64) id_municipio,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ TIPOS:
- Mais detalhes: https://cloud.google.com/bigquery/docs/reference/standard-sql/data-types
*/
CREATE VIEW basedosdados.br_mapbiomas_estatisticas.transicao_estado_bioma_de_para_anual AS
SELECT
SAFE_CAST(ano AS INT64) ano,
SAFE_CAST(estado_abrev AS STRING) estado_abrev,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ TIPOS:
- Mais detalhes: https://cloud.google.com/bigquery/docs/reference/standard-sql/data-types
*/
CREATE VIEW basedosdados.br_mapbiomas_estatisticas.transicao_estado_bioma_de_para_decenal AS
SELECT
SAFE_CAST(ano AS INT64) ano,
SAFE_CAST(estado_abrev AS STRING) estado_abrev,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ TIPOS:
- Mais detalhes: https://cloud.google.com/bigquery/docs/reference/standard-sql/data-types
*/
CREATE VIEW basedosdados.br_mapbiomas_estatisticas.transicao_estado_bioma_de_para_quinquenal AS
SELECT
SAFE_CAST(ano AS INT64) ano,
SAFE_CAST(estado_abrev AS STRING) estado_abrev,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ TIPOS:
- Mais detalhes: https://cloud.google.com/bigquery/docs/reference/standard-sql/data-types
*/
CREATE VIEW basedosdados.br_mapbiomas_estatisticas.transicao_municipio_de_para_anual AS
SELECT
SAFE_CAST(ano AS INT64) ano,
SAFE_CAST(estado_abrev AS STRING) estado_abrev,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ TIPOS:
- Mais detalhes: https://cloud.google.com/bigquery/docs/reference/standard-sql/data-types
*/
CREATE VIEW basedosdados.br_mapbiomas_estatisticas.transicao_municipio_de_para_decenal AS
SELECT
SAFE_CAST(ano AS INT64) ano,
SAFE_CAST(estado_abrev AS STRING) estado_abrev,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ TIPOS:
- Mais detalhes: https://cloud.google.com/bigquery/docs/reference/standard-sql/data-types
*/
CREATE VIEW basedosdados.br_mapbiomas_estatisticas.transicao_municipio_de_para_quinquenal AS
SELECT
SAFE_CAST(ano AS INT64) ano,
SAFE_CAST(estado_abrev AS STRING) estado_abrev,
Expand Down
1 change: 1 addition & 0 deletions bases/br_ms_sim/microdados/publish.sql
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ TIPOS:
- Mais detalhes: https://cloud.google.com/bigquery/docs/reference/standard-sql/data-types
*/
CREATE VIEW basedosdados.br_ms_sim.microdados AS
SELECT
SAFE_CAST(ano AS INT64) ano,
SAFE_CAST(estado_abrev AS STRING) estado_abrev,
Expand Down
1 change: 1 addition & 0 deletions bases/br_ms_sim/municipio/publish.sql
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ TIPOS:
- Mais detalhes: https://cloud.google.com/bigquery/docs/reference/standard-sql/data-types
*/
CREATE VIEW basedosdados.br_ms_sim.municipio AS
SELECT
SAFE_CAST(estado_abrev AS STRING) estado_abrev,
SAFE_CAST(id_municipio AS INT64) id_municipio,
Expand Down
1 change: 1 addition & 0 deletions bases/br_ms_sim/municipio_causa/publish.sql
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ TIPOS:
- Mais detalhes: https://cloud.google.com/bigquery/docs/reference/standard-sql/data-types
*/
CREATE VIEW basedosdados.br_ms_sim.municipio_causa AS
SELECT
SAFE_CAST(estado_abrev AS STRING) estado_abrev,
SAFE_CAST(id_municipio AS INT64) id_municipio,
Expand Down
1 change: 1 addition & 0 deletions bases/br_ms_sim/municipio_causa_idade/publish.sql
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ TIPOS:
- Mais detalhes: https://cloud.google.com/bigquery/docs/reference/standard-sql/data-types
*/
CREATE VIEW basedosdados.br_ms_sim.municipio_causa_idade AS
SELECT
SAFE_CAST(ano AS INT64) ano,
SAFE_CAST(estado_abrev AS STRING) estado_abrev,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ TIPOS:
- Mais detalhes: https://cloud.google.com/bigquery/docs/reference/standard-sql/data-types
*/
CREATE VIEW basedosdados.br_ms_sim.municipio_causa_idade_genero_raca AS
SELECT
SAFE_CAST(ano AS INT64) ano,
SAFE_CAST(estado_abrev AS STRING) estado_abrev,
Expand Down
3 changes: 2 additions & 1 deletion bases/br_sp_gov_orcamento/despesas/publish.sql
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ TIPOS:
- Mais detalhes: https://cloud.google.com/bigquery/docs/reference/standard-sql/data-types
*/
CREATE VIEW basedosdados.br_sp_gov_orcamento.despesas AS
SELECT
SAFE_CAST(1234567 as INT64) id_municipio,
SAFE_CAST(35 as INT64) id_estado,
Expand All @@ -38,4 +39,4 @@ SAFE_CAST(empenhado AS INT64) empenhado,
SAFE_CAST(liquidado AS INT64) liquidado,
SAFE_CAST(pago AS INT64) pago,
SAFE_CAST(pago_restos AS INT64) pago_restos
from basedosdados-staging.br_sp_gov_orcamento_staging.despesas as t
from basedosdados-staging.br_sp_gov_orcamento_staging.despesas as t
3 changes: 2 additions & 1 deletion bases/br_sp_gov_orcamento/receita_arrecadada/publish.sql
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ TIPOS:
- Mais detalhes: https://cloud.google.com/bigquery/docs/reference/standard-sql/data-types
*/
CREATE VIEW basedosdados.br_sp_gov_orcamento.receita_arrecadada AS
SELECT
SAFE_CAST(1234567 as INT64) id_municipio,
SAFE_CAST(35 as INT64) id_estado,
Expand All @@ -28,4 +29,4 @@ SELECT
SAFE_CAST(fonte_de_recursos AS STRING) fonte_de_recursos,
SAFE_CAST(receita AS STRING) receita,
SAFE_CAST(arrecadado AS INT64) arrecadado
from basedosdados-staging.br_sp_gov_orcamento_staging.receita_arrecadada as t
from basedosdados-staging.br_sp_gov_orcamento_staging.receita_arrecadada as t
3 changes: 2 additions & 1 deletion bases/br_sp_gov_orcamento/receita_prevista/publish.sql
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ TIPOS:
- Mais detalhes: https://cloud.google.com/bigquery/docs/reference/standard-sql/data-types
*/
CREATE VIEW basedosdados.br_sp_gov_orcamento.receita_prevista AS
SELECT
SAFE_CAST(1234567 as INT64) id_municipio,
SAFE_CAST(35 as INT64) id_estado,
Expand All @@ -28,4 +29,4 @@ SAFE_CAST(unidade_gestora AS STRING) unidade_gestora,
SAFE_CAST(fonte_de_recursos AS STRING) fonte_de_recursos,
SAFE_CAST(receita AS STRING) receita,
SAFE_CAST(previsto_do_ano AS INT64) previsto_do_ano
from basedosdados-staging.br_sp_gov_orcamento_staging.receita_prevista as t
from basedosdados-staging.br_sp_gov_orcamento_staging.receita_prevista as t
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ TIPOS:
- Mais detalhes: https://cloud.google.com/bigquery/docs/reference/standard-sql/data-types
*/
CREATE VIEW basedosdados.br_sp_ssp_seguranca.ocorrecencias_registradas AS
SELECT
SAFE_CAST(id_municipio AS INT64) id_municipio,
SAFE_CAST(id_estado AS INT64) id_estado,
Expand Down Expand Up @@ -47,4 +48,4 @@ SAFE_CAST(roubo_a_banco AS INT64) roubo_a_banco,
SAFE_CAST(roubo_de_carga AS INT64) roubo_de_carga,
SAFE_CAST(furto_outros AS INT64) furto_outros,
SAFE_CAST(furto_de_veiculo AS INT64) furto_de_veiculo
from basedosdados-staging.br_sp_ssp_seguranca_staging.ocorrecencias_registradas as t
from basedosdados-staging.br_sp_ssp_seguranca_staging.ocorrecencias_registradas as t
3 changes: 2 additions & 1 deletion bases/br_sp_ssp_seguranca/produtividade_policial/publish.sql
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ TIPOS:
- Mais detalhes: https://cloud.google.com/bigquery/docs/reference/standard-sql/data-types
*/
CREATE VIEW basedosdados.br_sp_ssp_seguranca.produtividade_policial AS
SELECT
SAFE_CAST(id_municipio AS INT64) id_municipio,
SAFE_CAST(id_estado AS INT64) id_estado,
Expand All @@ -37,4 +38,4 @@ SAFE_CAST(no_de_pessoas_presas_por_mandado AS INT64) no_de_pessoas_presas_por_ma
SAFE_CAST(no_de_prisoes_efetuadas AS INT64) no_de_prisoes_efetuadas,
SAFE_CAST(no_de_veiculos_recuperados AS INT64) no_de_veiculos_recuperados,
SAFE_CAST(tot_de_inqueritos_policiais_instaurados AS INT64) tot_de_inqueritos_policiais_instaurados
from basedosdados-staging.br_sp_ssp_seguranca_staging.produtividade_policial as t
from basedosdados-staging.br_sp_ssp_seguranca_staging.produtividade_policial as t
1 change: 1 addition & 0 deletions bases/br_tse_eleicoes/candidatos/publish.sql
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ TIPOS:
- Mais detalhes: https://cloud.google.com/bigquery/docs/reference/standard-sql/data-types
*/
CREATE VIEW basedosdados.br_tse_eleicoes.candidatos AS
SELECT
SAFE_CAST(ano AS INT64) ano,
SAFE_CAST(tipo_eleicao AS STRING) tipo_eleicao,
Expand Down
1 change: 1 addition & 0 deletions bases/br_tse_eleicoes/coligacoes/publish.sql
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ TIPOS:
- Mais detalhes: https://cloud.google.com/bigquery/docs/reference/standard-sql/data-types
*/
CREATE VIEW basedosdados.br_tse_eleicoes.coligacoes AS
SELECT
SAFE_CAST(ano AS INT64) ano,
SAFE_CAST(tipo_eleicao AS STRING) tipo_eleicao,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ TIPOS:
- Mais detalhes: https://cloud.google.com/bigquery/docs/reference/standard-sql/data-types
*/
CREATE VIEW basedosdados.br_tse_eleicoes.detalhes_votacao_municipio AS
SELECT
SAFE_CAST(ano AS INT64) ano,
SAFE_CAST(tipo_eleicao AS STRING) tipo_eleicao,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ TIPOS:
- Mais detalhes: https://cloud.google.com/bigquery/docs/reference/standard-sql/data-types
*/
CREATE VIEW basedosdados.br_tse_eleicoes.detalhes_votacao_municipio_zona AS
SELECT
SAFE_CAST(ano AS INT64) ano,
SAFE_CAST(tipo_eleicao AS STRING) tipo_eleicao,
Expand Down
1 change: 1 addition & 0 deletions bases/br_tse_eleicoes/detalhes_votacao_secao/publish.sql
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ TIPOS:
- Mais detalhes: https://cloud.google.com/bigquery/docs/reference/standard-sql/data-types
*/
CREATE VIEW basedosdados.br_tse_eleicoes.detalhes_votacao_secao AS
SELECT
SAFE_CAST(ano AS INT64) ano,
SAFE_CAST(tipo_eleicao AS STRING) tipo_eleicao,
Expand Down
1 change: 1 addition & 0 deletions bases/br_tse_eleicoes/local_secao/publish.sql
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ TIPOS:
- Mais detalhes: https://cloud.google.com/bigquery/docs/reference/standard-sql/data-types
*/
CREATE VIEW basedosdados.br_tse_eleicoes.local_secao AS
select
SAFE_CAST(ano AS INT64) ano,
SAFE_CAST(estado_abrev AS STRING) estado_abrev,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ TIPOS:
- Mais detalhes: https://cloud.google.com/bigquery/docs/reference/standard-sql/data-types
*/
CREATE VIEW basedosdados.br_tse_eleicoes.resultados_candidato_municipio AS
SELECT
SAFE_CAST(ano AS INT64) ano,
SAFE_CAST(tipo_eleicao AS STRING) tipo_eleicao,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ TIPOS:
- Mais detalhes: https://cloud.google.com/bigquery/docs/reference/standard-sql/data-types
*/
CREATE VIEW basedosdados.br_tse_eleicoes.resultados_candidato_municipio_zona AS
SELECT
SAFE_CAST(ano AS INT64) ano,
SAFE_CAST(tipo_eleicao AS STRING) tipo_eleicao,
Expand Down
Loading

0 comments on commit 5a71b99

Please sign in to comment.