[infra] table approve (#269)

* [infra] table approve

* add table-approve.yaml

* setup secrets

* print env keys

* fix secrets

* parametrize bucket names

* add changes

* check table partition

* update partion check

* test table approve 2 tables

* 1 table replace test

* print sqlpublish logs

* print logs

* add mod

* small fix sql publish

* change

* add changes

* add anvisa (big test)

* add header table.config

* delete datasets

* change push to master

* docker installs basedosdados

* add mobilidados

Co-authored-by: joaoc <[email protected]>

.github/workflows/table-approve.yml

@@ -0,0 +1,53 @@
+name: Update table when modified or added
+on:
+  push:
+    branches:
+    - master
+jobs:
+  get_changes:
+    runs-on: ubuntu-latest
+    steps:
+
+      - id: file_changes
+        uses: trilom/[email protected]
+
+      - name: File Changes
+        run: |
+          cp $HOME/files.json files.json
+
+      - name: Upload Changes
+        uses: actions/upload-artifact@v2
+        with:
+          name: push-changes
+          path: files.json
+
+  to_cloud_tasks:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      # https://github.com/google-github-actions/setup-gcloud/blob/master/setup-gcloud/README.md
+      - uses: google-github-actions/setup-gcloud@master
+        with:
+          version: '290.0.1'
+          project_id: ${{ secrets.GCP_MAIN_PROJECT_ID }}
+          service_account_key: ${{ secrets.GCP_SA_KEY_TASK_CALLER }}
+          export_default_credentials: true
+
+      - name: Download changes
+        uses: actions/download-artifact@v2
+        with:
+          name: push-changes
+
+      - name: Table Approve
+        uses: ./.github/workflows/table-approve
+        with:
+          project_id: ${{ secrets.GCP_MAIN_PROJECT_ID }}
+          GCP_TABLE_APPROVE_PROD: ${{ secrets.GCP_TABLE_APPROVE_PROD }}
+          GCP_TABLE_APPROVE_STAGING: ${{ secrets.GCP_TABLE_APPROVE_STAGING }}
+          destination_bucket_name: basedosdados
+          backup_bucket_name: basedosdados-backup
+
+
+

.github/workflows/table-approve/Dockerfile

@@ -0,0 +1,7 @@
+FROM python:3.9
+WORKDIR /app
+ENV PYTHONPATH=/app PYTHONUNBUFFERED=1 PYTHONIOENCODING=UTF-8
+
+RUN pip install --upgrade "basedosdados" toml
+COPY main.py /app/
+CMD ["python", "/app/main.py"]

.github/workflows/table-approve/action.yml

@@ -0,0 +1,6 @@
+name: "Table Approve"
+description: "Approves table to production"
+author: "Base dos Dados"
+runs:
+  using: "docker"
+  image: "Dockerfile"