Merge branch 'staging/apuracao-faixa-horaria-subsidio' of https://git…

…hub.com/prefeitura-rio/pipelines_rj_smtr into staging/apuracao-faixa-horaria-subsidio
prefeitura-rio · Aug 21, 2024 · 1b682c6 · 1b682c6
2 parents 6420b94 + 3a06b10
commit 1b682c6
Show file tree

Hide file tree

Showing 11 changed files with 490 additions and 1 deletion.
diff --git a/.github/workflows/cd-docs.yaml b/.github/workflows/cd-docs.yaml
@@ -0,0 +1,76 @@
+name: CD-docs
+
+on:
+  push:
+    branches:
+      - main
+      - staging/*
+
+env:
+  GKE_PROJECT_ID: ${{ secrets.GKE_PROJECT_ID }}
+  GKE_SA_KEY: ${{ secrets.GKE_SA_KEY }}
+  GKE_CLUSTER: ${{ secrets.GKE_APP_CLUSTER_NAME }}
+  GKE_ZONE: ${{ secrets.GKE_CLUSTER_ZONE }}
+  IMAGE_NAME: gcr.io/rj-smtr/dbt-docs
+
+jobs:
+  build-container:
+    name: Build and publish docs
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Setup Python version
+        uses: actions/setup-python@v2
+        with:
+          python-version: "3.9"
+
+      - run: |-
+          pip install --no-cache-dir -r requirements.txt
+
+      # Setup gcloud CLI
+      - name: Setup Google Cloud CLI
+        uses: google-github-actions/[email protected]
+        with:
+          service_account_key: ${{ secrets.GKE_SA_KEY }}
+          project_id: ${{ secrets.GKE_PROJECT_ID}}
+          export_default_credentials: true
+
+      - name: Get GKE credentials
+        uses: google-github-actions/[email protected]
+        with:
+          cluster_name: ${{ env.GKE_CLUSTER }}
+          location: ${{ env.GKE_ZONE }}
+          credentials: ${{ secrets.GKE_SA_KEY }}
+
+      # Configure Docker to use the gcloud command-line tool as a credential
+      # helper for authentication
+      - run: |-
+          gcloud --quiet auth configure-docker
+
+      - name: Generate Docs Static Files
+        run: |
+          cd queries
+          sh ./.kubernetes/docs/generate-docs.sh $KEY
+        env:
+          KEY: ${{ secrets.GKE_SA_KEY }}
+
+      - name: Build and publish image
+        run: |
+          cd queries
+          docker build -t $IMAGE_NAME:$GITHUB_SHA . -f Dockerfile-docs
+          docker push $IMAGE_NAME:$GITHUB_SHA
+
+      - name: Set up Kustomize
+        run: |-
+          cd queries/.kubernetes/docs
+          curl -sfLo kustomize https://github.com/kubernetes-sigs/kustomize/releases/download/v3.1.0/kustomize_3.1.0_linux_amd64
+          chmod u+x ./kustomize
+
+      - name: Deploy
+        run: |-
+          cd queries/.kubernetes/docs
+          ./kustomize edit set image gcr.io/PROJECT_ID/IMAGE_DBT:TAG=$IMAGE_NAME:$GITHUB_SHA
+          ./kustomize build . | kubectl apply -n dbt-docs -f -
+          kubectl rollout status -w -n dbt-docs deployment/dbt-docs
diff --git a/pipelines/constants.py b/pipelines/constants.py
@@ -2,7 +2,7 @@
 """
 Valores constantes gerais para pipelines da rj-smtr
 
-DBT 2024-07-03
+DBT 2024-08-21
 """
 
 from enum import Enum

diff --git a/queries/.kubernetes/deployment.yaml b/queries/.kubernetes/deployment.yaml
@@ -0,0 +1,107 @@
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: dbt-rpc-dev
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: dbt-rpc-dev
+  strategy:
+    rollingUpdate:
+      maxSurge: 1
+      maxUnavailable: 1
+  minReadySeconds: 5
+  template:
+    metadata:
+      labels:
+        app: dbt-rpc-dev
+    spec:
+      containers:
+        - name: dbt-rpc
+          image: gcr.io/PROJECT_ID/IMAGE_DBT:TAG
+          ports:
+            - containerPort: 80
+          volumeMounts:
+            - name: credentials-dev
+              mountPath: /credentials-dev/
+              readOnly: true
+            - name: credentials-prod
+              mountPath: /credentials-prod/
+              readOnly: true
+          command:
+            - "dbt"
+            - "rpc"
+            - "--profiles-dir"
+            - "."
+            - "--profile"
+            - "default"
+            - "--target"
+            - "dev"
+            - "--host"
+            - "0.0.0.0"
+            - "--port"
+            - "8580"
+      volumes:
+        - name: credentials-dev
+          secret:
+            secretName: credentials-dev
+        - name: credentials-prod
+          secret:
+            secretName: credentials-prod
+      restartPolicy: Always
+
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: dbt-rpc-prod
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: dbt-rpc-prod
+  strategy:
+    rollingUpdate:
+      maxSurge: 1
+      maxUnavailable: 1
+  minReadySeconds: 5
+  template:
+    metadata:
+      labels:
+        app: dbt-rpc-prod
+    spec:
+      containers:
+        - name: dbt-rpc
+          image: gcr.io/PROJECT_ID/IMAGE_DBT:TAG
+          ports:
+            - containerPort: 80
+          volumeMounts:
+            - name: credentials-dev
+              mountPath: /credentials-dev/
+              readOnly: true
+            - name: credentials-prod
+              mountPath: /credentials-prod/
+              readOnly: true
+          command:
+            - "dbt"
+            - "rpc"
+            - "--profiles-dir"
+            - "."
+            - "--profile"
+            - "default"
+            - "--target"
+            - "prod"
+            - "--host"
+            - "0.0.0.0"
+            - "--port"
+            - "8580"
+      volumes:
+        - name: credentials-dev
+          secret:
+            secretName: credentials-dev
+        - name: credentials-prod
+          secret:
+            secretName: credentials-prod
+      restartPolicy: Always
diff --git a/queries/.kubernetes/docs/deployment.yaml b/queries/.kubernetes/docs/deployment.yaml
@@ -0,0 +1,82 @@
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: dbt-docs
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: dbt-docs
+  strategy:
+    rollingUpdate:
+      maxSurge: 1
+      maxUnavailable: 1
+  minReadySeconds: 5
+  template:
+    metadata:
+      labels:
+        app: dbt-docs
+    spec:
+      containers:
+        - name: dbt-docs-serve
+          image: gcr.io/PROJECT_ID/IMAGE_DBT:TAG
+          ports:
+            - containerPort: 3000
+
+      restartPolicy: Always
+---
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app: dbt-docs
+  name: dbt-docs
+spec:
+  ports:
+    - port: 3000
+      targetPort: 3000
+  selector:
+    app: dbt-docs
+
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: dbt-docs-ingress
+  annotations:
+    kubernetes.io/ingress.class: nginx
+    nginx.ingress.kubernetes.io/rewrite-target: /
+    cert-manager.io/issuer: "letsencrypt-dbt-docs"
+    nginx.ingress.kubernetes.io/ssl-redirect: "true"
+spec:
+  tls:
+    - hosts:
+        - docs.mobilidade.rio
+      secretName: dbt-docs-tls
+  rules:
+    - host: docs.mobilidade.rio
+      http:
+        paths:
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: dbt-docs
+                port:
+                  number: 3000
+---
+apiVersion: cert-manager.io/v1
+kind: Issuer
+metadata:
+  name: letsencrypt-dbt-docs
+spec:
+  acme:
+    server: https://acme-v02.api.letsencrypt.org/directory
+    email: [email protected]
+    privateKeySecretRef:
+      name: dbt-docs-tls
+    solvers:
+      - http01:
+          ingress:
+            class: nginx
diff --git a/queries/.kubernetes/docs/generate-docs.sh b/queries/.kubernetes/docs/generate-docs.sh
@@ -0,0 +1,40 @@
+echo "Current working directory is:" && pwd
+
+echo "Creating directories..."
+
+mkdir ./credentials-dev
+
+mkdir ./credentials-prod
+
+mkdir ./profiles
+
+echo "Mounting files from env..."
+
+bash -c "echo $1 | base64 --decode > ./credentials-dev/dev.json"
+
+bash -c "echo $1 | base64 --decode > ./credentials-prod/prod.json"
+
+echo """
+queries:
+  target: dev
+  outputs:
+    dev:
+      type: bigquery
+      method: service-account
+      project: rj-smtr
+      dataset: dbt
+      location: US
+      threads: 2
+      keyfile: $PWD/credentials-dev/dev.json
+    prod:
+      type: bigquery
+      method: service-account
+      project: rj-smtr
+      dataset: dbt
+      location: US
+      threads: 2
+      keyfile: $PWD/credentials-prod/prod.json""" > profiles/profiles.yml
+
+ls ./profiles
+
+dbt docs generate --profiles-dir ./profiles
diff --git a/queries/.kubernetes/docs/kustomization.yaml b/queries/.kubernetes/docs/kustomization.yaml
@@ -0,0 +1,4 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - ./deployment.yaml
diff --git a/queries/.kubernetes/service.yaml b/queries/.kubernetes/service.yaml
@@ -0,0 +1,29 @@
+---
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app: dbt-rpc-dev
+  name: dbt-rpc-dev
+spec:
+  ports:
+    - name: "8580"
+      port: 8580
+      targetPort: 8580
+  selector:
+    app: dbt-rpc-dev
+
+---
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app: dbt-rpc-prod
+  name: dbt-rpc-prod
+spec:
+  ports:
+    - name: "8580"
+      port: 8580
+      targetPort: 8580
+  selector:
+    app: dbt-rpc-prod
diff --git a/queries/Dockerfile-docs b/queries/Dockerfile-docs
@@ -0,0 +1,19 @@
+FROM python:3.10-slim as dbt-generate
+
+# Setup virtual environment
+ENV VIRTUAL_ENV=/opt/venv
+RUN python3 -m venv $VIRTUAL_ENV
+ENV PATH="$VIRTUAL_ENV/bin:$PATH"
+
+# Copy and install dependencies
+WORKDIR /tmp
+COPY requirements-docs.txt ./requirements.txt
+RUN pip install --no-cache-dir -r requirements.txt && \
+    rm requirements.txt
+
+# Copy dbt project and profiles
+WORKDIR /dbt
+COPY . .
+
+# Generate and serve docs
+CMD ["dbt", "docs", "serve", "--profiles-dir", ".", "--port", "3000"]
diff --git a/queries/kustomization.yaml b/queries/kustomization.yaml
@@ -0,0 +1,5 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - ./.kubernetes/deployment.yaml
+  - ./.kubernetes/service.yaml
diff --git a/queries/requirements-docs.txt b/queries/requirements-docs.txt
@@ -0,0 +1,8 @@
+dbt-bigquery==1.7.2
+dbt-core==1.7.3
+dbt-extractor==0.5.1
+dbt-semantic-interfaces==0.4.2
+markupsafe==2.0.1
+setuptools==67.8.0
+pandas==2.1.4
+pandas-gbq==0.19.2