From 82def62a2b6008cd7e816572de01905913848fd0 Mon Sep 17 00:00:00 2001 From: Rafael Carvalho Pinheiro <74972217+pixuimpou@users.noreply.github.com> Date: Wed, 21 Aug 2024 09:13:14 -0300 Subject: [PATCH 1/2] registra flows (#156) --- pipelines/constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipelines/constants.py b/pipelines/constants.py index 30cd70cb6..320439249 100644 --- a/pipelines/constants.py +++ b/pipelines/constants.py @@ -2,7 +2,7 @@ """ Valores constantes gerais para pipelines da rj-smtr -DBT 2024-07-03 +DBT 2024-08-21 """ from enum import Enum From 491d7c684e3ba58ac16c85bfbb3a0223b32d6f3e Mon Sep 17 00:00:00 2001 From: Caio R S dos Santos Date: Wed, 21 Aug 2024 11:37:58 -0300 Subject: [PATCH 2/2] [FEATURE] Add CD for dbt docs (#151) * add docs cd * add requirements * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * fix repo secrets * fix env ref * fix env ref * redo all secrets for deploying * readd python deps * fix deploy shell script * add cd queries * fix relative paths * change cred mount path * fix typo * add ls for debugging * add more debug * new profiles strategy * run scripts at queries dirpath * rename profiles * add relative paths * fix dockerfile * add new context for docker building * fix requirements --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- .github/workflows/cd-docs.yaml | 76 +++++++++++++ queries/.kubernetes/deployment.yaml | 107 ++++++++++++++++++ queries/.kubernetes/docs/deployment.yaml | 82 ++++++++++++++ queries/.kubernetes/docs/generate-docs.sh | 40 +++++++ queries/.kubernetes/docs/kustomization.yaml | 4 + queries/.kubernetes/service.yaml | 29 +++++ queries/Dockerfile-docs | 19 ++++ queries/kustomization.yaml | 5 + queries/requirements-docs.txt | 8 ++ requirements.txt | 119 ++++++++++++++++++++ 10 files changed, 489 insertions(+) create mode 100644 .github/workflows/cd-docs.yaml create mode 100644 queries/.kubernetes/deployment.yaml create mode 100644 queries/.kubernetes/docs/deployment.yaml create mode 100644 queries/.kubernetes/docs/generate-docs.sh create mode 100644 queries/.kubernetes/docs/kustomization.yaml create mode 100644 queries/.kubernetes/service.yaml create mode 100644 queries/Dockerfile-docs create mode 100644 queries/kustomization.yaml create mode 100644 queries/requirements-docs.txt create mode 100644 requirements.txt diff --git a/.github/workflows/cd-docs.yaml b/.github/workflows/cd-docs.yaml new file mode 100644 index 000000000..575a2215c --- /dev/null +++ b/.github/workflows/cd-docs.yaml @@ -0,0 +1,76 @@ +name: CD-docs + +on: + push: + branches: + - main + - staging/* + +env: + GKE_PROJECT_ID: ${{ secrets.GKE_PROJECT_ID }} + GKE_SA_KEY: ${{ secrets.GKE_SA_KEY }} + GKE_CLUSTER: ${{ secrets.GKE_APP_CLUSTER_NAME }} + GKE_ZONE: ${{ secrets.GKE_CLUSTER_ZONE }} + IMAGE_NAME: gcr.io/rj-smtr/dbt-docs + +jobs: + build-container: + name: Build and publish docs + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Setup Python version + uses: actions/setup-python@v2 + with: + python-version: "3.9" + + - run: |- + pip install --no-cache-dir -r requirements.txt + + # Setup gcloud CLI + - name: Setup Google Cloud CLI + uses: google-github-actions/setup-gcloud@v0.2.1 + with: + service_account_key: ${{ secrets.GKE_SA_KEY }} + project_id: ${{ secrets.GKE_PROJECT_ID}} + export_default_credentials: true + + - name: Get GKE credentials + uses: google-github-actions/get-gke-credentials@v0.2.1 + with: + cluster_name: ${{ env.GKE_CLUSTER }} + location: ${{ env.GKE_ZONE }} + credentials: ${{ secrets.GKE_SA_KEY }} + + # Configure Docker to use the gcloud command-line tool as a credential + # helper for authentication + - run: |- + gcloud --quiet auth configure-docker + + - name: Generate Docs Static Files + run: | + cd queries + sh ./.kubernetes/docs/generate-docs.sh $KEY + env: + KEY: ${{ secrets.GKE_SA_KEY }} + + - name: Build and publish image + run: | + cd queries + docker build -t $IMAGE_NAME:$GITHUB_SHA . -f Dockerfile-docs + docker push $IMAGE_NAME:$GITHUB_SHA + + - name: Set up Kustomize + run: |- + cd queries/.kubernetes/docs + curl -sfLo kustomize https://github.com/kubernetes-sigs/kustomize/releases/download/v3.1.0/kustomize_3.1.0_linux_amd64 + chmod u+x ./kustomize + + - name: Deploy + run: |- + cd queries/.kubernetes/docs + ./kustomize edit set image gcr.io/PROJECT_ID/IMAGE_DBT:TAG=$IMAGE_NAME:$GITHUB_SHA + ./kustomize build . | kubectl apply -n dbt-docs -f - + kubectl rollout status -w -n dbt-docs deployment/dbt-docs diff --git a/queries/.kubernetes/deployment.yaml b/queries/.kubernetes/deployment.yaml new file mode 100644 index 000000000..fe3449976 --- /dev/null +++ b/queries/.kubernetes/deployment.yaml @@ -0,0 +1,107 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: dbt-rpc-dev +spec: + replicas: 1 + selector: + matchLabels: + app: dbt-rpc-dev + strategy: + rollingUpdate: + maxSurge: 1 + maxUnavailable: 1 + minReadySeconds: 5 + template: + metadata: + labels: + app: dbt-rpc-dev + spec: + containers: + - name: dbt-rpc + image: gcr.io/PROJECT_ID/IMAGE_DBT:TAG + ports: + - containerPort: 80 + volumeMounts: + - name: credentials-dev + mountPath: /credentials-dev/ + readOnly: true + - name: credentials-prod + mountPath: /credentials-prod/ + readOnly: true + command: + - "dbt" + - "rpc" + - "--profiles-dir" + - "." + - "--profile" + - "default" + - "--target" + - "dev" + - "--host" + - "0.0.0.0" + - "--port" + - "8580" + volumes: + - name: credentials-dev + secret: + secretName: credentials-dev + - name: credentials-prod + secret: + secretName: credentials-prod + restartPolicy: Always + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: dbt-rpc-prod +spec: + replicas: 1 + selector: + matchLabels: + app: dbt-rpc-prod + strategy: + rollingUpdate: + maxSurge: 1 + maxUnavailable: 1 + minReadySeconds: 5 + template: + metadata: + labels: + app: dbt-rpc-prod + spec: + containers: + - name: dbt-rpc + image: gcr.io/PROJECT_ID/IMAGE_DBT:TAG + ports: + - containerPort: 80 + volumeMounts: + - name: credentials-dev + mountPath: /credentials-dev/ + readOnly: true + - name: credentials-prod + mountPath: /credentials-prod/ + readOnly: true + command: + - "dbt" + - "rpc" + - "--profiles-dir" + - "." + - "--profile" + - "default" + - "--target" + - "prod" + - "--host" + - "0.0.0.0" + - "--port" + - "8580" + volumes: + - name: credentials-dev + secret: + secretName: credentials-dev + - name: credentials-prod + secret: + secretName: credentials-prod + restartPolicy: Always diff --git a/queries/.kubernetes/docs/deployment.yaml b/queries/.kubernetes/docs/deployment.yaml new file mode 100644 index 000000000..e14e4d5ac --- /dev/null +++ b/queries/.kubernetes/docs/deployment.yaml @@ -0,0 +1,82 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: dbt-docs +spec: + replicas: 1 + selector: + matchLabels: + app: dbt-docs + strategy: + rollingUpdate: + maxSurge: 1 + maxUnavailable: 1 + minReadySeconds: 5 + template: + metadata: + labels: + app: dbt-docs + spec: + containers: + - name: dbt-docs-serve + image: gcr.io/PROJECT_ID/IMAGE_DBT:TAG + ports: + - containerPort: 3000 + + restartPolicy: Always +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app: dbt-docs + name: dbt-docs +spec: + ports: + - port: 3000 + targetPort: 3000 + selector: + app: dbt-docs + +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: dbt-docs-ingress + annotations: + kubernetes.io/ingress.class: nginx + nginx.ingress.kubernetes.io/rewrite-target: / + cert-manager.io/issuer: "letsencrypt-dbt-docs" + nginx.ingress.kubernetes.io/ssl-redirect: "true" +spec: + tls: + - hosts: + - docs.mobilidade.rio + secretName: dbt-docs-tls + rules: + - host: docs.mobilidade.rio + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: dbt-docs + port: + number: 3000 +--- +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: letsencrypt-dbt-docs +spec: + acme: + server: https://acme-v02.api.letsencrypt.org/directory + email: gsperim@gmail.com + privateKeySecretRef: + name: dbt-docs-tls + solvers: + - http01: + ingress: + class: nginx diff --git a/queries/.kubernetes/docs/generate-docs.sh b/queries/.kubernetes/docs/generate-docs.sh new file mode 100644 index 000000000..4c3a25bfd --- /dev/null +++ b/queries/.kubernetes/docs/generate-docs.sh @@ -0,0 +1,40 @@ +echo "Current working directory is:" && pwd + +echo "Creating directories..." + +mkdir ./credentials-dev + +mkdir ./credentials-prod + +mkdir ./profiles + +echo "Mounting files from env..." + +bash -c "echo $1 | base64 --decode > ./credentials-dev/dev.json" + +bash -c "echo $1 | base64 --decode > ./credentials-prod/prod.json" + +echo """ +queries: + target: dev + outputs: + dev: + type: bigquery + method: service-account + project: rj-smtr + dataset: dbt + location: US + threads: 2 + keyfile: $PWD/credentials-dev/dev.json + prod: + type: bigquery + method: service-account + project: rj-smtr + dataset: dbt + location: US + threads: 2 + keyfile: $PWD/credentials-prod/prod.json""" > profiles/profiles.yml + +ls ./profiles + +dbt docs generate --profiles-dir ./profiles \ No newline at end of file diff --git a/queries/.kubernetes/docs/kustomization.yaml b/queries/.kubernetes/docs/kustomization.yaml new file mode 100644 index 000000000..2286bd1f1 --- /dev/null +++ b/queries/.kubernetes/docs/kustomization.yaml @@ -0,0 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ./deployment.yaml diff --git a/queries/.kubernetes/service.yaml b/queries/.kubernetes/service.yaml new file mode 100644 index 000000000..fe0b89b68 --- /dev/null +++ b/queries/.kubernetes/service.yaml @@ -0,0 +1,29 @@ +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app: dbt-rpc-dev + name: dbt-rpc-dev +spec: + ports: + - name: "8580" + port: 8580 + targetPort: 8580 + selector: + app: dbt-rpc-dev + +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app: dbt-rpc-prod + name: dbt-rpc-prod +spec: + ports: + - name: "8580" + port: 8580 + targetPort: 8580 + selector: + app: dbt-rpc-prod diff --git a/queries/Dockerfile-docs b/queries/Dockerfile-docs new file mode 100644 index 000000000..055bc6d22 --- /dev/null +++ b/queries/Dockerfile-docs @@ -0,0 +1,19 @@ +FROM python:3.10-slim as dbt-generate + +# Setup virtual environment +ENV VIRTUAL_ENV=/opt/venv +RUN python3 -m venv $VIRTUAL_ENV +ENV PATH="$VIRTUAL_ENV/bin:$PATH" + +# Copy and install dependencies +WORKDIR /tmp +COPY requirements-docs.txt ./requirements.txt +RUN pip install --no-cache-dir -r requirements.txt && \ + rm requirements.txt + +# Copy dbt project and profiles +WORKDIR /dbt +COPY . . + +# Generate and serve docs +CMD ["dbt", "docs", "serve", "--profiles-dir", ".", "--port", "3000"] \ No newline at end of file diff --git a/queries/kustomization.yaml b/queries/kustomization.yaml new file mode 100644 index 000000000..6db7fcdc2 --- /dev/null +++ b/queries/kustomization.yaml @@ -0,0 +1,5 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ./.kubernetes/deployment.yaml + - ./.kubernetes/service.yaml diff --git a/queries/requirements-docs.txt b/queries/requirements-docs.txt new file mode 100644 index 000000000..0a2d37b10 --- /dev/null +++ b/queries/requirements-docs.txt @@ -0,0 +1,8 @@ +dbt-bigquery==1.7.2 +dbt-core==1.7.3 +dbt-extractor==0.5.1 +dbt-semantic-interfaces==0.4.2 +markupsafe==2.0.1 +setuptools==67.8.0 +pandas==2.1.4 +pandas-gbq==0.19.2 \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 000000000..9715c7943 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,119 @@ +agate==1.7.1 +attrs==23.1.0 +Babel==2.14.0 +backoff==2.2.1 +basedosdados==2.0.0b13 +cachetools==5.3.2 +certifi==2023.11.17 +cffi==1.16.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +colorama==0.4.6 +croniter==2.0.1 +dask==2023.12.0 +db-dtypes==1.2.0 +dbt-bigquery==1.7.2 +dbt-core==1.7.3 +dbt-extractor==0.5.1 +dbt-semantic-interfaces==0.4.2 +distributed==2023.12.0 +docker==7.0.0 +fsspec==2023.12.2 +google-api-core==2.15.0 +google-api-python-client==2.110.0 +google-auth==2.25.2 +google-auth-httplib2==0.2.0 +google-auth-oauthlib==1.2.0 +google-cloud-bigquery==3.14.0 +google-cloud-bigquery-connection==1.14.0 +google-cloud-bigquery-storage==2.23.0 +google-cloud-core==2.4.1 +google-cloud-dataproc==5.8.0 +google-cloud-storage==2.13.0 +google-crc32c==1.5.0 +google-resumable-media==2.6.0 +googleapis-common-protos==1.62.0 +gql==3.4.1 +graphql-core==3.2.3 +grpc-google-iam-v1==0.13.0 +grpcio==1.60.0 +grpcio-status==1.60.0 +httplib2==0.22.0 +idna==3.6 +importlib-metadata==6.11.0 +importlib-resources==6.1.1 +infisical==1.5.0 +isodate==0.6.1 +Jinja2==3.1.2 +jsonschema==4.20.0 +jsonschema-specifications==2023.11.2 +leather==0.3.4 +locket==1.0.0 +Logbook==1.5.3 +loguru==0.7.0 +MarkupSafe==2.1.3 +marshmallow==3.20.1 +marshmallow-oneofschema==3.0.1 +mashumaro==3.11 +minimal-snowplow-tracker==0.0.2 +more-itertools==10.1.0 +msgpack==1.0.7 +multidict==6.0.4 +mypy-extensions==1.0.0 +networkx==3.2.1 +numpy==1.26.2 +oauthlib==3.2.2 +packaging==23.2 +pandas==2.1.4 +pandas-gbq==0.19.2 +parsedatetime==2.6 +partd==1.4.1 +pathspec==0.11.2 +pendulum==2.1.2 +prefect==1.4.1 +prefeitura-rio==1.1.3a1 +proto-plus==1.23.0 +protobuf==4.25.1 +psutil==5.9.6 +pyarrow==14.0.1 +pyasn1==0.5.1 +pyasn1-modules==0.3.0 +pycparser==2.21 +pycryptodomex==3.19.0 +pydantic==1.10.13 +pydata-google-auth==1.8.2 +PyNaCl==1.5.0 +pyparsing==3.1.1 +python-box==7.1.1 +python-dateutil==2.8.2 +python-slugify==8.0.1 +pytimeparse==1.1.8 +pytz==2023.3.post1 +pytzdata==2020.1 +PyYAML==6.0.1 +referencing==0.32.0 +requests==2.31.0 +requests-oauthlib==1.3.1 +requests-toolbelt==1.0.0 +rpds-py==0.13.2 +rsa==4.9 +sentry-sdk==1.39.0 +six==1.16.0 +sortedcontainers==2.4.0 +sqlparse==0.4.4 +tabulate==0.9.0 +tblib==3.0.0 +text-unidecode==1.3 +toml==0.10.2 +tomlkit==0.11.8 +toolz==0.12.0 +tornado==6.4 +tqdm==4.66.1 +typing_extensions==4.9.0 +tzdata==2023.3 +uritemplate==4.1.1 +urllib3==1.26.18 +yarl==1.9.4 +zict==3.0.0 +zipp==3.17.0