Teste de conexão com os servidores da SERPRO (#174)

* add fedora image build and test flow

* fix env on workflow

* fix docker build

* add wait for fedora image to be available

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* add new behavior to replace docker tag, fix wait for fedora image

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* fix replace docker tag

* add logs, fix github sha ref

* update flows and CD workflows

* add setup script

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* better setup

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* add setup as state_handler

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* remove binary write

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* fix fetching secret path from diff environment

* add requirements, add JDBC class

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* fix requirements

* update cd prod

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* fix linting

---------

Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

.github/workflows/build_fedora_docker.yaml

@@ -0,0 +1,70 @@
+name: Build Fedora Image
+
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - ".github/workflows/cd.yaml"
+      - "pipelines/serpro/*"
+      - "pyproject.toml"
+      - "Dockerfile-fedora"
+  pull_request:
+    branches:
+      - main
+    paths:
+      - ".github/workflows/cd_staging.yaml"
+      - "pipelines/serpro/*"
+      - "pyproject.toml"
+      - "Dockerfile-fedora"
+env:
+  GKE_PROJECT_ID: ${{ secrets.GKE_PROJECT_ID }}
+  GKE_SA_KEY: ${{ secrets.GKE_SA_KEY }}
+  GKE_CLUSTER: ${{ secrets.GKE_APP_CLUSTER_NAME }}
+  GKE_ZONE: ${{ secrets.GKE_CLUSTER_ZONE }}
+  IMAGE_NAME: gcr.io/rj-smtr/pipelines-fedora
+
+jobs:
+  build-container:
+    name: Build Fedora Image
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Setup Python version
+        uses: actions/setup-python@v2
+        with:
+          python-version: "3.9"
+
+      - run: |-
+          pip install --no-cache-dir -r requirements.txt
+
+      # - name: Update image tag in constants
+      #   run: |-
+      #     python .github/workflows/scripts/replace_docker_tag.py gcr.io/${{ env.GKE_PROJECT_ID }}/${{ env.IMAGE_NAME }}:${{ github.sha }}
+
+      # Setup gcloud CLI
+      - name: Setup Google Cloud CLI
+        uses: google-github-actions/[email protected]
+        with:
+          service_account_key: ${{ secrets.GKE_SA_KEY }}
+          project_id: ${{ secrets.GKE_PROJECT_ID}}
+          export_default_credentials: true
+
+      - name: Get GKE credentials
+        uses: google-github-actions/[email protected]
+        with:
+          cluster_name: ${{ env.GKE_CLUSTER }}
+          location: ${{ env.GKE_ZONE }}
+          credentials: ${{ secrets.GKE_SA_KEY }}
+
+      # Configure Docker to use the gcloud command-line tool as a credential
+      # helper for authentication
+      - run: |-
+          gcloud --quiet auth configure-docker
+
+      - name: Build and publish image
+        run: |
+          docker build -t $IMAGE_NAME:${{ github.sha}} . -f Dockerfile-fedora
+          docker push $IMAGE_NAME:${{ github.sha }}

.github/workflows/cd-docs.yaml

@@ -4,7 +4,6 @@ on:
   push:
     branches:
       - main
-      - staging/*
 
 env:
   GKE_PROJECT_ID: ${{ secrets.GKE_PROJECT_ID }}

.github/workflows/cd.yaml

@@ -46,6 +46,10 @@ jobs:
         run: |-
           python .github/workflows/scripts/replace_docker_tag.py ghcr.io/${{ github.repository_owner }}/${{ github.event.repository.name }} ${{ github.sha }}
 
+      - name: Update image tag in constants
+        run: |-
+          python .github/workflows/scripts/replace_docker_tag.py fedora gcr.io/rj-smtr/pipelines-fedora ${{ github.sha }}
+
       - name: Get changed files for code tree analysis
         id: files
         uses: Ana06/[email protected]
@@ -69,6 +73,14 @@ jobs:
           repo-token: ${{ secrets.GITHUB_TOKEN }}
           verbose: true
 
+      - name: Wait for Docker image to be available
+        uses: lewagon/[email protected]
+        with:
+          ref: ${{ github.event.pull_request.head.sha || github.sha }}
+          check-name: 'Build Fedora Image'
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+          verbose: true
+
       - name: Register Prefect flows
         run: |-
           python .github/workflows/scripts/register_flows.py --project $PREFECT__SERVER__PROJECT --path pipelines/ --schedule --filter-affected-flows

.github/workflows/cd_staging.yaml

@@ -46,7 +46,12 @@ jobs:
 
       - name: Update image tag in constants
         run: |-
-          python .github/workflows/scripts/replace_docker_tag.py ghcr.io/${{ github.repository_owner }}/${{ github.event.repository.name }} ${{ github.sha }}
+          python .github/workflows/scripts/replace_docker_tag.py debian ghcr.io/${{ github.repository_owner }}/${{ github.event.repository.name }} ${{ github.sha }}
+
+      - name: Update image tag in constants
+        run: |-
+          python .github/workflows/scripts/replace_docker_tag.py fedora gcr.io/rj-smtr/pipelines-fedora ${{ github.sha }}
+
 
       - name: Get changed files for code tree analysis
         id: files
@@ -71,6 +76,14 @@ jobs:
           repo-token: ${{ secrets.GITHUB_TOKEN }}
           verbose: true
 
+      - name: Wait for Docker image to be available
+        uses: lewagon/[email protected]
+        with:
+          ref: ${{ github.event.pull_request.head.sha || github.sha }}
+          check-name: 'Build Fedora Image'
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+          verbose: true
+
       - name: Register Prefect flows
         run: |-
           python .github/workflows/scripts/register_flows.py --project $PREFECT__SERVER__PROJECT --path pipelines/ --no-schedule --filter-affected-flows

.github/workflows/scripts/replace_docker_tag.py

@@ -11,16 +11,18 @@
 FILE_PATH = Path("./pipelines/constants.py")
 REPLACE_TAG = "AUTO_REPLACE_DOCKER_TAG"
 REPLACE_IMAGE = "AUTO_REPLACE_DOCKER_IMAGE"
+REPLACE_FEDORA_IMAGE = "AUTO_REPLACE_FEDORA_IMAGE"
+REPLACE_FEDORA_TAG = "AUTO_REPLACE_FEDORA_TAG"
 
 
 def get_name_version_from_args() -> List[str]:
     """
     Returns the version from the command line arguments.
     """
-    if len(argv) != 3:
-        print("Usage: replace_docker_tag.py <image_name> <version>")
+    if len(argv) != 4:
+        print("Usage: replace_docker_tag.py <debian/fedora> <image_name> <version>")
         exit(1)
-    return argv[1], argv[2]
+    return argv[1], argv[2], argv[3]
 
 
 def replace_in_text(orig_text: str, find_text: str, replace_text: str) -> str:
@@ -30,18 +32,25 @@ def replace_in_text(orig_text: str, find_text: str, replace_text: str) -> str:
     return orig_text.replace(find_text, replace_text)
 
 
-def update_file(file_path: Path, image_name: str, version: str) -> None:
+def update_file(file_path: Path, image_name: str, version: str, mode: str = None) -> None:
     """
     Updates the `DOCKER_TAG` variable in the `constants.py` file.
     """
     with file_path.open("r") as file:
         text = file.read()
-    text = replace_in_text(text, REPLACE_TAG, version)
-    text = replace_in_text(text, REPLACE_IMAGE, image_name)
+    if mode.lower() == "fedora":
+        replace_tag = REPLACE_FEDORA_TAG
+        replace_image = REPLACE_FEDORA_IMAGE
+    else:
+        replace_tag = REPLACE_TAG
+        replace_image = REPLACE_IMAGE
+    print(f"Will replace {replace_image}:{replace_tag} -> {image_name}:{version}")
+    text = replace_in_text(text, replace_tag, version)
+    text = replace_in_text(text, replace_image, image_name)
     with file_path.open("w") as file:
         file.write(text)
 
 
 if __name__ == "__main__":
-    image_name, version = get_name_version_from_args()
-    update_file(FILE_PATH, image_name, version)
+    mode, image_name, version = get_name_version_from_args()
+    update_file(FILE_PATH, image_name, version, mode=mode)

.gitignore

@@ -128,6 +128,9 @@ ENV/
 env.bak/
 venv.bak/
 
+# DBT
+queries/profiles
+
 # Spyder project settings
 .spyderproject
 .spyproject

Dockerfile-fedora

@@ -0,0 +1,25 @@
+# Build arguments
+# ARG PYTHON_VERSION=3.10-slim
+
+# Start Python image
+FROM fedora:latest
+# RUN python3 --version
+RUN dnf -y install python3.10 && dnf clean all
+RUN yum install java -y
+RUN yum install make -y
+# RUN dnf remove python3.12
+# Setting environment with prefect version
+ARG PREFECT_VERSION=1.4.1
+ENV PREFECT_VERSION $PREFECT_VERSION
+
+# Setup virtual environment and prefect
+ENV VIRTUAL_ENV=/opt/venv
+RUN python3.10 -m venv $VIRTUAL_ENV
+ENV PATH="$VIRTUAL_ENV/bin:$PATH"
+RUN python3.10 -m pip install --no-cache-dir -U "pip>=21.2.4" "prefect==$PREFECT_VERSION"
+
+# Install requirements
+WORKDIR /app
+COPY . .
+RUN python3.10 -m pip install --prefer-binary --no-cache-dir -U .
+RUN python3 -m pip install jaydebeapi

pipelines/constants.py

@@ -19,6 +19,9 @@ class constants(Enum):  # pylint: disable=c0103
     DOCKER_TAG = "AUTO_REPLACE_DOCKER_TAG"
     DOCKER_IMAGE_NAME = "AUTO_REPLACE_DOCKER_IMAGE"
     DOCKER_IMAGE = f"{DOCKER_IMAGE_NAME}:{DOCKER_TAG}"
+    DOCKER_FEDORA_TAG = "AUTO_REPLACE_FEDORA_TAG"
+    DOCKER_FEDORA_IMAGE_NAME = "AUTO_REPLACE_FEDORA_IMAGE"
+    DOCKER_IMAGE_FEDORA = f"{DOCKER_FEDORA_IMAGE_NAME}:{DOCKER_FEDORA_TAG}"
     GCS_FLOWS_BUCKET = "datario-public"
     # PROJECT_NAME = {"dev": "rj-smtr-dev", "prod": "rj-smtr"}
     # DEFAULT_BUCKET_NAME = {"dev": "br-rj-smtr-dev", "prod": "br-rj-smtr"}

pipelines/flows.py

@@ -19,4 +19,5 @@
 from pipelines.migration.controle_financeiro.flows import *  # noqa
 from pipelines.migration.projeto_subsidio_sppo.flows import *  # noqa
 from pipelines.migration.veiculo.flows import *  # noqa
+from pipelines.serpro.flows import *  # noqa
 from pipelines.treatment.bilhetagem.flows import *  # noqa

pipelines/serpro/flows.py

@@ -0,0 +1,19 @@
+# -*- coding: utf-8 -*-
+from prefect.run_configs import KubernetesRun
+from prefect.storage import GCS
+from prefeitura_rio.pipelines_utils.custom import Flow
+
+from pipelines.constants import constants as smtr_constants
+from pipelines.serpro.tasks import wait_sleeping
+from pipelines.serpro.utils import handler_setup_serpro
+
+with Flow("SMTR - Teste Conexão Serpro") as flow:
+    # setup_serpro()
+    wait_sleeping()
+
+flow.storage = GCS(smtr_constants.GCS_FLOWS_BUCKET.value)
+flow.run_config = KubernetesRun(
+    image=smtr_constants.DOCKER_IMAGE_FEDORA.value,
+    labels=[smtr_constants.RJ_SMTR_AGENT_LABEL.value],
+)
+flow.state_handlers = [handler_setup_serpro]

pipelines/serpro/tasks.py

@@ -0,0 +1,16 @@
+# -*- coding: utf-8 -*-
+from time import sleep
+
+from prefect import task
+
+from pipelines.utils.jdbc import JDBC
+
+
+@task
+def wait_sleeping(interval_seconds: int = 54000, wait=None):
+    sleep(interval_seconds)
+
+
+@task
+def get_db_object(secret_path="radar_serpro", environment: str = "dev"):
+    return JDBC(db_params_secret_path=secret_path, environment=environment)

pipelines/serpro/utils.py

@@ -0,0 +1,25 @@
+# -*- coding: utf-8 -*-
+import os
+
+from prefect.engine.state import State
+
+from pipelines.utils.secret import get_secret
+from pipelines.utils.utils import log
+
+
+def setup_serpro(secret_path: str = "radar_serpro"):
+    data = get_secret(secret_path=secret_path)["setup.sh"]
+    log("Got Secret")
+    os.popen("touch setup.sh")
+    with open("setup.sh", "w") as f:
+        f.write(data)
+    return os.popen("sh setup.sh")
+
+
+def handler_setup_serpro(obj, old_state: State, new_state: State) -> State:
+    """
+    State handler that will inject BD credentials into the environment.
+    """
+    if new_state.is_running():
+        setup_serpro()
+    return new_state

pipelines/utils/jdbc.py

@@ -0,0 +1,70 @@
+# -*- coding: utf-8 -*-
+from typing import List
+
+import jaydebeapi as jdb
+
+from pipelines.utils.secret import get_secret
+
+
+class JDBC:
+    def __init__(self, db_params_secret_path: str, environment: str = "staging") -> None:
+        self._environment = environment
+        self._secret_path = db_params_secret_path
+        self._conn_kwargs = self.get_conn_kwargs()
+        self._connection = self.connect()
+        self._cursor = self.get_cursor()
+
+    def get_conn_kwargs(self):
+
+        data = get_secret(secret_path=self._secret_path, environment=self._environment)
+        conn_kwargs = dict(
+            jclassname=data["jclassname"],
+            user=data["user"],
+            password=data["password"],
+            url=data["url"],
+            jars=[data["jars"]],
+        )
+        return conn_kwargs
+
+    def connect(self):
+        data = get_secret(secret_path=self._secret_path, environment=self._environment)
+
+        return jdb.connect(
+            jclassname=data["jclassname"],
+            url=data["url"],
+            jars=rf"{data['jars']}",
+            driver_args=[data["user"], data["password"]],
+        )
+
+    def get_cursor(self):
+        """
+        Returns a cursor for the JDBC database.
+        """
+        return self._connection.cursor()
+
+    def execute_query(self, query: str) -> None:
+        """
+        Execute query on the JDBC database.
+
+        Args:
+            query: The query to execute.
+        """
+        self._cursor.execute(query)
+
+    def get_columns(self) -> List[str]:
+        """
+        Returns the column names of the JDBC database.
+        """
+        return [column[0] for column in self._cursor.description]
+
+    def fetch_batch(self, batch_size: int) -> List[List]:
+        """
+        Fetches a batch of rows from the JDBC database.
+        """
+        return [list(item) for item in self._cursor.fetchmany(batch_size)]
+
+    def fetch_all(self) -> List[List]:
+        """
+        Fetches all rows from the JDBC database.
+        """
+        return [list(item) for item in self._cursor.fetchall()]

pipelines/utils/secret.py

@@ -19,7 +19,7 @@ def get_secret(secret_path: str = "/", secret_name: str = None, environment: str
     if not secret_path.startswith("/"):
         secret_path = f"/{secret_path}"
     if secret_path and not secret_name:
-        secrets = client.get_all_secrets(path=secret_path)
+        secrets = client.get_all_secrets(path=secret_path, environment=environment)
         return {s.secret_name.lower(): s.secret_value for s in secrets}
     secret = client.get_secret(secret_name=secret_name, path=secret_path, environment=environment)
     return {secret_name.lower(): secret.secret_value}