Merge branch 'main' into staging/flow-movidesk

prefeitura-rio · Dec 27, 2024 · 649f760 · 649f760
2 parents 062421c + 3081fd0
commit 649f760
Show file tree

Hide file tree

Showing 2 changed files with 3 additions and 21 deletions.
diff --git a/pipelines/migration/tasks.py b/pipelines/migration/tasks.py
@@ -1557,16 +1557,8 @@ def transform_raw_to_nested_structure(
 
                     content_columns = [c for c in data.columns if c not in primary_key]
                     data["content"] = data.apply(
-                        lambda row: json.dumps(
-                            {
-                                key: (
-                                    row[key]
-                                    if isinstance(row[key], (list, dict)) or not pd.isna(row[key])
-                                    else None
-                                )
-                                for key in content_columns
-                            },
-                            ensure_ascii=(
+                        lambda row: row[[c for c in content_columns]].to_json(
+                            force_ascii=(
                                 constants.CONTROLE_FINANCEIRO_DATASET_ID.value not in raw_filepath
                             ),
                         ),

diff --git a/pipelines/utils/pretreatment.py b/pipelines/utils/pretreatment.py
@@ -1,6 +1,5 @@
 # -*- coding: utf-8 -*-
 """Functions to pretreat data"""
-import json
 
 import pandas as pd
 from prefeitura_rio.pipelines_utils.logging import log
@@ -20,16 +19,7 @@ def transform_to_nested_structure(data: pd.DataFrame, primary_keys: list) -> pd.
     """
     content_columns = [c for c in data.columns if c not in primary_keys]
     data["content"] = data.apply(
-        lambda row: json.dumps(
-            {
-                key: (
-                    row[key]
-                    if isinstance(row[key], (list, dict)) or not pd.isna(row[key])
-                    else None
-                )
-                for key in content_columns
-            }
-        ),
+        lambda row: row[[c for c in content_columns]].to_json(),
         axis=1,
     )
     return data[primary_keys + ["content"]]