Skip to content

Commit

Permalink
Merge branch 'main' into staging/jae-integracao-matriz-smtr
Browse files Browse the repository at this point in the history
  • Loading branch information
mergify[bot] authored Dec 27, 2024
2 parents 5acda00 + 3081fd0 commit 2503c43
Show file tree
Hide file tree
Showing 2 changed files with 3 additions and 13 deletions.
8 changes: 2 additions & 6 deletions pipelines/migration/tasks.py
Original file line number Diff line number Diff line change
Expand Up @@ -1557,12 +1557,8 @@ def transform_raw_to_nested_structure(

content_columns = [c for c in data.columns if c not in primary_key]
data["content"] = data.apply(
lambda row: json.dumps(
{
key: value if not pd.isna(value) else None
for key, value in row[content_columns].to_dict().items()
},
ensure_ascii=(
lambda row: row[[c for c in content_columns]].to_json(
force_ascii=(
constants.CONTROLE_FINANCEIRO_DATASET_ID.value not in raw_filepath
),
),
Expand Down
8 changes: 1 addition & 7 deletions pipelines/utils/pretreatment.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
# -*- coding: utf-8 -*-
"""Functions to pretreat data"""
import json

import pandas as pd
from prefeitura_rio.pipelines_utils.logging import log
Expand All @@ -20,12 +19,7 @@ def transform_to_nested_structure(data: pd.DataFrame, primary_keys: list) -> pd.
"""
content_columns = [c for c in data.columns if c not in primary_keys]
data["content"] = data.apply(
lambda row: json.dumps(
{
key: value if not pd.isna(value) else None
for key, value in row[content_columns].to_dict().items()
}
),
lambda row: row[[c for c in content_columns]].to_json(),
axis=1,
)
return data[primary_keys + ["content"]]
Expand Down

0 comments on commit 2503c43

Please sign in to comment.