add pre_treatment_reader_args

prefeitura-rio · Oct 3, 2024 · 0cec8e0 · 0cec8e0
1 parent 59acf79
commit 0cec8e0
Show file tree

Hide file tree

Showing 2 changed files with 3 additions and 1 deletion.
diff --git a/pipelines/serpro/constants.py b/pipelines/serpro/constants.py
@@ -28,4 +28,5 @@ class constants(Enum):  # pylint: disable=c0103
                 SUBSTRING(auinf_dt_infracao, 1, 10) = '{date}'
         """,
         "primary_key": ["auinf_num_auto"],
+        "pre_treatment_reader_args": {"dtype": "object"},
     }
diff --git a/pipelines/serpro/flows.py b/pipelines/serpro/flows.py
@@ -67,8 +67,9 @@
     transform_raw_to_nested_structure_results = transform_raw_to_nested_structure.map(
         raw_filepath=raw_filepaths,
         filepath=local_filepaths,
-        primary_key=constants.SERPRO_CAPTURE_PARAMS.value["primary_key"],
+        primary_key=unmapped(constants.SERPRO_CAPTURE_PARAMS.value["primary_key"]),
         timestamp=timestamps,
+        reader_args=unmapped(constants.SERPRO_CAPTURE_PARAMS.value["pre_treatment_reader_args"]),
         error=unmapped(None),
     )