add: retries to dump_upload_batch and increase resources

prefeitura-rio · Jul 12, 2024 · 1e7a965 · 1e7a965
1 parent 80cd475
commit 1e7a965
Show file tree

Hide file tree

Showing 2 changed files with 5 additions and 5 deletions.
diff --git a/pipelines/rj_segovi/dump_db_1746/flows.py b/pipelines/rj_segovi/dump_db_1746/flows.py
@@ -25,10 +25,10 @@
     labels=[
         constants.RJ_SEGOVI_AGENT_LABEL.value,
     ],
-    cpu_limit="500m",
-    cpu_request="500m",
-    memory_limit="2Gi",
-    memory_request="2Gi",
+    cpu_limit="1000m",
+    cpu_request="1000m",
+    memory_limit="4Gi",
+    memory_request="4Gi",
 )
 
 _1746_default_parameters = {

diff --git a/pipelines/utils/dump_db/tasks.py b/pipelines/utils/dump_db/tasks.py
@@ -292,7 +292,7 @@ def parse_comma_separated_string_to_list(text: str) -> List[str]:
     return result
 
 
-@task
+@task(max_retries=3, retry_delay=timedelta(seconds=300))
 def dump_upload_batch(
     database: Database,
     batch_size: int,