Fix schedule, add vars in prod

stellar · Nov 12, 2024 · 1ece005 · 1ece005
1 parent 6a645fb
commit 1ece005
Show file tree

Hide file tree

Showing 6 changed files with 19 additions and 11 deletions.
diff --git a/airflow_variables_dev.json b/airflow_variables_dev.json
@@ -288,29 +288,29 @@
     "dbt": {
       "requests": {
         "cpu": "1",
-        "ephemeral-storage": "500Mi",
+        "ephemeral-storage": "1Gi",
         "memory": "600Mi"
       }
     },
     "default": {
       "requests": {
         "cpu": "0.3",
-        "ephemeral-storage": "500Mi",
+        "ephemeral-storage": "1Gi",
         "memory": "600Mi"
       }
     },
     "stellaretl": {
       "requests": {
         "cpu": "0.3",
-        "ephemeral-storage": "500Mi",
+        "ephemeral-storage": "1Gi",
         "memory": "600Mi"
       }
     },
     "elementaryreport": {
       "requests": {
         "cpu": "1",
         "ephemeral_storage": "2Gi",
-        "memory": "8Gi"
+        "memory": "4Gi"
       }
     }
   },

diff --git a/airflow_variables_prod.json b/airflow_variables_prod.json
@@ -303,6 +303,13 @@
         "ephemeral-storage": "1Gi",
         "memory": "1Gi"
       }
+    },
+    "elementaryreport": {
+      "requests": {
+        "cpu": "1",
+        "ephemeral_storage": "2Gi",
+        "memory": "4Gi"
+      }
     }
   },
   "sandbox_dataset": "crypto_stellar_internal_sandbox",
@@ -344,6 +351,7 @@
     "current_state": 1200,
     "default": 60,
     "elementary_dbt_data_quality": 2100,
+    "elementary_generate_report": 1200,
     "enriched_history_operations": 1800,
     "enriched_history_operations_with_exclude": 1800,
     "fee_stats": 360,

diff --git a/dags/dbt_data_quality_alerts_dag.py b/dags/dbt_data_quality_alerts_dag.py
@@ -29,6 +29,6 @@
 ) as dag:
 
     # Trigger elementary
-    elementary_alerts = elementary_task(dag, "dbt_data_quality", resource_cfg="dbt")
+    elementary_alerts = elementary_task(dag, "dbt_data_quality", "monitor", resource_cfg="dbt")
 
     elementary_alerts
diff --git a/dags/elementary_report_dag.py b/dags/elementary_report_dag.py
@@ -16,9 +16,9 @@
 with DAG(
     "elementary_report",
     default_args=get_default_dag_args(),
-    start_date=datetime(2024, 6, 25, 0, 0),
+    start_date=datetime(2024, 11, 11, 0, 0),
     description="This DAG creates elementary report and send it to slack",
-    schedule="*/60 * * * *",  # Runs every 60 minutes
+    schedule="0 0 * * MON",  # Runs every Monday
     user_defined_filters={
         "container_resources": lambda s: k8s.V1ResourceRequirements(requests=s),
     },
@@ -30,9 +30,9 @@
     elementary_alerts = elementary_task(
         dag,
         "generate_report",
+        "send-report",
         resource_cfg="elementaryreport",
-        command="send-report",
-        cmd_args=["--days-back", "7", "--profiles-dir", ".", "--executions-limit", "120"],
+        cmd_args=["--days-back", "7", "--profiles-dir", ".", "--executions-limit", "120", "--slack-file-name", f"elementary_report_{datetime.today().date()}.html"],
     )
 
     elementary_alerts
diff --git a/dags/stellar_etl_airflow/build_elementary_slack_alert_task.py b/dags/stellar_etl_airflow/build_elementary_slack_alert_task.py
@@ -20,7 +20,7 @@ def access_secret(secret_name, namespace):
 
 
 def elementary_task(
-    dag, task_name, cmd_args=[], resource_cfg="default", command="monitor"
+    dag, task_name, command, cmd_args=[], resource_cfg="default"
 ):
     namespace = conf.get("kubernetes", "NAMESPACE")
 

diff --git a/example_airflow_worker_config.yaml b/example_airflow_worker_config.yaml
@@ -154,7 +154,7 @@ spec:
           name: gcs-syncd
           resources:
             limits:
-              memory: 2400Mi
+              memory: 600Mi
             requests:
               cpu: 10m
               memory: 50Mi