From 1ece00598dc5583c85d196fed2632fc21666f1cf Mon Sep 17 00:00:00 2001
From: Amisha Singla <amishas157@gmail.com>
Date: Tue, 12 Nov 2024 13:08:49 -0600
Subject: [PATCH] Fix schedule, add vars in prod

---
 airflow_variables_dev.json                                | 8 ++++----
 airflow_variables_prod.json                               | 8 ++++++++
 dags/dbt_data_quality_alerts_dag.py                       | 2 +-
 dags/elementary_report_dag.py                             | 8 ++++----
 .../build_elementary_slack_alert_task.py                  | 2 +-
 example_airflow_worker_config.yaml                        | 2 +-
 6 files changed, 19 insertions(+), 11 deletions(-)

diff --git a/airflow_variables_dev.json b/airflow_variables_dev.json
index 71481ffc..141f6f5a 100644
--- a/airflow_variables_dev.json
+++ b/airflow_variables_dev.json
@@ -288,21 +288,21 @@
     "dbt": {
       "requests": {
         "cpu": "1",
-        "ephemeral-storage": "500Mi",
+        "ephemeral-storage": "1Gi",
         "memory": "600Mi"
       }
     },
     "default": {
       "requests": {
         "cpu": "0.3",
-        "ephemeral-storage": "500Mi",
+        "ephemeral-storage": "1Gi",
         "memory": "600Mi"
       }
     },
     "stellaretl": {
       "requests": {
         "cpu": "0.3",
-        "ephemeral-storage": "500Mi",
+        "ephemeral-storage": "1Gi",
         "memory": "600Mi"
       }
     },
@@ -310,7 +310,7 @@
       "requests": {
         "cpu": "1",
         "ephemeral_storage": "2Gi",
-        "memory": "8Gi"
+        "memory": "4Gi"
       }
     }
   },
diff --git a/airflow_variables_prod.json b/airflow_variables_prod.json
index 3ebff87d..d914f786 100644
--- a/airflow_variables_prod.json
+++ b/airflow_variables_prod.json
@@ -303,6 +303,13 @@
         "ephemeral-storage": "1Gi",
         "memory": "1Gi"
       }
+    },
+    "elementaryreport": {
+      "requests": {
+        "cpu": "1",
+        "ephemeral_storage": "2Gi",
+        "memory": "4Gi"
+      }
     }
   },
   "sandbox_dataset": "crypto_stellar_internal_sandbox",
@@ -344,6 +351,7 @@
     "current_state": 1200,
     "default": 60,
     "elementary_dbt_data_quality": 2100,
+    "elementary_generate_report": 1200,
     "enriched_history_operations": 1800,
     "enriched_history_operations_with_exclude": 1800,
     "fee_stats": 360,
diff --git a/dags/dbt_data_quality_alerts_dag.py b/dags/dbt_data_quality_alerts_dag.py
index 3a599218..b5a8de3b 100644
--- a/dags/dbt_data_quality_alerts_dag.py
+++ b/dags/dbt_data_quality_alerts_dag.py
@@ -29,6 +29,6 @@
 ) as dag:
 
     # Trigger elementary
-    elementary_alerts = elementary_task(dag, "dbt_data_quality", resource_cfg="dbt")
+    elementary_alerts = elementary_task(dag, "dbt_data_quality", "monitor", resource_cfg="dbt")
 
     elementary_alerts
diff --git a/dags/elementary_report_dag.py b/dags/elementary_report_dag.py
index 9f8c32cd..6ee1b4a9 100644
--- a/dags/elementary_report_dag.py
+++ b/dags/elementary_report_dag.py
@@ -16,9 +16,9 @@
 with DAG(
     "elementary_report",
     default_args=get_default_dag_args(),
-    start_date=datetime(2024, 6, 25, 0, 0),
+    start_date=datetime(2024, 11, 11, 0, 0),
     description="This DAG creates elementary report and send it to slack",
-    schedule="*/60 * * * *",  # Runs every 60 minutes
+    schedule="0 0 * * MON",  # Runs every Monday
     user_defined_filters={
         "container_resources": lambda s: k8s.V1ResourceRequirements(requests=s),
     },
@@ -30,9 +30,9 @@
     elementary_alerts = elementary_task(
         dag,
         "generate_report",
+        "send-report",
         resource_cfg="elementaryreport",
-        command="send-report",
-        cmd_args=["--days-back", "7", "--profiles-dir", ".", "--executions-limit", "120"],
+        cmd_args=["--days-back", "7", "--profiles-dir", ".", "--executions-limit", "120", "--slack-file-name", f"elementary_report_{datetime.today().date()}.html"],
     )
 
     elementary_alerts
diff --git a/dags/stellar_etl_airflow/build_elementary_slack_alert_task.py b/dags/stellar_etl_airflow/build_elementary_slack_alert_task.py
index 84ccad89..8435827c 100644
--- a/dags/stellar_etl_airflow/build_elementary_slack_alert_task.py
+++ b/dags/stellar_etl_airflow/build_elementary_slack_alert_task.py
@@ -20,7 +20,7 @@ def access_secret(secret_name, namespace):
 
 
 def elementary_task(
-    dag, task_name, cmd_args=[], resource_cfg="default", command="monitor"
+    dag, task_name, command, cmd_args=[], resource_cfg="default"
 ):
     namespace = conf.get("kubernetes", "NAMESPACE")
 
diff --git a/example_airflow_worker_config.yaml b/example_airflow_worker_config.yaml
index 0a8911dc..dd961858 100644
--- a/example_airflow_worker_config.yaml
+++ b/example_airflow_worker_config.yaml
@@ -154,7 +154,7 @@ spec:
           name: gcs-syncd
           resources:
             limits:
-              memory: 2400Mi
+              memory: 600Mi
             requests:
               cpu: 10m
               memory: 50Mi