Merge pull request #9280 from OpenMined/add_failed_alerts

ADD Failed Job Email alerts
OpenMined · Sep 11, 2024 · 04d2d5f · 04d2d5f
2 parents dceb057 + f647e71
commit 04d2d5f
Show file tree

Hide file tree

Showing 3 changed files with 133 additions and 1 deletion.
diff --git a/notebooks/scenarios/bigquery/020-configure-api.ipynb b/notebooks/scenarios/bigquery/020-configure-api.ipynb
@@ -524,7 +524,10 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "email_server.get_emails_for_user(user_email=ADMIN_EMAIL)"
+    "assert (\n",
+    "    \"Job Failed\"\n",
+    "    in email_server.get_emails_for_user(user_email=ADMIN_EMAIL)[0].email_content\n",
+    ")"
    ]
   },
   {
@@ -606,6 +609,11 @@
   }
  ],
  "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
   "language_info": {
    "codemirror_mode": {
     "name": "ipython",

diff --git a/packages/syft/src/syft/service/notification/email_templates.py b/packages/syft/src/syft/service/notification/email_templates.py
@@ -23,6 +23,107 @@ def email_body(notification: "Notification", context: AuthedServiceContext) -> s
         return ""
 
 
+@serializable(canonical_name="FailedJobTemplate", version=1)
+class FailedJobTemplate(EmailTemplate):
+    @staticmethod
+    def email_title(notification: "Notification", context: AuthedServiceContext) -> str:
+        return "Job Failed Notification"
+
+    @staticmethod
+    def email_body(notification: "Notification", context: AuthedServiceContext) -> str:
+        notification.linked_obj = cast(LinkedObject, notification.linked_obj)
+        queueitem_obj = notification.linked_obj.resolve_with_context(
+            context=context
+        ).unwrap()
+
+        worker_pool_obj = queueitem_obj.worker_pool.resolve_with_context(
+            context=context
+        ).unwrap()
+        method = queueitem_obj.method
+        if queueitem_obj.service == "apiservice":
+            method = queueitem_obj.kwargs.pop("path", "")
+            queueitem_obj.kwargs.pop("log_id")
+
+        head = """
+            <head>
+              <meta charset="UTF-8">
+              <meta name="viewport" content="width=device-width, initial-scale=1.0">
+              <title>Job Failed Notification</title>
+              <style>
+                body {
+                  font-family: Arial, sans-serif;
+                  background-color: #f4f4f4;
+                  margin: 0;
+                  padding: 0;
+                }
+                .container {
+                  width: 100%;
+                  max-width: 600px;
+                  margin: 0 auto;
+                  background-color: #ffffff;
+                  padding: 20px;
+                  border-radius: 8px;
+                  box-shadow: 0 2px 4px rgba(0, 0, 0, 0.1);
+                }
+                h1 {
+                  font-size: 24px;
+                  color: #333333;
+                }
+                p {
+                  font-size: 16px;
+                  color: #666666;
+                  line-height: 1.5;
+                }
+                .card {
+                  background-color: #f9f9f9;
+                  padding: 15px;
+                  border-radius: 8px;
+                  box-shadow: 0 1px 3px rgba(0, 0, 0, 0.1);
+                  margin-top: 20px;
+                }
+                .card h2 {
+                  font-size: 18px;
+                  color: #333333;
+                  margin-bottom: 10px;
+                }
+                .card p {
+                  font-size: 14px;
+                  color: #555555;
+                  margin: 5px 0;
+                }
+                .footer {
+                  font-size: 12px;
+                  color: #999999;
+                  margin-top: 30px;
+                }
+              </style>
+            </head>
+        """
+        body = f"""
+            <body>
+              <div class="container">
+                <h1>Job Failed Notification</h1>
+                <p>Hello,</p>
+                <p>We regret to inform you that your function job has encountered an
+                unexpected error and could not be completed successfully.</p>
+
+                <div class="card">
+                  <h2>Job Details</h2>
+                  <p><strong>Job ID:</strong> {queueitem_obj.job_id}</p>
+                  <p><strong>Worker Pool:</strong> {worker_pool_obj.name}</p>
+                  <p><strong>Method:</strong> {method}</p>
+                  <p><strong>Service:</strong> {queueitem_obj.service}</p>
+                  <p><strong>Arguments (args):</strong> {queueitem_obj.args}</p>
+                  <p><strong>Keyword Arguments (kwargs):</strong> {queueitem_obj.kwargs}</p>
+                </div>
+
+                <p class="footer">If you need assistance, feel free to reach out to our support team.</p>
+              </div>
+            </body>
+        """
+        return f"""<html>{head} {body}</html>"""
+
+
 @serializable(canonical_name="PasswordResetTemplate", version=1)
 class PasswordResetTemplate(EmailTemplate):
     @staticmethod

diff --git a/packages/syft/src/syft/service/queue/queue.py b/packages/syft/src/syft/service/queue/queue.py
@@ -17,11 +17,16 @@
 from ...server.worker_settings import WorkerSettings
 from ...service.context import AuthedServiceContext
 from ...store.document_store import NewBaseStash
+from ...store.linked_obj import LinkedObject
 from ...types.datetime import DateTime
 from ...types.errors import SyftException
 from ...types.uid import UID
 from ..job.job_stash import Job
 from ..job.job_stash import JobStatus
+from ..notification.email_templates import FailedJobTemplate
+from ..notification.notification_service import CreateNotification
+from ..notifier.notifier_enums import NOTIFIERS
+from ..queue.queue_service import QueueService
 from ..response import SyftError
 from ..response import SyftSuccess
 from ..worker.worker_stash import WorkerStash
@@ -218,6 +223,24 @@ def handle_message_multiprocessing(
         status = Status.COMPLETED
         job_status = JobStatus.COMPLETED
     except Exception as e:
+        root_context = AuthedServiceContext(
+            server=context.server,
+            credentials=worker.signing_key.verify_key,  # type: ignore
+        )
+        link = LinkedObject.with_context(
+            queue_item, context=root_context, service_type=QueueService
+        )
+        message = CreateNotification(
+            subject=f"Job {queue_item.job_id} failed!",
+            from_user_verify_key=worker.signing_key.verify_key,  # type: ignore
+            to_user_verify_key=credentials,
+            linked_obj=link,
+            notifier_types=[NOTIFIERS.EMAIL],
+            email_template=FailedJobTemplate,
+        )
+        method = worker.services.notification.send
+        result = method(context=root_context, notification=message)
+
         status = Status.ERRORED
         job_status = JobStatus.ERRORED
         logger.exception("Unhandled error in handle_message_multiprocessing")