Fix minor bugs, resolve TheHive observables creation issue, and add g…

…roup notifications

- Fixed a minor bug .

- Addressed an issue with the creation of observables in TheHive.

- Added support for group notifications.

Watcher/Watcher/common/mail_template/data_leak_group_template.py

@@ -67,7 +67,7 @@ def get_data_leak_group_template(keyword, alerts_number):
 
                 .word-list {
                     background: #f3f4f6;
-                    border-left: 10px solid #00267F;
+                    border-left: 4px solid #00267F;
                     padding: 15px 10px 15px 10px; 
                     margin: 20px 0;
                     border-radius: 0 4px 4px 0;

Watcher/Watcher/common/mail_template/data_leak_template.py

@@ -68,7 +68,7 @@ def get_data_leak_template(alert):
                 
                 .details {
                     background: #f3f4f6;
-                    border-left: 10px solid #00267F;
+                    border-left: 4px solid #00267F;
                     padding: 15px 10px 15px 10px; 
                     margin: 20px 0;
                     border-radius: 0 4px 4px 0;

Watcher/Watcher/common/mail_template/dns_finder_cert_transparency.py

@@ -67,7 +67,7 @@ def get_dns_finder_cert_transparency_template(alert):
                 
                 .word-list {
                     background: #f3f4f6;
-                    border-left: 10px solid #00267F;
+                    border-left: 4px solid #00267F;
                     padding: 15px 10px 15px 10px;
                     margin: 20px 0;
                     border-radius: 0 4px 4px 0;

Watcher/Watcher/common/mail_template/dns_finder_group_template.py

@@ -76,7 +76,7 @@ def get_dns_finder_group_template(dns_monitored, alerts_number):
                 
                 .details {{
                     background: #f3f4f6;
-                    border-left: 10px solid #00267F;
+                    border-left: 4px solid #00267F;
                     padding: 15px 10px 15px 10px; 
                     margin: 20px 0;
                     border-radius: 0 4px 4px 0;

Watcher/Watcher/common/mail_template/dns_finder_template.py

@@ -73,7 +73,7 @@ def get_dns_finder_template(alert):
                 
                 .details {{
                     background: #f3f4f6;
-                    border-left: 10px solid #00267F;
+                    border-left: 4px solid #00267F;
                     padding: 15px 10px 15px 10px;
                     margin: 20px 0;
                     border-radius: 0 4px 4px 0;

Watcher/Watcher/common/mail_template/site_monitoring_template.py

@@ -83,7 +83,7 @@ def get_site_monitoring_template(website_status, website_url, alert_id):
 
                 .details {
                     background: #f3f4f6;
-                    border-left: 10px solid #00267F;
+                    border-left: 4px solid #00267F;
                     padding: 15px 10px 15px 10px;
                     margin: 20px 0;
                     border-radius: 0 4px 4px 0;
@@ -131,7 +131,7 @@ def get_site_monitoring_template(website_status, website_url, alert_id):
                     <!-- Header -->
                     <td class="header" colspan="2">
                         <img src=\"""" + str(settings.WATCHER_LOGO_BASE64) + """ " alt="Watcher Logo">
-                        <h1>Website Monitoring Alert</h1>
+                        <h1>Website Monitoring</h1>
                     </td>
                 </tr>
                 <!-- Content -->

Watcher/Watcher/common/mail_template/threats_watcher_template.py

@@ -67,7 +67,7 @@ def get_threats_watcher_template(words_occurrence, email_words):
                 
                 .word-list {
                     background: #f3f4f6;
-                    border-left: 10px solid #00267F;
+                    border-left: 4px solid #00267F;
                     padding: 15px 10px 15px 10px; 
                     margin: 20px 0;
                     border-radius: 0 4px 4px 0;

Watcher/Watcher/common/utils/send_thehive_alerts.py

@@ -6,7 +6,6 @@
     create_new_alert
 )
 from site_monitoring.models import Site
-from common.core import generate_ref
 
 
 def get_ticket_id_for_domain(domain_name):
@@ -48,6 +47,7 @@ def post_to_thehive(url, data, headers, proxies):
 
 
 def send_thehive_alert(title, description, severity, tags, app_name, domain_name, observables=None, customFields=None, thehive_url=None, api_key=None):
+    from common.core import generate_ref
     """
     Send or update an alert in TheHive based on the application and ticket_id.
 
@@ -82,6 +82,9 @@ def send_thehive_alert(title, description, severity, tags, app_name, domain_name
 
     if app_name == 'website_monitoring' and not ticket_id:
         return
+
+    current_time = timezone.now().strftime("%H:%M:%S")
+    current_date = timezone.now().strftime("%d/%m/%y")
 
     # Handle the alert for 'website_monitoring' if ticket_id is found
     if app_name == 'website_monitoring':
@@ -94,7 +97,10 @@ def send_thehive_alert(title, description, severity, tags, app_name, domain_name
             app_name=app_name,
             observables=observables,
             customFields=customFields,
-            comment=f"New alert created by {app_name} on {timezone.now()}: New information reported and initial context added.",
+            comment=(
+                f"A change was processed by {app_name} application at {current_time} on {current_date}.\n"
+                "The associated observables have been handled in the dedicated section."
+            ),
             thehive_url=thehive_url,
             api_key=api_key
         )
@@ -110,7 +116,11 @@ def send_thehive_alert(title, description, severity, tags, app_name, domain_name
             app_name=app_name,
             observables=observables,
             customFields=customFields,
-            comment=f"New alert created by {app_name} on {timezone.now()}: New information reported and initial context added.",
+            comment=(
+                f"An alert was processed by {app_name} application at {current_time} on {current_date}.\n"
+                "The associated observables have been handled in the dedicated section."
+            ),
             thehive_url=thehive_url,
             api_key=api_key
         )
+

Watcher/Watcher/common/utils/update_thehive.py

@@ -1,6 +1,5 @@
 import requests
 from django.utils import timezone
-from common.core import generate_ref
 from django.conf import settings
 
 
@@ -65,7 +64,7 @@ def add_observables_to_item(item_type, item_id, observables_data, thehive_url, a
             # Send a POST request to add each observable to the item
             response = requests.post(url, headers=headers, json=observable, verify=False, proxies=proxies)
             response.raise_for_status()
-            added_observables.append(observable['data'])  # Add the observable data to the success list
+            added_observables.append(observable['data'])
         except requests.exceptions.RequestException as e:
             print(f"{timezone.now()} - Error while adding observable {observable['data']}: {e}")
 
@@ -135,11 +134,11 @@ def update_existing_alert_case(item_type, existing_item, observables, comment, t
         add_observables_to_item(item_type, item_id, observables_data, thehive_url, api_key)
 
     if comment:
-        print(f"{timezone.now()} - Adding comment to {item_type} with ID {item_id}...")
         add_comment_to_item(item_type, item_id, comment, thehive_url, api_key)
 
 
 def create_new_alert(ticket_id, title, description, severity, tags, app_name, observables, customFields, comment, thehive_url, api_key):
+    from common.core import generate_ref
     """
     Create a new alert in TheHive with the provided details.
 
@@ -183,7 +182,7 @@ def create_new_alert(ticket_id, title, description, severity, tags, app_name, ob
         response.raise_for_status()
         alert = response.json()
         alert_id = alert.get('_id')
-        print(f"{timezone.now()} - Alert created successfully with ID: {alert_id}")
+        print(f"{timezone.now()} - Alert successfully created on TheHive for {app_name}.")
 
         if observables:
             observables_data = create_observables(observables)

Watcher/Watcher/data_leak/core.py

@@ -10,9 +10,9 @@
 from django.db.models.functions import Length
 from json.decoder import JSONDecodeError
 from common.core import send_app_specific_notifications
+from common.core import send_app_specific_notifications_group
+from common.core import send_only_thehive_notifications
 from django.db.models import Q
-from django.core.mail import EmailMessage
-from common.mail_template.data_leak_group_template import get_data_leak_group_template
 
 
 def start_scheduler():
@@ -236,7 +236,7 @@ def check_keywords(keywords):
                     send_data_leak_notifications(alert)
             # if there is too many alerts, we send a group email
             if len(results) >= 6:
-                send_group_email(keyword, len(results))
+                send_data_leak_notifications_group(keyword, len(results), results)
 
     # now we check Pastebin for new pastes
     result = check_pastebin(keywords)
@@ -251,7 +251,7 @@ def check_keywords(keywords):
 
 def send_data_leak_notifications(alert):
     """
-    Sends notifications to Slack, Citadel, or TheHive based on data_leak.
+    Sends notifications to Slack, Citadel, TheHive or Email based on Data Leak.
     
     :param alert: Alert Object.
     """
@@ -270,34 +270,35 @@ def send_data_leak_notifications(alert):
     send_app_specific_notifications('data_leak', context_data, subscribers)
 
 
-def send_group_email(keyword, alerts_number):
+def send_data_leak_notifications_group(keyword, alerts_number, alerts):
     """
-    Send group e-mail for a specific keyword.
+    Sends grouped notifications to Slack, Citadel, TheHive or Email based on data_leak_group.
+    If the application is TheHive, individual notifications are sent for each alert.
 
-    :param keyword: Matched Keyword.
-    :param alerts_number: Number of alerts.
+    :param keyword: The keyword or term associated with the data leak.
+    :param alerts_number: The total number of alerts in the group.
+    :param alerts: The list of individual alerts to be processed and sent to TheHive.
     """
-    emails_to = [subscriber.user_rec.email for subscriber in Subscriber.objects.all()]
-
-    if emails_to: 
-        try:
-            # Construire l'email
-            subject = f"[{alerts_number} ALERTS] Data Leak"
-            body = get_data_leak_group_template(keyword, alerts_number)
-
-            email = EmailMessage(
-                subject=subject,
-                body=body,
-                from_email=settings.EMAIL_FROM,
-                to=emails_to,
-            )
-            email.content_subtype = "html"  
-
-            email.send(fail_silently=False)
-
-            for email_address in emails_to:
-                print(f"{timezone.now()} - Email sent to {email_address}")
-        except Exception as e:
-            print(f"{timezone.now()} - Email Error: {e}")
-    else:
-        print(f"{timezone.now()} - No subscriber, no email sent.")
+    subscribers = Subscriber.objects.filter(
+        Q(slack=True) | Q(citadel=True) | Q(thehive=True) | Q(email=True)
+    )
+
+
+    if not subscribers.exists():
+        print(f"{timezone.now()} - No subscribers for Data Leak group, no message sent.")
+        return
+
+    context_data_group = {
+        'keyword': keyword,
+        'alerts_number': alerts_number, 
+    }
+
+    send_app_specific_notifications_group('data_leak_group', context_data_group, subscribers)
+
+    for index, alert in enumerate(alerts):
+
+        context_data_thehive = {
+            'alert': alert,  
+        }
+
+        send_only_thehive_notifications('data_leak', context_data_thehive, subscribers)

Watcher/Watcher/dns_finder/core.py

@@ -10,9 +10,9 @@
 from .models import Alert, DnsMonitored, DnsTwisted, Subscriber, KeywordMonitored
 import certstream
 from common.core import send_app_specific_notifications
+from common.core import send_app_specific_notifications_group
+from common.core import send_only_thehive_notifications
 from django.db.models import Q
-from django.core.mail import EmailMessage
-from common.mail_template.dns_finder_group_template import get_dns_finder_group_template
 
 
 def start_scheduler():
@@ -145,7 +145,7 @@ def check_dnstwist(dns_monitored):
                 for alert in alerts_list:
                     send_dns_finder_notifications(alert)
             if len(alerts_list) >= 6:
-                send_group_email(dns_monitored, len(alerts_list))
+                send_dns_finder_notifications_group(dns_monitored, len(alerts_list), alerts_list)
         except ValueError:
             print('Decoding JSON has failed')
 
@@ -154,7 +154,7 @@ def check_dnstwist(dns_monitored):
 
 def send_dns_finder_notifications(alert):
     """
-    Sends notifications to Slack, Citadel, or TheHive based on dns_finder.
+    Sends notifications to Slack, Citadel, TheHive or Email based on DNS Finder.
     
     :param alert: Alert Object.
     """
@@ -182,37 +182,34 @@ def send_dns_finder_notifications(alert):
     send_app_specific_notifications('dns_finder', context_data, subscribers)
 
 
-def send_group_email(dns_monitored, alerts_number):
+def send_dns_finder_notifications_group(dns_monitored, alerts_number, alerts):
     """
-    Send group e-mail for a specific dns_monitored.
+    Sends grouped notifications to Slack, Citadel, TheHive or Email based on dns_finder_group.
+    If the application is TheHive, individual notifications are sent for each alert.
 
-    :param dns_monitored: DnsMonitored Object.
-    :param alerts_number: Number of alerts.
+    :param keyword: The keyword or term associated with the dns finder.
+    :param alerts_number: The total number of alerts in the group.
+    :param alerts: The list of individual alerts to be processed and sent to TheHive.
     """
-    # Collecter les emails des abonnés
-    emails_to = [subscriber.user_rec.email for subscriber in Subscriber.objects.all()]
+    subscribers = Subscriber.objects.filter(
+        Q(slack=True) | Q(citadel=True) | Q(thehive=True) | Q(email=True)
+    )
 
-    if emails_to:  # S'il y a au moins un abonné
-        try:
-            # Construire l'email
-            subject = f"[{alerts_number} ALERTS] DNS Finder"
-            body = get_dns_finder_group_template(dns_monitored, alerts_number)
-
-            email = EmailMessage(
-                subject=subject,
-                body=body,
-                from_email=settings.EMAIL_FROM,
-                to=emails_to,
-            )
-            email.content_subtype = "html"  
-
-            # Envoyer l'email
-            email.send(fail_silently=False)
-
-            # Confirmation des emails envoyés
-            for email_address in emails_to:
-                print(f"{timezone.now()} - Email sent to {email_address}")
-        except Exception as e:
-            print(f"{timezone.now()} - Email Error: {e}")
-    else:
-        print(f"{timezone.now()} - No subscriber, no email sent.")
+    if not subscribers.exists():
+        print(f"{timezone.now()} - No subscribers for DNS Finder group, no message sent.")
+        return
+
+    context_data_group = {
+        'dns_monitored': dns_monitored,
+        'alerts_number': alerts_number,
+    }
+
+    send_app_specific_notifications_group('dns_finder_group', context_data_group, subscribers)
+
+    for alert in alerts:
+
+        context_data_thehive = {
+            'alert': alert,  
+        }
+
+        send_only_thehive_notifications('dns_finder', context_data_thehive, subscribers)