From 77c9d14b3c99699e0f14bc2abf035a457066688a Mon Sep 17 00:00:00 2001 From: ygal Date: Wed, 4 Dec 2024 18:35:16 +0100 Subject: [PATCH] Creation of the common module and notification system Created a new Django App to centralize the generic functions of Watcher used by multiple modules. Refactored the notification system to integrate SMTPS protocol (replacing SMTP). Added automatic alert creation in TheHive via Feeder. Integrated notifications through the Citadel enterprise application (via APIs). Added Slack notifications (via APIs). --- Watcher/Watcher/common/__init__.py | 0 Watcher/Watcher/common/admin.py | 3 + Watcher/Watcher/common/apps.py | 6 + Watcher/Watcher/common/core.py | 508 ++++++++++++++++++ .../data_leak_group_template.py} | 2 +- .../mail_template/data_leak_template.py | 160 ++++++ .../dns_finder_cert_transparency.py} | 2 +- .../dns_finder_group_template.py} | 2 +- .../mail_template/dns_finder_template.py | 166 ++++++ .../mail_template/site_monitoring_template.py | 176 ++++++ .../mail_template/threats_watcher_template.py | 165 ++++++ Watcher/Watcher/common/migrations/__init__.py | 0 Watcher/Watcher/common/models.py | 3 + Watcher/Watcher/common/tests.py | 3 + .../common/utils/send_citadel_messages.py | 36 ++ .../common/utils/send_email_notifications.py | 36 ++ .../common/utils/send_slack_messages.py | 35 ++ .../common/utils/send_thehive_alerts.py | 111 ++++ .../Watcher/common/utils/update_thehive.py | 238 ++++++++ Watcher/Watcher/common/views.py | 3 + Watcher/Watcher/data_leak/admin.py | 15 +- Watcher/Watcher/data_leak/core.py | 96 +--- .../mail_template/default_template.py | 194 ------- ...ber_options_subscriber_citadel_and_more.py | 37 ++ Watcher/Watcher/data_leak/models.py | 13 +- Watcher/Watcher/dns_finder/admin.py | 15 +- Watcher/Watcher/dns_finder/api.py | 10 +- Watcher/Watcher/dns_finder/core.py | 134 +---- .../mail_template/default_template.py | 194 ------- .../0008_alter_subscriber_options_and_more.py | 41 ++ Watcher/Watcher/dns_finder/models.py | 14 +- Watcher/Watcher/dns_finder/serializers.py | 92 +--- Watcher/Watcher/dns_finder/urls.py | 3 +- .../Watcher/frontend/src/actions/DnsFinder.js | 17 - .../frontend/src/actions/SiteMonitoring.js | 17 - .../src/components/DnsFinder/Alerts.js | 46 +- .../SiteMonitoring/SuspiciousSites.js | 46 +- .../frontend/src/reducers/DnsFinder.js | 11 - .../frontend/src/reducers/SiteMonitoring.js | 11 - .../Watcher/frontend/static/frontend/main.js | 2 +- .../Watcher/frontend/static/img/misp_logo.png | Bin 0 -> 25095 bytes .../frontend/static/img/thehive_misp_logo.png | Bin 70998 -> 0 bytes Watcher/Watcher/site_monitoring/admin.py | 21 +- Watcher/Watcher/site_monitoring/api.py | 10 +- Watcher/Watcher/site_monitoring/core.py | 149 ++--- .../0029_alter_subscriber_options_and_more.py | 41 ++ Watcher/Watcher/site_monitoring/models.py | 16 +- .../Watcher/site_monitoring/serializers.py | 78 +-- Watcher/Watcher/site_monitoring/thehive.py | 189 ------- Watcher/Watcher/site_monitoring/urls.py | 3 +- Watcher/Watcher/threats_watcher/admin.py | 20 +- Watcher/Watcher/threats_watcher/core.py | 55 +- .../mail_template/default_template.py | 185 ------- ...scriber_email_subscriber_slack_and_more.py | 41 ++ Watcher/Watcher/threats_watcher/models.py | 12 +- Watcher/Watcher/watcher/settings.py | 30 +- 56 files changed, 2101 insertions(+), 1412 deletions(-) create mode 100644 Watcher/Watcher/common/__init__.py create mode 100644 Watcher/Watcher/common/admin.py create mode 100644 Watcher/Watcher/common/apps.py create mode 100644 Watcher/Watcher/common/core.py rename Watcher/Watcher/{data_leak/mail_template/group_template.py => common/mail_template/data_leak_group_template.py} (99%) create mode 100644 Watcher/Watcher/common/mail_template/data_leak_template.py rename Watcher/Watcher/{dns_finder/mail_template/default_template_cert_transparency.py => common/mail_template/dns_finder_cert_transparency.py} (99%) rename Watcher/Watcher/{dns_finder/mail_template/group_template.py => common/mail_template/dns_finder_group_template.py} (99%) create mode 100644 Watcher/Watcher/common/mail_template/dns_finder_template.py create mode 100644 Watcher/Watcher/common/mail_template/site_monitoring_template.py create mode 100644 Watcher/Watcher/common/mail_template/threats_watcher_template.py create mode 100644 Watcher/Watcher/common/migrations/__init__.py create mode 100644 Watcher/Watcher/common/models.py create mode 100644 Watcher/Watcher/common/tests.py create mode 100644 Watcher/Watcher/common/utils/send_citadel_messages.py create mode 100644 Watcher/Watcher/common/utils/send_email_notifications.py create mode 100644 Watcher/Watcher/common/utils/send_slack_messages.py create mode 100644 Watcher/Watcher/common/utils/send_thehive_alerts.py create mode 100644 Watcher/Watcher/common/utils/update_thehive.py create mode 100644 Watcher/Watcher/common/views.py delete mode 100644 Watcher/Watcher/data_leak/mail_template/default_template.py create mode 100644 Watcher/Watcher/data_leak/migrations/0011_alter_subscriber_options_subscriber_citadel_and_more.py delete mode 100644 Watcher/Watcher/dns_finder/mail_template/default_template.py create mode 100644 Watcher/Watcher/dns_finder/migrations/0008_alter_subscriber_options_and_more.py create mode 100644 Watcher/Watcher/frontend/static/img/misp_logo.png delete mode 100644 Watcher/Watcher/frontend/static/img/thehive_misp_logo.png create mode 100644 Watcher/Watcher/site_monitoring/migrations/0029_alter_subscriber_options_and_more.py delete mode 100644 Watcher/Watcher/site_monitoring/thehive.py delete mode 100644 Watcher/Watcher/threats_watcher/mail_template/default_template.py create mode 100644 Watcher/Watcher/threats_watcher/migrations/0015_subscriber_citadel_subscriber_email_subscriber_slack_and_more.py diff --git a/Watcher/Watcher/common/__init__.py b/Watcher/Watcher/common/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/Watcher/Watcher/common/admin.py b/Watcher/Watcher/common/admin.py new file mode 100644 index 0000000..8c38f3f --- /dev/null +++ b/Watcher/Watcher/common/admin.py @@ -0,0 +1,3 @@ +from django.contrib import admin + +# Register your models here. diff --git a/Watcher/Watcher/common/apps.py b/Watcher/Watcher/common/apps.py new file mode 100644 index 0000000..01cca2f --- /dev/null +++ b/Watcher/Watcher/common/apps.py @@ -0,0 +1,6 @@ +from django.apps import AppConfig + + +class CommonConfig(AppConfig): + default_auto_field = 'django.db.models.BigAutoField' + name = 'common' diff --git a/Watcher/Watcher/common/core.py b/Watcher/Watcher/common/core.py new file mode 100644 index 0000000..095eed4 --- /dev/null +++ b/Watcher/Watcher/common/core.py @@ -0,0 +1,508 @@ +from .utils.send_slack_messages import send_slack_message +from .utils.send_citadel_messages import send_citadel_message +from .utils.send_email_notifications import send_email_notifications +from django.utils import timezone +from django.conf import settings +import re +from html import unescape +from site_monitoring.models import Site +from .mail_template.threats_watcher_template import get_threats_watcher_template +from .mail_template.data_leak_template import get_data_leak_template +from .mail_template.data_leak_group_template import get_data_leak_group_template +from .mail_template.dns_finder_template import get_dns_finder_template +from .mail_template.dns_finder_group_template import get_dns_finder_group_template +from .mail_template.site_monitoring_template import get_site_monitoring_template + +thehive_url = settings.THE_HIVE_URL +api_key = settings.THEHIVE_API_KEY + +from datetime import datetime +from secrets import token_hex + +def generate_ref(): + """ + Generates a unique 'sourceRef' identifier for an alert. + + :return: Unique identifier as a string. + """ + ref = datetime.now().strftime("%y%m%d") + "-" + str(token_hex(3))[:5] + return ref + + +# Configuration for Slack +APP_CONFIG_SLACK = { + 'threats_watcher': { + 'content_template': ( + "*[THREATS WATCHER - WARNING] 🔥 Trendy Threats Detected 🔥*\n\n" + "Dear team,\n\n" + "Please find the new trendy word(s) detected below:\n\n" + "{words_list}\n\n" + "Please, find more details <{details_url}|here>." + ), + 'channel': settings.SLACK_CHANNEL, + 'url_suffix': '#/', + }, + 'data_leak': { + 'content_template': ( + "*[DATA LEAK - ALERT #{alert_pk}] 🔔 Data Leak Detected: {keyword_name} 🔔*\n\n" + "Dear team,\n\n" + "New Data Leakage Alert for {keyword_name} keyword:\n\n" + "*• Keyword:* {keyword_name}\n" + "*• Source:* {url}\n\n" + "Please, find more details <{details_url}|here>." + ), + 'channel': settings.SLACK_CHANNEL, + 'url_suffix': '#data_leak', + }, + 'website_monitoring': { + 'content_template': ( + "*[SITE MONITORING - INCIDENT] 🔔 {alert_type} on {domain_name} 🔔*\n\n" + "Dear team,\n\n" + "Please find the new incident detected below:\n\n" + "*• Type of alert:* {alert_type}\n" + "*• Domain name:* {domain_name}\n" + "*• Difference Score:* {difference_score}\n" + "*• New Ip:* {new_ip}\n" + "*• Old Ip:* {old_ip}\n" + "*• New Ip Second:* {new_ip_second}\n" + "*• Old IP Second:* {old_ip_second}\n" + "*• New MX Records:* {new_mx_records}\n" + "*• Old MX Records:* {old_mx_records}\n" + "*• New Mail Server:* {new_mail_A_record_ip}\n" + "*• Old Mail Server:* {old_mail_A_record_ip}\n\n" + "Please, find more details <{details_url}|here>." + ), + 'channel': settings.SLACK_CHANNEL, + 'url_suffix': '#/website_monitoring/', + }, + 'dns_finder': { + 'content_template': ( + "*[DNS FINDER - ALERT #{alert.pk}] 🚨 Suspicious DNS Detected: {alert.dns_twisted.domain_name} 🚨*\n\n" + "Dear team,\n\n" + "New Twisted DNS found: \n\n" + "*• Twisted DNS:* {alert.dns_twisted.domain_name}\n" + "*• Corporate Keyword:* {alert.dns_twisted.keyword_monitored}\n" + "*• Corporate DNS:* {alert.dns_twisted.dns_monitored}\n" + "*• Fuzzer:* {alert.dns_twisted.fuzzer}\n\n" + "Please, find more details <{details_url}|here>." + ), + 'channel': settings.SLACK_CHANNEL, + 'url_suffix': '#/dns_finder/', + }, +} + +# Configuration for Citadel +APP_CONFIG_CITADEL = { + 'threats_watcher': { + 'content_template': ( + "

[THREATS WATCHER - WARNING] 🔥 Trendy Threats Detected 🔥

" + "

Dear team,

" + "

Please find the new trendy word(s) detected below:\n

" + "\n" + "

Please, find more details here.

" + ), + 'citadel_room_id': settings.CITADEL_ROOM_ID, + 'url_suffix': '#/', + }, + 'data_leak': { + 'content_template': ( + "

[DATA LEAK - ALERT #{alert_pk}] 🔔 Data Leak Detected: {keyword_name} 🔔

" + "

Dear team,

" + "

New Data Leakage Alert for {keyword_name} keyword:

\n" + "" + "

Please, find more details here.

" + ), + 'citadel_room_id': settings.CITADEL_ROOM_ID, + 'url_suffix': '#data_leak', + }, + 'website_monitoring': { + 'content_template': ( + "

[SITE MONITORING - INCIDENT] 🔔 {alert_type} on {domain_name} 🔔

" + "

Dear team,

" + "

Please find the new incident detected below:

\n" + "" + "

Please, find more details here.

" + ), + 'citadel_room_id': settings.CITADEL_ROOM_ID, + 'url_suffix': '#/website_monitoring/', + }, + 'dns_finder': { + 'content_template': ( + "

[DNS FINDER - ALERT #{alert.pk}] 🚨 Suspicious DNS Detected: {alert.dns_twisted.domain_name} 🚨

" + "

Dear team,

" + "

New Twisted DNS found:

\n" + "" + "

Please, find more details here.

" + ), + 'citadel_room_id': settings.CITADEL_ROOM_ID, + 'url_suffix': '#/dns_finder/', + }, +} + +# Configuration for TheHive +APP_CONFIG_THEHIVE = { + 'threats_watcher': { + 'title': "Threats Watcher buzzword detected", + 'description_template': ( + "**Alert:**\n" + "**The following trendy words were detected:**\n" + "{words_list}" + ), + 'severity': 2, + 'tags': ["Threats Watcher", "Watcher", "Buzzword", "Trendy Words", "Threat Detection"] + }, + 'data_leak': { + 'title': "New Data Leakage - Alert #{alert_pk} for {keyword_name} keyword", + 'description_template': ( + "**Alert:**\n" + "**New data leak detected:**\n" + "*Keyword:* {keyword_name}\n" + "*Source:* {url}\n" + ), + 'severity': 3, + 'tags': ["Data Leak", "Watcher", "Sensitive Data", "Leak Detection"] + }, + 'website_monitoring': { + 'title': "Website Monitoring Detected - {alert_type} on {domain_name}", + 'description_template': ( + "**Alert:**\n" + "**New website monitoring incident detected:**\n" + "*Type of alert:* {alert_type}\n" + "*Domain name:* {domain_name}\n" + "*• Difference Score:* {difference_score}\n" + "*• New Ip:* {new_ip}\n" + "*• Old Ip:* {old_ip}\n" + "*• New Ip Second:* {new_ip_second}\n" + "*• Old IP Second:* {old_ip_second}\n" + "*• New MX Records:* {new_mx_records}\n" + "*• Old MX Records:* {old_mx_records}\n" + "*• New Mail Server:* {new_mail_A_record_ip}\n" + "*• Old Mail Server:* {old_mail_A_record_ip}\n" + ), + 'severity': 2, + 'tags': ["Website Monitoring", "Watcher", "Incident", "Website", "Domain Name"] + }, + 'dns_finder': { + 'title': "New Twisted DNS found - {alert.dns_twisted.domain_name}", + 'description_template': ( + "**Alert:**\n" + "**New Twisted DNS found:**\n" + "*Corporate Keyword:* {alert.dns_twisted.domain_name}\n" + "*Twisted DNS:* {alert.dns_twisted.keyword_monitored}\n" + "*Corporate DNS:* {alert.dns_twisted.dns_monitored}\n" + "*Fuzzer:* {alert.dns_twisted.fuzzer}\n" + ), + 'severity': 3, + 'tags': ["DNS Finder", "Watcher", "Twisted DNS", "Corporate Keywords", "Corporate DNS Assets"] + }, +} + + +# Configuration for Email +APP_CONFIG_EMAIL = { + 'threats_watcher': { + 'subject': "[WARNING] Threats Watcher buzzword detected", + 'template_func': get_threats_watcher_template, + }, + 'data_leak': { + 'subject': "[ALERT] Data Leak Detected", + 'template_func': get_data_leak_template, + }, + 'data_leak_group': { + 'subject': "[ALERT] Group Data Leak Detected", + 'template_func': get_data_leak_group_template, + }, + 'website_monitoring': { + 'subject': "[ALERT] Website Monitoring Detected", + 'template_func': get_site_monitoring_template, + }, + 'dns_finder': { + 'subject': "[ALERT] DNS Finder", + 'template_func': get_dns_finder_template, + }, + 'dns_finder_group': { + 'subject': "[ALERT] Group DNS Finder", + 'template_func': get_dns_finder_group_template, + }, +} + + + +def collect_observables(app_name, context_data): + """ + Collect observables specific to each app based on the context data provided. + """ + observables = [] + + if app_name == 'threats_watcher': + for word in context_data.get('email_words', []): + if re.match(r'^[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$', word): + observables.append({"dataType": "domain", "data": word}) + else: + observables.append({"dataType": "other", "data": word}) + + elif app_name == 'website_monitoring': + site = context_data.get('site') + alert_data = context_data.get('alert_data', {}) + if site: + observables.append({"dataType": "domain", "data": site.domain_name}) + if alert_data.get('new_ip'): + observables.append({"dataType": "ip", "data": alert_data['new_ip']}) + if alert_data.get('old_ip'): + observables.append({"dataType": "ip", "data": alert_data['old_ip']}) + if alert_data.get('new_ip_second'): + observables.append({"dataType": "ip", "data": alert_data['new_ip_second']}) + if alert_data.get('old_ip_second'): + observables.append({"dataType": "ip", "data": alert_data['old_ip_second']}) + if alert_data.get('new_mx_records'): + for mx_record in alert_data['new_mx_records']: + observables.append({"dataType": "domain", "data": mx_record}) + if alert_data.get('old_mx_records'): + for mx_record in alert_data['old_mx_records']: + observables.append({"dataType": "domain", "data": mx_record}) + if alert_data.get('new_mail_A_record_ip'): + observables.append({"dataType": "ip", "data": alert_data['new_mail_A_record_ip']}) + if alert_data.get('old_mail_A_record_ip'): + observables.append({"dataType": "ip", "data": alert_data['old_mail_A_record_ip']}) + + elif app_name == 'data_leak': + alert = context_data.get('alert') + if alert: + observables.append({"dataType": "url", "data": alert.url}) + observables.append({"dataType": "other", "data": alert.keyword.name}) + + elif app_name == 'dns_finder': + alert = context_data.get('alert') + if alert: + observables.append({"dataType": "domain", "data": alert.dns_twisted.domain_name}) + if alert.dns_twisted.keyword_monitored: + observables.append({"dataType": "other", "data": alert.dns_twisted.keyword_monitored.name}) + if alert.dns_twisted.dns_monitored: + observables.append({"dataType": "domain", "data": alert.dns_twisted.dns_monitored.domain_name}) + if alert.dns_twisted.fuzzer: + observables.append({"dataType": "other", "data": alert.dns_twisted.fuzzer}) + + return observables + +def remove_html_tags(text): + """Remove HTML tags from a text.""" + clean = re.compile('<.*?>') + return re.sub(clean, '', text) + +def send_app_specific_notifications(app_name, context_data, subscribers): + from .utils.send_thehive_alerts import send_thehive_alert + + """ + Send notifications based on the application type (Slack, Citadel, TheHive cases & alerts, Email). + Collect observables, format the notification content, and send them to respective channels. + """ + app_config_slack = APP_CONFIG_SLACK.get(app_name) + app_config_citadel = APP_CONFIG_CITADEL.get(app_name) + app_config_thehive = APP_CONFIG_THEHIVE.get(app_name) + app_config_email = APP_CONFIG_EMAIL.get(app_name) + + if not app_config_slack or not app_config_citadel or not app_config_thehive or not app_config_email: + return + + + if not subscribers.exists(): + return + + custom_field_key = settings.THE_HIVE_CUSTOM_FIELD + + observables = collect_observables(app_name, context_data) + + def send_notification(channel, content_template, subscribers_filter, send_func, **kwargs): + """Helper to format and send notification based on the channel.""" + if subscribers.filter(**subscribers_filter).exists(): + content = content_template.format(**kwargs) + send_func(content) + + try: + common_data = {} + if app_name == 'threats_watcher': + words_list = '\n'.join([remove_html_tags(unescape(word)) for word in context_data.get('email_words', [])]) + if not words_list: + return + common_data = { + 'words_list': words_list, + 'details_url': settings.WATCHER_URL + app_config_slack['url_suffix'], + 'app_name': 'threats_watcher' + } + + email_words = context_data.get('email_words', []) + email_body = get_threats_watcher_template(settings.WORDS_OCCURRENCE, email_words) + + + elif app_name == 'website_monitoring': + site = context_data.get('site') + alert_data = context_data.get('alert_data', {}) + if not site: + return + common_data = { + 'alert_type': alert_data.get('type', 'None'), + 'domain_name': site.domain_name, + 'difference_score': alert_data.get('difference_score', 'None'), + 'new_ip': alert_data.get('new_ip', 'None'), + 'old_ip': alert_data.get('old_ip', 'None'), + 'new_ip_second': alert_data.get('new_ip_second', 'None'), + 'old_ip_second': alert_data.get('old_ip_second', 'None'), + 'new_mx_records': alert_data.get('new_MX_records', 'None'), + 'old_mx_records': alert_data.get('old_MX_records', 'None'), + 'new_mail_A_record_ip': alert_data.get('new_mail_A_record_ip', 'None'), + 'old_mail_A_record_ip': alert_data.get('old_mail_A_record_ip', 'None'), + 'web_status': alert_data.get('web_status', 'N/A'), + 'timestamp': timezone.now().isoformat(), + 'details_url': settings.WATCHER_URL + app_config_slack['url_suffix'], + 'app_name': 'website_monitoring' + } + + email_words = context_data.get('alert', []) + website_url = site.domain_name + alert_id = alert_data.get('id', '-') + email_body = get_site_monitoring_template(website_url, alert_id, alert_data) + print(f"alert_data: {alert_data}") + + + elif app_name == 'data_leak': + alert = context_data.get('alert') + if not alert: + return + common_data = { + 'alert_pk': alert.pk, + 'keyword_name': alert.keyword.name, + 'url': alert.url, + 'content': alert.content[:200], + 'timestamp': alert.created_at.isoformat(), + 'details_url': settings.WATCHER_URL + app_config_slack['url_suffix'], + 'app_name': 'data_leak' + } + email_words = context_data.get('alert', []) + email_body = get_data_leak_template(alert) + + + elif app_name == 'dns_finder': + alert = context_data.get('alert') + if not alert: + print("Error: Alert object is None in context_data") + return + if not alert.dns_twisted: + print(f"Error: dns_twisted is missing in alert {alert.pk if alert.pk else 'Unknown'}") + return + if not alert.dns_twisted.domain_name: + print(f"Error: domain_name is missing in dns_twisted for alert {alert.pk if alert.pk else 'Unknown'}") + return + common_data = { + 'alert': alert, + 'details_url': settings.WATCHER_URL + app_config_slack['url_suffix'], + 'app_name': 'dns_finder' + } + email_words = context_data.get('alert', []) + email_body = get_dns_finder_template(alert) + + send_notification( + channel="slack", + content_template=app_config_slack['content_template'], + subscribers_filter={'slack': True}, + send_func=lambda content: send_slack_message(content, app_config_slack['channel'], app_name), + **common_data + ) + + citadel_title = app_config_citadel.get('title', f"[{app_name.upper()}] Incident Alert") + send_notification( + channel="citadel", + content_template=app_config_citadel['content_template'], + subscribers_filter={'citadel': True}, + send_func=lambda content: send_citadel_message( + content={ + "msgtype": "m.text", + "format": "org.matrix.custom.html", + "body": citadel_title + " - New Incident Alert", + "formatted_body": content.replace('\n', '
') + }, + room_id=app_config_citadel['citadel_room_id'], + app_name=common_data.get('app_name') + ), + title=citadel_title, + **common_data + ) + + if app_config_thehive: + formatted_title = app_config_thehive['title'].format(**common_data) + + domain_name = common_data.get('domain_name') + if domain_name: + try: + site = Site.objects.get(domain_name=domain_name) + except Site.DoesNotExist: + print(f"{timezone.now()} - Site with domain_name {domain_name} not found.") + return + + ticket_id = site.ticket_id + + send_notification( + channel="thehive", + content_template=app_config_thehive['description_template'], + subscribers_filter={'thehive': True}, + send_func=lambda content: send_thehive_alert( + title=formatted_title, + description=content, + severity=app_config_thehive['severity'], + tags=app_config_thehive['tags'], + customFields=app_config_thehive.get('customFields'), + app_name=app_name, + domain_name=site.domain_name, + observables=observables + ), + **common_data + ) + else: + if subscribers.filter(thehive=True).exists(): + send_thehive_alert( + title=formatted_title, + description=app_config_thehive['description_template'].format(**common_data), + severity=app_config_thehive['severity'], + tags=app_config_thehive['tags'], + app_name=app_name, + domain_name=None, + observables=observables, + customFields=app_config_thehive.get('customFields'), + thehive_url=thehive_url, + api_key=api_key + ) + + if app_config_email: + email_list = [subscriber.user_rec.email for subscriber in subscribers.filter(email=True)] + + if email_list: + email_subject = app_config_email['subject'].format(**common_data) + send_email_notifications(email_subject, email_body, email_list, app_name) + else: + pass + + if app_config_email: + # Récupération des abonnés ayant une adresse email + email_list = [subscriber.user_rec.email for subscriber in subscribers.filter(email=True)] + + except Exception as e: + print(f"Error sending notifications for {app_name}: {str(e)}") \ No newline at end of file diff --git a/Watcher/Watcher/data_leak/mail_template/group_template.py b/Watcher/Watcher/common/mail_template/data_leak_group_template.py similarity index 99% rename from Watcher/Watcher/data_leak/mail_template/group_template.py rename to Watcher/Watcher/common/mail_template/data_leak_group_template.py index 9fea5fa..ec8b669 100644 --- a/Watcher/Watcher/data_leak/mail_template/group_template.py +++ b/Watcher/Watcher/common/mail_template/data_leak_group_template.py @@ -1,7 +1,7 @@ from django.conf import settings -def get_group_template(keyword, alerts_number): +def get_data_leak_group_template(keyword, alerts_number): body = """\ diff --git a/Watcher/Watcher/common/mail_template/data_leak_template.py b/Watcher/Watcher/common/mail_template/data_leak_template.py new file mode 100644 index 0000000..f4ed007 --- /dev/null +++ b/Watcher/Watcher/common/mail_template/data_leak_template.py @@ -0,0 +1,160 @@ +from django.conf import settings + + +def get_data_leak_template(alert): + github_repo = "https://github.com/thalesgroup-cert/Watcher" + body = """\ + + + + + + +
+ +
+ Watcher Logo +

Data Leak Alert #""" + str(alert.pk) + """

+
+ + +
+

Dear team,

+ +

A new data leakage alert has been detected: for the keyword:

+ +
+

Keyword: """ + str(alert.keyword) + """

+

Source: """ + str(alert.url) + """

+
+ +

You can check more details here.

+ +

Kind Regards,
+ Watcher

+
+ + +
+ + GitHub + View Watcher on GitHub + +
+
+ +

[""" + str(settings.EMAIL_CLASSIFICATION) + """]

+ + + """ + return body diff --git a/Watcher/Watcher/dns_finder/mail_template/default_template_cert_transparency.py b/Watcher/Watcher/common/mail_template/dns_finder_cert_transparency.py similarity index 99% rename from Watcher/Watcher/dns_finder/mail_template/default_template_cert_transparency.py rename to Watcher/Watcher/common/mail_template/dns_finder_cert_transparency.py index e60cd56..10802bc 100644 --- a/Watcher/Watcher/dns_finder/mail_template/default_template_cert_transparency.py +++ b/Watcher/Watcher/common/mail_template/dns_finder_cert_transparency.py @@ -1,7 +1,7 @@ from django.conf import settings -def get_cert_transparency_template(alert): +def get_dns_finder_cert_transparency_template(alert): body = """\ diff --git a/Watcher/Watcher/dns_finder/mail_template/group_template.py b/Watcher/Watcher/common/mail_template/dns_finder_group_template.py similarity index 99% rename from Watcher/Watcher/dns_finder/mail_template/group_template.py rename to Watcher/Watcher/common/mail_template/dns_finder_group_template.py index 0840c83..9cf7055 100644 --- a/Watcher/Watcher/dns_finder/mail_template/group_template.py +++ b/Watcher/Watcher/common/mail_template/dns_finder_group_template.py @@ -1,7 +1,7 @@ from django.conf import settings -def get_group_template(dns_monitored, alerts_number): +def get_dns_finder_group_template(dns_monitored, alerts_number): body = """\ diff --git a/Watcher/Watcher/common/mail_template/dns_finder_template.py b/Watcher/Watcher/common/mail_template/dns_finder_template.py new file mode 100644 index 0000000..70aff93 --- /dev/null +++ b/Watcher/Watcher/common/mail_template/dns_finder_template.py @@ -0,0 +1,166 @@ +from django.conf import settings + +def get_dns_finder_template(alert): + """ + Génère un email HTML pour une alerte DNS Finder. + + :param alert: Objet contenant les informations sur l'alerte. + :return: Contenu HTML de l'email. + """ + body = f"""\ + + + + + + +
+ +
+ Watcher Logo +

DNS Finder: Alert #{alert.pk}

+
+ + +
+

Dear team,

+

A new Twisted DNS record has been detected:

+ +
+

Domain Name: {alert.dns_twisted.domain_name}

+

Asset Monitored: {alert.dns_twisted.dns_monitored}

+
+ +

You can check more details here.

+ +

Kind Regards,

+

Watcher

+
+ + +
+ + GitHub + View Watcher on GitHub + +
+
+ +

[{settings.EMAIL_CLASSIFICATION}]

+ + + """ + return body diff --git a/Watcher/Watcher/common/mail_template/site_monitoring_template.py b/Watcher/Watcher/common/mail_template/site_monitoring_template.py new file mode 100644 index 0000000..076f12c --- /dev/null +++ b/Watcher/Watcher/common/mail_template/site_monitoring_template.py @@ -0,0 +1,176 @@ +from django.conf import settings + + +def get_site_monitoring_template(website_status, website_url, alert_id): + """ + Génère le corps du mail HTML pour le monitoring de site avec les informations spécifiées. + + :param website_status: Le statut du site (OK, DOWN, etc.) + :param website_url: L'URL du site surveillé + :param alert_id: L'ID de l'alerte + :param ticket_id: L'ID du ticket associé + :return: Le corps du mail en HTML + """ + github_repo = "https://github.com/thalesgroup-cert/Watcher" + + alert_details = f""" +

Domain name: {website_status}

+

Type: {alert_id['type']}

+

New IP: {alert_id['new_ip']}, {alert_id['new_ip_second']}

+

Old IP: {alert_id['old_ip']}, {alert_id['old_ip_second']}

+ """ + + body = f"""\ + + + + + + +
+ +
+ Watcher Logo +

Website Monitoring Alert

+
+ + +
+

Dear team,

+ +

The current status of the monitored website is as follows:

+ +
+ {alert_details} +
+ +

You can check more details here.

+ +

Kind Regards,
+ Watcher

+
+ + +
+ + GitHub + View Watcher on GitHub + +
+
+ +

[{settings.EMAIL_CLASSIFICATION}]

+ + + """ + return body diff --git a/Watcher/Watcher/common/mail_template/threats_watcher_template.py b/Watcher/Watcher/common/mail_template/threats_watcher_template.py new file mode 100644 index 0000000..49ef666 --- /dev/null +++ b/Watcher/Watcher/common/mail_template/threats_watcher_template.py @@ -0,0 +1,165 @@ +from django.conf import settings + + +def get_threats_watcher_template(words_occurrence, email_words): + github_repo = "https://github.com/thalesgroup-cert/Watcher" + body = """\ + + + + + + +
+ +
+ Threats Watcher Logo +

Threats Watcher

+
+ + +
+

Dear team,

+ +

Please find below trendy word(s) that match at least """ + str(words_occurrence) + """ times:

+ +
+ """ + "

".join(email_words) + """ +

+ +

You can check more details here.

+ +

Kind Regards,
+ Watcher

+
+ + +
+ + + GitHub + View Watcher on GitHub + +
+
+ +

[""" + str(settings.EMAIL_CLASSIFICATION) + """]

+ + + """ + return body \ No newline at end of file diff --git a/Watcher/Watcher/common/migrations/__init__.py b/Watcher/Watcher/common/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/Watcher/Watcher/common/models.py b/Watcher/Watcher/common/models.py new file mode 100644 index 0000000..71a8362 --- /dev/null +++ b/Watcher/Watcher/common/models.py @@ -0,0 +1,3 @@ +from django.db import models + +# Create your models here. diff --git a/Watcher/Watcher/common/tests.py b/Watcher/Watcher/common/tests.py new file mode 100644 index 0000000..7ce503c --- /dev/null +++ b/Watcher/Watcher/common/tests.py @@ -0,0 +1,3 @@ +from django.test import TestCase + +# Create your tests here. diff --git a/Watcher/Watcher/common/utils/send_citadel_messages.py b/Watcher/Watcher/common/utils/send_citadel_messages.py new file mode 100644 index 0000000..5f1c5f6 --- /dev/null +++ b/Watcher/Watcher/common/utils/send_citadel_messages.py @@ -0,0 +1,36 @@ +import requests +from django.conf import settings +from django.utils import timezone + +def send_citadel_message(content, room_id, app_name): + """ + Sends a message to the specified Citadel room. + + Args: + content (dict): The content of the message (must contain 'msgtype' and 'body'). + room_id (str): The ID of the Citadel room to send the message to. + """ + + if not settings.CITADEL_API_TOKEN or not settings.CITADEL_ROOM_ID: + print(f"{str(timezone.now())} - No configuration for Citadel, not notifications sending.") + return + + url = f"{settings.CITADEL_URL}/_matrix/client/r0/rooms/{room_id}/send/m.room.message?access_token={settings.CITADEL_API_TOKEN}" + + headers = { + 'Content-Type': 'application/json', + } + + payload = { + 'msgtype': content.get('msgtype', 'm.text'), + 'body': content.get('body', ''), + 'format': content.get('format', None), + 'formatted_body': content.get('formatted_body', None) + } + + response = requests.post(url, headers=headers, json=payload) + + if response.status_code == 200: + print(f"{str(timezone.now())} - Message sent to Citadel successfully for {app_name}.") + else: + print(f"{str(timezone.now())} - Failed to send message to Citadel: {response.status_code} - {response.text}") \ No newline at end of file diff --git a/Watcher/Watcher/common/utils/send_email_notifications.py b/Watcher/Watcher/common/utils/send_email_notifications.py new file mode 100644 index 0000000..b83e937 --- /dev/null +++ b/Watcher/Watcher/common/utils/send_email_notifications.py @@ -0,0 +1,36 @@ +from email.mime.multipart import MIMEMultipart +from email.mime.text import MIMEText +import smtplib +from django.conf import settings +from datetime import datetime + + +def send_email_notifications(subject, body, emails_to, app_name): + """ + Send e-mail notifications to the list of recipients. + + Args: + subject (str): Subject of the email. + body (str): HTML body content of the email. + emails_to (list): List of recipient emails. + app_name (str): Name of the application sending the notification. + """ + + emails_to = [str(email) for email in emails_to if isinstance(email, str) or hasattr(email, 'email')] + + try: + msg = MIMEMultipart() + msg['From'] = settings.EMAIL_FROM + msg['To'] = ','.join(emails_to) + msg['Subject'] = subject + msg.attach(MIMEText(body, 'html', _charset='utf-8')) + text = msg.as_string() + + smtp_server = smtplib.SMTP(settings.SMTP_SERVER) + smtp_server.sendmail(settings.EMAIL_FROM, emails_to, text) + smtp_server.quit() + + print(f"{datetime.now()} - Email sent successfully for {app_name}.") + + except Exception as e: + print(f"{datetime.now()} - Failed to send email: {str(e)}") diff --git a/Watcher/Watcher/common/utils/send_slack_messages.py b/Watcher/Watcher/common/utils/send_slack_messages.py new file mode 100644 index 0000000..bb9f9e1 --- /dev/null +++ b/Watcher/Watcher/common/utils/send_slack_messages.py @@ -0,0 +1,35 @@ +import requests +from django.conf import settings +from django.utils import timezone + +def send_slack_message(content, channel, app_name): + """ + Sends a message to the specified Slack channel. + + Args: + content (str): The content of the message to send. + channel (str): The Slack channel where the message will be sent. + """ + + # if not settings.SLACK_API_TOKEN or not settings.SLACK_CHANNEL: + if not settings.SLACK_API_TOKEN or not settings.SLACK_CHANNEL: + print("No configuration for Slack, not notifications sending") + return + + url = 'https://slack.com/api/chat.postMessage' + headers = { + 'Authorization': f'Bearer {settings.SLACK_API_TOKEN}', + 'Content-Type': 'application/json', + } + + payload = { + 'channel': channel, + 'text': content, + } + + response = requests.post(url, headers=headers, json=payload) + + if response.status_code == 200: + print(f"{str(timezone.now())} - Message sent to Slack successfully for {app_name}.") + else: + print(f"{str(timezone.now())} - Failed to send message to Slack: {response.status_code} - {response.text}") diff --git a/Watcher/Watcher/common/utils/send_thehive_alerts.py b/Watcher/Watcher/common/utils/send_thehive_alerts.py new file mode 100644 index 0000000..5991ad9 --- /dev/null +++ b/Watcher/Watcher/common/utils/send_thehive_alerts.py @@ -0,0 +1,111 @@ +import requests +from django.conf import settings +from django.utils import timezone +from common.utils.update_thehive import ( + handle_alert_or_case, + create_new_alert +) +from site_monitoring.models import Site +from common.core import generate_ref + + +def get_ticket_id_for_domain(domain_name): + """ + Retrieve the ticket_id for a monitored domain from the Watcher database. + + :param domain_name: The domain name for which the ticket_id is being fetched. + :return: ticket_id if domain is found, otherwise None. + :rtype: str or None + """ + try: + domain = Site.objects.get(domain_name=domain_name) + return domain.ticket_id + except Site.DoesNotExist: + print(f"{timezone.now()} - Domain {domain_name} not found in Watcher database.") + return None + except Exception as e: + print(f"{timezone.now()} - Error while retrieving ticket_id for domain {domain_name}: {e}") + return None + + +def post_to_thehive(url, data, headers, proxies): + """ + Send a POST request to TheHive API and handle errors. + + :param url: The URL for TheHive API. + :param data: The data to send in the POST request. + :param headers: Headers to include in the request. + :param proxies: Proxies to use for the request. + :return: Response from TheHive as a JSON object if successful, None if failed. + :rtype: dict or None + """ + try: + response = requests.post(url, headers=headers, json=data, verify=False, proxies=proxies) + response.raise_for_status() + return response.json() + except requests.exceptions.RequestException as e: + return None + + +def send_thehive_alert(title, description, severity, tags, app_name, domain_name, observables=None, customFields=None, thehive_url=None, api_key=None): + """ + Send or update an alert in TheHive based on the application and ticket_id. + + :param title: The title of the alert. + :param description: The description of the alert. + :param severity: The severity level of the alert (integer). + :param tags: A list of tags associated with the alert. + :param app_name: The application triggering the alert (e.g., 'website_monitoring'). + :param domain_name: The domain name related to the alert (used for ticket_id lookup). + :param observables: Any observables (default is None). + :param customFields: Custom fields for the alert (default is None). + :param thehive_url: The URL of TheHive instance (default uses settings). + :param api_key: The API key for authenticating with TheHive (default uses settings). + :return: None + :rtype: None + """ + thehive_url = thehive_url or settings.THE_HIVE_URL + api_key = api_key or settings.THEHIVE_API_KEY + + ticket_id = None + + if app_name == 'website_monitoring' and domain_name: + ticket_id = get_ticket_id_for_domain(domain_name) + + if ticket_id is None: + ticket_id = generate_ref() + + if app_name == 'website_monitoring' and not ticket_id: + return + + # Handle the alert for 'website_monitoring' if ticket_id is found + if app_name == 'website_monitoring': + handle_alert_or_case( + ticket_id=ticket_id, + title=title, + description=description, + severity=severity, + tags=tags, + app_name=app_name, + observables=observables, + customFields=customFields, + comment=f"New alert created by {app_name} on {timezone.now()}: New information reported and initial context added.", + thehive_url=thehive_url, + api_key=api_key + ) + + # Create a new alert if the application is not 'website_monitoring' + else: + create_new_alert( + ticket_id=ticket_id, + title=title, + description=description, + severity=severity, + tags=tags, + app_name=app_name, + observables=observables, + customFields=customFields, + comment=f"New alert created by {app_name} on {timezone.now()}: New information reported and initial context added.", + thehive_url=thehive_url, + api_key=api_key + ) diff --git a/Watcher/Watcher/common/utils/update_thehive.py b/Watcher/Watcher/common/utils/update_thehive.py new file mode 100644 index 0000000..3e1ef87 --- /dev/null +++ b/Watcher/Watcher/common/utils/update_thehive.py @@ -0,0 +1,238 @@ +import requests +from django.utils import timezone +from common.core import generate_ref +from django.conf import settings + + +def search_thehive_for_ticket_id(watcher_id, thehive_url, api_key, item_type=None): + """ + Search TheHive for an item (alert or case) based on customFields.. + + :param watcher_id: The ID to search for. + :param thehive_url: The URL of TheHive instance. + :param api_key: The API key for authenticating with TheHive. + :param item_type: The type of item to search for ("alert" or "case"). + :return: The type and the first result if found, else None, None. + """ + headers = {'Content-Type': 'application/json', 'Authorization': f'Bearer {api_key}'} + proxies = {"http": None, "https": None} + + query = { + "query": [ + {"_name": f"list{item_type.capitalize()}"}, + {"_name": "filter", "_and": [ + {"_field": f"customFields.{settings.THE_HIVE_CUSTOM_FIELD}.string", "_value": watcher_id} + ]} + ] + } if item_type else None + + if not query: + return None, None + + url = f"{thehive_url}/api/v1/query" + try: + response = requests.post(url, headers=headers, json=query, verify=False, proxies=proxies) + response.raise_for_status() + results = response.json() + + if results: + return item_type, results[0] + except requests.exceptions.RequestException as e: + print(f"{timezone.now()} - Error searching for {item_type} with {settings.THE_HIVE_CUSTOM_FIELD} {watcher_id}: {e}") + + return None, None + + +def add_observables_to_item(item_type, item_id, observables_data, thehive_url, api_key): + """ + Add observables to an existing item (alert or case) in TheHive. + + :param item_type: The type of item ("alert" or "case"). + :param item_id: The ID of the item to add observables to. + :param observables_data: A list of observables to add. + :param thehive_url: The URL of TheHive instance. + :param api_key: The API key for authenticating with TheHive. + :return: None + """ + url = f"{thehive_url}/api/v1/{item_type}/{item_id}/observable" + headers = {'Content-Type': 'application/json', 'Authorization': f'Bearer {api_key}'} + proxies = {"http": None, "https": None} + + added_observables = [] + + for observable in observables_data: + try: + # Send a POST request to add each observable to the item + response = requests.post(url, headers=headers, json=observable, verify=False, proxies=proxies) + response.raise_for_status() + added_observables.append(observable['data']) # Add the observable data to the success list + except requests.exceptions.RequestException as e: + print(f"{timezone.now()} - Error while adding observable {observable['data']}: {e}") + + +def add_comment_to_item(item_type, item_id, comment, thehive_url, api_key): + """ + Add a comment to an existing item (alert or case) in TheHive. + + :param item_type: The type of item ("alert" or "case"). + :param item_id: The ID of the item to add a comment to. + :param comment: The comment to add to the item. + :param thehive_url: The URL of TheHive instance. + :param api_key: The API key for authenticating with TheHive. + :return: None + """ + url = f"{thehive_url}/api/v1/{item_type}/{item_id}/comment" + headers = {'Content-Type': 'application/json', 'Authorization': f'Bearer {api_key}'} + proxies = {"http": None, "https": None} + data = {"message": comment} + + try: + response = requests.post(url, headers=headers, json=data, verify=False, proxies=proxies) + response.raise_for_status() + except requests.exceptions.RequestException as e: + print(f"{timezone.now()} - Error while adding comment: {e}") + + +def create_observables(observables): + """ + Format observables for TheHive API. + + :param observables: A list of raw observables. + :return: A list of formatted observables ready for TheHive. + :rtype: list + """ + observables_data = [] + for obs in observables: + observable_data = { + "dataType": obs['dataType'], + "data": obs['data'], + "message": f"Observable added: {obs['data']} on {timezone.now()}", + "ioc": True, + "sighted": True, + "tlp": 2 + } + observables_data.append(observable_data) + return observables_data + + +def update_existing_alert_case(item_type, existing_item, observables, comment, thehive_url, api_key): + """ + Update an existing alert or case by adding observables and a comment. + + :param item_type: The type of item ("alert" or "case"). + :param existing_item: The existing item to update. + :param observables: The observables to add to the item. + :param comment: The comment to add to the item. + :param thehive_url: The URL of TheHive instance. + :param api_key: The API key for authenticating with TheHive. + :return: None + """ + item_id = existing_item["_id"] + + if observables: + observables_data = create_observables(observables) + print(f"{timezone.now()} - Adding observables to {item_type} with ID {item_id}...") + add_observables_to_item(item_type, item_id, observables_data, thehive_url, api_key) + + if comment: + print(f"{timezone.now()} - Adding comment to {item_type} with ID {item_id}...") + add_comment_to_item(item_type, item_id, comment, thehive_url, api_key) + + +def create_new_alert(ticket_id, title, description, severity, tags, app_name, observables, customFields, comment, thehive_url, api_key): + """ + Create a new alert in TheHive with the provided details. + + :param ticket_id: The ticket ID for the new alert. + :param title: The title of the alert. + :param description: The description of the alert. + :param severity: The severity level of the alert (integer). + :param tags: A list of tags associated with the alert. + :param app_name: The application triggering the alert. + :param observables: A list of observables to associate with the alert. + :param customFields: Custom fields to include in the alert. + :param comment: The comment to add to the alert. + :param thehive_url: The URL of TheHive instance. + :param api_key: The API key for authenticating with TheHive. + :return: The created alert if successful, None otherwise. + :rtype: dict or None + """ + if ticket_id is None: + ticket_id = generate_ref() + + alert_data = { + "title": title, + "description": description, + "severity": severity, + "tags": tags, + "type": app_name, + "source": "watcher", + "sourceRef": ticket_id, + "customFields": customFields or { + settings.THE_HIVE_CUSTOM_FIELD: {"string": ticket_id}, + "email-sender": {"string": settings.THE_HIVE_EMAIL_SENDER} + } + } + + headers = {'Content-Type': 'application/json', 'Authorization': f'Bearer {api_key}'} + proxies = {"http": None, "https": None} + + url = f"{thehive_url}/api/v1/alert" + try: + response = requests.post(url, headers=headers, json=alert_data, verify=False, proxies=proxies) + response.raise_for_status() + alert = response.json() + alert_id = alert.get('_id') + print(f"{timezone.now()} - Alert created successfully with ID: {alert_id}") + + if observables: + observables_data = create_observables(observables) + add_observables_to_item("alert", alert_id, observables_data, thehive_url, api_key) + + if comment: + add_comment_to_item("alert", alert_id, comment, thehive_url, api_key) + + return alert + + except requests.exceptions.RequestException as e: + print(f"{timezone.now()} - Failed to create alert: {e}") + return None + + +def handle_alert_or_case(ticket_id, observables, comment, title, description, severity, tags, app_name, customFields, thehive_url, api_key): + """ + Handle the creation or updating of alerts and cases in TheHive. + + :param ticket_id: The ID of the ticket (can be used to search for existing alerts/cases). + :param observables: A list of observables to add to the item. + :param comment: A comment to add to the item. + :param title: The title for the alert. + :param description: The description for the alert. + :param severity: The severity of the alert (integer). + :param tags: A list of tags for the alert. + :param app_name: The name of the application triggering the alert. + :param customFields: Custom fields to be included in the alert. + :param thehive_url: The URL of TheHive. + :param api_key: The API key for authentication. + :return: None + """ + if app_name != 'website_monitoring': + print(f"{timezone.now()} - Unsupported application: {app_name}.") + return + + # Search in TheHive by customFields. + alert_type, alert_item = search_thehive_for_ticket_id(ticket_id, thehive_url, api_key, item_type="alert") + case_type, case_item = search_thehive_for_ticket_id(ticket_id, thehive_url, api_key, item_type="case") + + if case_item: + print(f"{timezone.now()} - Case found for {settings.THE_HIVE_CUSTOM_FIELD} {ticket_id}. Updating...") + update_existing_alert_case("case", case_item, observables, comment, thehive_url, api_key) + elif alert_item: + print(f"{timezone.now()} - Alert found for {settings.THE_HIVE_CUSTOM_FIELD} {ticket_id}. Updating...") + update_existing_alert_case("alert", alert_item, observables, comment, thehive_url, api_key) + else: + create_new_alert( + ticket_id=ticket_id, title=title, description=description, severity=severity, + tags=tags, app_name=app_name, observables=observables, + customFields=customFields, comment=comment, thehive_url=thehive_url, api_key=api_key + ) \ No newline at end of file diff --git a/Watcher/Watcher/common/views.py b/Watcher/Watcher/common/views.py new file mode 100644 index 0000000..91ea44a --- /dev/null +++ b/Watcher/Watcher/common/views.py @@ -0,0 +1,3 @@ +from django.shortcuts import render + +# Create your views here. diff --git a/Watcher/Watcher/data_leak/admin.py b/Watcher/Watcher/data_leak/admin.py index c438565..0cb12fc 100644 --- a/Watcher/Watcher/data_leak/admin.py +++ b/Watcher/Watcher/data_leak/admin.py @@ -16,9 +16,18 @@ def __new__(cls, *args, **kwargs): @admin.register(Subscriber) class Subscriber(admin.ModelAdmin): - list_display = ['user_rec', 'created_at'] - list_filter = ['created_at'] - search_fields = ['user_rec'] + list_display = ('user_rec', 'created_at', 'email', 'thehive', 'slack', 'citadel') + list_filter = ('email', 'thehive', 'slack', 'citadel') + search_fields = ('user_rec__username',) + fieldsets = ( + (None, { + 'fields': ('user_rec', 'created_at') + }), + ('Notification Channels', { + 'fields': ('email', 'thehive', 'slack', 'citadel'), + 'description': "Select the notification channels for this subscriber." + }), + ) class AlertResource(resources.ModelResource): diff --git a/Watcher/Watcher/data_leak/core.py b/Watcher/Watcher/data_leak/core.py index c6ebb4e..58a4ff4 100644 --- a/Watcher/Watcher/data_leak/core.py +++ b/Watcher/Watcher/data_leak/core.py @@ -8,12 +8,9 @@ import tzlocal from django.conf import settings from django.db.models.functions import Length -from .mail_template.default_template import get_template -from .mail_template.group_template import get_group_template -from email.mime.multipart import MIMEMultipart -from email.mime.text import MIMEText from json.decoder import JSONDecodeError -import smtplib +from common.core import send_app_specific_notifications +from django.db.models import Q def start_scheduler(): @@ -26,7 +23,6 @@ def start_scheduler(): scheduler.add_job(main_data_leak, 'cron', day_of_week='mon-sun', minute='*/5', id='week_job', max_instances=10, replace_existing=True) - scheduler.add_job(cleanup, 'cron', day_of_week='mon-sun', hour='*/2', id='clean', replace_existing=True) scheduler.start() @@ -233,12 +229,8 @@ def check_keywords(keywords): for result in results: print(str(timezone.now()) + " - Create alert for: ", keyword, "url: ", result) alert = Alert.objects.create(keyword=Keyword.objects.get(name=keyword), url=result) - # limiting the number of specific email per alert - if len(results) < 6: - send_email(alert) - # if there is too many alerts, we send a group email - if len(results) >= 6: - send_group_email(keyword, len(results)) + send_data_leak_notifications(alert) + # now we check Pastebin for new pastes result = check_pastebin(keywords) @@ -248,75 +240,25 @@ def check_keywords(keywords): print(str(timezone.now()) + " - Create alert for: ", keyword, "url: ", url) alert = Alert.objects.create(keyword=Keyword.objects.get(name=keyword), url=url, content=paste_content_hits[url]) - send_email(alert) + send_data_leak_notifications(alert) -def send_email(alert): +def send_data_leak_notifications(alert): """ - Send e-mail alert. - - :param alert: Alert object. + Sends notifications to Slack, Citadel, or TheHive based on data_leak. + + :param alert: Alert Object. """ - emails_to = list() - # Get all subscribers email - for subscriber in Subscriber.objects.all(): - emails_to.append(subscriber.user_rec.email) - - # If there is at least one subscriber - if len(emails_to) > 0: - try: - msg = MIMEMultipart() - msg['From'] = settings.EMAIL_FROM - msg['To'] = ','.join(emails_to) - msg['Subject'] = str("[ALERT #" + str(alert.pk) + "] Data Leak") - body = get_template(alert) - msg.attach(MIMEText(body, 'html', _charset='utf-8')) - text = msg.as_string() - smtp_server = smtplib.SMTP(settings.SMTP_SERVER) - smtp_server.sendmail(settings.EMAIL_FROM, emails_to, text) - smtp_server.quit() - - except Exception as e: - # Print any error messages to stdout - print(str(timezone.now()) + " - Email Error : ", e) - finally: - for email in emails_to: - print(str(timezone.now()) + " - Email sent to ", email) - else: - print(str(timezone.now()) + " - No subscriber, no email sent.") + subscribers = Subscriber.objects.filter( + (Q(slack=True) | Q(citadel=True) | Q(thehive=True) | Q(email=True)) + ) + if not subscribers.exists(): + print(f"{timezone.now()} - No subscribers for Data Leak, no message sent.") + return -def send_group_email(keyword, alerts_number): - """ - Send group e-mail for a specific keyword. + context_data = { + 'alert': alert + } - :param keyword: Matched Keyword. - :param alerts_number: Number of alerts. - """ - emails_to = list() - # Get all subscribers email - for subscriber in Subscriber.objects.all(): - emails_to.append(subscriber.user_rec.email) - - # If there is at least one subscriber - if len(emails_to) > 0: - try: - msg = MIMEMultipart() - msg['From'] = settings.EMAIL_FROM - msg['To'] = ','.join(emails_to) - msg['Subject'] = str("[" + str(alerts_number) + " ALERTS] Data Leak") - body = get_group_template(keyword, alerts_number) - msg.attach(MIMEText(body, 'html', _charset='utf-8')) - text = msg.as_string() - smtp_server = smtplib.SMTP(settings.SMTP_SERVER) - smtp_server.sendmail(settings.EMAIL_FROM, emails_to, text) - smtp_server.quit() - - except Exception as e: - # Print any error messages to stdout - print(str(timezone.now()) + " - Email Error : ", e) - finally: - for email in emails_to: - print(str(timezone.now()) + " - Email sent to ", email) - else: - print(str(timezone.now()) + " - No subscriber, no email sent.") + send_app_specific_notifications('data_leak', context_data, subscribers) \ No newline at end of file diff --git a/Watcher/Watcher/data_leak/mail_template/default_template.py b/Watcher/Watcher/data_leak/mail_template/default_template.py deleted file mode 100644 index b2d2893..0000000 --- a/Watcher/Watcher/data_leak/mail_template/default_template.py +++ /dev/null @@ -1,194 +0,0 @@ -from django.conf import settings - - -def get_template(alert): - body = """\ - - - - - - - - - - - - -
-

Data Leak: Alert - """ - body += "#" + str( - alert.pk) + """ -

-
- - - - - - -
- - - - - - - - - - - - - - -

-
-

Dear team,

-

New Data Leakage Alert for - - """ - body += str( - alert.keyword) + """ keyword:

Source: - """ - body += str( - alert.url) + """

Details here.

-

- Best Regards, -

-

Watcher
-
-

-

-
-
- - - - - - - -
- - - - - - - -
-
-
- - - - -

[""" - body += str( - settings.EMAIL_CLASSIFICATION) + """]


- - - """ - return body diff --git a/Watcher/Watcher/data_leak/migrations/0011_alter_subscriber_options_subscriber_citadel_and_more.py b/Watcher/Watcher/data_leak/migrations/0011_alter_subscriber_options_subscriber_citadel_and_more.py new file mode 100644 index 0000000..8bdf836 --- /dev/null +++ b/Watcher/Watcher/data_leak/migrations/0011_alter_subscriber_options_subscriber_citadel_and_more.py @@ -0,0 +1,37 @@ +# Generated by Django 5.0.9 on 2024-11-29 15:26 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('data_leak', '0010_auto_20210517_0956'), + ] + + operations = [ + migrations.AlterModelOptions( + name='subscriber', + options={'verbose_name_plural': 'subscribers'}, + ), + migrations.AddField( + model_name='subscriber', + name='citadel', + field=models.BooleanField(default=False, verbose_name='Citadel'), + ), + migrations.AddField( + model_name='subscriber', + name='email', + field=models.BooleanField(default=False, verbose_name='E-mail'), + ), + migrations.AddField( + model_name='subscriber', + name='slack', + field=models.BooleanField(default=False, verbose_name='Slack'), + ), + migrations.AddField( + model_name='subscriber', + name='thehive', + field=models.BooleanField(default=False, verbose_name='TheHive'), + ), + ] diff --git a/Watcher/Watcher/data_leak/models.py b/Watcher/Watcher/data_leak/models.py index 884702c..4181f65 100644 --- a/Watcher/Watcher/data_leak/models.py +++ b/Watcher/Watcher/data_leak/models.py @@ -49,7 +49,18 @@ def __str__(self): class Subscriber(models.Model): """ - List of the email alert subscriber(s). + List of the alert subscriber(s). """ user_rec = models.ForeignKey(User, on_delete=models.CASCADE, related_name='data_leak') created_at = models.DateTimeField(default=timezone.now) + + email = models.BooleanField(default=False, verbose_name="E-mail") + thehive = models.BooleanField(default=False, verbose_name="TheHive") + slack = models.BooleanField(default=False, verbose_name="Slack") + citadel = models.BooleanField(default=False, verbose_name="Citadel") + + class Meta: + verbose_name_plural = 'subscribers' + + def __str__(self): + return f'{self.user_rec.username} - {self.created_at}' diff --git a/Watcher/Watcher/dns_finder/admin.py b/Watcher/Watcher/dns_finder/admin.py index 99f6e1e..4172a0b 100644 --- a/Watcher/Watcher/dns_finder/admin.py +++ b/Watcher/Watcher/dns_finder/admin.py @@ -99,6 +99,15 @@ def has_add_permission(self, request): @admin.register(Subscriber) class Subscriber(admin.ModelAdmin): - list_display = ['user_rec', 'created_at'] - list_filter = ['created_at'] - search_fields = ['user_rec'] + list_display = ('user_rec', 'created_at', 'email', 'thehive', 'slack', 'citadel') + list_filter = ('email', 'thehive', 'slack', 'citadel') + search_fields = ('user_rec__username',) + fieldsets = ( + (None, { + 'fields': ('user_rec', 'created_at') + }), + ('Notification Channels', { + 'fields': ('email', 'thehive', 'slack', 'citadel'), + 'description': "Select the notification channels for this subscriber." + }), + ) diff --git a/Watcher/Watcher/dns_finder/api.py b/Watcher/Watcher/dns_finder/api.py index 38852e2..217447e 100644 --- a/Watcher/Watcher/dns_finder/api.py +++ b/Watcher/Watcher/dns_finder/api.py @@ -1,6 +1,6 @@ from .models import DnsMonitored, DnsTwisted, Alert, KeywordMonitored from rest_framework import viewsets, permissions -from .serializers import AlertSerializer, DnsMonitoredSerializer, DnsTwistedSerializer, ThehiveSerializer, \ +from .serializers import AlertSerializer, DnsMonitoredSerializer, DnsTwistedSerializer, \ MISPSerializer, KeywordMonitoredSerializer @@ -53,14 +53,6 @@ def has_permission(self, request, view): return has_permission -# Thehive Viewset -class ThehiveViewSet(viewsets.ModelViewSet): - permission_classes = [ - ExportPermission - ] - serializer_class = ThehiveSerializer - - # MISP Viewset class MISPViewSet(viewsets.ModelViewSet): permission_classes = [ diff --git a/Watcher/Watcher/dns_finder/core.py b/Watcher/Watcher/dns_finder/core.py index b858d1a..9c3bf84 100644 --- a/Watcher/Watcher/dns_finder/core.py +++ b/Watcher/Watcher/dns_finder/core.py @@ -5,16 +5,12 @@ import json from django.utils import timezone from django.conf import settings -from .mail_template.default_template import get_template -from .mail_template.default_template_cert_transparency import get_cert_transparency_template -from .mail_template.group_template import get_group_template from apscheduler.schedulers.background import BackgroundScheduler import tzlocal from .models import Alert, DnsMonitored, DnsTwisted, Subscriber, KeywordMonitored -from email.mime.multipart import MIMEMultipart -from email.mime.text import MIMEText -import smtplib import certstream +from common.core import send_app_specific_notifications +from django.db.models import Q def start_scheduler(): @@ -31,6 +27,7 @@ def start_scheduler(): id='main_certificate_transparency', max_instances=2, replace_existing=True) + scheduler.start() @@ -62,7 +59,7 @@ def print_callback(message, context): print(str(timezone.now()) + " - " + "Keyword", keyword_monitored.name, "detected in :", domain) dns_twisted = DnsTwisted.objects.create(domain_name=domain, keyword_monitored=keyword_monitored) alert = Alert.objects.create(dns_twisted=dns_twisted) - send_email_cert_transparency(alert) + send_dns_finder_notifications(alert) def main_certificate_transparency(): @@ -135,119 +132,36 @@ def check_dnstwist(dns_monitored): fuzzer=twisted_website_dict['fuzzer']) alert = Alert.objects.create(dns_twisted=dns_twisted) alerts_list.append(alert) - # Send email alerts - if len(alerts_list) < 6: - for alert in alerts_list: - send_email(alert) - if len(alerts_list) >= 6: - send_group_email(dns_monitored, len(alerts_list)) + + for alert in alerts_list: + send_dns_finder_notifications(alert) + except ValueError: print('Decoding JSON has failed') print(str(timezone.now()) + " - " + "dnstwist: Successfully processed: ", dns_monitored.domain_name) -def send_email(alert): +def send_dns_finder_notifications(alert): """ - Send e-mail alert. - + Sends notifications to Slack, Citadel, or TheHive based on dns_finder. + :param alert: Alert Object. """ - emails_to = list() - # Get all subscribers email - for subscriber in Subscriber.objects.all(): - emails_to.append(subscriber.user_rec.email) + subscribers = Subscriber.objects.filter( + (Q(slack=True) | Q(citadel=True) | Q(thehive=True) | Q(email=True)) + ) - # If there is at least one subscriber - if len(emails_to) > 0: - try: - msg = MIMEMultipart() - msg['From'] = settings.EMAIL_FROM - msg['To'] = ','.join(emails_to) - msg['Subject'] = str("[ALERT #" + str(alert.pk) + "] DNS Finder") - body = get_template(alert) - msg.attach(MIMEText(body, 'html', _charset='utf-8')) - text = msg.as_string() - smtp_server = smtplib.SMTP(settings.SMTP_SERVER) - smtp_server.sendmail(settings.EMAIL_FROM, emails_to, text) - smtp_server.quit() - - except Exception as e: - # Print any error messages to stdout - print(str(timezone.now()) + " - Email Error : ", e) - finally: - for email in emails_to: - print(str(timezone.now()) + " - Email sent to ", email) - else: - print(str(timezone.now()) + " - No subscriber, no email sent.") - - -def send_group_email(dns_monitored, alerts_number): - """ - Send group e-mail for a specific dns_monitored. + if not subscribers.exists(): + print(f"{timezone.now()} - No subscribers for DNS Finder, no message sent.") + return - :param dns_monitored: DnsMonitored Object. - :param alerts_number: Number of alerts. - """ - emails_to = list() - # Get all subscribers email - for subscriber in Subscriber.objects.all(): - emails_to.append(subscriber.user_rec.email) + if not alert or not alert.dns_twisted or not alert.dns_twisted.domain_name: + print(f"Error: Invalid alert object or missing domain_name in dns_twisted for alert: {alert}") + return - # If there is at least one subscriber - if len(emails_to) > 0: - try: - msg = MIMEMultipart() - msg['From'] = settings.EMAIL_FROM - msg['To'] = ','.join(emails_to) - msg['Subject'] = str("[" + str(alerts_number) + " ALERTS] DNS Finder") - body = get_group_template(dns_monitored, alerts_number) - msg.attach(MIMEText(body, 'html', _charset='utf-8')) - text = msg.as_string() - smtp_server = smtplib.SMTP(settings.SMTP_SERVER) - smtp_server.sendmail(settings.EMAIL_FROM, emails_to, text) - smtp_server.quit() - - except Exception as e: - # Print any error messages to stdout - print(str(timezone.now()) + " - Email Error : ", e) - finally: - for email in emails_to: - print(str(timezone.now()) + " - Email sent to ", email) - else: - print(str(timezone.now()) + " - No subscriber, no email sent.") - - -def send_email_cert_transparency(alert): - """ - Send e-mail alert. - - :param alert: Alert Object. - """ - emails_to = list() - # Get all subscribers email - for subscriber in Subscriber.objects.all(): - emails_to.append(subscriber.user_rec.email) + context_data = { + 'alert': alert, + } - # If there is at least one subscriber - if len(emails_to) > 0: - try: - msg = MIMEMultipart() - msg['From'] = settings.EMAIL_FROM - msg['To'] = ','.join(emails_to) - msg['Subject'] = str("[ALERT #" + str(alert.pk) + "] DNS Finder") - body = get_cert_transparency_template(alert) - msg.attach(MIMEText(body, 'html', _charset='utf-8')) - text = msg.as_string() - smtp_server = smtplib.SMTP(settings.SMTP_SERVER) - smtp_server.sendmail(settings.EMAIL_FROM, emails_to, text) - smtp_server.quit() - - except Exception as e: - # Print any error messages to stdout - print(str(timezone.now()) + " - Email Error : ", e) - finally: - for email in emails_to: - print(str(timezone.now()) + " - Email sent to ", email) - else: - print(str(timezone.now()) + " - No subscriber, no email sent.") + send_app_specific_notifications('dns_finder', context_data, subscribers) \ No newline at end of file diff --git a/Watcher/Watcher/dns_finder/mail_template/default_template.py b/Watcher/Watcher/dns_finder/mail_template/default_template.py deleted file mode 100644 index c4546bc..0000000 --- a/Watcher/Watcher/dns_finder/mail_template/default_template.py +++ /dev/null @@ -1,194 +0,0 @@ -from django.conf import settings - - -def get_template(alert): - body = """\ - - - - - - - - - - - - -
-

DNS Finder: Alert - """ - body += "#" + str( - alert.pk) + """ -

-
- - - - - - -
- - - - - - - - - - - - - - -

-
-

Dear team,

-

New Twisted DNS found: - - """ - body += str( - alert.dns_twisted.domain_name) + """

Asset: - """ - body += str( - alert.dns_twisted.dns_monitored) + """

Details here.

-

- Best Regards, -

-

Watcher
-
-

-

-
-
- - - - - - - -
- - - - - - - -
-
-
- - - - -

[""" - body += str( - settings.EMAIL_CLASSIFICATION) + """]


- - - """ - return body diff --git a/Watcher/Watcher/dns_finder/migrations/0008_alter_subscriber_options_and_more.py b/Watcher/Watcher/dns_finder/migrations/0008_alter_subscriber_options_and_more.py new file mode 100644 index 0000000..2d91fd0 --- /dev/null +++ b/Watcher/Watcher/dns_finder/migrations/0008_alter_subscriber_options_and_more.py @@ -0,0 +1,41 @@ +# Generated by Django 5.0.9 on 2024-11-29 15:26 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('dns_finder', '0007_auto_20210517_0956'), + ] + + operations = [ + migrations.AlterModelOptions( + name='subscriber', + options={'verbose_name_plural': 'subscribers'}, + ), + migrations.RemoveField( + model_name='dnstwisted', + name='the_hive_case_id', + ), + migrations.AddField( + model_name='subscriber', + name='citadel', + field=models.BooleanField(default=False, verbose_name='Citadel'), + ), + migrations.AddField( + model_name='subscriber', + name='email', + field=models.BooleanField(default=False, verbose_name='E-mail'), + ), + migrations.AddField( + model_name='subscriber', + name='slack', + field=models.BooleanField(default=False, verbose_name='Slack'), + ), + migrations.AddField( + model_name='subscriber', + name='thehive', + field=models.BooleanField(default=False, verbose_name='TheHive'), + ), + ] diff --git a/Watcher/Watcher/dns_finder/models.py b/Watcher/Watcher/dns_finder/models.py index fffef52..802a8b0 100644 --- a/Watcher/Watcher/dns_finder/models.py +++ b/Watcher/Watcher/dns_finder/models.py @@ -43,7 +43,6 @@ class DnsTwisted(models.Model): dns_monitored = models.ForeignKey(DnsMonitored, on_delete=models.CASCADE, blank=True, null=True) keyword_monitored = models.ForeignKey(KeywordMonitored, on_delete=models.CASCADE, blank=True, null=True) fuzzer = models.CharField(max_length=100, blank=True, null=True) - the_hive_case_id = models.CharField(max_length=100, unique=True, blank=True, null=True) misp_event_id = models.IntegerField(unique=True, blank=True, null=True) created_at = models.DateTimeField(default=timezone.now) @@ -70,7 +69,18 @@ class Meta: class Subscriber(models.Model): """ - List of the email alert subscriber(s). + List of the alert subscriber(s). """ user_rec = models.ForeignKey(User, on_delete=models.CASCADE, related_name='dns_finder') created_at = models.DateTimeField(default=timezone.now) + + email = models.BooleanField(default=False, verbose_name="E-mail") + thehive = models.BooleanField(default=False, verbose_name="TheHive") + slack = models.BooleanField(default=False, verbose_name="Slack") + citadel = models.BooleanField(default=False, verbose_name="Citadel") + + class Meta: + verbose_name_plural = 'subscribers' + + def __str__(self): + return f'{self.user_rec.username} - {self.created_at}' diff --git a/Watcher/Watcher/dns_finder/serializers.py b/Watcher/Watcher/dns_finder/serializers.py index b1fed74..a8384fc 100644 --- a/Watcher/Watcher/dns_finder/serializers.py +++ b/Watcher/Watcher/dns_finder/serializers.py @@ -7,9 +7,6 @@ import requests from rest_framework.exceptions import NotFound, AuthenticationFailed -from thehive4py.api import TheHiveApi -from thehive4py.models import Case -from site_monitoring.thehive import create_observables, update_observables from pymisp import ExpandedPyMISP, MISPEvent from site_monitoring.misp import create_misp_tags, create_attributes, update_attributes @@ -124,91 +121,4 @@ def data(self): # just return success dictionary. you can change this to your need, but i dont think output should be user data after password change alert_id = self.validated_data['id'] alert = Alert.objects.get(pk=alert_id) - return {'id': alert_id, 'misp_event_id': DnsTwisted.objects.get(pk=alert.dns_twisted.pk).misp_event_id} - - -# Thehive Serializer -class ThehiveSerializer(serializers.Serializer): - id = serializers.IntegerField() - - def save(self): - alert_id = self.validated_data['id'] - alert = Alert.objects.get(pk=alert_id) - - dns_twisted = DnsTwisted.objects.get(pk=alert.dns_twisted.pk) - - # Getting IOCs related to the new twisted domain - if Site.objects.filter(domain_name=dns_twisted.domain_name): - already_in_monitoring = True - site = Site.objects.get(domain_name=dns_twisted.domain_name) - # Save the case id in database - DnsTwisted.objects.filter(pk=dns_twisted.pk).update(the_hive_case_id=site.the_hive_case_id) - else: - already_in_monitoring = False - site = Site.objects.create(domain_name=dns_twisted.domain_name, rtir=-999999999) - monitoring_init(site) - - site = Site.objects.get(pk=site.pk) - - # We now hav the IOCs related to the domain, we can remove it from monitoring - if not already_in_monitoring: - Site.objects.filter(pk=site.pk).delete() - - if site.the_hive_case_id is None: - site.the_hive_case_id = dns_twisted.the_hive_case_id - - # Test The Hive instance connection - try: - requests.get(settings.THE_HIVE_URL) - except requests.exceptions.SSLError as e: - print(str(timezone.now()) + " - ", e) - raise AuthenticationFailed("SSL Error: " + settings.THE_HIVE_URL) - except requests.exceptions.RequestException as e: - print(str(timezone.now()) + " - ", e) - raise NotFound("Not Found: " + settings.THE_HIVE_URL) - - hive_api = TheHiveApi(settings.THE_HIVE_URL, settings.THE_HIVE_KEY, cert=True) - - if site.the_hive_case_id is not None: - # If the case already exist, then we update IOCs - update_observables(hive_api, site) - else: - # If the case does not exist, then we create it - - # Prepare the case - case = Case(title='Suspicious domain name ' + site.domain_name, - owner=settings.THE_HIVE_CASE_ASSIGNEE, - severity=2, - tlp=2, - pap=2, - flag=False, - tags=['Watcher', 'Impersonation', 'Malicious Domain', 'Typosquatting'], - description='Suspicious domain name ' + site.domain_name) - - # Create the case - print(str(timezone.now()) + " - " + 'Create Case') - print('-----------------------------') - response = hive_api.create_case(case) - - if response.status_code == 201: - print(str(timezone.now()) + " - " + "OK") - case_id = response.json()['id'] - - # Save the case id in database - DnsTwisted.objects.filter(pk=dns_twisted.pk).update(the_hive_case_id=case_id) - if Site.objects.filter(domain_name=dns_twisted.domain_name): - Site.objects.filter(pk=site.pk).update(the_hive_case_id=case_id) - - # Create all IOCs observables - create_observables(hive_api, case_id, site) - else: - print(str(timezone.now()) + " - " + 'ko: {}/{}'.format(response.status_code, response.text)) - data = {'detail': response.json()['type'] + ": " + response.json()['message']} - raise serializers.ValidationError(data) - - @property - def data(self): - # just return success dictionary. you can change this to your need, but i dont think output should be user data after password change - alert_id = self.validated_data['id'] - alert = Alert.objects.get(pk=alert_id) - return {'id': alert_id, 'the_hive_case_id': DnsTwisted.objects.get(pk=alert.dns_twisted.pk).the_hive_case_id} + return {'id': alert_id, 'misp_event_id': DnsTwisted.objects.get(pk=alert.dns_twisted.pk).misp_event_id} \ No newline at end of file diff --git a/Watcher/Watcher/dns_finder/urls.py b/Watcher/Watcher/dns_finder/urls.py index 443e372..eaa044f 100644 --- a/Watcher/Watcher/dns_finder/urls.py +++ b/Watcher/Watcher/dns_finder/urls.py @@ -1,5 +1,5 @@ from rest_framework import routers -from .api import DnsMonitoredViewSet, DnsTwistedViewSet, AlertViewSet, ThehiveViewSet, MISPViewSet, KeywordMonitoredViewSet +from .api import DnsMonitoredViewSet, DnsTwistedViewSet, AlertViewSet, MISPViewSet, KeywordMonitoredViewSet from .core import start_scheduler @@ -8,7 +8,6 @@ router.register('api/dns_finder/keyword_monitored', KeywordMonitoredViewSet, 'keyword_monitored') router.register('api/dns_finder/dns_twisted', DnsTwistedViewSet, 'dns_twisted') router.register('api/dns_finder/alert', AlertViewSet, 'alert') -router.register('api/dns_finder/thehive', ThehiveViewSet, 'thehive') router.register('api/dns_finder/misp', MISPViewSet, 'misp') urlpatterns = router.urls diff --git a/Watcher/Watcher/frontend/src/actions/DnsFinder.js b/Watcher/Watcher/frontend/src/actions/DnsFinder.js index f008e8a..5efd902 100644 --- a/Watcher/Watcher/frontend/src/actions/DnsFinder.js +++ b/Watcher/Watcher/frontend/src/actions/DnsFinder.js @@ -7,7 +7,6 @@ import { ADD_DNS_MONITORED, PATCH_DNS_MONITORED, UPDATE_DNS_FINDER_ALERT, - EXPORT_THE_HIVE_DNS_FINDER, EXPORT_MISP_DNS_FINDER, GET_KEYWORD_MONITORED, DELETE_KEYWORD_MONITORED, @@ -173,22 +172,6 @@ export const updateAlertStatus = (id, status) => (dispatch, getState) => { ); }; -// EXPORT TO THE HIVE -export const exportToTheHive = (site) => (dispatch, getState) => { - axios - .post(`/api/dns_finder/thehive/`, site, tokenConfig(getState)) - .then(res => { - dispatch(createMessage({add: `Twisted DNS Exported to Thehive`})); - dispatch({ - type: EXPORT_THE_HIVE_DNS_FINDER, - payload: res.data - }); - }) - .catch(err => - dispatch(returnErrors(err.response.data, err.response.status)) - ); -}; - // EXPORT TO MISP export const exportToMISP = (site) => (dispatch, getState) => { axios diff --git a/Watcher/Watcher/frontend/src/actions/SiteMonitoring.js b/Watcher/Watcher/frontend/src/actions/SiteMonitoring.js index 32a9737..8b522f5 100755 --- a/Watcher/Watcher/frontend/src/actions/SiteMonitoring.js +++ b/Watcher/Watcher/frontend/src/actions/SiteMonitoring.js @@ -7,7 +7,6 @@ import { ADD_SITE, PATCH_SITE, UPDATE_SITE_ALERT, - EXPORT_THE_HIVE, EXPORT_MISP } from "./types"; import {createMessage, returnErrors} from "./messages"; @@ -107,22 +106,6 @@ export const updateSiteAlertStatus = (id, status) => (dispatch, getState) => { ); }; -// EXPORT TO THE HIVE -export const exportToTheHive = (site) => (dispatch, getState) => { - axios - .post(`/api/site_monitoring/thehive/`, site, tokenConfig(getState)) - .then(res => { - dispatch(createMessage({add: `Website Exported to Thehive`})); - dispatch({ - type: EXPORT_THE_HIVE, - payload: res.data - }); - }) - .catch(err => - dispatch(returnErrors(err.response.data, err.response.status)) - ); -}; - // EXPORT TO MISP export const exportToMISP = (site) => (dispatch, getState) => { axios diff --git a/Watcher/Watcher/frontend/src/components/DnsFinder/Alerts.js b/Watcher/Watcher/frontend/src/components/DnsFinder/Alerts.js index e4cfefa..ef2ac27 100755 --- a/Watcher/Watcher/frontend/src/components/DnsFinder/Alerts.js +++ b/Watcher/Watcher/frontend/src/components/DnsFinder/Alerts.js @@ -1,7 +1,7 @@ import React, {Component, Fragment} from 'react'; import {connect} from 'react-redux'; import PropTypes from 'prop-types'; -import {getAlerts, updateAlertStatus, exportToTheHive, exportToMISP} from "../../actions/DnsFinder"; +import {getAlerts, updateAlertStatus, exportToMISP} from "../../actions/DnsFinder"; import {addSite, getSites} from "../../actions/SiteMonitoring"; import Modal from "react-bootstrap/Modal"; import Button from "react-bootstrap/Button"; @@ -23,7 +23,6 @@ export class Alerts extends Component { id: 0, exportLoading: false, exportLoadingMISPTh: false, - theHiveCaseId: null, mispEventId: null, domainName: "" }; @@ -40,7 +39,6 @@ export class Alerts extends Component { alerts: PropTypes.array.isRequired, getAlerts: PropTypes.func.isRequired, updateAlertStatus: PropTypes.func.isRequired, - exportToTheHive: PropTypes.func.isRequired, exportToMISP: PropTypes.func.isRequired, auth: PropTypes.object.isRequired, error: PropTypes.object.isRequired, @@ -267,12 +265,11 @@ export class Alerts extends Component { return back; }; - displayExportModal = (id, domainName, theHiveCaseId, mispEventId) => { + displayExportModal = (id, domainName, mispEventId) => { this.setState({ showExportModal: true, id: id, domainName: domainName, - theHiveCaseId: theHiveCaseId, mispEventId: mispEventId }); }; @@ -285,21 +282,6 @@ export class Alerts extends Component { }); }; - let onSubmitTheHive; - onSubmitTheHive = e => { - e.preventDefault(); - const id = this.state.id; - const site = {id}; - - this.props.exportToTheHive(site); - this.setState({ - domainName: "", - id: 0, - exportLoadingMISPTh: id - }); - handleClose(); - }; - let onSubmitMisp; onSubmitMisp = e => { e.preventDefault(); @@ -315,14 +297,6 @@ export class Alerts extends Component { handleClose(); }; - const theHiveExportButton = ( - ); - const theHiveUpdateButton = ( - ); const mispExportButton = ( - {this.state.theHiveCaseId ? theHiveUpdateButton : theHiveExportButton} -
{this.state.mispEventId ? mispUpdateButton : mispExportButton}
@@ -411,7 +379,7 @@ export class Alerts extends Component { ); - const theHiveUpdateButton = ( - ); const mispExportButton = ( - {this.state.theHiveCaseId ? theHiveUpdateButton : theHiveExportButton} -
{this.state.mispEventId ? mispUpdateButton : mispExportButton}
@@ -500,7 +467,7 @@ export class SuspiciousSites extends Component {