Merge pull request #1072 from blacklanternsecurity/newsletters

Newsletters Module

.gitignore

@@ -1 +1,2 @@
 __pycache__/
+.coverage*

bbot/core/event/base.py

@@ -29,6 +29,7 @@
     split_host_port,
     tagify,
     validators,
+    truncate_string,
 )
 
 
@@ -489,7 +490,7 @@ def data_human(self):
         return self._data_human()
 
     def _data_human(self):
-        return str(self.data)
+        return truncate_string(str(self.data), n=2000)
 
     def _data_load(self, data):
         """

bbot/modules/newsletters.py

@@ -0,0 +1,46 @@
+# Created a new module called 'newsletters' that will scrape the websites (or recursive websites,
+# thanks to BBOT's sub-domain enumeration) looking for the presence of an 'email type' that also
+# contains a 'placeholder'. The combination of these two HTML items usually signify the presence
+# of an "Enter Your Email Here" type Newsletter Subscription service. This module could be used
+# to find newsletters for a future email bombing attack and/or find user-input fields that could
+# be be susceptible to overflows or injections.
+
+from .base import BaseModule
+import re
+from bs4 import BeautifulSoup
+
+# Known Websites with Newsletters
+# https://futureparty.com/
+# https://www.marketingbrew.com/
+# https://buffer.com/
+# https://www.milkkarten.net/
+# https://geekout.mattnavarra.com/
+
+deps_pip = ["beautifulsoup4"]
+
+
+class newsletters(BaseModule):
+    watched_events = ["HTTP_RESPONSE"]
+    produced_events = ["FINDING"]
+    flags = ["active", "safe"]
+    meta = {"description": "Searches for Newsletter Submission Entry Fields on Websites"}
+
+    # Parse through Website to find a Text Entry Box of 'type = email'
+    # and ensure that there is placeholder text within it.
+    def find_type(self, soup):
+        email_type = soup.find(type="email")
+        if email_type:
+            regex = re.compile(r"placeholder")
+            if regex.search(str(email_type)):
+                return True
+        return False
+
+    async def handle_event(self, event):
+        if event.data["status_code"] == 200:
+            soup = BeautifulSoup(event.data["body"], "html.parser")
+            result = self.find_type(soup)
+            if result:
+                description = f"Found a Newsletter Submission Form that could be used for email bombing attacks"
+                data = {"host": str(event.host), "description": description, "url": event.data["url"]}
+
+                await self.emit_event(data, "FINDING", event)

bbot/test/test_step_2/module_tests/base.py

@@ -47,6 +47,7 @@ class ModuleTestBase:
     module_name = None
     config_overrides = {}
     modules_overrides = []
+    log = logging.getLogger("bbot")
 
     class ModuleTest:
         def __init__(self, module_test_base, httpx_mock, httpserver, httpserver_ssl, monkeypatch, request):

bbot/test/test_step_2/module_tests/test_module_excavate.py

@@ -313,7 +313,6 @@ async def setup_before_prep(self, module_test):
         )
 
     def check(self, module_test, events):
-
         for serialize_type in ["Java", ".NET", "PHP (Array)", "PHP (String)", "PHP (Object)", "Possible Compressed"]:
             assert any(
                 e.type == "FINDING" and serialize_type in e.data["description"] for e in events

bbot/test/test_step_2/module_tests/test_module_newsletters.py

@@ -0,0 +1,57 @@
+from .base import ModuleTestBase
+
+# import logging
+
+
+class TestNewsletters(ModuleTestBase):
+    found_tgt = "http://127.0.0.1:8888/found"
+    missing_tgt = "http://127.0.0.1:8888/missing"
+    targets = [found_tgt, missing_tgt]
+    modules_overrides = ["speculate", "httpx", "newsletters"]
+
+    html_with_newsletter = """
+    <input aria-required="true" 
+    class="form-input form-input-text required" 
+    data-at="form-email" 
+    data-describedby="form-validation-error-box-element-5" 
+    data-label-inside="Enter your email" 
+    id="field-5f329905b4bfe1027b44513f94b50363-0" 
+    name="Enter your email" 
+    placeholder="Enter your email" 
+    required="" 
+    title="Enter your email" 
+    type="email" value=""/>
+    """
+
+    html_without_newsletter = """
+    <div>
+    <h1>Example Domain</h1>
+    <p>This domain is for use in illustrative examples in documents. You may use this
+    domain in literature without prior coordination or asking for permission.</p>
+    <p><a href="https://www.iana.org/domains/example">More information...</a></p>
+    </div>
+    """
+
+    async def setup_after_prep(self, module_test):
+        request_args = dict(uri="/found", headers={"test": "header"})
+        respond_args = dict(response_data=self.html_with_newsletter)
+        module_test.set_expect_requests(request_args, respond_args)
+        request_args = dict(uri="/missing", headers={"test": "header"})
+        respond_args = dict(response_data=self.html_without_newsletter)
+        module_test.set_expect_requests(request_args, respond_args)
+
+    def check(self, module_test, events):
+        found = False
+        missing = True
+        for event in events:
+            # self.log.info(f"event type: {event.type}")
+            if event.type == "FINDING":
+                # self.log.info(f"event data: {event.data}")
+                # Verify Positive Result
+                if event.data["url"] == self.found_tgt:
+                    found = True
+                # Verify Negative Result (should skip this statement if correct)
+                elif event.data["url"] == self.missing_tgt:
+                    missing = False
+        assert found, f"NEWSLETTER 'Found' Error - Expect status of True but got False"
+        assert missing, f"NEWSLETTER 'Missing' Error - Expect status of True but got False"