Merge branch 'dev' into baddns_module

.gitignore

@@ -1 +1,2 @@
 __pycache__/
+.coverage*

bbot/core/event/base.py

@@ -29,6 +29,7 @@
     split_host_port,
     tagify,
     validators,
+    truncate_string,
 )
 
 
@@ -489,7 +490,7 @@ def data_human(self):
         return self._data_human()
 
     def _data_human(self):
-        return str(self.data)
+        return truncate_string(str(self.data), n=2000)
 
     def _data_load(self, data):
         """

bbot/core/helpers/dns.py

@@ -334,6 +334,10 @@ async def _resolve_hostname(self, query, **kwargs):
 
         if results:
             self._last_dns_success = time.time()
+            self.debug(f"Answers for {query} with kwargs={kwargs}: {list(results)}")
+
+        if errors:
+            self.debug(f"Errors for {query} with kwargs={kwargs}: {errors}")
 
         return results, errors
 
@@ -1038,5 +1042,6 @@ def _get_dummy_module(self, name):
             dummy_module = self._dummy_modules[name]
         except KeyError:
             dummy_module = self.parent_helper._make_dummy_module(name=name, _type="DNS")
+            dummy_module.suppress_dupes = False
             self._dummy_modules[name] = dummy_module
         return dummy_module

bbot/modules/ajaxpro.py

@@ -17,22 +17,23 @@ async def handle_event(self, event):
         if event.type == "URL":
             if "dir" not in event.tags:
                 return False
-            probe_url = f"{event.data}ajaxpro/whatever.ashx"
-            probe = await self.helpers.request(probe_url)
-            if probe:
-                if probe.status_code == 200:
-                    probe_confirm = await self.helpers.request(f"{event.data}a/whatever.ashx")
-                    if probe_confirm:
-                        if probe_confirm.status_code != 200:
-                            await self.emit_event(
-                                {
-                                    "host": str(event.host),
-                                    "url": event.data,
-                                    "description": f"Ajaxpro Detected (Version Unconfirmed) Trigger: [{probe_url}]",
-                                },
-                                "FINDING",
-                                event,
-                            )
+            for stem in ["ajax", "ajaxpro"]:
+                probe_url = f"{event.data}{stem}/whatever.ashx"
+                probe = await self.helpers.request(probe_url)
+                if probe:
+                    if probe.status_code == 200:
+                        probe_confirm = await self.helpers.request(f"{event.data}a/whatever.ashx")
+                        if probe_confirm:
+                            if probe_confirm.status_code != 200:
+                                await self.emit_event(
+                                    {
+                                        "host": str(event.host),
+                                        "url": event.data,
+                                        "description": f"Ajaxpro Detected (Version Unconfirmed) Trigger: [{probe_url}]",
+                                    },
+                                    "FINDING",
+                                    event,
+                                )
 
         elif event.type == "HTTP_RESPONSE":
             resp_body = event.data.get("body", None)

bbot/modules/newsletters.py

@@ -0,0 +1,46 @@
+# Created a new module called 'newsletters' that will scrape the websites (or recursive websites,
+# thanks to BBOT's sub-domain enumeration) looking for the presence of an 'email type' that also
+# contains a 'placeholder'. The combination of these two HTML items usually signify the presence
+# of an "Enter Your Email Here" type Newsletter Subscription service. This module could be used
+# to find newsletters for a future email bombing attack and/or find user-input fields that could
+# be be susceptible to overflows or injections.
+
+from .base import BaseModule
+import re
+from bs4 import BeautifulSoup
+
+# Known Websites with Newsletters
+# https://futureparty.com/
+# https://www.marketingbrew.com/
+# https://buffer.com/
+# https://www.milkkarten.net/
+# https://geekout.mattnavarra.com/
+
+deps_pip = ["beautifulsoup4"]
+
+
+class newsletters(BaseModule):
+    watched_events = ["HTTP_RESPONSE"]
+    produced_events = ["FINDING"]
+    flags = ["active", "safe"]
+    meta = {"description": "Searches for Newsletter Submission Entry Fields on Websites"}
+
+    # Parse through Website to find a Text Entry Box of 'type = email'
+    # and ensure that there is placeholder text within it.
+    def find_type(self, soup):
+        email_type = soup.find(type="email")
+        if email_type:
+            regex = re.compile(r"placeholder")
+            if regex.search(str(email_type)):
+                return True
+        return False
+
+    async def handle_event(self, event):
+        if event.data["status_code"] == 200:
+            soup = BeautifulSoup(event.data["body"], "html.parser")
+            result = self.find_type(soup)
+            if result:
+                description = f"Found a Newsletter Submission Form that could be used for email bombing attacks"
+                data = {"host": str(event.host), "description": description, "url": event.data["url"]}
+
+                await self.emit_event(data, "FINDING", event)

bbot/modules/wafw00f.py

@@ -26,9 +26,15 @@ class wafw00f(BaseModule):
     in_scope_only = True
     per_hostport_only = True
 
+    async def filter_event(self, event):
+        http_status = getattr(event, "http_status", 0)
+        if not http_status or str(http_status).startswith("3"):
+            return False, f"Invalid HTTP status code: {http_status}"
+        return True, ""
+
     async def handle_event(self, event):
         url = f"{event.parsed.scheme}://{event.parsed.netloc}/"
-        WW = await self.scan.run_in_executor(wafw00f_main.WAFW00F, url)
+        WW = await self.scan.run_in_executor(wafw00f_main.WAFW00F, url, followredirect=False)
         waf_detections = await self.scan.run_in_executor(WW.identwaf)
         if waf_detections:
             for waf in waf_detections:

bbot/scanner/manager.py

@@ -319,6 +319,7 @@ async def _emit_event(self, event, **kwargs):
                                         f'Event validation failed for DNS child of {source_event}: "{record}" ({rdtype}): {e}'
                                     )
                     for child_event in dns_child_events:
+                        log.debug(f"Queueing DNS child for {event}: {child_event}")
                         self.queue_event(child_event)
 
         except ValidationError as e:

bbot/test/test_step_2/module_tests/base.py

@@ -47,6 +47,7 @@ class ModuleTestBase:
     module_name = None
     config_overrides = {}
     modules_overrides = []
+    log = logging.getLogger("bbot")
 
     class ModuleTest:
         def __init__(self, module_test_base, httpx_mock, httpserver, httpserver_ssl, monkeypatch, request):

bbot/test/test_step_2/module_tests/test_module_excavate.py

@@ -313,7 +313,6 @@ async def setup_before_prep(self, module_test):
         )
 
     def check(self, module_test, events):
-
         for serialize_type in ["Java", ".NET", "PHP (Array)", "PHP (String)", "PHP (Object)", "Possible Compressed"]:
             assert any(
                 e.type == "FINDING" and serialize_type in e.data["description"] for e in events

bbot/test/test_step_2/module_tests/test_module_newsletters.py

@@ -0,0 +1,57 @@
+from .base import ModuleTestBase
+
+# import logging
+
+
+class TestNewsletters(ModuleTestBase):
+    found_tgt = "http://127.0.0.1:8888/found"
+    missing_tgt = "http://127.0.0.1:8888/missing"
+    targets = [found_tgt, missing_tgt]
+    modules_overrides = ["speculate", "httpx", "newsletters"]
+
+    html_with_newsletter = """
+    <input aria-required="true" 
+    class="form-input form-input-text required" 
+    data-at="form-email" 
+    data-describedby="form-validation-error-box-element-5" 
+    data-label-inside="Enter your email" 
+    id="field-5f329905b4bfe1027b44513f94b50363-0" 
+    name="Enter your email" 
+    placeholder="Enter your email" 
+    required="" 
+    title="Enter your email" 
+    type="email" value=""/>
+    """
+
+    html_without_newsletter = """
+    <div>
+    <h1>Example Domain</h1>
+    <p>This domain is for use in illustrative examples in documents. You may use this
+    domain in literature without prior coordination or asking for permission.</p>
+    <p><a href="https://www.iana.org/domains/example">More information...</a></p>
+    </div>
+    """
+
+    async def setup_after_prep(self, module_test):
+        request_args = dict(uri="/found", headers={"test": "header"})
+        respond_args = dict(response_data=self.html_with_newsletter)
+        module_test.set_expect_requests(request_args, respond_args)
+        request_args = dict(uri="/missing", headers={"test": "header"})
+        respond_args = dict(response_data=self.html_without_newsletter)
+        module_test.set_expect_requests(request_args, respond_args)
+
+    def check(self, module_test, events):
+        found = False
+        missing = True
+        for event in events:
+            # self.log.info(f"event type: {event.type}")
+            if event.type == "FINDING":
+                # self.log.info(f"event data: {event.data}")
+                # Verify Positive Result
+                if event.data["url"] == self.found_tgt:
+                    found = True
+                # Verify Negative Result (should skip this statement if correct)
+                elif event.data["url"] == self.missing_tgt:
+                    missing = False
+        assert found, f"NEWSLETTER 'Found' Error - Expect status of True but got False"
+        assert missing, f"NEWSLETTER 'Missing' Error - Expect status of True but got False"

bbot/test/test_step_2/module_tests/test_module_wafw00f.py

@@ -12,3 +12,19 @@ async def setup_after_prep(self, module_test):
 
     def check(self, module_test, events):
         assert any(e.type == "WAF" and "LiteSpeed" in e.data["WAF"] for e in events)
+
+
+class TestWafw00f_noredirect(ModuleTestBase):
+    targets = ["http://127.0.0.1:8888"]
+    modules_overrides = ["httpx", "wafw00f"]
+
+    async def setup_after_prep(self, module_test):
+        expect_args = {"method": "GET", "uri": "/"}
+        respond_args = {"status": 301, "headers": {"Location": "/redirect"}}
+        module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args)
+        expect_args = {"method": "GET", "uri": "/redirect"}
+        respond_args = {"response_data": "Proudly powered by litespeed web server"}
+        module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args)
+
+    def check(self, module_test, events):
+        assert not any(e.type == "WAF" for e in events)

docs/modules/list_of_modules.md

@@ -25,6 +25,7 @@
 | hunt                 | scan     | No              | Watch for commonly-exploitable HTTP parameters                                                                                          | active, safe, web-thorough                                                          | HTTP_RESPONSE                                                                                                                           | FINDING                                                     |
 | iis_shortnames       | scan     | No              | Check for IIS shortname vulnerability                                                                                                   | active, iis-shortnames, safe, web-basic, web-thorough                               | URL                                                                                                                                     | URL_HINT                                                    |
 | masscan              | scan     | No              | Port scan with masscan. By default, scans top 100 ports.                                                                                | active, aggressive, portscan                                                        | IP_ADDRESS, IP_RANGE                                                                                                                    | OPEN_TCP_PORT                                               |
+| newsletters          | scan     | No              | Searches for Newsletter Submission Entry Fields on Websites                                                                             | active, safe                                                                        | HTTP_RESPONSE                                                                                                                           | FINDING                                                     |
 | nmap                 | scan     | No              | Port scan with nmap. By default, scans top 100 ports.                                                                                   | active, aggressive, portscan, web-thorough                                          | DNS_NAME, IP_ADDRESS, IP_RANGE                                                                                                          | OPEN_TCP_PORT                                               |
 | ntlm                 | scan     | No              | Watch for HTTP endpoints that support NTLM authentication                                                                               | active, safe, web-basic, web-thorough                                               | HTTP_RESPONSE, URL                                                                                                                      | DNS_NAME, FINDING                                           |
 | nuclei               | scan     | No              | Fast and customisable vulnerability scanner                                                                                             | active, aggressive, deadly                                                          | URL                                                                                                                                     | FINDING, VULNERABILITY                                      |

docs/scanning/advanced.md

@@ -61,7 +61,7 @@ Target:
 
 Modules:
   -m MODULE [MODULE ...], --modules MODULE [MODULE ...]
-                        Modules to enable. Choices: affiliates,ajaxpro,anubisdb,asn,azure_realm,azure_tenant,badsecrets,bevigil,binaryedge,bucket_amazon,bucket_azure,bucket_digitalocean,bucket_file_enum,bucket_firebase,bucket_google,builtwith,bypass403,c99,censys,certspotter,chaos,columbus,credshed,crobat,crt,dastardly,dehashed,digitorus,dnscommonsrv,dnsdumpster,dnszonetransfer,emailformat,ffuf,ffuf_shortnames,filedownload,fingerprintx,fullhunt,generic_ssrf,git,github_codesearch,github_org,gowitness,hackertarget,host_header,httpx,hunt,hunterio,iis_shortnames,internetdb,ip2location,ipneighbor,ipstack,leakix,masscan,massdns,myssl,nmap,nsec,ntlm,nuclei,oauth,otx,paramminer_cookies,paramminer_getparams,paramminer_headers,passivetotal,pgp,postman,rapiddns,riddler,robots,secretsdb,securitytrails,shodan_dns,sitedossier,skymem,smuggler,social,sslcert,subdomain_hijack,subdomaincenter,sublist3r,telerik,threatminer,url_manipulation,urlscan,vhost,viewdns,virustotal,wafw00f,wappalyzer,wayback,zoomeye
+                        Modules to enable. Choices: affiliates,ajaxpro,anubisdb,asn,azure_realm,azure_tenant,badsecrets,bevigil,binaryedge,bucket_amazon,bucket_azure,bucket_digitalocean,bucket_file_enum,bucket_firebase,bucket_google,builtwith,bypass403,c99,censys,certspotter,chaos,columbus,credshed,crobat,crt,dastardly,dehashed,digitorus,dnscommonsrv,dnsdumpster,dnszonetransfer,emailformat,ffuf,ffuf_shortnames,filedownload,fingerprintx,fullhunt,generic_ssrf,git,github_codesearch,github_org,gowitness,hackertarget,host_header,httpx,hunt,hunterio,iis_shortnames,internetdb,ip2location,ipneighbor,ipstack,leakix,masscan,massdns,myssl,newsletters,nmap,nsec,ntlm,nuclei,oauth,otx,paramminer_cookies,paramminer_getparams,paramminer_headers,passivetotal,pgp,postman,rapiddns,riddler,robots,secretsdb,securitytrails,shodan_dns,sitedossier,skymem,smuggler,social,sslcert,subdomain_hijack,subdomaincenter,sublist3r,telerik,threatminer,url_manipulation,urlscan,vhost,viewdns,virustotal,wafw00f,wappalyzer,wayback,zoomeye
   -l, --list-modules    List available modules.
   -em MODULE [MODULE ...], --exclude-modules MODULE [MODULE ...]
                         Exclude these modules.