Merge pull request #1048 from blacklanternsecurity/include-emails-wit…

…h-passwords

Include source emails in credential leaks

bbot/core/event/base.py

@@ -1246,10 +1246,11 @@ def make_event(
     """
 
     # allow tags to be either a string or an array
-    if tags is not None:
-        if isinstance(tags, str):
-            tags = [tags]
-        tags = list(tags)
+    if not tags:
+        tags = []
+    elif isinstance(tags, str):
+        tags = [tags]
+    tags = list(tags)
 
     if is_event(data):
         if scan is not None and not data.scan:

bbot/modules/credshed.py

@@ -1,9 +1,9 @@
 from contextlib import suppress
 
-from bbot.modules.templates.credential_leak import credential_leak
+from bbot.modules.base import BaseModule
 
 
-class credshed(credential_leak):
+class credshed(BaseModule):
     watched_events = ["DNS_NAME"]
     produced_events = ["PASSWORD", "HASHED_PASSWORD", "USERNAME", "EMAIL_ADDRESS"]
     flags = ["passive", "safe"]
@@ -17,6 +17,7 @@ class credshed(credential_leak):
         "password": "Credshed password",
         "credshed_url": "URL of credshed server",
     }
+    target_only = True
 
     async def setup(self):
         self.base_url = self.config.get("credshed_url", "").rstrip("/")
@@ -40,7 +41,7 @@ async def setup(self):
         return await super().setup()
 
     async def handle_event(self, event):
-        query = self.make_query(event)
+        query = event.data
         cs_query = await self.helpers.request(
             f"{self.base_url}/api/search",
             method="POST",
@@ -77,10 +78,10 @@ async def handle_event(self, event):
             email_event = self.make_event(email, "EMAIL_ADDRESS", source=event, tags=tags)
             if email_event is not None:
                 await self.emit_event(email_event)
-                if user and not self.already_seen(f"{email}:{user}"):
-                    await self.emit_event(user, "USERNAME", source=email_event, tags=tags)
-                if pw and not self.already_seen(f"{email}:{pw}"):
-                    await self.emit_event(pw, "PASSWORD", source=email_event, tags=tags)
+                if user:
+                    await self.emit_event(f"{email}:{user}", "USERNAME", source=email_event, tags=tags)
+                if pw:
+                    await self.emit_event(f"{email}:{pw}", "PASSWORD", source=email_event, tags=tags)
                 for h_pw in hashes:
-                    if h_pw and not self.already_seen(f"{email}:{h_pw}"):
-                        await self.emit_event(h_pw, "HASHED_PASSWORD", source=email_event, tags=tags)
+                    if h_pw:
+                        await self.emit_event(f"{email}:{h_pw}", "HASHED_PASSWORD", source=email_event, tags=tags)

bbot/modules/dehashed.py

@@ -1,15 +1,16 @@
 from contextlib import suppress
 
-from bbot.modules.templates.credential_leak import credential_leak
+from bbot.modules.base import BaseModule
 
 
-class dehashed(credential_leak):
+class dehashed(BaseModule):
     watched_events = ["DNS_NAME"]
     produced_events = ["PASSWORD", "HASHED_PASSWORD", "USERNAME"]
     flags = ["passive", "safe", "email-enum"]
     meta = {"description": "Execute queries against dehashed.com for exposed credentials", "auth_required": True}
     options = {"username": "", "api_key": ""}
     options_desc = {"username": "Email Address associated with your API key", "api_key": "DeHashed API Key"}
+    target_only = True
 
     base_url = "https://api.dehashed.com/search"
 
@@ -50,15 +51,15 @@ async def handle_event(self, event):
                     email_event = self.make_event(email, "EMAIL_ADDRESS", source=event, tags=tags)
                     if email_event is not None:
                         await self.emit_event(email_event)
-                        if user and not self.already_seen(f"{email}:{user}"):
-                            await self.emit_event(user, "USERNAME", source=email_event, tags=tags)
-                        if pw and not self.already_seen(f"{email}:{pw}"):
-                            await self.emit_event(pw, "PASSWORD", source=email_event, tags=tags)
-                        if h_pw and not self.already_seen(f"{email}:{h_pw}"):
-                            await self.emit_event(h_pw, "HASHED_PASSWORD", source=email_event, tags=tags)
+                        if user:
+                            await self.emit_event(f"{email}:{user}", "USERNAME", source=email_event, tags=tags)
+                        if pw:
+                            await self.emit_event(f"{email}:{pw}", "PASSWORD", source=email_event, tags=tags)
+                        if h_pw:
+                            await self.emit_event(f"{email}:{h_pw}", "HASHED_PASSWORD", source=email_event, tags=tags)
 
     async def query(self, event):
-        query = f"domain:{self.make_query(event)}"
+        query = f"domain:{event.data}"
         url = f"{self.base_url}?query={query}&size=10000&page=" + "{page}"
         page = 0
         num_entries = 0

bbot/modules/internal/speculate.py

@@ -1,6 +1,7 @@
 import random
 import ipaddress
 
+from bbot.core.helpers import validators
 from bbot.modules.internal.base import BaseInternalModule
 
 
@@ -21,6 +22,7 @@ class speculate(BaseInternalModule):
         "STORAGE_BUCKET",
         "SOCIAL",
         "AZURE_TENANT",
+        "USERNAME",
     ]
     produced_events = ["DNS_NAME", "OPEN_TCP_PORT", "IP_ADDRESS", "FINDING", "ORG_STUB"]
     flags = ["passive"]
@@ -156,6 +158,14 @@ async def handle_event(self, event):
                     stub_event.scope_distance = event.scope_distance
                 await self.emit_event(stub_event)
 
+        # USERNAME --> EMAIL
+        if event.type == "USERNAME":
+            email = event.data.split(":", 1)[-1]
+            if validators.soft_validate(email, "email"):
+                email_event = self.make_event(email, "EMAIL_ADDRESS", source=event, tags=["affiliate"])
+                email_event.scope_distance = event.scope_distance
+                await self.emit_event(email_event)
+
     async def filter_event(self, event):
         # don't accept errored DNS_NAMEs
         if any(t in event.tags for t in ("unresolved", "a-error", "aaaa-error")):

bbot/scanner/manager.py

@@ -275,7 +275,7 @@ async def _emit_event(self, event, **kwargs):
             if (
                 event.host
                 and event.type not in ("DNS_NAME", "DNS_NAME_UNRESOLVED", "IP_ADDRESS", "IP_RANGE")
-                and not str(event.module) == "speculate"
+                and not (event.type in ("OPEN_TCP_PORT", "URL_UNVERIFIED") and str(event.module) == "speculate")
             ):
                 source_module = self.scan.helpers._make_dummy_module("host", _type="internal")
                 source_module._priority = 4

bbot/test/test_step_1/test_events.py

@@ -250,6 +250,10 @@ async def test_events(events, scan, helpers, bbot_config):
     corrected_event3 = scan.make_event("wat.asdf.com", "IP_ADDRESS", dummy=True)
     assert corrected_event3.type == "DNS_NAME"
 
+    corrected_event4 = scan.make_event("[email protected]", "USERNAME", dummy=True)
+    assert corrected_event4.type == "EMAIL_ADDRESS"
+    assert "affiliate" in corrected_event4.tags
+
     test_vuln = scan.make_event(
         {"host": "EVILcorp.com", "severity": "iNfo ", "description": "asdf"}, "VULNERABILITY", dummy=True
     )

bbot/test/test_step_2/module_tests/test_module_credshed.py

@@ -74,18 +74,31 @@ def check(self, module_test, events):
         assert 1 == len([e for e in events if e.type == "EMAIL_ADDRESS" and e.data == "[email protected]"])
         assert 1 == len([e for e in events if e.type == "EMAIL_ADDRESS" and e.data == "[email protected]"])
         assert 1 == len(
-            [e for e in events if e.type == "HASHED_PASSWORD" and e.data == "539FE8942DEADBEEFBC49E6EB2F175AC"]
+            [
+                e
+                for e in events
+                if e.type == "HASHED_PASSWORD"
+                and e.data == "[email protected]:539FE8942DEADBEEFBC49E6EB2F175AC"
+            ]
         )
         assert 1 == len(
-            [e for e in events if e.type == "HASHED_PASSWORD" and e.data == "D2D8F0E9A4A2DEADBEEF1AC80F36D61F"]
+            [
+                e
+                for e in events
+                if e.type == "HASHED_PASSWORD"
+                and e.data == "[email protected]:D2D8F0E9A4A2DEADBEEF1AC80F36D61F"
+            ]
         )
         assert 1 == len(
             [
                 e
                 for e in events
                 if e.type == "HASHED_PASSWORD"
-                and e.data == "$2a$12$SHIC49jLIwsobdeadbeefuWb2BKWHUOk2yhpD77A0itiZI1vJqXHm"
+                and e.data
+                == "[email protected]:$2a$12$SHIC49jLIwsobdeadbeefuWb2BKWHUOk2yhpD77A0itiZI1vJqXHm"
             ]
         )
-        assert 1 == len([e for e in events if e.type == "PASSWORD" and e.data == "TimTamSlam69"])
-        assert 1 == len([e for e in events if e.type == "USERNAME" and e.data == "tim"])
+        assert 1 == len(
+            [e for e in events if e.type == "PASSWORD" and e.data == "[email protected]:TimTamSlam69"]
+        )
+        assert 1 == len([e for e in events if e.type == "USERNAME" and e.data == "[email protected]:tim"])

bbot/test/test_step_2/module_tests/test_module_dehashed.py

@@ -37,7 +37,11 @@
 
 
 class TestDehashed(ModuleTestBase):
-    config_overrides = {"modules": {"dehashed": {"username": "admin", "api_key": "deadbeef"}}}
+    modules_overrides = ["dehashed", "speculate"]
+    config_overrides = {
+        "scope_report_distance": 2,
+        "modules": {"dehashed": {"username": "admin", "api_key": "deadbeef"}},
+    }
 
     async def setup_before_prep(self, module_test):
         module_test.httpx_mock.add_response(
@@ -46,8 +50,9 @@ async def setup_before_prep(self, module_test):
         )
 
     def check(self, module_test, events):
-        assert len(events) == 9
-        assert 1 == len([e for e in events if e.type == "EMAIL_ADDRESS" and e.data == "[email protected]"])
+        assert len(events) == 11
+        assert 1 == len([e for e in events if e.type == "DNS_NAME" and e.data == "blacklanternsecurity.com"])
+        assert 1 == len([e for e in events if e.type == "ORG_STUB" and e.data == "blacklanternsecurity"])
         assert 1 == len(
             [
                 e
@@ -56,6 +61,22 @@ def check(self, module_test, events):
                 and e.data == "[email protected]"
                 and e.scope_distance == 1
                 and "affiliate" in e.tags
+            ]
+        )
+        assert 1 == len(
+            [
+                e
+                for e in events
+                if e.type == "DNS_NAME" and e.data == "bob.com" and e.scope_distance == 1 and "affiliate" in e.tags
+            ]
+        )
+        assert 1 == len([e for e in events if e.type == "EMAIL_ADDRESS" and e.data == "[email protected]"])
+        assert 1 == len(
+            [
+                e
+                for e in events
+                if e.type == "USERNAME"
+                and e.data == "[email protected]:[email protected]"
                 and e.source.data == "[email protected]"
             ]
         )
@@ -65,8 +86,11 @@ def check(self, module_test, events):
                 e
                 for e in events
                 if e.type == "HASHED_PASSWORD"
-                and e.data == "$2a$12$pVmwJ7pXEr3mE.DmCCE4fOUDdeadbeefd2KuCy/tq1ZUFyEOH2bve"
+                and e.data
+                == "[email protected]:$2a$12$pVmwJ7pXEr3mE.DmCCE4fOUDdeadbeefd2KuCy/tq1ZUFyEOH2bve"
             ]
         )
-        assert 1 == len([e for e in events if e.type == "PASSWORD" and e.data == "TimTamSlam69"])
-        assert 1 == len([e for e in events if e.type == "USERNAME" and e.data == "timmy"])
+        assert 1 == len(
+            [e for e in events if e.type == "PASSWORD" and e.data == "[email protected]:TimTamSlam69"]
+        )
+        assert 1 == len([e for e in events if e.type == "USERNAME" and e.data == "[email protected]:timmy"])