From ec1ea92ef8b739b9e455bbbe4bbf2ad6183256a8 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Thu, 1 Feb 2024 13:44:53 -0500 Subject: [PATCH] include source email address in PASSWORD, USERNAME, and HASHED_PASSWORD --- bbot/core/event/base.py | 9 ++--- bbot/modules/credshed.py | 19 +++++----- bbot/modules/dehashed.py | 19 +++++----- bbot/modules/internal/speculate.py | 10 ++++++ bbot/modules/templates/credential_leak.py | 33 ----------------- bbot/scanner/manager.py | 2 +- bbot/test/test_step_1/test_events.py | 4 +++ .../module_tests/test_module_credshed.py | 23 +++++++++--- .../module_tests/test_module_dehashed.py | 36 +++++++++++++++---- 9 files changed, 88 insertions(+), 67 deletions(-) delete mode 100644 bbot/modules/templates/credential_leak.py diff --git a/bbot/core/event/base.py b/bbot/core/event/base.py index f46faad012..895b8e2d5f 100644 --- a/bbot/core/event/base.py +++ b/bbot/core/event/base.py @@ -1246,10 +1246,11 @@ def make_event( """ # allow tags to be either a string or an array - if tags is not None: - if isinstance(tags, str): - tags = [tags] - tags = list(tags) + if not tags: + tags = [] + elif isinstance(tags, str): + tags = [tags] + tags = list(tags) if is_event(data): if scan is not None and not data.scan: diff --git a/bbot/modules/credshed.py b/bbot/modules/credshed.py index 09ed57377c..3826440077 100644 --- a/bbot/modules/credshed.py +++ b/bbot/modules/credshed.py @@ -1,9 +1,9 @@ from contextlib import suppress -from bbot.modules.templates.credential_leak import credential_leak +from bbot.modules.base import BaseModule -class credshed(credential_leak): +class credshed(BaseModule): watched_events = ["DNS_NAME"] produced_events = ["PASSWORD", "HASHED_PASSWORD", "USERNAME", "EMAIL_ADDRESS"] flags = ["passive", "safe"] @@ -17,6 +17,7 @@ class credshed(credential_leak): "password": "Credshed password", "credshed_url": "URL of credshed server", } + target_only = True async def setup(self): self.base_url = self.config.get("credshed_url", "").rstrip("/") @@ -40,7 +41,7 @@ async def setup(self): return await super().setup() async def handle_event(self, event): - query = self.make_query(event) + query = event.data cs_query = await self.helpers.request( f"{self.base_url}/api/search", method="POST", @@ -77,10 +78,10 @@ async def handle_event(self, event): email_event = self.make_event(email, "EMAIL_ADDRESS", source=event, tags=tags) if email_event is not None: await self.emit_event(email_event) - if user and not self.already_seen(f"{email}:{user}"): - await self.emit_event(user, "USERNAME", source=email_event, tags=tags) - if pw and not self.already_seen(f"{email}:{pw}"): - await self.emit_event(pw, "PASSWORD", source=email_event, tags=tags) + if user: + await self.emit_event(f"{email}:{user}", "USERNAME", source=email_event, tags=tags) + if pw: + await self.emit_event(f"{email}:{pw}", "PASSWORD", source=email_event, tags=tags) for h_pw in hashes: - if h_pw and not self.already_seen(f"{email}:{h_pw}"): - await self.emit_event(h_pw, "HASHED_PASSWORD", source=email_event, tags=tags) + if h_pw: + await self.emit_event(f"{email}:{h_pw}", "HASHED_PASSWORD", source=email_event, tags=tags) diff --git a/bbot/modules/dehashed.py b/bbot/modules/dehashed.py index 4b35467129..c1a35c4195 100644 --- a/bbot/modules/dehashed.py +++ b/bbot/modules/dehashed.py @@ -1,15 +1,16 @@ from contextlib import suppress -from bbot.modules.templates.credential_leak import credential_leak +from bbot.modules.base import BaseModule -class dehashed(credential_leak): +class dehashed(BaseModule): watched_events = ["DNS_NAME"] produced_events = ["PASSWORD", "HASHED_PASSWORD", "USERNAME"] flags = ["passive", "safe", "email-enum"] meta = {"description": "Execute queries against dehashed.com for exposed credentials", "auth_required": True} options = {"username": "", "api_key": ""} options_desc = {"username": "Email Address associated with your API key", "api_key": "DeHashed API Key"} + target_only = True base_url = "https://api.dehashed.com/search" @@ -50,15 +51,15 @@ async def handle_event(self, event): email_event = self.make_event(email, "EMAIL_ADDRESS", source=event, tags=tags) if email_event is not None: await self.emit_event(email_event) - if user and not self.already_seen(f"{email}:{user}"): - await self.emit_event(user, "USERNAME", source=email_event, tags=tags) - if pw and not self.already_seen(f"{email}:{pw}"): - await self.emit_event(pw, "PASSWORD", source=email_event, tags=tags) - if h_pw and not self.already_seen(f"{email}:{h_pw}"): - await self.emit_event(h_pw, "HASHED_PASSWORD", source=email_event, tags=tags) + if user: + await self.emit_event(f"{email}:{user}", "USERNAME", source=email_event, tags=tags) + if pw: + await self.emit_event(f"{email}:{pw}", "PASSWORD", source=email_event, tags=tags) + if h_pw: + await self.emit_event(f"{email}:{h_pw}", "HASHED_PASSWORD", source=email_event, tags=tags) async def query(self, event): - query = f"domain:{self.make_query(event)}" + query = f"domain:{event.data}" url = f"{self.base_url}?query={query}&size=10000&page=" + "{page}" page = 0 num_entries = 0 diff --git a/bbot/modules/internal/speculate.py b/bbot/modules/internal/speculate.py index 8f51d5d958..7aaf12d306 100644 --- a/bbot/modules/internal/speculate.py +++ b/bbot/modules/internal/speculate.py @@ -1,6 +1,7 @@ import random import ipaddress +from bbot.core.helpers import validators from bbot.modules.internal.base import BaseInternalModule @@ -21,6 +22,7 @@ class speculate(BaseInternalModule): "STORAGE_BUCKET", "SOCIAL", "AZURE_TENANT", + "USERNAME", ] produced_events = ["DNS_NAME", "OPEN_TCP_PORT", "IP_ADDRESS", "FINDING", "ORG_STUB"] flags = ["passive"] @@ -156,6 +158,14 @@ async def handle_event(self, event): stub_event.scope_distance = event.scope_distance await self.emit_event(stub_event) + # USERNAME --> EMAIL + if event.type == "USERNAME": + email = event.data.split(":", 1)[-1] + if validators.soft_validate(email, "email"): + email_event = self.make_event(email, "EMAIL_ADDRESS", source=event, tags=["affiliate"]) + email_event.scope_distance = event.scope_distance + await self.emit_event(email_event) + async def filter_event(self, event): # don't accept errored DNS_NAMEs if any(t in event.tags for t in ("unresolved", "a-error", "aaaa-error")): diff --git a/bbot/modules/templates/credential_leak.py b/bbot/modules/templates/credential_leak.py deleted file mode 100644 index 5085e197af..0000000000 --- a/bbot/modules/templates/credential_leak.py +++ /dev/null @@ -1,33 +0,0 @@ -from bbot.modules.base import BaseModule - - -class credential_leak(BaseModule): - """ - A typical free API-based subdomain enumeration module - Inherited by many other modules including sublist3r, dnsdumpster, etc. - """ - - async def setup(self): - self.queries_processed = set() - self.data_seen = set() - return True - - async def filter_event(self, event): - query = self.make_query(event) - query_hash = hash(query) - if query_hash not in self.queries_processed: - self.queries_processed.add(query_hash) - return True - return False, f'Already processed "{query}"' - - def make_query(self, event): - if "target" in event.tags: - return event.data - _, domain = self.helpers.split_domain(event.data) - return domain - - def already_seen(self, item): - h = hash(item) - already_seen = h in self.data_seen - self.data_seen.add(h) - return already_seen diff --git a/bbot/scanner/manager.py b/bbot/scanner/manager.py index a511e63165..c0f5982304 100644 --- a/bbot/scanner/manager.py +++ b/bbot/scanner/manager.py @@ -275,7 +275,7 @@ async def _emit_event(self, event, **kwargs): if ( event.host and event.type not in ("DNS_NAME", "DNS_NAME_UNRESOLVED", "IP_ADDRESS", "IP_RANGE") - and not str(event.module) == "speculate" + and not (event.type in ("OPEN_TCP_PORT", "URL_UNVERIFIED") and str(event.module) == "speculate") ): source_module = self.scan.helpers._make_dummy_module("host", _type="internal") source_module._priority = 4 diff --git a/bbot/test/test_step_1/test_events.py b/bbot/test/test_step_1/test_events.py index 0f30f64986..ccbf17c1f1 100644 --- a/bbot/test/test_step_1/test_events.py +++ b/bbot/test/test_step_1/test_events.py @@ -250,6 +250,10 @@ async def test_events(events, scan, helpers, bbot_config): corrected_event3 = scan.make_event("wat.asdf.com", "IP_ADDRESS", dummy=True) assert corrected_event3.type == "DNS_NAME" + corrected_event4 = scan.make_event("bob@evilcorp.com", "USERNAME", dummy=True) + assert corrected_event4.type == "EMAIL_ADDRESS" + assert "affiliate" in corrected_event4.tags + test_vuln = scan.make_event( {"host": "EVILcorp.com", "severity": "iNfo ", "description": "asdf"}, "VULNERABILITY", dummy=True ) diff --git a/bbot/test/test_step_2/module_tests/test_module_credshed.py b/bbot/test/test_step_2/module_tests/test_module_credshed.py index 7de6424123..4b95660779 100644 --- a/bbot/test/test_step_2/module_tests/test_module_credshed.py +++ b/bbot/test/test_step_2/module_tests/test_module_credshed.py @@ -74,18 +74,31 @@ def check(self, module_test, events): assert 1 == len([e for e in events if e.type == "EMAIL_ADDRESS" and e.data == "judy@blacklanternsecurity.com"]) assert 1 == len([e for e in events if e.type == "EMAIL_ADDRESS" and e.data == "tim@blacklanternsecurity.com"]) assert 1 == len( - [e for e in events if e.type == "HASHED_PASSWORD" and e.data == "539FE8942DEADBEEFBC49E6EB2F175AC"] + [ + e + for e in events + if e.type == "HASHED_PASSWORD" + and e.data == "judy@blacklanternsecurity.com:539FE8942DEADBEEFBC49E6EB2F175AC" + ] ) assert 1 == len( - [e for e in events if e.type == "HASHED_PASSWORD" and e.data == "D2D8F0E9A4A2DEADBEEF1AC80F36D61F"] + [ + e + for e in events + if e.type == "HASHED_PASSWORD" + and e.data == "judy@blacklanternsecurity.com:D2D8F0E9A4A2DEADBEEF1AC80F36D61F" + ] ) assert 1 == len( [ e for e in events if e.type == "HASHED_PASSWORD" - and e.data == "$2a$12$SHIC49jLIwsobdeadbeefuWb2BKWHUOk2yhpD77A0itiZI1vJqXHm" + and e.data + == "judy@blacklanternsecurity.com:$2a$12$SHIC49jLIwsobdeadbeefuWb2BKWHUOk2yhpD77A0itiZI1vJqXHm" ] ) - assert 1 == len([e for e in events if e.type == "PASSWORD" and e.data == "TimTamSlam69"]) - assert 1 == len([e for e in events if e.type == "USERNAME" and e.data == "tim"]) + assert 1 == len( + [e for e in events if e.type == "PASSWORD" and e.data == "tim@blacklanternsecurity.com:TimTamSlam69"] + ) + assert 1 == len([e for e in events if e.type == "USERNAME" and e.data == "tim@blacklanternsecurity.com:tim"]) diff --git a/bbot/test/test_step_2/module_tests/test_module_dehashed.py b/bbot/test/test_step_2/module_tests/test_module_dehashed.py index 8b20c85c59..ab1cc20aa7 100644 --- a/bbot/test/test_step_2/module_tests/test_module_dehashed.py +++ b/bbot/test/test_step_2/module_tests/test_module_dehashed.py @@ -37,7 +37,11 @@ class TestDehashed(ModuleTestBase): - config_overrides = {"modules": {"dehashed": {"username": "admin", "api_key": "deadbeef"}}} + modules_overrides = ["dehashed", "speculate"] + config_overrides = { + "scope_report_distance": 2, + "modules": {"dehashed": {"username": "admin", "api_key": "deadbeef"}}, + } async def setup_before_prep(self, module_test): module_test.httpx_mock.add_response( @@ -46,8 +50,9 @@ async def setup_before_prep(self, module_test): ) def check(self, module_test, events): - assert len(events) == 9 - assert 1 == len([e for e in events if e.type == "EMAIL_ADDRESS" and e.data == "bob@blacklanternsecurity.com"]) + assert len(events) == 11 + assert 1 == len([e for e in events if e.type == "DNS_NAME" and e.data == "blacklanternsecurity.com"]) + assert 1 == len([e for e in events if e.type == "ORG_STUB" and e.data == "blacklanternsecurity"]) assert 1 == len( [ e @@ -56,6 +61,22 @@ def check(self, module_test, events): and e.data == "bob@bob.com" and e.scope_distance == 1 and "affiliate" in e.tags + ] + ) + assert 1 == len( + [ + e + for e in events + if e.type == "DNS_NAME" and e.data == "bob.com" and e.scope_distance == 1 and "affiliate" in e.tags + ] + ) + assert 1 == len([e for e in events if e.type == "EMAIL_ADDRESS" and e.data == "bob@blacklanternsecurity.com"]) + assert 1 == len( + [ + e + for e in events + if e.type == "USERNAME" + and e.data == "bob@blacklanternsecurity.com:bob@bob.com" and e.source.data == "bob@blacklanternsecurity.com" ] ) @@ -65,8 +86,11 @@ def check(self, module_test, events): e for e in events if e.type == "HASHED_PASSWORD" - and e.data == "$2a$12$pVmwJ7pXEr3mE.DmCCE4fOUDdeadbeefd2KuCy/tq1ZUFyEOH2bve" + and e.data + == "bob@blacklanternsecurity.com:$2a$12$pVmwJ7pXEr3mE.DmCCE4fOUDdeadbeefd2KuCy/tq1ZUFyEOH2bve" ] ) - assert 1 == len([e for e in events if e.type == "PASSWORD" and e.data == "TimTamSlam69"]) - assert 1 == len([e for e in events if e.type == "USERNAME" and e.data == "timmy"]) + assert 1 == len( + [e for e in events if e.type == "PASSWORD" and e.data == "tim@blacklanternsecurity.com:TimTamSlam69"] + ) + assert 1 == len([e for e in events if e.type == "USERNAME" and e.data == "tim@blacklanternsecurity.com:timmy"])