From e7c716c74efc32614df61cdb07cc77e4a984c5bd Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Fri, 19 Jan 2024 11:31:39 -0500 Subject: [PATCH 1/8] asyncify emit_event, default qsize=100 --- bbot/modules/ajaxpro.py | 4 +- bbot/modules/azure_realm.py | 2 +- bbot/modules/azure_tenant.py | 4 +- bbot/modules/badsecrets.py | 6 +-- bbot/modules/base.py | 38 ++++---------- bbot/modules/bevigil.py | 4 +- bbot/modules/bucket_file_enum.py | 2 +- bbot/modules/builtwith.py | 4 +- bbot/modules/bypass403.py | 4 +- bbot/modules/credshed.py | 8 +-- bbot/modules/deadly/dastardly.py | 4 +- bbot/modules/deadly/ffuf.py | 2 +- bbot/modules/deadly/nuclei.py | 4 +- bbot/modules/deadly/vhost.py | 4 +- bbot/modules/dehashed.py | 8 +-- bbot/modules/dnscommonsrv.py | 2 +- bbot/modules/dnszonetransfer.py | 8 +-- bbot/modules/emailformat.py | 2 +- bbot/modules/ffuf_shortnames.py | 16 +++--- bbot/modules/fingerprintx.py | 2 +- bbot/modules/generic_ssrf.py | 2 +- bbot/modules/git.py | 2 +- bbot/modules/github_codesearch.py | 4 +- bbot/modules/github_org.py | 6 +-- bbot/modules/gowitness.py | 6 +-- bbot/modules/host_header.py | 8 +-- bbot/modules/httpx.py | 4 +- bbot/modules/hunt.py | 2 +- bbot/modules/hunterio.py | 6 +-- bbot/modules/iis_shortnames.py | 4 +- bbot/modules/internal/excavate.py | 50 +++++++++---------- bbot/modules/internal/speculate.py | 12 ++--- bbot/modules/internetdb.py | 12 ++--- bbot/modules/ip2location.py | 2 +- bbot/modules/ipneighbor.py | 2 +- bbot/modules/ipstack.py | 2 +- bbot/modules/masscan.py | 12 ++--- bbot/modules/massdns.py | 10 ++-- bbot/modules/nmap.py | 4 +- bbot/modules/nsec.py | 4 +- bbot/modules/ntlm.py | 4 +- bbot/modules/oauth.py | 8 +-- bbot/modules/output/asset_inventory.py | 8 +-- bbot/modules/paramminer_headers.py | 2 +- bbot/modules/pgp.py | 2 +- bbot/modules/postman.py | 2 +- bbot/modules/report/asn.py | 4 +- bbot/modules/robots.py | 2 +- bbot/modules/secretsdb.py | 2 +- bbot/modules/sitedossier.py | 2 +- bbot/modules/skymem.py | 4 +- bbot/modules/smuggler.py | 2 +- bbot/modules/social.py | 2 +- bbot/modules/sslcert.py | 2 +- bbot/modules/subdomain_hijack.py | 4 +- bbot/modules/telerik.py | 14 +++--- bbot/modules/templates/bucket.py | 6 +-- bbot/modules/templates/subdomain_enum.py | 2 +- bbot/modules/url_manipulation.py | 2 +- bbot/modules/urlscan.py | 8 +-- bbot/modules/viewdns.py | 2 +- bbot/modules/wafw00f.py | 4 +- bbot/modules/wappalyzer.py | 2 +- bbot/modules/wayback.py | 2 +- bbot/modules/zoomeye.py | 2 +- .../test_step_1/test_manager_deduplication.py | 4 +- .../test_manager_scope_accuracy.py | 2 +- bbot/test/test_step_1/test_modules_basic.py | 2 +- docs/contribution.md | 2 +- 69 files changed, 189 insertions(+), 197 deletions(-) diff --git a/bbot/modules/ajaxpro.py b/bbot/modules/ajaxpro.py index 924f888357..3a00613151 100644 --- a/bbot/modules/ajaxpro.py +++ b/bbot/modules/ajaxpro.py @@ -24,7 +24,7 @@ async def handle_event(self, event): probe_confirm = await self.helpers.request(f"{event.data}a/whatever.ashx") if probe_confirm: if probe_confirm.status_code != 200: - self.emit_event( + await self.emit_event( { "host": str(event.host), "url": event.data, @@ -40,7 +40,7 @@ async def handle_event(self, event): ajaxpro_regex_result = self.ajaxpro_regex.search(resp_body) if ajaxpro_regex_result: ajax_pro_path = ajaxpro_regex_result.group(0) - self.emit_event( + await self.emit_event( { "host": str(event.host), "url": event.data["url"], diff --git a/bbot/modules/azure_realm.py b/bbot/modules/azure_realm.py index f299ca3dc1..a3d6ad6ba3 100644 --- a/bbot/modules/azure_realm.py +++ b/bbot/modules/azure_realm.py @@ -22,7 +22,7 @@ async def handle_event(self, event): auth_url, "URL_UNVERIFIED", source=event, tags=["affiliate", "ms-auth-url"] ) url_event.source_domain = domain - self.emit_event(url_event) + await self.emit_event(url_event) async def getuserrealm(self, domain): url = f"https://login.microsoftonline.com/getuserrealm.srf?login=test@{domain}" diff --git a/bbot/modules/azure_tenant.py b/bbot/modules/azure_tenant.py index a15fbecf46..909acbe205 100644 --- a/bbot/modules/azure_tenant.py +++ b/bbot/modules/azure_tenant.py @@ -34,7 +34,7 @@ async def handle_event(self, event): self.verbose(f'Found {len(domains):,} domains under tenant for "{query}": {", ".join(sorted(domains))}') for domain in domains: if domain != query: - self.emit_event(domain, "DNS_NAME", source=event, tags=["affiliate", "azure-tenant"]) + await self.emit_event(domain, "DNS_NAME", source=event, tags=["affiliate", "azure-tenant"]) # tenant names if domain.lower().endswith(".onmicrosoft.com"): tenantname = domain.split(".")[0].lower() @@ -44,7 +44,7 @@ async def handle_event(self, event): event_data = {"tenant-names": sorted(tenant_names), "domains": sorted(domains)} if tenant_id is not None: event_data["tenant-id"] = tenant_id - self.emit_event(event_data, "AZURE_TENANT", source=event) + await self.emit_event(event_data, "AZURE_TENANT", source=event) async def query(self, domain): url = f"{self.base_url}/autodiscover/autodiscover.svc" diff --git a/bbot/modules/badsecrets.py b/bbot/modules/badsecrets.py index 0e188bced6..a7a31a48c8 100644 --- a/bbot/modules/badsecrets.py +++ b/bbot/modules/badsecrets.py @@ -52,11 +52,11 @@ async def handle_event(self, event): "url": event.data["url"], "host": str(event.host), } - self.emit_event(data, "VULNERABILITY", event) + await self.emit_event(data, "VULNERABILITY", event) elif r["type"] == "IdentifyOnly": # There is little value to presenting a non-vulnerable asp.net viewstate, as it is not crackable without a Matrioshka brain. Just emit a technology instead. if r["detecting_module"] == "ASPNET_Viewstate": - self.emit_event( + await self.emit_event( {"technology": "microsoft asp.net", "url": event.data["url"], "host": str(event.host)}, "TECHNOLOGY", event, @@ -67,4 +67,4 @@ async def handle_event(self, event): "url": event.data["url"], "host": str(event.host), } - self.emit_event(data, "FINDING", event) + await self.emit_event(data, "FINDING", event) diff --git a/bbot/modules/base.py b/bbot/modules/base.py index 7a4a783422..6ae93c7ea5 100644 --- a/bbot/modules/base.py +++ b/bbot/modules/base.py @@ -104,7 +104,7 @@ class BaseModule: _preserve_graph = False _stats_exclude = False - _qsize = 0 + _qsize = 100 _priority = 3 _name = "base" _type = "scan" @@ -381,7 +381,7 @@ def make_event(self, *args, **kwargs): Examples: >>> new_event = self.make_event("1.2.3.4", source=event) - >>> self.emit_event(new_event) + >>> await self.emit_event(new_event) Returns: Event or None: The created event, or None if a validation error occurred and raise_error was False. @@ -401,7 +401,7 @@ def make_event(self, *args, **kwargs): event.module = self return event - def emit_event(self, *args, **kwargs): + async def emit_event(self, *args, **kwargs): """Emit an event to the event queue and distribute it to interested modules. This is how modules "return" data. @@ -419,10 +419,10 @@ def emit_event(self, *args, **kwargs): ``` Examples: - >>> self.emit_event("www.evilcorp.com", source=event, tags=["affiliate"]) + >>> await self.emit_event("www.evilcorp.com", source=event, tags=["affiliate"]) >>> new_event = self.make_event("1.2.3.4", source=event) - >>> self.emit_event(new_event) + >>> await self.emit_event(new_event) Returns: None @@ -438,27 +438,7 @@ def emit_event(self, *args, **kwargs): emit_kwargs[o] = v event = self.make_event(*args, **event_kwargs) if event: - self.queue_outgoing_event(event, **emit_kwargs) - - async def emit_event_wait(self, *args, **kwargs): - """Emit an event to the event queue and await until there is space in the outgoing queue. - - This method is similar to `emit_event`, but it waits until there's sufficient space in the outgoing - event queue before emitting the event. It utilizes the queue size threshold defined in `self._qsize`. - - Args: - *args: Positional arguments to be passed to `emit_event()` for event creation. - **kwargs: Keyword arguments to be passed to `emit_event()` for event creation or configuration. - - Returns: - None - - See Also: - emit_event: For emitting an event without waiting on the queue size. - """ - while self.outgoing_event_queue.qsize() > self._qsize: - await self.helpers.sleep(0.2) - return self.emit_event(*args, **kwargs) + await self.queue_outgoing_event(event, **emit_kwargs) async def _events_waiting(self): """ @@ -808,7 +788,7 @@ async def queue_event(self, event, precheck=True): except AttributeError: self.debug(f"Not in an acceptable state to queue incoming event") - def queue_outgoing_event(self, event, **kwargs): + async def queue_outgoing_event(self, event, **kwargs): """ Queues an outgoing event to the module's outgoing event queue for further processing. @@ -829,7 +809,7 @@ def queue_outgoing_event(self, event, **kwargs): AttributeError: If the module is not in an acceptable state to queue outgoing events. """ try: - self.outgoing_event_queue.put_nowait((event, kwargs)) + await self.outgoing_event_queue.put((event, kwargs)) except AttributeError: self.debug(f"Not in an acceptable state to queue outgoing event") @@ -1076,7 +1056,7 @@ def incoming_event_queue(self): @property def outgoing_event_queue(self): if self._outgoing_event_queue is None: - self._outgoing_event_queue = ShuffleQueue() + self._outgoing_event_queue = ShuffleQueue(self._qsize) return self._outgoing_event_queue @property diff --git a/bbot/modules/bevigil.py b/bbot/modules/bevigil.py index e6c9990dd5..ff868e969b 100644 --- a/bbot/modules/bevigil.py +++ b/bbot/modules/bevigil.py @@ -29,13 +29,13 @@ async def handle_event(self, event): subdomains = await self.query(query, request_fn=self.request_subdomains, parse_fn=self.parse_subdomains) if subdomains: for subdomain in subdomains: - self.emit_event(subdomain, "DNS_NAME", source=event) + await self.emit_event(subdomain, "DNS_NAME", source=event) if self.urls: urls = await self.query(query, request_fn=self.request_urls, parse_fn=self.parse_urls) if urls: for parsed_url in await self.scan.run_in_executor(self.helpers.validators.collapse_urls, urls): - self.emit_event(parsed_url.geturl(), "URL_UNVERIFIED", source=event) + await self.emit_event(parsed_url.geturl(), "URL_UNVERIFIED", source=event) async def request_subdomains(self, query): url = f"{self.base_url}/{self.helpers.quote(query)}/subdomains/" diff --git a/bbot/modules/bucket_file_enum.py b/bbot/modules/bucket_file_enum.py index 7eb6926c0f..facaa021e3 100644 --- a/bbot/modules/bucket_file_enum.py +++ b/bbot/modules/bucket_file_enum.py @@ -42,7 +42,7 @@ async def handle_aws(self, event): bucket_file = url + "/" + key file_extension = self.helpers.get_file_extension(key) if file_extension not in self.scan.url_extension_blacklist: - self.emit_event(bucket_file, "URL_UNVERIFIED", source=event, tags="filedownload") + await self.emit_event(bucket_file, "URL_UNVERIFIED", source=event, tags="filedownload") urls_emitted += 1 if urls_emitted >= self.file_limit: return diff --git a/bbot/modules/builtwith.py b/bbot/modules/builtwith.py index 25a46ddf53..0b5793657a 100644 --- a/bbot/modules/builtwith.py +++ b/bbot/modules/builtwith.py @@ -33,14 +33,14 @@ async def handle_event(self, event): if subdomains: for s in subdomains: if s != event: - self.emit_event(s, "DNS_NAME", source=event) + await self.emit_event(s, "DNS_NAME", source=event) # redirects if self.config.get("redirects", True): redirects = await self.query(query, parse_fn=self.parse_redirects, request_fn=self.request_redirects) if redirects: for r in redirects: if r != event: - self.emit_event(r, "DNS_NAME", source=event, tags=["affiliate"]) + await self.emit_event(r, "DNS_NAME", source=event, tags=["affiliate"]) async def request_domains(self, query): url = f"{self.base_url}/v20/api.json?KEY={self.api_key}&LOOKUP={query}&NOMETA=yes&NOATTR=yes&HIDETEXT=yes&HIDEDL=yes" diff --git a/bbot/modules/bypass403.py b/bbot/modules/bypass403.py index 182f4b4db2..e4d8ed3f1e 100644 --- a/bbot/modules/bypass403.py +++ b/bbot/modules/bypass403.py @@ -132,7 +132,7 @@ async def handle_event(self, event): if results is None: return if len(results) > collapse_threshold: - self.emit_event( + await self.emit_event( { "description": f"403 Bypass MULTIPLE SIGNATURES (exceeded threshold {str(collapse_threshold)})", "host": str(event.host), @@ -143,7 +143,7 @@ async def handle_event(self, event): ) else: for description in results: - self.emit_event( + await self.emit_event( {"description": description, "host": str(event.host), "url": event.data}, "FINDING", source=event, diff --git a/bbot/modules/credshed.py b/bbot/modules/credshed.py index c493199d79..09ed57377c 100644 --- a/bbot/modules/credshed.py +++ b/bbot/modules/credshed.py @@ -76,11 +76,11 @@ async def handle_event(self, event): email_event = self.make_event(email, "EMAIL_ADDRESS", source=event, tags=tags) if email_event is not None: - self.emit_event(email_event) + await self.emit_event(email_event) if user and not self.already_seen(f"{email}:{user}"): - self.emit_event(user, "USERNAME", source=email_event, tags=tags) + await self.emit_event(user, "USERNAME", source=email_event, tags=tags) if pw and not self.already_seen(f"{email}:{pw}"): - self.emit_event(pw, "PASSWORD", source=email_event, tags=tags) + await self.emit_event(pw, "PASSWORD", source=email_event, tags=tags) for h_pw in hashes: if h_pw and not self.already_seen(f"{email}:{h_pw}"): - self.emit_event(h_pw, "HASHED_PASSWORD", source=email_event, tags=tags) + await self.emit_event(h_pw, "HASHED_PASSWORD", source=email_event, tags=tags) diff --git a/bbot/modules/deadly/dastardly.py b/bbot/modules/deadly/dastardly.py index 403e9511ed..47fd834fd7 100644 --- a/bbot/modules/deadly/dastardly.py +++ b/bbot/modules/deadly/dastardly.py @@ -60,7 +60,7 @@ async def handle_event(self, event): for testcase in testsuite.testcases: for failure in testcase.failures: if failure.severity == "Info": - self.emit_event( + await self.emit_event( { "host": str(event.host), "url": url, @@ -70,7 +70,7 @@ async def handle_event(self, event): event, ) else: - self.emit_event( + await self.emit_event( { "severity": failure.severity, "host": str(event.host), diff --git a/bbot/modules/deadly/ffuf.py b/bbot/modules/deadly/ffuf.py index cadaf4990e..f7ba3d96e0 100644 --- a/bbot/modules/deadly/ffuf.py +++ b/bbot/modules/deadly/ffuf.py @@ -83,7 +83,7 @@ async def handle_event(self, event): filters = await self.baseline_ffuf(fixed_url, exts=exts) async for r in self.execute_ffuf(self.tempfile, fixed_url, exts=exts, filters=filters): - self.emit_event(r["url"], "URL_UNVERIFIED", source=event, tags=[f"status-{r['status']}"]) + await self.emit_event(r["url"], "URL_UNVERIFIED", source=event, tags=[f"status-{r['status']}"]) async def filter_event(self, event): if "endpoint" in event.tags: diff --git a/bbot/modules/deadly/nuclei.py b/bbot/modules/deadly/nuclei.py index 33bcbe7e06..f3ef4ad67d 100644 --- a/bbot/modules/deadly/nuclei.py +++ b/bbot/modules/deadly/nuclei.py @@ -150,7 +150,7 @@ async def handle_batch(self, *events): description_string += f" Extracted Data: [{','.join(extracted_results)}]" if severity in ["INFO", "UNKNOWN"]: - self.emit_event( + await self.emit_event( { "host": str(source_event.host), "url": url, @@ -160,7 +160,7 @@ async def handle_batch(self, *events): source_event, ) else: - self.emit_event( + await self.emit_event( { "severity": severity, "host": str(source_event.host), diff --git a/bbot/modules/deadly/vhost.py b/bbot/modules/deadly/vhost.py index f4675e10fe..e2908dbbe4 100644 --- a/bbot/modules/deadly/vhost.py +++ b/bbot/modules/deadly/vhost.py @@ -92,9 +92,9 @@ async def ffuf_vhost(self, host, basehost, event, wordlist=None, skip_dns_host=F found_vhost_b64 = r["input"]["FUZZ"] vhost_dict = {"host": str(event.host), "url": host, "vhost": base64.b64decode(found_vhost_b64).decode()} if f"{vhost_dict['vhost']}{basehost}" != event.parsed.netloc: - self.emit_event(vhost_dict, "VHOST", source=event) + await self.emit_event(vhost_dict, "VHOST", source=event) if skip_dns_host == False: - self.emit_event(f"{vhost_dict['vhost']}{basehost}", "DNS_NAME", source=event, tags=["vhost"]) + await self.emit_event(f"{vhost_dict['vhost']}{basehost}", "DNS_NAME", source=event, tags=["vhost"]) yield vhost_dict["vhost"] diff --git a/bbot/modules/dehashed.py b/bbot/modules/dehashed.py index a09de454e5..4b35467129 100644 --- a/bbot/modules/dehashed.py +++ b/bbot/modules/dehashed.py @@ -49,13 +49,13 @@ async def handle_event(self, event): if email: email_event = self.make_event(email, "EMAIL_ADDRESS", source=event, tags=tags) if email_event is not None: - self.emit_event(email_event) + await self.emit_event(email_event) if user and not self.already_seen(f"{email}:{user}"): - self.emit_event(user, "USERNAME", source=email_event, tags=tags) + await self.emit_event(user, "USERNAME", source=email_event, tags=tags) if pw and not self.already_seen(f"{email}:{pw}"): - self.emit_event(pw, "PASSWORD", source=email_event, tags=tags) + await self.emit_event(pw, "PASSWORD", source=email_event, tags=tags) if h_pw and not self.already_seen(f"{email}:{h_pw}"): - self.emit_event(h_pw, "HASHED_PASSWORD", source=email_event, tags=tags) + await self.emit_event(h_pw, "HASHED_PASSWORD", source=email_event, tags=tags) async def query(self, event): query = f"domain:{self.make_query(event)}" diff --git a/bbot/modules/dnscommonsrv.py b/bbot/modules/dnscommonsrv.py index 538f516211..958b6b6127 100644 --- a/bbot/modules/dnscommonsrv.py +++ b/bbot/modules/dnscommonsrv.py @@ -106,4 +106,4 @@ async def handle_event(self, event): queries = [event.data] + [f"{srv}.{event.data}" for srv in common_srvs] async for query, results in self.helpers.resolve_batch(queries, type="srv"): if results: - self.emit_event(query, "DNS_NAME", tags=["srv-record"], source=event) + await self.emit_event(query, "DNS_NAME", tags=["srv-record"], source=event) diff --git a/bbot/modules/dnszonetransfer.py b/bbot/modules/dnszonetransfer.py index 1ec5bf5eb3..0a48526dc5 100644 --- a/bbot/modules/dnszonetransfer.py +++ b/bbot/modules/dnszonetransfer.py @@ -42,7 +42,9 @@ async def handle_event(self, event): continue self.hugesuccess(f"Successful zone transfer against {nameserver} for domain {domain}!") finding_description = f"Successful DNS zone transfer against {nameserver} for {domain}" - self.emit_event({"host": str(event.host), "description": finding_description}, "FINDING", source=event) + await self.emit_event( + {"host": str(event.host), "description": finding_description}, "FINDING", source=event + ) for name, ttl, rdata in zone.iterate_rdatas(): if str(name) == "@": parent_data = domain @@ -52,13 +54,13 @@ async def handle_event(self, event): if not parent_event or parent_event == event: parent_event = event else: - self.emit_event(parent_event) + await self.emit_event(parent_event) for rdtype, t in self.helpers.dns.extract_targets(rdata): if not self.helpers.is_ip(t): t = f"{t}.{domain}" module = self.helpers.dns._get_dummy_module(rdtype) child_event = self.scan.make_event(t, "DNS_NAME", parent_event, module=module) - self.emit_event(child_event) + await self.emit_event(child_event) else: self.debug(f"No data returned by {nameserver} for domain {domain}") diff --git a/bbot/modules/emailformat.py b/bbot/modules/emailformat.py index 3fd47ee2db..000c3d5cf3 100644 --- a/bbot/modules/emailformat.py +++ b/bbot/modules/emailformat.py @@ -19,4 +19,4 @@ async def handle_event(self, event): return for email in self.helpers.extract_emails(r.text): if email.endswith(query): - self.emit_event(email, "EMAIL_ADDRESS", source=event) + await self.emit_event(email, "EMAIL_ADDRESS", source=event) diff --git a/bbot/modules/ffuf_shortnames.py b/bbot/modules/ffuf_shortnames.py index ca45402e62..c062f09dfc 100644 --- a/bbot/modules/ffuf_shortnames.py +++ b/bbot/modules/ffuf_shortnames.py @@ -158,12 +158,14 @@ async def handle_event(self, event): if "shortname-file" in event.tags: for ext in used_extensions: async for r in self.execute_ffuf(tempfile, root_url, suffix=f".{ext}"): - self.emit_event(r["url"], "URL_UNVERIFIED", source=event, tags=[f"status-{r['status']}"]) + await self.emit_event( + r["url"], "URL_UNVERIFIED", source=event, tags=[f"status-{r['status']}"] + ) elif "shortname-directory" in event.tags: async for r in self.execute_ffuf(tempfile, root_url, exts=["/"]): r_url = f"{r['url'].rstrip('/')}/" - self.emit_event(r_url, "URL_UNVERIFIED", source=event, tags=[f"status-{r['status']}"]) + await self.emit_event(r_url, "URL_UNVERIFIED", source=event, tags=[f"status-{r['status']}"]) if self.config.get("find_delimiters"): if "shortname-directory" in event.tags: @@ -175,7 +177,9 @@ async def handle_event(self, event): async for r in self.execute_ffuf( tempfile, root_url, prefix=f"{prefix}{delimeter}", exts=["/"] ): - self.emit_event(r["url"], "URL_UNVERIFIED", source=event, tags=[f"status-{r['status']}"]) + await self.emit_event( + r["url"], "URL_UNVERIFIED", source=event, tags=[f"status-{r['status']}"] + ) elif "shortname-file" in event.tags: for ext in used_extensions: @@ -187,7 +191,7 @@ async def handle_event(self, event): async for r in self.execute_ffuf( tempfile, root_url, prefix=f"{prefix}{delimeter}", suffix=f".{ext}" ): - self.emit_event( + await self.emit_event( r["url"], "URL_UNVERIFIED", source=event, tags=[f"status-{r['status']}"] ) @@ -217,7 +221,7 @@ async def finish(self): ) async for r in self.execute_ffuf(tempfile, url, prefix=prefix, exts=["/"]): - self.emit_event( + await self.emit_event( r["url"], "URL_UNVERIFIED", source=self.shortname_to_event[hint], @@ -233,7 +237,7 @@ async def finish(self): async for r in self.execute_ffuf( tempfile, url, prefix=prefix, suffix=f".{ext}" ): - self.emit_event( + await self.emit_event( r["url"], "URL_UNVERIFIED", source=self.shortname_to_event[hint], diff --git a/bbot/modules/fingerprintx.py b/bbot/modules/fingerprintx.py index be5695541d..a7d8f2ea02 100644 --- a/bbot/modules/fingerprintx.py +++ b/bbot/modules/fingerprintx.py @@ -52,4 +52,4 @@ async def handle_batch(self, *events): protocol_data["port"] = port if banner: protocol_data["banner"] = banner - self.emit_event(protocol_data, "PROTOCOL", source=source_event, tags=tags) + await self.emit_event(protocol_data, "PROTOCOL", source=source_event, tags=tags) diff --git a/bbot/modules/generic_ssrf.py b/bbot/modules/generic_ssrf.py index bf2c37097d..d4045993b2 100644 --- a/bbot/modules/generic_ssrf.py +++ b/bbot/modules/generic_ssrf.py @@ -200,7 +200,7 @@ def interactsh_callback(self, r): matched_severity = match[2] matched_read_response = str(match[3]) - self.emit_event( + await self.emit_event( { "severity": matched_severity, "host": str(matched_event.host), diff --git a/bbot/modules/git.py b/bbot/modules/git.py index dafe151d11..fc61402de2 100644 --- a/bbot/modules/git.py +++ b/bbot/modules/git.py @@ -29,7 +29,7 @@ async def handle_event(self, event): text = "" if text: if getattr(result, "status_code", 0) == 200 and "[core]" in text and not self.fp_regex.match(text): - self.emit_event( + await self.emit_event( {"host": str(event.host), "url": url, "description": f"Exposed .git config at {url}"}, "FINDING", event, diff --git a/bbot/modules/github_codesearch.py b/bbot/modules/github_codesearch.py index c983357229..a138b43997 100644 --- a/bbot/modules/github_codesearch.py +++ b/bbot/modules/github_codesearch.py @@ -21,13 +21,13 @@ async def handle_event(self, event): repo_event = self.make_event({"url": repo_url}, "CODE_REPOSITORY", source=event) if repo_event is None: continue - self.emit_event(repo_event) + await self.emit_event(repo_event) for raw_url in raw_urls: url_event = self.make_event(raw_url, "URL_UNVERIFIED", source=repo_event, tags=["httpx-safe"]) if not url_event: continue url_event.scope_distance = repo_event.scope_distance - self.emit_event(url_event) + await self.emit_event(url_event) async def query(self, query): repos = {} diff --git a/bbot/modules/github_org.py b/bbot/modules/github_org.py index 66182a2a65..66fa038a78 100644 --- a/bbot/modules/github_org.py +++ b/bbot/modules/github_org.py @@ -57,7 +57,7 @@ async def handle_event(self, event): for repo_url in repos: repo_event = self.make_event({"url": repo_url}, "CODE_REPOSITORY", source=event) repo_event.scope_distance = event.scope_distance - self.emit_event(repo_event) + await self.emit_event(repo_event) # find members from org (SOCIAL --> SOCIAL) if is_org and self.include_members: @@ -66,7 +66,7 @@ async def handle_event(self, event): for member in org_members: event_data = {"platform": "github", "profile_name": member, "url": f"https://github.com/{member}"} member_event = self.make_event(event_data, "SOCIAL", tags="github-org-member", source=event) - self.emit_event(member_event) + await self.emit_event(member_event) # find valid orgs from stub (ORG_STUB --> SOCIAL) elif event.type == "ORG_STUB": @@ -80,7 +80,7 @@ async def handle_event(self, event): event_data = {"platform": "github", "profile_name": user, "url": f"https://github.com/{user}"} github_org_event = self.make_event(event_data, "SOCIAL", tags="github-org", source=event) github_org_event.scope_distance = event.scope_distance - self.emit_event(github_org_event) + await self.emit_event(github_org_event) async def query_org_repos(self, query): repos = [] diff --git a/bbot/modules/gowitness.py b/bbot/modules/gowitness.py index 5f4c4a5e81..5592eaa65b 100644 --- a/bbot/modules/gowitness.py +++ b/bbot/modules/gowitness.py @@ -137,7 +137,7 @@ async def handle_batch(self, *events): filename = screenshot["filename"] webscreenshot_data = {"filename": filename, "url": final_url} source_event = events[url] - self.emit_event(webscreenshot_data, "WEBSCREENSHOT", source=source_event) + await self.emit_event(webscreenshot_data, "WEBSCREENSHOT", source=source_event) # emit URLs for url, row in self.new_network_logs.items(): @@ -151,7 +151,7 @@ async def handle_batch(self, *events): if self.helpers.is_spider_danger(source_event, url): tags.append("spider-danger") if url and url.startswith("http"): - self.emit_event(url, "URL_UNVERIFIED", source=source_event, tags=tags) + await self.emit_event(url, "URL_UNVERIFIED", source=source_event, tags=tags) # emit technologies for _, row in self.new_technologies.items(): @@ -160,7 +160,7 @@ async def handle_batch(self, *events): source_event = events[source_url] technology = row["value"] tech_data = {"technology": technology, "url": source_url, "host": str(source_event.host)} - self.emit_event(tech_data, "TECHNOLOGY", source=source_event) + await self.emit_event(tech_data, "TECHNOLOGY", source=source_event) def construct_command(self): # base executable diff --git a/bbot/modules/host_header.py b/bbot/modules/host_header.py index bae7219906..bec77c15a3 100644 --- a/bbot/modules/host_header.py +++ b/bbot/modules/host_header.py @@ -30,7 +30,7 @@ async def setup(self): def rand_string(self, *args, **kwargs): return self.helpers.rand_string(*args, **kwargs) - def interactsh_callback(self, r): + async def interactsh_callback(self, r): full_id = r.get("full-id", None) if full_id: if "." in full_id: @@ -40,7 +40,7 @@ def interactsh_callback(self, r): matched_event = match[0] matched_technique = match[1] - self.emit_event( + await self.emit_event( { "host": str(matched_event.host), "url": matched_event.data["url"], @@ -128,7 +128,7 @@ async def handle_event(self, event): split_output = output.split("\n") if " 4" in split_output: - self.emit_event( + await self.emit_event( { "host": str(event.host), "url": event.data["url"], @@ -168,7 +168,7 @@ async def handle_event(self, event): # emit all the domain reflections we found for dr in domain_reflections: - self.emit_event( + await self.emit_event( { "host": str(event.host), "url": event.data["url"], diff --git a/bbot/modules/httpx.py b/bbot/modules/httpx.py index 0c12ad740d..393d2a402a 100644 --- a/bbot/modules/httpx.py +++ b/bbot/modules/httpx.py @@ -153,11 +153,11 @@ async def handle_batch(self, *events): url_event = self.make_event(url, "URL", source_event, tags=tags) if url_event: if url_event != source_event: - self.emit_event(url_event) + await self.emit_event(url_event) else: url_event._resolved.set() # HTTP response - self.emit_event(j, "HTTP_RESPONSE", url_event, tags=url_event.tags) + await self.emit_event(j, "HTTP_RESPONSE", url_event, tags=url_event.tags) for tempdir in Path(tempfile.gettempdir()).iterdir(): if tempdir.is_dir() and self.httpx_tempdir_regex.match(tempdir.name): diff --git a/bbot/modules/hunt.py b/bbot/modules/hunt.py index 0ccf0391be..add45b665d 100644 --- a/bbot/modules/hunt.py +++ b/bbot/modules/hunt.py @@ -289,4 +289,4 @@ async def handle_event(self, event): url = event.data.get("url", "") if url: data["url"] = url - self.emit_event(data, "FINDING", event) + await self.emit_event(data, "FINDING", event) diff --git a/bbot/modules/hunterio.py b/bbot/modules/hunterio.py index 1e65c6e4c7..792ca6d98b 100644 --- a/bbot/modules/hunterio.py +++ b/bbot/modules/hunterio.py @@ -26,14 +26,14 @@ async def handle_event(self, event): if email: email_event = self.make_event(email, "EMAIL_ADDRESS", event) if email_event: - self.emit_event(email_event) + await self.emit_event(email_event) for source in sources: domain = source.get("domain", "") if domain: - self.emit_event(domain, "DNS_NAME", email_event) + await self.emit_event(domain, "DNS_NAME", email_event) url = source.get("uri", "") if url: - self.emit_event(url, "URL_UNVERIFIED", email_event) + await self.emit_event(url, "URL_UNVERIFIED", email_event) async def query(self, query): emails = [] diff --git a/bbot/modules/iis_shortnames.py b/bbot/modules/iis_shortnames.py index ff7941dc2c..7d558a23aa 100644 --- a/bbot/modules/iis_shortnames.py +++ b/bbot/modules/iis_shortnames.py @@ -221,7 +221,7 @@ class safety_counter_obj: technique_strings.append(f"{method} ({technique})") description = f"IIS Shortname Vulnerability Detected. Potentially Vulnerable Method/Techniques: [{','.join(technique_strings)}]" - self.emit_event( + await self.emit_event( {"severity": "LOW", "host": str(event.host), "url": normalized_url, "description": description}, "VULNERABILITY", event, @@ -314,7 +314,7 @@ class safety_counter_obj: hint_type = "shortname-file" else: hint_type = "shortname-directory" - self.emit_event(f"{normalized_url}/{url_hint}", "URL_HINT", event, tags=[hint_type]) + await self.emit_event(f"{normalized_url}/{url_hint}", "URL_HINT", event, tags=[hint_type]) async def filter_event(self, event): if "dir" in event.tags: diff --git a/bbot/modules/internal/excavate.py b/bbot/modules/internal/excavate.py index f3f15f83f9..f5c028c95d 100644 --- a/bbot/modules/internal/excavate.py +++ b/bbot/modules/internal/excavate.py @@ -23,7 +23,7 @@ async def search(self, content, event, **kwargs): async for result, name in self._search(content, event, **kwargs): results.add(result) for result in results: - self.report(result, name, event, **kwargs) + await self.report(result, name, event, **kwargs) async def _search(self, content, event, **kwargs): for name, regex in self.compiled_regexes.items(): @@ -32,7 +32,7 @@ async def _search(self, content, event, **kwargs): for result in regex.findall(content): yield result, name - def report(self, result, name, event): + async def report(self, result, name, event): pass @@ -48,10 +48,10 @@ async def search(self, content, event, **kwargs): async for csp, name in self._search(content, event, **kwargs): extracted_domains = self.extract_domains(csp) for domain in extracted_domains: - self.report(domain, event, **kwargs) + await self.report(domain, event, **kwargs) - def report(self, domain, event, **kwargs): - self.excavate.emit_event(domain, "DNS_NAME", source=event, tags=["affiliate"]) + async def report(self, domain, event, **kwargs): + await self.excavate.emit_event(domain, "DNS_NAME", source=event, tags=["affiliate"]) class HostnameExtractor(BaseExtractor): @@ -62,8 +62,8 @@ def __init__(self, excavate): self.regexes[f"dns_name_{i+1}"] = r.pattern super().__init__(excavate) - def report(self, result, name, event, **kwargs): - self.excavate.emit_event(result, "DNS_NAME", source=event) + async def report(self, result, name, event, **kwargs): + await self.excavate.emit_event(result, "DNS_NAME", source=event) class URLExtractor(BaseExtractor): @@ -95,7 +95,7 @@ async def search(self, content, event, **kwargs): urls_found = 0 for result, name in results: - url_event = self.report(result, name, event, **kwargs) + url_event = await self.report(result, name, event, **kwargs) if url_event is not None: url_in_scope = self.excavate.scan.in_scope(url_event) is_spider_danger = self.excavate.helpers.is_spider_danger(event, result) @@ -115,7 +115,7 @@ async def search(self, content, event, **kwargs): url_event.add_tag("spider-danger") self.excavate.debug(f"Found URL [{result}] from parsing [{event.data.get('url')}] with regex [{name}]") - self.excavate.emit_event(url_event) + await self.excavate.emit_event(url_event) if url_in_scope: urls_found += 1 @@ -150,7 +150,7 @@ async def _search(self, content, event, **kwargs): yield result, name - def report(self, result, name, event, **kwargs): + async def report(self, result, name, event, **kwargs): parsed_uri = None try: parsed_uri = self.excavate.helpers.urlparse(result) @@ -168,7 +168,7 @@ def report(self, result, name, event, **kwargs): parsed_url = getattr(event, "parsed", None) if parsed_url: event_data["url"] = parsed_url.geturl() - self.excavate.emit_event( + await self.excavate.emit_event( event_data, "FINDING", source=event, @@ -177,7 +177,7 @@ def report(self, result, name, event, **kwargs): protocol_data = {"protocol": parsed_uri.scheme, "host": str(host)} if port: protocol_data["port"] = port - self.excavate.emit_event( + await self.excavate.emit_event( protocol_data, "PROTOCOL", source=event, @@ -192,12 +192,12 @@ class EmailExtractor(BaseExtractor): regexes = {"email": _email_regex} tld_blacklist = ["png", "jpg", "jpeg", "bmp", "ico", "gif", "svg", "css", "ttf", "woff", "woff2"] - def report(self, result, name, event, **kwargs): + async def report(self, result, name, event, **kwargs): result = result.lower() tld = result.split(".")[-1] if tld not in self.tld_blacklist: self.excavate.debug(f"Found email address [{result}] from parsing [{event.data.get('url')}]") - self.excavate.emit_event(result, "EMAIL_ADDRESS", source=event) + await self.excavate.emit_event(result, "EMAIL_ADDRESS", source=event) class ErrorExtractor(BaseExtractor): @@ -218,10 +218,10 @@ class ErrorExtractor(BaseExtractor): "ASP.NET:4": r"Error ([\d-]+) \([\dA-F]+\)", } - def report(self, result, name, event, **kwargs): + async def report(self, result, name, event, **kwargs): self.excavate.debug(f"Found error message from parsing [{event.data.get('url')}] with regex [{name}]") description = f"Error message Detected at Error Type: {name}" - self.excavate.emit_event( + await self.excavate.emit_event( {"host": str(event.host), "url": event.data.get("url", ""), "description": description}, "FINDING", source=event, @@ -231,7 +231,7 @@ def report(self, result, name, event, **kwargs): class JWTExtractor(BaseExtractor): regexes = {"JWT": r"eyJ(?:[\w-]*\.)(?:[\w-]*\.)[\w-]*"} - def report(self, result, name, event, **kwargs): + async def report(self, result, name, event, **kwargs): self.excavate.debug(f"Found JWT candidate [{result}]") try: j.decode(result, options={"verify_signature": False}) @@ -240,7 +240,7 @@ def report(self, result, name, event, **kwargs): if jwt_headers["alg"].upper()[0:2] == "HS": tags = ["crackable"] description = f"JWT Identified [{result}]" - self.excavate.emit_event( + await self.excavate.emit_event( {"host": str(event.host), "url": event.data.get("url", ""), "description": description}, "FINDING", event, @@ -259,9 +259,9 @@ class SerializationExtractor(BaseExtractor): "Possible Compressed": r"H4sIAAAAAAAA[a-zA-Z0-9+/]+={,2}", } - def report(self, result, name, event, **kwargs): + async def report(self, result, name, event, **kwargs): description = f"{name} serialized object found" - self.excavate.emit_event( + await self.excavate.emit_event( {"host": str(event.host), "url": event.data.get("url"), "description": description}, "FINDING", event ) @@ -272,9 +272,9 @@ class FunctionalityExtractor(BaseExtractor): "Web Service WSDL": r"(?i)((?:http|https)://[^\s]*?.(?:wsdl))", } - def report(self, result, name, event, **kwargs): + async def report(self, result, name, event, **kwargs): description = f"{name} found" - self.excavate.emit_event( + await self.excavate.emit_event( {"host": str(event.host), "url": event.data.get("url"), "description": description}, "FINDING", event ) @@ -314,7 +314,7 @@ class JavascriptExtractor(BaseExtractor): "possible_creds_var": r"(?:password|passwd|pwd|pass)\s*=+\s*['\"][^\s'\"]{1,60}['\"]", } - def report(self, result, name, event, **kwargs): + async def report(self, result, name, event, **kwargs): # ensure that basic auth matches aren't false positives if name == "authorization_basic": try: @@ -326,7 +326,7 @@ def report(self, result, name, event, **kwargs): self.excavate.debug(f"Found Possible Secret in Javascript [{result}]") description = f"Possible secret in JS [{result}] Signature [{name}]" - self.excavate.emit_event( + await self.excavate.emit_event( {"host": str(event.host), "url": event.data.get("url", ""), "description": description}, "FINDING", event ) @@ -384,7 +384,7 @@ async def handle_event(self, event): # inherit web spider distance from parent (don't increment) source_web_spider_distance = getattr(event, "web_spider_distance", 0) url_event.web_spider_distance = source_web_spider_distance - self.emit_event(url_event) + await self.emit_event(url_event) else: self.verbose(f"Exceeded max HTTP redirects ({self.max_redirects}): {location}") diff --git a/bbot/modules/internal/speculate.py b/bbot/modules/internal/speculate.py index 9b57b0e817..8f51d5d958 100644 --- a/bbot/modules/internal/speculate.py +++ b/bbot/modules/internal/speculate.py @@ -73,13 +73,13 @@ async def handle_event(self, event): ips = list(net) random.shuffle(ips) for ip in ips: - self.emit_event(ip, "IP_ADDRESS", source=event, internal=True) + await self.emit_event(ip, "IP_ADDRESS", source=event, internal=True) # parent domains if event.type == "DNS_NAME": parent = self.helpers.parent_domain(event.data) if parent != event.data: - self.emit_event(parent, "DNS_NAME", source=event, internal=True) + await self.emit_event(parent, "DNS_NAME", source=event, internal=True) # generate open ports @@ -91,7 +91,7 @@ async def handle_event(self, event): if event.type == "URL" or (event.type == "URL_UNVERIFIED" and self.open_port_consumers): # only speculate port from a URL if it wouldn't be speculated naturally from the host if event.host and (event.port not in self.ports or not speculate_open_ports): - self.emit_event( + await self.emit_event( self.helpers.make_netloc(event.host, event.port), "OPEN_TCP_PORT", source=event, @@ -108,7 +108,7 @@ async def handle_event(self, event): # inherit web spider distance from parent (don't increment) source_web_spider_distance = getattr(event, "web_spider_distance", 0) url_event.web_spider_distance = source_web_spider_distance - self.emit_event(url_event) + await self.emit_event(url_event) # from hosts if speculate_open_ports: @@ -120,7 +120,7 @@ async def handle_event(self, event): if event.type == "IP_ADDRESS" or usable_dns: for port in self.ports: - self.emit_event( + await self.emit_event( self.helpers.make_netloc(event.data, port), "OPEN_TCP_PORT", source=event, @@ -154,7 +154,7 @@ async def handle_event(self, event): stub_event = self.make_event(stub, "ORG_STUB", source=event) if event.scope_distance > 0: stub_event.scope_distance = event.scope_distance - self.emit_event(stub_event) + await self.emit_event(stub_event) async def filter_event(self, event): # don't accept errored DNS_NAMEs diff --git a/bbot/modules/internetdb.py b/bbot/modules/internetdb.py index 0384a4d4dd..b3e98b9fca 100644 --- a/bbot/modules/internetdb.py +++ b/bbot/modules/internetdb.py @@ -76,7 +76,7 @@ async def handle_event(self, event): return if data: if r.status_code == 200: - self._parse_response(data=data, event=event) + await self._parse_response(data=data, event=event) elif r.status_code == 404: detail = data.get("detail", "") if detail: @@ -86,22 +86,22 @@ async def handle_event(self, event): err_msg = data.get("msg", "") self.verbose(f"Shodan error for {ip}: {err_data}: {err_msg}") - def _parse_response(self, data: dict, event): + async def _parse_response(self, data: dict, event): """Handles emitting events from returned JSON""" data: dict # has keys: cpes, hostnames, ip, ports, tags, vulns # ip is a string, ports is a list of ports, the rest is a list of strings for hostname in data.get("hostnames", []): - self.emit_event(hostname, "DNS_NAME", source=event) + await self.emit_event(hostname, "DNS_NAME", source=event) for cpe in data.get("cpes", []): - self.emit_event({"technology": cpe, "host": str(event.host)}, "TECHNOLOGY", source=event) + await self.emit_event({"technology": cpe, "host": str(event.host)}, "TECHNOLOGY", source=event) for port in data.get("ports", []): - self.emit_event( + await self.emit_event( self.helpers.make_netloc(event.data, port), "OPEN_TCP_PORT", source=event, internal=True, quick=True ) vulns = data.get("vulns", []) if vulns: vulns_str = ", ".join([str(v) for v in vulns]) - self.emit_event( + await self.emit_event( {"description": f"Shodan reported verified vulnerabilities: {vulns_str}", "host": str(event.host)}, "FINDING", source=event, diff --git a/bbot/modules/ip2location.py b/bbot/modules/ip2location.py index d0a66ce4cf..e192b2abba 100644 --- a/bbot/modules/ip2location.py +++ b/bbot/modules/ip2location.py @@ -57,4 +57,4 @@ async def handle_event(self, event): if error_msg: self.warning(error_msg) elif geo_data: - self.emit_event(geo_data, "GEOLOCATION", event) + await self.emit_event(geo_data, "GEOLOCATION", event) diff --git a/bbot/modules/ipneighbor.py b/bbot/modules/ipneighbor.py index b6688abee7..2b139f8078 100644 --- a/bbot/modules/ipneighbor.py +++ b/bbot/modules/ipneighbor.py @@ -34,4 +34,4 @@ async def handle_event(self, event): ip_event = self.make_event(str(ip), "IP_ADDRESS", event, internal=True) # keep the scope distance low to give it one more hop for DNS resolution # ip_event.scope_distance = max(1, event.scope_distance) - self.emit_event(ip_event) + await self.emit_event(ip_event) diff --git a/bbot/modules/ipstack.py b/bbot/modules/ipstack.py index a8a143fdc2..98f1395056 100644 --- a/bbot/modules/ipstack.py +++ b/bbot/modules/ipstack.py @@ -47,4 +47,4 @@ async def handle_event(self, event): if error_msg: self.warning(error_msg) elif geo_data: - self.emit_event(geo_data, "GEOLOCATION", event) + await self.emit_event(geo_data, "GEOLOCATION", event) diff --git a/bbot/modules/masscan.py b/bbot/modules/masscan.py index 4d9c9db689..02598a2150 100644 --- a/bbot/modules/masscan.py +++ b/bbot/modules/masscan.py @@ -193,18 +193,18 @@ def process_output(self, line, result_callback): result = self.helpers.make_netloc(result, port_number) if source is None: source = self.make_event(ip, "IP_ADDRESS", source=self.get_source_event(ip)) - self.emit_event(source) + await self.emit_event(source) result_callback(result, source=source) def append_alive_host(self, host, source): host_event = self.make_event(host, "IP_ADDRESS", source=self.get_source_event(host)) self.alive_hosts[host] = host_event self._write_ping_result(host) - self.emit_event(host_event) + await self.emit_event(host_event) def emit_open_tcp_port(self, data, source): self._write_syn_result(data) - self.emit_event(data, "OPEN_TCP_PORT", source=source) + await self.emit_event(data, "OPEN_TCP_PORT", source=source) def emit_from_cache(self): ip_events = {} @@ -220,7 +220,7 @@ def emit_from_cache(self): break ip_event = self.make_event(ip, "IP_ADDRESS", source=self.get_source_event(ip)) ip_events[ip] = ip_event - self.emit_event(ip_event) + await self.emit_event(ip_event) # syn scan if self.syn_cache.is_file(): cached_syns = list(self.helpers.read_file(self.syn_cache)) @@ -237,8 +237,8 @@ def emit_from_cache(self): if source_event is None: self.verbose(f"Source event not found for {line}") source_event = self.make_event(line, "IP_ADDRESS", source=self.get_source_event(line)) - self.emit_event(source_event) - self.emit_event(line, "OPEN_TCP_PORT", source=source_event) + await self.emit_event(source_event) + await self.emit_event(line, "OPEN_TCP_PORT", source=source_event) def get_source_event(self, host): source_event = self.scan.whitelist.get(host) diff --git a/bbot/modules/massdns.py b/bbot/modules/massdns.py index 5a0865476e..a345b79f69 100644 --- a/bbot/modules/massdns.py +++ b/bbot/modules/massdns.py @@ -119,7 +119,7 @@ async def handle_event(self, event): self.info(f"Brute-forcing subdomains for {query} (source: {event.data})") for hostname in await self.massdns(query, self.subdomain_list): - self.emit_result(hostname, event, query) + await self.emit_result(hostname, event, query) def abort_if(self, event): if not event.scope_distance == 0: @@ -127,12 +127,12 @@ def abort_if(self, event): if "wildcard" in event.tags: return True, "event is a wildcard" - def emit_result(self, result, source_event, query, tags=None): + async def emit_result(self, result, source_event, query, tags=None): if not result == source_event: kwargs = {"abort_if": self.abort_if} if tags is not None: kwargs["tags"] = tags - self.emit_event(result, "DNS_NAME", source_event, **kwargs) + await self.emit_event(result, "DNS_NAME", source_event, **kwargs) def already_processed(self, hostname): if hash(hostname) in self.processed: @@ -380,7 +380,9 @@ def add_mutation(_domain_hash, m): if source_event is None: self.warning(f"Could not correlate source event from: {hostname}") source_event = self.scan.root_event - self.emit_result(hostname, source_event, query, tags=[f"mutation-{self._mutation_run}"]) + await self.emit_result( + hostname, source_event, query, tags=[f"mutation-{self._mutation_run}"] + ) if results: found_mutations = True continue diff --git a/bbot/modules/nmap.py b/bbot/modules/nmap.py index aeb28f9aeb..6d8a1293b9 100644 --- a/bbot/modules/nmap.py +++ b/bbot/modules/nmap.py @@ -52,10 +52,10 @@ async def handle_batch(self, *events): for port in host.open_ports: port_number = int(port.split("/")[0]) netloc = self.helpers.make_netloc(host.address, port_number) - self.emit_event(netloc, "OPEN_TCP_PORT", source=source_event) + await self.emit_event(netloc, "OPEN_TCP_PORT", source=source_event) for hostname in host.hostnames: netloc = self.helpers.make_netloc(hostname, port_number) - self.emit_event(netloc, "OPEN_TCP_PORT", source=source_event) + await self.emit_event(netloc, "OPEN_TCP_PORT", source=source_event) finally: output_file.unlink(missing_ok=True) diff --git a/bbot/modules/nsec.py b/bbot/modules/nsec.py index bfd770d440..7d313c140e 100644 --- a/bbot/modules/nsec.py +++ b/bbot/modules/nsec.py @@ -18,12 +18,12 @@ async def handle_event(self, event): async for result in self.nsec_walk(event.data): if not emitted_finding: emitted_finding = True - self.emit_event( + await self.emit_event( {"host": event.data, "description": f"DNSSEC NSEC Zone Walking Enabled for domain: {event.data}"}, "FINDING", source=event, ) - self.emit_event(result, "DNS_NAME", source=event) + await self.emit_event(result, "DNS_NAME", source=event) async def get_nsec_record(self, domain): domain = domain.replace("\\000.", "") diff --git a/bbot/modules/ntlm.py b/bbot/modules/ntlm.py index 76e93c595a..c69beb941d 100644 --- a/bbot/modules/ntlm.py +++ b/bbot/modules/ntlm.py @@ -87,7 +87,7 @@ async def handle_event(self, event): for result, request_url in await self.handle_url(event): if result and request_url: self.found.add(found_hash) - self.emit_event( + await self.emit_event( { "host": str(event.host), "url": request_url, @@ -98,7 +98,7 @@ async def handle_event(self, event): ) fqdn = result.get("FQDN", "") if fqdn: - self.emit_event(fqdn, "DNS_NAME", source=event) + await self.emit_event(fqdn, "DNS_NAME", source=event) break async def filter_event(self, event): diff --git a/bbot/modules/oauth.py b/bbot/modules/oauth.py index f4d5925110..33cb38959b 100644 --- a/bbot/modules/oauth.py +++ b/bbot/modules/oauth.py @@ -66,16 +66,16 @@ async def handle_event(self, event): source=event, ) finding_event.source_domain = source_domain - self.emit_event(finding_event) + await self.emit_event(finding_event) url_event = self.make_event( token_endpoint, "URL_UNVERIFIED", source=event, tags=["affiliate", "oauth-token-endpoint"] ) url_event.source_domain = source_domain - self.emit_event(url_event) + await self.emit_event(url_event) for result in oidc_results: if result not in (domain, event.data): event_type = "URL_UNVERIFIED" if self.helpers.is_url(result) else "DNS_NAME" - self.emit_event(result, event_type, source=event, tags=["affiliate"]) + await self.emit_event(result, event_type, source=event, tags=["affiliate"]) for oauth_task in oauth_tasks: url = await oauth_task @@ -90,7 +90,7 @@ async def handle_event(self, event): source=event, ) oauth_finding.source_domain = source_domain - self.emit_event(oauth_finding) + await self.emit_event(oauth_finding) def url_and_base(self, url): yield url diff --git a/bbot/modules/output/asset_inventory.py b/bbot/modules/output/asset_inventory.py index acb833daae..b16dd18fbf 100644 --- a/bbot/modules/output/asset_inventory.py +++ b/bbot/modules/output/asset_inventory.py @@ -173,19 +173,19 @@ async def finish(self): self.add_custom_headers(list(asset.custom_fields)) if not is_ip(asset.host): host_event = self.make_event(asset.host, "DNS_NAME", source=self.scan.root_event) - self.emit_event(host_event) + await self.emit_event(host_event) for port in asset.ports: netloc = self.helpers.make_netloc(asset.host, port) open_port_event = self.make_event(netloc, "OPEN_TCP_PORT", source=host_event) - self.emit_event(open_port_event) + await self.emit_event(open_port_event) else: for ip in asset.ip_addresses: ip_event = self.make_event(ip, "IP_ADDRESS", source=self.scan.root_event) - self.emit_event(ip_event) + await self.emit_event(ip_event) for port in asset.ports: netloc = self.helpers.make_netloc(ip, port) open_port_event = self.make_event(netloc, "OPEN_TCP_PORT", source=ip_event) - self.emit_event(open_port_event) + await self.emit_event(open_port_event) else: self.warning( f"use_previous=True was set but no previous asset inventory was found at {self.output_file}" diff --git a/bbot/modules/paramminer_headers.py b/bbot/modules/paramminer_headers.py index 65880d9c8d..7044ae90ab 100644 --- a/bbot/modules/paramminer_headers.py +++ b/bbot/modules/paramminer_headers.py @@ -126,7 +126,7 @@ def process_results(self, event, results): if reflection: tags = ["http_reflection"] description = f"[Paramminer] {self.compare_mode.capitalize()}: [{result}] Reasons: [{reasons}] Reflection: [{str(reflection)}]" - self.emit_event( + await self.emit_event( {"host": str(event.host), "url": url, "description": description}, "FINDING", event, diff --git a/bbot/modules/pgp.py b/bbot/modules/pgp.py index c1e0773c37..2c378f5853 100644 --- a/bbot/modules/pgp.py +++ b/bbot/modules/pgp.py @@ -20,7 +20,7 @@ async def handle_event(self, event): if results: for hostname in results: if not hostname == event: - self.emit_event(hostname, "EMAIL_ADDRESS", event, abort_if=self.abort_if) + await self.emit_event(hostname, "EMAIL_ADDRESS", event, abort_if=self.abort_if) async def query(self, query): results = set() diff --git a/bbot/modules/postman.py b/bbot/modules/postman.py index 619107b533..57d361b848 100644 --- a/bbot/modules/postman.py +++ b/bbot/modules/postman.py @@ -26,7 +26,7 @@ async def handle_event(self, event): query = self.make_query(event) self.verbose(f"Searching for any postman workspaces, collections, requests belonging to {query}") for url in await self.query(query): - self.emit_event(url, "URL_UNVERIFIED", source=event, tags="httpx-safe") + await self.emit_event(url, "URL_UNVERIFIED", source=event, tags="httpx-safe") async def query(self, query): interesting_urls = [] diff --git a/bbot/modules/report/asn.py b/bbot/modules/report/asn.py index a8f57709aa..db8612a1e3 100644 --- a/bbot/modules/report/asn.py +++ b/bbot/modules/report/asn.py @@ -42,9 +42,9 @@ async def handle_event(self, event): emails = asn.pop("emails", []) self.cache_put(asn) asn_event = self.make_event(asn, "ASN", source=event) - self.emit_event(asn_event) + await self.emit_event(asn_event) for email in emails: - self.emit_event(email, "EMAIL_ADDRESS", source=asn_event) + await self.emit_event(email, "EMAIL_ADDRESS", source=asn_event) async def report(self): asn_data = sorted(self.asn_cache.items(), key=lambda x: self.asn_counts[x[0]], reverse=True) diff --git a/bbot/modules/robots.py b/bbot/modules/robots.py index fd873b799b..717900beee 100644 --- a/bbot/modules/robots.py +++ b/bbot/modules/robots.py @@ -48,4 +48,4 @@ async def handle_event(self, event): tags = [] if self.helpers.is_spider_danger(event, unverified_url): tags.append("spider-danger") - self.emit_event(unverified_url, "URL_UNVERIFIED", source=event, tags=tags) + await self.emit_event(unverified_url, "URL_UNVERIFIED", source=event, tags=tags) diff --git a/bbot/modules/secretsdb.py b/bbot/modules/secretsdb.py index 83f305c611..3fc8ad5392 100644 --- a/bbot/modules/secretsdb.py +++ b/bbot/modules/secretsdb.py @@ -54,7 +54,7 @@ async def handle_event(self, event): parsed_url = getattr(event, "parsed", None) if parsed_url: event_data["url"] = parsed_url.geturl() - self.emit_event( + await self.emit_event( event_data, "FINDING", source=event, diff --git a/bbot/modules/sitedossier.py b/bbot/modules/sitedossier.py index 87358a955d..0c797296af 100644 --- a/bbot/modules/sitedossier.py +++ b/bbot/modules/sitedossier.py @@ -19,7 +19,7 @@ async def handle_event(self, event): self.verbose(e) continue if hostname and hostname.endswith(f".{query}") and not hostname == event.data: - await self.emit_event_wait(hostname, "DNS_NAME", event, abort_if=self.abort_if) + await self.emit_event(hostname, "DNS_NAME", event, abort_if=self.abort_if) async def query(self, query, parse_fn=None, request_fn=None): results = set() diff --git a/bbot/modules/skymem.py b/bbot/modules/skymem.py index 71d0e883e7..4bd76c70d8 100644 --- a/bbot/modules/skymem.py +++ b/bbot/modules/skymem.py @@ -19,7 +19,7 @@ async def handle_event(self, event): if not r: return for email in self.helpers.extract_emails(r.text): - self.emit_event(email, "EMAIL_ADDRESS", source=event) + await self.emit_event(email, "EMAIL_ADDRESS", source=event) # iterate through other pages domain_ids = re.findall(r' {e.host}" description += f" ({source_hosts_str})" - self.emit_event({"host": event.host, "url": url, "description": description}, "FINDING", source=event) + await self.emit_event( + {"host": event.host, "url": url, "description": description}, "FINDING", source=event + ) else: self.debug(reason) diff --git a/bbot/modules/telerik.py b/bbot/modules/telerik.py index 71ebc4e089..6cbdfcf196 100644 --- a/bbot/modules/telerik.py +++ b/bbot/modules/telerik.py @@ -211,7 +211,7 @@ async def handle_event(self, event): version = "<= 2019 (Either Pre-2017 (vulnerable), or 2017-2019 w/ Encrypt-Then-Mac)" description = f"Telerik RAU AXD Handler detected. Verbose Errors Enabled: [{str(verbose_errors)}] Version Guess: [{version}]" - self.emit_event( + await self.emit_event( {"host": str(event.host), "url": f"{event.data}{webresource}", "description": description}, "FINDING", event, @@ -237,7 +237,7 @@ async def handle_event(self, event): description = f"[CVE-2017-11317] [{str(version)}] {webresource}" if "fileInfo" in output.stdout: self.debug(f"Confirmed Vulnerable Telerik (version: {str(version)}") - self.emit_event( + await self.emit_event( { "severity": "CRITICAL", "description": description, @@ -276,7 +276,7 @@ async def handle_event(self, event): await self.helpers.cancel_tasks(tasks) self.debug(f"Detected Telerik UI instance ({dh})") description = f"Telerik DialogHandler detected" - self.emit_event( + await self.emit_event( {"host": str(event.host), "url": f"{event.data}{dh}", "description": description}, "FINDING", event, @@ -297,7 +297,7 @@ async def handle_event(self, event): if validate_result.status_code != 500: self.debug(f"Detected Telerik UI instance (Telerik.Web.UI.SpellCheckHandler.axd)") description = f"Telerik SpellCheckHandler detected" - self.emit_event( + await self.emit_event( { "host": str(event.host), "url": f"{event.data}{spellcheckhandler}", @@ -317,7 +317,7 @@ async def handle_event(self, event): chartimagehandler_error = "ChartImage.axd?ImageName=" result_error, _ = await self.test_detector(event.data, chartimagehandler_error) if result_error.status_code != 200: - self.emit_event( + await self.emit_event( { "host": str(event.host), "url": f"{event.data}{chartimagehandler}", @@ -331,7 +331,7 @@ async def handle_event(self, event): resp_body = event.data.get("body", None) if resp_body: if '":{"SerializedParameters":"' in resp_body: - self.emit_event( + await self.emit_event( { "host": str(event.host), "url": event.data["url"], @@ -341,7 +341,7 @@ async def handle_event(self, event): event, ) elif '"_serializedConfiguration":"' in resp_body: - self.emit_event( + await self.emit_event( { "host": str(event.host), "url": event.data["url"], diff --git a/bbot/modules/templates/bucket.py b/bbot/modules/templates/bucket.py index eef8f5beed..f9681385b3 100644 --- a/bbot/modules/templates/bucket.py +++ b/bbot/modules/templates/bucket.py @@ -52,7 +52,7 @@ async def handle_dns_name(self, event): for d in self.delimiters: buckets.add(d.join(split)) async for bucket_name, url, tags in self.brute_buckets(buckets, permutations=self.permutations): - self.emit_event({"name": bucket_name, "url": url}, "STORAGE_BUCKET", source=event, tags=tags) + await self.emit_event({"name": bucket_name, "url": url}, "STORAGE_BUCKET", source=event, tags=tags) async def handle_storage_bucket(self, event): url = event.data["url"] @@ -61,12 +61,12 @@ async def handle_storage_bucket(self, event): description, tags = await self._check_bucket_open(bucket_name, url) if description: event_data = {"host": event.host, "url": url, "description": description} - self.emit_event(event_data, "FINDING", source=event, tags=tags) + await self.emit_event(event_data, "FINDING", source=event, tags=tags) async for bucket_name, url, tags in self.brute_buckets( [bucket_name], permutations=self.permutations, omit_base=True ): - self.emit_event({"name": bucket_name, "url": url}, "STORAGE_BUCKET", source=event, tags=tags) + await self.emit_event({"name": bucket_name, "url": url}, "STORAGE_BUCKET", source=event, tags=tags) async def brute_buckets(self, buckets, permutations=False, omit_base=False): buckets = set(buckets) diff --git a/bbot/modules/templates/subdomain_enum.py b/bbot/modules/templates/subdomain_enum.py index 0a00676287..790b35515b 100644 --- a/bbot/modules/templates/subdomain_enum.py +++ b/bbot/modules/templates/subdomain_enum.py @@ -38,7 +38,7 @@ async def handle_event(self, event): self.verbose(e) continue if hostname and hostname.endswith(f".{query}") and not hostname == event.data: - self.emit_event(hostname, "DNS_NAME", event, abort_if=self.abort_if) + await self.emit_event(hostname, "DNS_NAME", event, abort_if=self.abort_if) async def request_url(self, query): url = f"{self.base_url}/subdomains/{self.helpers.quote(query)}" diff --git a/bbot/modules/url_manipulation.py b/bbot/modules/url_manipulation.py index f4d598c636..983595fe16 100644 --- a/bbot/modules/url_manipulation.py +++ b/bbot/modules/url_manipulation.py @@ -74,7 +74,7 @@ async def handle_event(self, event): if "body" in reasons: reported_signature = f"Modified URL: {sig[1]}" description = f"Url Manipulation: [{','.join(reasons)}] Sig: [{reported_signature}]" - self.emit_event( + await self.emit_event( {"description": description, "host": str(event.host), "url": event.data}, "FINDING", source=event, diff --git a/bbot/modules/urlscan.py b/bbot/modules/urlscan.py index f1efe08e54..4c3811af09 100644 --- a/bbot/modules/urlscan.py +++ b/bbot/modules/urlscan.py @@ -25,16 +25,18 @@ async def handle_event(self, event): domain_event = self.make_event(domain, "DNS_NAME", source=event) if domain_event: if str(domain_event.host).endswith(query) and not str(domain_event.host) == str(event.host): - self.emit_event(domain_event, abort_if=self.abort_if) + await self.emit_event(domain_event, abort_if=self.abort_if) source_event = domain_event if url: url_event = self.make_event(url, "URL_UNVERIFIED", source=source_event) if url_event: if str(url_event.host).endswith(query): if self.urls: - self.emit_event(url_event, abort_if=self.abort_if) + await self.emit_event(url_event, abort_if=self.abort_if) else: - self.emit_event(str(url_event.host), "DNS_NAME", source=event, abort_if=self.abort_if) + await self.emit_event( + str(url_event.host), "DNS_NAME", source=event, abort_if=self.abort_if + ) else: self.debug(f"{url_event.host} does not match {query}") diff --git a/bbot/modules/viewdns.py b/bbot/modules/viewdns.py index c2a5e44317..d9a5898459 100644 --- a/bbot/modules/viewdns.py +++ b/bbot/modules/viewdns.py @@ -26,7 +26,7 @@ async def setup(self): async def handle_event(self, event): _, query = self.helpers.split_domain(event.data) for domain, _ in await self.query(query): - self.emit_event(domain, "DNS_NAME", source=event, tags=["affiliate"]) + await self.emit_event(domain, "DNS_NAME", source=event, tags=["affiliate"]) async def query(self, query): results = set() diff --git a/bbot/modules/wafw00f.py b/bbot/modules/wafw00f.py index f80979619a..89c7ee1fad 100644 --- a/bbot/modules/wafw00f.py +++ b/bbot/modules/wafw00f.py @@ -32,12 +32,12 @@ async def handle_event(self, event): waf_detections = await self.scan.run_in_executor(WW.identwaf) if waf_detections: for waf in waf_detections: - self.emit_event({"host": str(event.host), "url": url, "WAF": waf}, "WAF", source=event) + await self.emit_event({"host": str(event.host), "url": url, "WAF": waf}, "WAF", source=event) else: if self.config.get("generic_detect") == True: generic = await self.scan.run_in_executor(WW.genericdetect) if generic: - self.emit_event( + await self.emit_event( { "host": str(event.host), "url": url, diff --git a/bbot/modules/wappalyzer.py b/bbot/modules/wappalyzer.py index 6d30fc0576..c87274a29b 100644 --- a/bbot/modules/wappalyzer.py +++ b/bbot/modules/wappalyzer.py @@ -28,7 +28,7 @@ async def setup(self): async def handle_event(self, event): for res in await self.scan.run_in_executor(self.wappalyze, event.data): - self.emit_event( + await self.emit_event( {"technology": res.lower(), "url": event.data["url"], "host": str(event.host)}, "TECHNOLOGY", event ) diff --git a/bbot/modules/wayback.py b/bbot/modules/wayback.py index 4bec112bff..bf4fb769e6 100644 --- a/bbot/modules/wayback.py +++ b/bbot/modules/wayback.py @@ -27,7 +27,7 @@ async def setup(self): async def handle_event(self, event): query = self.make_query(event) for result, event_type in await self.query(query): - self.emit_event(result, event_type, event, abort_if=self.abort_if) + await self.emit_event(result, event_type, event, abort_if=self.abort_if) async def query(self, query): results = set() diff --git a/bbot/modules/zoomeye.py b/bbot/modules/zoomeye.py index 3c83fa828f..b1d4e76704 100644 --- a/bbot/modules/zoomeye.py +++ b/bbot/modules/zoomeye.py @@ -36,7 +36,7 @@ async def handle_event(self, event): tags = [] if not hostname.endswith(f".{query}"): tags = ["affiliate"] - self.emit_event(hostname, "DNS_NAME", event, tags=tags) + await self.emit_event(hostname, "DNS_NAME", event, tags=tags) async def query(self, query): results = set() diff --git a/bbot/test/test_step_1/test_manager_deduplication.py b/bbot/test/test_step_1/test_manager_deduplication.py index 305796bea5..e046988ae3 100644 --- a/bbot/test/test_step_1/test_manager_deduplication.py +++ b/bbot/test/test_step_1/test_manager_deduplication.py @@ -15,7 +15,7 @@ async def setup(self): async def handle_event(self, event): self.events.append(event) - self.emit_event(f"{self.name}.test.notreal", "DNS_NAME", source=event) + await self.emit_event(f"{self.name}.test.notreal", "DNS_NAME", source=event) class EverythingModule(DefaultModule): _name = "everything_module" @@ -27,7 +27,7 @@ class EverythingModule(DefaultModule): async def handle_event(self, event): self.events.append(event) if event.type == "DNS_NAME": - self.emit_event(f"{event.data}:88", "OPEN_TCP_PORT", source=event) + await self.emit_event(f"{event.data}:88", "OPEN_TCP_PORT", source=event) class NoSuppressDupes(DefaultModule): _name = "no_suppress_dupes" diff --git a/bbot/test/test_step_1/test_manager_scope_accuracy.py b/bbot/test/test_step_1/test_manager_scope_accuracy.py index 08ac2c3ae8..e8e5da391e 100644 --- a/bbot/test/test_step_1/test_manager_scope_accuracy.py +++ b/bbot/test/test_step_1/test_manager_scope_accuracy.py @@ -257,7 +257,7 @@ async def filter_event(self, event): return False, "bleh" async def handle_event(self, event): - self.emit_event( + await self.emit_event( {"host": str(event.host), "description": "yep", "severity": "CRITICAL"}, "VULNERABILITY", source=event ) diff --git a/bbot/test/test_step_1/test_modules_basic.py b/bbot/test/test_step_1/test_modules_basic.py index b943144ac2..7a25bb4ea1 100644 --- a/bbot/test/test_step_1/test_modules_basic.py +++ b/bbot/test/test_step_1/test_modules_basic.py @@ -311,7 +311,7 @@ class dummy(BaseModule): watched_events = ["*"] async def handle_event(self, event): - self.emit_event( + await self.emit_event( {"host": "www.evilcorp.com", "url": "http://www.evilcorp.com", "description": "asdf"}, "FINDING", event ) diff --git a/docs/contribution.md b/docs/contribution.md index 65b074adba..2d36cfe446 100644 --- a/docs/contribution.md +++ b/docs/contribution.md @@ -74,7 +74,7 @@ class MyModule(BaseModule): self.hugeinfo(f"GOT EVENT: {event}") for ip in await self.helpers.resolve(event.data): self.hugesuccess(f"EMITTING IP_ADDRESS: {ip}") - self.emit_event(ip, "IP_ADDRESS", source=event) + await self.emit_event(ip, "IP_ADDRESS", source=event) ``` After saving the module, you can run it simply by specifying it with `-m`: From 451e67cf90e9bf556fdcd467e14824ecc630010a Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Fri, 19 Jan 2024 12:04:48 -0500 Subject: [PATCH 2/8] fix tests --- bbot/modules/generic_ssrf.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bbot/modules/generic_ssrf.py b/bbot/modules/generic_ssrf.py index d4045993b2..9d75f4a9e6 100644 --- a/bbot/modules/generic_ssrf.py +++ b/bbot/modules/generic_ssrf.py @@ -188,7 +188,7 @@ async def handle_event(self, event): for s in self.submodules.values(): await s.test(event) - def interactsh_callback(self, r): + async def interactsh_callback(self, r): full_id = r.get("full-id", None) if full_id: if "." in full_id: @@ -229,6 +229,6 @@ async def finish(self): await self.helpers.sleep(5) try: for r in await self.interactsh_instance.poll(): - self.interactsh_callback(r) + await self.interactsh_callback(r) except InteractshError as e: self.debug(f"Error in interact.sh: {e}") From b4fea8a72b8142356942e34d70bbfe082c71a04f Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Fri, 19 Jan 2024 12:59:11 -0500 Subject: [PATCH 3/8] fix tests --- bbot/modules/masscan.py | 14 +++++++------- bbot/modules/paramminer_headers.py | 4 ++-- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/bbot/modules/masscan.py b/bbot/modules/masscan.py index 02598a2150..15881d5b2f 100644 --- a/bbot/modules/masscan.py +++ b/bbot/modules/masscan.py @@ -105,7 +105,7 @@ async def setup(self): async def handle_batch(self, *events): if self.use_cache: - self.emit_from_cache() + await self.emit_from_cache() else: targets = [str(e.data) for e in events] if not targets: @@ -138,7 +138,7 @@ async def masscan(self, targets, result_callback, ping=False): try: with open(stats_file, "w") as stats_fh: async for line in self.helpers.run_live(command, sudo=True, stderr=stats_fh): - self.process_output(line, result_callback=result_callback) + await self.process_output(line, result_callback=result_callback) finally: for file in (stats_file, target_file): file.unlink() @@ -169,7 +169,7 @@ def _build_masscan_command(self, target_file=None, dry_run=False, ping=False): command += ("--echo",) return command - def process_output(self, line, result_callback): + async def process_output(self, line, result_callback): try: j = json.loads(line) except Exception: @@ -194,19 +194,19 @@ def process_output(self, line, result_callback): if source is None: source = self.make_event(ip, "IP_ADDRESS", source=self.get_source_event(ip)) await self.emit_event(source) - result_callback(result, source=source) + await result_callback(result, source=source) - def append_alive_host(self, host, source): + async def append_alive_host(self, host, source): host_event = self.make_event(host, "IP_ADDRESS", source=self.get_source_event(host)) self.alive_hosts[host] = host_event self._write_ping_result(host) await self.emit_event(host_event) - def emit_open_tcp_port(self, data, source): + async def emit_open_tcp_port(self, data, source): self._write_syn_result(data) await self.emit_event(data, "OPEN_TCP_PORT", source=source) - def emit_from_cache(self): + async def emit_from_cache(self): ip_events = {} # ping scan if self.ping_cache.is_file(): diff --git a/bbot/modules/paramminer_headers.py b/bbot/modules/paramminer_headers.py index 7044ae90ab..3d3861621f 100644 --- a/bbot/modules/paramminer_headers.py +++ b/bbot/modules/paramminer_headers.py @@ -119,7 +119,7 @@ async def do_mining(self, wl, url, batch_size, compare_helper): pass return results - def process_results(self, event, results): + async def process_results(self, event, results): url = event.data.get("url") for result, reasons, reflection in results: tags = [] @@ -171,7 +171,7 @@ async def handle_event(self, event): results = await self.do_mining(wl, url, batch_size, compare_helper) except HttpCompareError as e: self.debug(f"Encountered HttpCompareError: [{e}] for URL [{event.data}]") - self.process_results(event, results) + await self.process_results(event, results) async def count_test(self, url): baseline = await self.helpers.request(url) From ca3f3c28716efdeba57f6e686e150f183e6f2f5d Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Fri, 19 Jan 2024 17:21:32 -0500 Subject: [PATCH 4/8] fix tests --- bbot/core/helpers/helper.py | 4 +-- bbot/core/helpers/interactsh.py | 5 ++-- bbot/modules/host_header.py | 2 +- bbot/test/conftest.py | 22 ++++++++++++-- bbot/test/test_step_1/test_web.py | 48 +++++++++++++++++++++++++++---- 5 files changed, 68 insertions(+), 13 deletions(-) diff --git a/bbot/core/helpers/helper.py b/bbot/core/helpers/helper.py index dbe19f20c6..36c6346c93 100644 --- a/bbot/core/helpers/helper.py +++ b/bbot/core/helpers/helper.py @@ -77,8 +77,8 @@ def __init__(self, config, scan=None): # cloud helpers self.cloud = CloudHelper(self) - def interactsh(self): - return Interactsh(self) + def interactsh(self, *args, **kwargs): + return Interactsh(self, *args, **kwargs) def http_compare(self, url, allow_redirects=False, include_cache_buster=True): return HttpCompare(url, self, allow_redirects=allow_redirects, include_cache_buster=include_cache_buster) diff --git a/bbot/core/helpers/interactsh.py b/bbot/core/helpers/interactsh.py index 95f76d2f5f..929637dfe6 100644 --- a/bbot/core/helpers/interactsh.py +++ b/bbot/core/helpers/interactsh.py @@ -78,12 +78,13 @@ class Interactsh: ``` """ - def __init__(self, parent_helper): + def __init__(self, parent_helper, poll_interval=10): self.parent_helper = parent_helper self.server = None self.correlation_id = None self.custom_server = self.parent_helper.config.get("interactsh_server", None) self.token = self.parent_helper.config.get("interactsh_token", None) + self.poll_interval = poll_interval self._poll_task = None async def register(self, callback=None): @@ -279,7 +280,7 @@ async def _poll_loop(self, callback): log.warning(e) log.trace(traceback.format_exc()) if not data_list: - await asyncio.sleep(10) + await asyncio.sleep(self.poll_interval) continue for data in data_list: if data: diff --git a/bbot/modules/host_header.py b/bbot/modules/host_header.py index bec77c15a3..4adaa766a7 100644 --- a/bbot/modules/host_header.py +++ b/bbot/modules/host_header.py @@ -58,7 +58,7 @@ async def finish(self): await self.helpers.sleep(5) try: for r in await self.interactsh_instance.poll(): - self.interactsh_callback(r) + await self.interactsh_callback(r) except InteractshError as e: self.debug(f"Error in interact.sh: {e}") diff --git a/bbot/test/conftest.py b/bbot/test/conftest.py index 684ec18a23..b60a0633d8 100644 --- a/bbot/test/conftest.py +++ b/bbot/test/conftest.py @@ -1,10 +1,12 @@ import ssl import shutil import pytest +import asyncio import logging from pathlib import Path from pytest_httpserver import HTTPServer +from bbot.core.helpers.misc import execute_sync_or_async from bbot.core.helpers.interactsh import server_list as interactsh_servers @@ -98,20 +100,34 @@ class Interactsh_mock: def __init__(self): self.interactions = [] self.correlation_id = "deadbeef-dead-beef-dead-beefdeadbeef" + self.stop = False def mock_interaction(self, subdomain_tag): self.interactions.append(subdomain_tag) async def register(self, callback=None): + if callable(callback): + asyncio.create_task(self.poll_loop(callback)) return "fakedomain.fakeinteractsh.com" async def deregister(self, callback=None): - pass + self.stop = True - async def poll(self): + async def poll_loop(self, callback=None): + while not self.stop: + data_list = await self.poll(callback) + if not data_list: + await asyncio.sleep(1) + continue + + async def poll(self, callback=None): poll_results = [] for subdomain_tag in self.interactions: - poll_results.append({"full-id": f"{subdomain_tag}.fakedomain.fakeinteractsh.com", "protocol": "HTTP"}) + result = {"full-id": f"{subdomain_tag}.fakedomain.fakeinteractsh.com", "protocol": "HTTP"} + poll_results.append(result) + if callback is not None: + await execute_sync_or_async(callback, result) + self.interactions = [] return poll_results diff --git a/bbot/test/test_step_1/test_web.py b/bbot/test/test_step_1/test_web.py index 9179d42e63..13edaf7258 100644 --- a/bbot/test/test_step_1/test_web.py +++ b/bbot/test/test_step_1/test_web.py @@ -104,23 +104,61 @@ async def test_web_helpers(bbot_scanner, bbot_config, bbot_httpserver): async def test_web_interactsh(bbot_scanner, bbot_config, bbot_httpserver): from bbot.core.helpers.interactsh import server_list + sync_called = False + async_called = False + + sync_correct_url = False + async_correct_url = False + scan1 = bbot_scanner("8.8.8.8", config=bbot_config) + scan1.status = "RUNNING" - interactsh_client = scan1.helpers.interactsh() + interactsh_client = scan1.helpers.interactsh(poll_interval=3) + interactsh_client2 = scan1.helpers.interactsh(poll_interval=3) async def async_callback(data): - log.debug(f"interactsh poll: {data}") + nonlocal async_called + nonlocal async_correct_url + async_called = True + d = data.get("raw-request", "") + async_correct_url |= "bbot_interactsh_test" in d + log.debug(f"interactsh poll (async): {d}") + + def sync_callback(data): + nonlocal sync_called + nonlocal sync_correct_url + sync_called = True + d = data.get("raw-request", "") + sync_correct_url |= "bbot_interactsh_test" in d + log.debug(f"interactsh poll (sync): {d}") interactsh_domain = await interactsh_client.register(callback=async_callback) - url = f"https://{interactsh_domain}/bbot_interactsh_test" + url = f"http://{interactsh_domain}/bbot_interactsh_test" response = await scan1.helpers.request(url) assert response.status_code == 200 - await asyncio.sleep(10) assert any(interactsh_domain.endswith(f"{s}") for s in server_list) + + interactsh_domain2 = await interactsh_client2.register(callback=sync_callback) + url2 = f"http://{interactsh_domain2}/bbot_interactsh_test" + response2 = await scan1.helpers.request(url2) + assert response2.status_code == 200 + assert any(interactsh_domain2.endswith(f"{s}") for s in server_list) + + await asyncio.sleep(10) + data_list = await interactsh_client.poll() + data_list2 = await interactsh_client2.poll() assert isinstance(data_list, list) - assert any("bbot_interactsh_test" in d.get("raw-request", "") for d in data_list) + assert isinstance(data_list2, list) + assert await interactsh_client.deregister() is None + assert await interactsh_client2.deregister() is None + + assert sync_called, "Interactsh synchrononous callback was not called" + assert async_called, "Interactsh async callback was not called" + + assert sync_correct_url, f"Data content was not correct for {url2}" + assert async_correct_url, f"Data content was not correct for {url}" @pytest.mark.asyncio From 48836874d277aa9ec212d11f98a8f0ca04e30d0c Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Mon, 22 Jan 2024 09:49:47 -0500 Subject: [PATCH 5/8] fix tests again --- bbot/modules/paramminer_headers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bbot/modules/paramminer_headers.py b/bbot/modules/paramminer_headers.py index 3d3861621f..3458edaa93 100644 --- a/bbot/modules/paramminer_headers.py +++ b/bbot/modules/paramminer_headers.py @@ -247,4 +247,4 @@ async def finish(self): results = await self.do_mining(untested_matches_copy, url, batch_size, compare_helper) except HttpCompareError as e: self.debug(f"Encountered HttpCompareError: [{e}] for URL [{url}]") - self.process_results(event, results) + await self.process_results(event, results) From e60f4b7010908d0cc8d58842b413c3c991614339 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Mon, 22 Jan 2024 11:35:01 -0500 Subject: [PATCH 6/8] fixed postman tests --- bbot/test/test_step_2/module_tests/test_module_postman.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bbot/test/test_step_2/module_tests/test_module_postman.py b/bbot/test/test_step_2/module_tests/test_module_postman.py index 2879b4838a..4f75151621 100644 --- a/bbot/test/test_step_2/module_tests/test_module_postman.py +++ b/bbot/test/test_step_2/module_tests/test_module_postman.py @@ -183,10 +183,10 @@ async def setup_after_prep(self, module_test): old_emit_event = module_test.module.emit_event - def new_emit_event(event_data, event_type, **kwargs): + async def new_emit_event(event_data, event_type, **kwargs): if event_data.startswith("https://www.postman.com"): event_data = event_data.replace("https://www.postman.com", "http://127.0.0.1:8888") - old_emit_event(event_data, event_type, **kwargs) + await old_emit_event(event_data, event_type, **kwargs) module_test.monkeypatch.setattr(module_test.module, "emit_event", new_emit_event) module_test.scan.helpers.dns.mock_dns({("asdf.blacklanternsecurity.com", "A"): "127.0.0.1"}) From 6df8bffa2189e5fd2945950b3f6924c72a551ee8 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Mon, 22 Jan 2024 14:32:37 -0500 Subject: [PATCH 7/8] improve task counting for batch modules --- bbot/core/helpers/async_helpers.py | 19 ++++++++++--------- bbot/modules/base.py | 5 +++-- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/bbot/core/helpers/async_helpers.py b/bbot/core/helpers/async_helpers.py index 916764e53e..4cc7011615 100644 --- a/bbot/core/helpers/async_helpers.py +++ b/bbot/core/helpers/async_helpers.py @@ -57,33 +57,34 @@ def __init__(self): @property def value(self): - return len(self.tasks) + return sum([t.n for t in self.tasks.values()]) - def count(self, task_name, _log=True): + def count(self, task_name, n=1, _log=True): if callable(task_name): task_name = f"{task_name.__qualname__}()" - return self.Task(self, task_name, _log) + return self.Task(self, task_name, n=n, _log=_log) class Task: - def __init__(self, manager, task_name, _log=True): + def __init__(self, manager, task_name, n=1, _log=True): self.manager = manager self.task_name = task_name self.task_id = None self.start_time = None self.log = _log + self.n = n async def __aenter__(self): - self.task_id = uuid.uuid4() # generate a unique ID for the task + self.task_id = uuid.uuid4() # if self.log: # log.trace(f"Starting task {self.task_name} ({self.task_id})") - async with self.manager.lock: # acquire the lock + async with self.manager.lock: self.start_time = datetime.now() self.manager.tasks[self.task_id] = self - return self.task_id # this will be passed as 'task_id' to __aexit__ + return self async def __aexit__(self, exc_type, exc_val, exc_tb): - async with self.manager.lock: # acquire the lock - self.manager.tasks.pop(self.task_id, None) # remove only current task + async with self.manager.lock: + self.manager.tasks.pop(self.task_id, None) # if self.log: # log.trace(f"Finished task {self.task_name} ({self.task_id})") diff --git a/bbot/modules/base.py b/bbot/modules/base.py index 6ae93c7ea5..d049f711a5 100644 --- a/bbot/modules/base.py +++ b/bbot/modules/base.py @@ -175,7 +175,7 @@ async def handle_event(self, event): """ pass - def handle_batch(self, *events): + async def handle_batch(self, *events): """Handles incoming events in batches for optimized processing. This method is automatically called when multiple events that match any in `watched_events` are encountered and the `batch_size` attribute is set to a value greater than 1. Override this method to implement custom batch event-handling logic for your module. @@ -350,13 +350,14 @@ async def _handle_batch(self): - If a "FINISHED" event is found, invokes 'finish()' method of the module. """ finish = False - async with self._task_counter.count(f"{self.name}.handle_batch()"): + async with self._task_counter.count(f"{self.name}.handle_batch()") as counter: submitted = False if self.batch_size <= 1: return if self.num_incoming_events > 0: events, finish = await self._events_waiting() if events and not self.errored: + counter.n = len(events) self.debug(f"Handling batch of {len(events):,} events") submitted = True async with self.scan._acatch(f"{self.name}.handle_batch()"): From 3778ff977671d3a55d8b499556f428eee008e8e9 Mon Sep 17 00:00:00 2001 From: TheTechromancer Date: Mon, 22 Jan 2024 14:32:50 -0500 Subject: [PATCH 8/8] lovecraftian entities --- bbot/core/helpers/names_generator.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/bbot/core/helpers/names_generator.py b/bbot/core/helpers/names_generator.py index 49ed866d67..16432cb0e5 100644 --- a/bbot/core/helpers/names_generator.py +++ b/bbot/core/helpers/names_generator.py @@ -298,6 +298,7 @@ "ashley", "audrey", "austin", + "azathoth", "baggins", "bailey", "barbara", @@ -347,8 +348,10 @@ "courtney", "craig", "crystal", + "cthulu", "curtis", "cynthia", + "dagon", "dale", "dandelion", "daniel", @@ -554,6 +557,7 @@ "noah", "norma", "norman", + "nyarlathotep", "obama", "olivia", "padme",